Commit 4fbe4a03 authored by jz143's avatar jz143

use resque for job queue

parent 8ee8e552
......@@ -14,7 +14,7 @@ gem 'uglifier', '>= 1.3.0'
# Use CoffeeScript for .coffee assets and views
gem 'coffee-rails', '~> 4.1.0'
# See https://github.com/rails/execjs#readme for more supported runtimes
gem 'therubyracer', platforms: :ruby
gem 'therubyracer', platforms: :ruby, group: :production
# Use jquery as the JavaScript library
gem 'jquery-rails'
......@@ -74,7 +74,6 @@ gem "pundit"
gem "plist", :git => "https://github.com/jiehanzheng/plist.git", :ref => "8dc72ad"
# job queue
gem 'resque', :git => "https://github.com/resque/resque.git", :ref => "9c023f8c12f7e31caf8b7a73ae5094934a0ddf17"
gem 'resque-status', '~> 0.5.0'
gem 'resque', '~> 1.26.0'
gem 'bootstrap-sass', '~> 3.3.5'
......@@ -5,18 +5,6 @@ GIT
specs:
plist (3.1.1)
GIT
remote: https://github.com/resque/resque.git
revision: 9c023f8c12f7e31caf8b7a73ae5094934a0ddf17
ref: 9c023f8c12f7e31caf8b7a73ae5094934a0ddf17
specs:
resque (1.25.2)
mono_logger (~> 1.0)
multi_json (~> 1.0)
redis-namespace (~> 1.3)
sinatra (>= 0.9.2)
vegas (~> 0.1.2)
GEM
remote: https://rubygems.org/
specs:
......@@ -175,8 +163,12 @@ GEM
redis-namespace (1.5.2)
redis (~> 3.0, >= 3.0.4)
ref (2.0.0)
resque-status (0.5.0)
resque (~> 1.19)
resque (1.26.0)
mono_logger (~> 1.0)
multi_json (~> 1.0)
redis-namespace (~> 1.3)
sinatra (>= 0.9.2)
vegas (~> 0.1.2)
sass (3.4.17)
sass-rails (5.0.3)
railties (>= 4.0.0, < 5.0)
......@@ -190,8 +182,8 @@ GEM
simple_form (3.2.0)
actionpack (~> 4.0)
activemodel (~> 4.0)
sinatra (1.4.6)
rack (~> 1.4)
sinatra (1.4.7)
rack (~> 1.5)
rack-protection (~> 1.4)
tilt (>= 1.3, < 3)
spring (1.3.6)
......@@ -249,8 +241,7 @@ DEPENDENCIES
plist!
pundit
rails (= 4.2.4)
resque!
resque-status (~> 0.5.0)
resque (~> 1.26.0)
sass-rails (~> 5.0)
sdoc (~> 0.4.0)
simple_form
......
......@@ -21,7 +21,7 @@ class AppFilesController < ApplicationController
if params[:download_token] == @app_file.download_token
send_file @app_file.archive.path
else
render nothing: true
head :bad_request
end
end
......
class SessionsController < ApplicationController
def create_during_test
if Rails.env.test?
if Rails.env.test? || Rails.env.development?
session[:user_id] = params[:user_id]
end
redirect_to :root
......
require 'open-uri'
class BuildIpaJob < ActiveJob::Base
queue_as :mac_server
queue_as :mac_online
def perform(app_file_id)
ActiveRecord::Base.clear_active_connections!
# get corresponding app_file record
app_file = AppFile.find(app_file_id)
......@@ -18,16 +20,19 @@ class BuildIpaJob < ActiveJob::Base
app_file.signing!
tmpdir = Dir.mktmpdir
success = false
begin
log << "#{self.class.name} using tmp dir: #{tmpdir}\n"
# fetch app file from remote web server (assuming we are worker on another machine)
# we use remote's HTTP send_file here to reduce ruby workload
File.open(File.join(tmpdir, 'Unsigned.app.zip'), "wb") do |file|
# XXX: this looks like a hack
default_url_options = Rails.application.config.action_mailer.default_url_options
include Rails.application.routes.url_helpers
open(app_app_file_download_url(app_file.app, app_file, default_url_options), "rb") do |read_file|
url_options = Rails.application.config.action_mailer.default_url_options
url_options[:format] = 'zip'
url_options[:download_token] = app_file.download_token
download_url = Rails.application.routes.url_helpers.app_app_file_download_url(app_file.app, app_file, url_options)
log << "Downloading from #{download_url}\n"
open(download_url, "rb") do |read_file|
file.write(read_file.read)
end
end
......@@ -64,6 +69,7 @@ class BuildIpaJob < ActiveJob::Base
log << "--- END ENTITLEMENTS ---\n"
# call signing script, collect output
app_file.signed_ipa_path_on_signing_server = File.join(tmpdir, 'Signed.ipa')
signing_script_args = [
'floatsign.sh',
File.join(tmpdir, 'UnsignedArchiveContent', app_folder), # app archive to sign
......@@ -72,7 +78,7 @@ class BuildIpaJob < ActiveJob::Base
'-p', Rails.root.join('resources', 'ios_provisioning_profiles', 'CoLab_Member_Apps_20151.mobileprovision').to_s,
'-b', app_file.app.ios_application_identifier,
'-t', File.join(tmpdir, 'floatsign_tmp'),
File.join(tmpdir, 'Signed.ipa')
app_file.signed_ipa_path_on_signing_server
]
log << "Signing with args: " << signing_script_args.collect { |a| "\"#{a}\"" }.join(' ') << "\n"
......@@ -86,20 +92,25 @@ class BuildIpaJob < ActiveJob::Base
end
app_file.pending_transfer_to_web!
FetchedSignedIpaJob.perform_later app_file.id
success = true
rescue => e
app_file.signing_failed!
log << "Fatal error: " << e.to_s << "\n"
ensure
FileUtils.remove_entry tmpdir
log << "Fatal error: " << e.to_s << "\n"
end
else
log << "Refusing to codesign when app_file is not freshly uploaded\n"
end
# save log to database
app_file.code_signing_log = log
app_file.save!
# save log and states to database without validation and callbacks
app_file.update_columns(
code_signing_log: log,
signed_ipa_path_on_signing_server: app_file.signed_ipa_path_on_signing_server)
if success
FetchedSignedIpaJob.perform_later app_file.id
else
FileUtils.remove_entry tmpdir
end
end
end
class FetchedSignedIpaJob < ActiveJob::Base
queue_as :web
queue_as :web_background
def perform(app_file_id)
ActiveRecord::Base.clear_active_connections!
# get corresponding app_file record
app_file = AppFile.find(app_file_id)
log = app_file.code_signing_log
begin
app_file.downloading_to_web!
# get file to app_file.signed_ipa_location
log << "scp from sign:#{app_file.signed_ipa_path_on_signing_server} to #{app_file.signed_ipa_location}\n"
SSH::SCP.download!(Settings.signing_worker.host, Settings.signing_worker.user,
Net::SCP.download!(Settings.signing_worker.host, Settings.signing_worker.user,
app_file.signed_ipa_path_on_signing_server, app_file.signed_ipa_location)
# remove remote file
# remove remote tmpdir and file
log << "removing sign:#{File.dirname(app_file.signed_ipa_path_on_signing_server)}\n"
Net::SSH.start(Settings.signing_worker.host, Settings.signing_worker.user) do |ssh|
ssh.exec "rm -rf \"#{File.dirname(app_file.signed_ipa_path_on_signing_server)}\""
......@@ -21,11 +25,14 @@ class FetchedSignedIpaJob < ActiveJob::Base
# remove user uploaded unsigned package to save space
log << "removing local, unsigned copy\n"
app_file.archive.destroy
app_file.signed!
rescue => e
app_file.download_to_web_failed!
log << "Fatal error: " << e.to_s << "\n"
ensure
end
# save log and states to database without validation and callbacks
app_file.update_column(:code_signing_log, log)
end
end
......@@ -6,6 +6,8 @@ class AppFile < ActiveRecord::Base
has_attached_file :archive
validates_attachment_file_name :archive, matches: [/apk\Z/, /zip\Z/]
validates_with AttachmentPresenceValidator, attributes: :archive, :on => :create
validates_with AttachmentSizeValidator, attributes: :archive, less_than: 300.megabytes
enum code_signing_status: { unsigned: 0, downloading_to_mac: 4, signing: 1, signing_failed: 2, pending_transfer_to_web: 5, downloading_to_web: 6, download_to_web_failed: 7, signed: 3 }
after_commit :queue_for_ios_code_signing, :on => :create
......@@ -21,7 +23,7 @@ class AppFile < ActiveRecord::Base
if is_ios_app?
# make a secret random key for direct download
require 'securerandom'
download_token = SecureRandom.hex
update_column(:download_token, SecureRandom.hex)
BuildIpaJob.perform_later id
end
......
......@@ -22,5 +22,9 @@ module Appstore
# Do not swallow errors in after_commit/after_rollback callbacks.
config.active_record.raise_in_transactional_callbacks = true
# Be sure to have the adapter's gem in your Gemfile and follow
# the adapter's specific installation and deployment instructions.
config.active_job.queue_adapter = :resque
end
end
rails_root = ENV['RAILS_ROOT'] || File.dirname(__FILE__) + '/../..'
rails_env = ENV['RAILS_ENV'] || 'development'
resque_config = YAML.load_file(rails_root + '/config/resque.yml')
Resque.redis = resque_config[rails_env]
development: localhost:6379
test: localhost:6379
production: appstore.colab.duke.edu:6379
require 'resque/tasks'
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment