use resque for job queue

Gemfile

@@ -14,7 +14,7 @@ gem 'uglifier', '>= 1.3.0'
 # Use CoffeeScript for .coffee assets and views
 gem 'coffee-rails', '~> 4.1.0'
 # See https://github.com/rails/execjs#readme for more supported runtimes
-gem 'therubyracer', platforms: :ruby
+gem 'therubyracer', platforms: :ruby, group: :production
 
 # Use jquery as the JavaScript library
 gem 'jquery-rails'
@@ -74,7 +74,6 @@ gem "pundit"
 gem "plist", :git => "https://github.com/jiehanzheng/plist.git", :ref => "8dc72ad"
 
 # job queue
-gem 'resque', :git => "https://github.com/resque/resque.git", :ref => "9c023f8c12f7e31caf8b7a73ae5094934a0ddf17"
-gem 'resque-status', '~> 0.5.0'
+gem 'resque', '~> 1.26.0'
 
 gem 'bootstrap-sass', '~> 3.3.5'

Gemfile.lock

@@ -5,18 +5,6 @@ GIT
   specs:
     plist (3.1.1)
 
-GIT
-  remote: https://github.com/resque/resque.git
-  revision: 9c023f8c12f7e31caf8b7a73ae5094934a0ddf17
-  ref: 9c023f8c12f7e31caf8b7a73ae5094934a0ddf17
-  specs:
-    resque (1.25.2)
-      mono_logger (~> 1.0)
-      multi_json (~> 1.0)
-      redis-namespace (~> 1.3)
-      sinatra (>= 0.9.2)
-      vegas (~> 0.1.2)
-
 GEM
   remote: https://rubygems.org/
   specs:
@@ -175,8 +163,12 @@ GEM
     redis-namespace (1.5.2)
       redis (~> 3.0, >= 3.0.4)
     ref (2.0.0)
-    resque-status (0.5.0)
-      resque (~> 1.19)
+    resque (1.26.0)
+      mono_logger (~> 1.0)
+      multi_json (~> 1.0)
+      redis-namespace (~> 1.3)
+      sinatra (>= 0.9.2)
+      vegas (~> 0.1.2)
     sass (3.4.17)
     sass-rails (5.0.3)
       railties (>= 4.0.0, < 5.0)
@@ -190,8 +182,8 @@ GEM
     simple_form (3.2.0)
       actionpack (~> 4.0)
       activemodel (~> 4.0)
-    sinatra (1.4.6)
-      rack (~> 1.4)
+    sinatra (1.4.7)
+      rack (~> 1.5)
       rack-protection (~> 1.4)
       tilt (>= 1.3, < 3)
     spring (1.3.6)
@@ -249,8 +241,7 @@ DEPENDENCIES
   plist!
   pundit
   rails (= 4.2.4)
-  resque!
-  resque-status (~> 0.5.0)
+  resque (~> 1.26.0)
   sass-rails (~> 5.0)
   sdoc (~> 0.4.0)
   simple_form

app/controllers/app_files_controller.rb

@@ -21,7 +21,7 @@ class AppFilesController < ApplicationController
         if params[:download_token] == @app_file.download_token
           send_file @app_file.archive.path
         else
-          render nothing: true
+          head :bad_request
         end
       end
 

app/controllers/sessions_controller.rb

 class SessionsController < ApplicationController
   def create_during_test
-    if Rails.env.test?
+    if Rails.env.test? || Rails.env.development?
       session[:user_id] = params[:user_id]
     end
     redirect_to :root

app/jobs/build_ipa_job.rb

 require 'open-uri'
 
 class BuildIpaJob < ActiveJob::Base
-  queue_as :mac_server
+  queue_as :mac_online
 
   def perform(app_file_id)
+    ActiveRecord::Base.clear_active_connections!
+
     # get corresponding app_file record
     app_file = AppFile.find(app_file_id)
 
@@ -18,16 +20,19 @@ class BuildIpaJob < ActiveJob::Base
       app_file.signing!
 
       tmpdir = Dir.mktmpdir
+      success = false
       begin
         log << "#{self.class.name} using tmp dir: #{tmpdir}\n"
 
         # fetch app file from remote web server (assuming we are worker on another machine)
         # we use remote's HTTP send_file here to reduce ruby workload
         File.open(File.join(tmpdir, 'Unsigned.app.zip'), "wb") do |file|
-          # XXX: this looks like a hack
-          default_url_options = Rails.application.config.action_mailer.default_url_options
-          include Rails.application.routes.url_helpers
-          open(app_app_file_download_url(app_file.app, app_file, default_url_options), "rb") do |read_file|
+          url_options = Rails.application.config.action_mailer.default_url_options
+          url_options[:format] = 'zip'
+          url_options[:download_token] = app_file.download_token
+          download_url = Rails.application.routes.url_helpers.app_app_file_download_url(app_file.app, app_file, url_options)
+          log << "Downloading from #{download_url}\n"
+          open(download_url, "rb") do |read_file|
             file.write(read_file.read)
           end
         end
@@ -64,6 +69,7 @@ class BuildIpaJob < ActiveJob::Base
         log << "--- END ENTITLEMENTS ---\n"
 
         # call signing script, collect output
+        app_file.signed_ipa_path_on_signing_server = File.join(tmpdir, 'Signed.ipa')
         signing_script_args = [
           'floatsign.sh',
           File.join(tmpdir, 'UnsignedArchiveContent', app_folder),  # app archive to sign
@@ -72,7 +78,7 @@ class BuildIpaJob < ActiveJob::Base
           '-p', Rails.root.join('resources', 'ios_provisioning_profiles', 'CoLab_Member_Apps_20151.mobileprovision').to_s,
           '-b', app_file.app.ios_application_identifier,
           '-t', File.join(tmpdir, 'floatsign_tmp'),
-          File.join(tmpdir, 'Signed.ipa')
+          app_file.signed_ipa_path_on_signing_server
         ]
 
         log << "Signing with args: " << signing_script_args.collect { |a| "\"#{a}\"" }.join(' ') << "\n"
@@ -86,20 +92,25 @@ class BuildIpaJob < ActiveJob::Base
         end
 
         app_file.pending_transfer_to_web!
-        FetchedSignedIpaJob.perform_later app_file.id
+        success = true
       rescue => e
         app_file.signing_failed!
-        log << "Fatal error: " << e.to_s << "\n"
-      ensure
-        FileUtils.remove_entry tmpdir
+        log << "Fatal error: " << e.to_s << "\n"        
       end
 
     else
       log << "Refusing to codesign when app_file is not freshly uploaded\n"
     end
 
-    # save log to database
-    app_file.code_signing_log = log
-    app_file.save!
+    # save log and states to database without validation and callbacks
+    app_file.update_columns(
+      code_signing_log: log,
+      signed_ipa_path_on_signing_server: app_file.signed_ipa_path_on_signing_server)
+
+    if success
+      FetchedSignedIpaJob.perform_later app_file.id
+    else
+      FileUtils.remove_entry tmpdir
+    end
   end
 end

app/jobs/fetched_signed_ipa_job.rb

 class FetchedSignedIpaJob < ActiveJob::Base
-  queue_as :web
+  queue_as :web_background
 
   def perform(app_file_id)
+    ActiveRecord::Base.clear_active_connections!
+
     # get corresponding app_file record
     app_file = AppFile.find(app_file_id)
 
     log = app_file.code_signing_log
     begin
+      app_file.downloading_to_web!
+
       # get file to app_file.signed_ipa_location
       log << "scp from sign:#{app_file.signed_ipa_path_on_signing_server} to #{app_file.signed_ipa_location}\n"
-      SSH::SCP.download!(Settings.signing_worker.host, Settings.signing_worker.user,
+      Net::SCP.download!(Settings.signing_worker.host, Settings.signing_worker.user,
         app_file.signed_ipa_path_on_signing_server, app_file.signed_ipa_location)
 
-      # remove remote file
+      # remove remote tmpdir and file
       log << "removing sign:#{File.dirname(app_file.signed_ipa_path_on_signing_server)}\n"
       Net::SSH.start(Settings.signing_worker.host, Settings.signing_worker.user) do |ssh|
         ssh.exec "rm -rf \"#{File.dirname(app_file.signed_ipa_path_on_signing_server)}\""
@@ -21,11 +25,14 @@ class FetchedSignedIpaJob < ActiveJob::Base
       # remove user uploaded unsigned package to save space
       log << "removing local, unsigned copy\n"
       app_file.archive.destroy
+
+      app_file.signed!
     rescue => e
       app_file.download_to_web_failed!
       log << "Fatal error: " << e.to_s << "\n"
-    ensure
     end
-    
+
+    # save log and states to database without validation and callbacks
+    app_file.update_column(:code_signing_log, log)
   end
 end

app/models/app_file.rb

@@ -6,6 +6,8 @@ class AppFile < ActiveRecord::Base
   
   has_attached_file :archive
   validates_attachment_file_name :archive, matches: [/apk\Z/, /zip\Z/]
+  validates_with AttachmentPresenceValidator, attributes: :archive, :on => :create
+  validates_with AttachmentSizeValidator, attributes: :archive, less_than: 300.megabytes
 
   enum code_signing_status: { unsigned: 0, downloading_to_mac: 4, signing: 1, signing_failed: 2, pending_transfer_to_web: 5, downloading_to_web: 6, download_to_web_failed: 7, signed: 3 }
   after_commit :queue_for_ios_code_signing, :on => :create
@@ -21,7 +23,7 @@ class AppFile < ActiveRecord::Base
     if is_ios_app?
       # make a secret random key for direct download
       require 'securerandom'
-      download_token = SecureRandom.hex
+      update_column(:download_token, SecureRandom.hex)
 
       BuildIpaJob.perform_later id
     end

config/application.rb

@@ -22,5 +22,9 @@ module Appstore
 
     # Do not swallow errors in after_commit/after_rollback callbacks.
     config.active_record.raise_in_transactional_callbacks = true
+
+    # Be sure to have the adapter's gem in your Gemfile and follow
+    # the adapter's specific installation and deployment instructions.
+    config.active_job.queue_adapter = :resque
   end
 end

config/initializers/resque.rb

+rails_root = ENV['RAILS_ROOT'] || File.dirname(__FILE__) + '/../..'
+rails_env = ENV['RAILS_ENV'] || 'development'
+
+resque_config = YAML.load_file(rails_root + '/config/resque.yml')
+Resque.redis = resque_config[rails_env]

config/resque.yml

+development: localhost:6379
+test: localhost:6379
+production: appstore.colab.duke.edu:6379

lib/tasks/resque.rake

+require 'resque/tasks'