Commit 24b48865 authored by jz143's avatar jz143

Merge branch 'job_queue'

parents 87d9ba08 4fbe4a03
......@@ -14,7 +14,7 @@ gem 'uglifier', '>= 1.3.0'
# Use CoffeeScript for .coffee assets and views
gem 'coffee-rails', '~> 4.1.0'
# See https://github.com/rails/execjs#readme for more supported runtimes
gem 'therubyracer', platforms: :ruby
gem 'therubyracer', platforms: :ruby, group: :production
# Use jquery as the JavaScript library
gem 'jquery-rails'
......@@ -64,7 +64,8 @@ gem 'simple_form'
# authentication
# gem 'devise', "~> 3.5.2"
gem 'net-ssh'
gem 'net-ssh', '~> 3.1.0'
gem 'net-scp', '~> 1.2.1'
# authorization
gem "pundit"
......@@ -78,4 +79,7 @@ gem "slack-notifier", '~> 1.5.1'
gem "omniauth", '~> 1.3.1'
gem "omniauth-duke-oauth2", :git => 'git@gitlab.oit.duke.edu:colab/omniauth-duke-oauth2.git', :ref => '5eaf6759'
# job queue
gem 'resque', '~> 1.26.0'
gem 'bootstrap-sass', '~> 3.3.5'
......@@ -125,13 +125,14 @@ GEM
mimemagic (0.3.0)
mini_portile (0.6.2)
minitest (5.8.0)
mono_logger (1.1.0)
multi_json (1.11.2)
multi_xml (0.5.5)
multipart-post (2.0.0)
mysql2 (0.3.20)
net-scp (1.2.1)
net-ssh (>= 2.6.5)
net-ssh (3.0.1)
net-ssh (3.1.1)
nokogiri (1.6.6.2)
mini_portile (~> 0.6.0)
oauth2 (1.1.0)
......@@ -155,6 +156,8 @@ GEM
pundit (1.0.1)
activesupport (>= 3.0.0)
rack (1.6.4)
rack-protection (1.5.3)
rack
rack-test (0.6.3)
rack (>= 1.0)
rails (4.2.4)
......@@ -183,7 +186,16 @@ GEM
thor (>= 0.18.1, < 2.0)
rake (10.4.2)
rdoc (4.2.0)
redis (3.2.2)
redis-namespace (1.5.2)
redis (~> 3.0, >= 3.0.4)
ref (2.0.0)
resque (1.26.0)
mono_logger (~> 1.0)
multi_json (~> 1.0)
redis-namespace (~> 1.3)
sinatra (>= 0.9.2)
vegas (~> 0.1.2)
sass (3.4.17)
sass-rails (5.0.3)
railties (>= 4.0.0, < 5.0)
......@@ -197,6 +209,10 @@ GEM
simple_form (3.2.0)
actionpack (~> 4.0)
activemodel (~> 4.0)
sinatra (1.4.7)
rack (~> 1.5)
rack-protection (~> 1.4)
tilt (>= 1.3, < 3)
slack-notifier (1.5.1)
spring (1.3.6)
sprockets (3.3.3)
......@@ -223,6 +239,8 @@ GEM
uglifier (2.7.1)
execjs (>= 0.3.0)
json (>= 1.8.0)
vegas (0.1.11)
rack (>= 1.0.0)
web-console (2.2.1)
activemodel (>= 4.0)
binding_of_caller (>= 0.7.2)
......@@ -245,13 +263,15 @@ DEPENDENCIES
jbuilder (~> 2.0)
jquery-rails
mysql2 (~> 0.3.18)
net-ssh
net-scp (~> 1.2.1)
net-ssh (~> 3.1.0)
omniauth (~> 1.3.1)
omniauth-duke-oauth2!
paperclip (~> 4.3)
plist!
pundit
rails (= 4.2.4)
resque (~> 1.26.0)
sass-rails (~> 5.0)
sdoc (~> 0.4.0)
simple_form
......@@ -264,4 +284,4 @@ DEPENDENCIES
web-console (~> 2.0)
BUNDLED WITH
1.10.6
1.11.2
......@@ -17,6 +17,14 @@ class AppFilesController < ApplicationController
def download
@app_file = AppFile.find(params[:app_file_id])
respond_to do |format|
format.zip do
if params[:download_token] == @app_file.download_token
send_file @app_file.archive.path
else
head :bad_request
end
end
format.apk do
send_file @app_file.archive.path
@app_file.increment!(:download_count)
......@@ -24,11 +32,15 @@ class AppFilesController < ApplicationController
format.plist do
render 'manifest.plist'
@app_file.increment!(:download_count)
end
format.ipa do
send_file @app_file.signed_ipa_location, :type => 'application/octet-stream'
@app_file.increment!(:download_count)
if @app_file.signed?
send_file @app_file.signed_ipa_location, :type => 'application/octet-stream'
else
render json: { :error => 'Signed ipa is not available yet' }
end
end
end
end
......
class SessionsController < ApplicationController
def create_during_test
if Rails.env.test?
if Rails.env.test? || Rails.env.development?
session[:user_id] = params[:user_id]
end
redirect_to :root
......
require 'open-uri'
class BuildIpaJob < ActiveJob::Base
queue_as :default
queue_as :mac_online
def perform(app_file_id)
ActiveRecord::Base.clear_active_connections!
# get corresponding app_file record
app_file = AppFile.find(app_file_id)
if app_file.platform != 'iOS'
return # this job doesn't sign non-iOS apps
# this job doesn't sign non-iOS apps
unless app_file.is_ios_app?
return
end
log = ""
......@@ -15,19 +20,23 @@ class BuildIpaJob < ActiveJob::Base
app_file.signing!
tmpdir = Dir.mktmpdir
success = false
begin
log << "#{self.class.name} using tmp dir: #{tmpdir}\n"
# ensure we have the .app file on this machine
# NOTE - in the future, add the ability to copy from remote
unless File.file?(app_file.archive.path)
raise "App file (#{app_file.archive.path}) doesn't exist on worker"
# fetch app file from remote web server (assuming we are worker on another machine)
# we use remote's HTTP send_file here to reduce ruby workload
File.open(File.join(tmpdir, 'Unsigned.app.zip'), "wb") do |file|
url_options = Rails.application.config.action_mailer.default_url_options
url_options[:format] = 'zip'
url_options[:download_token] = app_file.download_token
download_url = Rails.application.routes.url_helpers.app_app_file_download_url(app_file.app, app_file, url_options)
log << "Downloading from #{download_url}\n"
open(download_url, "rb") do |read_file|
file.write(read_file.read)
end
end
# copy file to tmp folder
# NOTE - future: copy file from sshfs to tmp folder
FileUtils.copy_file(app_file.archive.path, File.join(tmpdir, 'Unsigned.app.zip'))
# Mac automatically zips .app folder, so we need to unzip it
log << "Unzipping archive...\n"
IO.popen(['unzip', File.join(tmpdir, 'Unsigned.app.zip'), '-d', File.join(tmpdir, 'UnsignedArchiveContent')]) do |unzip_io|
......@@ -60,6 +69,7 @@ class BuildIpaJob < ActiveJob::Base
log << "--- END ENTITLEMENTS ---\n"
# call signing script, collect output
app_file.signed_ipa_path_on_signing_server = File.join(tmpdir, 'Signed.ipa')
signing_script_args = [
'floatsign.sh',
File.join(tmpdir, 'UnsignedArchiveContent', app_folder), # app archive to sign
......@@ -68,7 +78,7 @@ class BuildIpaJob < ActiveJob::Base
'-p', Rails.root.join('resources', 'ios_provisioning_profiles', 'CoLab_Member_Apps_20151.mobileprovision').to_s,
'-b', app_file.app.ios_application_identifier,
'-t', File.join(tmpdir, 'floatsign_tmp'),
File.join(tmpdir, 'Signed.ipa')
app_file.signed_ipa_path_on_signing_server
]
log << "Signing with args: " << signing_script_args.collect { |a| "\"#{a}\"" }.join(' ') << "\n"
......@@ -81,23 +91,26 @@ class BuildIpaJob < ActiveJob::Base
raise "Signing script exited with non-zero status (#{$?})"
end
# save ipa file to the same directory as unsigned .app file
FileUtils.copy_file(File.join(tmpdir, 'Signed.ipa'), app_file.signed_ipa_location)
app_file.signed!
app_file.pending_transfer_to_web!
success = true
rescue => e
app_file.signing_failed!
log << "Fatal error: " << e.to_s << "\n"
ensure
FileUtils.remove_entry tmpdir
log << "Fatal error: " << e.to_s << "\n"
end
else
log << "Refusing to codesign when app_file is not freshly uploaded\n"
end
# save log to database
app_file.code_signing_log = log
app_file.save!
# save log and states to database without validation and callbacks
app_file.update_columns(
code_signing_log: log,
signed_ipa_path_on_signing_server: app_file.signed_ipa_path_on_signing_server)
if success
FetchedSignedIpaJob.perform_later app_file.id
else
FileUtils.remove_entry tmpdir
end
end
end
class FetchedSignedIpaJob < ActiveJob::Base
queue_as :web_background
def perform(app_file_id)
ActiveRecord::Base.clear_active_connections!
# get corresponding app_file record
app_file = AppFile.find(app_file_id)
log = app_file.code_signing_log
begin
app_file.downloading_to_web!
# get file to app_file.signed_ipa_location
log << "scp from sign:#{app_file.signed_ipa_path_on_signing_server} to #{app_file.signed_ipa_location}\n"
Net::SCP.download!(Settings.signing_worker.host, Settings.signing_worker.user,
app_file.signed_ipa_path_on_signing_server, app_file.signed_ipa_location)
# remove remote tmpdir and file
log << "removing sign:#{File.dirname(app_file.signed_ipa_path_on_signing_server)}\n"
Net::SSH.start(Settings.signing_worker.host, Settings.signing_worker.user) do |ssh|
ssh.exec "rm -rf \"#{File.dirname(app_file.signed_ipa_path_on_signing_server)}\""
end
# remove user uploaded unsigned package to save space
log << "removing local, unsigned copy\n"
app_file.archive.destroy
app_file.signed!
rescue => e
app_file.download_to_web_failed!
log << "Fatal error: " << e.to_s << "\n"
end
# save log and states to database without validation and callbacks
app_file.update_column(:code_signing_log, log)
end
end
......@@ -6,8 +6,10 @@ class AppFile < ActiveRecord::Base
has_attached_file :archive
validates_attachment_file_name :archive, matches: [/apk\Z/, /zip\Z/]
validates_with AttachmentPresenceValidator, attributes: :archive, :on => :create
validates_with AttachmentSizeValidator, attributes: :archive, less_than: 300.megabytes
enum code_signing_status: [ :unsigned, :signing, :signing_failed, :signed ]
enum code_signing_status: { unsigned: 0, downloading_to_mac: 4, signing: 1, signing_failed: 2, pending_transfer_to_web: 5, downloading_to_web: 6, download_to_web_failed: 7, signed: 3 }
after_commit :queue_for_ios_code_signing, :on => :create
def signed_ipa_location
......@@ -18,10 +20,13 @@ class AppFile < ActiveRecord::Base
puts "code_signing_status = " + code_signing_status
puts "platform = " + platform
puts "id = " + id.to_s
# if is_ios_app? && code_signing_status == :unsigned
# puts "QUEUEING THE APP"
if is_ios_app?
# make a secret random key for direct download
require 'securerandom'
update_column(:download_token, SecureRandom.hex)
BuildIpaJob.perform_later id
# end
end
end
def is_ios_app?
......
......@@ -22,5 +22,9 @@ module Appstore
# Do not swallow errors in after_commit/after_rollback callbacks.
config.active_record.raise_in_transactional_callbacks = true
# Be sure to have the adapter's gem in your Gemfile and follow
# the adapter's specific installation and deployment instructions.
config.active_job.queue_adapter = :resque
end
end
rails_root = ENV['RAILS_ROOT'] || File.dirname(__FILE__) + '/../..'
rails_env = ENV['RAILS_ENV'] || 'development'
resque_config = YAML.load_file(rails_root + '/config/resque.yml')
Resque.redis = resque_config[rails_env]
development: localhost:6379
test: localhost:6379
production: appstore.colab.duke.edu:6379
web_server:
host: localhost
user:
signing_worker:
host: localhost
user:
web:
host: appstore.colab.duke.edu
user: appstore
signing_worker:
host: istudio-osxserver.oit.duke.edu
user: appstore
class AddDownloadTokenToAppFile < ActiveRecord::Migration
def change
add_column :app_files, :download_token, :string
add_column :app_files, :signed_ipa_path_on_signing_server, :string
end
end
......@@ -11,22 +11,24 @@
#
# It's strongly recommended that you check this file into your version control system.
ActiveRecord::Schema.define(version: 20151124175550) do
ActiveRecord::Schema.define(version: 20160321154537) do
create_table "app_files", force: :cascade do |t|
t.string "platform"
t.string "version"
t.integer "download_count", default: 0
t.integer "download_count", default: 0
t.integer "app_id"
t.datetime "created_at", null: false
t.datetime "updated_at", null: false
t.datetime "created_at", null: false
t.datetime "updated_at", null: false
t.string "archive_file_name"
t.string "archive_content_type"
t.integer "archive_file_size"
t.datetime "archive_updated_at"
t.integer "user_id"
t.integer "code_signing_status", default: 0
t.integer "code_signing_status", default: 0
t.text "code_signing_log"
t.string "download_token"
t.string "signed_ipa_path_on_signing_server"
end
add_index "app_files", ["app_id"], name: "index_app_files_on_app_id"
......
require 'resque/tasks'
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment