From 3034c00bb33d46750d55d25ca9b95f1301ca4ab1 Mon Sep 17 00:00:00 2001 From: Pete Matsyburka Date: Tue, 18 Jun 2024 16:51:42 +0300 Subject: [PATCH] use sidekiq --- .gitignore | 1 + .rubocop.yml | 3 + Dockerfile | 2 +- Gemfile | 4 +- app/controllers/api/submissions_controller.rb | 6 +- .../api/submitter_form_views_controller.rb | 2 +- app/controllers/api/submitters_controller.rb | 2 +- .../api/templates_clone_controller.rb | 2 +- app/controllers/api/templates_controller.rb | 2 +- app/controllers/start_form_controller.rb | 2 +- app/controllers/submissions_controller.rb | 4 +- .../submitters_send_email_controller.rb | 2 +- app/controllers/templates_controller.rb | 4 +- .../templates_uploads_controller.rb | 2 +- .../webhook_settings_controller.rb | 2 +- app/jobs/process_submitter_completion_job.rb | 6 +- ...send_form_completed_webhook_request_job.rb | 17 +++-- .../send_form_started_webhook_request_job.rb | 17 +++-- .../send_form_viewed_webhook_request_job.rb | 17 +++-- ...submission_archived_webhook_request_job.rb | 17 +++-- ..._submission_created_webhook_request_job.rb | 17 +++-- .../send_submitter_invitation_email_job.rb | 4 +- ...nd_template_created_webhook_request_job.rb | 17 +++-- ...nd_template_updated_webhook_request_job.rb | 17 +++-- app/models/user.rb | 6 ++ config/application.rb | 2 +- config/boot.rb | 36 +-------- config/database.yml | 6 +- config/dotenv.rb | 46 ++++++++++++ config/environments/development.rb | 2 +- config/environments/production.rb | 2 +- config/environments/test.rb | 2 +- config/initializers/sidekiq.rb | 10 +-- config/puma.rb | 13 +++- config/routes.rb | 7 +- lib/puma/plugin/redis_server.rb | 70 ++++++++++++++++++ lib/puma/plugin/sidekiq_embed.rb | 74 +++++++++++++++++++ lib/submitters.rb | 2 +- lib/submitters/submit_values.rb | 4 +- spec/rails_helper.rb | 11 +++ spec/system/submit_form_spec.rb | 2 +- 41 files changed, 332 insertions(+), 132 deletions(-) create mode 100644 config/dotenv.rb create mode 100644 lib/puma/plugin/redis_server.rb create mode 100644 lib/puma/plugin/sidekiq_embed.rb diff --git a/.gitignore b/.gitignore index dd1db6a54..5f01e7181 100644 --- a/.gitignore +++ b/.gitignore @@ -36,3 +36,4 @@ yarn-debug.log* /attachments /docuseal /ee +dump.rdb diff --git a/.rubocop.yml b/.rubocop.yml index 4c2cd0429..7e2f433d2 100644 --- a/.rubocop.yml +++ b/.rubocop.yml @@ -76,5 +76,8 @@ Rails/SkipsModelValidations: Rails/ApplicationController: Enabled: false +Rails/Output: + Enabled: false + Capybara/ClickLinkOrButtonStyle: Enabled: false diff --git a/Dockerfile b/Dockerfile index c12b174f4..3d9f037f3 100644 --- a/Dockerfile +++ b/Dockerfile @@ -41,7 +41,7 @@ ENV OPENSSL_CONF=/app/openssl_legacy.cnf WORKDIR /app -RUN apk add --no-cache sqlite-dev libpq-dev mariadb-dev vips-dev vips-poppler poppler-utils vips-heif gcompat ttf-freefont && mkdir /fonts && rm /usr/share/fonts/freefont/FreeSans.otf +RUN apk add --no-cache sqlite-dev libpq-dev mariadb-dev vips-dev vips-poppler poppler-utils redis vips-heif gcompat ttf-freefont && mkdir /fonts && rm /usr/share/fonts/freefont/FreeSans.otf RUN echo $'.include = /etc/ssl/openssl.cnf\n\ \n\ diff --git a/Gemfile b/Gemfile index 10a7a26fd..7286a785d 100644 --- a/Gemfile +++ b/Gemfile @@ -27,7 +27,7 @@ gem 'pagy' gem 'pg', require: false gem 'premailer-rails' gem 'pretender' -gem 'puma' +gem 'puma', require: false gem 'rack' gem 'rails' gem 'rails_autolink' @@ -37,7 +37,7 @@ gem 'rqrcode' gem 'ruby-vips' gem 'rubyXL' gem 'shakapacker' -gem 'sidekiq', require: ENV.key?('REDIS_URL') +gem 'sidekiq' gem 'sqlite3', require: false, force_ruby_platform: true gem 'strip_attributes' gem 'turbo-rails' diff --git a/app/controllers/api/submissions_controller.rb b/app/controllers/api/submissions_controller.rb index 928205d1c..57076f7b7 100644 --- a/app/controllers/api/submissions_controller.rb +++ b/app/controllers/api/submissions_controller.rb @@ -64,14 +64,14 @@ def create submissions = create_submissions(@template, params) submissions.each do |submission| - SendSubmissionCreatedWebhookRequestJob.perform_later({ 'submission_id' => submission.id }) + SendSubmissionCreatedWebhookRequestJob.perform_async({ 'submission_id' => submission.id }) end Submissions.send_signature_requests(submissions) submissions.each do |submission| if submission.submitters.all?(&:completed_at?) && submission.submitters.last - ProcessSubmitterCompletionJob.perform_later({ 'submitter_id' => submission.submitters.last.id }) + ProcessSubmitterCompletionJob.perform_async({ 'submitter_id' => submission.submitters.last.id }) end end @@ -94,7 +94,7 @@ def destroy else @submission.update!(archived_at: Time.current) - SendSubmissionArchivedWebhookRequestJob.perform_later('submission_id' => @submission.id) + SendSubmissionArchivedWebhookRequestJob.perform_async('submission_id' => @submission.id) end render json: @submission.as_json(only: %i[id archived_at]) diff --git a/app/controllers/api/submitter_form_views_controller.rb b/app/controllers/api/submitter_form_views_controller.rb index dd24d0a55..b582095a4 100644 --- a/app/controllers/api/submitter_form_views_controller.rb +++ b/app/controllers/api/submitter_form_views_controller.rb @@ -13,7 +13,7 @@ def create SubmissionEvents.create_with_tracking_data(submitter, 'view_form', request) - SendFormViewedWebhookRequestJob.perform_later({ 'submitter_id' => submitter.id }) + SendFormViewedWebhookRequestJob.perform_async({ 'submitter_id' => submitter.id }) render json: {} end diff --git a/app/controllers/api/submitters_controller.rb b/app/controllers/api/submitters_controller.rb index e45b87ccc..6ee1ec35d 100644 --- a/app/controllers/api/submitters_controller.rb +++ b/app/controllers/api/submitters_controller.rb @@ -66,7 +66,7 @@ def update end if @submitter.completed_at? - ProcessSubmitterCompletionJob.perform_later({ 'submitter_id' => @submitter.id }) + ProcessSubmitterCompletionJob.perform_async({ 'submitter_id' => @submitter.id }) elsif normalized_params[:send_email] || normalized_params[:send_sms] Submitters.send_signature_requests([@submitter]) end diff --git a/app/controllers/api/templates_clone_controller.rb b/app/controllers/api/templates_clone_controller.rb index 3e881bcef..2c0407828 100644 --- a/app/controllers/api/templates_clone_controller.rb +++ b/app/controllers/api/templates_clone_controller.rb @@ -20,7 +20,7 @@ def create Templates::CloneAttachments.call(template: cloned_template, original_template: @template) - SendTemplateCreatedWebhookRequestJob.perform_later('template_id' => cloned_template.id) + SendTemplateCreatedWebhookRequestJob.perform_async('template_id' => cloned_template.id) render json: Templates::SerializeForApi.call(cloned_template) end diff --git a/app/controllers/api/templates_controller.rb b/app/controllers/api/templates_controller.rb index 830afc222..cecb193c9 100644 --- a/app/controllers/api/templates_controller.rb +++ b/app/controllers/api/templates_controller.rb @@ -65,7 +65,7 @@ def update @template.update!(template_params) - SendTemplateUpdatedWebhookRequestJob.perform_later('template_id' => @template.id) + SendTemplateUpdatedWebhookRequestJob.perform_async('template_id' => @template.id) render json: @template.as_json(only: %i[id updated_at]) end diff --git a/app/controllers/start_form_controller.rb b/app/controllers/start_form_controller.rb index b268a4136..d5f033c21 100644 --- a/app/controllers/start_form_controller.rb +++ b/app/controllers/start_form_controller.rb @@ -37,7 +37,7 @@ def update if @submitter.save if is_new_record - SendSubmissionCreatedWebhookRequestJob.perform_later({ 'submission_id' => @submitter.submission.id }) + SendSubmissionCreatedWebhookRequestJob.perform_async({ 'submission_id' => @submitter.submission.id }) end redirect_to submit_form_path(@submitter.slug) diff --git a/app/controllers/submissions_controller.rb b/app/controllers/submissions_controller.rb index dc45d5953..0de0f7895 100644 --- a/app/controllers/submissions_controller.rb +++ b/app/controllers/submissions_controller.rb @@ -45,7 +45,7 @@ def create end submissions.each do |submission| - SendSubmissionCreatedWebhookRequestJob.perform_later({ 'submission_id' => submission.id }) + SendSubmissionCreatedWebhookRequestJob.perform_async({ 'submission_id' => submission.id }) end Submissions.send_signature_requests(submissions) @@ -56,7 +56,7 @@ def create def destroy @submission.update!(archived_at: Time.current) - SendSubmissionArchivedWebhookRequestJob.perform_later('submission_id' => @submission.id) + SendSubmissionArchivedWebhookRequestJob.perform_async('submission_id' => @submission.id) redirect_back(fallback_location: template_path(@submission.template), notice: 'Submission has been archived') end diff --git a/app/controllers/submitters_send_email_controller.rb b/app/controllers/submitters_send_email_controller.rb index 5ac98ead2..97cc393d2 100644 --- a/app/controllers/submitters_send_email_controller.rb +++ b/app/controllers/submitters_send_email_controller.rb @@ -13,7 +13,7 @@ def create alert: 'Email has been sent already.') end - SendSubmitterInvitationEmailJob.perform_later('submitter_id' => @submitter.id) + SendSubmitterInvitationEmailJob.perform_async('submitter_id' => @submitter.id) @submitter.sent_at ||= Time.current @submitter.save! diff --git a/app/controllers/templates_controller.rb b/app/controllers/templates_controller.rb index a7a682f68..ef91a0050 100644 --- a/app/controllers/templates_controller.rb +++ b/app/controllers/templates_controller.rb @@ -61,7 +61,7 @@ def create if @template.save Templates::CloneAttachments.call(template: @template, original_template: @base_template) if @base_template - SendTemplateUpdatedWebhookRequestJob.perform_later('template_id' => @template.id) + SendTemplateUpdatedWebhookRequestJob.perform_async('template_id' => @template.id) maybe_redirect_to_template(@template) else @@ -72,7 +72,7 @@ def create def update @template.update!(template_params) - SendTemplateUpdatedWebhookRequestJob.perform_later('template_id' => @template.id) + SendTemplateUpdatedWebhookRequestJob.perform_async('template_id' => @template.id) head :ok end diff --git a/app/controllers/templates_uploads_controller.rb b/app/controllers/templates_uploads_controller.rb index 4c0eea23c..4519eb787 100644 --- a/app/controllers/templates_uploads_controller.rb +++ b/app/controllers/templates_uploads_controller.rb @@ -18,7 +18,7 @@ def create @template.update!(schema:) - SendTemplateCreatedWebhookRequestJob.perform_later('template_id' => @template.id) + SendTemplateCreatedWebhookRequestJob.perform_async('template_id' => @template.id) redirect_to edit_template_path(@template) rescue Templates::CreateAttachments::PdfEncrypted diff --git a/app/controllers/webhook_settings_controller.rb b/app/controllers/webhook_settings_controller.rb index 146a759c1..d61aa30dd 100644 --- a/app/controllers/webhook_settings_controller.rb +++ b/app/controllers/webhook_settings_controller.rb @@ -17,7 +17,7 @@ def create def update submitter = current_account.submitters.where.not(completed_at: nil).order(:id).last - SendFormCompletedWebhookRequestJob.perform_later({ 'submitter_id' => submitter.id }) + SendFormCompletedWebhookRequestJob.perform_async({ 'submitter_id' => submitter.id }) redirect_back(fallback_location: settings_webhooks_path, notice: 'Webhook request has been sent.') end diff --git a/app/jobs/process_submitter_completion_job.rb b/app/jobs/process_submitter_completion_job.rb index b3fd92572..64bec6e96 100644 --- a/app/jobs/process_submitter_completion_job.rb +++ b/app/jobs/process_submitter_completion_job.rb @@ -1,6 +1,8 @@ # frozen_string_literal: true -class ProcessSubmitterCompletionJob < ApplicationJob +class ProcessSubmitterCompletionJob + include Sidekiq::Job + def perform(params = {}) submitter = Submitter.find(params['submitter_id']) @@ -20,7 +22,7 @@ def perform(params = {}) return if Accounts.load_webhook_url(submitter.account).blank? - SendFormCompletedWebhookRequestJob.perform_later({ 'submitter_id' => submitter.id }) + SendFormCompletedWebhookRequestJob.perform_async({ 'submitter_id' => submitter.id }) end def enqueue_completed_emails(submitter) diff --git a/app/jobs/send_form_completed_webhook_request_job.rb b/app/jobs/send_form_completed_webhook_request_job.rb index 104cd5293..8f417278f 100644 --- a/app/jobs/send_form_completed_webhook_request_job.rb +++ b/app/jobs/send_form_completed_webhook_request_job.rb @@ -1,7 +1,9 @@ # frozen_string_literal: true -class SendFormCompletedWebhookRequestJob < ApplicationJob - queue_as :webhooks +class SendFormCompletedWebhookRequestJob + include Sidekiq::Job + + sidekiq_options queue: :webhooks USER_AGENT = 'DocuSeal.co Webhook' @@ -38,12 +40,11 @@ def perform(params = {}) if (resp.nil? || resp.status.to_i >= 400) && attempt <= MAX_ATTEMPTS && (!Docuseal.multitenant? || submitter.account.account_configs.exists?(key: :plan)) - SendFormCompletedWebhookRequestJob.set(wait: (2**attempt).minutes) - .perform_later({ - 'submitter_id' => submitter.id, - 'attempt' => attempt + 1, - 'last_status' => resp&.status.to_i - }) + SendFormCompletedWebhookRequestJob.perform_in((2**attempt).minutes, { + 'submitter_id' => submitter.id, + 'attempt' => attempt + 1, + 'last_status' => resp&.status.to_i + }) end end end diff --git a/app/jobs/send_form_started_webhook_request_job.rb b/app/jobs/send_form_started_webhook_request_job.rb index 2770c6d80..9bf8f6c83 100644 --- a/app/jobs/send_form_started_webhook_request_job.rb +++ b/app/jobs/send_form_started_webhook_request_job.rb @@ -1,7 +1,9 @@ # frozen_string_literal: true -class SendFormStartedWebhookRequestJob < ApplicationJob - queue_as :webhooks +class SendFormStartedWebhookRequestJob + include Sidekiq::Job + + sidekiq_options queue: :webhooks USER_AGENT = 'DocuSeal.co Webhook' @@ -36,12 +38,11 @@ def perform(params = {}) if (resp.nil? || resp.status.to_i >= 400) && attempt <= MAX_ATTEMPTS && (!Docuseal.multitenant? || submitter.account.account_configs.exists?(key: :plan)) - SendFormStartedWebhookRequestJob.set(wait: (2**attempt).minutes) - .perform_later({ - 'submitter_id' => submitter.id, - 'attempt' => attempt + 1, - 'last_status' => resp&.status.to_i - }) + SendFormStartedWebhookRequestJob.perform_in((2**attempt).minutes, { + 'submitter_id' => submitter.id, + 'attempt' => attempt + 1, + 'last_status' => resp&.status.to_i + }) end end end diff --git a/app/jobs/send_form_viewed_webhook_request_job.rb b/app/jobs/send_form_viewed_webhook_request_job.rb index 6bc51c947..9cf6cc545 100644 --- a/app/jobs/send_form_viewed_webhook_request_job.rb +++ b/app/jobs/send_form_viewed_webhook_request_job.rb @@ -1,7 +1,9 @@ # frozen_string_literal: true -class SendFormViewedWebhookRequestJob < ApplicationJob - queue_as :webhooks +class SendFormViewedWebhookRequestJob + include Sidekiq::Job + + sidekiq_options queue: :webhooks USER_AGENT = 'DocuSeal.co Webhook' @@ -36,12 +38,11 @@ def perform(params = {}) if (resp.nil? || resp.status.to_i >= 400) && attempt <= MAX_ATTEMPTS && (!Docuseal.multitenant? || submitter.account.account_configs.exists?(key: :plan)) - SendFormViewedWebhookRequestJob.set(wait: (2**attempt).minutes) - .perform_later({ - 'submitter_id' => submitter.id, - 'attempt' => attempt + 1, - 'last_status' => resp&.status.to_i - }) + SendFormViewedWebhookRequestJob.perform_in((2**attempt).minutes, { + 'submitter_id' => submitter.id, + 'attempt' => attempt + 1, + 'last_status' => resp&.status.to_i + }) end end end diff --git a/app/jobs/send_submission_archived_webhook_request_job.rb b/app/jobs/send_submission_archived_webhook_request_job.rb index 3b9ae202d..76273ceec 100644 --- a/app/jobs/send_submission_archived_webhook_request_job.rb +++ b/app/jobs/send_submission_archived_webhook_request_job.rb @@ -1,7 +1,9 @@ # frozen_string_literal: true -class SendSubmissionArchivedWebhookRequestJob < ApplicationJob - queue_as :webhooks +class SendSubmissionArchivedWebhookRequestJob + include Sidekiq::Job + + sidekiq_options queue: :webhooks USER_AGENT = 'DocuSeal.co Webhook' @@ -34,12 +36,11 @@ def perform(params = {}) if (resp.nil? || resp.status.to_i >= 400) && attempt <= MAX_ATTEMPTS && (!Docuseal.multitenant? || submission.account.account_configs.exists?(key: :plan)) - SendSubmissionArchivedWebhookRequestJob.set(wait: (2**attempt).minutes) - .perform_later({ - 'submission_id' => submission.id, - 'attempt' => attempt + 1, - 'last_status' => resp&.status.to_i - }) + SendSubmissionArchivedWebhookRequestJob.perform_in((2**attempt).minutes, { + 'submission_id' => submission.id, + 'attempt' => attempt + 1, + 'last_status' => resp&.status.to_i + }) end end end diff --git a/app/jobs/send_submission_created_webhook_request_job.rb b/app/jobs/send_submission_created_webhook_request_job.rb index 13dc26237..e5ec70700 100644 --- a/app/jobs/send_submission_created_webhook_request_job.rb +++ b/app/jobs/send_submission_created_webhook_request_job.rb @@ -1,7 +1,9 @@ # frozen_string_literal: true -class SendSubmissionCreatedWebhookRequestJob < ApplicationJob - queue_as :webhooks +class SendSubmissionCreatedWebhookRequestJob + include Sidekiq::Job + + sidekiq_options queue: :webhooks USER_AGENT = 'DocuSeal.co Webhook' @@ -34,12 +36,11 @@ def perform(params = {}) if (resp.nil? || resp.status.to_i >= 400) && attempt <= MAX_ATTEMPTS && (!Docuseal.multitenant? || submission.account.account_configs.exists?(key: :plan)) - SendSubmissionCreatedWebhookRequestJob.set(wait: (2**attempt).minutes) - .perform_later({ - 'submission_id' => submission.id, - 'attempt' => attempt + 1, - 'last_status' => resp&.status.to_i - }) + SendSubmissionCreatedWebhookRequestJob.perform_in((2**attempt).minutes, { + 'submission_id' => submission.id, + 'attempt' => attempt + 1, + 'last_status' => resp&.status.to_i + }) end end end diff --git a/app/jobs/send_submitter_invitation_email_job.rb b/app/jobs/send_submitter_invitation_email_job.rb index 861974290..7ec9e4672 100644 --- a/app/jobs/send_submitter_invitation_email_job.rb +++ b/app/jobs/send_submitter_invitation_email_job.rb @@ -1,6 +1,8 @@ # frozen_string_literal: true -class SendSubmitterInvitationEmailJob < ApplicationJob +class SendSubmitterInvitationEmailJob + include Sidekiq::Job + def perform(params = {}) submitter = Submitter.find(params['submitter_id']) diff --git a/app/jobs/send_template_created_webhook_request_job.rb b/app/jobs/send_template_created_webhook_request_job.rb index f508850fd..ed6434509 100644 --- a/app/jobs/send_template_created_webhook_request_job.rb +++ b/app/jobs/send_template_created_webhook_request_job.rb @@ -1,7 +1,9 @@ # frozen_string_literal: true -class SendTemplateCreatedWebhookRequestJob < ApplicationJob - queue_as :webhooks +class SendTemplateCreatedWebhookRequestJob + include Sidekiq::Job + + sidekiq_options queue: :webhooks USER_AGENT = 'DocuSeal.co Webhook' @@ -34,12 +36,11 @@ def perform(params = {}) if (resp.nil? || resp.status.to_i >= 400) && attempt <= MAX_ATTEMPTS && (!Docuseal.multitenant? || template.account.account_configs.exists?(key: :plan)) - SendTemplateCreatedWebhookRequestJob.set(wait: (2**attempt).minutes) - .perform_later({ - 'template_id' => template.id, - 'attempt' => attempt + 1, - 'last_status' => resp&.status.to_i - }) + SendTemplateCreatedWebhookRequestJob.perform_in((2**attempt).minutes, { + 'template_id' => template.id, + 'attempt' => attempt + 1, + 'last_status' => resp&.status.to_i + }) end end end diff --git a/app/jobs/send_template_updated_webhook_request_job.rb b/app/jobs/send_template_updated_webhook_request_job.rb index 5d622ba2d..638ba196b 100644 --- a/app/jobs/send_template_updated_webhook_request_job.rb +++ b/app/jobs/send_template_updated_webhook_request_job.rb @@ -1,7 +1,9 @@ # frozen_string_literal: true -class SendTemplateUpdatedWebhookRequestJob < ApplicationJob - queue_as :webhooks +class SendTemplateUpdatedWebhookRequestJob + include Sidekiq::Job + + sidekiq_options queue: :webhooks USER_AGENT = 'DocuSeal.co Webhook' @@ -34,12 +36,11 @@ def perform(params = {}) if (resp.nil? || resp.status.to_i >= 400) && attempt <= MAX_ATTEMPTS && (!Docuseal.multitenant? || template.account.account_configs.exists?(key: :plan)) - SendTemplateUpdatedWebhookRequestJob.set(wait: (2**attempt).minutes) - .perform_later({ - 'template_id' => template.id, - 'attempt' => attempt + 1, - 'last_status' => resp&.status.to_i - }) + SendTemplateUpdatedWebhookRequestJob.perform_in((2**attempt).minutes, { + 'template_id' => template.id, + 'attempt' => attempt + 1, + 'last_status' => resp&.status.to_i + }) end end end diff --git a/app/models/user.rb b/app/models/user.rb index 1948bfe01..6af98ce0e 100644 --- a/app/models/user.rb +++ b/app/models/user.rb @@ -81,6 +81,12 @@ def remember_me true end + def sidekiq? + return true if Rails.env.development? + + role == 'admin' + end + def self.sign_in_after_reset_password if PasswordsController::Current.user.present? !PasswordsController::Current.user.otp_required_for_login diff --git a/config/application.rb b/config/application.rb index ffb886f89..a1ab003c6 100644 --- a/config/application.rb +++ b/config/application.rb @@ -19,7 +19,7 @@ module DocuSeal class Application < Rails::Application config.load_defaults 7.1 - config.autoload_lib(ignore: %w[assets tasks]) + config.autoload_lib(ignore: %w[assets tasks puma]) config.active_storage.routes_prefix = '' diff --git a/config/boot.rb b/config/boot.rb index 481cacb77..e4231ebc8 100644 --- a/config/boot.rb +++ b/config/boot.rb @@ -2,41 +2,7 @@ ENV['BUNDLE_GEMFILE'] ||= File.expand_path('../Gemfile', __dir__) -if ENV['RAILS_ENV'] == 'production' && ENV['SECRET_KEY_BASE'].to_s.empty? - require 'dotenv' - require 'securerandom' - - dotenv_path = "#{ENV.fetch('WORKDIR', '.')}/docuseal.env" - - unless File.exist?(dotenv_path) - default_env = <<~TEXT - DATABASE_URL= # keep empty to use sqlite or specify postgresql database URL - SECRET_KEY_BASE=#{SecureRandom.hex(64)} - TEXT - - File.write(dotenv_path, default_env) - end - - database_url = ENV.fetch('DATABASE_URL', nil) - - Dotenv.load(dotenv_path) - - ENV['DATABASE_URL'] = ENV['DATABASE_URL'].to_s.empty? ? database_url : ENV.fetch('DATABASE_URL', nil) -end - -if ENV['DATABASE_URL'].to_s.split('@').last.to_s.split('/').first.to_s.include?('_') - require 'addressable' - - url = Addressable::URI.parse(ENV.fetch('DATABASE_URL', '')) - - ENV['DATABASE_HOST'] = url.host - ENV['DATABASE_PORT'] = (url.port || 5432).to_s - ENV['DATABASE_USER'] = url.user - ENV['DATABASE_PASSWORD'] = url.password - ENV['DATABASE_NAME'] = url.path.to_s.delete_prefix('/') - - ENV.delete('DATABASE_URL') -end +require_relative 'dotenv' require 'bundler/setup' # Set up gems listed in the Gemfile. require 'bootsnap/setup' # Speed up boot time by caching expensive operations. diff --git a/config/database.yml b/config/database.yml index a4c503abd..f26c54efc 100644 --- a/config/database.yml +++ b/config/database.yml @@ -1,7 +1,7 @@ default: &default adapter: postgresql encoding: unicode - pool: <%= ENV.fetch('RAILS_MAX_THREADS', 15) %> + pool: <%= ENV.fetch('RAILS_MAX_THREADS', 15).to_i + ENV.fetch('SIDEKIQ_THREADS', 5).to_i %> development: <<: *default @@ -22,12 +22,14 @@ production: <% elsif ENV['DATABASE_URL'].to_s.empty? %> adapter: sqlite3 database: <%= ENV['WORKDIR'] || '.' %>/db.sqlite3 + pool: <%= ENV.fetch('RAILS_MAX_THREADS', 15).to_i + ENV.fetch('SIDEKIQ_THREADS', 5).to_i %> + timeout: 5000 <% elsif ENV['DATABASE_URL'].match?(/\Apostgres/) %> <<: *default url: <%= ENV['DATABASE_URL'] %> <% elsif ENV['DATABASE_URL'].match?(/\Amysql/) %> adapter: mysql2 encoding: utf8mb4 - pool: <%= ENV.fetch('RAILS_MAX_THREADS', 15) %> + pool: <%= ENV.fetch('RAILS_MAX_THREADS', 15).to_i + ENV.fetch('SIDEKIQ_THREADS', 5).to_i %> url: <%= ENV['DATABASE_URL'] %> <% end %> diff --git a/config/dotenv.rb b/config/dotenv.rb new file mode 100644 index 000000000..70ffa3287 --- /dev/null +++ b/config/dotenv.rb @@ -0,0 +1,46 @@ +# frozen_string_literal: true + +if ENV['RAILS_ENV'] == 'production' && ENV['SECRET_KEY_BASE'].to_s.empty? + require 'dotenv' + require 'securerandom' + + dotenv_path = "#{ENV.fetch('WORKDIR', '.')}/docuseal.env" + + unless File.exist?(dotenv_path) + default_env = <<~TEXT + DATABASE_URL= # keep empty to use sqlite or specify postgresql database URL + SECRET_KEY_BASE=#{SecureRandom.hex(64)} + TEXT + + File.write(dotenv_path, default_env) + end + + database_url = ENV.fetch('DATABASE_URL', nil) + + Dotenv.load(dotenv_path) + + ENV['DATABASE_URL'] = ENV['DATABASE_URL'].to_s.empty? ? database_url : ENV.fetch('DATABASE_URL', nil) +end + +if ENV['DATABASE_URL'].to_s.split('@').last.to_s.split('/').first.to_s.include?('_') + require 'addressable' + + url = Addressable::URI.parse(ENV.fetch('DATABASE_URL', '')) + + ENV['DATABASE_HOST'] = url.host + ENV['DATABASE_PORT'] = (url.port || 5432).to_s + ENV['DATABASE_USER'] = url.user + ENV['DATABASE_PASSWORD'] = url.password + ENV['DATABASE_NAME'] = url.path.to_s.delete_prefix('/') + + ENV.delete('DATABASE_URL') +end + +if ENV['REDIS_URL'].to_s.empty? + require 'digest' + + redis_password = Digest::SHA1.hexdigest("redis#{ENV.fetch('SECRET_KEY_BASE', '')}") + + ENV['REDIS_URL'] = "redis://default:#{redis_password}@0.0.0.0:6379/0" + ENV['LOCAL_REDIS_URL'] = ENV.fetch('REDIS_URL', nil) +end diff --git a/config/environments/development.rb b/config/environments/development.rb index d619764df..e8cc2e815 100644 --- a/config/environments/development.rb +++ b/config/environments/development.rb @@ -47,7 +47,7 @@ config.cache_store = :null_store end - config.active_job.queue_adapter = :sidekiq if defined?(Sidekiq) + config.active_job.queue_adapter = :sidekiq # Store uploaded files on the local file system (see config/storage.yml for options). config.active_storage.service = :disk diff --git a/config/environments/production.rb b/config/environments/production.rb index e6cd5573c..3dff8fd73 100644 --- a/config/environments/production.rb +++ b/config/environments/production.rb @@ -27,7 +27,7 @@ config.action_controller.perform_caching = true config.active_record.sqlite3_production_warning = false - config.active_job.queue_adapter = :sidekiq if defined?(Sidekiq) + config.active_job.queue_adapter = :sidekiq # Ensures that a master key has been made available in either ENV["RAILS_MASTER_KEY"] # or in config/master.key. This key is used to decrypt credentials (and other encrypted files). diff --git a/config/environments/test.rb b/config/environments/test.rb index 116b53747..7944c00a7 100644 --- a/config/environments/test.rb +++ b/config/environments/test.rb @@ -44,7 +44,7 @@ # The :test delivery method accumulates sent emails in the # ActionMailer::Base.deliveries array. config.action_mailer.delivery_method = :test - config.active_job.queue_adapter = :test + config.active_job.queue_adapter = :sidekiq # Print deprecation notices to the stderr. config.active_support.deprecation = :stderr diff --git a/config/initializers/sidekiq.rb b/config/initializers/sidekiq.rb index 4733a1c17..1cb926cad 100644 --- a/config/initializers/sidekiq.rb +++ b/config/initializers/sidekiq.rb @@ -1,16 +1,14 @@ # frozen_string_literal: true -if defined?(Sidekiq) - require 'sidekiq/web' +require 'sidekiq/web' if defined?(Puma) +if !ENV['SIDEKIQ_BASIC_AUTH_PASSWORD'].to_s.empty? && defined?(Sidekiq::Web) Sidekiq::Web.use(Rack::Auth::Basic) do |_, password| - next true if Rails.env.development? - ActiveSupport::SecurityUtils.secure_compare( Digest::SHA256.hexdigest(password), Digest::SHA256.hexdigest(ENV.fetch('SIDEKIQ_BASIC_AUTH_PASSWORD')) ) end - - Sidekiq.strict_args! end + +Sidekiq.strict_args! diff --git a/config/puma.rb b/config/puma.rb index ef31eadf9..964a32983 100644 --- a/config/puma.rb +++ b/config/puma.rb @@ -5,7 +5,9 @@ # Any libraries that use thread pools should be configured to match # the maximum value specified for Puma. Default is set to 5 threads for minimum # and maximum; this matches the default thread size of Active Record. -# + +require_relative 'dotenv' + max_threads_count = ENV.fetch('RAILS_MAX_THREADS', 15) min_threads_count = ENV.fetch('RAILS_MIN_THREADS') { max_threads_count } threads min_threads_count, max_threads_count @@ -41,5 +43,10 @@ # # preload_app! -# Allow puma to be restarted by `bin/rails restart` command. -plugin :tmp_restart +if ENV['MULTITENANT'] != 'true' || ENV['DEMO'] == 'true' + require_relative '../lib/puma/plugin/redis_server' + require_relative '../lib/puma/plugin/sidekiq_embed' + + plugin :sidekiq_embed + plugin :redis_server +end diff --git a/config/routes.rb b/config/routes.rb index 171868be2..ad4c7d153 100644 --- a/config/routes.rb +++ b/config/routes.rb @@ -2,7 +2,12 @@ Rails.application.routes.draw do mount LetterOpenerWeb::Engine, at: '/letter_opener' if Rails.env.development? - mount Sidekiq::Web => '/sidekiq' if defined?(Sidekiq) + + if !Docuseal.multitenant? && defined?(Sidekiq::Web) + authenticated :user, ->(u) { u.sidekiq? } do + mount Sidekiq::Web => '/jobs' + end + end root 'dashboard#index' diff --git a/lib/puma/plugin/redis_server.rb b/lib/puma/plugin/redis_server.rb new file mode 100644 index 000000000..03b29dbfc --- /dev/null +++ b/lib/puma/plugin/redis_server.rb @@ -0,0 +1,70 @@ +# frozen_string_literal: true + +require 'puma/plugin' + +# rubocop:disable Metrics +Puma::Plugin.create do + def start(launcher) + return if ENV['LOCAL_REDIS_URL'].to_s.empty? + + @puma_pid = $PROCESS_ID + + launcher.events.on_booted do + @redis_server_pid = fork_redis + end + + in_background { monitor_redis } + + at_exit do + stop_redis_server if Process.pid == @puma_pid + end + + launcher.events.on_stopped { stop_redis_server } + launcher.events.on_restart { stop_redis_server } + end + + private + + def monitor_redis + loop do + if redis_dead? + Process.kill(:INT, @puma_pid) + + break + end + + sleep 5 + end + end + + def redis_dead? + return false unless @redis_server_pid + + Process.waitpid(@redis_server_pid, Process::WNOHANG) + + false + rescue Errno::ECHILD, Errno::ESRCH + true + end + + def fork_redis + fork do + Process.setsid + + Dir.chdir(ENV.fetch('WORKDIR', nil)) unless ENV['WORKDIR'].to_s.empty? + + exec('redis-server', '--requirepass', Digest::SHA1.hexdigest("redis#{ENV.fetch('SECRET_KEY_BASE', '')}"), + out: '/dev/null') + end + end + + def stop_redis_server + if @redis_server_pid + Process.kill(:INT, @redis_server_pid) + Process.wait(@redis_server_pid) + end + rescue Errno::ECHILD, Errno::ESRCH + nil + end +end +# rubocop:enable Metrics diff --git a/lib/puma/plugin/sidekiq_embed.rb b/lib/puma/plugin/sidekiq_embed.rb new file mode 100644 index 000000000..67ba2b79e --- /dev/null +++ b/lib/puma/plugin/sidekiq_embed.rb @@ -0,0 +1,74 @@ +# frozen_string_literal: true + +require 'puma/plugin' + +# rubocop:disable Metrics +Puma::Plugin.create do + def config(cfg) + return if cfg.instance_variable_get(:@options)[:workers] <= 0 + + cfg.on_worker_boot { start_sidekiq! } + + cfg.on_worker_shutdown { @sidekiq&.stop } + cfg.on_refork { @sidekiq&.stop } + end + + def start(launcher) + launcher.events.on_booted do + next if Puma.stats_hash[:workers].to_i != 0 + + start_sidekiq! + end + + launcher.events.on_stopped { Thread.new { @sidekiq&.stop }.join } + launcher.events.on_restart { Thread.new { @sidekiq&.stop }.join } + end + + def fire_event(config, event) + arr = config[:lifecycle_events][event] + + arr.each(&:call) + + arr.clear + end + + def start_sidekiq! + Thread.new do + wait_for_redis! + + configs = Sidekiq.configure_embed do |config| + config.logger.level = Logger::INFO + sidekiq_config = YAML.load_file('config/sidekiq.yml') + config.queues = sidekiq_config['queues'] + config.concurrency = ENV.fetch('SIDEKIQ_THREADS', 5).to_i + config.merge!(sidekiq_config) + config[:max_retries] = 13 + + ActiveSupport.run_load_hooks(:sidekiq_config, config) + end.instance_variable_get(:@config) + + @sidekiq = Sidekiq::Launcher.new(configs, embedded: true) + + @sidekiq.run + + fire_event(configs, :startup) + end + end + + def wait_for_redis! + attempt = 0 + + loop do + attempt += 1 + + sleep (attempt - 1) / 10.0 + + RedisClient.new(url: ENV.fetch('REDIS_URL', nil)).call('GET', '1') + + break + rescue RedisClient::CannotConnectError + raise('Unable to connect to redis') if attempt > 10 + end + end +end +# rubocop:enable Metrics diff --git a/lib/submitters.rb b/lib/submitters.rb index 2be8b6d5e..ebe3a326a 100644 --- a/lib/submitters.rb +++ b/lib/submitters.rb @@ -73,7 +73,7 @@ def send_signature_requests(submitters) next if submitter.email.blank? next if submitter.preferences['send_email'] == false - SendSubmitterInvitationEmailJob.perform_later('submitter_id' => submitter.id) + SendSubmitterInvitationEmailJob.perform_async('submitter_id' => submitter.id) end end end diff --git a/lib/submitters/submit_values.rb b/lib/submitters/submit_values.rb index f65c9a59e..f71f132a0 100644 --- a/lib/submitters/submit_values.rb +++ b/lib/submitters/submit_values.rb @@ -14,14 +14,14 @@ def call(submitter, params, request) unless submitter.submission_events.exists?(event_type: 'start_form') SubmissionEvents.create_with_tracking_data(submitter, 'start_form', request) - SendFormStartedWebhookRequestJob.perform_later({ 'submitter_id' => submitter.id }) + SendFormStartedWebhookRequestJob.perform_async({ 'submitter_id' => submitter.id }) end update_submitter!(submitter, params, request) submitter.submission.save! - ProcessSubmitterCompletionJob.perform_later({ 'submitter_id' => submitter.id }) if submitter.completed_at? + ProcessSubmitterCompletionJob.perform_async({ 'submitter_id' => submitter.id }) if submitter.completed_at? submitter end diff --git a/spec/rails_helper.rb b/spec/rails_helper.rb index 49d72854b..ce6dd9854 100644 --- a/spec/rails_helper.rb +++ b/spec/rails_helper.rb @@ -9,6 +9,9 @@ require 'capybara/cuprite' require 'capybara/rspec' require 'webmock/rspec' +require 'sidekiq/testing' + +Sidekiq::Testing.fake! WebMock.disable_net_connect!(allow_localhost: true) @@ -56,4 +59,12 @@ driven_by :headless_cuprite end end + + config.before do + Sidekiq::Worker.clear_all + end + + config.before do |example| + Sidekiq::Testing.inline! if example.metadata[:sidekiq] == :inline + end end diff --git a/spec/system/submit_form_spec.rb b/spec/system/submit_form_spec.rb index 187cee1e0..67ccedc45 100644 --- a/spec/system/submit_form_spec.rb +++ b/spec/system/submit_form_spec.rb @@ -81,7 +81,7 @@ expect do click_on 'submit' - end.to change(enqueued_jobs, :size).by(2) + end.to change(enqueued_jobs, :size).by(1) end end end