Add job and task for attachment cleanup (#224)

* Add job and task for attachment cleanup

* files and fixtures for tests

* rake task working, tests added.

* fix issues with test fixtures

* trying to fix tests

* purge files before object destroy

* handle file deletion in batches

* improve readability

* fix and simplify tests

app/jobs/attachments_cleanup_job.rb

@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+# delete/destroy attachements that have been deleted X days ago
+class AttachmentsCleanupJob < ApplicationJob
+  queue_as :default
+
+  # Finds all deleted attachments more than `days_old`` days old, and destroys them
+  # Params:
+  # +days_old+:: positive integer. Number of days old and older to destroy. Default is 7
+  def perform(days_old = 7)
+    if !days_old.instance_of?(Integer) || (days_old < 1)
+      err = "'#{days_old}' is not a positive integer!"
+      Rails.logger.error err
+      raise err
+    end
+
+    Rails.logger.info "Cleaning up all deleted attachments which are at least #{days_old} days old."
+
+    # SELECT "attachments".* FROM "attachments"
+    #   WHERE "attachments"."deleted_at" IS NOT NULL AND "attachements"."deleted_at" <= $1
+    attachments_to_delete = Attachment.only_deleted.where(deleted_at: ..(Date.yesterday.midnight - days_old.day))
+    attachments_to_delete.find_in_batches(batch_size: 50) do |group|
+      group.each do |att|
+        att.file.purge
+        att.really_destroy!
+      end
+    end
+  end
+end

lib/tasks/attachments.rake

@@ -0,0 +1,10 @@
+# frozen_string_literal: true
+
+namespace :attachments do
+  desc 'destroys all attachments older than :days_old, Default = 7'
+  task :cleanup, [:days_old] => [:environment] do |_t, args|
+    args.with_defaults(days_old: 7)
+    puts "Running attachment cleanup job for files deleted more than '#{args.days_old}' day ago."
+    AttachmentsCleanupJob.perform_now(args.days_old.to_i)
+  end
+end

test/fixtures/active_storage/attachments.yml

@@ -2,3 +2,18 @@ attachment1_file_test_file_fastq:
   name: file
   record: attachment1 (Attachment)
   blob: attachment1_file_test_file_fastq_blob
+
+attachmentA_file_test_file_fastq:
+  name: file
+  record: attachmentA (Attachment)
+  blob: attachmentA_file_test_file_fastq_blob
+
+attachmentB_file_test_file_fastq:
+  name: file
+  record: attachmentB (Attachment)
+  blob: attachmentB_file_test_file_fastq_blob
+
+attachmentC_file_test_file_fastq:
+  name: file
+  record: attachmentC (Attachment)
+  blob: attachmentC_file_test_file_fastq_blob

test/fixtures/active_storage/blobs.yml

@@ -5,3 +5,9 @@ test_file_fastq_blob: <%= ActiveStorage::FixtureSet.blob filename: "test_file.fa
 testsample_illumina_pe_forward_blob: <%= ActiveStorage::FixtureSet.blob filename: "TestSample_S1_L001_R1_001.fastq", service_name: "test_fixtures" %>
 
 testsample_illumina_pe_reverse_blob: <%= ActiveStorage::FixtureSet.blob filename: "TestSample_S1_L001_R2_001.fastq", service_name: "test_fixtures" %>
+
+attachmentA_file_test_file_fastq_blob: <%= ActiveStorage::FixtureSet.blob filename: "test_file_A.fastq", service_name: "test_fixtures" %>
+
+attachmentB_file_test_file_fastq_blob: <%= ActiveStorage::FixtureSet.blob filename: "test_file_B.fastq", service_name: "test_fixtures" %>
+
+attachmentC_file_test_file_fastq_blob: <%= ActiveStorage::FixtureSet.blob filename: "test_file_C.fastq", service_name: "test_fixtures" %>

test/fixtures/attachments.yml

@@ -3,3 +3,15 @@
 attachment1:
   metadata: {}
   attachable: sample1 (Sample)
+
+attachmentA:
+  metadata: {}
+  attachable: sampleA (Sample)
+
+attachmentB:
+  metadata: {}
+  attachable: sampleA (Sample)
+
+attachmentC:
+  metadata: {}
+  attachable: sampleA (Sample)

test/fixtures/files/test_file_A.fastq

@@ -0,0 +1,16 @@
+@M01648:39:000000000-A2K80:1:1101:16244:1505 1:N:0:1
+ATTCTAAATCACGCTGATGCAAAGAACTGATAAAACAATTGTGATTTTCTTTTAAGGCTAGATGTAAAGCACTGCCCCATTTTCCCGCACCAATAACCGCTATACGCATCAACCAAGCTTTGTTTTTAAAATTTCATTTACTTTTGCAGGATTAAAAGCTCCTTTGCCTTCTTTCATTGTTTGACCCACAAAATATCCAAAAAGTCTGTCTATACNNNTNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNN
++
+BBBBAFFFFFFFGGF?AGGGGGHHHHHHG4FDGHGGGDHHHAFGHHGGHHFHHGHFFFHHHGHGHHFHFHFHFFFHGGGHHHGGGHGGGGGHEHHHHHCGCG?DGFGGGGHHHHGGEF3EGB?GGFFFCHHBGGHHHHHEGFFHHF2GGAFHEBGHHBGH2GFGF1GHHHHHHHHFBGH=FC0DDCCFGDHHH0<DH0=CDHCD00==00;0<;0###;################################
+@M01648:39:000000000-A2K80:1:1101:13924:1520 1:N:0:1
+TTGGGGTTTGCAAATCTATATGGAACAGAACTTGTTATCTATCTTAAATCCATCATTCCTATTGATAAAAAAGAACTTGATGATGTTTTAAGCGAGGTGGGTAATGTTATGGCTGTTGTACTTTACTCTATGGTGATTGTAGCGATATTTCAAGGTGCGCTTTTTGGTTTAATCCCAATATTTTATGGTTATGTTGGAATTTTTATGGGGGGAANNNGNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNN
++
+BBBBBFFFFF5FGFGGFGGGG6E5B4GFGHHG555FFGHHFGFEGHHHFGGGFHHFHHFHFGFHBFFHHHBBEEFGFHHHHHHFFGHFHFG3F3E1EGEC1/?FGFGG4GDF3?2FHEF24??BG43BFEBBFGFHBBG3BBBGEEA</BD333F22B2///AFFF<//?<11<<1?FGB?=FGBDF11=111F1<<.1BFCFH0<<0F..--;.###.################################
+@M01648:39:000000000-A2K80:1:1101:15711:1539 1:N:0:1
+CAGTTATGAATACTTATTATATAGGGCTTTTAAAGAATTCTAAAAATCCAAAAGATGTAGAAGTGGGGAATTCTTTGGGTATTATTTTTCCTAATCAAGACAATAGAGGAACGCATATTAACATTAGCGGTATTGCTATGACAAAATCAAGTAAAAATCAAGATGCGGCTAAAAAATTTATGGAATTTATGTTAAGTCCTGAAATTCAATAAATTNNNANNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNN
++
+A3A>AFFFBFDFGGGFFFGGGGGHHACHHHHCGGHGFHHFFFAFGFFFHHGFCH2FGHHFEHHD3FEEAEHHHGFGHHGGHEFGHHHHHHHHFHHGFFBFHHHHGHHHHHFFGEEEGHHFGFGHHHFG?CEEGHHHGHHEHHHHGHFEFFHHHHHHGFBHHFDHFHCGGCFHHHHGAGFGH11DGHCGHGF>1100G0DBFB0GHH0=D0=D0<;###:################################
+@M01648:39:000000000-A2K80:1:1101:15176:1589 1:N:0:1
+CCCATAGGTATTTAAAGCATTTATAAGCTCCATTCTTTTTGCACTAAAATGCAGGTTAGTACTTTCTAAATCTTGTAAATATTTTGTACTTTCAAATTTTGTCATAAAACTTGATTGCATAAACACGGATAAATCTAAAACTTCACTTTTGAAATTGTCACACAAATTTTTATAATTTTTTACA
++
+BBBBBFFF5DFFGGGGGGFGGGHHHHHHHHHGHHHHHHHGCHHFHHGFHHGGHGHAFGHHHHHHHHHGHGHHHHHHHHGHHHFHHHHFHHHHFHEHHGHHHHHHHFBFGGHGHHHGHHHHHHHHHHGGGGGGHHHHHFBFFGHHHHHHHHHGHHHHHHDGDHHGECGHHHHHGHHHHHHHGGHG

test/fixtures/files/test_file_B.fastq

@@ -0,0 +1,16 @@
+@M01648:39:000000000-A2K80:1:1101:16244:1505 1:N:0:1
+ATTCTAAATCACGCTGATGCAAAGAACTGATAAAACAATTGTGATTTTCTTTTAAGGCTAGATGTAAAGCACTGCCCCATTTTCCCGCACCAATAACCGCTATACGCATCAACCAAGCTTTGTTTTTAAAATTTCATTTACTTTTGCAGGATTAAAAGCTCCTTTGCCTTCTTTCATTGTTTGACCCACAAAATATCCAAAAAGTCTGTCTATACNNNTNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNN
++
+BBBBAFFFFFFFGGF?AGGGGGHHHHHHG4FDGHGGGDHHHAFGHHGGHHFHHGHFFFHHHGHGHHFHFHFHFFFHGGGHHHGGGHGGGGGHEHHHHHCGCG?DGFGGGGHHHHGGEF3EGB?GGFFFCHHBGGHHHHHEGFFHHF2GGAFHEBGHHBGH2GFGF1GHHHHHHHHFBGH=FC0DDCCFGDHHH0<DH0=CDHCD00==00;0<;0###;################################
+@M01648:39:000000000-A2K80:1:1101:13924:1520 1:N:0:1
+TTGGGGTTTGCAAATCTATATGGAACAGAACTTGTTATCTATCTTAAATCCATCATTCCTATTGATAAAAAAGAACTTGATGATGTTTTAAGCGAGGTGGGTAATGTTATGGCTGTTGTACTTTACTCTATGGTGATTGTAGCGATATTTCAAGGTGCGCTTTTTGGTTTAATCCCAATATTTTATGGTTATGTTGGAATTTTTATGGGGGGAANNNGNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNN
++
+BBBBBFFFFF5FGFGGFGGGG6E5B4GFGHHG555FFGHHFGFEGHHHFGGGFHHFHHFHFGFHBFFHHHBBEEFGFHHHHHHFFGHFHFG3F3E1EGEC1/?FGFGG4GDF3?2FHEF24??BG43BFEBBFGFHBBG3BBBGEEA</BD333F22B2///AFFF<//?<11<<1?FGB?=FGBDF11=111F1<<.1BFCFH0<<0F..--;.###.################################
+@M01648:39:000000000-A2K80:1:1101:15711:1539 1:N:0:1
+CAGTTATGAATACTTATTATATAGGGCTTTTAAAGAATTCTAAAAATCCAAAAGATGTAGAAGTGGGGAATTCTTTGGGTATTATTTTTCCTAATCAAGACAATAGAGGAACGCATATTAACATTAGCGGTATTGCTATGACAAAATCAAGTAAAAATCAAGATGCGGCTAAAAAATTTATGGAATTTATGTTAAGTCCTGAAATTCAATAAATTNNNANNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNN
++
+A3A>AFFFBFDFGGGFFFGGGGGHHACHHHHCGGHGFHHFFFAFGFFFHHGFCH2FGHHFEHHD3FEEAEHHHGFGHHGGHEFGHHHHHHHHFHHGFFBFHHHHGHHHHHFFGEEEGHHFGFGHHHFG?CEEGHHHGHHEHHHHGHFEFFHHHHHHGFBHHFDHFHCGGCFHHHHGAGFGH11DGHCGHGF>1100G0DBFB0GHH0=D0=D0<;###:################################
+@M01648:39:000000000-A2K80:1:1101:15176:1589 1:N:0:1
+GGCATAGGTATTTAAAGCATTTATAAGCTCCATTCTTTTTGCACTAAAATGCAGGTTAGTACTTTCTAAATCTTGTAAATATTTTGTACTTTCAAATTTTGTCATAAAACTTGATTGCATAAACACGGATAAATCTAAAACTTCACTTTTGAAATTGTCACACAAATTTTTATAATTTTTTACA
++
+BBBBBFFF5DFFGGGGGGFGGGHHHHHHHHHGHHHHHHHGCHHFHHGFHHGGHGHAFGHHHHHHHHHGHGHHHHHHHHGHHHFHHHHFHHHHFHEHHGHHHHHHHFBFGGHGHHHGHHHHHHHHHHGGGGGGHHHHHFBFFGHHHHHHHHHGHHHHHHDGDHHGECGHHHHHGHHHHHHHGGHG

test/fixtures/files/test_file_C.fastq

@@ -0,0 +1,16 @@
+@M01648:39:000000000-A2K80:1:1101:16244:1505 1:N:0:1
+TTTCTAAATCACGCTGATGCAAAGAACTGATAAAACAATTGTGATTTTCTTTTAAGGCTAGATGTAAAGCACTGCCCCATTTTCCCGCACCAATAACCGCTATACGCATCAACCAAGCTTTGTTTTTAAAATTTCATTTACTTTTGCAGGATTAAAAGCTCCTTTGCCTTCTTTCATTGTTTGACCCACAAAATATCCAAAAAGTCTGTCTATACNNNTNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNN
++
+BBBBAFFFFFFFGGF?AGGGGGHHHHHHG4FDGHGGGDHHHAFGHHGGHHFHHGHFFFHHHGHGHHFHFHFHFFFHGGGHHHGGGHGGGGGHEHHHHHCGCG?DGFGGGGHHHHGGEF3EGB?GGFFFCHHBGGHHHHHEGFFHHF2GGAFHEBGHHBGH2GFGF1GHHHHHHHHFBGH=FC0DDCCFGDHHH0<DH0=CDHCD00==00;0<;0###;################################
+@M01648:39:000000000-A2K80:1:1101:13924:1520 1:N:0:1
+AAATTTCTTTGCAAATCTATATGGAACAGAACTTGTTATCTATCTTAAATCCATCATTCCTATTGATAAAAAAGAACTTGATGATGTTTTAAGCGAGGTGGGTAATGTTATGGCTGTTGTACTTTACTCTATGGTGATTGTAGCGATATTTCAAGGTGCGCTTTTTGGTTTAATCCCAATATTTTATGGTTATGTTGGAATTTTTATGGGGGGAANNNGNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNN
++
+BBBBBFFFFF5FGFGGFGGGG6E5B4GFGHHG555FFGHHFGFEGHHHFGGGFHHFHHFHFGFHBFFHHHBBEEFGFHHHHHHFFGHFHFG3F3E1EGEC1/?FGFGG4GDF3?2FHEF24??BG43BFEBBFGFHBBG3BBBGEEA</BD333F22B2///AFFF<//?<11<<1?FGB?=FGBDF11=111F1<<.1BFCFH0<<0F..--;.###.################################
+@M01648:39:000000000-A2K80:1:1101:15711:1539 1:N:0:1
+CAGTTATGAATACTTATTATATAGGGCTTTTAAAGAATTCTAAAAATCCAAAAGATGTAGAAGTGGGGAATTCTTTGGGTATTATTTTTCCTAATCAAGACAATAGAGGAACGCATATTAACATTAGCGGTATTGCTATGACAAAATCAAGTAAAAATCAAGATGCGGCTAAAAAATTTATGGAATTTATGTTAAGTCCTGAAATTCAATAAATTNNNANNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNN
++
+A3A>AFFFBFDFGGGFFFGGGGGHHACHHHHCGGHGFHHFFFAFGFFFHHGFCH2FGHHFEHHD3FEEAEHHHGFGHHGGHEFGHHHHHHHHFHHGFFBFHHHHGHHHHHFFGEEEGHHFGFGHHHFG?CEEGHHHGHHEHHHHGHFEFFHHHHHHGFBHHFDHFHCGGCFHHHHGAGFGH11DGHCGHGF>1100G0DBFB0GHH0=D0=D0<;###:################################
+@M01648:39:000000000-A2K80:1:1101:15176:1589 1:N:0:1
+GGCATAGGTATTTAAAGCATTTATAAGCTCCATTCTTTTTGCACTAAAATGCAGGTTAGTACTTTCTAAATCTTGTAAATATTTTGTACTTTCAAATTTTGTCATAAAACTTGATTGCATAAACACGGATAAATCTAAAACTTCACTTTTGAAATTGTCACACAAATTTTTATAATTTTTTACA
++
+BBBBBFFF5DFFGGGGGGFGGGHHHHHHHHHGHHHHHHHGCHHFHHGFHHGGHGHAFGHHHHHHHHHGHGHHHHHHHHGHHHFHHHHFHHHHFHEHHGHHHHHHHFBFGGHGHHHGHHHHHHHHHHGGGGGGHHHHHFBFFGHHHHHHHHHGHHHHHHDGDHHGECGHHHHHGHHHHHHHGGHG

test/fixtures/namespaces/project_namespaces.yml

@@ -223,3 +223,11 @@ project26_namespace:
   type: Project
   parent_id: <%= ActiveRecord::FixtureSet.identify(:john_doe_namespace) %>
   owner_id: <%= ActiveRecord::FixtureSet.identify(:john_doe) %>
+
+projectA_namespace:
+  name: Project A
+  path: project-A
+  description: Project A description
+  type: Project
+  parent_id: <%= ActiveRecord::FixtureSet.identify(:jeff_doe_namespace) %>
+  owner_id: <%= ActiveRecord::FixtureSet.identify(:jeff_doe) %>

test/fixtures/namespaces/user_namespaces.yml

@@ -23,3 +23,9 @@ david_doe_namespace:
   path: david.doe_at_localhost
   type: User
   owner_id: <%= ActiveRecord::FixtureSet.identify(:david_doe) %>
+
+jeff_doe_namespace:
+  name: jeff.doe@localhost
+  path: jeff.doe_at_localhost
+  type: User
+  owner_id: <%= ActiveRecord::FixtureSet.identify(:jeff_doe) %>

test/fixtures/projects.yml

@@ -111,3 +111,7 @@ project25:
 project26:
   creator_id: <%= ActiveRecord::FixtureSet.identify(:john_doe) %>
   namespace_id: <%= ActiveRecord::FixtureSet.identify(:project26_namespace) %>
+
+projectA:
+  creator_id: <%= ActiveRecord::FixtureSet.identify(:jeff_doe) %>
+  namespace_id: <%= ActiveRecord::FixtureSet.identify(:projectA_namespace) %>

test/fixtures/routes.yml

@@ -22,6 +22,11 @@ john_doe_namespace_route:
   path: john.doe_at_localhost
   source: john_doe_namespace (Namespace)
 
+jeff_doe_namespace_route:
+  name: jeff.doe@localhost
+  path: jeff.doe_at_localhost
+  source: jeff_doe_namespace (Namespace)
+
 jane_doe_namespace_route:
   name: jane.doe@localhost
   path: jane.doe_at_localhost

test/fixtures/samples.yml

@@ -42,3 +42,8 @@ sample27:
   description: Sample 2 description.
   project_id: <%= ActiveRecord::FixtureSet.identify(:project26) %>
 
+sampleA:
+  name: Project A Sample 1
+  description: Sample 1 description.
+  project_id: <%= ActiveRecord::FixtureSet.identify(:projectA) %>
+

test/jobs/attachments_cleanup_job_test.rb

@@ -0,0 +1,137 @@
+# frozen_string_literal: true
+
+require 'test_helper'
+
+class AttachmentsCleanupJobTest < ActiveJob::TestCase
+  def setup
+    @attachment1 = attachments(:attachmentA)
+    @attachment2 = attachments(:attachmentB)
+    @attachment3 = attachments(:attachmentC)
+  end
+
+  test 'valid attachments pretest' do
+    assert @attachment1.valid?
+    assert @attachment2.valid?
+    assert @attachment3.valid?
+  end
+
+  test 'deletion after default 7 days' do
+    # delete (soft) first attachment
+    assert_nil @attachment1.deleted_at
+    @attachment1.destroy
+    assert_not_nil @attachment1.deleted_at
+    # go forward 4 days
+    travel 4.days
+    # delete (soft) second attachment
+    assert_nil @attachment2.deleted_at
+    @attachment2.destroy
+    assert_not_nil @attachment2.deleted_at
+    # go forward 5 more days
+    travel 5.days
+    # verify files are only soft deleted
+    assert_not_nil @attachment1.deleted_at
+    assert_not_nil @attachment2.deleted_at
+    assert_nil @attachment3.deleted_at
+
+    # run job and verify file/object count changes
+    assert_difference -> { ActiveStorage::Attachment.count } => -1,
+                      -> { Attachment.only_deleted.count } => -1,
+                      -> { Attachment.all.count } => 0 do
+      AttachmentsCleanupJob.perform_now
+    end
+
+    # verify attachment exist or not
+    id_list = Attachment.all.map(&:id)
+    assert_not(id_list.include?(@attachment1.id))
+    assert_not(id_list.include?(@attachment2.id))
+    assert(Attachment.only_deleted.map(&:id).include?(@attachment2.id))
+    assert(id_list.include?(@attachment3.id))
+  end
+
+  test 'deletion after specified 14 days' do
+    # delete (soft) first attachment
+    assert_nil @attachment1.deleted_at
+    @attachment1.destroy
+    assert_not_nil @attachment1.deleted_at
+    # go forward 10 days
+    travel 10.days
+    # delete (soft) second attachment
+    assert_nil @attachment2.deleted_at
+    @attachment2.destroy
+    assert_not_nil @attachment2.deleted_at
+    # go forward 6 more days
+    travel 6.days
+    # verify files are only soft deleted
+    assert_not_nil @attachment1.deleted_at
+    assert_not_nil @attachment2.deleted_at
+    assert_nil @attachment3.deleted_at
+
+    # run job and verify file/object count changes
+    assert_difference -> { ActiveStorage::Attachment.count } => -1,
+                      -> { Attachment.only_deleted.count } => -1,
+                      -> { Attachment.all.count } => 0 do
+      AttachmentsCleanupJob.perform_now(14)
+    end
+
+    # verify attachment exist or not
+    id_list = Attachment.all.map(&:id)
+    assert_not(id_list.include?(@attachment1.id))
+    assert_not(id_list.include?(@attachment2.id))
+    assert(Attachment.only_deleted.map(&:id).include?(@attachment2.id))
+    assert(id_list.include?(@attachment3.id))
+  end
+
+  test 'deletion multiple' do
+    # delete (soft) first attachment
+    assert_nil @attachment1.deleted_at
+    @attachment1.destroy
+    assert_not_nil @attachment1.deleted_at
+    # delete (soft) second attachment
+    assert_nil @attachment2.deleted_at
+    @attachment2.destroy
+    assert_not_nil @attachment2.deleted_at
+    # go forward 9 days
+    travel 9.days
+    # verify files are only soft deleted
+    assert_not_nil @attachment1.deleted_at
+    assert_not_nil @attachment2.deleted_at
+    assert_nil @attachment3.deleted_at
+
+    # run job and verify file/object count changes
+    assert_difference -> { ActiveStorage::Attachment.count } => -2,
+                      -> { Attachment.only_deleted.count } => -2,
+                      -> { Attachment.all.count } => 0 do
+      AttachmentsCleanupJob.perform_now
+    end
+
+    # verify attachment exist or not
+    id_list = Attachment.all.map(&:id)
+    assert_not(id_list.include?(@attachment1.id))
+    assert_not(id_list.include?(@attachment2.id))
+    assert(id_list.include?(@attachment3.id))
+  end
+
+  test 'invalid argument string' do
+    assert_raise(Exception) do
+      AttachmentsCleanupJob.perform_now('this is not a number')
+    end
+  end
+
+  test 'invalid argument negative' do
+    assert_raise(Exception) do
+      AttachmentsCleanupJob.perform_now(-1)
+    end
+  end
+
+  test 'invalid argument zero' do
+    assert_raise(Exception) do
+      AttachmentsCleanupJob.perform_now(0)
+    end
+  end
+
+  test 'invalid argument int as string' do
+    assert_raise(Exception) do
+      AttachmentsCleanupJob.perform_now('1')
+    end
+  end
+end