WIP Bulk importer

.dockworker/src/FileSystem/RecursivePathFileOperatorTrait.php

@@ -0,0 +1,101 @@
+<?php
+
+namespace Dockworker\FileSystem;
+
+use RecursiveDirectoryIterator;
+use RecursiveIteratorIterator;
+
+/**
+ * Provides methods to operate on a series of files recursively in a path.
+ */
+trait RecursivePathFileOperatorTrait {
+
+  /**
+   * Filepaths to operate on.
+   *
+   * @var string[]
+   */
+  private $recursivePathOperatorFiles = [];
+
+  /**
+   * Adds files recursively to the operation queue.
+   *
+   * @param string[] $paths
+   *   An array of file paths to traverse recursively.
+   * @param string[] $extension_filter
+   *   The extensions to include when populating to queue.
+   */
+  protected function addRecursivePathFilesFromPath(array $paths, array $extension_filter = []) {
+    foreach ($paths as $path) {
+      if (file_exists($path)) {
+        $directory = new RecursiveDirectoryIterator($path);
+        $iterator = new RecursiveIteratorIterator($directory);
+        $files = [];
+        foreach ($iterator as $info) {
+          $files[] = $info->getPathname();
+        }
+        $this->recursivePathOperatorFiles = array_merge(
+          $this->recursivePathOperatorFiles,
+          self::filterArrayFilesByExtension($files, $extension_filter)
+        );
+      }
+    }
+  }
+
+  /**
+   * Filters an array of files, removing any that do not match given extensions.
+   *
+   * @param string[] $files
+   *   The array of files to filter.
+   * @param string[] $extension_filter
+   *   An array of extensions to keep in the file list.
+   *
+   * @return string[]
+   *   The filtered array of files.
+   */
+  protected static function filterArrayFilesByExtension(array $files, $extension_filter = []) {
+    foreach ($files as $file_key => $filename) {
+      $fileExt = pathinfo($filename, PATHINFO_EXTENSION);
+      if (!empty($extension_filter) && !in_array($fileExt, $extension_filter)) {
+        unset($files[$file_key]);
+      }
+    }
+    return $files;
+  }
+
+  /**
+   * Clears the list of files in the operation queue.
+   */
+  protected function clearRecursivePathFiles() {
+    $this->recursivePathOperatorFiles = [];
+  }
+
+  /**
+   * Returns the list of current files in the operation queue.
+   *
+   * @return string[]
+   *   The list of currently queued files.
+   */
+  protected function getRecursivePathFiles() {
+    return $this->recursivePathOperatorFiles;
+  }
+
+  /**
+   * Returns the list of current files as a string.
+   *
+   * @param string $separator
+   *   The separator to use when formatting the output list.
+   * @param string $quote
+   *   The string/character used when formatting the output list.
+   *
+   * @return string
+   *   The formatted list of current files.
+   */
+  protected function getRecursivePathStringFileList($separator = ' ', $quote = '\'') {
+    if (!empty($this->recursivePathOperatorFiles)) {
+      return $quote . implode("$quote$separator$quote", $this->recursivePathOperatorFiles) . $quote;
+    }
+    return '';
+  }
+
+}

.dockworker/src/Robo/Plugin/Commands/HerbariumBulkImportImagesCommandDeprecated.php → .dockworker/src/Robo/Plugin/Commands/HerbariumBulkImportImagesCommands.php

@@ -2,16 +2,18 @@
 
 namespace Dockworker\Robo\Plugin\CommandsDeprecated;
 
-use Dockworker\PersistentLocalDockworkerDataTrait;
-use Dockworker\RecursivePathFileOperatorTrait;
-use Dockworker\Robo\Plugin\Commands\DrupalDeploymentDrushCommands;
+use Dockworker\DockworkerDrupalCommands;
+use Dockworker\FileSystem\RecursivePathFileOperatorTrait;
+use Dockworker\IO\DockworkerIOTrait;
+use Dockworker\Storage\ApplicationPersistentDataStorageTrait;
 
 /**
  * Defines the commands used to bulk import archival masters into the CMH.
  */
-class HerbariumBulkImportImagesCommand extends DrupalDeploymentDrushCommands {
+class HerbariumBulkImportImagesCommands extends DockworkerDrupalCommands {
 
-  use PersistentLocalDockworkerDataTrait;
+  use ApplicationPersistentDataStorageTrait;
+  use DockworkerIOTrait;
   use RecursivePathFileOperatorTrait;
 
   /**
@@ -129,19 +131,27 @@ public function addHerbariumArchivalMastersFromTree(
     $this->targetDrupalUid = $options['uid'];
     $this->targetCommitMessage = $options['commit-message'];
     $this->targetDeployEnv = $env;
-    $this->options = $options;
 
-    $this->setUpArchivalMasterQueue();
-    $this->importQueuedArchivalMasters();
+    $this->io()->title('Herbarium Archival Master Import');
+    $this->io()->section('Import Options');
+
+    // Query the user for an ID string for this import. If none is given, generate one randomly.
+    $options['import_id'] = $this->dockworkerIO->ask(
+      'Provide an ID for this import?',
+      'herbarium-import-' . date('Y-m-d-H-i-s')
+    );
+
+    $this->setUpArchivalMasterQueue($options);
+    $this->importQueuedArchivalMasters($options);
   }
 
   /**
    * Queues any files in the tree that are to be imported as archival masters.
    */
-  protected function setUpArchivalMasterQueue() {
+  protected function setUpArchivalMasterQueue($options) {
     $this->addRecursivePathFilesFromPath(
       [$this->sourceTreePath],
-      [$this->options['issue-page-extension']]
+      [$options['issue-page-extension']]
     );
   }
 
@@ -151,10 +161,11 @@ protected function setUpArchivalMasterQueue() {
    * @throws \Dockworker\DockworkerException
    * @throws \Exception
    */
-  protected function importQueuedArchivalMasters() {
+  protected function importQueuedArchivalMasters($options) {
     $import_files = $this->getRecursivePathFiles();
     if (!empty($import_files)) {
-      $this->initLocalDockworkerConfig('bulk_imports');
+
+
       $this->targetPodId = $this->k8sGetLatestPod(
         $this->targetDeployEnv,
         'deployment',
@@ -171,12 +182,12 @@ protected function importQueuedArchivalMasters() {
           if (!in_array($this->curFileName, $imported_items)) {
             $this->curFileAccessionId = $this->getAccessionIdFromFilepath(
               $this->curFilePath,
-              $this->options['issue-page-extension']
+              $options['issue-page-extension']
             );
             $this->curFileNodeId = $this->getNidFromAccessionId();
             if (!empty($this->curFileNodeId)) {
               $this->io()->title($this->curFileName);
-              $this->importArchivalMaster();
+              $this->importArchivalMaster($options);
               $imported_items[] = $this->curFileName;
               $this->curLocalDockworkerConfiguration->set('dockworker.imported_items', $imported_items);
               $this->witeLocalDockworkerConfig();
@@ -237,9 +248,9 @@ private function getNidFromAccessionId() : string {
    *
    * @throws \Dockworker\DockworkerException
    */
-  protected function importArchivalMaster() {
+  protected function importArchivalMaster($options) {
     $this->copyArchivalMasterToPod();
-    $this->executeArchivalMasterDeployedImport();
+    $this->executeArchivalMasterDeployedImport($options);
   }
 
   /**
@@ -262,7 +273,7 @@ protected function copyArchivalMasterToPod() {
    *
    * @throws \Dockworker\DockworkerException
    */
-  protected function executeArchivalMasterDeployedImport() {
+  protected function executeArchivalMasterDeployedImport($options) {
     $this->say("Importing archival master [NID#$this->curFileNodeId]...");
     $cmd = sprintf(
       '$DRUSH eval "' .
@@ -272,7 +283,7 @@ protected function executeArchivalMasterDeployedImport() {
       $this->targetDrupalUid,
       $this->curFileNodeId,
       "/tmp/$this->curFileName",
-      $this->options['commit-message']
+      $options['commit-message']
     );
     $this->kubernetesPodExecCommand(
       $this->targetPodId,
@@ -281,4 +292,20 @@ protected function executeArchivalMasterDeployedImport() {
     );
   }
 
+  protected function getCurrentImportedItems($import_id) {
+    $cur = $this->getApplicationPersistentDataConfigurationItem(
+      'bulk_imports',
+      $import_id
+    );
+    if (empty($cur)) {
+      $this->setApplicationPersistentDataConfigurationItem(
+        'bulk_imports',
+        $import_id,
+        []
+      );
+      return [];
+    }
+    return $cur;
+  }
+
 }