From 43ed44d6b40269c12d5cc0de16af7abe3399b429 Mon Sep 17 00:00:00 2001 From: Thiyagu K Date: Fri, 11 Oct 2024 17:02:06 +0000 Subject: [PATCH 01/13] quickstart and latest transfer check --- .../src/check_latest_transfer_operation.php | 57 +++++++++ storagetransfer/src/quickstart.php | 2 +- .../test/checkLatestTransferOperationTest.php | 108 ++++++++++++++++++ 3 files changed, 166 insertions(+), 1 deletion(-) create mode 100644 storagetransfer/src/check_latest_transfer_operation.php create mode 100644 storagetransfer/test/checkLatestTransferOperationTest.php diff --git a/storagetransfer/src/check_latest_transfer_operation.php b/storagetransfer/src/check_latest_transfer_operation.php new file mode 100644 index 0000000000..9c5ab11837 --- /dev/null +++ b/storagetransfer/src/check_latest_transfer_operation.php @@ -0,0 +1,57 @@ + $projectId, + 'job_name' => $jobName + ]); + + $client = new StorageTransferServiceClient(); + $request = $client->getTransferJob($transferJob); + $latestOperationName = $request->getLatestOperationName(); + + if($latestOperationName){ + $transferOperation = $client->getOperationsClient()->getOperation($latestOperationName); + + $operation = $transferOperation->getMetadata(); + + printf('Latest transfer operation for %s is: %s ' . PHP_EOL, $jobName, $operation->serializeToJsonString()); + } else { + printf('Transfer job %s has not ran yet.' . PHP_EOL, $jobName); + } +} +# [END storagetransfer_get_latest_transfer_operation] + +// The following 2 lines are only needed to run the samples +require_once __DIR__ . '/../../testing/sample_helpers.php'; +\Google\Cloud\Samples\execute_sample(__FILE__, __NAMESPACE__, $argv); diff --git a/storagetransfer/src/quickstart.php b/storagetransfer/src/quickstart.php index 85383317cd..b4affdf58a 100644 --- a/storagetransfer/src/quickstart.php +++ b/storagetransfer/src/quickstart.php @@ -41,7 +41,7 @@ function quickstart($projectId, $sourceGcsBucketName, $sinkGcsBucketName) $transferJob = new TransferJob([ 'project_id' => $projectId, 'transfer_spec' => new TransferSpec([ - 'gcs_data_sink' => new GcsData(['bucket_name' => $sourceGcsBucketName]), + 'gcs_data_sink' => new GcsData(['bucket_name' => $sinkGcsBucketName]), 'gcs_data_source' => new GcsData(['bucket_name' => $sourceGcsBucketName]) ]), 'status' => Status::ENABLED diff --git a/storagetransfer/test/checkLatestTransferOperationTest.php b/storagetransfer/test/checkLatestTransferOperationTest.php new file mode 100644 index 0000000000..4d7830d712 --- /dev/null +++ b/storagetransfer/test/checkLatestTransferOperationTest.php @@ -0,0 +1,108 @@ +createBucket( + sprintf('php-source-bucket-%s', $uniqueBucketId) + ); + self::$sinkBucket = self::$storage->createBucket( + sprintf('php-sink-bucket-%s', $uniqueBucketId) + ); + + self::grantStsPermissions(self::$sourceBucket); + self::grantStsPermissions(self::$sinkBucket); + } + + public static function tearDownAfterClass(): void + { + self::$sourceBucket->delete(); + self::$sinkBucket->delete(); + } + + public function testCheckLatestTransferOperation() + { + $transferData = $this->runFunctionSnippet('quickstart', [ + self::$projectId, self::$sinkBucket->name(), self::$sourceBucket->name() + ]); + preg_match('/transferJobs\/\d+/', $transferData, $match); + self::$jobName = $match[0]; + + $output = $this->runFunctionSnippet('check_latest_transfer_operation', [ + self::$projectId, self::$jobName + ]); + + $this->assertMatchesRegularExpression('/transferJobs\/.*/', $output); + + $transferJob = new TransferJob([ + 'name' => self::$jobName, + 'status' => Status::DELETED + ]); + $request = (new UpdateTransferJobRequest()) + ->setJobName(self::$jobName) + ->setProjectId(self::$projectId) + ->setTransferJob($transferJob); + + self::$sts->updateTransferJob($request); + } + + private static function grantStsPermissions($bucket) + { + $request2 = (new GetGoogleServiceAccountRequest()) + ->setProjectId(self::$projectId); + $googleServiceAccount = self::$sts->getGoogleServiceAccount($request2); + $email = $googleServiceAccount->getAccountEmail(); + $members = ['serviceAccount:' . $email]; + + $policy = $bucket->iam()->policy(['requestedPolicyVersion' => 3]); + $policy['version'] = 3; + + array_push( + $policy['bindings'], + ['role' => 'roles/storage.objectViewer', 'members' => $members], + ['role' => 'roles/storage.legacyBucketReader', 'members' => $members], + ['role' => 'roles/storage.legacyBucketWriter', 'members' => $members] + ); + + $bucket->iam()->setPolicy($policy); + } +} From 3a999b60d65794ec73ca68564bee298e9a13ea71 Mon Sep 17 00:00:00 2001 From: Thiyagu K Date: Tue, 15 Oct 2024 05:40:57 +0000 Subject: [PATCH 02/13] added samples for nearline and combine test cases --- storagetransfer/src/nearline_request.php | 102 +++++++++++++++++ storagetransfer/test/StorageTransferTest.php | 45 +++++++- .../test/checkLatestTransferOperationTest.php | 108 ------------------ 3 files changed, 144 insertions(+), 111 deletions(-) create mode 100644 storagetransfer/src/nearline_request.php delete mode 100644 storagetransfer/test/checkLatestTransferOperationTest.php diff --git a/storagetransfer/src/nearline_request.php b/storagetransfer/src/nearline_request.php new file mode 100644 index 0000000000..c54de2ddd6 --- /dev/null +++ b/storagetransfer/src/nearline_request.php @@ -0,0 +1,102 @@ + $dateTime->format('Y'), + 'month' => $dateTime->format('m') + 1, + 'day' => $dateTime->format('d'), + ]); + + $time = new TimeOfDay([ + 'hours' => $dateTime->format('H'), + 'minutes' => $dateTime->format('i'), + 'seconds' => $dateTime->format('s'), + ]); + + $transferJob = new TransferJob([ + 'project_id' => $projectId, + 'description' => $description, + 'schedule' => new Schedule([ + 'schedule_start_date' => $date, + 'start_time_of_day' => $time + ]), + 'transfer_spec' => new TransferSpec([ + 'gcs_data_source' => new GcsData(['bucket_name' => $sourceGcsBucketName]), + 'gcs_data_sink' => new GcsData(['bucket_name' => $sinkGcsBucketName]), + 'object_conditions' => new ObjectConditions([ + 'min_time_elapsed_since_last_modification' => new ProtobufDuration([ + 'seconds' => 2592000 + ]) + ]), + 'transfer_options' => new TransferOptions(['delete_objects_from_source_after_transfer' => true]) + ]), + 'status' => Status::ENABLED + ]); + + $client = new StorageTransferServiceClient(); + $request = (new CreateTransferJobRequest()) + ->setTransferJob($transferJob); + $response = $client->createTransferJob($request); + $request2 = (new RunTransferJobRequest()) + ->setJobName($response->getName()) + ->setProjectId($projectId); + $client->runTransferJob($request2); + + printf('Created and ran transfer job : %s' . PHP_EOL, $response->getName()); +} +# [END storagetransfer_transfer_to_nearline] + +// The following 2 lines are only needed to run the samples +require_once __DIR__ . '/../../testing/sample_helpers.php'; +\Google\Cloud\Samples\execute_sample(__FILE__, __NAMESPACE__, $argv); diff --git a/storagetransfer/test/StorageTransferTest.php b/storagetransfer/test/StorageTransferTest.php index c23060f6e0..57c8ab1cea 100644 --- a/storagetransfer/test/StorageTransferTest.php +++ b/storagetransfer/test/StorageTransferTest.php @@ -1,6 +1,6 @@ runFunctionSnippet('quickstart', [ self::$projectId, self::$sinkBucket->name(), self::$sourceBucket->name() ]); + $this->assertMatchesRegularExpression('/transferJobs\/.*/', $output); + + self::deleteTransferJob($output); + } + + public function testCheckLatestTransferOperation() + { + $transferData = $this->runFunctionSnippet('quickstart', [ + self::$projectId, self::$sinkBucket->name(), self::$sourceBucket->name() + ]); + preg_match('/transferJobs\/\d+/', $transferData, $match); + $jobName = $match[0]; + + $output = $this->runFunctionSnippet('check_latest_transfer_operation', [ + self::$projectId, $jobName + ]); $this->assertMatchesRegularExpression('/transferJobs\/.*/', $output); + self::deleteTransferJob($output); + } + + public function testNearlineRequest() + { + $description = sprintf('My transfer job from %s -> %s', self::$sourceBucket->name(), self::$sinkBucket->name()); + $date = new DateTime('now'); + $startDate = $date->format('Y-m-d H:i:s'); + + $output = $this->runFunctionSnippet('nearline_request', [ + self::$projectId, $description, self::$sourceBucket->name(), self::$sinkBucket->name(), $startDate + ]); + + $this->assertMatchesRegularExpression('/Created and ran transfer job : transferJobs\/.*/', $output); + + self::deleteTransferJob($output); + } + + // deletes a transfer job created by a sample to clean up + private static function deleteTransferJob($output) + { preg_match('/transferJobs\/\d+/', $output, $match); $jobName = $match[0]; + $transferJob = new TransferJob([ 'name' => $jobName, 'status' => Status::DELETED diff --git a/storagetransfer/test/checkLatestTransferOperationTest.php b/storagetransfer/test/checkLatestTransferOperationTest.php deleted file mode 100644 index 4d7830d712..0000000000 --- a/storagetransfer/test/checkLatestTransferOperationTest.php +++ /dev/null @@ -1,108 +0,0 @@ -createBucket( - sprintf('php-source-bucket-%s', $uniqueBucketId) - ); - self::$sinkBucket = self::$storage->createBucket( - sprintf('php-sink-bucket-%s', $uniqueBucketId) - ); - - self::grantStsPermissions(self::$sourceBucket); - self::grantStsPermissions(self::$sinkBucket); - } - - public static function tearDownAfterClass(): void - { - self::$sourceBucket->delete(); - self::$sinkBucket->delete(); - } - - public function testCheckLatestTransferOperation() - { - $transferData = $this->runFunctionSnippet('quickstart', [ - self::$projectId, self::$sinkBucket->name(), self::$sourceBucket->name() - ]); - preg_match('/transferJobs\/\d+/', $transferData, $match); - self::$jobName = $match[0]; - - $output = $this->runFunctionSnippet('check_latest_transfer_operation', [ - self::$projectId, self::$jobName - ]); - - $this->assertMatchesRegularExpression('/transferJobs\/.*/', $output); - - $transferJob = new TransferJob([ - 'name' => self::$jobName, - 'status' => Status::DELETED - ]); - $request = (new UpdateTransferJobRequest()) - ->setJobName(self::$jobName) - ->setProjectId(self::$projectId) - ->setTransferJob($transferJob); - - self::$sts->updateTransferJob($request); - } - - private static function grantStsPermissions($bucket) - { - $request2 = (new GetGoogleServiceAccountRequest()) - ->setProjectId(self::$projectId); - $googleServiceAccount = self::$sts->getGoogleServiceAccount($request2); - $email = $googleServiceAccount->getAccountEmail(); - $members = ['serviceAccount:' . $email]; - - $policy = $bucket->iam()->policy(['requestedPolicyVersion' => 3]); - $policy['version'] = 3; - - array_push( - $policy['bindings'], - ['role' => 'roles/storage.objectViewer', 'members' => $members], - ['role' => 'roles/storage.legacyBucketReader', 'members' => $members], - ['role' => 'roles/storage.legacyBucketWriter', 'members' => $members] - ); - - $bucket->iam()->setPolicy($policy); - } -} From 09bef5e1439d7c9d35f87a70b45862926193872f Mon Sep 17 00:00:00 2001 From: Thiyagu K Date: Wed, 16 Oct 2024 18:36:40 +0000 Subject: [PATCH 03/13] added samples for manifest and test cases --- storagetransfer/src/manifest_request.php | 74 ++++++++++++++++++++ storagetransfer/test/StorageTransferTest.php | 60 ++++++++++++++-- 2 files changed, 127 insertions(+), 7 deletions(-) create mode 100644 storagetransfer/src/manifest_request.php diff --git a/storagetransfer/src/manifest_request.php b/storagetransfer/src/manifest_request.php new file mode 100644 index 0000000000..88b7038a8d --- /dev/null +++ b/storagetransfer/src/manifest_request.php @@ -0,0 +1,74 @@ + $projectId, + 'transfer_spec' => new TransferSpec([ + 'source_agent_pool_name' => $sourceAgentPoolName, + 'posix_data_source' => new PosixFilesystem(['root_directory' => $rootDirectory]), + 'gcs_data_sink' => new GcsData(['bucket_name' => $sinkGcsBucketName]), + 'transfer_manifest' => new TransferManifest(['location' => $manifestLocation]) + ]), + 'status' => Status::ENABLED + ]); + + $client = new StorageTransferServiceClient(); + $request = (new CreateTransferJobRequest()) + ->setTransferJob($transferJob); + $response = $client->createTransferJob($request); + $request2 = (new RunTransferJobRequest()) + ->setJobName($response->getName()) + ->setProjectId($projectId); + $client->runTransferJob($request2); + + printf('Created and ran transfer job from %s to %s using manifest %s with name %s ' . PHP_EOL, $rootDirectory, $sinkGcsBucketName, $manifestLocation, $response->getName()); +} +# [END storagetransfer_manifest_request] + +// The following 2 lines are only needed to run the samples +require_once __DIR__ . '/../../testing/sample_helpers.php'; +\Google\Cloud\Samples\execute_sample(__FILE__, __NAMESPACE__, $argv); diff --git a/storagetransfer/test/StorageTransferTest.php b/storagetransfer/test/StorageTransferTest.php index 57c8ab1cea..65747b75bc 100644 --- a/storagetransfer/test/StorageTransferTest.php +++ b/storagetransfer/test/StorageTransferTest.php @@ -35,6 +35,7 @@ class StorageTransferTest extends TestCase private static $storage; private static $sourceBucket; private static $sinkBucket; + private static $sourceAgentPoolName; public static function setUpBeforeClass(): void { @@ -48,6 +49,7 @@ public static function setUpBeforeClass(): void self::$sinkBucket = self::$storage->createBucket( sprintf('php-sink-bucket-%s', $uniqueBucketId) ); + self::$sourceAgentPoolName = ''; self::grantStsPermissions(self::$sourceBucket); self::grantStsPermissions(self::$sinkBucket); @@ -66,7 +68,8 @@ public function testQuickstart() ]); $this->assertMatchesRegularExpression('/transferJobs\/.*/', $output); - self::deleteTransferJob($output); + preg_match('/transferJobs\/\d+/', $output, $match); + self::deleteTransferJob($match[0]); } public function testCheckLatestTransferOperation() @@ -83,7 +86,8 @@ public function testCheckLatestTransferOperation() $this->assertMatchesRegularExpression('/transferJobs\/.*/', $output); - self::deleteTransferJob($output); + preg_match('/transferJobs\/\d+/', $output, $match); + self::deleteTransferJob($match[0]); } public function testNearlineRequest() @@ -98,15 +102,57 @@ public function testNearlineRequest() $this->assertMatchesRegularExpression('/Created and ran transfer job : transferJobs\/.*/', $output); - self::deleteTransferJob($output); + preg_match('/transferJobs\/\d+/', $output, $match); + self::deleteTransferJob($match[0]); } - // deletes a transfer job created by a sample to clean up - private static function deleteTransferJob($output) + public function testManifestRequest() { - preg_match('/transferJobs\/\d+/', $output, $match); - $jobName = $match[0]; + try { + $manifestName = 'manifest.csv'; + $rootDirectory = sys_get_temp_dir() . '/sts-manifest-request-test'; + if (!is_dir($rootDirectory)) { + mkdir($rootDirectory, 0700, true); + } + $tempFile = $rootDirectory . '/text.txt'; + + // Write test data to the temporary file + $testData = 'test data'; + file_put_contents($tempFile, $testData); + + // Escape double quotes for CSV content + $csvContent = '"' . str_replace('"', '""', 'text.txt') . '"'; + $tempManifestObject = fopen('php://temp', 'r+'); // Create a temporary file stream + + // Write CSV content to the temporary manifest + fwrite($tempManifestObject, $csvContent); + + // Upload the temporary manifest to GCS bucket (replace with your library) + self::$sinkBucket->upload( + $tempManifestObject, + [ + 'name' => $manifestName + ] + ); + $manifestLocation = sprintf('gs://%s/%s', self::$sinkBucket->name(), $manifestName); + + $output = $this->runFunctionSnippet('manifest_request', [ + self::$projectId, self::$sourceAgentPoolName, $rootDirectory, self::$sinkBucket->name(), $manifestLocation + ]); + + $this->assertMatchesRegularExpression('/transferJobs\/.*/', $output); + } finally { + unlink($tempFile); + rmdir($rootDirectory); + self::$sinkBucket->object($manifestName)->delete(); + preg_match('/transferJobs\/\w+/', $output, $match); + self::deleteTransferJob($match[0]); + } + } + // deletes a transfer job created by a sample to clean up + private static function deleteTransferJob($jobName) + { $transferJob = new TransferJob([ 'name' => $jobName, 'status' => Status::DELETED From 0756ae1418cee79731b65c8abcb6721d88267c67 Mon Sep 17 00:00:00 2001 From: Thiyagu K Date: Wed, 16 Oct 2024 18:47:37 +0000 Subject: [PATCH 04/13] added samples and test cases for posix to GCS --- storagetransfer/src/posix_request.php | 71 ++++++++++++++++++++ storagetransfer/test/StorageTransferTest.php | 26 +++++++ 2 files changed, 97 insertions(+) create mode 100644 storagetransfer/src/posix_request.php diff --git a/storagetransfer/src/posix_request.php b/storagetransfer/src/posix_request.php new file mode 100644 index 0000000000..fc1c2bc7d2 --- /dev/null +++ b/storagetransfer/src/posix_request.php @@ -0,0 +1,71 @@ + $projectId, + 'transfer_spec' => new TransferSpec([ + 'source_agent_pool_name' => $sourceAgentPoolName, + 'posix_data_source' => new PosixFilesystem(['root_directory' => $rootDirectory]), + 'gcs_data_sink' => new GcsData(['bucket_name' => $sinkGcsBucketName]) + ]), + 'status' => Status::ENABLED + ]); + + $client = new StorageTransferServiceClient(); + $request = (new CreateTransferJobRequest()) + ->setTransferJob($transferJob); + $response = $client->createTransferJob($request); + $request2 = (new RunTransferJobRequest()) + ->setJobName($response->getName()) + ->setProjectId($projectId); + $client->runTransferJob($request2); + + printf('Created and ran transfer job from %s to %s with name %s ' . PHP_EOL, $rootDirectory, $sinkGcsBucketName, $response->getName()); +} +# [END storagetransfer_transfer_from_posix] + +// The following 2 lines are only needed to run the samples +require_once __DIR__ . '/../../testing/sample_helpers.php'; +\Google\Cloud\Samples\execute_sample(__FILE__, __NAMESPACE__, $argv); diff --git a/storagetransfer/test/StorageTransferTest.php b/storagetransfer/test/StorageTransferTest.php index 65747b75bc..88049fde36 100644 --- a/storagetransfer/test/StorageTransferTest.php +++ b/storagetransfer/test/StorageTransferTest.php @@ -150,6 +150,32 @@ public function testManifestRequest() } } + public function testPosixRequest() + { + try { + $rootDirectory = sys_get_temp_dir() . '/sts-manifest-request-test'; + if (!is_dir($rootDirectory)) { + mkdir($rootDirectory, 0700, true); + } + $tempFile = $rootDirectory . '/text.txt'; + + // Write test data to the temporary file + $testData = 'test data'; + file_put_contents($tempFile, $testData); + + $output = $this->runFunctionSnippet('posix_request', [ + self::$projectId, self::$sourceAgentPoolName, $rootDirectory, self::$sinkBucket->name() + ]); + + $this->assertMatchesRegularExpression('/transferJobs\/.*/', $output); + } finally { + unlink($tempFile); + rmdir($rootDirectory); + preg_match('/transferJobs\/\w+/', $output, $match); + self::deleteTransferJob($match[0]); + } + } + // deletes a transfer job created by a sample to clean up private static function deleteTransferJob($jobName) { From 5a9b2e801241c914261fe32f85973939180649f3 Mon Sep 17 00:00:00 2001 From: Thiyagu K Date: Thu, 17 Oct 2024 15:53:51 +0000 Subject: [PATCH 05/13] fixes --- storagetransfer/src/manifest_request.php | 8 ++++---- storagetransfer/src/nearline_request.php | 8 ++++---- storagetransfer/src/posix_request.php | 9 ++++----- storagetransfer/src/quickstart.php | 8 ++++---- storagetransfer/test/StorageTransferTest.php | 6 ++++-- 5 files changed, 20 insertions(+), 19 deletions(-) diff --git a/storagetransfer/src/manifest_request.php b/storagetransfer/src/manifest_request.php index 88b7038a8d..26f7c3583e 100644 --- a/storagetransfer/src/manifest_request.php +++ b/storagetransfer/src/manifest_request.php @@ -57,13 +57,13 @@ function manifest_request($projectId, $sourceAgentPoolName, $rootDirectory, $sin ]); $client = new StorageTransferServiceClient(); - $request = (new CreateTransferJobRequest()) + $createRequest = (new CreateTransferJobRequest()) ->setTransferJob($transferJob); - $response = $client->createTransferJob($request); - $request2 = (new RunTransferJobRequest()) + $response = $client->createTransferJob($createRequest); + $runRequest = (new RunTransferJobRequest()) ->setJobName($response->getName()) ->setProjectId($projectId); - $client->runTransferJob($request2); + $client->runTransferJob($runRequest); printf('Created and ran transfer job from %s to %s using manifest %s with name %s ' . PHP_EOL, $rootDirectory, $sinkGcsBucketName, $manifestLocation, $response->getName()); } diff --git a/storagetransfer/src/nearline_request.php b/storagetransfer/src/nearline_request.php index c54de2ddd6..55ad0b0247 100644 --- a/storagetransfer/src/nearline_request.php +++ b/storagetransfer/src/nearline_request.php @@ -85,13 +85,13 @@ function nearline_request($projectId, $description, $sourceGcsBucketName, $sinkG ]); $client = new StorageTransferServiceClient(); - $request = (new CreateTransferJobRequest()) + $createRequest = (new CreateTransferJobRequest()) ->setTransferJob($transferJob); - $response = $client->createTransferJob($request); - $request2 = (new RunTransferJobRequest()) + $response = $client->createTransferJob($createRequest); + $runRequest = (new RunTransferJobRequest()) ->setJobName($response->getName()) ->setProjectId($projectId); - $client->runTransferJob($request2); + $client->runTransferJob($runRequest); printf('Created and ran transfer job : %s' . PHP_EOL, $response->getName()); } diff --git a/storagetransfer/src/posix_request.php b/storagetransfer/src/posix_request.php index fc1c2bc7d2..30133788c2 100644 --- a/storagetransfer/src/posix_request.php +++ b/storagetransfer/src/posix_request.php @@ -35,7 +35,6 @@ * @param string $sourceAgentPoolName The agent pool associated with the POSIX data source. * @param string $rootDirectory The root directory path on the source filesystem. * @param string $sinkGcsBucketName The name of the GCS bucket to transfer objects to. - * @param string $manifestLocation Transfer manifest location. Must be a `gs:` URL. */ function posix_request($projectId, $sourceAgentPoolName, $rootDirectory, $sinkGcsBucketName) { @@ -54,13 +53,13 @@ function posix_request($projectId, $sourceAgentPoolName, $rootDirectory, $sinkGc ]); $client = new StorageTransferServiceClient(); - $request = (new CreateTransferJobRequest()) + $createRequest = (new CreateTransferJobRequest()) ->setTransferJob($transferJob); - $response = $client->createTransferJob($request); - $request2 = (new RunTransferJobRequest()) + $response = $client->createTransferJob($createRequest); + $runRequest = (new RunTransferJobRequest()) ->setJobName($response->getName()) ->setProjectId($projectId); - $client->runTransferJob($request2); + $client->runTransferJob($runRequest); printf('Created and ran transfer job from %s to %s with name %s ' . PHP_EOL, $rootDirectory, $sinkGcsBucketName, $response->getName()); } diff --git a/storagetransfer/src/quickstart.php b/storagetransfer/src/quickstart.php index b4affdf58a..278db790ae 100644 --- a/storagetransfer/src/quickstart.php +++ b/storagetransfer/src/quickstart.php @@ -48,13 +48,13 @@ function quickstart($projectId, $sourceGcsBucketName, $sinkGcsBucketName) ]); $client = new StorageTransferServiceClient(); - $request = (new CreateTransferJobRequest()) + $createRequest = (new CreateTransferJobRequest()) ->setTransferJob($transferJob); - $response = $client->createTransferJob($request); - $request2 = (new RunTransferJobRequest()) + $response = $client->createTransferJob($createRequest); + $runRequest = (new RunTransferJobRequest()) ->setJobName($response->getName()) ->setProjectId($projectId); - $client->runTransferJob($request2); + $client->runTransferJob($runRequest); printf('Created and ran transfer job from %s to %s with name %s ' . PHP_EOL, $sourceGcsBucketName, $sinkGcsBucketName, $response->getName()); } diff --git a/storagetransfer/test/StorageTransferTest.php b/storagetransfer/test/StorageTransferTest.php index 88049fde36..e3aa537d42 100644 --- a/storagetransfer/test/StorageTransferTest.php +++ b/storagetransfer/test/StorageTransferTest.php @@ -32,6 +32,7 @@ class StorageTransferTest extends TestCase use TestTrait; private static $sts; + private static $root; private static $storage; private static $sourceBucket; private static $sinkBucket; @@ -39,6 +40,7 @@ class StorageTransferTest extends TestCase public static function setUpBeforeClass(): void { + self::$root = sys_get_temp_dir(); self::checkProjectEnvVars(); self::$storage = new StorageClient(); self::$sts = new StorageTransferServiceClient(); @@ -110,7 +112,7 @@ public function testManifestRequest() { try { $manifestName = 'manifest.csv'; - $rootDirectory = sys_get_temp_dir() . '/sts-manifest-request-test'; + $rootDirectory = self::$root . '/sts-manifest-request-test'; if (!is_dir($rootDirectory)) { mkdir($rootDirectory, 0700, true); } @@ -153,7 +155,7 @@ public function testManifestRequest() public function testPosixRequest() { try { - $rootDirectory = sys_get_temp_dir() . '/sts-manifest-request-test'; + $rootDirectory = self::$root . '/sts-manifest-request-test'; if (!is_dir($rootDirectory)) { mkdir($rootDirectory, 0700, true); } From 21b22ce549714651310ecf959b6b272454212673 Mon Sep 17 00:00:00 2001 From: Thiyagu K Date: Thu, 17 Oct 2024 15:58:38 +0000 Subject: [PATCH 06/13] variables fixes --- storagetransfer/src/manifest_request.php | 8 ++++---- storagetransfer/src/nearline_request.php | 8 ++++---- storagetransfer/src/quickstart.php | 8 ++++---- 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/storagetransfer/src/manifest_request.php b/storagetransfer/src/manifest_request.php index 88b7038a8d..26f7c3583e 100644 --- a/storagetransfer/src/manifest_request.php +++ b/storagetransfer/src/manifest_request.php @@ -57,13 +57,13 @@ function manifest_request($projectId, $sourceAgentPoolName, $rootDirectory, $sin ]); $client = new StorageTransferServiceClient(); - $request = (new CreateTransferJobRequest()) + $createRequest = (new CreateTransferJobRequest()) ->setTransferJob($transferJob); - $response = $client->createTransferJob($request); - $request2 = (new RunTransferJobRequest()) + $response = $client->createTransferJob($createRequest); + $runRequest = (new RunTransferJobRequest()) ->setJobName($response->getName()) ->setProjectId($projectId); - $client->runTransferJob($request2); + $client->runTransferJob($runRequest); printf('Created and ran transfer job from %s to %s using manifest %s with name %s ' . PHP_EOL, $rootDirectory, $sinkGcsBucketName, $manifestLocation, $response->getName()); } diff --git a/storagetransfer/src/nearline_request.php b/storagetransfer/src/nearline_request.php index c54de2ddd6..55ad0b0247 100644 --- a/storagetransfer/src/nearline_request.php +++ b/storagetransfer/src/nearline_request.php @@ -85,13 +85,13 @@ function nearline_request($projectId, $description, $sourceGcsBucketName, $sinkG ]); $client = new StorageTransferServiceClient(); - $request = (new CreateTransferJobRequest()) + $createRequest = (new CreateTransferJobRequest()) ->setTransferJob($transferJob); - $response = $client->createTransferJob($request); - $request2 = (new RunTransferJobRequest()) + $response = $client->createTransferJob($createRequest); + $runRequest = (new RunTransferJobRequest()) ->setJobName($response->getName()) ->setProjectId($projectId); - $client->runTransferJob($request2); + $client->runTransferJob($runRequest); printf('Created and ran transfer job : %s' . PHP_EOL, $response->getName()); } diff --git a/storagetransfer/src/quickstart.php b/storagetransfer/src/quickstart.php index b4affdf58a..278db790ae 100644 --- a/storagetransfer/src/quickstart.php +++ b/storagetransfer/src/quickstart.php @@ -48,13 +48,13 @@ function quickstart($projectId, $sourceGcsBucketName, $sinkGcsBucketName) ]); $client = new StorageTransferServiceClient(); - $request = (new CreateTransferJobRequest()) + $createRequest = (new CreateTransferJobRequest()) ->setTransferJob($transferJob); - $response = $client->createTransferJob($request); - $request2 = (new RunTransferJobRequest()) + $response = $client->createTransferJob($createRequest); + $runRequest = (new RunTransferJobRequest()) ->setJobName($response->getName()) ->setProjectId($projectId); - $client->runTransferJob($request2); + $client->runTransferJob($runRequest); printf('Created and ran transfer job from %s to %s with name %s ' . PHP_EOL, $sourceGcsBucketName, $sinkGcsBucketName, $response->getName()); } From 83cd2efe9a11856ec85590c040d436834221d9c7 Mon Sep 17 00:00:00 2001 From: Thiyagu K Date: Thu, 17 Oct 2024 16:06:58 +0000 Subject: [PATCH 07/13] added samples and test cases for posix to posix --- .../src/posix_to_posix_request.php | 76 +++++++++++++++++++ storagetransfer/test/StorageTransferTest.php | 38 +++++++++- 2 files changed, 112 insertions(+), 2 deletions(-) create mode 100644 storagetransfer/src/posix_to_posix_request.php diff --git a/storagetransfer/src/posix_to_posix_request.php b/storagetransfer/src/posix_to_posix_request.php new file mode 100644 index 0000000000..97c5a15a82 --- /dev/null +++ b/storagetransfer/src/posix_to_posix_request.php @@ -0,0 +1,76 @@ + $projectId, + 'transfer_spec' => new TransferSpec([ + 'source_agent_pool_name' => $sourceAgentPoolName, + 'sink_agent_pool_name' => $sinkAgentPoolName, + 'posix_data_source' => new PosixFilesystem(['root_directory' => $rootDirectory]), + 'posix_data_sink' => new PosixFilesystem(['root_directory' => $destinationDirectory]), + 'gcs_intermediate_data_location' => new GcsData(['bucket_name' => $bucketName]) + ]), + 'status' => Status::ENABLED + ]); + + $client = new StorageTransferServiceClient(); + $createRequest = (new CreateTransferJobRequest()) + ->setTransferJob($transferJob); + $response = $client->createTransferJob($createRequest); + $runRequest = (new RunTransferJobRequest()) + ->setJobName($response->getName()) + ->setProjectId($projectId); + $client->runTransferJob($runRequest); + + printf('Created and ran transfer job from %s to %s with name %s ' . PHP_EOL, $rootDirectory, $destinationDirectory, $response->getName()); +} +# [END storagetransfer_transfer_posix_to_posix] + +// The following 2 lines are only needed to run the samples +require_once __DIR__ . '/../../testing/sample_helpers.php'; +\Google\Cloud\Samples\execute_sample(__FILE__, __NAMESPACE__, $argv); diff --git a/storagetransfer/test/StorageTransferTest.php b/storagetransfer/test/StorageTransferTest.php index 88049fde36..2000ef1cb5 100644 --- a/storagetransfer/test/StorageTransferTest.php +++ b/storagetransfer/test/StorageTransferTest.php @@ -32,6 +32,7 @@ class StorageTransferTest extends TestCase use TestTrait; private static $sts; + private static $root; private static $storage; private static $sourceBucket; private static $sinkBucket; @@ -39,6 +40,7 @@ class StorageTransferTest extends TestCase public static function setUpBeforeClass(): void { + self::$root = sys_get_temp_dir(); self::checkProjectEnvVars(); self::$storage = new StorageClient(); self::$sts = new StorageTransferServiceClient(); @@ -110,7 +112,7 @@ public function testManifestRequest() { try { $manifestName = 'manifest.csv'; - $rootDirectory = sys_get_temp_dir() . '/sts-manifest-request-test'; + $rootDirectory = self::$root . '/sts-manifest-request-test'; if (!is_dir($rootDirectory)) { mkdir($rootDirectory, 0700, true); } @@ -153,7 +155,7 @@ public function testManifestRequest() public function testPosixRequest() { try { - $rootDirectory = sys_get_temp_dir() . '/sts-manifest-request-test'; + $rootDirectory = self::$root . '/sts-manifest-request-test'; if (!is_dir($rootDirectory)) { mkdir($rootDirectory, 0700, true); } @@ -176,6 +178,38 @@ public function testPosixRequest() } } + public function testPosixToPosixRequest() + { + try { + $sinkAgentPoolName = ''; + $rootDirectory = self::$root . '/sts-posix-test-source'; + $destinationDirectory = self::$root . '/sts-posix-test-sink'; + if (!is_dir($rootDirectory)) { + mkdir($rootDirectory, 0700, true); + } + if (!is_dir($destinationDirectory)) { + mkdir($destinationDirectory, 0700, true); + } + $tempFile = $rootDirectory . '/text.txt'; + + // Write test data to the temporary file + $testData = 'test data'; + file_put_contents($tempFile, $testData); + + $output = $this->runFunctionSnippet('posix_to_posix_request', [ + self::$projectId, self::$sourceAgentPoolName, $sinkAgentPoolName, $rootDirectory, $destinationDirectory, self::$sinkBucket->name() + ]); + + $this->assertMatchesRegularExpression('/transferJobs\/.*/', $output); + } finally { + unlink($tempFile); + rmdir($rootDirectory); + rmdir($destinationDirectory); + preg_match('/transferJobs\/\w+/', $output, $match); + self::deleteTransferJob($match[0]); + } + } + // deletes a transfer job created by a sample to clean up private static function deleteTransferJob($jobName) { From 2c22adf9e0f85a8bfe8939632a1a3d6b0b76c9ac Mon Sep 17 00:00:00 2001 From: Thiyagu K Date: Thu, 17 Oct 2024 16:11:58 +0000 Subject: [PATCH 08/13] added samples and test cases for GCS to posix --- storagetransfer/src/posix_download.php | 75 ++++++++++++++++++++ storagetransfer/test/StorageTransferTest.php | 41 ++++++++++- 2 files changed, 114 insertions(+), 2 deletions(-) create mode 100644 storagetransfer/src/posix_download.php diff --git a/storagetransfer/src/posix_download.php b/storagetransfer/src/posix_download.php new file mode 100644 index 0000000000..7c1408dadc --- /dev/null +++ b/storagetransfer/src/posix_download.php @@ -0,0 +1,75 @@ + $projectId, + 'transfer_spec' => new TransferSpec([ + 'sink_agent_pool_name' => $sinkAgentPoolName, + 'gcs_data_source' => new GcsData([ + 'bucket_name' => $gcsSourceBucket, + 'path' => $gcsSourcePath + ]), + 'posix_data_sink' => new PosixFilesystem(['root_directory' => $rootDirectory]) + ]), + 'status' => Status::ENABLED + ]); + + $client = new StorageTransferServiceClient(); + $createRequest = (new CreateTransferJobRequest()) + ->setTransferJob($transferJob); + $response = $client->createTransferJob($createRequest); + $runRequest = (new RunTransferJobRequest()) + ->setJobName($response->getName()) + ->setProjectId($projectId); + $client->runTransferJob($runRequest); + + printf('Created and ran a transfer job from %s to %s with name %s ' . PHP_EOL, $gcsSourcePath, $rootDirectory, $response->getName()); +} +# [END storagetransfer_download_to_posix] + +// The following 2 lines are only needed to run the samples +require_once __DIR__ . '/../../testing/sample_helpers.php'; +\Google\Cloud\Samples\execute_sample(__FILE__, __NAMESPACE__, $argv); diff --git a/storagetransfer/test/StorageTransferTest.php b/storagetransfer/test/StorageTransferTest.php index 88049fde36..835f3682e9 100644 --- a/storagetransfer/test/StorageTransferTest.php +++ b/storagetransfer/test/StorageTransferTest.php @@ -32,6 +32,7 @@ class StorageTransferTest extends TestCase use TestTrait; private static $sts; + private static $root; private static $storage; private static $sourceBucket; private static $sinkBucket; @@ -39,6 +40,7 @@ class StorageTransferTest extends TestCase public static function setUpBeforeClass(): void { + self::$root = sys_get_temp_dir(); self::checkProjectEnvVars(); self::$storage = new StorageClient(); self::$sts = new StorageTransferServiceClient(); @@ -110,7 +112,7 @@ public function testManifestRequest() { try { $manifestName = 'manifest.csv'; - $rootDirectory = sys_get_temp_dir() . '/sts-manifest-request-test'; + $rootDirectory = self::$root . '/sts-manifest-request-test'; if (!is_dir($rootDirectory)) { mkdir($rootDirectory, 0700, true); } @@ -153,7 +155,7 @@ public function testManifestRequest() public function testPosixRequest() { try { - $rootDirectory = sys_get_temp_dir() . '/sts-manifest-request-test'; + $rootDirectory = self::$root . '/sts-manifest-request-test'; if (!is_dir($rootDirectory)) { mkdir($rootDirectory, 0700, true); } @@ -176,6 +178,41 @@ public function testPosixRequest() } } + public function testDownloadToPosix() + { + try { + $tempFileName = 'text.txt'; + $sinkAgentPoolName = ''; + $rootDirectory = self::$root . '/sts-download-to-posix-test'; + $gcsSourcePath = 'sts-manifest-request-test/'; + if (!is_dir($rootDirectory)) { + mkdir($rootDirectory, 0700, true); + } + $tempFile = $rootDirectory . '/' . $tempFileName; + file_put_contents($tempFile, 'test data'); + + // Upload the temporary file to GCS + self::$sourceBucket->upload( + fopen($tempFile, 'r'), + [ + 'name' => $tempFileName + ] + ); + + $output = $this->runFunctionSnippet('posix_download', [ + self::$projectId, $sinkAgentPoolName, self::$sourceBucket->name(), $gcsSourcePath, $rootDirectory + ]); + + $this->assertMatchesRegularExpression('/transferJobs\/.*/', $output); + } finally { + unlink($tempFile); + rmdir($rootDirectory); + self::$sourceBucket->object($tempFileName)->delete(); + preg_match('/transferJobs\/\w+/', $output, $match); + self::deleteTransferJob($match[0]); + } + } + // deletes a transfer job created by a sample to clean up private static function deleteTransferJob($jobName) { From fc988fd9a395460d6e27fbadd3ef99637c1ea97e Mon Sep 17 00:00:00 2001 From: Thiyagu K Date: Thu, 31 Oct 2024 10:42:36 +0000 Subject: [PATCH 09/13] resolve conflict --- storagetransfer/test/StorageTransferTest.php | 32 ++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/storagetransfer/test/StorageTransferTest.php b/storagetransfer/test/StorageTransferTest.php index 835f3682e9..c7c15c9d80 100644 --- a/storagetransfer/test/StorageTransferTest.php +++ b/storagetransfer/test/StorageTransferTest.php @@ -178,6 +178,38 @@ public function testPosixRequest() } } + public function testPosixToPosixRequest() + { + try { + $sinkAgentPoolName = ''; + $rootDirectory = self::$root . '/sts-posix-test-source'; + $destinationDirectory = self::$root . '/sts-posix-test-sink'; + if (!is_dir($rootDirectory)) { + mkdir($rootDirectory, 0700, true); + } + if (!is_dir($destinationDirectory)) { + mkdir($destinationDirectory, 0700, true); + } + $tempFile = $rootDirectory . '/text.txt'; + + // Write test data to the temporary file + $testData = 'test data'; + file_put_contents($tempFile, $testData); + + $output = $this->runFunctionSnippet('posix_to_posix_request', [ + self::$projectId, self::$sourceAgentPoolName, $sinkAgentPoolName, $rootDirectory, $destinationDirectory, self::$sinkBucket->name() + ]); + + $this->assertMatchesRegularExpression('/transferJobs\/.*/', $output); + } finally { + unlink($tempFile); + rmdir($rootDirectory); + rmdir($destinationDirectory); + preg_match('/transferJobs\/\w+/', $output, $match); + self::deleteTransferJob($match[0]); + } + } + public function testDownloadToPosix() { try { From 55ba8e39654751173652d4cbe3fa39fd75452231 Mon Sep 17 00:00:00 2001 From: Thiyagu K Date: Mon, 4 Nov 2024 05:50:02 +0000 Subject: [PATCH 10/13] added typehints added typehints --- .../src/check_latest_transfer_operation.php | 8 +++++--- storagetransfer/src/manifest_request.php | 9 +++++++-- storagetransfer/src/nearline_request.php | 9 +++++++-- storagetransfer/src/posix_download.php | 10 ++++++++-- storagetransfer/src/posix_request.php | 8 ++++++-- storagetransfer/src/posix_to_posix_request.php | 10 ++++++++-- storagetransfer/src/quickstart.php | 7 +++++-- 7 files changed, 46 insertions(+), 15 deletions(-) diff --git a/storagetransfer/src/check_latest_transfer_operation.php b/storagetransfer/src/check_latest_transfer_operation.php index 9c5ab11837..fcd9c6df4f 100644 --- a/storagetransfer/src/check_latest_transfer_operation.php +++ b/storagetransfer/src/check_latest_transfer_operation.php @@ -27,8 +27,10 @@ * @param string $projectId Your Google Cloud project ID. * @param string $jobName Storage Transfer Service job name. */ -function check_latest_transfer_operation($projectId, $jobName) -{ +function check_latest_transfer_operation( + string $projectId, + string $jobName +): void { // $project = 'my-project-id'; // $jobName = 'myJob/1234567890'; $transferJob = new GetTransferJobRequest([ @@ -40,7 +42,7 @@ function check_latest_transfer_operation($projectId, $jobName) $request = $client->getTransferJob($transferJob); $latestOperationName = $request->getLatestOperationName(); - if($latestOperationName){ + if ($latestOperationName) { $transferOperation = $client->getOperationsClient()->getOperation($latestOperationName); $operation = $transferOperation->getMetadata(); diff --git a/storagetransfer/src/manifest_request.php b/storagetransfer/src/manifest_request.php index 26f7c3583e..cc52e5512f 100644 --- a/storagetransfer/src/manifest_request.php +++ b/storagetransfer/src/manifest_request.php @@ -38,8 +38,13 @@ * @param string $sinkGcsBucketName The name of the GCS bucket to transfer objects to. * @param string $manifestLocation Transfer manifest location. Must be a `gs:` URL. */ -function manifest_request($projectId, $sourceAgentPoolName, $rootDirectory, $sinkGcsBucketName, $manifestLocation) -{ +function manifest_request( + string $projectId, + string $sourceAgentPoolName, + string $rootDirectory, + string $sinkGcsBucketName, + string $manifestLocation +): void { // $project = 'my-project-id'; // $sourceAgentPoolName = 'projects/my-project/agentPools/transfer_service_default'; // $rootDirectory = '/directory/to/transfer/source'; diff --git a/storagetransfer/src/nearline_request.php b/storagetransfer/src/nearline_request.php index 55ad0b0247..ea7ae7fe67 100644 --- a/storagetransfer/src/nearline_request.php +++ b/storagetransfer/src/nearline_request.php @@ -43,8 +43,13 @@ * @param string $sinkGcsBucketName The name of the GCS bucket to transfer objects to. * @param string $startDate Date to start daily migration. */ -function nearline_request($projectId, $description, $sourceGcsBucketName, $sinkGcsBucketName, $startDate) -{ +function nearline_request( + string $projectId, + string $description, + string $sourceGcsBucketName, + string $sinkGcsBucketName, + string $startDate +): void { // $project = 'my-project-id'; // $description = 'My transfer job'; // $sourceGcsBucketName = 'my-source-bucket'; diff --git a/storagetransfer/src/posix_download.php b/storagetransfer/src/posix_download.php index 7c1408dadc..2a50f3543e 100644 --- a/storagetransfer/src/posix_download.php +++ b/storagetransfer/src/posix_download.php @@ -1,4 +1,5 @@ Date: Wed, 6 Nov 2024 08:22:48 +0000 Subject: [PATCH 11/13] added samples and test cases for event driven GCS transfer added samples and test cases for event driven GCS transfer --- storagetransfer/composer.json | 3 +- .../src/event_driven_gcs_transfer.php | 71 +++++++++++++ storagetransfer/test/StorageTransferTest.php | 100 ++++++++++++++++-- 3 files changed, 165 insertions(+), 9 deletions(-) create mode 100644 storagetransfer/src/event_driven_gcs_transfer.php diff --git a/storagetransfer/composer.json b/storagetransfer/composer.json index c4c7b78edb..e18cf526e7 100644 --- a/storagetransfer/composer.json +++ b/storagetransfer/composer.json @@ -4,6 +4,7 @@ "paragonie/random_compat": "^9.0.0" }, "require-dev": { - "google/cloud-storage": "^1.20.1" + "google/cloud-storage": "^1.20.1", + "google/cloud-pubsub": "^2.0" } } diff --git a/storagetransfer/src/event_driven_gcs_transfer.php b/storagetransfer/src/event_driven_gcs_transfer.php new file mode 100644 index 0000000000..a31e399ce7 --- /dev/null +++ b/storagetransfer/src/event_driven_gcs_transfer.php @@ -0,0 +1,71 @@ + $projectId, + 'transfer_spec' => new TransferSpec([ + 'gcs_data_sink' => new GcsData(['bucket_name' => $sinkGcsBucketName]), + 'gcs_data_source' => new GcsData(['bucket_name' => $sourceGcsBucketName]) + ]), + 'event_stream' => new EventStream(['name' => $pubsubId]), + 'status' => Status::ENABLED + ]); + + $client = new StorageTransferServiceClient(); + $createRequest = (new CreateTransferJobRequest()) + ->setTransferJob($transferJob); + $response = $client->createTransferJob($createRequest); + + printf('Created an event driven transfer from %s to %s with name %s .' . PHP_EOL, $sourceGcsBucketName, $sinkGcsBucketName, $response->getName()); +} +# [END storagetransfer_create_event_driven_gcs_transfer] + +// The following 2 lines are only needed to run the samples +require_once __DIR__ . '/../../testing/sample_helpers.php'; +\Google\Cloud\Samples\execute_sample(__FILE__, __NAMESPACE__, $argv); diff --git a/storagetransfer/test/StorageTransferTest.php b/storagetransfer/test/StorageTransferTest.php index c7c15c9d80..c356fbac53 100644 --- a/storagetransfer/test/StorageTransferTest.php +++ b/storagetransfer/test/StorageTransferTest.php @@ -1,4 +1,5 @@ createTopic( + sprintf('php-pubsub-sts-topic-%s', $uniqueBucketId) + ); + + self::$subscription = self::$topic->subscription( + sprintf('php-pubsub-sts-subscription-%s', $uniqueBucketId) + ); + self::$subscription->create(); + + self::grantStsPubSubPermissions(); } public static function tearDownAfterClass(): void { self::$sourceBucket->delete(); self::$sinkBucket->delete(); + self::$topic->delete(); + self::$subscription->delete(); } public function testQuickstart() { $output = $this->runFunctionSnippet('quickstart', [ - self::$projectId, self::$sinkBucket->name(), self::$sourceBucket->name() + self::$projectId, + self::$sinkBucket->name(), + self::$sourceBucket->name() ]); $this->assertMatchesRegularExpression('/transferJobs\/.*/', $output); @@ -77,13 +98,16 @@ public function testQuickstart() public function testCheckLatestTransferOperation() { $transferData = $this->runFunctionSnippet('quickstart', [ - self::$projectId, self::$sinkBucket->name(), self::$sourceBucket->name() + self::$projectId, + self::$sinkBucket->name(), + self::$sourceBucket->name() ]); preg_match('/transferJobs\/\d+/', $transferData, $match); $jobName = $match[0]; $output = $this->runFunctionSnippet('check_latest_transfer_operation', [ - self::$projectId, $jobName + self::$projectId, + $jobName ]); $this->assertMatchesRegularExpression('/transferJobs\/.*/', $output); @@ -99,7 +123,11 @@ public function testNearlineRequest() $startDate = $date->format('Y-m-d H:i:s'); $output = $this->runFunctionSnippet('nearline_request', [ - self::$projectId, $description, self::$sourceBucket->name(), self::$sinkBucket->name(), $startDate + self::$projectId, + $description, + self::$sourceBucket->name(), + self::$sinkBucket->name(), + $startDate ]); $this->assertMatchesRegularExpression('/Created and ran transfer job : transferJobs\/.*/', $output); @@ -139,7 +167,11 @@ public function testManifestRequest() $manifestLocation = sprintf('gs://%s/%s', self::$sinkBucket->name(), $manifestName); $output = $this->runFunctionSnippet('manifest_request', [ - self::$projectId, self::$sourceAgentPoolName, $rootDirectory, self::$sinkBucket->name(), $manifestLocation + self::$projectId, + self::$sourceAgentPoolName, + $rootDirectory, + self::$sinkBucket->name(), + $manifestLocation ]); $this->assertMatchesRegularExpression('/transferJobs\/.*/', $output); @@ -166,7 +198,10 @@ public function testPosixRequest() file_put_contents($tempFile, $testData); $output = $this->runFunctionSnippet('posix_request', [ - self::$projectId, self::$sourceAgentPoolName, $rootDirectory, self::$sinkBucket->name() + self::$projectId, + self::$sourceAgentPoolName, + $rootDirectory, + self::$sinkBucket->name() ]); $this->assertMatchesRegularExpression('/transferJobs\/.*/', $output); @@ -197,7 +232,12 @@ public function testPosixToPosixRequest() file_put_contents($tempFile, $testData); $output = $this->runFunctionSnippet('posix_to_posix_request', [ - self::$projectId, self::$sourceAgentPoolName, $sinkAgentPoolName, $rootDirectory, $destinationDirectory, self::$sinkBucket->name() + self::$projectId, + self::$sourceAgentPoolName, + $sinkAgentPoolName, + $rootDirectory, + $destinationDirectory, + self::$sinkBucket->name() ]); $this->assertMatchesRegularExpression('/transferJobs\/.*/', $output); @@ -232,7 +272,11 @@ public function testDownloadToPosix() ); $output = $this->runFunctionSnippet('posix_download', [ - self::$projectId, $sinkAgentPoolName, self::$sourceBucket->name(), $gcsSourcePath, $rootDirectory + self::$projectId, + $sinkAgentPoolName, + self::$sourceBucket->name(), + $gcsSourcePath, + $rootDirectory ]); $this->assertMatchesRegularExpression('/transferJobs\/.*/', $output); @@ -245,6 +289,23 @@ public function testDownloadToPosix() } } + public function testEventDrivenGCSRequest() + { + try { + $output = $this->runFunctionSnippet('event_driven_gcs_transfer', [ + self::$projectId, + self::$sourceBucket->name(), + self::$sinkBucket->name(), + self::$subscription->name() + ]); + + $this->assertMatchesRegularExpression('/transferJobs\/.*/', $output); + } finally { + preg_match('/transferJobs\/\w+/', $output, $match); + self::deleteTransferJob($match[0]); + } + } + // deletes a transfer job created by a sample to clean up private static function deleteTransferJob($jobName) { @@ -280,4 +341,27 @@ private static function grantStsPermissions($bucket) $bucket->iam()->setPolicy($policy); } + + private static function grantStsPubSubPermissions() + { + $request2 = (new GetGoogleServiceAccountRequest()) + ->setProjectId(self::$projectId); + $googleServiceAccount = self::$sts->getGoogleServiceAccount($request2); + $email = $googleServiceAccount->getAccountEmail(); + $members = ['serviceAccount:' . $email]; + + $topicPolicy = self::$topic->iam()->policy(); + $topicPolicy['bindings'][] = [ + 'role' => 'roles/pubsub.publisher', + 'members' => $members + ]; + self::$topic->iam()->setPolicy($topicPolicy); + + $subscriptionPolicy = self::$subscription->iam()->policy(); + $subscriptionPolicy['bindings'][] = [ + 'role' => 'roles/pubsub.subscriber', + 'members' => $members + ]; + self::$subscription->iam()->setPolicy($subscriptionPolicy); + } } From 61f4ac5321f75a5eb336cdffb010639d079be663 Mon Sep 17 00:00:00 2001 From: Thiyagu K Date: Mon, 18 Nov 2024 13:14:42 +0530 Subject: [PATCH 12/13] lint fix --- compute/firewall/src/print_firewall_rule.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compute/firewall/src/print_firewall_rule.php b/compute/firewall/src/print_firewall_rule.php index 65ee07f757..bab5a7bc5e 100644 --- a/compute/firewall/src/print_firewall_rule.php +++ b/compute/firewall/src/print_firewall_rule.php @@ -54,7 +54,7 @@ function print_firewall_rule(string $projectId, string $firewallRuleName) printf('Self Link: %s' . PHP_EOL, $response->getSelfLink()); printf('Logging Enabled: %s' . PHP_EOL, var_export($response->getLogConfig()->getEnable(), true)); print('--Allowed--' . PHP_EOL); - foreach ($response->getAllowed()as $item) { + foreach ($response->getAllowed() as $item) { printf('Protocol: %s' . PHP_EOL, $item->getIPProtocol()); foreach ($item->getPorts() as $ports) { printf(' - Ports: %s' . PHP_EOL, $ports); From 2248b4603961910853ded1a0a33c4806e8ab253b Mon Sep 17 00:00:00 2001 From: Brent Shaffer Date: Fri, 13 Dec 2024 13:33:10 -0800 Subject: [PATCH 13/13] use latest library version --- storagetransfer/composer.json | 2 +- storagetransfer/src/check_latest_transfer_operation.php | 5 ++--- storagetransfer/src/nearline_request.php | 2 +- 3 files changed, 4 insertions(+), 5 deletions(-) diff --git a/storagetransfer/composer.json b/storagetransfer/composer.json index e18cf526e7..91a80dc7db 100644 --- a/storagetransfer/composer.json +++ b/storagetransfer/composer.json @@ -1,6 +1,6 @@ { "require": { - "google/cloud-storage-transfer": "^1.4", + "google/cloud-storage-transfer": "^2.0", "paragonie/random_compat": "^9.0.0" }, "require-dev": { diff --git a/storagetransfer/src/check_latest_transfer_operation.php b/storagetransfer/src/check_latest_transfer_operation.php index fcd9c6df4f..5f2f3ceefe 100644 --- a/storagetransfer/src/check_latest_transfer_operation.php +++ b/storagetransfer/src/check_latest_transfer_operation.php @@ -43,9 +43,8 @@ function check_latest_transfer_operation( $latestOperationName = $request->getLatestOperationName(); if ($latestOperationName) { - $transferOperation = $client->getOperationsClient()->getOperation($latestOperationName); - - $operation = $transferOperation->getMetadata(); + $transferOperation = $client->resumeOperation($latestOperationName); + $operation = $transferOperation->getLastProtoResponse(); printf('Latest transfer operation for %s is: %s ' . PHP_EOL, $jobName, $operation->serializeToJsonString()); } else { diff --git a/storagetransfer/src/nearline_request.php b/storagetransfer/src/nearline_request.php index ea7ae7fe67..c5f95c0095 100644 --- a/storagetransfer/src/nearline_request.php +++ b/storagetransfer/src/nearline_request.php @@ -59,7 +59,7 @@ function nearline_request( $dateTime = new DateTime($startDate); $date = new Date([ 'year' => $dateTime->format('Y'), - 'month' => $dateTime->format('m') + 1, + 'month' => $dateTime->format('m'), 'day' => $dateTime->format('d'), ]);