From a6bdb5a97d8e8ae850110a5816a2a89c241c3fe5 Mon Sep 17 00:00:00 2001 From: Arne Franken Date: Mon, 27 May 2024 00:06:07 +0200 Subject: [PATCH 1/8] Support versionId in APIs --- .../testing/s3mock/ObjectController.java | 25 ++++-- .../adobe/testing/s3mock/dto/CopySource.java | 21 +++-- .../s3mock/dto/VersioningConfiguration.java | 81 +++++++++++++++++++ .../testing/s3mock/util/AwsHttpHeaders.java | 1 + .../s3mock/util/AwsHttpParameters.java | 4 +- .../testing/s3mock/dto/CopySourceTest.kt | 15 +++- .../s3mock/dto/VersioningConfigurationTest.kt | 44 ++++++++++ ...gConfigurationTest_testDeserialization.xml | 22 +++++ ...ingConfigurationTest_testSerialization.xml | 21 +++++ 9 files changed, 218 insertions(+), 16 deletions(-) create mode 100644 server/src/main/java/com/adobe/testing/s3mock/dto/VersioningConfiguration.java create mode 100644 server/src/test/kotlin/com/adobe/testing/s3mock/dto/VersioningConfigurationTest.kt create mode 100644 server/src/test/resources/com/adobe/testing/s3mock/dto/VersioningConfigurationTest_testDeserialization.xml create mode 100644 server/src/test/resources/com/adobe/testing/s3mock/dto/VersioningConfigurationTest_testSerialization.xml diff --git a/server/src/main/java/com/adobe/testing/s3mock/ObjectController.java b/server/src/main/java/com/adobe/testing/s3mock/ObjectController.java index c970d5fea..d78e891c8 100644 --- a/server/src/main/java/com/adobe/testing/s3mock/ObjectController.java +++ b/server/src/main/java/com/adobe/testing/s3mock/ObjectController.java @@ -47,6 +47,7 @@ import static com.adobe.testing.s3mock.util.AwsHttpParameters.NOT_UPLOAD_ID; import static com.adobe.testing.s3mock.util.AwsHttpParameters.RETENTION; import static com.adobe.testing.s3mock.util.AwsHttpParameters.TAGGING; +import static com.adobe.testing.s3mock.util.AwsHttpParameters.VERSION_ID; import static com.adobe.testing.s3mock.util.HeaderUtil.checksumAlgorithmFromHeader; import static com.adobe.testing.s3mock.util.HeaderUtil.checksumAlgorithmFromSdk; import static com.adobe.testing.s3mock.util.HeaderUtil.checksumFrom; @@ -190,6 +191,7 @@ public ResponseEntity headObject(@PathVariable String bucketName, @RequestHeader(value = IF_NONE_MATCH, required = false) List noneMatch, @RequestHeader(value = IF_MODIFIED_SINCE, required = false) List ifModifiedSince, @RequestHeader(value = IF_UNMODIFIED_SINCE, required = false) List ifUnmodifiedSince, + @RequestParam(value = VERSION_ID, required = false) String versionId, @RequestParam Map queryParams) { bucketService.verifyBucketExists(bucketName); @@ -232,7 +234,8 @@ public ResponseEntity headObject(@PathVariable String bucketName, } ) public ResponseEntity deleteObject(@PathVariable String bucketName, - @PathVariable ObjectKey key) { + @PathVariable ObjectKey key, + @RequestParam(value = VERSION_ID, required = false) String versionId) { bucketService.verifyBucketExists(bucketName); var deleted = objectService.deleteObject(bucketName, key.key()); @@ -270,6 +273,7 @@ public ResponseEntity getObject(@PathVariable String buck @RequestHeader(value = IF_NONE_MATCH, required = false) List noneMatch, @RequestHeader(value = IF_MODIFIED_SINCE, required = false) List ifModifiedSince, @RequestHeader(value = IF_UNMODIFIED_SINCE, required = false) List ifUnmodifiedSince, + @RequestParam(value = VERSION_ID, required = false) String versionId, @RequestParam Map queryParams) { bucketService.verifyBucketExists(bucketName); @@ -322,6 +326,7 @@ public ResponseEntity getObject(@PathVariable String buck public ResponseEntity putObjectAcl(@PathVariable final String bucketName, @PathVariable ObjectKey key, @RequestHeader(value = X_AMZ_ACL, required = false) ObjectCannedACL cannedAcl, + @RequestParam(value = VERSION_ID, required = false) String versionId, @RequestBody(required = false) AccessControlPolicy body) { bucketService.verifyBucketExists(bucketName); objectService.verifyObjectExists(bucketName, key.key()); @@ -360,7 +365,8 @@ public ResponseEntity putObjectAcl(@PathVariable final String bucketName, produces = APPLICATION_XML_VALUE ) public ResponseEntity getObjectAcl(@PathVariable final String bucketName, - @PathVariable ObjectKey key) { + @PathVariable ObjectKey key, + @RequestParam(value = VERSION_ID, required = false) String versionId) { bucketService.verifyBucketExists(bucketName); objectService.verifyObjectExists(bucketName, key.key()); var acl = objectService.getAcl(bucketName, key.key()); @@ -384,7 +390,8 @@ public ResponseEntity getObjectAcl(@PathVariable final Stri } ) public ResponseEntity getObjectTagging(@PathVariable String bucketName, - @PathVariable ObjectKey key) { + @PathVariable ObjectKey key, + @RequestParam(value = VERSION_ID, required = false) String versionId) { bucketService.verifyBucketExists(bucketName); var s3ObjectMetadata = objectService.verifyObjectExists(bucketName, key.key()); @@ -412,6 +419,7 @@ public ResponseEntity getObjectTagging(@PathVariable String bucketName, ) public ResponseEntity putObjectTagging(@PathVariable String bucketName, @PathVariable ObjectKey key, + @RequestParam(value = VERSION_ID, required = false) String versionId, @RequestBody Tagging body) { bucketService.verifyBucketExists(bucketName); @@ -439,7 +447,8 @@ public ResponseEntity putObjectTagging(@PathVariable String bucketName, produces = APPLICATION_XML_VALUE ) public ResponseEntity getLegalHold(@PathVariable String bucketName, - @PathVariable ObjectKey key) { + @PathVariable ObjectKey key, + @RequestParam(value = VERSION_ID, required = false) String versionId) { bucketService.verifyBucketExists(bucketName); bucketService.verifyBucketObjectLockEnabled(bucketName); var s3ObjectMetadata = objectService.verifyObjectLockConfiguration(bucketName, key.key()); @@ -464,6 +473,7 @@ public ResponseEntity getLegalHold(@PathVariable String bucketName, ) public ResponseEntity putLegalHold(@PathVariable String bucketName, @PathVariable ObjectKey key, + @RequestParam(value = VERSION_ID, required = false) String versionId, @RequestBody LegalHold body) { bucketService.verifyBucketExists(bucketName); bucketService.verifyBucketObjectLockEnabled(bucketName); @@ -490,7 +500,8 @@ public ResponseEntity putLegalHold(@PathVariable String bucketName, produces = APPLICATION_XML_VALUE ) public ResponseEntity getObjectRetention(@PathVariable String bucketName, - @PathVariable ObjectKey key) { + @PathVariable ObjectKey key, + @RequestParam(value = VERSION_ID, required = false) String versionId) { bucketService.verifyBucketExists(bucketName); bucketService.verifyBucketObjectLockEnabled(bucketName); var s3ObjectMetadata = objectService.verifyObjectLockConfiguration(bucketName, key.key()); @@ -515,6 +526,7 @@ public ResponseEntity getObjectRetention(@PathVariable String bucketN ) public ResponseEntity putObjectRetention(@PathVariable String bucketName, @PathVariable ObjectKey key, + @RequestParam(value = VERSION_ID, required = false) String versionId, @RequestBody Retention body) { bucketService.verifyBucketExists(bucketName); bucketService.verifyBucketObjectLockEnabled(bucketName); @@ -547,7 +559,8 @@ public ResponseEntity getObjectAttributes( @RequestHeader(value = IF_NONE_MATCH, required = false) List noneMatch, @RequestHeader(value = IF_MODIFIED_SINCE, required = false) List ifModifiedSince, @RequestHeader(value = IF_UNMODIFIED_SINCE, required = false) List ifUnmodifiedSince, - @RequestHeader(value = X_AMZ_OBJECT_ATTRIBUTES) List objectAttributes) { + @RequestHeader(value = X_AMZ_OBJECT_ATTRIBUTES) List objectAttributes, + @RequestParam(value = VERSION_ID, required = false) String versionId) { bucketService.verifyBucketExists(bucketName); //this is for either an object request, or a parts request. diff --git a/server/src/main/java/com/adobe/testing/s3mock/dto/CopySource.java b/server/src/main/java/com/adobe/testing/s3mock/dto/CopySource.java index ba39a5d49..2e2cca4b2 100644 --- a/server/src/main/java/com/adobe/testing/s3mock/dto/CopySource.java +++ b/server/src/main/java/com/adobe/testing/s3mock/dto/CopySource.java @@ -1,5 +1,5 @@ /* - * Copyright 2017-2022 Adobe. + * Copyright 2017-2024 Adobe. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -25,13 +25,14 @@ */ public record CopySource( String bucket, - String key + String key, + String versionId ) { - public static final String DELIMITER = "/"; + static final String DELIMITER = "/"; /** - * Creates a {@link CopySource} expecting the given String represents the source as {@code - * /{bucket}/{key}}. + * Creates a {@link CopySource} expecting the given String to represent the source as {@code + * /{bucket}/{key}[?versionId={versionId}]}. * * @param copySource The object references. * @@ -41,8 +42,14 @@ public record CopySource( public CopySource(String copySource) { //inefficient duplicate parsing of incoming String, call to default constructor must be the //first statement... - this(extractBucketAndKeyArray(SdkHttpUtils.urlDecode(copySource))[0], - extractBucketAndKeyArray(SdkHttpUtils.urlDecode(copySource))[1]); + this(extractBucketAndKeyArray( + SdkHttpUtils.urlDecode(copySource) + )[0], + extractBucketAndKeyArray( + SdkHttpUtils.urlDecode(copySource) + )[1], + null //TODO: support versionId + ); } /** diff --git a/server/src/main/java/com/adobe/testing/s3mock/dto/VersioningConfiguration.java b/server/src/main/java/com/adobe/testing/s3mock/dto/VersioningConfiguration.java new file mode 100644 index 000000000..0edd64a7e --- /dev/null +++ b/server/src/main/java/com/adobe/testing/s3mock/dto/VersioningConfiguration.java @@ -0,0 +1,81 @@ +/* + * Copyright 2017-2024 Adobe. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.adobe.testing.s3mock.dto; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonRootName; +import com.fasterxml.jackson.annotation.JsonValue; +import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlProperty; + +/** + * API Reference. + */ +@JsonRootName("VersioningConfiguration") +public record VersioningConfiguration( + @JsonProperty("MfaDelete") + MFADelete mfaDelete, + @JsonProperty("Status") + Status status, + //workaround for adding xmlns attribute to root element only. + @JacksonXmlProperty(isAttribute = true, localName = "xmlns") + String xmlns +) { + + public VersioningConfiguration { + if (xmlns == null) { + xmlns = "http://s3.amazonaws.com/doc/2006-03-01/"; + } + } + + enum MFADelete { + ENABLED("Enabled"), + DISABLED("Disabled"); + + private final String value; + + @JsonCreator + MFADelete(String value) { + this.value = value; + } + + @Override + @JsonValue + public String toString() { + return value; + } + } + + enum Status { + ENABLED("Enabled"), + SUSPENDED("Suspended"); + + private final String value; + + @JsonCreator + Status(String value) { + this.value = value; + } + + @Override + @JsonValue + public String toString() { + return value; + } + } + +} diff --git a/server/src/main/java/com/adobe/testing/s3mock/util/AwsHttpHeaders.java b/server/src/main/java/com/adobe/testing/s3mock/util/AwsHttpHeaders.java index 744336e50..1b75b05b2 100644 --- a/server/src/main/java/com/adobe/testing/s3mock/util/AwsHttpHeaders.java +++ b/server/src/main/java/com/adobe/testing/s3mock/util/AwsHttpHeaders.java @@ -50,6 +50,7 @@ public final class AwsHttpHeaders { public static final String X_AMZ_TAGGING = "x-amz-tagging"; public static final String CONTENT_MD5 = "Content-MD5"; + public static final String X_AMZ_VERSION_ID = "x-amz-version-id"; public static final String X_AMZ_DELETE_MARKER = "x-amz-delete-marker"; public static final String X_AMZ_BUCKET_OBJECT_LOCK_ENABLED = "x-amz-bucket-object-lock-enabled"; diff --git a/server/src/main/java/com/adobe/testing/s3mock/util/AwsHttpParameters.java b/server/src/main/java/com/adobe/testing/s3mock/util/AwsHttpParameters.java index b7a82908d..609a908c7 100644 --- a/server/src/main/java/com/adobe/testing/s3mock/util/AwsHttpParameters.java +++ b/server/src/main/java/com/adobe/testing/s3mock/util/AwsHttpParameters.java @@ -1,5 +1,5 @@ /* - * Copyright 2017-2023 Adobe. + * Copyright 2017-2024 Adobe. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -58,6 +58,8 @@ public class AwsHttpParameters { public static final String LOCATION = "location"; public static final String NOT_LOCATION = NOT + LOCATION; + public static final String VERSION_ID = "versionId"; + private AwsHttpParameters() { // private constructor for utility classes } diff --git a/server/src/test/kotlin/com/adobe/testing/s3mock/dto/CopySourceTest.kt b/server/src/test/kotlin/com/adobe/testing/s3mock/dto/CopySourceTest.kt index 1a0ca9908..630a38384 100644 --- a/server/src/test/kotlin/com/adobe/testing/s3mock/dto/CopySourceTest.kt +++ b/server/src/test/kotlin/com/adobe/testing/s3mock/dto/CopySourceTest.kt @@ -17,6 +17,7 @@ package com.adobe.testing.s3mock.dto import org.assertj.core.api.Assertions.assertThat import org.assertj.core.api.Assertions.assertThatThrownBy +import org.junit.jupiter.api.Disabled import org.junit.jupiter.api.Test import java.util.UUID @@ -26,7 +27,7 @@ import java.util.UUID internal class CopySourceTest { @Test fun fromPrefixedCopySourceString() { - val copySource = CopySource(CopySource.DELIMITER + VALID_COPY_SOURCE) + val copySource = CopySource("/$VALID_COPY_SOURCE") assertThat(copySource.bucket).isEqualTo(BUCKET) assertThat(copySource.key).isEqualTo(KEY) @@ -56,9 +57,19 @@ internal class CopySourceTest { .isInstanceOf(NullPointerException::class.java) } + @Test + @Disabled + fun fromCopySourceWithVersion() { + val copySource = CopySource(COPY_SOURCE_WITH_VERSION) + + assertThat(copySource.bucket).isEqualTo(BUCKET) + assertThat(copySource.key).isEqualTo(KEY) + } + companion object { private val BUCKET = UUID.randomUUID().toString() private val KEY = UUID.randomUUID().toString() - private val VALID_COPY_SOURCE = BUCKET + CopySource.DELIMITER + KEY + private val VALID_COPY_SOURCE = "$BUCKET/$KEY" + private val COPY_SOURCE_WITH_VERSION = "$VALID_COPY_SOURCE?versionId=123" } } diff --git a/server/src/test/kotlin/com/adobe/testing/s3mock/dto/VersioningConfigurationTest.kt b/server/src/test/kotlin/com/adobe/testing/s3mock/dto/VersioningConfigurationTest.kt new file mode 100644 index 000000000..e9caeeb2d --- /dev/null +++ b/server/src/test/kotlin/com/adobe/testing/s3mock/dto/VersioningConfigurationTest.kt @@ -0,0 +1,44 @@ +/* + * Copyright 2017-2024 Adobe. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.adobe.testing.s3mock.dto + +import com.adobe.testing.s3mock.dto.DtoTestUtil.deserialize +import com.adobe.testing.s3mock.dto.DtoTestUtil.serializeAndAssert +import com.adobe.testing.s3mock.dto.VersioningConfiguration.MFADelete +import com.adobe.testing.s3mock.dto.VersioningConfiguration.Status +import org.assertj.core.api.Assertions.assertThat +import org.junit.jupiter.api.Test +import org.junit.jupiter.api.TestInfo +import java.io.IOException + +internal class VersioningConfigurationTest { + + @Test + @Throws(IOException::class) + fun testSerialization(testInfo: TestInfo) { + val iut = VersioningConfiguration(null, Status.SUSPENDED, null) + serializeAndAssert(iut, testInfo) + } + + @Test + @Throws(IOException::class) + fun testDeserialization(testInfo: TestInfo) { + val iut = deserialize(VersioningConfiguration::class.java, testInfo) + assertThat(iut.status).isEqualTo(Status.ENABLED) + assertThat(iut.mfaDelete).isEqualTo(MFADelete.ENABLED) + } +} diff --git a/server/src/test/resources/com/adobe/testing/s3mock/dto/VersioningConfigurationTest_testDeserialization.xml b/server/src/test/resources/com/adobe/testing/s3mock/dto/VersioningConfigurationTest_testDeserialization.xml new file mode 100644 index 000000000..70aac4f01 --- /dev/null +++ b/server/src/test/resources/com/adobe/testing/s3mock/dto/VersioningConfigurationTest_testDeserialization.xml @@ -0,0 +1,22 @@ + + + + Enabled + Enabled + diff --git a/server/src/test/resources/com/adobe/testing/s3mock/dto/VersioningConfigurationTest_testSerialization.xml b/server/src/test/resources/com/adobe/testing/s3mock/dto/VersioningConfigurationTest_testSerialization.xml new file mode 100644 index 000000000..f0e725408 --- /dev/null +++ b/server/src/test/resources/com/adobe/testing/s3mock/dto/VersioningConfigurationTest_testSerialization.xml @@ -0,0 +1,21 @@ + + + + Suspended + From 00739c58f0eb99bd737ec82b3b6ebc77f555e0c2 Mon Sep 17 00:00:00 2001 From: Arne Franken Date: Tue, 28 May 2024 23:04:06 +0200 Subject: [PATCH 2/8] Store ACLs with ObjectMetadata This is a step to storing versions, as ACLs are stored by version in S3. --- .../testing/s3mock/store/ObjectStore.java | 83 +++++++++---------- .../s3mock/store/S3ObjectMetadata.java | 4 +- .../testing/s3mock/ObjectControllerTest.kt | 1 + .../s3mock/TaggingHeaderConverterTest.kt | 2 +- .../testing/s3mock/service/ServiceTestBase.kt | 3 +- .../testing/s3mock/util/HeaderUtilTest.kt | 3 +- 6 files changed, 48 insertions(+), 48 deletions(-) diff --git a/server/src/main/java/com/adobe/testing/s3mock/store/ObjectStore.java b/server/src/main/java/com/adobe/testing/s3mock/store/ObjectStore.java index d2fe0db22..66f708be8 100644 --- a/server/src/main/java/com/adobe/testing/s3mock/store/ObjectStore.java +++ b/server/src/main/java/com/adobe/testing/s3mock/store/ObjectStore.java @@ -32,7 +32,6 @@ import com.adobe.testing.s3mock.dto.Tag; import com.fasterxml.jackson.databind.ObjectMapper; import java.io.IOException; -import java.nio.charset.Charset; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; @@ -54,7 +53,6 @@ public class ObjectStore extends StoreBase { private static final Logger LOG = LoggerFactory.getLogger(ObjectStore.class); private static final String META_FILE = "objectMetadata.json"; - private static final String ACL_FILE = "objectAcl.json"; private static final String DATA_FILE = "binaryData"; /** @@ -129,7 +127,8 @@ public S3ObjectMetadata storeS3ObjectMetadata(BucketMetadata bucket, encryptionHeaders, checksumAlgorithm, checksum, - storageClass + storageClass, + null ); writeMetafile(bucket, s3ObjectMetadata); return s3ObjectMetadata; @@ -170,7 +169,8 @@ public void storeObjectTags(BucketMetadata bucket, UUID id, List tags) { s3ObjectMetadata.encryptionHeaders(), s3ObjectMetadata.checksumAlgorithm(), s3ObjectMetadata.checksum(), - s3ObjectMetadata.storageClass() + s3ObjectMetadata.storageClass(), + s3ObjectMetadata.policy() )); } } @@ -203,7 +203,8 @@ public void storeLegalHold(BucketMetadata bucket, UUID id, LegalHold legalHold) s3ObjectMetadata.encryptionHeaders(), s3ObjectMetadata.checksumAlgorithm(), s3ObjectMetadata.checksum(), - s3ObjectMetadata.storageClass() + s3ObjectMetadata.storageClass(), + s3ObjectMetadata.policy() )); } } @@ -216,16 +217,38 @@ public void storeLegalHold(BucketMetadata bucket, UUID id, LegalHold legalHold) * @param policy the ACL. */ public void storeAcl(BucketMetadata bucket, UUID id, AccessControlPolicy policy) { - writeAclFile(bucket, id, policy); + synchronized (lockStore.get(id)) { + var s3ObjectMetadata = getS3ObjectMetadata(bucket, id); + writeMetafile(bucket, new S3ObjectMetadata( + s3ObjectMetadata.id(), + s3ObjectMetadata.key(), + s3ObjectMetadata.size(), + s3ObjectMetadata.modificationDate(), + s3ObjectMetadata.etag(), + s3ObjectMetadata.contentType(), + s3ObjectMetadata.lastModified(), + s3ObjectMetadata.dataPath(), + s3ObjectMetadata.userMetadata(), + s3ObjectMetadata.tags(), + s3ObjectMetadata.legalHold(), + s3ObjectMetadata.retention(), + s3ObjectMetadata.owner(), + s3ObjectMetadata.storeHeaders(), + s3ObjectMetadata.encryptionHeaders(), + s3ObjectMetadata.checksumAlgorithm(), + s3ObjectMetadata.checksum(), + s3ObjectMetadata.storageClass(), + policy + ) + ); + } } public AccessControlPolicy readAcl(BucketMetadata bucket, UUID id) { - var policy = readAclFile(bucket, id); - if (policy == null) { - var s3ObjectMetadata = getS3ObjectMetadata(bucket, id); - return privateCannedAcl(s3ObjectMetadata.owner()); - } - return policy; + var s3ObjectMetadata = getS3ObjectMetadata(bucket, id); + return s3ObjectMetadata.policy() == null + ? privateCannedAcl(s3ObjectMetadata.owner()) + : s3ObjectMetadata.policy(); } /** @@ -256,7 +279,8 @@ public void storeRetention(BucketMetadata bucket, UUID id, Retention retention) s3ObjectMetadata.encryptionHeaders(), s3ObjectMetadata.checksumAlgorithm(), s3ObjectMetadata.checksum(), - s3ObjectMetadata.storageClass() + s3ObjectMetadata.storageClass(), + s3ObjectMetadata.policy() )); } } @@ -371,7 +395,8 @@ public CopyObjectResult pretendToCopyS3Object(BucketMetadata sourceBucket, ? sourceObject.encryptionHeaders() : encryptionHeaders, sourceObject.checksumAlgorithm(), sourceObject.checksum(), - storageClass != null ? storageClass : sourceObject.storageClass() + storageClass != null ? storageClass : sourceObject.storageClass(), + sourceObject.policy() )); return new CopyObjectResult(sourceObject.modificationDate(), sourceObject.etag()); } @@ -453,10 +478,6 @@ private Path getMetaFilePath(BucketMetadata bucket, UUID id) { return Paths.get(getObjectFolderPath(bucket, id).toString(), META_FILE); } - private Path getAclFilePath(BucketMetadata bucket, UUID id) { - return Paths.get(getObjectFolderPath(bucket, id).toString(), ACL_FILE); - } - private Path getDataFilePath(BucketMetadata bucket, UUID id) { return Paths.get(getObjectFolderPath(bucket, id).toString(), DATA_FILE); } @@ -472,30 +493,4 @@ private void writeMetafile(BucketMetadata bucket, S3ObjectMetadata s3ObjectMetad throw new IllegalStateException("Could not write object metadata-file " + id, e); } } - - private AccessControlPolicy readAclFile(BucketMetadata bucket, UUID id) { - try { - synchronized (lockStore.get(id)) { - var aclFile = getAclFilePath(bucket, id).toFile(); - if (!aclFile.exists()) { - return null; - } - var toDeserialize = FileUtils.readFileToString(aclFile, Charset.defaultCharset()); - return objectMapper.readValue(toDeserialize, AccessControlPolicy.class); - } - } catch (IOException e) { - throw new IllegalStateException("Could not read object acl-file " + id, e); - } - } - - private void writeAclFile(BucketMetadata bucket, UUID id, AccessControlPolicy policy) { - try { - synchronized (lockStore.get(id)) { - var aclFile = getAclFilePath(bucket, id).toFile(); - FileUtils.write(aclFile, objectMapper.writeValueAsString(policy), Charset.defaultCharset()); - } - } catch (IOException e) { - throw new IllegalStateException("Could not write object acl-file " + id, e); - } - } } diff --git a/server/src/main/java/com/adobe/testing/s3mock/store/S3ObjectMetadata.java b/server/src/main/java/com/adobe/testing/s3mock/store/S3ObjectMetadata.java index 354f8439e..c95f3a151 100644 --- a/server/src/main/java/com/adobe/testing/s3mock/store/S3ObjectMetadata.java +++ b/server/src/main/java/com/adobe/testing/s3mock/store/S3ObjectMetadata.java @@ -18,6 +18,7 @@ import static com.adobe.testing.s3mock.util.EtagUtil.normalizeEtag; +import com.adobe.testing.s3mock.dto.AccessControlPolicy; import com.adobe.testing.s3mock.dto.ChecksumAlgorithm; import com.adobe.testing.s3mock.dto.LegalHold; import com.adobe.testing.s3mock.dto.Owner; @@ -55,7 +56,8 @@ public record S3ObjectMetadata( Map encryptionHeaders, ChecksumAlgorithm checksumAlgorithm, String checksum, - StorageClass storageClass + StorageClass storageClass, + AccessControlPolicy policy ) { public S3ObjectMetadata { diff --git a/server/src/test/kotlin/com/adobe/testing/s3mock/ObjectControllerTest.kt b/server/src/test/kotlin/com/adobe/testing/s3mock/ObjectControllerTest.kt index 1444b53cd..d74bdbada 100644 --- a/server/src/test/kotlin/com/adobe/testing/s3mock/ObjectControllerTest.kt +++ b/server/src/test/kotlin/com/adobe/testing/s3mock/ObjectControllerTest.kt @@ -635,6 +635,7 @@ internal class ObjectControllerTest : BaseControllerTest() { encryptionHeaders(encryption, encryptionKey), null, null, + null, null ) } diff --git a/server/src/test/kotlin/com/adobe/testing/s3mock/TaggingHeaderConverterTest.kt b/server/src/test/kotlin/com/adobe/testing/s3mock/TaggingHeaderConverterTest.kt index 4089bb087..23ed26017 100644 --- a/server/src/test/kotlin/com/adobe/testing/s3mock/TaggingHeaderConverterTest.kt +++ b/server/src/test/kotlin/com/adobe/testing/s3mock/TaggingHeaderConverterTest.kt @@ -57,6 +57,6 @@ internal class TaggingHeaderConverterTest { } private fun tag(i: Int): String { - return String.format("tag%d=value%d", i, i) + return "tag$i=value$i" } } diff --git a/server/src/test/kotlin/com/adobe/testing/s3mock/service/ServiceTestBase.kt b/server/src/test/kotlin/com/adobe/testing/s3mock/service/ServiceTestBase.kt index 35104d391..422d3b4df 100644 --- a/server/src/test/kotlin/com/adobe/testing/s3mock/service/ServiceTestBase.kt +++ b/server/src/test/kotlin/com/adobe/testing/s3mock/service/ServiceTestBase.kt @@ -119,7 +119,8 @@ internal abstract class ServiceTestBase { null, null, null, - StorageClass.STANDARD + StorageClass.STANDARD, + null ) } diff --git a/server/src/test/kotlin/com/adobe/testing/s3mock/util/HeaderUtilTest.kt b/server/src/test/kotlin/com/adobe/testing/s3mock/util/HeaderUtilTest.kt index 38284b403..296148085 100644 --- a/server/src/test/kotlin/com/adobe/testing/s3mock/util/HeaderUtilTest.kt +++ b/server/src/test/kotlin/com/adobe/testing/s3mock/util/HeaderUtilTest.kt @@ -77,7 +77,8 @@ internal class HeaderUtilTest { null, null, null, - StorageClass.STANDARD + StorageClass.STANDARD, + null ) } From 580093aac7108a904c8aa5e20fe4f2ff7c041b70 Mon Sep 17 00:00:00 2001 From: Arne Franken Date: Wed, 29 May 2024 23:18:13 +0200 Subject: [PATCH 3/8] Simpler FaviconController --- .../java/com/adobe/testing/s3mock/FaviconController.java | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/server/src/main/java/com/adobe/testing/s3mock/FaviconController.java b/server/src/main/java/com/adobe/testing/s3mock/FaviconController.java index 0c4c1e459..afe6ab8a0 100644 --- a/server/src/main/java/com/adobe/testing/s3mock/FaviconController.java +++ b/server/src/main/java/com/adobe/testing/s3mock/FaviconController.java @@ -1,5 +1,5 @@ /* - * Copyright 2017-2023 Adobe. + * Copyright 2017-2024 Adobe. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -16,20 +16,18 @@ package com.adobe.testing.s3mock; -import org.springframework.stereotype.Controller; import org.springframework.web.bind.annotation.GetMapping; import org.springframework.web.bind.annotation.RequestMapping; -import org.springframework.web.bind.annotation.ResponseBody; +import org.springframework.web.bind.annotation.RestController; /** * Spring Boot 2.2+ does not include the default favicon.ico anymore. * This is needed to check if the S3 Mock is up (at least in our examples and some use-cases) */ -@Controller +@RestController @RequestMapping class FaviconController { @GetMapping("favicon.ico") - @ResponseBody void favicon() { // Method is intentionally empty. } From b5c3a19983d31f6f784d71e91f53205c06f87d7b Mon Sep 17 00:00:00 2001 From: Arne Franken Date: Wed, 29 May 2024 23:58:01 +0200 Subject: [PATCH 4/8] Version-compatible CopySource --- .../adobe/testing/s3mock/dto/CopySource.java | 22 +++++++++---------- .../testing/s3mock/dto/CopySourceTest.kt | 14 ++++++------ 2 files changed, 18 insertions(+), 18 deletions(-) diff --git a/server/src/main/java/com/adobe/testing/s3mock/dto/CopySource.java b/server/src/main/java/com/adobe/testing/s3mock/dto/CopySource.java index 2e2cca4b2..31623fe01 100644 --- a/server/src/main/java/com/adobe/testing/s3mock/dto/CopySource.java +++ b/server/src/main/java/com/adobe/testing/s3mock/dto/CopySource.java @@ -39,17 +39,17 @@ public record CopySource( * @throws IllegalArgumentException If {@code copySource} could not be parsed. * @throws NullPointerException If {@code copySource} is null. */ - public CopySource(String copySource) { - //inefficient duplicate parsing of incoming String, call to default constructor must be the - //first statement... - this(extractBucketAndKeyArray( - SdkHttpUtils.urlDecode(copySource) - )[0], - extractBucketAndKeyArray( - SdkHttpUtils.urlDecode(copySource) - )[1], - null //TODO: support versionId - ); + public static CopySource from(String copySource) { + var bucketAndKey = extractBucketAndKeyArray(SdkHttpUtils.urlDecode(copySource)); + var bucket = requireNonNull(bucketAndKey[0]); + var key = requireNonNull(bucketAndKey[1]); + String versionId = null; + if (key.contains("?versionId=")) { + String[] keyAndVersionId = key.split("\\?versionId="); + key = keyAndVersionId[0]; + versionId = keyAndVersionId[1]; + } + return new CopySource(bucket, key, versionId); } /** diff --git a/server/src/test/kotlin/com/adobe/testing/s3mock/dto/CopySourceTest.kt b/server/src/test/kotlin/com/adobe/testing/s3mock/dto/CopySourceTest.kt index 630a38384..e853ae5b6 100644 --- a/server/src/test/kotlin/com/adobe/testing/s3mock/dto/CopySourceTest.kt +++ b/server/src/test/kotlin/com/adobe/testing/s3mock/dto/CopySourceTest.kt @@ -27,7 +27,7 @@ import java.util.UUID internal class CopySourceTest { @Test fun fromPrefixedCopySourceString() { - val copySource = CopySource("/$VALID_COPY_SOURCE") + val copySource = CopySource.from("/$VALID_COPY_SOURCE") assertThat(copySource.bucket).isEqualTo(BUCKET) assertThat(copySource.key).isEqualTo(KEY) @@ -35,7 +35,7 @@ internal class CopySourceTest { @Test fun fromCopySourceString() { - val copySource = CopySource(VALID_COPY_SOURCE) + val copySource = CopySource.from(VALID_COPY_SOURCE) assertThat(copySource.bucket).isEqualTo(BUCKET) assertThat(copySource.key).isEqualTo(KEY) @@ -44,7 +44,7 @@ internal class CopySourceTest { @Test fun invalidCopySource() { assertThatThrownBy { - CopySource(UUID.randomUUID().toString()) + CopySource.from(UUID.randomUUID().toString()) } .isInstanceOf(IllegalArgumentException::class.java) } @@ -52,15 +52,14 @@ internal class CopySourceTest { @Test fun nullCopySource() { assertThatThrownBy { - CopySource(null) + CopySource.from(null) } .isInstanceOf(NullPointerException::class.java) } @Test - @Disabled fun fromCopySourceWithVersion() { - val copySource = CopySource(COPY_SOURCE_WITH_VERSION) + val copySource = CopySource.from(COPY_SOURCE_WITH_VERSION) assertThat(copySource.bucket).isEqualTo(BUCKET) assertThat(copySource.key).isEqualTo(KEY) @@ -69,7 +68,8 @@ internal class CopySourceTest { companion object { private val BUCKET = UUID.randomUUID().toString() private val KEY = UUID.randomUUID().toString() + private const val VERSION = "123" private val VALID_COPY_SOURCE = "$BUCKET/$KEY" - private val COPY_SOURCE_WITH_VERSION = "$VALID_COPY_SOURCE?versionId=123" + private val COPY_SOURCE_WITH_VERSION = "$VALID_COPY_SOURCE?versionId=$VERSION" } } From f366d20078e69fe0b46c71a707f3960e631416c8 Mon Sep 17 00:00:00 2001 From: Arne Franken Date: Thu, 30 May 2024 18:40:59 +0200 Subject: [PATCH 5/8] Support Bucket Versioning Config --- README.md | 4 +- .../adobe/testing/s3mock/its/BucketV2IT.kt | 52 +++++++++++++++ .../testing/s3mock/BucketController.java | 65 ++++++++++++++++++- .../testing/s3mock/service/BucketService.java | 16 +++++ .../testing/s3mock/store/BucketMetadata.java | 25 ++++++- .../testing/s3mock/store/BucketStore.java | 9 +++ .../s3mock/util/AwsHttpParameters.java | 19 +++--- .../s3mock/service/MultipartServiceTest.kt | 2 +- .../testing/s3mock/service/ServiceTestBase.kt | 1 + .../s3mock/store/StoreConfigurationTest.kt | 2 +- .../testing/s3mock/store/StoreTestBase.kt | 1 + 11 files changed, 179 insertions(+), 17 deletions(-) diff --git a/README.md b/README.md index c95d0dc52..732533b55 100755 --- a/README.md +++ b/README.md @@ -102,7 +102,7 @@ Of these [operations of the Amazon S3 API](https://docs.aws.amazon.com/AmazonS3/ | [GetBucketReplication](https://docs.aws.amazon.com/AmazonS3/latest/API/API_GetBucketReplication.html) | :x: | | | [GetBucketRequestPayment](https://docs.aws.amazon.com/AmazonS3/latest/API/API_GetBucketRequestPayment.html) | :x: | | | [GetBucketTagging](https://docs.aws.amazon.com/AmazonS3/latest/API/API_GetBucketTagging.html) | :x: | | -| [GetBucketVersioning](https://docs.aws.amazon.com/AmazonS3/latest/API/API_GetBucketVersioning.html) | :x: | | +| [GetBucketVersioning](https://docs.aws.amazon.com/AmazonS3/latest/API/API_GetBucketVersioning.html) | :white_check_mark: | | | [GetBucketWebsite](https://docs.aws.amazon.com/AmazonS3/latest/API/API_GetBucketWebsite.html) | :x: | | | [GetObject](https://docs.aws.amazon.com/AmazonS3/latest/API/API_GetObject.html) | :white_check_mark: | | | [GetObjectAcl](https://docs.aws.amazon.com/AmazonS3/latest/API/API_GetObjectAcl.html) | :white_check_mark: | | @@ -143,7 +143,7 @@ Of these [operations of the Amazon S3 API](https://docs.aws.amazon.com/AmazonS3/ | [PutBucketReplication](https://docs.aws.amazon.com/AmazonS3/latest/API/API_PutBucketReplication.html) | :x: | | | [PutBucketRequestPayment](https://docs.aws.amazon.com/AmazonS3/latest/API/API_PutBucketRequestPayment.html) | :x: | | | [PutBucketTagging](https://docs.aws.amazon.com/AmazonS3/latest/API/API_PutBucketTagging.html) | :x: | | -| [PutBucketVersioning](https://docs.aws.amazon.com/AmazonS3/latest/API/API_PutBucketVersioning.html) | :x: | | +| [PutBucketVersioning](https://docs.aws.amazon.com/AmazonS3/latest/API/API_PutBucketVersioning.html) | :white_check_mark: | | | [PutBucketWebsite](https://docs.aws.amazon.com/AmazonS3/latest/API/API_PutBucketWebsite.html) | :x: | | | [PutObject](https://docs.aws.amazon.com/AmazonS3/latest/API/API_PutObject.html) | :white_check_mark: | | | [PutObjectAcl](https://docs.aws.amazon.com/AmazonS3/latest/API/API_PutObjectAcl.html) | :white_check_mark: | | diff --git a/integration-tests/src/test/kotlin/com/adobe/testing/s3mock/its/BucketV2IT.kt b/integration-tests/src/test/kotlin/com/adobe/testing/s3mock/its/BucketV2IT.kt index 446731a09..0fed4e0dd 100644 --- a/integration-tests/src/test/kotlin/com/adobe/testing/s3mock/its/BucketV2IT.kt +++ b/integration-tests/src/test/kotlin/com/adobe/testing/s3mock/its/BucketV2IT.kt @@ -26,18 +26,24 @@ import software.amazon.awssdk.awscore.exception.AwsServiceException import software.amazon.awssdk.services.s3.S3Client import software.amazon.awssdk.services.s3.model.AbortIncompleteMultipartUpload import software.amazon.awssdk.services.s3.model.BucketLifecycleConfiguration +import software.amazon.awssdk.services.s3.model.BucketVersioningStatus import software.amazon.awssdk.services.s3.model.CreateBucketRequest import software.amazon.awssdk.services.s3.model.DeleteBucketLifecycleRequest import software.amazon.awssdk.services.s3.model.DeleteBucketRequest import software.amazon.awssdk.services.s3.model.ExpirationStatus import software.amazon.awssdk.services.s3.model.GetBucketLifecycleConfigurationRequest import software.amazon.awssdk.services.s3.model.GetBucketLocationRequest +import software.amazon.awssdk.services.s3.model.GetBucketVersioningRequest import software.amazon.awssdk.services.s3.model.HeadBucketRequest import software.amazon.awssdk.services.s3.model.LifecycleExpiration import software.amazon.awssdk.services.s3.model.LifecycleRule import software.amazon.awssdk.services.s3.model.LifecycleRuleFilter +import software.amazon.awssdk.services.s3.model.MFADelete +import software.amazon.awssdk.services.s3.model.MFADeleteStatus import software.amazon.awssdk.services.s3.model.NoSuchBucketException import software.amazon.awssdk.services.s3.model.PutBucketLifecycleConfigurationRequest +import software.amazon.awssdk.services.s3.model.PutBucketVersioningRequest +import software.amazon.awssdk.services.s3.model.VersioningConfiguration import java.util.concurrent.TimeUnit /** @@ -79,6 +85,52 @@ internal class BucketV2IT : S3TestBase() { assertThat(bucketLocation.locationConstraint().toString()).isEqualTo("eu-west-1") } + @Test + @S3VerifiedSuccess(year = 2024) + fun getDefaultBucketVersioning(testInfo: TestInfo) { + val bucketName = givenBucketV2(testInfo) + + s3ClientV2.getBucketVersioning( + GetBucketVersioningRequest + .builder() + .bucket(bucketName) + .build() + ).also { + assertThat(it.status()).isNull() + assertThat(it.mfaDelete()).isNull() + } + } + + @Test + @S3VerifiedFailure(year = 2024, reason = "No real Mfa value") + fun putAndGetBucketVersioning(testInfo: TestInfo) { + val bucketName = givenBucketV2(testInfo) + s3ClientV2.putBucketVersioning( + PutBucketVersioningRequest + .builder() + .bucket(bucketName) + .mfa("fakeMfaValue") + .versioningConfiguration( + VersioningConfiguration + .builder() + .status(BucketVersioningStatus.ENABLED) + .mfaDelete(MFADelete.ENABLED) + .build() + ) + .build() + ) + + s3ClientV2.getBucketVersioning( + GetBucketVersioningRequest + .builder() + .bucket(bucketName) + .build() + ).also { + assertThat(it.status()).isEqualTo(BucketVersioningStatus.ENABLED) + assertThat(it.mfaDelete()).isEqualTo(MFADeleteStatus.ENABLED) + } + } + @Test @S3VerifiedSuccess(year = 2024) fun duplicateBucketCreation(testInfo: TestInfo) { diff --git a/server/src/main/java/com/adobe/testing/s3mock/BucketController.java b/server/src/main/java/com/adobe/testing/s3mock/BucketController.java index dd49b735d..506e5d36f 100644 --- a/server/src/main/java/com/adobe/testing/s3mock/BucketController.java +++ b/server/src/main/java/com/adobe/testing/s3mock/BucketController.java @@ -30,9 +30,11 @@ import static com.adobe.testing.s3mock.util.AwsHttpParameters.NOT_LOCATION; import static com.adobe.testing.s3mock.util.AwsHttpParameters.NOT_OBJECT_LOCK; import static com.adobe.testing.s3mock.util.AwsHttpParameters.NOT_UPLOADS; +import static com.adobe.testing.s3mock.util.AwsHttpParameters.NOT_VERSIONING; import static com.adobe.testing.s3mock.util.AwsHttpParameters.NOT_VERSIONS; import static com.adobe.testing.s3mock.util.AwsHttpParameters.OBJECT_LOCK; import static com.adobe.testing.s3mock.util.AwsHttpParameters.START_AFTER; +import static com.adobe.testing.s3mock.util.AwsHttpParameters.VERSIONING; import static com.adobe.testing.s3mock.util.AwsHttpParameters.VERSIONS; import static com.adobe.testing.s3mock.util.AwsHttpParameters.VERSION_ID_MARKER; import static org.springframework.http.MediaType.APPLICATION_XML_VALUE; @@ -44,6 +46,7 @@ import com.adobe.testing.s3mock.dto.ListVersionsResult; import com.adobe.testing.s3mock.dto.LocationConstraint; import com.adobe.testing.s3mock.dto.ObjectLockConfiguration; +import com.adobe.testing.s3mock.dto.VersioningConfiguration; import com.adobe.testing.s3mock.service.BucketService; import org.springframework.http.ResponseEntity; import org.springframework.stereotype.Controller; @@ -116,7 +119,8 @@ public ResponseEntity listBuckets() { }, params = { NOT_OBJECT_LOCK, - NOT_LIFECYCLE + NOT_LIFECYCLE, + NOT_VERSIONING } ) public ResponseEntity createBucket(@PathVariable final String bucketName, @@ -180,6 +184,62 @@ public ResponseEntity deleteBucket(@PathVariable String bucketName) { return ResponseEntity.noContent().build(); } + /** + * Get VersioningConfiguration of a bucket. + * API Reference + * + * @param bucketName name of the Bucket. + * + * @return 200, VersioningConfiguration + */ + @GetMapping( + value = { + //AWS SDK V2 pattern + "/{bucketName:.+}", + //AWS SDK V1 pattern + "/{bucketName:.+}/" + }, + params = { + VERSIONING, + NOT_LIST_TYPE + }, + produces = APPLICATION_XML_VALUE + ) + public ResponseEntity getVersioningConfiguration( + @PathVariable String bucketName) { + bucketService.verifyBucketExists(bucketName); + var configuration = bucketService.getVersioningConfiguration(bucketName); + return ResponseEntity.ok(configuration); + } + + /** + * Put VersioningConfiguration of a bucket. + * API Reference + * + * @param bucketName name of the Bucket. + * + * @return 200 + */ + @PutMapping( + value = { + //AWS SDK V2 pattern + "/{bucketName:.+}", + //AWS SDK V1 pattern + "/{bucketName:.+}/" + }, + params = { + VERSIONING + }, + consumes = APPLICATION_XML_VALUE + ) + public ResponseEntity putVersioningConfiguration( + @PathVariable String bucketName, + @RequestBody VersioningConfiguration configuration) { + bucketService.verifyBucketExists(bucketName); + bucketService.setVersioningConfiguration(bucketName, configuration); + return ResponseEntity.ok().build(); + } + /** * Get ObjectLockConfiguration of a bucket. * API Reference @@ -360,7 +420,8 @@ public ResponseEntity getBucketLocation( NOT_LIST_TYPE, NOT_LIFECYCLE, NOT_LOCATION, - NOT_VERSIONS + NOT_VERSIONS, + NOT_VERSIONING }, produces = APPLICATION_XML_VALUE ) diff --git a/server/src/main/java/com/adobe/testing/s3mock/service/BucketService.java b/server/src/main/java/com/adobe/testing/s3mock/service/BucketService.java index d8169ea88..00711b041 100644 --- a/server/src/main/java/com/adobe/testing/s3mock/service/BucketService.java +++ b/server/src/main/java/com/adobe/testing/s3mock/service/BucketService.java @@ -40,6 +40,7 @@ import com.adobe.testing.s3mock.dto.ObjectVersion; import com.adobe.testing.s3mock.dto.Prefix; import com.adobe.testing.s3mock.dto.S3Object; +import com.adobe.testing.s3mock.dto.VersioningConfiguration; import com.adobe.testing.s3mock.store.BucketStore; import com.adobe.testing.s3mock.store.ObjectStore; import java.util.ArrayList; @@ -113,6 +114,21 @@ public boolean deleteBucket(String bucketName) { return bucketStore.deleteBucket(bucketName); } + public void setVersioningConfiguration(String bucketName, VersioningConfiguration configuration) { + var bucketMetadata = bucketStore.getBucketMetadata(bucketName); + bucketStore.storeVersioningConfiguration(bucketMetadata, configuration); + } + + public VersioningConfiguration getVersioningConfiguration(String bucketName) { + var bucketMetadata = bucketStore.getBucketMetadata(bucketName); + var configuration = bucketMetadata.versioningConfiguration(); + if (configuration != null) { + return configuration; + } else { + throw NOT_FOUND_BUCKET_OBJECT_LOCK; + } + } + public void setObjectLockConfiguration(String bucketName, ObjectLockConfiguration configuration) { var bucketMetadata = bucketStore.getBucketMetadata(bucketName); bucketStore.storeObjectLockConfiguration(bucketMetadata, configuration); diff --git a/server/src/main/java/com/adobe/testing/s3mock/store/BucketMetadata.java b/server/src/main/java/com/adobe/testing/s3mock/store/BucketMetadata.java index b0b984bc6..e0c271d2a 100644 --- a/server/src/main/java/com/adobe/testing/s3mock/store/BucketMetadata.java +++ b/server/src/main/java/com/adobe/testing/s3mock/store/BucketMetadata.java @@ -18,6 +18,7 @@ import com.adobe.testing.s3mock.dto.BucketLifecycleConfiguration; import com.adobe.testing.s3mock.dto.ObjectLockConfiguration; +import com.adobe.testing.s3mock.dto.VersioningConfiguration; import java.nio.file.Path; import java.util.HashMap; import java.util.Map; @@ -30,6 +31,7 @@ public record BucketMetadata( String name, String creationDate, + VersioningConfiguration versioningConfiguration, ObjectLockConfiguration objectLockConfiguration, BucketLifecycleConfiguration bucketLifecycleConfiguration, ObjectOwnership objectOwnership, @@ -38,12 +40,14 @@ public record BucketMetadata( ) { public BucketMetadata(String name, String creationDate, + VersioningConfiguration versioningConfiguration, ObjectLockConfiguration objectLockConfiguration, BucketLifecycleConfiguration bucketLifecycleConfiguration, ObjectOwnership objectOwnership, Path path) { this(name, creationDate, + versioningConfiguration, objectLockConfiguration, bucketLifecycleConfiguration, objectOwnership, @@ -51,9 +55,23 @@ public BucketMetadata(String name, String creationDate, new HashMap<>()); } + public BucketMetadata withVersioningConfiguration( + VersioningConfiguration versioningConfiguration) { + return new BucketMetadata(name(), + creationDate(), + versioningConfiguration, + objectLockConfiguration(), + bucketLifecycleConfiguration(), + objectOwnership(), + path()); + } + public BucketMetadata withObjectLockConfiguration( ObjectLockConfiguration objectLockConfiguration) { - return new BucketMetadata(name(), creationDate(), objectLockConfiguration, + return new BucketMetadata(name(), + creationDate(), + versioningConfiguration(), + objectLockConfiguration, bucketLifecycleConfiguration(), objectOwnership(), path()); @@ -61,7 +79,10 @@ public BucketMetadata withObjectLockConfiguration( public BucketMetadata withBucketLifecycleConfiguration( BucketLifecycleConfiguration bucketLifecycleConfiguration) { - return new BucketMetadata(name(), creationDate(), objectLockConfiguration(), + return new BucketMetadata(name(), + creationDate(), + versioningConfiguration(), + objectLockConfiguration(), bucketLifecycleConfiguration, objectOwnership(), path()); diff --git a/server/src/main/java/com/adobe/testing/s3mock/store/BucketStore.java b/server/src/main/java/com/adobe/testing/s3mock/store/BucketStore.java index 5e2fc5ae8..3ec6348d9 100644 --- a/server/src/main/java/com/adobe/testing/s3mock/store/BucketStore.java +++ b/server/src/main/java/com/adobe/testing/s3mock/store/BucketStore.java @@ -19,6 +19,7 @@ import com.adobe.testing.s3mock.dto.BucketLifecycleConfiguration; import com.adobe.testing.s3mock.dto.ObjectLockConfiguration; import com.adobe.testing.s3mock.dto.ObjectLockEnabled; +import com.adobe.testing.s3mock.dto.VersioningConfiguration; import com.fasterxml.jackson.databind.ObjectMapper; import java.io.File; import java.io.IOException; @@ -190,6 +191,7 @@ public BucketMetadata createBucket(String bucketName, var newBucketMetadata = new BucketMetadata( bucketName, s3ObjectDateFormat.format(LocalDateTime.now()), + new VersioningConfiguration(null, null, null), objectLockEnabled ? new ObjectLockConfiguration(ObjectLockEnabled.ENABLED, null) : null, null, @@ -229,6 +231,13 @@ public void storeObjectLockConfiguration(BucketMetadata metadata, } } + public void storeVersioningConfiguration(BucketMetadata metadata, + VersioningConfiguration configuration) { + synchronized (lockStore.get(metadata.name())) { + writeToDisk(metadata.withVersioningConfiguration(configuration)); + } + } + public void storeBucketLifecycleConfiguration(BucketMetadata metadata, BucketLifecycleConfiguration configuration) { synchronized (lockStore.get(metadata.name())) { diff --git a/server/src/main/java/com/adobe/testing/s3mock/util/AwsHttpParameters.java b/server/src/main/java/com/adobe/testing/s3mock/util/AwsHttpParameters.java index 609a908c7..b21f482b0 100644 --- a/server/src/main/java/com/adobe/testing/s3mock/util/AwsHttpParameters.java +++ b/server/src/main/java/com/adobe/testing/s3mock/util/AwsHttpParameters.java @@ -21,30 +21,30 @@ */ public class AwsHttpParameters { - private static final String NOT = "!"; - - public static final String ACL = "acl"; - public static final String NOT_ACL = NOT + ACL; public static final String CONTINUATION_TOKEN = "continuation-token"; public static final String DELETE = "delete"; public static final String ENCODING_TYPE = "encoding-type"; public static final String KEY_MARKER = "key-marker"; public static final String VERSION_ID_MARKER = "version-id-marker"; public static final String LIST_TYPE_V2 = "list-type=2"; - public static final String VERSIONS = "versions"; - public static final String NOT_VERSIONS = "!versions"; public static final String NOT_LIST_TYPE = "!list-type"; public static final String MAX_KEYS = "max-keys"; public static final String PART_NUMBER = "partNumber"; public static final String START_AFTER = "start-after"; + public static final String VERSION_ID = "versionId"; + + private static final String NOT = "!"; + + public static final String ACL = "acl"; + public static final String NOT_ACL = NOT + ACL; + public static final String VERSIONS = "versions"; + public static final String NOT_VERSIONS = NOT + VERSIONS; public static final String TAGGING = "tagging"; public static final String NOT_TAGGING = NOT + TAGGING; public static final String UPLOADS = "uploads"; public static final String NOT_UPLOADS = NOT + UPLOADS; - public static final String UPLOAD_ID = "uploadId"; public static final String NOT_UPLOAD_ID = NOT + UPLOAD_ID; - public static final String LEGAL_HOLD = "legal-hold"; public static final String NOT_LEGAL_HOLD = NOT + LEGAL_HOLD; public static final String OBJECT_LOCK = "object-lock"; @@ -57,8 +57,9 @@ public class AwsHttpParameters { public static final String NOT_ATTRIBUTES = NOT + ATTRIBUTES; public static final String LOCATION = "location"; public static final String NOT_LOCATION = NOT + LOCATION; + public static final String VERSIONING = "versioning"; + public static final String NOT_VERSIONING = NOT + VERSIONING; - public static final String VERSION_ID = "versionId"; private AwsHttpParameters() { // private constructor for utility classes diff --git a/server/src/test/kotlin/com/adobe/testing/s3mock/service/MultipartServiceTest.kt b/server/src/test/kotlin/com/adobe/testing/s3mock/service/MultipartServiceTest.kt index 8147432b2..812415c19 100644 --- a/server/src/test/kotlin/com/adobe/testing/s3mock/service/MultipartServiceTest.kt +++ b/server/src/test/kotlin/com/adobe/testing/s3mock/service/MultipartServiceTest.kt @@ -158,7 +158,7 @@ internal class MultipartServiceTest : ServiceTestBase() { val uploadId = "uploadId" val bucketName = "bucketName" whenever(bucketStore.getBucketMetadata(bucketName)) - .thenReturn(BucketMetadata(null, null, null, null, null, null)) + .thenReturn(BucketMetadata(null, null, null, null, null, null, null)) whenever( multipartStore.getMultipartUpload( ArgumentMatchers.any( diff --git a/server/src/test/kotlin/com/adobe/testing/s3mock/service/ServiceTestBase.kt b/server/src/test/kotlin/com/adobe/testing/s3mock/service/ServiceTestBase.kt index 422d3b4df..ad882a22b 100644 --- a/server/src/test/kotlin/com/adobe/testing/s3mock/service/ServiceTestBase.kt +++ b/server/src/test/kotlin/com/adobe/testing/s3mock/service/ServiceTestBase.kt @@ -130,6 +130,7 @@ internal abstract class ServiceTestBase { Date().toString(), null, null, + null, BUCKET_OWNER_ENFORCED, Files.createTempDirectory(bucketName) ) diff --git a/server/src/test/kotlin/com/adobe/testing/s3mock/store/StoreConfigurationTest.kt b/server/src/test/kotlin/com/adobe/testing/s3mock/store/StoreConfigurationTest.kt index bd938a9f5..0c966338a 100644 --- a/server/src/test/kotlin/com/adobe/testing/s3mock/store/StoreConfigurationTest.kt +++ b/server/src/test/kotlin/com/adobe/testing/s3mock/store/StoreConfigurationTest.kt @@ -61,7 +61,7 @@ internal class StoreConfigurationTest { val bucketMetadata = BucketMetadata( existingBucketName, Instant.now().toString(), - null, null, ObjectOwnership.BUCKET_OWNER_ENFORCED, existingBucket + null, null, null, ObjectOwnership.BUCKET_OWNER_ENFORCED, existingBucket ) val metaFile = Paths.get(existingBucket.toString(), BUCKET_META_FILE) OBJECT_MAPPER.writeValue(metaFile.toFile(), bucketMetadata) diff --git a/server/src/test/kotlin/com/adobe/testing/s3mock/store/StoreTestBase.kt b/server/src/test/kotlin/com/adobe/testing/s3mock/store/StoreTestBase.kt index 1e32619a4..9df530933 100644 --- a/server/src/test/kotlin/com/adobe/testing/s3mock/store/StoreTestBase.kt +++ b/server/src/test/kotlin/com/adobe/testing/s3mock/store/StoreTestBase.kt @@ -37,6 +37,7 @@ internal abstract class StoreTestBase { Date().toString(), null, null, + null, ObjectOwnership.BUCKET_OWNER_ENFORCED, Paths.get(rootFolder.toString(), bucketName), mapOf() From df320d2cf7de685f50b09d73202015ede5dc7ebe Mon Sep 17 00:00:00 2001 From: Arne Franken Date: Sat, 1 Jun 2024 10:03:00 +0200 Subject: [PATCH 6/8] Prepare version storage --- .../testing/s3mock/store/ObjectStore.java | 5 ++- .../s3mock/store/S3ObjectVersions.java | 43 +++++++++++++++++++ .../s3mock/store/S3ObjectVersionsTest.kt | 34 +++++++++++++++ 3 files changed, 80 insertions(+), 2 deletions(-) create mode 100644 server/src/main/java/com/adobe/testing/s3mock/store/S3ObjectVersions.java create mode 100644 server/src/test/kotlin/com/adobe/testing/s3mock/store/S3ObjectVersionsTest.kt diff --git a/server/src/main/java/com/adobe/testing/s3mock/store/ObjectStore.java b/server/src/main/java/com/adobe/testing/s3mock/store/ObjectStore.java index 66f708be8..68e67d8da 100644 --- a/server/src/main/java/com/adobe/testing/s3mock/store/ObjectStore.java +++ b/server/src/main/java/com/adobe/testing/s3mock/store/ObjectStore.java @@ -54,6 +54,7 @@ public class ObjectStore extends StoreBase { private static final Logger LOG = LoggerFactory.getLogger(ObjectStore.class); private static final String META_FILE = "objectMetadata.json"; private static final String DATA_FILE = "binaryData"; + private static final String VERSIONS_FILE = "versions.json"; /** * This map stores one lock object per S3Object ID. @@ -475,11 +476,11 @@ private Path getObjectFolderPath(BucketMetadata bucket, UUID id) { } private Path getMetaFilePath(BucketMetadata bucket, UUID id) { - return Paths.get(getObjectFolderPath(bucket, id).toString(), META_FILE); + return getObjectFolderPath(bucket, id).resolve(META_FILE); } private Path getDataFilePath(BucketMetadata bucket, UUID id) { - return Paths.get(getObjectFolderPath(bucket, id).toString(), DATA_FILE); + return getObjectFolderPath(bucket, id).resolve(DATA_FILE); } private void writeMetafile(BucketMetadata bucket, S3ObjectMetadata s3ObjectMetadata) { diff --git a/server/src/main/java/com/adobe/testing/s3mock/store/S3ObjectVersions.java b/server/src/main/java/com/adobe/testing/s3mock/store/S3ObjectVersions.java new file mode 100644 index 000000000..a8483d650 --- /dev/null +++ b/server/src/main/java/com/adobe/testing/s3mock/store/S3ObjectVersions.java @@ -0,0 +1,43 @@ +/* + * Copyright 2017-2024 Adobe. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.adobe.testing.s3mock.store; + +import java.util.HashMap; +import java.util.Map; +import java.util.UUID; +import java.util.concurrent.atomic.AtomicInteger; + +public record S3ObjectVersions( + UUID id, + Map versions, + AtomicInteger latestVersionPointer +) { + + public S3ObjectVersions(UUID id) { + this(id, new HashMap<>(), new AtomicInteger(0)); + } + + public String createVersion() { + String versionId = UUID.randomUUID().toString(); + versions.put(latestVersionPointer.incrementAndGet(), versionId); + return versionId; + } + + public String getLatestVersion() { + return versions.get(latestVersionPointer.get()); + } +} diff --git a/server/src/test/kotlin/com/adobe/testing/s3mock/store/S3ObjectVersionsTest.kt b/server/src/test/kotlin/com/adobe/testing/s3mock/store/S3ObjectVersionsTest.kt new file mode 100644 index 000000000..393120d8a --- /dev/null +++ b/server/src/test/kotlin/com/adobe/testing/s3mock/store/S3ObjectVersionsTest.kt @@ -0,0 +1,34 @@ +/* + * Copyright 2017-2024 Adobe. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.adobe.testing.s3mock.store + +import org.assertj.core.api.Assertions.assertThat +import org.junit.jupiter.api.Test +import java.util.UUID + +internal class S3ObjectVersionsTest { + @Test + fun testVersions() { + val iut = S3ObjectVersions(UUID.randomUUID()) + assertThat(iut.latestVersion).isNull() + assertThat(iut.latestVersionPointer.get()).isZero() + + val version = iut.createVersion() + assertThat(version).isNotBlank() + assertThat(iut.latestVersionPointer.get()).isOne() + assertThat(iut.latestVersion).isEqualTo(version) + } +} From df72f15216817695158c345ac189f8c6a43c734b Mon Sep 17 00:00:00 2001 From: Arne Franken Date: Wed, 24 Jul 2024 23:09:30 +0200 Subject: [PATCH 7/8] Prepare VersionsV2IT --- .../adobe/testing/s3mock/its/VersionsV2IT.kt | 25 +++++++++++++++++++ 1 file changed, 25 insertions(+) create mode 100644 integration-tests/src/test/kotlin/com/adobe/testing/s3mock/its/VersionsV2IT.kt diff --git a/integration-tests/src/test/kotlin/com/adobe/testing/s3mock/its/VersionsV2IT.kt b/integration-tests/src/test/kotlin/com/adobe/testing/s3mock/its/VersionsV2IT.kt new file mode 100644 index 000000000..7abbe21c3 --- /dev/null +++ b/integration-tests/src/test/kotlin/com/adobe/testing/s3mock/its/VersionsV2IT.kt @@ -0,0 +1,25 @@ +/* + * Copyright 2017-2024 Adobe. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.adobe.testing.s3mock.its + +import software.amazon.awssdk.services.s3.S3Client + +internal class VersionsV2IT : S3TestBase() { + private val s3ClientV2: S3Client = createS3ClientV2() + + +} From fb59d36e858e9e2dbd76e081bf7f1a6bed79cd81 Mon Sep 17 00:00:00 2001 From: afranken Date: Wed, 18 Dec 2024 22:20:00 +0100 Subject: [PATCH 8/8] POM cleanup: ordering, scoping --- integration-tests/pom.xml | 2 ++ server/pom.xml | 24 +++++++++++++----------- testsupport/common/pom.xml | 20 ++++++++++---------- testsupport/junit4/pom.xml | 1 + testsupport/junit5/pom.xml | 8 ++++---- testsupport/testcontainers/pom.xml | 18 +++++++++--------- testsupport/testng/pom.xml | 8 ++++---- 7 files changed, 43 insertions(+), 38 deletions(-) diff --git a/integration-tests/pom.xml b/integration-tests/pom.xml index d8c119080..9633efc18 100644 --- a/integration-tests/pom.xml +++ b/integration-tests/pom.xml @@ -40,6 +40,7 @@ com.adobe.testing s3mock-docker pom + test * @@ -70,6 +71,7 @@ org.awaitility awaitility + test org.jetbrains.kotlin diff --git a/server/pom.xml b/server/pom.xml index a83384b03..69c389619 100644 --- a/server/pom.xml +++ b/server/pom.xml @@ -35,6 +35,10 @@ + + org.springframework.boot + spring-boot-starter-web + com.fasterxml.jackson.dataformat jackson-dataformat-xml @@ -59,11 +63,6 @@ software.amazon.awssdk s3 - - software.amazon.awssdk - auth - test - commons-codec commons-codec @@ -82,9 +81,15 @@ httpcore test + + software.amazon.awssdk + auth + test + org.glassfish.jaxb jaxb-runtime + test org.junit.jupiter @@ -101,10 +106,6 @@ spring-boot-configuration-processor true - - org.springframework.boot - spring-boot-starter-actuator - org.springframework.boot spring-boot-devtools @@ -112,12 +113,13 @@ org.springframework.boot - spring-boot-starter-test + spring-boot-starter-actuator test org.springframework.boot - spring-boot-starter-web + spring-boot-starter-test + test org.xmlunit diff --git a/testsupport/common/pom.xml b/testsupport/common/pom.xml index fcc411536..4382aaaa3 100644 --- a/testsupport/common/pom.xml +++ b/testsupport/common/pom.xml @@ -31,13 +31,21 @@ S3Mock - Testsupport - Common + + com.amazonaws + aws-java-sdk-s3 + com.adobe.testing s3mock - com.amazonaws - aws-java-sdk-s3 + software.amazon.awssdk + s3 + + + software.amazon.awssdk + url-connection-client org.assertj @@ -72,13 +80,5 @@ aws-xml-protocol test - - software.amazon.awssdk - s3 - - - software.amazon.awssdk - url-connection-client - diff --git a/testsupport/junit4/pom.xml b/testsupport/junit4/pom.xml index b7a2325ff..6e18ef05d 100644 --- a/testsupport/junit4/pom.xml +++ b/testsupport/junit4/pom.xml @@ -48,6 +48,7 @@ org.springframework.boot spring-boot-starter-actuator + test diff --git a/testsupport/junit5/pom.xml b/testsupport/junit5/pom.xml index 3391039ca..0c20e44ca 100644 --- a/testsupport/junit5/pom.xml +++ b/testsupport/junit5/pom.xml @@ -36,15 +36,15 @@ s3mock-testsupport-common + + org.junit.jupiter + junit-jupiter-api + org.assertj assertj-core test - - org.junit.jupiter - junit-jupiter-api - org.junit.jupiter diff --git a/testsupport/testcontainers/pom.xml b/testsupport/testcontainers/pom.xml index f4a49e863..b2e9130b0 100644 --- a/testsupport/testcontainers/pom.xml +++ b/testsupport/testcontainers/pom.xml @@ -31,11 +31,6 @@ S3Mock - Testsupport - Testcontainers - - com.adobe.testing - s3mock - test - com.adobe.testing @@ -48,6 +43,15 @@ + + org.testcontainers + testcontainers + + + com.adobe.testing + s3mock + test + org.assertj assertj-core @@ -63,10 +67,6 @@ junit-jupiter test - - org.testcontainers - testcontainers - software.amazon.awssdk aws-query-protocol diff --git a/testsupport/testng/pom.xml b/testsupport/testng/pom.xml index 2fb504d47..18dad796f 100644 --- a/testsupport/testng/pom.xml +++ b/testsupport/testng/pom.xml @@ -35,15 +35,15 @@ com.adobe.testing s3mock-testsupport-common + + org.testng + testng + org.assertj assertj-core test - - org.testng - testng -