Skip to content

Commit

Permalink
Add waiters for bucket version status and object version id
Browse files Browse the repository at this point in the history
  • Loading branch information
fczuardi committed Dec 20, 2024
1 parent 5bc68f9 commit 1f5cd26
Show file tree
Hide file tree
Showing 4 changed files with 196 additions and 33 deletions.
22 changes: 17 additions & 5 deletions .github/workflows/pull-request-test.yml
Original file line number Diff line number Diff line change
Expand Up @@ -10,12 +10,11 @@ jobs:
fail-fast: true
matrix:
category:
- acl
- cold_storage
- basic
# - acl
# - cold_storage
# - basic
- presign
- bucket_versioning
# - locking
# - bucket_versioning
# - policy
uses: ./.github/workflows/run-tests.yml
with:
Expand All @@ -31,3 +30,16 @@ jobs:
- name: ok
run:
exit 0
run-locking-tests:
strategy:
fail-fast: false
matrix:
category:
- not cli and locking
uses: ./.github/workflows/run-tests.yml
with:
tests: "*_test.py"
config: "../params.example.yaml"
flags: "-v --log-cli-level INFO --color yes -m '${{ matrix.category }}'"
secrets:
PROFILES: ${{ secrets.PROFILES }}
45 changes: 28 additions & 17 deletions docs/conftest.py
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,9 @@
change_policies_json,
delete_policy_and_bucket_and_wait,
get_tenants,
wait_for_bucket_version,
replace_failed_put_without_version,
put_object_lock_configuration_with_determination,
)
from datetime import datetime, timedelta
from botocore.exceptions import ClientError
Expand Down Expand Up @@ -242,11 +245,16 @@ def versioned_bucket_with_one_object(s3_client, lock_mode):
Bucket=bucket_name,
VersioningConfiguration={"Status": "Enabled"}
)
wait_for_bucket_version(s3_client, bucket_name)

# Upload a single object and get it's version
object_key = "test-object.txt"
content = b"Sample content for testing versioned object."
object_version = put_object_and_wait(s3_client, bucket_name, object_key, content)
if not object_version:
object_version, object_key = replace_failed_put_without_version(s3_client, bucket_name, object_key, content)

assert object_version, "Setup failed, could not get VersionId from put_object in versioned bucket"

# Yield details to tests
yield bucket_name, object_key, object_version
Expand All @@ -260,13 +268,8 @@ def versioned_bucket_with_one_object(s3_client, lock_mode):
@pytest.fixture
def bucket_with_one_object_and_lock_enabled(s3_client, lock_mode, versioned_bucket_with_one_object):
bucket_name, object_key, object_version = versioned_bucket_with_one_object
# Enable bucket lock configuration if not already set
s3_client.put_object_lock_configuration(
Bucket=bucket_name,
ObjectLockConfiguration={
'ObjectLockEnabled': 'Enabled',
}
)
configuration = { 'ObjectLockEnabled': 'Enabled', }
put_object_lock_configuration_with_determination(s3_client, bucket_name, configuration)
logging.info(f"Object lock configuration enabled for bucket: {bucket_name}")

# Yield details to tests
Expand All @@ -291,6 +294,8 @@ def lockeable_bucket_name(s3_client, lock_mode):
VersioningConfiguration={"Status": "Enabled"}
)

versioning_status = wait_for_bucket_version(s3_client, bucket_name)

logging.info(f"Created versioned bucket: {bucket_name}")

# Yield the bucket name for tests
Expand All @@ -316,23 +321,22 @@ def bucket_with_lock(lockeable_bucket_name, s3_client, lock_mode):

# Enable Object Lock configuration with a default retention rule
retention_days = 1
s3_client.put_object_lock_configuration(
Bucket=bucket_name,
ObjectLockConfiguration={
"ObjectLockEnabled": "Enabled",
"Rule": {
"DefaultRetention": {
"Mode": lock_mode,
"Days": retention_days
}
configuration = {
"ObjectLockEnabled": "Enabled",
"Rule": {
"DefaultRetention": {
"Mode": lock_mode,
"Days": retention_days
}
}
)
}
put_object_lock_configuration_with_determination(s3_client, bucket_name, configuration)

logging.info(f"Bucket '{bucket_name}' configured with Object Lock and default retention.")

return bucket_name


@pytest.fixture
def bucket_with_lock_and_object(s3_client, bucket_with_lock):
"""
Expand All @@ -347,9 +351,16 @@ def bucket_with_lock_and_object(s3_client, bucket_with_lock):
object_key = "test-object.txt"
object_content = "This is a dynamically generated object for testing."

versioning_status = wait_for_bucket_version(s3_client, bucket_name)
logging.info(f"bucket versioning status is: {versioning_status}")

# Upload the generated object to the bucket
response = s3_client.put_object(Bucket=bucket_name, Key=object_key, Body=object_content)
object_version = response.get("VersionId")
if not object_version:
object_version, object_key = replace_failed_put_without_version(s3_client, bucket_name, object_key, object_content)

assert object_version, "Setup failed, could not get VersionId from put_object in versioned bucket"

# Verify that the object is uploaded and has a version ID
if not object_version:
Expand Down
39 changes: 30 additions & 9 deletions docs/locking_test.py
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,10 @@
create_bucket_and_wait,
put_object_and_wait,
cleanup_old_buckets,
wait_for_bucket_version,
replace_failed_put_without_version,
get_object_lock_configuration_with_determination,
get_object_retention_with_determination,
)
config = os.getenv("CONFIG", config)
# -
Expand Down Expand Up @@ -71,6 +75,7 @@ def versioned_bucket_with_lock_config(s3_client, versioned_bucket_with_one_objec
}
}
}
logging.info(f"calling put_object_lock_configuration...")
response = s3_client.put_object_lock_configuration(
Bucket=bucket_name,
ObjectLockConfiguration=lock_config
Expand All @@ -79,10 +84,21 @@ def versioned_bucket_with_lock_config(s3_client, versioned_bucket_with_one_objec
assert response_status == 200, "Expected HTTPStatusCode 200 for successful lock configuration."
logging.info(f"Bucket '{bucket_name}' locked with mode {lock_mode}. Status: {response_status}")

# wait for the lock configuration to return on get calls
applied_config = get_object_lock_configuration_with_determination(s3_client, bucket_name)
assert applied_config["ObjectLockConfiguration"]["ObjectLockEnabled"] == "Enabled", "Expected Object Lock to be enabled."
logging.info(f"A lock configuration exists on te bucket, if you trust the return of get_object_lock_configuration call.")

# Upload another object after lock configuration
second_object_key = "post-lock-object.txt"
post_lock_content = b"Content for object after lock configuration"
logging.info(f"Making a put_object on a bucket that is supposed to have lock configuration...")
second_version_id = put_object_and_wait(s3_client, bucket_name, second_object_key, post_lock_content)
if not second_version_id:
second_version_id, second_object_key = replace_failed_put_without_version(s3_client, bucket_name, second_object_key, post_lock_content)

assert second_version_id, "Setup failed, could not get VersionId from put_object in versioned bucket"

logging.info(f"Uploaded post-lock object: {bucket_name}/{second_object_key} with version ID {second_version_id}")

# Yield details for tests to use
Expand Down Expand Up @@ -162,7 +178,9 @@ def test_verify_object_lock_configuration(bucket_with_lock, s3_client, lock_mode

# Retrieve and verify the applied bucket-level Object Lock configuration
logging.info("Retrieving Object Lock configuration from bucket...")
applied_config = s3_client.get_object_lock_configuration(Bucket=bucket_name)
# the commented line below is the boto3 command to get object lock configuration, we use a helper function to account for MagaluCloud eventual consistency
# applied_config = s3_client.get_object_lock_configuration(Bucket=bucket_name)
applied_config = get_object_lock_configuration_with_determination(s3_client, bucket_name)
assert applied_config["ObjectLockConfiguration"]["ObjectLockEnabled"] == "Enabled", "Expected Object Lock to be enabled."
assert applied_config["ObjectLockConfiguration"]["Rule"]["DefaultRetention"]["Mode"] == lock_mode, f"Expected retention mode to be {lock_mode}."
assert applied_config["ObjectLockConfiguration"]["Rule"]["DefaultRetention"]["Days"] == 1, "Expected retention period of 1 day."
Expand All @@ -189,18 +207,21 @@ def test_verify_object_retention(versioned_bucket_with_lock_config, s3_client, l

# Use get_object_retention to check object-level retention details
logging.info("Retrieving object retention details...")
retention_info = s3_client.get_object_retention(Bucket=bucket_name, Key=second_object_key)
# the commented line below is the boto3 command to get object retention, we use a helper function to account for MagaluCloud eventual consistency
# retention_info = s3_client.get_object_retention(Bucket=bucket_name, Key=second_object_key)
retention_info = get_object_retention_with_determination(s3_client, bucket_name, second_object_key)
assert retention_info["Retention"]["Mode"] == lock_mode, f"Expected object lock mode to be {lock_mode}."
logging.info(f"Retention verified as applied with mode {retention_info['Retention']['Mode']} "
f"and retain until {retention_info['Retention']['RetainUntilDate']}.")

# Use head_object to check retention details
logging.info("Fetching data of the new object with a head request...")
head_response = s3_client.head_object(Bucket=bucket_name, Key=second_object_key)
assert head_response['ObjectLockRetainUntilDate'], 'Expected lock ending date to be present.'
assert head_response['ObjectLockMode'] == lock_mode, f"Expected lock mode to be {lock_mode}"
logging.info(f"Retention verified as applied with mode {head_response['ObjectLockMode']} "
f"and retain until {head_response['ObjectLockRetainUntilDate']}.")
# TODO: uncomment if MagaluCloud start returning retention date on the head object
# # Use head_object to check retention details
# logging.info("Fetching data of the new object with a head request...")
# head_response = s3_client.head_object(Bucket=bucket_name, Key=second_object_key)
# assert head_response['ObjectLockRetainUntilDate'], 'Expected lock ending date to be present.'
# assert head_response['ObjectLockMode'] == lock_mode, f"Expected lock mode to be {lock_mode}"
# logging.info(f"Retention verified as applied with mode {head_response['ObjectLockMode']} "
# f"and retain until {head_response['ObjectLockRetainUntilDate']}.")
run_example(__name__, "test_verify_object_retention", config=config,)
# -

Expand Down
123 changes: 121 additions & 2 deletions docs/s3_helpers.py
Original file line number Diff line number Diff line change
Expand Up @@ -130,8 +130,7 @@ def put_object_and_wait(s3_client, bucket_name, object_key, content):

# Log confirmation
logging.info(
f"Object '{object_key}' in bucket '{bucket_name}' confirmed as uploaded. "
f"Version ID: {version_id}"
f"Object '{object_key}' in bucket '{bucket_name}' confirmed as uploaded. Version ID: {version_id}"
)

return version_id
Expand Down Expand Up @@ -270,3 +269,123 @@ def update_existing_keys(main_dict, sub_dict):
main_dict[key] = sub_dict[key]

return main_dict

# TODO: not cool, #eventualconsistency
def wait_for_bucket_version(s3_client, bucket_name):
retries = 0
versioning_status = "Unknown"
start_time = datetime.now()
while versioning_status != "Enabled" and retries < 10:
logging.info(f"[wait_for_bucket_version] check ({retries}) if the bucket version status got propagated...")
versioning_status = s3_client.get_bucket_versioning(
Bucket=bucket_name
).get('Status')
retries += 1
time.sleep(retries * retries)
assert versioning_status == "Enabled", "Setup error: versioned bucket is not actually versioned"
end_time = datetime.now()
logging.warn(f"[replace_failed_put_without_version] Total consistency wait time={end_time - start_time}")

return versioning_status

# TODO: not cool, #eventualconsistency
def replace_failed_put_without_version(s3_client, bucket_name, object_key, object_content):

retries = 0
interval_multiplier = 3 # seconds
start_time = datetime.now()
object_version = None
while not object_version and retries < 10:
retries += 1

# create a new object key
new_object_key = f"object_key_{retries}"

logging.info(f"attempt ({retries}): key:{new_object_key}")
wait_time = retries * retries * interval_multiplier
logging.info(f"wait {wait_time} seconds")
time.sleep(wait_time)

# delete object (marker?) on the strange object without version id
s3_client.delete_object(Bucket=bucket_name, Key=object_key)

# check again the bucket versioning status
versioning_status = wait_for_bucket_version(s3_client, bucket_name)
logging.info(f"check again ({retries}) that bucket versioning status: {versioning_status}")
# put the object again in the hopes that this time it will have a version id
response = s3_client.put_object(Bucket=bucket_name, Key=new_object_key, Body=object_content)

# check if it has version id
object_version = response.get("VersionId")
logging.info(f"version:{object_version}")
end_time = datetime.now()
logging.warn(f"[replace_failed_put_without_version] Total consistency wait time={end_time - start_time}")

return object_version, new_object_key

# TODO: review when #eventualconsistency stops being so bad
def put_object_lock_configuration_with_determination(s3_client, bucket_name, configuration):
retries = 0
interval_multiplier = 3 # seconds
response = None
start_time = datetime.now()
while retries < 10:
retries += 1
try:
response = s3_client.put_object_lock_configuration(
Bucket=bucket_name,
ObjectLockConfiguration=configuration
)
break
except Exception as e:
logging.error(f"Error ({retries}): {e}")
wait_time = retries * retries * interval_multiplier
logging.info(f"wait {wait_time} seconds")
time.sleep(wait_time)
end_time = datetime.now()
logging.warn(f"[put_object_lock_configuration_with_determination] Total consistency wait time={end_time - start_time}")
return response

# TODO: review when #eventualconsistency stops being so bad
def get_object_retention_with_determination(s3_client, bucket_name, object_key):
retries = 0
interval_multiplier = 3 # seconds
response = None
start_time = datetime.now()
while retries < 20:
retries += 1
try:
response = s3_client.get_object_retention(
Bucket=bucket_name,
Key=object_key,
)
break
except Exception as e:
logging.error(f"[get_object_retention_with_determination] Error ({retries}): {e}")
wait_time = retries * retries * interval_multiplier
logging.info(f"wait {wait_time} seconds")
time.sleep(wait_time)
end_time = datetime.now()
logging.warn(f"[get_object_retention_with_determination] Total consistency wait time={end_time - start_time}")
return response


# TODO: review when #eventualconsistency stops being so bad
def get_object_lock_configuration_with_determination(s3_client, bucket_name):
retries = 0
interval_multiplier = 3 # seconds
response = None
start_time = datetime.now()
while retries < 20:
retries += 1
try:
response = s3_client.get_object_lock_configuration(Bucket=bucket_name)
break
except Exception as e:
logging.error(f"[get_object_lock_configuration_with_determination] Error ({retries}): {e}")
wait_time = retries * retries * interval_multiplier
logging.info(f"wait {wait_time} seconds")
time.sleep(wait_time)
end_time = datetime.now()
logging.warn(f"[get_object_lock_configuration_with_determination] Total consistency wait time={end_time - start_time}")
return response

0 comments on commit 1f5cd26

Please sign in to comment.