mirror of
https://github.com/minio/minio.git
synced 2025-11-11 06:20:14 -05:00
Support bucket versioning (#9377)
- Implement a new xl.json 2.0.0 format to support, this moves the entire marshaling logic to POSIX layer, top layer always consumes a common FileInfo construct which simplifies the metadata reads. - Implement list object versions - Migrate to siphash from crchash for new deployments for object placements. Fixes #2111
This commit is contained in:
Harshavardhana
GitHub
@@ -1518,177 +1518,177 @@ function test_serverside_encryption_error() {
|
||||
return $rv
|
||||
}
|
||||
|
||||
# WORM bucket tests.
|
||||
function test_worm_bucket() {
|
||||
# log start time
|
||||
start_time=$(get_time)
|
||||
# # WORM bucket tests.
|
||||
# function test_worm_bucket() {
|
||||
# # log start time
|
||||
# start_time=$(get_time)
|
||||
|
||||
# Make bucket
|
||||
bucket_name="awscli-mint-test-bucket-$RANDOM"
|
||||
function="${AWS} s3api create-bucket --bucket ${bucket_name} --object-lock-enabled-for-bucket"
|
||||
# # Make bucket
|
||||
# bucket_name="awscli-mint-test-bucket-$RANDOM"
|
||||
# function="${AWS} s3api create-bucket --bucket ${bucket_name} --object-lock-enabled-for-bucket"
|
||||
|
||||
# execute the test
|
||||
out=$($function 2>&1)
|
||||
rv=$?
|
||||
# # execute the test
|
||||
# out=$($function 2>&1)
|
||||
# rv=$?
|
||||
|
||||
if [ $rv -ne 0 ]; then
|
||||
# if this functionality is not implemented return right away.
|
||||
if echo "$out" | grep -q "NotImplemented"; then
|
||||
${AWS} s3 rb s3://"${bucket_name}" --force > /dev/null 2>&1
|
||||
return 0
|
||||
fi
|
||||
fi
|
||||
# if [ $rv -ne 0 ]; then
|
||||
# # if this functionality is not implemented return right away.
|
||||
# if echo "$out" | grep -q "NotImplemented"; then
|
||||
# ${AWS} s3 rb s3://"${bucket_name}" --force > /dev/null 2>&1
|
||||
# return 0
|
||||
# fi
|
||||
# fi
|
||||
|
||||
# if make bucket succeeds set object lock configuration
|
||||
if [ $rv -eq 0 ]; then
|
||||
function="${AWS} s3api put-object-lock-configuration --bucket ${bucket_name} --object-lock-configuration ObjectLockEnabled=Enabled"
|
||||
out=$($function 2>&1)
|
||||
rv=$?
|
||||
if [ $rv -ne 0 ]; then
|
||||
# if this functionality is not implemented return right away.
|
||||
if echo "$out" | grep -q "NotImplemented"; then
|
||||
${AWS} s3 rb s3://"${bucket_name}" --force > /dev/null 2>&1
|
||||
return 0
|
||||
fi
|
||||
fi
|
||||
else
|
||||
# if make bucket fails, $bucket_name has the error output
|
||||
out="${bucket_name}"
|
||||
fi
|
||||
# # if make bucket succeeds set object lock configuration
|
||||
# if [ $rv -eq 0 ]; then
|
||||
# function="${AWS} s3api put-object-lock-configuration --bucket ${bucket_name} --object-lock-configuration ObjectLockEnabled=Enabled"
|
||||
# out=$($function 2>&1)
|
||||
# rv=$?
|
||||
# if [ $rv -ne 0 ]; then
|
||||
# # if this functionality is not implemented return right away.
|
||||
# if echo "$out" | grep -q "NotImplemented"; then
|
||||
# ${AWS} s3 rb s3://"${bucket_name}" --force > /dev/null 2>&1
|
||||
# return 0
|
||||
# fi
|
||||
# fi
|
||||
# else
|
||||
# # if make bucket fails, $bucket_name has the error output
|
||||
# out="${bucket_name}"
|
||||
# fi
|
||||
|
||||
# if setting object lock configuration succeeds, upload a file first time
|
||||
if [ $rv -eq 0 ]; then
|
||||
function="${AWS} s3api put-object --body ${MINT_DATA_DIR}/datafile-1-kB --bucket ${bucket_name} --key datafile-1-kB"
|
||||
out=$($function 2>&1)
|
||||
rv=$?
|
||||
else
|
||||
# if make bucket fails, $bucket_name has the error output
|
||||
out="${bucket_name}"
|
||||
fi
|
||||
# # if setting object lock configuration succeeds, upload a file first time
|
||||
# if [ $rv -eq 0 ]; then
|
||||
# function="${AWS} s3api put-object --body ${MINT_DATA_DIR}/datafile-1-kB --bucket ${bucket_name} --key datafile-1-kB"
|
||||
# out=$($function 2>&1)
|
||||
# rv=$?
|
||||
# else
|
||||
# # if make bucket fails, $bucket_name has the error output
|
||||
# out="${bucket_name}"
|
||||
# fi
|
||||
|
||||
# second time upload of same file should fail due to WORM setting
|
||||
if [ $rv -eq 0 ]; then
|
||||
function="${AWS} s3api put-object --body ${MINT_DATA_DIR}/datafile-1-kB --bucket ${bucket_name} --key datafile-1-kB"
|
||||
out=$($function 2>&1)
|
||||
rv=$?
|
||||
else
|
||||
out="First time object upload failed"
|
||||
fi
|
||||
# # second time upload will succeed and there shall be now two versions of the object
|
||||
# if [ $rv -eq 0 ]; then
|
||||
# function="${AWS} s3api put-object --body ${MINT_DATA_DIR}/datafile-1-kB --bucket ${bucket_name} --key datafile-1-kB"
|
||||
# out=$($function 2>&1)
|
||||
# rv=$?
|
||||
# else
|
||||
# out="First time object upload failed"
|
||||
# fi
|
||||
|
||||
if [ $rv -eq 0 ]; then
|
||||
log_success "$(get_duration "$start_time")" "${test_function}"
|
||||
else
|
||||
# cleanup is not possible due to one day validity of object lock configurataion
|
||||
log_failure "$(get_duration "$start_time")" "${function}" "${out}"
|
||||
fi
|
||||
# if [ $rv -eq 0 ]; then
|
||||
# log_success "$(get_duration "$start_time")" "${test_function}"
|
||||
# else
|
||||
# # cleanup is not possible due to one day validity of object lock configurataion
|
||||
# log_failure "$(get_duration "$start_time")" "${function}" "${out}"
|
||||
# fi
|
||||
|
||||
return $rv
|
||||
}
|
||||
# return $rv
|
||||
# }
|
||||
|
||||
# Tests creating and deleting an object with legal hold.
|
||||
function test_legal_hold() {
|
||||
# log start time
|
||||
start_time=$(get_time)
|
||||
# # Tests creating and deleting an object with legal hold.
|
||||
# function test_legal_hold() {
|
||||
# # log start time
|
||||
# start_time=$(get_time)
|
||||
|
||||
# Make bucket
|
||||
bucket_name="awscli-mint-test-bucket-$RANDOM"
|
||||
function="${AWS} s3api create-bucket --bucket ${bucket_name} --object-lock-enabled-for-bucket"
|
||||
# # Make bucket
|
||||
# bucket_name="awscli-mint-test-bucket-$RANDOM"
|
||||
# function="${AWS} s3api create-bucket --bucket ${bucket_name} --object-lock-enabled-for-bucket"
|
||||
|
||||
# execute the test
|
||||
out=$($function 2>&1)
|
||||
rv=$?
|
||||
# # execute the test
|
||||
# out=$($function 2>&1)
|
||||
# rv=$?
|
||||
|
||||
if [ $rv -ne 0 ]; then
|
||||
# if this functionality is not implemented return right away.
|
||||
if echo "$out" | grep -q "NotImplemented"; then
|
||||
${AWS} s3 rb s3://"${bucket_name}" --force > /dev/null 2>&1
|
||||
return 0
|
||||
fi
|
||||
fi
|
||||
# if [ $rv -ne 0 ]; then
|
||||
# # if this functionality is not implemented return right away.
|
||||
# if echo "$out" | grep -q "NotImplemented"; then
|
||||
# ${AWS} s3 rb s3://"${bucket_name}" --force > /dev/null 2>&1
|
||||
# return 0
|
||||
# fi
|
||||
# fi
|
||||
|
||||
# if make bucket succeeds upload a file
|
||||
if [ $rv -eq 0 ]; then
|
||||
function="${AWS} s3api put-object --body ${MINT_DATA_DIR}/datafile-1-kB --bucket ${bucket_name} --key datafile-1-kB --object-lock-legal-hold-status ON"
|
||||
out=$($function 2>&1)
|
||||
errcnt=$(echo "$out" | sed -n '/Bucket is missing ObjectLockConfiguration/p' | wc -l)
|
||||
# skip test for gateways
|
||||
if [ "$errcnt" -eq 1 ]; then
|
||||
return 0
|
||||
fi
|
||||
rv=$?
|
||||
else
|
||||
# if make bucket fails, $bucket_name has the error output
|
||||
out="${bucket_name}"
|
||||
fi
|
||||
# # if make bucket succeeds upload a file
|
||||
# if [ $rv -eq 0 ]; then
|
||||
# function="${AWS} s3api put-object --body ${MINT_DATA_DIR}/datafile-1-kB --bucket ${bucket_name} --key datafile-1-kB --object-lock-legal-hold-status ON"
|
||||
# out=$($function 2>&1)
|
||||
# errcnt=$(echo "$out" | sed -n '/Bucket is missing ObjectLockConfiguration/p' | wc -l)
|
||||
# # skip test for gateways
|
||||
# if [ "$errcnt" -eq 1 ]; then
|
||||
# return 0
|
||||
# fi
|
||||
# rv=$?
|
||||
# else
|
||||
# # if make bucket fails, $bucket_name has the error output
|
||||
# out="${bucket_name}"
|
||||
# fi
|
||||
|
||||
# if upload succeeds stat the file
|
||||
if [ $rv -eq 0 ]; then
|
||||
function="${AWS} s3api head-object --bucket ${bucket_name} --key datafile-1-kB"
|
||||
# save the ref to function being tested, so it can be logged
|
||||
test_function=${function}
|
||||
out=$($function 2>&1)
|
||||
lhold=$(echo "$out" | jq -r .ObjectLockLegalHoldStatus)
|
||||
rv=$?
|
||||
fi
|
||||
# # if upload succeeds stat the file
|
||||
# if [ $rv -eq 0 ]; then
|
||||
# function="${AWS} s3api head-object --bucket ${bucket_name} --key datafile-1-kB"
|
||||
# # save the ref to function being tested, so it can be logged
|
||||
# test_function=${function}
|
||||
# out=$($function 2>&1)
|
||||
# lhold=$(echo "$out" | jq -r .ObjectLockLegalHoldStatus)
|
||||
# rv=$?
|
||||
# fi
|
||||
|
||||
# if head-object succeeds, verify metadata has legal hold status
|
||||
if [ $rv -eq 0 ]; then
|
||||
if [ "${lhold}" == "" ]; then
|
||||
rv=1
|
||||
out="Legal hold was not applied"
|
||||
fi
|
||||
if [ "${lhold}" == "OFF" ]; then
|
||||
rv=1
|
||||
out="Legal hold was not applied"
|
||||
fi
|
||||
fi
|
||||
if [ $rv -eq 0 ]; then
|
||||
function="${AWS} s3api put-object-legal-hold --bucket ${bucket_name} --key datafile-1-kB --legal-hold Status=OFF"
|
||||
out=$($function 2>&1)
|
||||
rv=$?
|
||||
else
|
||||
# if make bucket fails, $bucket_name has the error output
|
||||
out="${bucket_name}"
|
||||
fi
|
||||
# if upload succeeds download the file
|
||||
if [ $rv -eq 0 ]; then
|
||||
function="${AWS} s3api get-object-legal-hold --bucket ${bucket_name} --key datafile-1-kB"
|
||||
# save the ref to function being tested, so it can be logged
|
||||
test_function=${function}
|
||||
out=$($function 2>&1)
|
||||
lhold=$(echo "$out" | jq -r .LegalHold.Status)
|
||||
rv=$?
|
||||
fi
|
||||
# # if head-object succeeds, verify metadata has legal hold status
|
||||
# if [ $rv -eq 0 ]; then
|
||||
# if [ "${lhold}" == "" ]; then
|
||||
# rv=1
|
||||
# out="Legal hold was not applied"
|
||||
# fi
|
||||
# if [ "${lhold}" == "OFF" ]; then
|
||||
# rv=1
|
||||
# out="Legal hold was not applied"
|
||||
# fi
|
||||
# fi
|
||||
# if [ $rv -eq 0 ]; then
|
||||
# function="${AWS} s3api put-object-legal-hold --bucket ${bucket_name} --key datafile-1-kB --legal-hold Status=OFF"
|
||||
# out=$($function 2>&1)
|
||||
# rv=$?
|
||||
# else
|
||||
# # if make bucket fails, $bucket_name has the error output
|
||||
# out="${bucket_name}"
|
||||
# fi
|
||||
# # if upload succeeds download the file
|
||||
# if [ $rv -eq 0 ]; then
|
||||
# function="${AWS} s3api get-object-legal-hold --bucket ${bucket_name} --key datafile-1-kB"
|
||||
# # save the ref to function being tested, so it can be logged
|
||||
# test_function=${function}
|
||||
# out=$($function 2>&1)
|
||||
# lhold=$(echo "$out" | jq -r .LegalHold.Status)
|
||||
# rv=$?
|
||||
# fi
|
||||
|
||||
# if head-object succeeds, verify metadata has legal hold status
|
||||
if [ $rv -eq 0 ]; then
|
||||
if [ "${lhold}" == "" ]; then
|
||||
rv=1
|
||||
out="Legal hold was not applied"
|
||||
fi
|
||||
if [ "${lhold}" == "ON" ]; then
|
||||
rv=1
|
||||
out="Legal hold status not turned off"
|
||||
fi
|
||||
fi
|
||||
# Attempt a delete on prefix shouldn't delete the directory since we have an object inside it.
|
||||
if [ $rv -eq 0 ]; then
|
||||
function="${AWS} s3api delete-object --bucket ${bucket_name} --key datafile-1-kB"
|
||||
# save the ref to function being tested, so it can be logged
|
||||
test_function=${function}
|
||||
out=$($function 2>&1)
|
||||
rv=$?
|
||||
fi
|
||||
if [ $rv -eq 0 ]; then
|
||||
log_success "$(get_duration "$start_time")" "${test_function}"
|
||||
else
|
||||
# clean up and log error
|
||||
${AWS} s3 rb s3://"${bucket_name}" --force > /dev/null 2>&1
|
||||
log_failure "$(get_duration "$start_time")" "${function}" "${out}"
|
||||
fi
|
||||
# # if head-object succeeds, verify metadata has legal hold status
|
||||
# if [ $rv -eq 0 ]; then
|
||||
# if [ "${lhold}" == "" ]; then
|
||||
# rv=1
|
||||
# out="Legal hold was not applied"
|
||||
# fi
|
||||
# if [ "${lhold}" == "ON" ]; then
|
||||
# rv=1
|
||||
# out="Legal hold status not turned off"
|
||||
# fi
|
||||
# fi
|
||||
# # Attempt a delete on prefix shouldn't delete the directory since we have an object inside it.
|
||||
# if [ $rv -eq 0 ]; then
|
||||
# function="${AWS} s3api delete-object --bucket ${bucket_name} --key datafile-1-kB"
|
||||
# # save the ref to function being tested, so it can be logged
|
||||
# test_function=${function}
|
||||
# out=$($function 2>&1)
|
||||
# rv=$?
|
||||
# fi
|
||||
# if [ $rv -eq 0 ]; then
|
||||
# log_success "$(get_duration "$start_time")" "${test_function}"
|
||||
# else
|
||||
# # clean up and log error
|
||||
# ${AWS} s3 rb s3://"${bucket_name}" --force > /dev/null 2>&1
|
||||
# log_failure "$(get_duration "$start_time")" "${function}" "${out}"
|
||||
# fi
|
||||
|
||||
return $rv
|
||||
}
|
||||
# return $rv
|
||||
# }
|
||||
|
||||
# main handler for all the tests.
|
||||
main() {
|
||||
@@ -1716,9 +1716,9 @@ main() {
|
||||
# Error tests
|
||||
test_list_objects_error && \
|
||||
test_put_object_error && \
|
||||
test_serverside_encryption_error && \
|
||||
test_worm_bucket && \
|
||||
test_legal_hold
|
||||
test_serverside_encryption_error
|
||||
# test_worm_bucket && \
|
||||
# test_legal_hold
|
||||
|
||||
return $?
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user