tests: add more

* CLI 'ais get --latest' * CLI 'ais cp --latest' and 'ais cp --sync' (in progress) Signed-off-by: Alex Aizman <[email protected]>
NVIDIA · Jan 8, 2024 · 6cb2c3a · 6cb2c3a
1 parent 54d3f9c
commit 6cb2c3a
Show file tree

Hide file tree

Showing 5 changed files with 149 additions and 5 deletions.
diff --git a/ais/test/scripts/remais-get-validate.sh b/ais/test/scripts/remais-get-validate.sh
@@ -85,6 +85,21 @@ checksum=$(ais ls "$bucket/lorem-duis" --cached -H -props checksum | awk '{print
 echo "3. out-of-band PUT: 2nd version (overwrite)"
 echo $duis | AIS_ENDPOINT=$rendpoint ais put - "$rbucket/lorem-duis" 1>/dev/null || exit $?
 
+######### --latest
+
+echo "3.1 'get --latest' without changing bucket props"
+
+ais get "$bucket/lorem-duis" /dev/null --latest 1>/dev/null
+checksum=$(ais ls "$bucket/lorem-duis" --cached -H -props checksum | awk '{print $2}')
+[[ "$checksum" == "$sum2"  ]] || { echo "FAIL: $checksum != $sum2"; exit 1; }
+
+echo "3.2 restore the state prior to step 3.1"
+
+echo $lorem | ais put - "$bucket/lorem-duis" 1>/dev/null || exit $?
+echo $duis  | AIS_ENDPOINT=$rendpoint ais put - "$rbucket/lorem-duis" $host 1>/dev/null || exit $?
+
+######### end of --latest
+
 echo "4. warm GET and check (expecting the first version's checksum)"
 ais get "$bucket/lorem-duis" /dev/null 1>/dev/null
 checksum=$(ais ls "$bucket/lorem-duis" --cached -H -props checksum | awk '{print $2}')

diff --git a/ais/test/scripts/s3-cp-latest-prefix.sh b/ais/test/scripts/s3-cp-latest-prefix.sh
@@ -0,0 +1,114 @@
+#!/bin/bash
+
+## Prerequisites: #################################################################################
+# - s3 bucket
+# - s3cmd, $PATH-executable and configured to access the bucket out-of-band
+# - aistore cluster, also configured to access the same bucket
+#
+## Example usage:
+## ./ais/test/scripts/s3-cp-latest-prefix.sh --bucket s3://abc       ########################
+
+lorem='Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat.'
+
+duis='Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum. Et harum quidem..'
+
+## Command line options (and their respective defaults)
+bucket="s3://abc"
+
+## constants
+sum1="xxhash[ad97df912d23103f]"
+sum2="xxhash[ecb5ed42299ea74d]"
+
+host="--host=s3.amazonaws.com"
+
+while (( "$#" )); do
+  case "${1}" in
+    --bucket) bucket=$2; shift; shift;;
+    *) echo "fatal: unknown argument '${1}'"; exit 1;;
+  esac
+done
+
+## uncomment for verbose output
+## set -x
+
+## establish existence
+ais show bucket $bucket -c 1>/dev/null || exit $?
+
+## remember existing bucket's versioning; disable if need be
+sync=$(ais bucket props show ${bucket} versioning.synchronize -H | awk '{print $2}')
+[[ "$sync" == "false"  ]] || ais bucket props set $bucket versioning.synchronize=false
+
+cleanup() {
+  rc=$?
+  ais object rm "$bucket/lorem-duis" 1>/dev/null 2>&1
+  [[ "$sync" == "true"  ]] || ais bucket props set $bucket versioning.synchronize=false 1>/dev/null 2>&1
+  exit $rc
+}
+
+trap cleanup EXIT INT TERM
+
+echo -e
+ais show performance counters --regex "(GET-COLD$|VERSION-CHANGE$|DELETE)"
+echo -e
+
+echo "1. out-of-band PUT: 1st version"
+echo $lorem | s3cmd put - "$bucket/lorem-duis" $host 1>/dev/null || exit $?
+
+echo "2. copy, and check"
+ais cp "$bucket/lorem-duis" --wait
+checksum=$(ais ls "$bucket/lorem-duis" --cached -H -props checksum | awk '{print $2}')
+[[ "$checksum" == "$sum1"  ]] || { echo "FAIL: $checksum != $sum1"; exit 1; }
+
+echo "3. out-of-band PUT: 2nd version (overwrite)"
+echo $duis | s3cmd put - "$bucket/lorem-duis" $host 1>/dev/null || exit $?
+
+echo "4. copy and check (expecting the first version's checksum)"
+ais cp "$bucket/lorem-duis" --wait
+checksum=$(ais ls "$bucket/lorem-duis" --cached -H -props checksum | awk '{print $2}')
+[[ "$checksum" != "$sum2"  ]] || { echo "FAIL: $checksum == $sum2"; exit 1; }
+
+echo "5. query cold-get count (statistics)"
+cnt1=$(ais show performance counters --regex GET-COLD -H | awk '{sum+=$2;}END{print sum;}')
+
+echo "6. copy latest: detect version change and update in-cluster copy"
+ais cp "$bucket/lorem-duis" --latest --wait
+checksum=$(ais ls "$bucket/lorem-duis" --cached -H -props checksum | awk '{print $2}')
+[[ "$checksum" == "$sum2"  ]] || { echo "FAIL: $checksum != $sum2"; exit 1; }
+
+echo "7. cold-get counter must increment"
+cnt2=$(ais show performance counters --regex GET-COLD -H | awk '{sum+=$2;}END{print sum;}')
+[[ $cnt2 == $(($cnt1+1)) ]] || { echo "FAIL: $cnt2 != $(($cnt1+1))"; exit 1; }
+
+echo "8. warm GET must remain \"warm\" and cold-get-count must not increment"
+ais get "$bucket/lorem-duis" /dev/null 1>/dev/null
+checksum=$(ais ls "$bucket/lorem-duis" --cached -H -props checksum | awk '{print $2}')
+[[ "$checksum" == "$sum2"  ]] || { echo "FAIL: $checksum != $sum2"; exit 1; }
+
+cnt3=$(ais show performance counters --regex GET-COLD -H | awk '{sum+=$2;}END{print sum;}')
+[[ $cnt3 == $cnt2 ]] || { echo "FAIL: $cnt3 != $cnt2"; exit 1; }
+
+echo "9. out-of-band DELETE"
+s3cmd del "$bucket/lorem-duis" $host 1>/dev/null || exit $?
+
+echo "10. copy with '--latest' (expecting no changes)"
+ais cp "$bucket/lorem-duis" --latest --wait
+checksum=$(ais ls "$bucket/lorem-duis" --cached -H -props checksum | awk '{print $2}')
+[[ "$checksum" == "$sum2"  ]] || { echo "FAIL: $checksum != $sum2"; exit 1; }
+
+echo "11. remember 'remote-deleted' counter"
+cnt4=$(ais show performance counters --regex REMOTE-DEL -H | awk '{sum+=$2;}END{print sum;}')
+
+echo "12. run 'cp --latest' one last time, and make sure the object \"disappears\""
+ais cp "$bucket/lorem-duis" --latest --wait 2>/dev/null
+[[ $? == 0 ]] || { echo "FAIL: expecting 'prefetch --wait' to return Ok, got $?"; exit 1; }
+
+echo "13. 'remote-deleted' counter must increment"
+cnt5=$(ais show performance counters --regex REMOTE-DEL -H | awk '{sum+=$2;}END{print sum;}')
+[[ $cnt5 == $(($cnt4+1)) ]] || { echo "FAIL: $cnt5 != $(($cnt4+1))"; exit 1; }
+
+
+ais ls "$bucket/lorem-duis" --cached --silent -H 2>/dev/null
+[[ $? != 0 ]] || { echo "FAIL: expecting 'show object' error, got $?"; exit 1; }
+
+echo -e
+ais show performance counters --regex "(GET-COLD$|VERSION-CHANGE$|DELETE)"
diff --git a/ais/test/scripts/s3-get-validate.sh b/ais/test/scripts/s3-get-validate.sh
@@ -65,6 +65,21 @@ checksum=$(ais ls "$bucket/lorem-duis" --cached -H -props checksum | awk '{print
 echo "3. out-of-band PUT: 2nd version (overwrite)"
 echo $duis | s3cmd put - "$bucket/lorem-duis" $host 1>/dev/null || exit $?
 
+######### --latest
+
+echo "3.1 'get --latest' without changing bucket props"
+
+ais get "$bucket/lorem-duis" /dev/null --latest 1>/dev/null
+checksum=$(ais ls "$bucket/lorem-duis" --cached -H -props checksum | awk '{print $2}')
+[[ "$checksum" == "$sum2"  ]] || { echo "FAIL: $checksum != $sum2"; exit 1; }
+
+echo "3.2 restore the state prior to step 3.1"
+
+echo $lorem | ais put - "$bucket/lorem-duis" 1>/dev/null || exit $?
+echo $duis  | s3cmd put - "$bucket/lorem-duis" $host 1>/dev/null || exit $?
+
+######### end of --latest
+
 echo "4. warm GET and check (expecting the first version's checksum)"
 ais get "$bucket/lorem-duis" /dev/null 1>/dev/null
 checksum=$(ais ls "$bucket/lorem-duis" --cached -H -props checksum | awk '{print $2}')

diff --git a/ais/test/scripts/s3-prefetch-latest-prefix.sh b/ais/test/scripts/s3-prefetch-latest-prefix.sh
@@ -95,15 +95,15 @@ ais prefetch "$bucket/lorem-duis" --wait
 checksum=$(ais ls "$bucket/lorem-duis" --cached -H -props checksum | awk '{print $2}')
 [[ "$checksum" == "$sum2"  ]] || { echo "FAIL: $checksum != $sum2"; exit 1; }
 
-echo "11. remember 'remote-deleted' counter and enable version synchronization"
+echo "11. remember 'remote-deleted' counter _and_ enable version synchronization"
 cnt4=$(ais show performance counters --regex REMOTE-DEL -H | awk '{sum+=$2;}END{print sum;}')
 ais bucket props set $bucket versioning.synchronize=true
 
 echo "12. run 'prefetch --latest' one last time, and make sure the object \"disappears\""
 ais prefetch "$bucket/lorem-duis" --latest --wait 2>/dev/null
 [[ $? == 0 ]] || { echo "FAIL: expecting 'prefetch --wait' to return Ok, got $?"; exit 1; }
 
-echo "13. 'remote-deleted' counter must increment"
+echo "13. 'remote-deleted' counter must increment (because 'versioning.synchronize=true')"
 cnt5=$(ais show performance counters --regex REMOTE-DEL -H | awk '{sum+=$2;}END{print sum;}')
 [[ $cnt5 == $(($cnt4+1)) ]] || { echo "FAIL: $cnt5 != $(($cnt4+1))"; exit 1; }
 

diff --git a/ais/test/scripts/s3-prefetch-latest-range.sh b/ais/test/scripts/s3-prefetch-latest-range.sh
@@ -95,15 +95,15 @@ ais prefetch "$bucket/shard-{001..009}" --wait
 checksum=$(ais ls "$bucket/shard-001" --cached -H -props checksum | awk '{print $2}')
 [[ "$checksum" == "$sum2"  ]] || { echo "FAIL: $checksum != $sum2"; exit 1; }
 
-echo "11. remember 'remote-deleted' counter and enable version synchronization"
+echo "11. remember 'remote-deleted' counter _and_ enable version synchronization"
 cnt4=$(ais show performance counters --regex REMOTE-DEL -H | awk '{sum+=$2;}END{print sum;}')
 ais bucket props set $bucket versioning.synchronize=true
 
-echo "12. run 'prefetch --latest' one last time, and make sure the object \"disappears\""
+echo "12. run 'prefetch --latest' one last time; make sure the object \"disappears\""
 ais prefetch "$bucket/shard-{001..009}" --latest --wait 2>/dev/null
 [[ $? == 0 ]] || { echo "FAIL: expecting 'prefetch --wait' to return Ok, got $?"; exit 1; }
 
-echo "13. 'remote-deleted' counter must increment"
+echo "13. 'remote-deleted' counter must increment (because 'versioning.synchronize=true')"
 cnt5=$(ais show performance counters --regex REMOTE-DEL -H | awk '{sum+=$2;}END{print sum;}')
 [[ $cnt5 == $(($cnt4+1)) ]] || { echo "FAIL: $cnt5 != $(($cnt4+1))"; exit 1; }