[config change] versioning vs remote version changed or deleted

* `versioning.synchronize` (formerly, `versioning.sync_warm_get`)
  - prev. commit: 931b611
* unify; add comments, references
* extend copy-bucket control (to be continued)

Signed-off-by: Alex Aizman <[email protected]>

ais/test/scripts/remais-get-validate.sh

@@ -56,14 +56,14 @@ ais show bucket $bucket --add 1>/dev/null || exit 1
 ## remember existing bucket 'validate_warm_get' setting; disable if need be
 validate=$(ais bucket props show ${bucket} versioning.validate_warm_get -H | awk '{print $2}')
 [[ "$validate" == "false"  ]] || ais bucket props set $bucket versioning.validate_warm_get=false
-sync=$(ais bucket props show ${bucket} versioning.sync_warm_get -H | awk '{print $2}')
-[[ "$sync" == "false"  ]] || ais bucket props set $bucket versioning.sync_warm_get=false
+sync=$(ais bucket props show ${bucket} versioning.synchronize -H | awk '{print $2}')
+[[ "$sync" == "false"  ]] || ais bucket props set $bucket versioning.synchronize=false
 
 cleanup() {
   rc=$?
   ais object rm "$bucket/lorem-duis" 1>/dev/null 2>&1
   [[ "$validate" == "true"  ]] || ais bucket props set $bucket versioning.validate_warm_get=false 1>/dev/null 2>&1
-  [[ "$sync" == "true"  ]] || ais bucket props set $bucket versioning.sync_warm_get=false 1>/dev/null 2>&1
+  [[ "$sync" == "true"  ]] || ais bucket props set $bucket versioning.synchronize=false 1>/dev/null 2>&1
   [[ "$exists" == "true" ]] || ais rmb $bucket -y 1>/dev/null 2>&1
   exit $rc
 }
@@ -122,8 +122,8 @@ ais bucket props set $bucket versioning.validate_warm_get=false
 ais get "$bucket/lorem-duis" /dev/null --silent 1>/dev/null 2>&1
 [[ $? == 0 ]] || { echo "FAIL: expecting warm GET to succeed, got $?"; exit 1; }
 
-echo "12. remember 'remote-deleted' counter and update bucket props: enable sync-warm-get"
-ais bucket props set $bucket versioning.sync_warm_get=true
+echo "12. remember 'remote-deleted' counter and enable version synchronization"
+ais bucket props set $bucket versioning.synchronize=true
 
 cnt4=$(ais show performance counters --regex DELETED -H | awk '{sum+=$2;}END{print sum;}')
 

ais/test/scripts/remais-prefetch-latest.sh

@@ -53,17 +53,17 @@ AIS_ENDPOINT=$rendpoint ais show bucket $rbucket -c 1>/dev/null 2>&1 || exists=f
 ## add it to _this_ cluster's BMD
 ais show bucket $bucket --add 1>/dev/null || exit 1
 
-## remember existing bucket 'validate_warm_get' setting; disable if need be
+## remember existing bucket 'versioning' settings; disable synchronization if need be
 validate=$(ais bucket props show ${bucket} versioning.validate_warm_get -H | awk '{print $2}')
 [[ "$validate" == "false"  ]] || ais bucket props set $bucket versioning.validate_warm_get=false
-sync=$(ais bucket props show ${bucket} versioning.sync_warm_get -H | awk '{print $2}')
-[[ "$sync" == "false"  ]] || ais bucket props set $bucket versioning.sync_warm_get=false
+sync=$(ais bucket props show ${bucket} versioning.synchronize -H | awk '{print $2}')
+[[ "$sync" == "false"  ]] || ais bucket props set $bucket versioning.synchronize=false
 
 cleanup() {
   rc=$?
   ais object rm "$bucket/lorem-duis" 1>/dev/null 2>&1
   [[ "$validate" == "true"  ]] || ais bucket props set $bucket versioning.validate_warm_get=false 1>/dev/null 2>&1
-  [[ "$sync" == "true"  ]] || ais bucket props set $bucket versioning.sync_warm_get=false 1>/dev/null 2>&1
+  [[ "$sync" == "true"  ]] || ais bucket props set $bucket versioning.synchronize=false 1>/dev/null 2>&1
   [[ "$exists" == "true" ]] || ais rmb $bucket -y 1>/dev/null 2>&1
   exit $rc
 }
@@ -118,9 +118,9 @@ ais prefetch "$bucket/lorem-duis" --wait
 checksum=$(ais ls "$bucket/lorem-duis" --cached -H -props checksum | awk '{print $2}')
 [[ "$checksum" == "$sum2"  ]] || { echo "FAIL: $checksum != $sum2"; exit 1; }
 
-echo "11. remember 'remote-deleted' counter and update bucket props: set sync-warm-get = true"
+echo "11. remember 'remote-deleted' counter, enable version synchronization on the bucket"
 cnt4=$(ais show performance counters --regex REMOTE-DEL -H | awk '{sum+=$2;}END{print sum;}')
-ais bucket props set $bucket versioning.sync_warm_get=true
+ais bucket props set $bucket versioning.synchronize=true
 
 echo "12. run 'prefetch --latest' one last time, and make sure the object \"disappears\""
 ais prefetch "$bucket/lorem-duis" --latest --wait 2>/dev/null

ais/test/scripts/s3-get-validate.sh

@@ -37,14 +37,14 @@ ais show bucket $bucket -c 1>/dev/null || exit $?
 ## remember existing bucket's versioning; disable if need be
 validate=$(ais bucket props show ${bucket} versioning.validate_warm_get -H | awk '{print $2}')
 [[ "$validate" == "false"  ]] || ais bucket props set $bucket versioning.validate_warm_get=false
-sync=$(ais bucket props show ${bucket} versioning.sync_warm_get -H | awk '{print $2}')
-[[ "$sync" == "false"  ]] || ais bucket props set $bucket versioning.sync_warm_get=false
+sync=$(ais bucket props show ${bucket} versioning.synchronize -H | awk '{print $2}')
+[[ "$sync" == "false"  ]] || ais bucket props set $bucket versioning.synchronize=false
 
 cleanup() {
   rc=$?
   ais object rm "$bucket/lorem-duis" 1>/dev/null 2>&1
   [[ "$validate" == "true"  ]] || ais bucket props set $bucket versioning.validate_warm_get=false 1>/dev/null 2>&1
-  [[ "$sync" == "true"  ]] || ais bucket props set $bucket versioning.sync_warm_get=false 1>/dev/null 2>&1
+  [[ "$sync" == "true"  ]] || ais bucket props set $bucket versioning.synchronize=false 1>/dev/null 2>&1
   exit $rc
 }
 
@@ -103,8 +103,8 @@ echo "11. warm GET must be fine"
 ais get "$bucket/lorem-duis" /dev/null --silent 1>/dev/null 2>&1
 [[ $? == 0 ]] || { echo "FAIL: expecting warm GET to succeed, got $?"; exit 1; }
 
-echo "12. remember 'remote-deleted' counter and update bucket props: set sync-warm-get = true"
-ais bucket props set $bucket versioning.sync_warm_get=true
+echo "12. remember 'remote-deleted' counter and enable version synchronization"
+ais bucket props set $bucket versioning.synchronize=true
 
 cnt4=$(ais show performance counters --regex DELETED -H | awk '{sum+=$2;}END{print sum;}')
 

ais/test/scripts/s3-prefetch-latest-prefix.sh

@@ -35,13 +35,13 @@ done
 ais show bucket $bucket -c 1>/dev/null || exit $?
 
 ## remember existing bucket's versioning; disable if need be
-sync=$(ais bucket props show ${bucket} versioning.sync_warm_get -H | awk '{print $2}')
-[[ "$sync" == "false"  ]] || ais bucket props set $bucket versioning.sync_warm_get=false
+sync=$(ais bucket props show ${bucket} versioning.synchronize -H | awk '{print $2}')
+[[ "$sync" == "false"  ]] || ais bucket props set $bucket versioning.synchronize=false
 
 cleanup() {
   rc=$?
   ais object rm "$bucket/lorem-duis" 1>/dev/null 2>&1
-  [[ "$sync" == "true"  ]] || ais bucket props set $bucket versioning.sync_warm_get=false 1>/dev/null 2>&1
+  [[ "$sync" == "true"  ]] || ais bucket props set $bucket versioning.synchronize=false 1>/dev/null 2>&1
   exit $rc
 }
 
@@ -95,9 +95,9 @@ ais prefetch "$bucket/lorem-duis" --wait
 checksum=$(ais ls "$bucket/lorem-duis" --cached -H -props checksum | awk '{print $2}')
 [[ "$checksum" == "$sum2"  ]] || { echo "FAIL: $checksum != $sum2"; exit 1; }
 
-echo "11. remember 'remote-deleted' counter and update bucket props: set sync-warm-get = true"
+echo "11. remember 'remote-deleted' counter and enable version synchronization"
 cnt4=$(ais show performance counters --regex REMOTE-DEL -H | awk '{sum+=$2;}END{print sum;}')
-ais bucket props set $bucket versioning.sync_warm_get=true
+ais bucket props set $bucket versioning.synchronize=true
 
 echo "12. run 'prefetch --latest' one last time, and make sure the object \"disappears\""
 ais prefetch "$bucket/lorem-duis" --latest --wait 2>/dev/null

ais/test/scripts/s3-prefetch-latest-range.sh

@@ -35,13 +35,13 @@ done
 ais show bucket $bucket -c 1>/dev/null || exit $?
 
 ## remember existing bucket's versioning; disable if need be
-sync=$(ais bucket props show ${bucket} versioning.sync_warm_get -H | awk '{print $2}')
-[[ "$sync" == "false"  ]] || ais bucket props set $bucket versioning.sync_warm_get=false
+sync=$(ais bucket props show ${bucket} versioning.synchronize -H | awk '{print $2}')
+[[ "$sync" == "false"  ]] || ais bucket props set $bucket versioning.synchronize=false
 
 cleanup() {
   rc=$?
   ais object rm "$bucket/shard-001" 1>/dev/null 2>&1
-  [[ "$sync" == "true"  ]] || ais bucket props set $bucket versioning.sync_warm_get=false 1>/dev/null 2>&1
+  [[ "$sync" == "true"  ]] || ais bucket props set $bucket versioning.synchronize=false 1>/dev/null 2>&1
   exit $rc
 }
 
@@ -95,9 +95,9 @@ ais prefetch "$bucket/shard-{001..009}" --wait
 checksum=$(ais ls "$bucket/shard-001" --cached -H -props checksum | awk '{print $2}')
 [[ "$checksum" == "$sum2"  ]] || { echo "FAIL: $checksum != $sum2"; exit 1; }
 
-echo "11. remember 'remote-deleted' counter and update bucket props: set sync-warm-get = true"
+echo "11. remember 'remote-deleted' counter and enable version synchronization"
 cnt4=$(ais show performance counters --regex REMOTE-DEL -H | awk '{sum+=$2;}END{print sum;}')
-ais bucket props set $bucket versioning.sync_warm_get=true
+ais bucket props set $bucket versioning.synchronize=true
 
 echo "12. run 'prefetch --latest' one last time, and make sure the object \"disappears\""
 ais prefetch "$bucket/shard-{001..009}" --latest --wait 2>/dev/null

api/apc/multiobj.go

@@ -4,8 +4,6 @@
  */
 package apc
 
-// TODO: ContinueOnError vs. cmn.SupportedReactions - unify
-
 type (
 	// List of object names _or_ a template specifying { optional Prefix, zero or more Ranges }
 	ListRange struct {
@@ -15,7 +13,7 @@ type (
 	PrefetchMsg struct {
 		ListRange
 		ContinueOnError bool `json:"coer"`
-		LatestVer       bool `json:"latest-ver"` // see also: QparamLatestVer, 'versioning.*_warm_get'
+		LatestVer       bool `json:"latest-ver"` // see also: QparamLatestVer, 'versioning.validate_warm_get'
 	}
 
 	// ArchiveMsg contains the parameters (all except the destination bucket)

api/apc/tcb.go

@@ -14,11 +14,12 @@ import (
 // copy & (offline) transform bucket to bucket
 type (
 	CopyBckMsg struct {
-		Prepend   string `json:"prepend"`    // destination naming, as in: dest-obj-name = Prepend + source-obj-name
-		Prefix    string `json:"prefix"`     // prefix to select matching _source_ objects or virtual directories
-		DryRun    bool   `json:"dry_run"`    // visit all source objects, don't make any modifications
-		Force     bool   `json:"force"`      // force running in presence of "limited coexistence" type conflicts
-		LatestVer bool   `json:"latest-ver"` // see also: QparamLatestVer, 'versioning.*_warm_get'
+		Prepend   string `json:"prepend"`     // destination naming, as in: dest-obj-name = Prepend + source-obj-name
+		Prefix    string `json:"prefix"`      // prefix to select matching _source_ objects or virtual directories
+		DryRun    bool   `json:"dry_run"`     // visit all source objects, don't make any modifications
+		Force     bool   `json:"force"`       // force running in presence of "limited coexistence" type conflicts
+		LatestVer bool   `json:"latest-ver"`  // see also: QparamLatestVer, 'versioning.validate_warm_get', PrefetchMsg
+		Sync      bool   `json:"synchronize"` // see also: 'versioning.synchronize'
 	}
 	Transform struct {
 		Name    string       `json:"id,omitempty"`

cmd/cli/cli/bucket_hdlr.go

@@ -189,7 +189,7 @@ var (
 		Usage: "update bucket properties; the command accepts both JSON-formatted input and plain Name=Value pairs, e.g.:\n" +
 			indent1 + "\t* ais bucket props set ais://nnn backend_bck=s3://mmm\n" +
 			indent1 + "\t* ais bucket props set ais://nnn backend_bck=none\n" +
-			indent1 + "\t* ais bucket props set gs://vvv versioning.validate_warm_get=false versioning.sync_warm_get=true\n" +
+			indent1 + "\t* ais bucket props set gs://vvv versioning.validate_warm_get=false versioning.synchronize=true\n" +
 			indent1 + "\t* ais bucket props set gs://vvv mirror.enabled=true mirror.copies=4 checksum.type=md5\n" +
 			indent1 + "\t* ais bucket props set s3://mmm ec.enabled true ec.data_slices 6 ec.parity_slices 4 --force\n" +
 			indent1 + "\tReferences:\n" +

cmd/cli/cli/const.go

@@ -488,7 +488,7 @@ var (
 	bucketPropsFlag = cli.StringFlag{
 		Name: "props",
 		Usage: "create bucket with the specified (non-default) properties, e.g.:\n" +
-			indent1 + "\t* ais create ais://mmm --props=\"versioning.validate_warm_get=false versioning.sync_warm_get=true\"\n" +
+			indent1 + "\t* ais create ais://mmm --props=\"versioning.validate_warm_get=false versioning.synchronize=true\"\n" +
 			indent1 + "\t* ais create ais://nnn --props='mirror.enabled=true mirror.copies=4 checksum.type=md5'\n" +
 			indent1 + "\t(tip: use '--props' to override properties that a new bucket inherits from cluster config at creation time;\n" +
 			indent1 + "\t see also: 'ais bucket props show' and 'ais bucket props set')",

cmd/cli/go.mod

@@ -4,7 +4,7 @@ go 1.21
 
 // direct
 require (
-	github.com/NVIDIA/aistore v1.3.22-0.20240105162653-406ce359fcc2
+	github.com/NVIDIA/aistore v1.3.22-0.20240105234838-8194c9a877ff
 	github.com/fatih/color v1.16.0
 	github.com/json-iterator/go v1.1.12
 	github.com/onsi/ginkgo v1.16.5

cmd/cli/go.sum

@@ -1,7 +1,7 @@
 code.cloudfoundry.org/bytefmt v0.0.0-20190710193110-1eb035ffe2b6/go.mod h1:wN/zk7mhREp/oviagqUXY3EwuHhWyOvAdsn5Y4CzOrc=
 github.com/BurntSushi/toml v1.3.2/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ=
-github.com/NVIDIA/aistore v1.3.22-0.20240105162653-406ce359fcc2 h1:3hHoaIhrzUBCQtFFmLdjbvVPzng94ZNHn4Ve9xOyBEI=
-github.com/NVIDIA/aistore v1.3.22-0.20240105162653-406ce359fcc2/go.mod h1:jpWmGuqxnY+akx81S5eqHhGdgSENm0mVYRrVbpCW4/I=
+github.com/NVIDIA/aistore v1.3.22-0.20240105234838-8194c9a877ff h1:zD3bVhXL4J3neokP/C2OF4mAPoFFuGNjKCAX/75gWck=
+github.com/NVIDIA/aistore v1.3.22-0.20240105234838-8194c9a877ff/go.mod h1:jpWmGuqxnY+akx81S5eqHhGdgSENm0mVYRrVbpCW4/I=
 github.com/OneOfOne/xxhash v1.2.8 h1:31czK/TI9sNkxIKfaUfGlU47BAxQ0ztGgd9vPyqimf8=
 github.com/OneOfOne/xxhash v1.2.8/go.mod h1:eZbhyaAYD41SGSSsnmcpxVoRiQ/MPUTjUdIIOT9Um7Q=
 github.com/VividCortex/ewma v1.1.1/go.mod h1:2Tkkvm3sRDVXaiyucHiACn4cqf7DpdyLvmxzcbUokwA=

cmn/config.go

@@ -384,31 +384,30 @@ type (
 		// Determines if versioning is enabled
 		Enabled bool `json:"enabled"`
 
-		// Validate and, possibly, synchronize(*) remote object's version when reading its
-		// in-cluster ("cached") copy, scenarios including (but not limited to):
+		// Validate remote version and, possibly, update in-cluster ("cached") copy.
+		// Scenarios include (but are not limited to):
 		// - warm GET
-		// - prefetch bucket (**)
+		// - prefetch bucket (*)
 		// - prefetch multiple objects (see api/multiobj.go)
 		// - copy bucket
 		// Applies to Cloud and remote AIS buckets - generally, buckets that have remote backends
-		// that in turn provide some form of object versioning
-		// (*)  Update in-cluster copy to the _latest_
-		// (**) Xactions (jobs) that read multiple objects (e.g., prefetch and copy-bucket)
-		//      provide their own, operation-level, option to syncronize in-cluster versions
-		//      (which doesn't require changing bucket configuration)
+		// that in turn provide some form of object versioning.
+		// (*) Xactions (ie., jobs) that read-access multiple objects (e.g., prefetch, copy-bucket)
+		// may support operation-scope option to synchronize remote content (to aistore) - the option
+		// not requiring changing bucket configuration.
 		// See also:
-		// - apc.QparamLatestVer, "latest-ver"
+		// - apc.QparamLatestVer, apc.PrefetchMsg, apc.CopyBckMsg
 		ValidateWarmGet bool `json:"validate_warm_get"`
 
 		// A stronger variant of the above that in addition entails:
 		// - deleting in-cluster object if its remote ("cached") counterpart does not exist
-		// See also: apc.QparamSync
-		SyncWarmGet bool `json:"sync_warm_get"`
+		// See also: apc.QparamSync, apc.CopyBckMsg
+		Sync bool `json:"synchronize"`
 	}
 	VersionConfToSet struct {
 		Enabled         *bool `json:"enabled,omitempty"`
 		ValidateWarmGet *bool `json:"validate_warm_get,omitempty"`
-		SyncWarmGet     *bool `json:"sync_warm_get,omitempty"`
+		Sync            *bool `json:"synchronize,omitempty"`
 	}
 
 	NetConf struct {

cmn/tests/iter_fields_test.go

@@ -79,7 +79,7 @@ var _ = Describe("IterFields", func() {
 
 					"versioning.enabled":           false,
 					"versioning.validate_warm_get": false,
-					"versioning.sync_warm_get":     false,
+					"versioning.synchronize":       false,
 
 					"checksum.type":              cos.ChecksumXXHash,
 					"checksum.validate_warm_get": false,
@@ -135,7 +135,7 @@ var _ = Describe("IterFields", func() {
 
 					"versioning.enabled":           (*bool)(nil),
 					"versioning.validate_warm_get": (*bool)(nil),
-					"versioning.sync_warm_get":     (*bool)(nil),
+					"versioning.synchronize":       (*bool)(nil),
 
 					"checksum.type":              apc.String(cos.ChecksumXXHash),
 					"checksum.validate_warm_get": (*bool)(nil),

core/ldp.go

@@ -127,7 +127,7 @@ func (lom *LOM) CheckRemoteMD(rlocked bool) (bool, int, error) {
 		return lom.Equal(oa), errCode, nil
 	}
 
-	if errCode == http.StatusNotFound && lom.VersionConf().SyncWarmGet {
+	if errCode == http.StatusNotFound && lom.VersionConf().Sync {
 		errDel := lom.Remove(rlocked /*force through rlock*/)
 		if errDel != nil {
 			errCode, err = 0, errDel

core/lom.go

@@ -143,7 +143,7 @@ func (lom *LOM) ValidateWarmGet(qparam string /*apc.QparamLatestVer*/) bool {
 	case !lom.Bck().IsCloud() && !lom.Bck().IsRemoteAIS():
 		return false
 	case qparam == "":
-		return lom.VersionConf().ValidateWarmGet || lom.VersionConf().SyncWarmGet // bucket prop
+		return lom.VersionConf().ValidateWarmGet || lom.VersionConf().Sync // bucket prop
 	case qparam == "true":
 		return true
 	default:

core/target.go

@@ -115,7 +115,7 @@ type (
 		OWT        cmn.OWT
 		Finalize   bool // copies and EC (as in poi.finalize())
 		DryRun     bool
-		SyncRemote bool // use instead of bucket's 'versioning.sync_warm_get'
+		SyncRemote bool // use instead of bucket's 'versioning.synchronize'
 		LatestVer  bool // use instead of bucket's 'versioning.validate_warm_get'; see also: QparamLatestVer
 	}
 )

docs/out_of_band.md

@@ -79,7 +79,7 @@ $ ais bucket props show BUCKET versioning
 PROPERTY                         VALUE
 versioning.enabled               ...
 versioning.validate_warm_get     true
-versioning.sync_warm_get         false
+versioning.synchronize           false
 ```
 
 No assumption is being made on whether any of the above is present (except, of course, the size aka "Content-Length").
@@ -113,18 +113,18 @@ $ ais bucket props show BUCKET versioning
 PROPERTY                         VALUE
 versioning.enabled               ...
 versioning.validate_warm_get     true
-versioning.sync_warm_get         false  ## <<<<<<<<<<<<<<<< note!
+versioning.synchronize           false  ## <<<<<<<<<<<<<<<< note!
 ```
 
-The knob called `versioning.sync_warm_get` is simply a stronger variant of the `versioning.validate_warm_get`;
+The knob called `versioning.synchronize` is simply a stronger variant of the `versioning.validate_warm_get`;
 that entails both:
 
 1. validating remote object version, and
 2. deleting in-cluster object if its remote ("cached") counterpart does not exist.
 
 To recap:
 
-if an attempt to read remote metadata returns "object not found", and `versioning.sync_warm_get` is set to `true`, then
+if an attempt to read remote metadata returns "object not found", and `versioning.synchronize` is set to `true`, then
 we go ahead and delete the object locally, thus effectively _synchronizing_ in-cluster content with it's remote source.
 
 ## GET latest version

ext/dload/api.go

@@ -112,7 +112,7 @@ type (
 		Base
 		Prefix string `json:"prefix"`
 		Suffix string `json:"suffix"`
-		Sync   bool   `json:"sync"`
+		Sync   bool   `json:"synchronize"`
 	}
 
 	SingleBody struct {

ext/dload/dispatcher.go

@@ -203,10 +203,10 @@ func (d *dispatcher) dispatchDownload(job jobif) (ok bool) {
 	diffResolver := NewDiffResolver(&defaultDiffResolverCtx{})
 	go diffResolver.Start()
 
-	// In case of `!job.Sync()` we don't want to traverse the whole bucket.
-	// We just want to download requested objects so we know exactly which
-	// objects must be checked (compared) and which not. Therefore, only traverse
-	// bucket when we need to sync the objects.
+	// In case of `!job.Sync()` we don't want to traverse entire bucket.
+	// We just want to download requested objects and to find out which
+	// objects must be checked (latest version-wise). Therefore, "walk"
+	// bucket only when we need to sync the objects.
 	if job.Sync() {
 		go diffResolver.walk(job)
 	}