list-objects to report 'version-changed' (new)

* when size doesn't change ... * part two, prev. commit: 4fc8fa0 Signed-off-by: Alex Aizman <[email protected]>
NVIDIA · Jan 12, 2024 · 821eb0f · 821eb0f
1 parent 4fc8fa0
commit 821eb0f
Show file tree

Hide file tree

Showing 7 changed files with 96 additions and 21 deletions.
diff --git a/cmd/cli/cli/bucket_hdlr.go b/cmd/cli/cli/bucket_hdlr.go
@@ -101,6 +101,7 @@ var (
 			unitsFlag,
 			silentFlag,
 			dontWaitFlag,
+			verChangedFlag,
 		},
 
 		cmdLRU: {

diff --git a/cmd/cli/cli/const.go b/cmd/cli/cli/const.go
@@ -497,7 +497,7 @@ var (
 			indent4 + "\traw - do not convert to (or from) human-readable format",
 	}
 
-	// Bucket
+	// list-objects
 	startAfterFlag = cli.StringFlag{
 		Name:  "start-after",
 		Usage: "list bucket's content alphabetically starting with the first name _after_ the specified",
@@ -523,7 +523,16 @@ var (
 		Name:  "paged",
 		Usage: "list objects page by page, one page at a time (see also '--page-size' and '--limit')",
 	}
-	showUnmatchedFlag = cli.BoolFlag{Name: "show-unmatched", Usage: "list also objects that were _not_ matched by regex and/or template (range)"}
+	showUnmatchedFlag = cli.BoolFlag{
+		Name:  "show-unmatched",
+		Usage: "list also objects that were _not_ matched by regex and/or template (range)",
+	}
+	verChangedFlag = cli.BoolFlag{
+		Name: "check-version",
+		Usage: "check whether listed remote objects and their in-cluster copies are identical, ie., have the same versions\n" +
+			indent4 + "\t- applies to remote buckets only\n" +
+			indent4 + "\t- see related: 'ais get --latest', 'ais cp --sync', 'ais prefetch --latest'",
+	}
 
 	keepMDFlag       = cli.BoolFlag{Name: "keep-md", Usage: "keep bucket metadata"}
 	dataSlicesFlag   = cli.IntFlag{Name: "data-slices,data,d", Usage: "number of data slices", Required: true}

diff --git a/cmd/cli/cli/ls.go b/cmd/cli/cli/ls.go
@@ -271,8 +271,17 @@ func listObjects(c *cli.Context, bck cmn.Bck, prefix string, listArch bool) erro
 	if bck.IsRemote() {
 		addCachedCol = true
 		msg.SetFlag(apc.LsBckPresent) // default
+		if flagIsSet(c, verChangedFlag) {
+			msg.SetFlag(apc.LsVerChanged)
+		}
+	} else if flagIsSet(c, verChangedFlag) {
+		return fmt.Errorf("flag %s applies to remote buckets only (have: %s)", qflprn(verChangedFlag), bck)
 	}
+
 	if flagIsSet(c, listObjCachedFlag) {
+		if flagIsSet(c, verChangedFlag) {
+			return fmt.Errorf(errFmtExclusive, qflprn(verChangedFlag), qflprn(listObjCachedFlag))
+		}
 		msg.SetFlag(apc.LsObjCached)
 		addCachedCol = false // redundant
 	}
@@ -302,11 +311,13 @@ func listObjects(c *cli.Context, bck cmn.Bck, prefix string, listArch bool) erro
 		debug.Assert(apc.LsPropsSepa == ",", "',' is documented in 'objPropsFlag' usage and elsewhere")
 		props = splitCsv(propsStr) // split apc.LsPropsSepa
 	}
-	// NOTE: compare w/ `showObjProps()`
+
 	if flagIsSet(c, nameOnlyFlag) {
+		if flagIsSet(c, verChangedFlag) {
+			return fmt.Errorf(errFmtExclusive, qflprn(verChangedFlag), qflprn(nameOnlyFlag))
+		}
 		if len(props) > 2 {
-			warn := fmt.Sprintf("flag %s is incompatible with the value of %s",
-				qflprn(nameOnlyFlag), qflprn(objPropsFlag))
+			warn := fmt.Sprintf("flag %s is incompatible with the value of %s", qflprn(nameOnlyFlag), qflprn(objPropsFlag))
 			actionWarn(c, warn)
 		}
 		msg.SetFlag(apc.LsNameOnly)
@@ -316,6 +327,8 @@ func listObjects(c *cli.Context, bck cmn.Bck, prefix string, listArch bool) erro
 			msg.AddProps(apc.GetPropsName)
 			msg.AddProps(apc.GetPropsSize)
 			msg.SetFlag(apc.LsNameSize)
+		} else if flagIsSet(c, verChangedFlag) {
+			msg.AddProps(apc.GetPropsDefaultCloud...) // TODO: maybe too crude
 		} else {
 			msg.AddProps(apc.GetPropsMinimal...)
 		}
@@ -359,7 +372,8 @@ func listObjects(c *cli.Context, bck cmn.Bck, prefix string, listArch bool) erro
 			} else {
 				toPrint = objList.Entries
 			}
-			err = printLso(c, toPrint, lstFilter, msg.Props, addCachedCol)
+			err = printLso(c, toPrint, lstFilter, msg.Props,
+				addCachedCol, bck.IsRemote(), msg.IsFlagSet(apc.LsVerChanged))
 			if err != nil {
 				return err
 			}
@@ -397,7 +411,8 @@ func listObjects(c *cli.Context, bck cmn.Bck, prefix string, listArch bool) erro
 	if err != nil {
 		return lsoErr(msg, err)
 	}
-	return printLso(c, objList.Entries, lstFilter, msg.Props, addCachedCol)
+	return printLso(c, objList.Entries, lstFilter, msg.Props,
+		addCachedCol, bck.IsRemote(), msg.IsFlagSet(apc.LsVerChanged))
 }
 
 func lsoErr(msg *apc.LsoMsg, err error) error {
@@ -436,24 +451,26 @@ func _setPage(c *cli.Context, bck cmn.Bck) (pageSize, limit int, err error) {
 }
 
 // NOTE: in addition to CACHED, may also dynamically add STATUS column
-func printLso(c *cli.Context, entries cmn.LsoEntries, lstFilter *lstFilter, props string, addCachedCol bool) error {
+func printLso(c *cli.Context, entries cmn.LsoEntries, lstFilter *lstFilter, props string,
+	addCachedCol, isRemote, addStatusCol bool) error {
 	var (
 		hideHeader     = flagIsSet(c, noHeaderFlag)
 		hideFooter     = flagIsSet(c, noFooterFlag)
 		matched, other = lstFilter.apply(entries)
 		units, errU    = parseUnitsFlag(c, unitsFlag)
-		addStatusCol   bool
 	)
 	if errU != nil {
 		return errU
 	}
 
 	propsList := splitCsv(props)
-	if addCachedCol && !cos.StringInSlice(apc.GetPropsStatus, propsList) {
-		for _, e := range entries {
-			if e.IsVerChanged() {
-				addStatusCol = true
-				break
+	if isRemote && !addStatusCol {
+		if addCachedCol && !cos.StringInSlice(apc.GetPropsStatus, propsList) {
+			for _, e := range entries {
+				if e.IsVerChanged() {
+					addStatusCol = true
+					break
+				}
 			}
 		}
 	}

diff --git a/cmd/cli/go.mod b/cmd/cli/go.mod
@@ -4,7 +4,7 @@ go 1.21
 
 // direct
 require (
-	github.com/NVIDIA/aistore v1.3.22-0.20240111170850-3e694cd1def0
+	github.com/NVIDIA/aistore v1.3.22-0.20240111185148-4fc8fa04104b
 	github.com/fatih/color v1.16.0
 	github.com/json-iterator/go v1.1.12
 	github.com/onsi/ginkgo v1.16.5

diff --git a/cmd/cli/go.sum b/cmd/cli/go.sum
@@ -1,7 +1,7 @@
 code.cloudfoundry.org/bytefmt v0.0.0-20190710193110-1eb035ffe2b6/go.mod h1:wN/zk7mhREp/oviagqUXY3EwuHhWyOvAdsn5Y4CzOrc=
 github.com/BurntSushi/toml v1.3.2/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ=
-github.com/NVIDIA/aistore v1.3.22-0.20240111170850-3e694cd1def0 h1:q3lxEeA6c3KM8VMrNH4S+nGEB/f739R/2+CCI55AnAs=
-github.com/NVIDIA/aistore v1.3.22-0.20240111170850-3e694cd1def0/go.mod h1:jpWmGuqxnY+akx81S5eqHhGdgSENm0mVYRrVbpCW4/I=
+github.com/NVIDIA/aistore v1.3.22-0.20240111185148-4fc8fa04104b h1:aNuTMxkImvqn6LNXdKz7Tjlba4IKjn16+PoLzqFn9gY=
+github.com/NVIDIA/aistore v1.3.22-0.20240111185148-4fc8fa04104b/go.mod h1:jpWmGuqxnY+akx81S5eqHhGdgSENm0mVYRrVbpCW4/I=
 github.com/OneOfOne/xxhash v1.2.8 h1:31czK/TI9sNkxIKfaUfGlU47BAxQ0ztGgd9vPyqimf8=
 github.com/OneOfOne/xxhash v1.2.8/go.mod h1:eZbhyaAYD41SGSSsnmcpxVoRiQ/MPUTjUdIIOT9Um7Q=
 github.com/VividCortex/ewma v1.1.1/go.mod h1:2Tkkvm3sRDVXaiyucHiACn4cqf7DpdyLvmxzcbUokwA=

diff --git a/cmn/objattrs.go b/cmn/objattrs.go
@@ -93,6 +93,31 @@ func (oa *ObjAttrs) SetSize(size int64) {
 
 func CustomMD2S(md cos.StrKVs) string { return fmt.Sprintf("%+v", md) }
 
+func S2CustomMD(custom string) (md cos.StrKVs) {
+	if len(custom) < 8 || !strings.HasPrefix(custom, "map[") { // Sprintf above
+		return nil
+	}
+	s := custom[4 : len(custom)-1]
+	lst := strings.Split(s, " ")
+	md = make(cos.StrKVs, len(lst))
+	parseCustom(md, lst, VersionObjMD)
+	parseCustom(md, lst, SourceObjMD)
+	parseCustom(md, lst, CRC32CObjMD)
+	parseCustom(md, lst, MD5ObjMD)
+	parseCustom(md, lst, ETag)
+	return md
+}
+
+func parseCustom(md cos.StrKVs, lst []string, key string) {
+	keyX := key + ":"
+	for _, kv := range lst {
+		if strings.HasPrefix(kv, keyX) {
+			md[key] = kv[len(keyX):]
+			return
+		}
+	}
+}
+
 func (oa *ObjAttrs) GetCustomMD() cos.StrKVs   { return oa.CustomMD }
 func (oa *ObjAttrs) SetCustomMD(md cos.StrKVs) { oa.CustomMD = md }
 
@@ -183,6 +208,14 @@ func (oa *ObjAttrs) FromHeader(hdr http.Header) (cksum *cos.Cksum) {
 	return
 }
 
+func (oa *ObjAttrs) FromLsoEntry(e *LsoEntry) {
+	oa.Size = e.Size
+	oa.Ver = e.Version
+
+	// entry.Custom = cmn.CustomMD2S(custom)
+	_ = CustomMD2S(nil)
+}
+
 // local <=> remote equality in the context of cold-GET and download. This function
 // decides whether we need to go ahead and re-read the object from its remote location.
 //

diff --git a/xact/xs/wanted_lso.go b/xact/xs/wanted_lso.go
@@ -48,10 +48,10 @@ func (wi *walkInfo) setWanted(e *cmn.LsoEntry, lom *core.LOM) {
 		case apc.GetPropsCached: // via obj.SetPresent()
 
 		case apc.GetPropsSize:
-			if lom.Bucket().IsRemote() && e.Size > 0 && lom.SizeBytes() != e.Size {
-				e.SetVerChanged()
-			} else if wi.msg.IsFlagSet(apc.LsVerChanged) {
-				debug.Assert(false, "'LsVerChanged' not implemented yet") // TODO -- FIXME
+			if lom.Bucket().IsRemote() { // TODO: micro-optimize
+				if e.Size > 0 && lom.SizeBytes() != e.Size {
+					e.SetVerChanged()
+				}
 			}
 			e.Size = lom.SizeBytes()
 		case apc.GetPropsVersion:
@@ -75,4 +75,19 @@ func (wi *walkInfo) setWanted(e *cmn.LsoEntry, lom *core.LOM) {
 			debug.Assert(false, name)
 		}
 	}
+	if !wi.msg.IsFlagSet(apc.LsVerChanged) || e.IsVerChanged() || !lom.Bucket().IsRemote() /*ditto*/ {
+		return
+	}
+	//
+	// extensive version-changed check
+	//
+	md := cmn.S2CustomMD(e.Custom)
+	if len(md) > 0 {
+		var oa cmn.ObjAttrs
+		oa.CustomMD = md
+		oa.Size = e.Size
+		if !lom.Equal(&oa) {
+			e.SetVerChanged()
+		}
+	}
 }