WIP

Signed-off-by: Miloslav Trmač <[email protected]>

storage/storage_dest.go

@@ -343,6 +343,38 @@ func (f *zstdFetcher) GetBlobAt(chunks []chunked.ImageSourceChunk) (chan io.Read
 // If the call fails with ErrFallbackToOrdinaryLayerDownload, the caller can fall back to PutBlobWithOptions.
 // The fallback _must not_ be done otherwise.
 func (s *storageImageDestination) PutBlobPartial(ctx context.Context, chunkAccessor private.BlobChunkAccessor, srcInfo types.BlobInfo, options private.PutBlobPartialOptions) (_ private.UploadedBlob, retErr error) {
+	// The identity of partially-pulled layers is, as long as we keep compatibility with tar-like consumers,
+	// unfixably ambiguous: there are two possible “views” of the same file (same compressed digest),
+	// the traditional “view” that decompresses the primary stream and consumes a tar file,
+	// and the partial-pull “view” that starts with the TOC.
+	// The two “views” have two separate metadata sets, and the only way to ensure they are consistent would be to
+	// read the full primary stream (and authenticate it against the compressed digest), and ensure the metadata
+	// and layer contents exactly match the partially-pulled contents - making the partial
+	// pull completely pointless.
+	//
+	// Instead, we require the image to “commit” to uncompressed layer digest values via the config's RootFS.DiffIDs array:
+	// they are already naturally computed for traditionally-pulled layers, and for partially-pulled layers we
+	// do the optimal partial pull, and then reconstruct the uncompressed tar stream just to (expensively) compute this digest.
+	//
+	// So, if the input image doesn't have RootFS.DiffIDs (i.e. is schema1), don't bother with partial pulls at all,
+	// and always use the traditional “view”.
+	// (Hypothetically, the user can opt out of the DiffID commitment by a c/storage option, giving up security for performance,
+	// but schema1 images don't support layer annotations anyway, so neither zstd:chunked nor estargz layers can
+	// benefit from partial pulls anyway — so this is not giving up on any functionality.)
+	untrustedDiffID, err := s.untrustedLayerDiffID(options.LayerIndex)
+	if err != nil {
+		if errors.Is(err, errUntrustedLayerDiffIDNotYetAvailable) {
+			// PutBlobPartial is a private API, so all callers are within c/image, and should have called
+			// NoteOriginalOCIConfig first.
+			return private.UploadedBlob{}, fmt.Errorf("internal error: in PutBlobPartial, untrustedLayerDiffID returned errUntrustedLayerDiffIDNotYetAvailable")
+		}
+		var e untrustedLayerDiffIDUnknownError
+		if errors.As(err, &e) {
+			return private.UploadedBlob{}, private.NewErrFallbackToOrdinaryLayerDownload(err)
+		}
+		return private.UploadedBlob{}, err
+	}
+
 	fetcher := zstdFetcher{
 		chunkAccessor: chunkAccessor,
 		ctx:           ctx,
@@ -379,6 +411,7 @@ func (s *storageImageDestination) PutBlobPartial(ctx context.Context, chunkAcces
 	blobDigest := srcInfo.Digest
 
 	s.lock.Lock()
+	// For true partial pulls, c/storage decides whether to compute the uncompressed digest based on an option in the config file.
 	if out.UncompressedDigest != "" {
 		s.lockProtected.indexToDiffID[options.LayerIndex] = out.UncompressedDigest
 		if out.TOCDigest != "" {