Skip to content

Commit

Permalink
Merge pull request #1176 from tonistiigi/cache-refs
Browse files Browse the repository at this point in the history 
Update storage management from root labels to leases
  • Loading branch information
@AkihiroSuda
AkihiroSuda authored Oct 18, 2019
2 parents df3f7e0 + 339d4b2 commit a038b5d
Show file tree
Hide file tree
Showing 27 changed files with 2,000 additions and 831 deletions.
63 changes: 35 additions & 28 deletions cache/blobs/blobs.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,13 +2,12 @@ package blobs

import (
"context"
"time"

"github.com/containerd/containerd/content"
"github.com/containerd/containerd/diff"
"github.com/containerd/containerd/leases"
"github.com/containerd/containerd/mount"
"github.com/moby/buildkit/cache"
"github.com/moby/buildkit/snapshot"
"github.com/moby/buildkit/util/flightcontrol"
"github.com/moby/buildkit/util/winlayers"
digest "github.com/opencontainers/go-digest"
Expand All @@ -28,11 +27,17 @@ type DiffPair struct {

var ErrNoBlobs = errors.Errorf("no blobs for snapshot")

func GetDiffPairs(ctx context.Context, contentStore content.Store, snapshotter snapshot.Snapshotter, differ diff.Comparer, ref cache.ImmutableRef, createBlobs bool) ([]DiffPair, error) {
// GetDiffPairs returns the DiffID/Blobsum pairs for a giver reference and saves it.
// Caller must hold a lease when calling this function.
func GetDiffPairs(ctx context.Context, contentStore content.Store, differ diff.Comparer, ref cache.ImmutableRef, createBlobs bool) ([]DiffPair, error) {
if ref == nil {
return nil, nil
}

if _, ok := leases.FromContext(ctx); !ok {
return nil, errors.Errorf("missing lease requirement for GetDiffPairs")
}

if err := ref.Finalize(ctx, true); err != nil {
return nil, err
}
Expand All @@ -41,22 +46,23 @@ func GetDiffPairs(ctx context.Context, contentStore content.Store, snapshotter s
ctx = winlayers.UseWindowsLayerMode(ctx)
}

return getDiffPairs(ctx, contentStore, snapshotter, differ, ref, createBlobs)
return getDiffPairs(ctx, contentStore, differ, ref, createBlobs)
}

func getDiffPairs(ctx context.Context, contentStore content.Store, snapshotter snapshot.Snapshotter, differ diff.Comparer, ref cache.ImmutableRef, createBlobs bool) ([]DiffPair, error) {
func getDiffPairs(ctx context.Context, contentStore content.Store, differ diff.Comparer, ref cache.ImmutableRef, createBlobs bool) ([]DiffPair, error) {
if ref == nil {
return nil, nil
}

baseCtx := ctx
eg, ctx := errgroup.WithContext(ctx)
var diffPairs []DiffPair
var currentPair DiffPair
var currentDescr ocispec.Descriptor
parent := ref.Parent()
if parent != nil {
defer parent.Release(context.TODO())
eg.Go(func() error {
dp, err := getDiffPairs(ctx, contentStore, snapshotter, differ, parent, createBlobs)
dp, err := getDiffPairs(ctx, contentStore, differ, parent, createBlobs)
if err != nil {
return err
}
Expand All @@ -66,12 +72,9 @@ func getDiffPairs(ctx context.Context, contentStore content.Store, snapshotter s
}
eg.Go(func() error {
dp, err := g.Do(ctx, ref.ID(), func(ctx context.Context) (interface{}, error) {
diffID, blob, err := snapshotter.GetBlob(ctx, ref.ID())
if err != nil {
return nil, err
}
if blob != "" {
return DiffPair{DiffID: diffID, Blobsum: blob}, nil
refInfo := ref.Info()
if refInfo.Blob != "" {
return nil, nil
} else if !createBlobs {
return nil, errors.WithStack(ErrNoBlobs)
}
Expand Down Expand Up @@ -107,9 +110,6 @@ func getDiffPairs(ctx context.Context, contentStore content.Store, snapshotter s
descr, err := differ.Compare(ctx, lower, upper,
diff.WithMediaType(ocispec.MediaTypeImageLayerGzip),
diff.WithReference(ref.ID()),
diff.WithLabels(map[string]string{
"containerd.io/gc.root": time.Now().UTC().Format(time.RFC3339Nano),
}),
)
if err != nil {
return nil, err
Expand All @@ -118,30 +118,37 @@ func getDiffPairs(ctx context.Context, contentStore content.Store, snapshotter s
if err != nil {
return nil, err
}
diffIDStr, ok := info.Labels[containerdUncompressed]
if !ok {
if diffID, ok := info.Labels[containerdUncompressed]; !ok {
return nil, errors.Errorf("invalid differ response with no diffID: %v", descr.Digest)
} else {
if descr.Annotations == nil {
descr.Annotations = map[string]string{}
}
descr.Annotations[containerdUncompressed] = diffID
}
diffIDDigest, err := digest.Parse(diffIDStr)
if err != nil {
return nil, err
}
if err := snapshotter.SetBlob(ctx, ref.ID(), diffIDDigest, descr.Digest); err != nil {
return nil, err
}
return DiffPair{DiffID: diffIDDigest, Blobsum: descr.Digest}, nil
return descr, nil

})
if err != nil {
return err
}
currentPair = dp.(DiffPair)

if dp != nil {
currentDescr = dp.(ocispec.Descriptor)
}
return nil
})
err := eg.Wait()
if err != nil {
return nil, err
}
return append(diffPairs, currentPair), nil
if currentDescr.Digest != "" {
if err := ref.SetBlob(baseCtx, currentDescr); err != nil {
return nil, err
}
}
refInfo := ref.Info()
return append(diffPairs, DiffPair{DiffID: refInfo.DiffID, Blobsum: refInfo.Blob}), nil
}

func isTypeWindows(ref cache.ImmutableRef) bool {
Expand Down
60 changes: 41 additions & 19 deletions cache/contenthash/checksum_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,16 +11,21 @@ import (
"testing"
"time"

"github.com/containerd/containerd/content/local"
ctdmetadata "github.com/containerd/containerd/metadata"
"github.com/containerd/containerd/snapshots"
"github.com/containerd/containerd/snapshots/native"
"github.com/moby/buildkit/cache"
"github.com/moby/buildkit/cache/metadata"
"github.com/moby/buildkit/snapshot"
containerdsnapshot "github.com/moby/buildkit/snapshot/containerd"
"github.com/moby/buildkit/util/leaseutil"
digest "github.com/opencontainers/go-digest"
"github.com/pkg/errors"
"github.com/stretchr/testify/require"
"github.com/tonistiigi/fsutil"
fstypes "github.com/tonistiigi/fsutil/types"
bolt "go.etcd.io/bbolt"
)

const (
Expand All @@ -37,7 +42,7 @@ func TestChecksumSymlinkNoParentScan(t *testing.T) {

snapshotter, err := native.NewSnapshotter(filepath.Join(tmpdir, "snapshots"))
require.NoError(t, err)
cm := setupCacheManager(t, tmpdir, "native", snapshotter)
cm, _ := setupCacheManager(t, tmpdir, "native", snapshotter)
defer cm.Close()

ch := []string{
Expand Down Expand Up @@ -66,7 +71,7 @@ func TestChecksumHardlinks(t *testing.T) {

snapshotter, err := native.NewSnapshotter(filepath.Join(tmpdir, "snapshots"))
require.NoError(t, err)
cm := setupCacheManager(t, tmpdir, "native", snapshotter)
cm, _ := setupCacheManager(t, tmpdir, "native", snapshotter)
defer cm.Close()

ch := []string{
Expand Down Expand Up @@ -149,7 +154,7 @@ func TestChecksumWildcard(t *testing.T) {

snapshotter, err := native.NewSnapshotter(filepath.Join(tmpdir, "snapshots"))
require.NoError(t, err)
cm := setupCacheManager(t, tmpdir, "native", snapshotter)
cm, _ := setupCacheManager(t, tmpdir, "native", snapshotter)
defer cm.Close()

ch := []string{
Expand Down Expand Up @@ -204,7 +209,7 @@ func TestSymlinksNoFollow(t *testing.T) {

snapshotter, err := native.NewSnapshotter(filepath.Join(tmpdir, "snapshots"))
require.NoError(t, err)
cm := setupCacheManager(t, tmpdir, "native", snapshotter)
cm, _ := setupCacheManager(t, tmpdir, "native", snapshotter)
defer cm.Close()

ch := []string{
Expand Down Expand Up @@ -263,7 +268,7 @@ func TestChecksumBasicFile(t *testing.T) {

snapshotter, err := native.NewSnapshotter(filepath.Join(tmpdir, "snapshots"))
require.NoError(t, err)
cm := setupCacheManager(t, tmpdir, "native", snapshotter)
cm, _ := setupCacheManager(t, tmpdir, "native", snapshotter)
defer cm.Close()

ch := []string{
Expand Down Expand Up @@ -413,7 +418,7 @@ func TestHandleChange(t *testing.T) {

snapshotter, err := native.NewSnapshotter(filepath.Join(tmpdir, "snapshots"))
require.NoError(t, err)
cm := setupCacheManager(t, tmpdir, "native", snapshotter)
cm, _ := setupCacheManager(t, tmpdir, "native", snapshotter)
defer cm.Close()

ch := []string{
Expand Down Expand Up @@ -491,7 +496,7 @@ func TestHandleRecursiveDir(t *testing.T) {

snapshotter, err := native.NewSnapshotter(filepath.Join(tmpdir, "snapshots"))
require.NoError(t, err)
cm := setupCacheManager(t, tmpdir, "native", snapshotter)
cm, _ := setupCacheManager(t, tmpdir, "native", snapshotter)
defer cm.Close()

ch := []string{
Expand Down Expand Up @@ -540,7 +545,7 @@ func TestChecksumUnorderedFiles(t *testing.T) {

snapshotter, err := native.NewSnapshotter(filepath.Join(tmpdir, "snapshots"))
require.NoError(t, err)
cm := setupCacheManager(t, tmpdir, "native", snapshotter)
cm, _ := setupCacheManager(t, tmpdir, "native", snapshotter)
defer cm.Close()

ch := []string{
Expand Down Expand Up @@ -593,7 +598,7 @@ func TestSymlinkInPathScan(t *testing.T) {

snapshotter, err := native.NewSnapshotter(filepath.Join(tmpdir, "snapshots"))
require.NoError(t, err)
cm := setupCacheManager(t, tmpdir, "native", snapshotter)
cm, _ := setupCacheManager(t, tmpdir, "native", snapshotter)
defer cm.Close()

ch := []string{
Expand Down Expand Up @@ -624,7 +629,7 @@ func TestSymlinkNeedsScan(t *testing.T) {

snapshotter, err := native.NewSnapshotter(filepath.Join(tmpdir, "snapshots"))
require.NoError(t, err)
cm := setupCacheManager(t, tmpdir, "native", snapshotter)
cm, _ := setupCacheManager(t, tmpdir, "native", snapshotter)
defer cm.Close()

ch := []string{
Expand Down Expand Up @@ -657,7 +662,7 @@ func TestSymlinkAbsDirSuffix(t *testing.T) {

snapshotter, err := native.NewSnapshotter(filepath.Join(tmpdir, "snapshots"))
require.NoError(t, err)
cm := setupCacheManager(t, tmpdir, "native", snapshotter)
cm, _ := setupCacheManager(t, tmpdir, "native", snapshotter)
defer cm.Close()

ch := []string{
Expand All @@ -684,7 +689,7 @@ func TestSymlinkThroughParent(t *testing.T) {

snapshotter, err := native.NewSnapshotter(filepath.Join(tmpdir, "snapshots"))
require.NoError(t, err)
cm := setupCacheManager(t, tmpdir, "native", snapshotter)
cm, _ := setupCacheManager(t, tmpdir, "native", snapshotter)
defer cm.Close()

ch := []string{
Expand Down Expand Up @@ -739,7 +744,7 @@ func TestSymlinkInPathHandleChange(t *testing.T) {

snapshotter, err := native.NewSnapshotter(filepath.Join(tmpdir, "snapshots"))
require.NoError(t, err)
cm := setupCacheManager(t, tmpdir, "native", snapshotter)
cm, _ := setupCacheManager(t, tmpdir, "native", snapshotter)
defer cm.Close()

ch := []string{
Expand Down Expand Up @@ -802,7 +807,7 @@ func TestPersistence(t *testing.T) {

snapshotter, err := native.NewSnapshotter(filepath.Join(tmpdir, "snapshots"))
require.NoError(t, err)
cm := setupCacheManager(t, tmpdir, "native", snapshotter)
cm, closeBolt := setupCacheManager(t, tmpdir, "native", snapshotter)
defer cm.Close()

ch := []string{
Expand Down Expand Up @@ -838,8 +843,10 @@ func TestPersistence(t *testing.T) {

// we can't close snapshotter and open it twice (especially, its internal bbolt store)
cm.Close()
closeBolt()
getDefaultManager().lru.Purge()
cm = setupCacheManager(t, tmpdir, "native", snapshotter)
cm, closeBolt = setupCacheManager(t, tmpdir, "native", snapshotter)
defer closeBolt()
defer cm.Close()

ref, err = cm.Get(context.TODO(), id)
Expand Down Expand Up @@ -872,17 +879,32 @@ func createRef(t *testing.T, cm cache.Manager, files []string) cache.ImmutableRe
return ref
}

func setupCacheManager(t *testing.T, tmpdir string, snapshotterName string, snapshotter snapshots.Snapshotter) cache.Manager {
func setupCacheManager(t *testing.T, tmpdir string, snapshotterName string, snapshotter snapshots.Snapshotter) (cache.Manager, func()) {
md, err := metadata.NewStore(filepath.Join(tmpdir, "metadata.db"))
require.NoError(t, err)

store, err := local.NewStore(tmpdir)
require.NoError(t, err)

db, err := bolt.Open(filepath.Join(tmpdir, "containerdmeta.db"), 0644, nil)
require.NoError(t, err)

mdb := ctdmetadata.NewDB(db, store, map[string]snapshots.Snapshotter{
snapshotterName: snapshotter,
})

cm, err := cache.NewManager(cache.ManagerOpt{
Snapshotter: snapshot.FromContainerdSnapshotter(snapshotterName, snapshotter, nil),
MetadataStore: md,
Snapshotter: snapshot.FromContainerdSnapshotter(snapshotterName, containerdsnapshot.NSSnapshotter("buildkit", mdb.Snapshotter(snapshotterName)), nil),
MetadataStore: md,
LeaseManager: leaseutil.WithNamespace(ctdmetadata.NewLeaseManager(mdb), "buildkit"),
ContentStore: mdb.ContentStore(),
GarbageCollect: mdb.GarbageCollect,
})
require.NoError(t, err)

return cm
return cm, func() {
db.Close()
}
}

// these test helpers are from tonistiigi/fsutil
Expand Down
Loading

0 comments on commit a038b5d

Please sign in to comment.