diff --git a/lib/images/README.md b/lib/images/README.md index 696f8ce95..3be9dab31 100644 --- a/lib/images/README.md +++ b/lib/images/README.md @@ -5,7 +5,9 @@ Converts OCI images to bootable erofs disks for Cloud Hypervisor VMs. ## Architecture ``` -OCI Registry → go-containerregistry → OCI Layout → umoci → rootfs/ → mkfs.erofs → disk.erofs +OCI Registry → go-containerregistry → OCI Layout → umoci + native Linux: shared base layers → base.erofs + final layer artifact → VM overlay + fallback: all layers → rootfs/ → mkfs.erofs → disk.erofs ``` ## Design Decisions @@ -65,7 +67,10 @@ Content-addressable storage with tag symlinks (similar to Docker/Unikraft): rootfs.erofs latest -> abc123def456... # Tag symlink to digest 3.18 -> def456abc123... # Another tag - layers/ # Shared materialized layer artifacts + bases/ # Shared composed read-only base disks + 789abc... + rootfs.erofs + layers/ # Materialized final-layer artifacts abc123def456.../ layer.erofs artifact.erofs.json @@ -86,7 +91,8 @@ Content-addressable storage with tag symlinks (similar to Docker/Unikraft): - Natural hierarchy: All versions of an image grouped under repository - Easy inspection: Clear which digest belongs to which image - Layer caching: All images share the same blob storage, layers deduplicated automatically -- Materialized layer artifacts are reference-protected and reconciled by the layer lifecycle manager; stale temporary trees are age-gated before removal. +- Native Linux images share a composed base disk; the final layer is copied into each VM's writable overlay at instance creation. +- Materialized layer artifacts and shared bases are reference-protected and reconciled by the image lifecycle manager; stale temporary trees are age-gated before removal. **Design:** - Images stored by manifest digest (content hash) @@ -95,6 +101,7 @@ Content-addressable storage with tag symlinks (similar to Docker/Unikraft): - Pulling same tag twice updates the symlink if digest changed - OCI cache uses digest hex as layout tag for true content-addressable caching - Shared blob storage enables automatic layer deduplication across all images +- Shared base disks avoid exporting and storing a complete rootfs for each image variant - Orphaned digests are automatically deleted when the last tag referencing them is removed - Symlinks only created after successful build (status: ready) - Disk accounting uses logical file sizes, matching image metadata and storage admission rather than filesystem block allocation. diff --git a/lib/images/compose.go b/lib/images/compose.go index 9df8c6175..e8030c30a 100644 --- a/lib/images/compose.go +++ b/lib/images/compose.go @@ -8,23 +8,30 @@ import ( "path/filepath" ) -// composeRootfs validates the persisted model and merges its layers into -// dest in manifest order, reading each layer blob from the shared OCI cache. -// Whiteout and opaque-directory markers are interpreted as each layer is -// applied. Any previous tree at dest is replaced: callers must not read dest -// concurrently, and a failure between the remove and the rename leaves dest -// absent. The export root is always 0755 regardless of the last layer's tar -// root entry, matching the mode the previous unpack path created. A crash -// can also strand .compose-* staging directories in dest's parent, the same -// way .unpack-* directories can strand under layer builds. +// composeRootfs validates the persisted model and merges its layers into dest +// in manifest order. Whiteouts are applied to the composed tree. func (c *ociClient) composeRootfs(ctx context.Context, dest, layoutTag string, model *imageManifestModel) error { if err := validateManifestModel(layoutTag, model); err != nil { return fmt.Errorf("validate manifest model: %w", err) } + return c.composeLayerTree(ctx, dest, model.Layers) +} + +func (c *ociClient) composeLayers(ctx context.Context, dest string, layers []layerDescriptor) error { + return c.composeLayerTree(ctx, dest, layers) +} + +func (c *ociClient) composeLayerTree(ctx context.Context, dest string, layers []layerDescriptor) error { parent := filepath.Dir(dest) if err := os.MkdirAll(parent, 0755); err != nil { return fmt.Errorf("create compose parent: %w", err) } + leftovers, _ := filepath.Glob(filepath.Join(parent, ".compose-*")) + for _, leftover := range leftovers { + if err := removePath(leftover); err != nil { + slog.Warn("failed to remove stale compose staging directory", "dir", leftover, "error", err) + } + } staging, err := os.MkdirTemp(parent, ".compose-*") if err != nil { return fmt.Errorf("create compose directory: %w", err) @@ -34,14 +41,9 @@ func (c *ociClient) composeRootfs(ctx context.Context, dest, layoutTag string, m slog.Warn("failed to remove compose staging directory", "dir", staging, "error", err) } }() - - for i, desc := range model.Layers { - if _, err := unpackCachedLayer(ctx, c.cacheBlobDir(), desc, staging, composeOnDiskFormat()); err != nil { - return fmt.Errorf("apply layer %d: %w", i, err) - } + if err := c.composeLayerList(ctx, staging, layers); err != nil { + return err } - // The export directory must stay traversable by other readers; MkdirTemp - // creates it 0700. if err := os.Chmod(staging, 0755); err != nil { return fmt.Errorf("set compose directory mode: %w", err) } @@ -53,3 +55,12 @@ func (c *ociClient) composeRootfs(ctx context.Context, dest, layoutTag string, m } return nil } + +func (c *ociClient) composeLayerList(ctx context.Context, dest string, layers []layerDescriptor) error { + for i, desc := range layers { + if _, err := unpackCachedLayer(ctx, c.cacheBlobDir(), desc, dest, composeOnDiskFormat()); err != nil { + return fmt.Errorf("apply layer %d: %w", i, err) + } + } + return nil +} diff --git a/lib/images/disk.go b/lib/images/disk.go index 68184b785..96b3a0b6f 100644 --- a/lib/images/disk.go +++ b/lib/images/disk.go @@ -196,40 +196,34 @@ func convertToExt4(ctx context.Context, rootfsDir, diskPath string) (int64, erro // Align to sector boundary (required by macOS Virtualization.framework) diskSizeBytes = alignToSector(diskSizeBytes) - // Ensure parent directory exists + return createExt4Disk(ctx, rootfsDir, diskPath, diskSizeBytes) +} + +func createExt4Disk(ctx context.Context, rootfsDir, diskPath string, diskSizeBytes int64) (int64, error) { if err := os.MkdirAll(filepath.Dir(diskPath), 0755); err != nil { return 0, fmt.Errorf("create disk parent dir: %w", err) } - - // Create sparse file - f, err := os.Create(diskPath) + file, err := os.OpenFile(diskPath, os.O_CREATE|os.O_TRUNC|os.O_RDWR, 0644) if err != nil { return 0, fmt.Errorf("create disk file: %w", err) } - if err := f.Truncate(diskSizeBytes); err != nil { - f.Close() + if err := file.Truncate(alignToSector(diskSizeBytes)); err != nil { + _ = file.Close() return 0, fmt.Errorf("truncate disk file: %w", err) } - f.Close() + if err := file.Close(); err != nil { + return 0, fmt.Errorf("close disk file: %w", err) + } - // Format as ext4 with rootfs contents using mkfs.ext4 - // -b 4096: 4KB blocks (standard, matches VM page size and sector alignment) - // -O ^has_journal: Disable journal (not needed for read-only VM mounts) - // -d: Copy directory contents into filesystem - // -F: Force creation (file not block device) cmd := exec.CommandContext(ctx, mkfsExt4Binary(), "-b", "4096", "-O", "^has_journal", "-d", rootfsDir, "-F", diskPath) output, err := cmd.CombinedOutput() if err != nil { return 0, fmt.Errorf("mkfs.ext4 failed: %w, output: %s", err, output) } - - // Verify final size is sector-aligned (mkfs.ext4 should preserve our truncated size) stat, err := os.Stat(diskPath) if err != nil { return 0, fmt.Errorf("stat disk: %w", err) } - - // Re-align if mkfs.ext4 changed the size (shouldn't happen with -F on a regular file) if stat.Size()%sectorSize != 0 { alignedSize := alignToSector(stat.Size()) if err := os.Truncate(diskPath, alignedSize); err != nil { @@ -237,10 +231,20 @@ func convertToExt4(ctx context.Context, rootfsDir, diskPath string) (int64, erro } return alignedSize, nil } - return stat.Size(), nil } +// CreateExt4DiskFromRootfs creates a fixed-size ext4 disk populated with a directory tree. +func CreateExt4DiskFromRootfs(rootfsDir, diskPath string, sizeBytes int64) error { + return CreateExt4DiskFromRootfsWithContext(context.Background(), rootfsDir, diskPath, sizeBytes) +} + +// CreateExt4DiskFromRootfsWithContext is the cancellable form of CreateExt4DiskFromRootfs. +func CreateExt4DiskFromRootfsWithContext(ctx context.Context, rootfsDir, diskPath string, sizeBytes int64) error { + _, err := createExt4Disk(ctx, rootfsDir, diskPath, sizeBytes) + return err +} + // convertToErofs converts a rootfs directory to an erofs disk image using mkfs.erofs func convertToErofs(ctx context.Context, rootfsDir, diskPath string) (int64, error) { // Ensure parent directory exists diff --git a/lib/images/disk_usage.go b/lib/images/disk_usage.go index ca72c1994..e2a6c72ed 100644 --- a/lib/images/disk_usage.go +++ b/lib/images/disk_usage.go @@ -6,6 +6,7 @@ import ( "fmt" "os" "path/filepath" + "strings" "syscall" ) @@ -75,11 +76,13 @@ func totalReadyImageBytesFromMetadataWithContext(ctx context.Context, imagesDir if globErr != nil { return fmt.Errorf("find ready image rootfs for %s: %w", path, globErr) } + sharedBase := false for _, rootfsPath := range rootfsPaths { rootfsInfo, statErr := os.Stat(rootfsPath) if statErr != nil { continue } + sharedBase = isSharedBaseLink(rootfsPath, filepath.Join(imagesDir, "bases")) if !markUniqueRootfs(rootfsInfo, seenRootfs) { return nil } @@ -87,6 +90,9 @@ func totalReadyImageBytesFromMetadataWithContext(ctx context.Context, imagesDir } if meta.SizeBytes > 0 { + if sharedBase { + return nil + } total += meta.SizeBytes return nil } @@ -233,7 +239,11 @@ func (s *layerStore) computeDiskUsageTotals(ctx context.Context) (int64, int64, if err != nil { return 0, 0, err } - return readyImageBytes, ociCacheBytes + layerArtifactBytes, nil + baseBytes, err := totalFileBytesWithContext(ctx, s.paths.ImageBasesDir(), "shared image bases") + if err != nil { + return 0, 0, err + } + return readyImageBytes + baseBytes, ociCacheBytes + layerArtifactBytes, nil } func totalRootfsBytesInDigestDirWithContext(ctx context.Context, digestDir string, seen map[rootfsIdentity]struct{}) (int64, error) { @@ -273,6 +283,23 @@ func totalRootfsBytesInDigestDirWithContext(ctx context.Context, digestDir strin return total, nil } +func isSharedBaseLink(path, basesDir string) bool { + info, err := os.Lstat(path) + if err != nil || info.Mode()&os.ModeSymlink == 0 { + return false + } + resolved, err := filepath.EvalSymlinks(path) + if err != nil { + return false + } + canonicalBasesDir, err := filepath.EvalSymlinks(basesDir) + if err != nil { + return false + } + relative, err := filepath.Rel(canonicalBasesDir, resolved) + return err == nil && relative != "." && relative != ".." && !strings.HasPrefix(relative, ".."+string(filepath.Separator)) +} + type rootfsIdentity struct { dev uint64 ino uint64 diff --git a/lib/images/layer_artifact.go b/lib/images/layer_artifact.go index 1e7897c98..30aad9bfe 100644 --- a/lib/images/layer_artifact.go +++ b/lib/images/layer_artifact.go @@ -11,6 +11,7 @@ import ( "io/fs" "log/slog" "os" + "os/exec" "path/filepath" "runtime" "strings" @@ -147,6 +148,9 @@ func probeLayerArtifactSupport(layersDir string) bool { if !supportsLayerArtifacts() || os.MkdirAll(layersDir, 0755) != nil { return false } + if _, err := exec.LookPath("fsck.erofs"); err != nil { + return false + } probeDir, err := os.MkdirTemp(layersDir, ".probe-*") if err != nil { return false @@ -166,7 +170,7 @@ func probeLayerArtifactSupport(layersDir string) bool { } func (m *manager) layerArtifactSupport() bool { - return m.layers.artifactsSupported + return m.layers != nil && m.layers.artifactsSupported } func (m *manager) materializeLayerArtifact(ctx context.Context, desc layerDescriptor) (*layerArtifact, error) { @@ -245,9 +249,8 @@ func (s *layerStore) clearLayerCache(layerHex string) error { // The layer is unpacked into an isolated temp directory, converted to the // default image format, and installed atomically. Normal failures remove the // temp directory; a crash mid-build can leave a stale .unpack-* directory -// behind, which reconciliation landing with the pull integration is expected -// to sweep. No production caller yet: pull integration and -// composition land in later changes. +// behind, which the startup sweep in layer_gc.go removes once it ages past +// the eviction grace period. // // Concurrent callers share one build. The build itself is detached from the // initiating caller's cancellation so one cancelled pull cannot fail every diff --git a/lib/images/layer_artifact_test.go b/lib/images/layer_artifact_test.go index de117aadb..34c6d2b5b 100644 --- a/lib/images/layer_artifact_test.go +++ b/lib/images/layer_artifact_test.go @@ -34,23 +34,25 @@ const whiteoutPrefix = ".wh." const testTarGzMediaType = "application/vnd.oci.image.layer.v1.tar+gzip" -// writeLayerTestLayout writes img into the shared OCI cache of p tagged with -// the image's digest, mirroring pullToOCILayout. +// writeLayerTestLayout writes images into the shared OCI cache of p tagged +// with each image's digest, mirroring pullToOCILayout. func testLayerArtifactManager(p *paths.Paths) *manager { store := newLayerStore(p, 1) store.artifactsSupported = true return &manager{paths: p, layers: store} } -func writeLayerTestLayout(t *testing.T, p *paths.Paths, img gcr.Image) { +func writeLayerTestLayout(t *testing.T, p *paths.Paths, imgs ...gcr.Image) { t.Helper() - digest, err := img.Digest() - require.NoError(t, err) layoutPath, err := layout.Write(p.SystemOCICache(), empty.Index) require.NoError(t, err) - require.NoError(t, layoutPath.AppendImage(img, layout.WithAnnotations(map[string]string{ - "org.opencontainers.image.ref.name": digestToLayoutTag(digest.String()), - }))) + for _, img := range imgs { + digest, err := img.Digest() + require.NoError(t, err) + require.NoError(t, layoutPath.AppendImage(img, layout.WithAnnotations(map[string]string{ + "org.opencontainers.image.ref.name": digestToLayoutTag(digest.String()), + }))) + } } func layerDescFromImage(t *testing.T, img gcr.Image, index int) layerDescriptor { diff --git a/lib/images/layer_gc.go b/lib/images/layer_gc.go new file mode 100644 index 000000000..a7e4a6ad9 --- /dev/null +++ b/lib/images/layer_gc.go @@ -0,0 +1,340 @@ +package images + +import ( + "context" + "fmt" + "io/fs" + "log/slog" + "os" + "path/filepath" + "strings" + "sync" + "time" +) + +// layerEvictionGracePeriod keeps freshly written layer artifacts and temp +// directories out of cleanup so recovery and eviction never race builds that +// are still writing them. +const layerEvictionGracePeriod = 10 * time.Minute + +// referencedLayerDigests returns the set of layer blob digests referenced by +// the manifest models of every image in the images tree — both content and +// legacy layouts write their model as a manifest.json with the digest as its +// parent directory — plus the digests currently referenced by in-flight +// builds. Layer artifacts in this set are protected from eviction. An +// unreadable manifest aborts the scan so eviction fails closed. +// +// Callers must hold createMu so the in-flight map read is ordered with model +// writes during finalization. +func (m *manager) referencedLayerDigests() (map[string]struct{}, error) { + refs := make(map[string]struct{}, len(m.inflightLayerRefs)) + for digestHex := range m.inflightLayerRefs { + refs[digestHex] = struct{}{} + } + err := filepath.WalkDir(m.paths.ImagesDir(), func(path string, entry fs.DirEntry, err error) error { + if err != nil { + if os.IsNotExist(err) { + return nil + } + // An incomplete walk means an incomplete reference set; the caller + // must not evict against it. + return err + } + if entry.IsDir() || entry.Name() != "manifest.json" { + return nil + } + digestHex := filepath.Base(filepath.Dir(path)) + model, readErr := readManifestModelAt(path, digestHex) + if readErr != nil { + return fmt.Errorf("read manifest model %s: %w", path, readErr) + } + if model == nil { + return nil + } + for _, layer := range model.Layers { + refs[strings.TrimPrefix(layer.Digest, "sha256:")] = struct{}{} + } + return nil + }) + if err != nil { + return nil, fmt.Errorf("walk image manifests: %w", err) + } + return refs, nil +} + +// inflightLayerRef is the handle returned by retainInflightLayers. Its +// release is idempotent: finalization releases the refs as soon as the +// manifest model is durable, and the build's deferred release becomes a +// no-op afterwards. +type inflightLayerRef struct { + once sync.Once + digestHexes []string +} + +func (r *inflightLayerRef) release(m *manager) { + r.once.Do(func() { + m.createMu.Lock() + defer m.createMu.Unlock() + m.releaseInflightLayerRefsLocked(r.digestHexes) + }) +} + +// releaseLocked is release for callers already holding createMu. +func (r *inflightLayerRef) releaseLocked(m *manager) { + r.once.Do(func() { m.releaseInflightLayerRefsLocked(r.digestHexes) }) +} + +type inflightBaseRef struct { + once sync.Once + digestHex string +} + +func (r *inflightBaseRef) release(m *manager) { + r.once.Do(func() { + m.createMu.Lock() + defer m.createMu.Unlock() + m.releaseInflightBaseRefLocked(r.digestHex) + }) +} + +func (r *inflightBaseRef) releaseLocked(m *manager) { + r.once.Do(func() { m.releaseInflightBaseRefLocked(r.digestHex) }) +} + +// retainInflightLayers registers one in-flight reference per digest so +// reconciliation cannot evict layers a build is materializing. +func (m *manager) retainInflightLayers(digestHexes []string) *inflightLayerRef { + m.createMu.Lock() + for _, digestHex := range digestHexes { + m.inflightLayerRefs[digestHex]++ + } + m.createMu.Unlock() + return &inflightLayerRef{digestHexes: digestHexes} +} + +func (m *manager) releaseInflightLayerRefsLocked(digestHexes []string) { + for _, digestHex := range digestHexes { + if m.inflightLayerRefs[digestHex] <= 1 { + delete(m.inflightLayerRefs, digestHex) + } else { + m.inflightLayerRefs[digestHex]-- + } + } +} + +func (m *manager) retainInflightBase(digest string) *inflightBaseRef { + digestHex := strings.TrimPrefix(digest, "sha256:") + m.createMu.Lock() + m.inflightBaseRefs[digestHex]++ + m.createMu.Unlock() + return &inflightBaseRef{digestHex: digestHex} +} + +func (m *manager) releaseInflightBaseRefLocked(digestHex string) { + if m.inflightBaseRefs[digestHex] <= 1 { + delete(m.inflightBaseRefs, digestHex) + } else { + m.inflightBaseRefs[digestHex]-- + } +} + +// reconcileLayerStore evicts unreferenced layer artifacts and refreshes the +// cached disk usage totals so accounting reflects the removals. +func (m *manager) reconcileLayerStore() { + if m.layers == nil { + return + } + m.createMu.Lock() + defer m.createMu.Unlock() + m.reconcileLayerStoreLocked() +} + +func (m *manager) scheduleLayerReconcile() { + if m.layerEvictionGrace <= 0 { + return + } + m.reconcileMu.Lock() + defer m.reconcileMu.Unlock() + if m.reconcileTimer != nil { + return + } + m.reconcileTimer = time.AfterFunc(m.layerEvictionGrace, func() { + m.reconcileMu.Lock() + m.reconcileTimer = nil + m.reconcileMu.Unlock() + m.reconcileLayerStore() + }) +} + +// reconcileLayerStoreLocked is used by lifecycle operations that already hold +// createMu. Serializing reconciliation with manifest finalization prevents an +// eviction scan from racing a newly committed layer reference. +func (m *manager) reconcileLayerStoreLocked() { + if m.layers == nil { + return + } + m.evictUnreferencedLayerArtifacts() + m.evictUnreferencedImageBases() + m.layers.refreshDiskUsageTotals() +} + +// evictUnreferencedLayerArtifacts removes layer artifacts that no image +// manifest model references, deleting the digest directory entirely. Artifacts +// newer than the grace period are kept so in-flight builds never lose work. +func (m *manager) evictUnreferencedImageBases() { + refs := make(map[string]struct{}, len(m.inflightBaseRefs)) + for digestHex := range m.inflightBaseRefs { + refs[digestHex] = struct{}{} + } + if err := filepath.WalkDir(m.paths.ImagesDir(), func(path string, entry fs.DirEntry, err error) error { + if err != nil { + if os.IsNotExist(err) { + return nil + } + return err + } + if entry.IsDir() || entry.Name() != "manifest.json" { + return nil + } + model, err := readManifestModelAt(path, filepath.Base(filepath.Dir(path))) + if err != nil { + return err + } + if model != nil && model.BaseDigest != "" { + refs[strings.TrimPrefix(model.BaseDigest, "sha256:")] = struct{}{} + } + return nil + }); err != nil { + slog.Warn("skipping shared base eviction: incomplete reference scan", "error", err) + return + } + entries, err := os.ReadDir(m.paths.ImageBasesDir()) + if err != nil { + if !os.IsNotExist(err) { + slog.Warn("shared base eviction failed to list bases", "error", err) + } + return + } + cutoff := time.Now().Add(-m.layerEvictionGrace) + for _, entry := range entries { + if !entry.IsDir() { + continue + } + if _, ok := refs[entry.Name()]; ok { + continue + } + path := filepath.Join(m.paths.ImageBasesDir(), entry.Name()) + info, err := os.Stat(path) + if err != nil || info.ModTime().After(cutoff) { + continue + } + if err := removePath(path); err != nil { + slog.Warn("failed to evict unreferenced shared base", "digest", entry.Name(), "error", err) + } + } +} + +func (m *manager) evictUnreferencedLayerArtifacts() { + refs, err := m.referencedLayerDigests() + if err != nil { + // Evicting against a truncated reference set would delete artifacts + // belonging to images the walk never reached. + slog.Warn("skipping layer eviction: incomplete reference scan", "error", err) + return + } + + layersDir := m.paths.ImageLayersDir() + entries, err := os.ReadDir(layersDir) + if err != nil { + if !os.IsNotExist(err) { + slog.Warn("layer eviction failed to list layer store", "error", err) + } + return + } + + cutoff := time.Now().Add(-m.layerEvictionGrace) + evicted := 0 + var evictedBytes int64 + for _, entry := range entries { + if !entry.IsDir() { + continue + } + digestHex := entry.Name() + if _, referenced := refs[digestHex]; referenced { + continue + } + dirPath := filepath.Join(layersDir, digestHex) + info, statErr := os.Stat(dirPath) + if statErr != nil || info.ModTime().After(cutoff) { + continue + } + size, err := dirSize(dirPath) + if err != nil { + slog.Warn("failed to measure layer artifact size", "digest", digestHex, "error", err) + } + // removePath clears read-only directories restored from layer + // metadata, which os.RemoveAll cannot unlink through. + if err := removePath(dirPath); err != nil { + slog.Warn("failed to evict unreferenced layer artifact", "digest", digestHex, "error", err) + continue + } + evicted++ + evictedBytes += size + } + if evicted > 0 { + slog.Info("evicted unreferenced layer artifacts", "count", evicted, "bytes", evictedBytes) + m.recordLayerArtifactsEvicted(context.Background(), int64(evicted)) + } +} + +// isStaleTempDirName reports whether a directory name matches the temp +// prefixes builds use for staging, installs, and tag promotion. +func isStaleTempDirName(name string) bool { + for _, prefix := range []string{".unpack-", ".install-", ".tag-stage-"} { + if strings.HasPrefix(name, prefix) { + return true + } + } + return false +} + +// cleanStaleImageTempDirs removes temp directories left behind by builds that +// were interrupted mid-install, mid-materialization, or mid-tag promotion. +// The walk covers the whole images tree: layer and content staging dirs plus +// .tag-stage-* dirs, which are created under images//. Only +// directories older than the grace period are removed so live builds are +// never disturbed. +// +// This must stay a startup-only sweep: a staging dir's own mtime only moves +// when its direct children change, so a deep extraction running longer than +// the grace period can look stale while actively writing. A periodic sweep +// would need a heartbeat or a live-build registry first. +func (m *manager) cleanStaleImageTempDirs() { + cutoff := time.Now().Add(-m.layerEvictionGrace) + err := filepath.WalkDir(m.paths.ImagesDir(), func(path string, entry fs.DirEntry, err error) error { + if err != nil { + if os.IsNotExist(err) { + return nil + } + return err + } + if !entry.IsDir() { + return nil + } + name := entry.Name() + if !isStaleTempDirName(name) { + return nil + } + if info, err := entry.Info(); err == nil && info.ModTime().Before(cutoff) { + // removePath clears the read-only directories umoci restores from + // layer metadata, which os.RemoveAll cannot unlink through. + if err := removePath(path); err != nil { + slog.Warn("failed to remove stale image temp dir", "dir", path, "error", err) + } + } + return fs.SkipDir + }) + if err != nil { + slog.Warn("failed to clean stale image temp dirs", "root", m.paths.ImagesDir(), "error", err) + } +} diff --git a/lib/images/lifecycle_test.go b/lib/images/lifecycle_test.go new file mode 100644 index 000000000..38883bb19 --- /dev/null +++ b/lib/images/lifecycle_test.go @@ -0,0 +1,252 @@ +package images + +import ( + "context" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" + "time" + + gcr "github.com/google/go-containerregistry/pkg/v1" + "github.com/google/go-containerregistry/pkg/v1/empty" + "github.com/google/go-containerregistry/pkg/v1/mutate" + "github.com/kernel/hypeman/lib/paths" + "github.com/stretchr/testify/require" +) + +func layerStoreHexes(t *testing.T, p *paths.Paths) map[string]struct{} { + t.Helper() + entries, err := os.ReadDir(p.ImageLayersDir()) + require.NoError(t, err) + hexes := make(map[string]struct{}) + for _, entry := range entries { + if entry.IsDir() { + hexes[entry.Name()] = struct{}{} + } + } + return hexes +} + +func imageDigest(t *testing.T, img gcr.Image) string { + t.Helper() + digest, err := img.Digest() + require.NoError(t, err) + return digest.String() +} + +func imageBaseHexes(t *testing.T, p *paths.Paths) map[string]struct{} { + t.Helper() + entries, err := os.ReadDir(p.ImageBasesDir()) + require.NoError(t, err) + hexes := make(map[string]struct{}) + for _, entry := range entries { + if entry.IsDir() { + hexes[entry.Name()] = struct{}{} + } + } + return hexes +} + +// importAndWait imports an image and blocks until its build reaches ready. +func importAndWait(t *testing.T, m *manager, ctx context.Context, repo, tag, digest string) { + t.Helper() + events := make(chan StatusEvent, 2) + layoutTag := digestToLayoutTag(digest) + m.subscribeToReady(layoutTag, events) + defer m.unsubscribeFromReady(layoutTag, events) + _, err := m.ImportLocalImage(ctx, repo, tag, digest) + require.NoError(t, err) + select { + case event := <-events: + require.Equal(t, StatusReady, event.Status) + case <-time.After(30 * time.Second): + t.Fatalf("image %s did not become ready", repo) + } +} + +// TestSharedLayersMaterializeOnceAndEvictWithReferences is the end-to-end +// lifecycle: two images share a composed base, the final-layer artifacts remain +// independently referenced, and the base is evicted only after both images go. +func TestSharedLayersMaterializeOnceAndEvictWithReferences(t *testing.T) { + if !supportsLayerArtifacts() { + t.Skip("native overlayfs layer artifacts are not supported") + } + if _, err := exec.LookPath("mkfs.erofs"); err != nil { + t.Skip("mkfs.erofs not available") + } + dataDir := t.TempDir() + p := paths.New(dataDir) + mgr, err := NewManager(p, 1, nil) + require.NoError(t, err) + m := mgr.(*manager) + m.layerEvictionGrace = 0 + + base := syntheticLayer(t, "base.txt", "shared base content") + topA := syntheticLayer(t, "a.txt", "app A payload") + topB := syntheticLayer(t, "b.txt", "app B payload") + + imgA, err := mutate.AppendLayers(empty.Image, base, topA) + require.NoError(t, err) + imgB, err := mutate.AppendLayers(empty.Image, base, topB) + require.NoError(t, err) + + writeLayerTestLayout(t, p, imgA, imgB) + digestA, digestB := imageDigest(t, imgA), imageDigest(t, imgB) + + baseManifest, err := imgA.Manifest() + require.NoError(t, err) + topAHex := baseManifest.Layers[1].Digest.Hex + topBManifest, err := imgB.Manifest() + require.NoError(t, err) + topBHex := topBManifest.Layers[1].Digest.Hex + + ctx := context.Background() + const repoA = "kernel.local/apps/app-a" + const repoB = "kernel.local/apps/app-b" + + importAndWait(t, m, ctx, repoA, "v1", digestA) + importAndWait(t, m, ctx, repoB, "v1", digestB) + + modelA, err := readManifestModel(p, strings.TrimPrefix(digestA, "sha256:")) + require.NoError(t, err) + require.Equal(t, 1, modelA.BaseLayerCount) + baseHex := strings.TrimPrefix(modelA.BaseDigest, "sha256:") + + // Only final layers are materialized; the composed base is stored once. + hexes := layerStoreHexes(t, p) + require.Len(t, hexes, 2) + require.Contains(t, hexes, topAHex) + require.Contains(t, hexes, topBHex) + require.Equal(t, map[string]struct{}{baseHex: {}}, imageBaseHexes(t, p)) + + // Deleting image A evicts only its unique layer; the shared base survives. + require.NoError(t, m.DeleteImage(ctx, repoA+"@"+digestA)) + hexes = layerStoreHexes(t, p) + require.Len(t, hexes, 1) + require.Contains(t, hexes, topBHex) + require.NotContains(t, hexes, topAHex) + require.Contains(t, imageBaseHexes(t, p), baseHex, "shared base must survive while referenced") + + // Deleting image B removes the last references: everything is evicted. + require.NoError(t, m.DeleteImage(ctx, repoB+"@"+digestB)) + hexes = layerStoreHexes(t, p) + require.Empty(t, hexes, "unreferenced layer artifacts must be evicted") + require.Empty(t, imageBaseHexes(t, p), "unreferenced shared bases must be evicted") +} + +func newLifecycleTestManager(p *paths.Paths) *manager { + return &manager{ + paths: p, + layers: newLayerStore(p, 1), + inflightPulls: make(map[string]*inflightImagePull), + inflightLayerRefs: make(map[string]int), + } +} + +func TestSharedBaseBytesCountedOnce(t *testing.T) { + p := paths.New(t.TempDir()) + m := newLifecycleTestManager(p) + baseHex := strings.Repeat("a", 64) + basePath := p.ImageBasePath(baseHex) + require.NoError(t, os.MkdirAll(filepath.Dir(basePath), 0o755)) + require.NoError(t, os.WriteFile(basePath, []byte("shared-base"), 0o644)) + + for _, digestHex := range []string{strings.Repeat("b", 64), strings.Repeat("c", 64)} { + contentDir := p.ImageContentDir(digestHex) + require.NoError(t, os.MkdirAll(contentDir, 0o755)) + rootfsPath := p.ImageContentPath(digestHex) + relative, err := filepath.Rel(filepath.Dir(rootfsPath), basePath) + require.NoError(t, err) + require.NoError(t, os.Symlink(relative, rootfsPath)) + require.NoError(t, writeMetadataFile(p.ImageContentMetadata(digestHex), &imageMetadata{ + Digest: "sha256:" + digestHex, Status: StatusReady, SizeBytes: int64(len("shared-base")), + })) + } + + readyBytes, _, err := m.layers.getDiskUsageTotals(context.Background()) + require.NoError(t, err) + require.Equal(t, int64(len("shared-base")), readyBytes) +} + +// TestLayerArtifactsCountedInOneBucket verifies the accounting contract: ready +// image bytes and the OCI+layer cache total stay disjoint, so consumers summing +// TotalImageBytes and TotalOCICacheBytes count layer bytes exactly once. +func TestLayerArtifactsCountedInOneBucket(t *testing.T) { + p := paths.New(t.TempDir()) + m := newLifecycleTestManager(p) + + digestHex := "cd01cd01cd01cd01cd01cd01cd01cd01cd01cd01cd01cd01cd01cd01cd01cd01" + require.NoError(t, os.MkdirAll(p.ImageLayerDir(digestHex), 0o755)) + payload := make([]byte, 4096) + require.NoError(t, os.WriteFile(p.ImageLayerArtifactForFormat(digestHex, string(DefaultImageFormat)), payload, 0o644)) + + readyBytes, cacheBytes, err := m.layers.getDiskUsageTotals(context.Background()) + require.NoError(t, err) + require.GreaterOrEqual(t, cacheBytes, int64(len(payload))) + + totalBytes, err := m.TotalImageBytes(context.Background()) + require.NoError(t, err) + require.Equal(t, readyBytes, totalBytes) + + cacheTotal, err := m.TotalOCICacheBytes(context.Background()) + require.NoError(t, err) + require.Equal(t, cacheBytes, cacheTotal) +} + +func TestCleanStaleImageTempDirsRemovesOnlyOldDirectories(t *testing.T) { + p := paths.New(t.TempDir()) + m := newLifecycleTestManager(p) + m.layerEvictionGrace = time.Hour + + layersDir := p.ImageLayersDir() + staleDir := filepath.Join(layersDir, "ab12", ".unpack-stale") + freshDir := filepath.Join(layersDir, "cd34", ".unpack-fresh") + require.NoError(t, os.MkdirAll(staleDir, 0o755)) + require.NoError(t, os.MkdirAll(freshDir, 0o755)) + old := time.Now().Add(-2 * time.Hour) + require.NoError(t, os.Chtimes(staleDir, old, old)) + + m.cleanStaleImageTempDirs() + + _, err := os.Stat(staleDir) + require.True(t, os.IsNotExist(err), "stale temp dir must be removed") + _, err = os.Stat(freshDir) + require.NoError(t, err, "fresh temp dir must survive cleanup") +} + +func TestEvictionKeepsReferencedAndFreshArtifacts(t *testing.T) { + p := paths.New(t.TempDir()) + m := newLifecycleTestManager(p) + m.layerEvictionGrace = time.Hour + + referencedHex := "ef01ef01ef01ef01ef01ef01ef01ef01ef01ef01ef01ef01ef01ef01ef01ef01" + orphanFreshHex := "ab23ab23ab23ab23ab23ab23ab23ab23ab23ab23ab23ab23ab23ab23ab23ab23" + + // A manifest model referencing one layer protects it regardless of age. + model := &imageManifestModel{ + SchemaVersion: manifestModelSchemaVersion, + Digest: "sha256:" + referencedHex, + RootFSType: "layers", + Config: manifestConfigRef{ + Digest: "sha256:" + strings.Repeat("c", 64), + MediaType: "application/vnd.oci.image.config.v1+json", + DiffIDs: []string{"sha256:" + referencedHex}, + }, + Layers: []layerDescriptor{{Digest: "sha256:" + referencedHex, DiffID: "sha256:" + referencedHex}}, + } + require.NoError(t, writeManifestModel(p, referencedHex, model)) + require.NoError(t, os.MkdirAll(p.ImageLayerDir(referencedHex), 0o755)) + require.NoError(t, os.WriteFile(p.ImageLayerArtifactForFormat(referencedHex, string(DefaultImageFormat)), []byte("kept"), 0o644)) + + // An unreferenced but fresh artifact is protected by the grace period. + require.NoError(t, os.MkdirAll(p.ImageLayerDir(orphanFreshHex), 0o755)) + require.NoError(t, os.WriteFile(p.ImageLayerArtifactForFormat(orphanFreshHex, string(DefaultImageFormat)), []byte("fresh"), 0o644)) + + m.reconcileLayerStore() + + hexes := layerStoreHexes(t, p) + require.Contains(t, hexes, referencedHex) + require.Contains(t, hexes, orphanFreshHex) +} diff --git a/lib/images/manager.go b/lib/images/manager.go index e0b52fa7a..968b375f6 100644 --- a/lib/images/manager.go +++ b/lib/images/manager.go @@ -52,7 +52,7 @@ type Manager interface { TagImage(ctx context.Context, source, target string) (*Image, error) DeleteImage(ctx context.Context, name string) error RecoverInterruptedBuilds() - // TotalImageBytes returns the total size of all ready images on disk. + // TotalImageBytes returns ready rootfs and shared-base bytes on disk. // Used by the resource manager for disk capacity tracking. TotalImageBytes(ctx context.Context) (int64, error) // TotalOCICacheBytes returns the total size of the OCI and materialized layer caches. @@ -76,6 +76,11 @@ type manager struct { queue *queue.Queue createMu sync.Mutex layers *layerStore + inflightLayerRefs map[string]int + inflightBaseRefs map[string]int + layerEvictionGrace time.Duration + reconcileMu sync.Mutex + reconcileTimer *time.Timer tagGenerations map[string]uint64 requestedTags map[string]string // newest pull's digest per requested tag metrics *Metrics @@ -105,6 +110,9 @@ func NewManager(p *paths.Paths, maxConcurrentBuilds int, meter metric.Meter) (Ma ociClient: ociClient, queue: queue.New(maxConcurrentBuilds), inflightPulls: make(map[string]*inflightImagePull), + inflightLayerRefs: make(map[string]int), + inflightBaseRefs: make(map[string]int), + layerEvictionGrace: layerEvictionGracePeriod, borrowedCredentialsTimeout: DefaultBorrowedCredentialsTimeout, readySubscribers: make(map[string][]chan StatusEvent), tagGenerations: make(map[string]uint64), @@ -121,6 +129,13 @@ func NewManager(p *paths.Paths, maxConcurrentBuilds int, meter metric.Meter) (Ma } m.RecoverInterruptedBuilds() + // Sweep temp dirs and evict layer orphans an unclean shutdown may have + // left behind. Recovered builds re-enqueued above run concurrently; they + // re-materialize from the blob cache if this pass evicts their previous + // attempt's unreferenced artifacts. + m.cleanStaleImageTempDirs() + m.reconcileLayerStore() + m.scheduleLayerReconcile() // Keep legacy images readable in their existing layout and promote them only // when an operation needs shared content, such as a cross-repository tag. // Avoiding a startup-wide migration keeps startup bounded and independent of @@ -485,15 +500,32 @@ func (m *manager) buildImage(ctx context.Context, ref *ResolvedRef, credentials // still pulls the architecture its digest identifies. Uses the cache if the // digest is already pulled. pullRef := ref.DigestRef() - result, err := m.ociClient.pullAndExportWithAuth(ctx, pullRef, ref.Digest(), tempDir, credentials) + result, err := m.ociClient.pullManifestWithPlatformAuth(ctx, pullRef, ref.Digest(), vmPlatform(), credentials) m.recordPullResultMetrics(ctx, ref.Digest(), result) if err != nil { - m.updateStatusByDigest(ref, StatusFailed, fmt.Errorf("pull and export: %w", err), buildID) + m.updateStatusByDigest(ref, StatusFailed, fmt.Errorf("pull manifest: %w", err), buildID) m.recordPullMetrics(ctx, "failed") return } m.recordPullMetrics(ctx, "success") + materialized, materializeErr := m.materializeLayerArtifacts(ctx, result) + var baseLease *inflightBaseRef + defer func() { + if materialized != nil { + materialized.release(m) + } + if baseLease != nil { + baseLease.release(m) + } + m.reconcileLayerStore() + m.scheduleLayerReconcile() + }() + if materializeErr != nil { + slog.Warn("layer materialization failed; using flattened rootfs", + "digest", ref.DigestHex(), "error", materializeErr) + } + // Check if this digest already exists and is ready (deduplication) if meta, err := readMetadata(m.paths, ref.Repository(), ref.DigestHex()); err == nil { if meta.Status == StatusReady { @@ -514,21 +546,36 @@ func (m *manager) buildImage(ctx context.Context, ref *ResolvedRef, credentials m.updateStatusByDigest(ref, StatusConverting, nil, buildID) diskPath := resolveImageLayout(m.paths, ref.Repository(), ref.DigestHex()).disk - // Keep the temporary filesystem beside its final path so finalization stays - // atomic even when system/builds and images are on different filesystems. diskTempPath := diskPath + ".tmp-" + buildID defer os.Remove(diskTempPath) - // Use default image format (erofs on Linux, ext4 on Darwin) - convertStart := time.Now() - diskSize, err := ExportRootfs(tempDir, diskTempPath, DefaultImageFormat) - m.recordImageBuildPhase(ctx, ref.Digest(), "filesystem_export", time.Since(convertStart), phaseStatus(err), "not_applicable") + layered := materializeErr == nil && materialized != nil && m.layerArtifactSupport() && result.Manifest != nil && len(result.Manifest.Layers) > 1 + var diskSize int64 + var baseDigest string + if layered { + result.Manifest.BaseLayerCount = len(result.Manifest.Layers) - 1 + baseDigest = sharedBaseDigest(result.Manifest) + baseLease = m.retainInflightBase(baseDigest) + baseStart := time.Now() + baseDigest, diskSize, err = m.buildSharedBase(ctx, result.Manifest, buildDir) + m.recordImageBuildPhase(ctx, ref.Digest(), "shared_base_export", time.Since(baseStart), phaseStatus(err), "not_applicable") + } else { + err = result.measure("layer_unpack", func() error { + return m.ociClient.composeRootfs(ctx, tempDir, digestToLayoutTag(ref.Digest()), result.Manifest) + }) + m.recordImageBuildPhase(ctx, ref.Digest(), "layer_unpack", result.Phases[len(result.Phases)-1].Duration, phaseStatus(err), "not_applicable") + if err == nil { + convertStart := time.Now() + diskSize, err = ExportRootfsWithContext(ctx, tempDir, diskTempPath, DefaultImageFormat) + m.recordImageBuildPhase(ctx, ref.Digest(), "filesystem_export", time.Since(convertStart), phaseStatus(err), "not_applicable") + } + } if err != nil { - m.updateStatusByDigest(ref, StatusFailed, fmt.Errorf("convert to %s: %w", DefaultImageFormat, err), buildID) + m.updateStatusByDigest(ref, StatusFailed, fmt.Errorf("prepare image filesystem: %w", err), buildID) return } finalizeStart := time.Now() - err = m.finalizeImage(ref, result, diskSize, buildID, diskTempPath) + err = m.finalizeImage(ref, result, diskSize, buildID, diskTempPath, materialized, baseLease, baseDigest) m.recordImageBuildPhase(ctx, ref.Digest(), "finalize", time.Since(finalizeStart), phaseStatus(err), "not_applicable") if err != nil { if errors.Is(err, errStaleBuild) { @@ -541,7 +588,22 @@ func (m *manager) buildImage(ctx context.Context, ref *ResolvedRef, credentials buildStatus = "success" } -func (m *manager) finalizeImage(ref *ResolvedRef, result *pullResult, diskSize int64, buildID, diskTempPath string) error { +func (m *manager) materializeLayerArtifacts(ctx context.Context, result *pullResult) (*inflightLayerRef, error) { + if result.Manifest == nil || layerArtifactFormat() == "" || !m.layerArtifactSupport() || len(result.Manifest.Layers) < 2 { + return nil, nil + } + // The shared base is a composed disk. Only the final layer needs a + // materialized overlay representation for instance creation. + desc := result.Manifest.Layers[len(result.Manifest.Layers)-1] + digestHex := strings.TrimPrefix(desc.Digest, "sha256:") + handle := m.retainInflightLayers([]string{digestHex}) + if _, err := m.materializeLayerArtifact(ctx, desc); err != nil { + return handle, err + } + return handle, nil +} + +func (m *manager) finalizeImage(ref *ResolvedRef, result *pullResult, diskSize int64, buildID, diskTempPath string, materialized *inflightLayerRef, baseLease *inflightBaseRef, baseDigest string) error { m.createMu.Lock() defer m.createMu.Unlock() @@ -567,10 +629,23 @@ func (m *manager) finalizeImage(ref *ResolvedRef, result *pullResult, diskSize i diskInstalled := false modelWritten := false - if err := installAtomically(layout.disk, func(path string) error { - return os.Rename(diskTempPath, path) - }); err != nil { - return fmt.Errorf("install image disk: %w", err) + if baseDigest == "" { + if err := installAtomically(layout.disk, func(path string) error { + return os.Rename(diskTempPath, path) + }); err != nil { + return fmt.Errorf("install image disk: %w", err) + } + } else { + basePath := m.paths.ImageBasePath(strings.TrimPrefix(baseDigest, "sha256:")) + if err := installAtomically(layout.disk, func(path string) error { + relative, err := filepath.Rel(filepath.Dir(layout.disk), basePath) + if err != nil { + return err + } + return os.Symlink(relative, path) + }); err != nil { + return fmt.Errorf("install image base link: %w", err) + } } diskInstalled = true @@ -580,10 +655,24 @@ func (m *manager) finalizeImage(ref *ResolvedRef, result *pullResult, diskSize i if result.Manifest != nil { model := *result.Manifest model.Platform = actualPlatform.String() + if baseDigest != "" { + model.BaseDigest = baseDigest + model.BaseLayerCount = len(model.Layers) - 1 + } if err := writeManifestModelAt(modelPath, ref.DigestHex(), &model); err != nil { return rollbackFinalization(layout, modelPath, diskInstalled, modelWritten, fmt.Errorf("write manifest model: %w", err)) } modelWritten = true + // The model now protects these layers on disk, so the in-flight refs + // can go while still holding createMu: a delete racing the ready + // notification then reconciles against durable state. The build's + // deferred release is a no-op after this. + if materialized != nil { + materialized.releaseLocked(m) + } + if baseLease != nil { + baseLease.releaseLocked(m) + } } meta.Status = StatusReady @@ -817,7 +906,8 @@ func (m *manager) deleteDigestImage(repository, digestHex string) error { return err } m.clearRequestedDigest(digestHex) - m.layers.refreshDiskUsageTotals() + m.reconcileLayerStoreLocked() + m.scheduleLayerReconcile() return nil } @@ -851,11 +941,13 @@ func (m *manager) deleteTaggedImage(repository, tag string) error { if err := removeDigestIfUnreferenced(m.paths, repository, digestHex, true); err != nil { return fmt.Errorf("delete orphaned digest %s: %w", digestHex, err) } - m.layers.refreshDiskUsageTotals() + m.reconcileLayerStoreLocked() + m.scheduleLayerReconcile() return nil } -// TotalImageBytes returns the total size of all ready images on disk. +// TotalImageBytes returns ready rootfs and shared-base bytes on disk. Shared +// layer artifacts are accounted separately via TotalOCICacheBytes. func (m *manager) TotalImageBytes(ctx context.Context) (int64, error) { readyImageBytes, _, err := m.layers.getDiskUsageTotals(ctx) if err != nil { diff --git a/lib/images/manager_test.go b/lib/images/manager_test.go index 13a6835e2..03a757d0c 100644 --- a/lib/images/manager_test.go +++ b/lib/images/manager_test.go @@ -736,7 +736,7 @@ func TestDeleteAndRecreateDuringBuildTail(t *testing.T) { m.updateStatusByDigest(staleRef, StatusFailed, errors.New("stale build"), firstMeta.BuildID) staleBundle, err := m.ociClient.extractOCIImageBundle(digestHex) require.NoError(t, err) - require.ErrorIs(t, m.finalizeImage(staleRef, &pullResult{Metadata: staleBundle.Meta}, 1, firstMeta.BuildID, ""), errStaleBuild) + require.ErrorIs(t, m.finalizeImage(staleRef, &pullResult{Metadata: staleBundle.Meta}, 1, firstMeta.BuildID, "", nil, nil, ""), errStaleBuild) currentMeta, err = readMetadata(p, repo, digestHex) require.NoError(t, err) require.Equal(t, StatusPending, currentMeta.Status) diff --git a/lib/images/manifest_model.go b/lib/images/manifest_model.go index 63e45decf..65a1f5014 100644 --- a/lib/images/manifest_model.go +++ b/lib/images/manifest_model.go @@ -18,13 +18,15 @@ import ( // manifest digest, the resolved platform, the image config with its diff ids, // and the ordered layer descriptors. type imageManifestModel struct { - SchemaVersion int `json:"schema_version"` - Digest string `json:"digest"` // manifest digest, sha256:... - MediaType string `json:"media_type,omitempty"` - Platform string `json:"platform"` // os/arch[/variant] - Config manifestConfigRef `json:"config"` - RootFSType string `json:"rootfs_type"` - Layers []layerDescriptor `json:"layers"` // manifest order, base layer first + SchemaVersion int `json:"schema_version"` + Digest string `json:"digest"` // manifest digest, sha256:... + MediaType string `json:"media_type,omitempty"` + Platform string `json:"platform"` // os/arch[/variant] + Config manifestConfigRef `json:"config"` + RootFSType string `json:"rootfs_type"` + Layers []layerDescriptor `json:"layers"` // manifest order, base layer first + BaseDigest string `json:"base_digest,omitempty"` + BaseLayerCount int `json:"base_layer_count,omitempty"` } const manifestModelSchemaVersion = 1 @@ -79,6 +81,23 @@ func validateManifestModel(digestHex string, model *imageManifestModel) error { if model.RootFSType != "layers" { return fmt.Errorf("unsupported manifest rootfs type: %q", model.RootFSType) } + if model.BaseLayerCount < 0 || model.BaseLayerCount > len(model.Layers) { + return fmt.Errorf("invalid base layer count: %d", model.BaseLayerCount) + } + if (model.BaseDigest == "") != (model.BaseLayerCount == 0) { + return fmt.Errorf("base digest and base layer count must be set together") + } + if model.BaseDigest != "" { + if _, err := parseSHA256Digest(model.BaseDigest); err != nil { + return fmt.Errorf("invalid base digest: %q", model.BaseDigest) + } + if model.BaseLayerCount == len(model.Layers) { + return fmt.Errorf("base digest must leave an image layer for the overlay") + } + if model.BaseDigest != sharedBaseDigest(model) { + return fmt.Errorf("base digest does not match base layers") + } + } if err := validateManifestConfig(digestHex, model); err != nil { return err } @@ -156,7 +175,13 @@ func writeManifestModelAt(path, digestHex string, model *imageManifestModel) err // Missing models return (nil, nil): images converted before the manifest model // existed only have a flattened rootfs. func readManifestModel(p *paths.Paths, digestHex string) (*imageManifestModel, error) { - data, err := os.ReadFile(p.ImageContentManifestModel(digestHex)) + return readManifestModelAt(p.ImageContentManifestModel(digestHex), digestHex) +} + +// readManifestModelAt loads the manifest model at path, if present. Missing +// files return (nil, nil). +func readManifestModelAt(path, digestHex string) (*imageManifestModel, error) { + data, err := os.ReadFile(path) if err != nil { if os.IsNotExist(err) { return nil, nil diff --git a/lib/images/metrics.go b/lib/images/metrics.go index d860885b2..e58075905 100644 --- a/lib/images/metrics.go +++ b/lib/images/metrics.go @@ -11,11 +11,12 @@ import ( // Metrics holds the metrics instruments for image operations. type Metrics struct { - buildDuration metric.Float64Histogram - buildPhaseDuration metric.Float64Histogram - ociLayerCount metric.Int64Histogram - ociCompressedBytes metric.Int64Histogram - pullsTotal metric.Int64Counter + buildDuration metric.Float64Histogram + buildPhaseDuration metric.Float64Histogram + ociLayerCount metric.Int64Histogram + ociCompressedBytes metric.Int64Histogram + pullsTotal metric.Int64Counter + layerArtifactsEvicted metric.Int64Counter } // newMetrics creates and registers all image metrics. @@ -78,6 +79,14 @@ func newMetrics(meter metric.Meter, m *manager) (*Metrics, error) { return nil, err } + layerArtifactsEvicted, err := meter.Int64Counter( + "hypeman_images_layer_artifacts_evicted_total", + metric.WithDescription("Total number of shared layer artifacts evicted after their last reference was removed"), + ) + if err != nil { + return nil, err + } + // Register observable gauges for queue length and total images buildQueueLength, err := meter.Int64ObservableGauge( "hypeman_images_build_queue_length", @@ -123,11 +132,12 @@ func newMetrics(meter metric.Meter, m *manager) (*Metrics, error) { } return &Metrics{ - buildDuration: buildDuration, - buildPhaseDuration: buildPhaseDuration, - ociLayerCount: ociLayerCount, - ociCompressedBytes: ociCompressedBytes, - pullsTotal: pullsTotal, + buildDuration: buildDuration, + buildPhaseDuration: buildPhaseDuration, + ociLayerCount: ociLayerCount, + ociCompressedBytes: ociCompressedBytes, + pullsTotal: pullsTotal, + layerArtifactsEvicted: layerArtifactsEvicted, }, nil } @@ -169,3 +179,11 @@ func (m *manager) recordOCIImageMetrics(ctx context.Context, layerCount int, com m.metrics.ociLayerCount.Record(ctx, int64(layerCount), attrs) m.metrics.ociCompressedBytes.Record(ctx, compressedBytes, attrs) } + +// recordLayerArtifactsEvicted counts layer artifacts removed by eviction. +func (m *manager) recordLayerArtifactsEvicted(ctx context.Context, evicted int64) { + if m.metrics == nil { + return + } + m.metrics.layerArtifactsEvicted.Add(ctx, evicted) +} diff --git a/lib/images/oci.go b/lib/images/oci.go index 9fd191024..ec6791a1c 100644 --- a/lib/images/oci.go +++ b/lib/images/oci.go @@ -246,13 +246,21 @@ func (c *ociClient) pullAndExportWithPlatform(ctx context.Context, imageRef, dig } func (c *ociClient) pullAndExportWithPlatformAuth(ctx context.Context, imageRef, digest, exportDir string, platform gcr.Platform, credentials *authn.AuthConfig) (*pullResult, error) { - // Use a shared OCI layout for all images to enable automatic layer caching - // The cacheDir itself is the OCI layout root with shared blobs/sha256/ directory - // The digest is ALWAYS known at this point (from inspectManifest or digest reference) + result, err := c.pullManifestWithPlatformAuth(ctx, imageRef, digest, platform, credentials) + if err != nil { + return result, err + } + if err := result.measure("layer_unpack", func() error { + return c.composeRootfs(ctx, exportDir, digestToLayoutTag(digest), result.Manifest) + }); err != nil { + return result, fmt.Errorf("unpack layers: %w", err) + } + return result, nil +} + +func (c *ociClient) pullManifestWithPlatformAuth(ctx context.Context, imageRef, digest string, platform gcr.Platform, credentials *authn.AuthConfig) (*pullResult, error) { layoutTag := digestToLayoutTag(digest) result := &pullResult{Digest: digest} - - // Check if this digest is already cached cacheLookupStart := time.Now() result.CacheHit = c.existsInLayout(layoutTag) result.Phases = append(result.Phases, imageBuildPhaseMeasurement{ @@ -261,37 +269,25 @@ func (c *ociClient) pullAndExportWithPlatformAuth(ctx context.Context, imageRef, Status: "success", }) if !result.CacheHit { - // Not cached, pull it using digest-based tag if err := result.measure("registry_pull", func() error { return c.pullToOCILayoutWithPlatformAuth(ctx, imageRef, layoutTag, platform, credentials) }); err != nil { return result, fmt.Errorf("pull to oci layout: %w", err) } } - // If cached, we skip the pull entirely - // Extract metadata (from cache or freshly pulled) var bundle *ociImageBundle - err := result.measure("metadata_extract", func() error { + if err := result.measure("metadata_extract", func() error { var err error bundle, err = c.extractOCIImageBundle(layoutTag) return err - }) - if err != nil { + }); err != nil { return result, fmt.Errorf("extract metadata: %w", err) } result.Metadata = bundle.Meta result.Manifest = bundle.Model result.LayerCount = bundle.LayerCount result.CompressedBytes = bundle.CompressedBytes - - // Compose the rootfs from the shared layer blobs in manifest order. - if err := result.measure("layer_unpack", func() error { - return c.composeRootfs(ctx, exportDir, layoutTag, bundle.Model) - }); err != nil { - return result, fmt.Errorf("unpack layers: %w", err) - } - return result, nil } diff --git a/lib/images/runtime.go b/lib/images/runtime.go new file mode 100644 index 000000000..26a58a077 --- /dev/null +++ b/lib/images/runtime.go @@ -0,0 +1,219 @@ +package images + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "errors" + "fmt" + "os" + "os/exec" + "path/filepath" + "strings" + + "github.com/kernel/hypeman/lib/paths" +) + +// LayeredRuntimeManager prepares the writable overlay for an image whose +// read-only rootfs is a shared composed base. +type LayeredRuntimeManager interface { + RequiredOverlayBytes(ctx context.Context, image *Image) (int64, error) + PrepareInstanceOverlay(ctx context.Context, image *Image, overlayPath string, sizeBytes int64) error +} + +func sharedBaseDigest(model *imageManifestModel) string { + hash := sha256.New() + hash.Write([]byte("hypeman-base-v1\x00")) + for _, layer := range model.Layers[:model.BaseLayerCount] { + hash.Write([]byte(layer.Digest)) + hash.Write([]byte{0}) + } + return "sha256:" + hex.EncodeToString(hash.Sum(nil)) +} + +func (m *manager) buildSharedBase(ctx context.Context, model *imageManifestModel, buildDir string) (string, int64, error) { + baseRef := sharedBaseDigest(model) + result := m.layers.flights.DoChan("base\x00"+baseRef, func() (any, error) { + buildCtx, cancel := context.WithTimeout(context.Background(), layerBuildTimeout) + defer cancel() + select { + case m.layers.builds <- struct{}{}: + defer func() { <-m.layers.builds }() + case <-buildCtx.Done(): + return nil, buildCtx.Err() + } + m.layers.beginLayerBuild() + defer m.layers.endLayerBuild() + return m.buildSharedBaseOnce(buildCtx, model, buildDir, baseRef) + }) + select { + case <-ctx.Done(): + return "", 0, ctx.Err() + case shared := <-result: + if shared.Err != nil { + return "", 0, shared.Err + } + built := shared.Val.(sharedBaseBuildResult) + return built.digest, built.size, nil + } +} + +type sharedBaseBuildResult struct { + digest string + size int64 +} + +func (m *manager) buildSharedBaseOnce(ctx context.Context, model *imageManifestModel, buildDir, baseRef string) (sharedBaseBuildResult, error) { + if model.BaseLayerCount == 0 || model.BaseLayerCount >= len(model.Layers) { + return sharedBaseBuildResult{}, fmt.Errorf("invalid shared base layer count: %d", model.BaseLayerCount) + } + baseHex := strings.TrimPrefix(baseRef, "sha256:") + basePath := m.paths.ImageBasePath(baseHex) + if stat, err := os.Stat(basePath); err == nil { + if stat.Mode().IsRegular() && stat.Size() > 0 { + return sharedBaseBuildResult{digest: baseRef, size: stat.Size()}, nil + } + if err := removePath(basePath); err != nil { + return sharedBaseBuildResult{}, fmt.Errorf("discard invalid shared base: %w", err) + } + } else if !os.IsNotExist(err) { + return sharedBaseBuildResult{}, fmt.Errorf("stat shared base: %w", err) + } + + baseDir := filepath.Join(buildDir, "base-rootfs") + if err := m.ociClient.composeLayers(ctx, baseDir, model.Layers[:model.BaseLayerCount]); err != nil { + return sharedBaseBuildResult{}, fmt.Errorf("compose shared base: %w", err) + } + if err := os.MkdirAll(filepath.Dir(basePath), 0755); err != nil { + return sharedBaseBuildResult{}, fmt.Errorf("create shared base directory: %w", err) + } + tempDir, err := os.MkdirTemp(filepath.Dir(basePath), ".base-build-*") + if err != nil { + return sharedBaseBuildResult{}, fmt.Errorf("create shared base staging directory: %w", err) + } + defer removePath(tempDir) + tempPath := filepath.Join(tempDir, filepath.Base(basePath)) + size, err := ExportRootfsWithContext(ctx, baseDir, tempPath, DefaultImageFormat) + if err != nil { + return sharedBaseBuildResult{}, fmt.Errorf("export shared base: %w", err) + } + if err := installAtomically(basePath, func(path string) error { return os.Rename(tempPath, path) }); err != nil { + return sharedBaseBuildResult{}, fmt.Errorf("install shared base: %w", err) + } + return sharedBaseBuildResult{digest: baseRef, size: size}, nil +} + +func (m *manager) runtimeLayerArtifact(ctx context.Context, desc layerDescriptor) (*layerArtifact, error) { + layerHex := strings.TrimPrefix(desc.Digest, "sha256:") + record, err := readLayerRecord(m.paths, layerHex) + if err != nil && !errors.Is(err, errCorruptLayerRecord) { + return nil, err + } + if err != nil || record == nil || !record.matches(desc) { + return m.materializeLayerArtifact(ctx, desc) + } + info, err := os.Stat(layerArtifactPath(m.paths, layerHex)) + if err != nil || !info.Mode().IsRegular() || info.Size() != record.SizeBytes { + return m.materializeLayerArtifact(ctx, desc) + } + return record, nil +} + +func (m *manager) RequiredOverlayBytes(ctx context.Context, image *Image) (int64, error) { + model, err := readRuntimeManifestModel(m.paths, image.Name, strings.TrimPrefix(image.Digest, "sha256:")) + if err != nil { + return 0, err + } + if model == nil || model.BaseDigest == "" { + return 0, nil + } + if model.BaseLayerCount != len(model.Layers)-1 { + return 0, fmt.Errorf("unsupported layered image plan: %d base layers, %d total layers", model.BaseLayerCount, len(model.Layers)) + } + desc := model.Layers[model.BaseLayerCount] + record, err := m.runtimeLayerArtifact(ctx, desc) + if err != nil { + return 0, fmt.Errorf("prepare image layer artifact: %w", err) + } + return alignToSector(record.UnpackedBytes + record.UnpackedBytes/2), nil +} + +func (m *manager) PrepareInstanceOverlay(ctx context.Context, image *Image, overlayPath string, sizeBytes int64) error { + digestHex := strings.TrimPrefix(image.Digest, "sha256:") + model, err := readRuntimeManifestModel(m.paths, image.Name, digestHex) + if err != nil { + return fmt.Errorf("read image manifest model: %w", err) + } + if model == nil || model.BaseDigest == "" { + return CreateEmptyExt4Disk(overlayPath, sizeBytes) + } + if model.BaseLayerCount != len(model.Layers)-1 { + return fmt.Errorf("unsupported layered image plan: %d base layers, %d total layers", model.BaseLayerCount, len(model.Layers)) + } + basePath := m.paths.ImageBasePath(strings.TrimPrefix(model.BaseDigest, "sha256:")) + if _, err := os.Stat(basePath); err != nil { + return fmt.Errorf("stat shared image base: %w", err) + } + + desc := model.Layers[model.BaseLayerCount] + layerHex := strings.TrimPrefix(desc.Digest, "sha256:") + if _, err := m.runtimeLayerArtifact(ctx, desc); err != nil { + return fmt.Errorf("prepare image layer artifact: %w", err) + } + artifactPath := layerArtifactPath(m.paths, layerHex) + if _, err := os.Stat(artifactPath); err != nil { + return fmt.Errorf("stat image layer artifact: %w", err) + } + + stage, err := os.MkdirTemp(filepath.Dir(overlayPath), ".overlay-stage-*") + if err != nil { + return fmt.Errorf("create overlay staging directory: %w", err) + } + defer removePath(stage) + upper := filepath.Join(stage, "upper") + if err := os.Mkdir(upper, 0755); err != nil { + return fmt.Errorf("create overlay upper directory: %w", err) + } + work := filepath.Join(stage, "work") + if err := os.Mkdir(work, 0755); err != nil { + return fmt.Errorf("create overlay work directory: %w", err) + } + if err := extractLayerArtifact(ctx, artifactPath, upper); err != nil { + return fmt.Errorf("copy image layer into overlay: %w", err) + } + // The merged root inherits the upperdir's mode. fsck.erofs may restore the + // extracted layer root's mode, so set it after extraction. + if err := os.Chmod(upper, 0755); err != nil { + return fmt.Errorf("set overlay upper directory mode: %w", err) + } + + tempPath := overlayPath + ".tmp" + defer os.Remove(tempPath) + if err := CreateExt4DiskFromRootfsWithContext(ctx, stage, tempPath, sizeBytes); err != nil { + return fmt.Errorf("create populated overlay disk: %w", err) + } + if err := os.Rename(tempPath, overlayPath); err != nil { + return fmt.Errorf("install populated overlay disk: %w", err) + } + return nil +} + +func readRuntimeManifestModel(p *paths.Paths, imageName, digestHex string) (*imageManifestModel, error) { + if imageName != "" { + ref, err := ParseNormalizedRef(imageName) + if err != nil { + return nil, err + } + layout := resolveImageLayout(p, ref.Repository(), digestHex) + return readManifestModelAt(manifestModelPath(p, layout, digestHex), digestHex) + } + return readManifestModel(p, digestHex) +} + +func extractLayerArtifact(ctx context.Context, artifactPath, dest string) error { + cmd := exec.CommandContext(ctx, "fsck.erofs", "--extract="+dest, "--xattrs", "--preserve", artifactPath) + if output, err := cmd.CombinedOutput(); err != nil { + return fmt.Errorf("extract layer artifact: %w, output: %s", err, output) + } + return nil +} diff --git a/lib/images/runtime_test.go b/lib/images/runtime_test.go new file mode 100644 index 000000000..1751b37d5 --- /dev/null +++ b/lib/images/runtime_test.go @@ -0,0 +1,41 @@ +package images + +import ( + "strings" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestSharedBaseDigestDependsOnOrderedBaseLayers(t *testing.T) { + model := &imageManifestModel{ + BaseLayerCount: 2, + Layers: []layerDescriptor{ + {Digest: "sha256:" + strings.Repeat("a", 64)}, + {Digest: "sha256:" + strings.Repeat("b", 64)}, + {Digest: "sha256:" + strings.Repeat("c", 64)}, + }, + } + base := sharedBaseDigest(model) + + model.Layers[2].Digest = "sha256:" + strings.Repeat("d", 64) + require.Equal(t, base, sharedBaseDigest(model)) + + model.Layers[1].Digest = "sha256:" + strings.Repeat("d", 64) + require.NotEqual(t, base, sharedBaseDigest(model)) + require.True(t, strings.HasPrefix(base, "sha256:")) +} + +func TestValidateManifestModelRejectsInvalidSharedBase(t *testing.T) { + digest := strings.Repeat("a", 64) + model := &imageManifestModel{ + SchemaVersion: manifestModelSchemaVersion, + Digest: "sha256:" + digest, + RootFSType: "layers", + BaseDigest: "sha256:" + strings.Repeat("b", 64), + BaseLayerCount: 2, + Config: manifestConfigRef{Digest: "sha256:" + strings.Repeat("c", 64), MediaType: "application/vnd.oci.image.config.v1+json", DiffIDs: []string{"sha256:" + strings.Repeat("d", 64)}}, + Layers: []layerDescriptor{{Digest: "sha256:" + strings.Repeat("e", 64), DiffID: "sha256:" + strings.Repeat("d", 64)}}, + } + require.ErrorContains(t, validateManifestModel(digest, model), "base layer count") +} diff --git a/lib/images/storage.go b/lib/images/storage.go index 5b1d9e566..156ad18e4 100644 --- a/lib/images/storage.go +++ b/lib/images/storage.go @@ -292,8 +292,9 @@ func promoteImageToContent(p *paths.Paths, sourceRepository, digestHex string, s if err := writeMetadataFile(p.ImageContentMetadata(digestHex), &pendingMeta); err != nil { return fmt.Errorf("write content metadata: %w", err) } - if err := installAtomically(p.ImageContentPath(digestHex), func(path string) error { - return os.Link(sourceDiskPath, path) + contentDiskPath := p.ImageContentPath(digestHex) + if err := installAtomically(contentDiskPath, func(path string) error { + return linkImageDisk(sourceDiskPath, path, contentDiskPath) }); err != nil { return fmt.Errorf("link source disk: %w", err) } @@ -358,6 +359,28 @@ func promoteLegacyTags(p *paths.Paths, repository, digestHex string) error { return nil } +func linkImageDisk(sourcePath, targetPath, finalTargetPath string) error { + info, err := os.Lstat(sourcePath) + if err != nil { + return err + } + if info.Mode()&os.ModeSymlink == 0 { + return os.Link(sourcePath, targetPath) + } + target, err := os.Readlink(sourcePath) + if err != nil { + return err + } + if !filepath.IsAbs(target) { + target = filepath.Join(filepath.Dir(sourcePath), target) + } + relative, err := filepath.Rel(filepath.Dir(finalTargetPath), target) + if err != nil { + return err + } + return os.Symlink(relative, targetPath) +} + func installAtomically(path string, install func(string) error) error { if err := os.MkdirAll(filepath.Dir(path), 0755); err != nil { return err diff --git a/lib/images/storage_test.go b/lib/images/storage_test.go index b6479d615..0c6c2aaa1 100644 --- a/lib/images/storage_test.go +++ b/lib/images/storage_test.go @@ -3,6 +3,7 @@ package images import ( "os" "path/filepath" + "strings" "testing" "time" @@ -272,6 +273,51 @@ func TestPromoteImageToContentReplacesFailedContent(t *testing.T) { require.FileExists(t, p.ImageContentPath(digest)) } +func TestPromoteLayeredImageRebasesDiskLink(t *testing.T) { + p := paths.New(t.TempDir()) + repository := "docker.io/library/alpine" + digest := strings.Repeat("a", 64) + seedLegacy(t, p, repository, "latest", digest) + + model := &imageManifestModel{ + SchemaVersion: manifestModelSchemaVersion, + Digest: "sha256:" + digest, + RootFSType: "layers", + BaseLayerCount: 1, + Config: manifestConfigRef{ + Digest: "sha256:" + strings.Repeat("c", 64), + MediaType: "application/vnd.oci.image.config.v1+json", + DiffIDs: []string{"sha256:" + strings.Repeat("d", 64), "sha256:" + strings.Repeat("e", 64)}, + }, + Layers: []layerDescriptor{ + {Digest: "sha256:" + strings.Repeat("f", 64), DiffID: "sha256:" + strings.Repeat("d", 64)}, + {Digest: "sha256:" + strings.Repeat("0", 64), DiffID: "sha256:" + strings.Repeat("e", 64)}, + }, + } + baseDigest := strings.TrimPrefix(sharedBaseDigest(model), "sha256:") + model.BaseDigest = "sha256:" + baseDigest + basePath := p.ImageBasePath(baseDigest) + require.NoError(t, os.MkdirAll(filepath.Dir(basePath), 0o755)) + require.NoError(t, os.WriteFile(basePath, []byte("base"), 0o644)) + legacyDisk := p.ImageDigestPath(repository, digest) + require.NoError(t, os.Remove(legacyDisk)) + relative, err := filepath.Rel(filepath.Dir(legacyDisk), basePath) + require.NoError(t, err) + require.NoError(t, os.Symlink(relative, legacyDisk)) + legacyManifest := filepath.Join(p.ImageDigestDir(repository, digest), "manifest.json") + require.NoError(t, writeManifestModelAt(legacyManifest, digest, model)) + legacyMeta, err := readMetadata(p, repository, digest) + require.NoError(t, err) + + require.NoError(t, promoteImageToContent(p, repository, digest, legacyMeta)) + contentDisk := p.ImageContentPath(digest) + resolved, err := filepath.EvalSymlinks(contentDisk) + require.NoError(t, err) + expected, err := filepath.EvalSymlinks(basePath) + require.NoError(t, err) + require.Equal(t, expected, resolved) +} + func TestPromoteLegacyTagsRemovesStaleLegacySymlinks(t *testing.T) { p := paths.New(t.TempDir()) repository := "docker.io/library/alpine" diff --git a/lib/images/tag.go b/lib/images/tag.go index af7582476..c64415b43 100644 --- a/lib/images/tag.go +++ b/lib/images/tag.go @@ -92,7 +92,7 @@ func (m *manager) cleanupReplacedTag(ref *NormalizedRef, previousDigest, digestH if err := removeDigestIfUnreferenced(m.paths, ref.Repository(), previousDigest, true); err != nil { slog.Warn("failed to collect replaced image content", "repository", ref.Repository(), "digest", previousDigest, "error", err) } - m.layers.refreshDiskUsageTotals() + m.reconcileLayerStoreLocked() } func parseTagReferences(source, target string) (*NormalizedRef, *NormalizedRef, error) { diff --git a/lib/instances/admission.go b/lib/instances/admission.go index cc22a099f..8b2741aab 100644 --- a/lib/instances/admission.go +++ b/lib/instances/admission.go @@ -18,5 +18,12 @@ func storedDiskReservationBytes(stored *StoredMetadata) int64 { if stored == nil { return 0 } - return requestedDiskReservationBytes(stored.OverlaySize, stored.Volumes) + return requestedDiskReservationBytes(effectiveOverlayDiskSize(stored), stored.Volumes) +} + +func effectiveOverlayDiskSize(stored *StoredMetadata) int64 { + if stored.OverlayDiskSize > 0 { + return stored.OverlayDiskSize + } + return stored.OverlaySize } diff --git a/lib/instances/admission_allocations.go b/lib/instances/admission_allocations.go index 17d0f5e94..e4e47797a 100644 --- a/lib/instances/admission_allocations.go +++ b/lib/instances/admission_allocations.go @@ -145,7 +145,7 @@ func (m *manager) allocationFromStoredMetadata(stored *StoredMetadata, active bo Name: stored.Name, Vcpus: stored.Vcpus, MemoryBytes: stored.Size + stored.HotplugSize, - OverlayBytes: stored.OverlaySize, + OverlayBytes: effectiveOverlayDiskSize(stored), VolumeOverlayBytes: volumeOverlayBytes, NetworkDownloadBps: stored.NetworkBandwidthDownload, NetworkUploadBps: stored.NetworkBandwidthUpload, diff --git a/lib/instances/create.go b/lib/instances/create.go index 504b58e65..9b270e993 100644 --- a/lib/instances/create.go +++ b/lib/instances/create.go @@ -225,7 +225,18 @@ func (m *manager) createInstance( return nil, fmt.Errorf("total memory %d (size + hotplug_size) exceeds maximum allowed %d per instance", totalMemory, m.limits.MaxMemoryPerInstance) } - diskBytes := requestedDiskReservationBytes(overlaySize, req.Volumes) + overlayDiskSize := overlaySize + if preparer, ok := m.imageManager.(images.LayeredRuntimeManager); ok { + extraOverlayBytes, err := preparer.RequiredOverlayBytes(ctx, imageInfo) + if err != nil { + return nil, fmt.Errorf("calculate image overlay size: %w", err) + } + overlayDiskSize += extraOverlayBytes + if overlayDiskSize < overlaySize { + return nil, fmt.Errorf("image overlay size overflows") + } + } + diskBytes := requestedDiskReservationBytes(overlayDiskSize, req.Volumes) reservedResources := false // Reserve aggregate resources for this create while it is in flight. @@ -341,6 +352,7 @@ func (m *manager) createInstance( Size: size, HotplugSize: hotplugSize, OverlaySize: overlaySize, + OverlayDiskSize: overlayDiskSize, Vcpus: vcpus, NetworkBandwidthDownload: req.NetworkBandwidthDownload, // Will be set by caller if using resource manager NetworkBandwidthUpload: req.NetworkBandwidthUpload, // Will be set by caller if using resource manager @@ -410,9 +422,16 @@ func (m *manager) createInstance( log.InfoContext(ctx, "configured vGPU", "instance_id", id, "profile", gpuProfile, "uuid", gpuDevice.MdevUUID) } - // 13. Create overlay disk with specified size - log.DebugContext(ctx, "creating overlay disk", "instance_id", id, "size_bytes", stored.OverlaySize) - if err := m.createOverlayDisk(id, stored.OverlaySize); err != nil { + // 13. Create overlay disk with specified size. Layered images populate its + // upper directory with the image's final layer; test fakes and legacy images + // retain the empty-overlay path. + log.DebugContext(ctx, "creating overlay disk", "instance_id", id, "size_bytes", overlayDiskSize) + if preparer, ok := m.imageManager.(images.LayeredRuntimeManager); ok { + if err := preparer.PrepareInstanceOverlay(ctx, imageInfo, m.paths.InstanceOverlay(id), overlayDiskSize); err != nil { + log.ErrorContext(ctx, "failed to create layered overlay disk", "instance_id", id, "error", err) + return nil, fmt.Errorf("create overlay disk: %w", err) + } + } else if err := m.createOverlayDisk(id, overlayDiskSize); err != nil { log.ErrorContext(ctx, "failed to create overlay disk", "instance_id", id, "error", err) return nil, fmt.Errorf("create overlay disk: %w", err) } diff --git a/lib/instances/types.go b/lib/instances/types.go index efa057b31..45585b7fb 100644 --- a/lib/instances/types.go +++ b/lib/instances/types.go @@ -96,7 +96,8 @@ type StoredMetadata struct { // Resources (matching Cloud Hypervisor terminology) Size int64 // Base memory in bytes HotplugSize int64 // Hotplug memory in bytes - OverlaySize int64 // Overlay disk size in bytes + OverlaySize int64 // User-requested writable overlay capacity + OverlayDiskSize int64 `json:"overlay_disk_size,omitempty"` // Physical overlay disk size including image data Vcpus int NetworkBandwidthDownload int64 // Download rate limit in bytes/sec (external→VM), 0 = auto NetworkBandwidthUpload int64 // Upload rate limit in bytes/sec (VM→external), 0 = auto diff --git a/lib/network/bridge_linux.go b/lib/network/bridge_linux.go index 2e5bfc986..88b1b252a 100644 --- a/lib/network/bridge_linux.go +++ b/lib/network/bridge_linux.go @@ -21,7 +21,7 @@ import ( "golang.org/x/sys/unix" ) -const netlinkDumpRetryCount = 3 +const netlinkDumpRetryCount = 10 const iptablesWaitSeconds = "5" func newIPTablesCommand(args ...string) *exec.Cmd { @@ -46,7 +46,7 @@ func listBridgeAddrsWithRetry(link netlink.Link) ([]netlink.Addr, error) { return nil, listErr } err = listErr - time.Sleep(10 * time.Millisecond) + time.Sleep(time.Duration(i+1) * 10 * time.Millisecond) } return nil, err } @@ -62,7 +62,7 @@ func listLinksWithRetry() ([]netlink.Link, error) { return nil, listErr } err = listErr - time.Sleep(10 * time.Millisecond) + time.Sleep(time.Duration(i+1) * 10 * time.Millisecond) } return nil, err } diff --git a/lib/paths/paths.go b/lib/paths/paths.go index 9086242e6..3386e015c 100644 --- a/lib/paths/paths.go +++ b/lib/paths/paths.go @@ -178,6 +178,20 @@ func (p *Paths) ImageRepositoryTagSymlink(repository, tag string) string { return filepath.Join(p.ImageRepositoriesDir(), repository, tag) } +// ImageBasesDir returns the root directory of shared composed base images. +func (p *Paths) ImageBasesDir() string { + return filepath.Join(p.dataDir, "images", "bases") +} + +// ImageBasePath returns the path to one shared base rootfs disk. +func (p *Paths) ImageBasePath(digestHex string) string { + ext := "erofs" + if runtime.GOOS == "darwin" { + ext = "ext4" + } + return filepath.Join(p.ImageBasesDir(), digestHex, "rootfs."+ext) +} + // ImageLayersDir returns the root directory of the per-layer artifact store. // Layer artifacts are content-addressed by the compressed layer blob digest. func (p *Paths) ImageLayersDir() string { diff --git a/lib/resources/resource.go b/lib/resources/resource.go index f8f624e93..300446b4a 100644 --- a/lib/resources/resource.go +++ b/lib/resources/resource.go @@ -889,7 +889,7 @@ func (m *Manager) MaxImageStorageBytes() int64 { return int64(float64(capacity) * fraction) } -// CurrentImageStorageBytes returns the current image storage usage (OCI cache + rootfs). +// CurrentImageStorageBytes returns the current image storage usage (ready rootfs + OCI cache and shared layers). func (m *Manager) CurrentImageStorageBytes(ctx context.Context) (int64, error) { if m.imageLister == nil { return 0, nil