Shared-layer lifecycle: accounting, eviction, recovery, materialization

lib/images/README.md

@@ -5,7 +5,9 @@ Converts OCI images to bootable erofs disks for Cloud Hypervisor VMs.
 ## Architecture
 
 ```
-OCI Registry → go-containerregistry → OCI Layout → umoci → rootfs/ → mkfs.erofs → disk.erofs
+OCI Registry → go-containerregistry → OCI Layout → umoci
+  native Linux: shared base layers → base.erofs + final layer artifact → VM overlay
+  fallback: all layers → rootfs/ → mkfs.erofs → disk.erofs
 ```
 
 ## Design Decisions
@@ -65,7 +67,10 @@ Content-addressable storage with tag symlinks (similar to Docker/Unikraft):
         rootfs.erofs
       latest -> abc123def456...   # Tag symlink to digest
       3.18 -> def456abc123...     # Another tag
-    layers/                       # Shared materialized layer artifacts
+    bases/                        # Shared composed read-only base disks
+      789abc...
+        rootfs.erofs
+    layers/                       # Materialized final-layer artifacts
       abc123def456.../
         layer.erofs
         artifact.erofs.json
@@ -86,7 +91,8 @@ Content-addressable storage with tag symlinks (similar to Docker/Unikraft):
 - Natural hierarchy: All versions of an image grouped under repository
 - Easy inspection: Clear which digest belongs to which image
 - Layer caching: All images share the same blob storage, layers deduplicated automatically
-- Materialized layer artifacts are reference-protected and reconciled by the layer lifecycle manager; stale temporary trees are age-gated before removal.
+- Native Linux images share a composed base disk; the final layer is copied into each VM's writable overlay at instance creation.
+- Materialized layer artifacts and shared bases are reference-protected and reconciled by the image lifecycle manager; stale temporary trees are age-gated before removal.
 
 **Design:**
 - Images stored by manifest digest (content hash)
@@ -95,6 +101,7 @@ Content-addressable storage with tag symlinks (similar to Docker/Unikraft):
 - Pulling same tag twice updates the symlink if digest changed
 - OCI cache uses digest hex as layout tag for true content-addressable caching
 - Shared blob storage enables automatic layer deduplication across all images
+- Shared base disks avoid exporting and storing a complete rootfs for each image variant
 - Orphaned digests are automatically deleted when the last tag referencing them is removed
 - Symlinks only created after successful build (status: ready)
 - Disk accounting uses logical file sizes, matching image metadata and storage admission rather than filesystem block allocation.

lib/images/compose.go

@@ -8,23 +8,30 @@ import (
 	"path/filepath"
 )
 
-// composeRootfs validates the persisted model and merges its layers into
-// dest in manifest order, reading each layer blob from the shared OCI cache.
-// Whiteout and opaque-directory markers are interpreted as each layer is
-// applied. Any previous tree at dest is replaced: callers must not read dest
-// concurrently, and a failure between the remove and the rename leaves dest
-// absent. The export root is always 0755 regardless of the last layer's tar
-// root entry, matching the mode the previous unpack path created. A crash
-// can also strand .compose-* staging directories in dest's parent, the same
-// way .unpack-* directories can strand under layer builds.
+// composeRootfs validates the persisted model and merges its layers into dest
+// in manifest order. Whiteouts are applied to the composed tree.
 func (c *ociClient) composeRootfs(ctx context.Context, dest, layoutTag string, model *imageManifestModel) error {
 	if err := validateManifestModel(layoutTag, model); err != nil {
 		return fmt.Errorf("validate manifest model: %w", err)
 	}
+	return c.composeLayerTree(ctx, dest, model.Layers)
+}
+
+func (c *ociClient) composeLayers(ctx context.Context, dest string, layers []layerDescriptor) error {
+	return c.composeLayerTree(ctx, dest, layers)
+}
+
+func (c *ociClient) composeLayerTree(ctx context.Context, dest string, layers []layerDescriptor) error {
 	parent := filepath.Dir(dest)
 	if err := os.MkdirAll(parent, 0755); err != nil {
 		return fmt.Errorf("create compose parent: %w", err)
 	}
+	leftovers, _ := filepath.Glob(filepath.Join(parent, ".compose-*"))
+	for _, leftover := range leftovers {
+		if err := removePath(leftover); err != nil {
+			slog.Warn("failed to remove stale compose staging directory", "dir", leftover, "error", err)
+		}
+	}
 	staging, err := os.MkdirTemp(parent, ".compose-*")
 	if err != nil {
 		return fmt.Errorf("create compose directory: %w", err)
@@ -34,14 +41,9 @@ func (c *ociClient) composeRootfs(ctx context.Context, dest, layoutTag string, m
 			slog.Warn("failed to remove compose staging directory", "dir", staging, "error", err)
 		}
 	}()
-
-	for i, desc := range model.Layers {
-		if _, err := unpackCachedLayer(ctx, c.cacheBlobDir(), desc, staging, composeOnDiskFormat()); err != nil {
-			return fmt.Errorf("apply layer %d: %w", i, err)
-		}
+	if err := c.composeLayerList(ctx, staging, layers); err != nil {
+		return err
 	}
-	// The export directory must stay traversable by other readers; MkdirTemp
-	// creates it 0700.
 	if err := os.Chmod(staging, 0755); err != nil {
 		return fmt.Errorf("set compose directory mode: %w", err)
 	}
@@ -53,3 +55,12 @@ func (c *ociClient) composeRootfs(ctx context.Context, dest, layoutTag string, m
 	}
 	return nil
 }
+
+func (c *ociClient) composeLayerList(ctx context.Context, dest string, layers []layerDescriptor) error {
+	for i, desc := range layers {
+		if _, err := unpackCachedLayer(ctx, c.cacheBlobDir(), desc, dest, composeOnDiskFormat()); err != nil {
+			return fmt.Errorf("apply layer %d: %w", i, err)
+		}
+	}
+	return nil
+}

lib/images/disk.go

@@ -196,51 +196,55 @@ func convertToExt4(ctx context.Context, rootfsDir, diskPath string) (int64, erro
 	// Align to sector boundary (required by macOS Virtualization.framework)
 	diskSizeBytes = alignToSector(diskSizeBytes)
 
-	// Ensure parent directory exists
+	return createExt4Disk(ctx, rootfsDir, diskPath, diskSizeBytes)
+}
+
+func createExt4Disk(ctx context.Context, rootfsDir, diskPath string, diskSizeBytes int64) (int64, error) {
 	if err := os.MkdirAll(filepath.Dir(diskPath), 0755); err != nil {
 		return 0, fmt.Errorf("create disk parent dir: %w", err)
 	}
-
-	// Create sparse file
-	f, err := os.Create(diskPath)
+	file, err := os.OpenFile(diskPath, os.O_CREATE|os.O_TRUNC|os.O_RDWR, 0644)
 	if err != nil {
 		return 0, fmt.Errorf("create disk file: %w", err)
 	}
-	if err := f.Truncate(diskSizeBytes); err != nil {
-		f.Close()
+	if err := file.Truncate(alignToSector(diskSizeBytes)); err != nil {
+		_ = file.Close()
 		return 0, fmt.Errorf("truncate disk file: %w", err)
 	}
-	f.Close()
+	if err := file.Close(); err != nil {
+		return 0, fmt.Errorf("close disk file: %w", err)
+	}
 
-	// Format as ext4 with rootfs contents using mkfs.ext4
-	// -b 4096: 4KB blocks (standard, matches VM page size and sector alignment)
-	// -O ^has_journal: Disable journal (not needed for read-only VM mounts)
-	// -d: Copy directory contents into filesystem
-	// -F: Force creation (file not block device)
 	cmd := exec.CommandContext(ctx, mkfsExt4Binary(), "-b", "4096", "-O", "^has_journal", "-d", rootfsDir, "-F", diskPath)
 	output, err := cmd.CombinedOutput()
 	if err != nil {
 		return 0, fmt.Errorf("mkfs.ext4 failed: %w, output: %s", err, output)
 	}
-
-	// Verify final size is sector-aligned (mkfs.ext4 should preserve our truncated size)
 	stat, err := os.Stat(diskPath)
 	if err != nil {
 		return 0, fmt.Errorf("stat disk: %w", err)
 	}
-
-	// Re-align if mkfs.ext4 changed the size (shouldn't happen with -F on a regular file)
 	if stat.Size()%sectorSize != 0 {
 		alignedSize := alignToSector(stat.Size())
 		if err := os.Truncate(diskPath, alignedSize); err != nil {
 			return 0, fmt.Errorf("align disk to sector boundary: %w", err)
 		}
 		return alignedSize, nil
 	}
-
 	return stat.Size(), nil
 }
 
+// CreateExt4DiskFromRootfs creates a fixed-size ext4 disk populated with a directory tree.
+func CreateExt4DiskFromRootfs(rootfsDir, diskPath string, sizeBytes int64) error {
+	return CreateExt4DiskFromRootfsWithContext(context.Background(), rootfsDir, diskPath, sizeBytes)
+}
+
+// CreateExt4DiskFromRootfsWithContext is the cancellable form of CreateExt4DiskFromRootfs.
+func CreateExt4DiskFromRootfsWithContext(ctx context.Context, rootfsDir, diskPath string, sizeBytes int64) error {
+	_, err := createExt4Disk(ctx, rootfsDir, diskPath, sizeBytes)
+	return err
+}
+
 // convertToErofs converts a rootfs directory to an erofs disk image using mkfs.erofs
 func convertToErofs(ctx context.Context, rootfsDir, diskPath string) (int64, error) {
 	// Ensure parent directory exists

lib/images/disk_usage.go

@@ -6,6 +6,7 @@ import (
 	"fmt"
 	"os"
 	"path/filepath"
+	"strings"
 	"syscall"
 )
 
@@ -75,18 +76,23 @@ func totalReadyImageBytesFromMetadataWithContext(ctx context.Context, imagesDir
 			if globErr != nil {
 				return fmt.Errorf("find ready image rootfs for %s: %w", path, globErr)
 			}
+			sharedBase := false
 			for _, rootfsPath := range rootfsPaths {
 				rootfsInfo, statErr := os.Stat(rootfsPath)
 				if statErr != nil {
 					continue
 				}
+				sharedBase = isSharedBaseLink(rootfsPath, filepath.Join(imagesDir, "bases"))
 				if !markUniqueRootfs(rootfsInfo, seenRootfs) {
 					return nil
 				}
 				break
 			}
 
 			if meta.SizeBytes > 0 {
+				if sharedBase {
+					return nil
+				}
 				total += meta.SizeBytes
 				return nil
 			}
@@ -233,7 +239,11 @@ func (s *layerStore) computeDiskUsageTotals(ctx context.Context) (int64, int64,
 	if err != nil {
 		return 0, 0, err
 	}
-	return readyImageBytes, ociCacheBytes + layerArtifactBytes, nil
+	baseBytes, err := totalFileBytesWithContext(ctx, s.paths.ImageBasesDir(), "shared image bases")
+	if err != nil {
+		return 0, 0, err
+	}
+	return readyImageBytes + baseBytes, ociCacheBytes + layerArtifactBytes, nil
 }
 
 func totalRootfsBytesInDigestDirWithContext(ctx context.Context, digestDir string, seen map[rootfsIdentity]struct{}) (int64, error) {
@@ -273,6 +283,23 @@ func totalRootfsBytesInDigestDirWithContext(ctx context.Context, digestDir strin
 	return total, nil
 }
 
+func isSharedBaseLink(path, basesDir string) bool {
+	info, err := os.Lstat(path)
+	if err != nil || info.Mode()&os.ModeSymlink == 0 {
+		return false
+	}
+	resolved, err := filepath.EvalSymlinks(path)
+	if err != nil {
+		return false
+	}
+	canonicalBasesDir, err := filepath.EvalSymlinks(basesDir)
+	if err != nil {
+		return false
+	}
+	relative, err := filepath.Rel(canonicalBasesDir, resolved)
+	return err == nil && relative != "." && relative != ".." && !strings.HasPrefix(relative, ".."+string(filepath.Separator))
+}
+
 type rootfsIdentity struct {
 	dev uint64
 	ino uint64

lib/images/layer_artifact.go

@@ -11,6 +11,7 @@ import (
 	"io/fs"
 	"log/slog"
 	"os"
+	"os/exec"
 	"path/filepath"
 	"runtime"
 	"strings"
@@ -147,6 +148,9 @@ func probeLayerArtifactSupport(layersDir string) bool {
 	if !supportsLayerArtifacts() || os.MkdirAll(layersDir, 0755) != nil {
 		return false
 	}
+	if _, err := exec.LookPath("fsck.erofs"); err != nil {
+		return false
+	}
 	probeDir, err := os.MkdirTemp(layersDir, ".probe-*")
 	if err != nil {
 		return false
@@ -166,7 +170,7 @@ func probeLayerArtifactSupport(layersDir string) bool {
 }
 
 func (m *manager) layerArtifactSupport() bool {
-	return m.layers.artifactsSupported
+	return m.layers != nil && m.layers.artifactsSupported
 }
 
 func (m *manager) materializeLayerArtifact(ctx context.Context, desc layerDescriptor) (*layerArtifact, error) {
@@ -245,9 +249,8 @@ func (s *layerStore) clearLayerCache(layerHex string) error {
 // The layer is unpacked into an isolated temp directory, converted to the
 // default image format, and installed atomically. Normal failures remove the
 // temp directory; a crash mid-build can leave a stale .unpack-* directory
-// behind, which reconciliation landing with the pull integration is expected
-// to sweep. No production caller yet: pull integration and
-// composition land in later changes.
+// behind, which the startup sweep in layer_gc.go removes once it ages past
+// the eviction grace period.
 //
 // Concurrent callers share one build. The build itself is detached from the
 // initiating caller's cancellation so one cancelled pull cannot fail every

lib/images/layer_artifact_test.go

@@ -34,23 +34,25 @@ const whiteoutPrefix = ".wh."
 
 const testTarGzMediaType = "application/vnd.oci.image.layer.v1.tar+gzip"
 
-// writeLayerTestLayout writes img into the shared OCI cache of p tagged with
-// the image's digest, mirroring pullToOCILayout.
+// writeLayerTestLayout writes images into the shared OCI cache of p tagged
+// with each image's digest, mirroring pullToOCILayout.
 func testLayerArtifactManager(p *paths.Paths) *manager {
 	store := newLayerStore(p, 1)
 	store.artifactsSupported = true
 	return &manager{paths: p, layers: store}
 }
 
-func writeLayerTestLayout(t *testing.T, p *paths.Paths, img gcr.Image) {
+func writeLayerTestLayout(t *testing.T, p *paths.Paths, imgs ...gcr.Image) {
 	t.Helper()
-	digest, err := img.Digest()
-	require.NoError(t, err)
 	layoutPath, err := layout.Write(p.SystemOCICache(), empty.Index)
 	require.NoError(t, err)
-	require.NoError(t, layoutPath.AppendImage(img, layout.WithAnnotations(map[string]string{
-		"org.opencontainers.image.ref.name": digestToLayoutTag(digest.String()),
-	})))
+	for _, img := range imgs {
+		digest, err := img.Digest()
+		require.NoError(t, err)
+		require.NoError(t, layoutPath.AppendImage(img, layout.WithAnnotations(map[string]string{
+			"org.opencontainers.image.ref.name": digestToLayoutTag(digest.String()),
+		})))
+	}
 }
 
 func layerDescFromImage(t *testing.T, img gcr.Image, index int) layerDescriptor {