diff --git a/storage/drivers/overlay/overlay.go b/storage/drivers/overlay/overlay.go index b488256097..8e1baab9e1 100644 --- a/storage/drivers/overlay/overlay.go +++ b/storage/drivers/overlay/overlay.go @@ -112,6 +112,7 @@ type overlayOptions struct { mountOptions string ignoreChownErrors bool forceMask *os.FileMode + dirmetaDelegate bool useComposefs bool } @@ -465,7 +466,9 @@ func Init(home string, options graphdriver.Options) (graphdriver.Driver, error) } func parseOptions(options []string) (*overlayOptions, error) { - o := &overlayOptions{} + o := &overlayOptions{ + dirmetaDelegate: true, // default on + } for _, option := range options { key, val, err := parsers.ParseKeyValueOpt(option) if err != nil { @@ -577,6 +580,12 @@ func parseOptions(options []string) (*overlayOptions, error) { if err != nil { return nil, err } + case "dirmeta_delegate": + logrus.Debugf("overlay: dirmeta_delegate=%s", val) + o.dirmetaDelegate, err = strconv.ParseBool(val) + if err != nil { + return nil, err + } case "force_mask": logrus.Debugf("overlay: force_mask=%s", val) var mask int64 @@ -2485,6 +2494,7 @@ func (d *Driver) applyDiff(target string, options graphdriver.ApplyDiffOpts) (si ForceMask: d.options.forceMask, WhiteoutFormat: d.getWhiteoutFormat(), InUserNS: unshare.IsRootless(), + DirmetaDelegate: d.options.dirmetaDelegate, }); err != nil { return 0, err } diff --git a/storage/pkg/archive/archive.go b/storage/pkg/archive/archive.go index bce24e5af5..70cb0c2faf 100644 --- a/storage/pkg/archive/archive.go +++ b/storage/pkg/archive/archive.go @@ -70,6 +70,12 @@ type ( ForceMask *os.FileMode // Timestamp, if set, will be set in each header as create/mod/access time Timestamp *time.Time + // DirmetaDelegate, if set, causes implicitly-created parent directories + // (structural directories not present as entries in the tar stream) to be + // marked with the overlay dirmeta_delegate xattr. This tells overlayfs to + // delegate metadata (timestamps, ownership, mode) for these directories to + // a lower layer, preserving the meaningful metadata from base image layers. + DirmetaDelegate bool } ) @@ -1135,7 +1141,11 @@ loop: parent := filepath.Dir(hdr.Name) parentPath := filepath.Join(dest, parent) if err := fileutils.Lexists(parentPath); err != nil && os.IsNotExist(err) { - err = idtools.MkdirAllAndChownNew(parentPath, 0o777, rootIDs) + if options.DirmetaDelegate { + err = mkdirAllAndChownWithDirmetaDelegate(parentPath, 0o777, rootIDs) + } else { + err = idtools.MkdirAllAndChownNew(parentPath, 0o777, rootIDs) + } if err != nil { return err } diff --git a/storage/pkg/archive/archive_linux.go b/storage/pkg/archive/archive_linux.go index 4613ee32f7..33782c5c12 100644 --- a/storage/pkg/archive/archive_linux.go +++ b/storage/pkg/archive/archive_linux.go @@ -176,6 +176,83 @@ func isWhiteOut(stat os.FileInfo) bool { return major(uint64(s.Rdev)) == 0 && minor(uint64(s.Rdev)) == 0 //nolint:unconvert } +// setDirmetaDelegateXattr sets the overlay dirmeta_delegate xattr on a directory, +// telling overlayfs to delegate metadata lookups to a lower layer. +func setDirmetaDelegateXattr(path string) error { + xattrName := GetOverlayXattrName("dirmeta_delegate") + return system.Lsetxattr(path, xattrName, []byte("y"), 0) +} + +// mkdirAllWithDirmetaDelegate creates all directories in path that don't exist, +// and sets the overlay dirmeta_delegate xattr on each newly-created directory. +// This marks them as structural directories whose metadata should be delegated +// to a lower overlayfs layer. +func mkdirAllWithDirmetaDelegate(path string, mode os.FileMode) error { + // Find the deepest existing ancestor so we know which dirs are new + existing := path + var toCreate []string + for { + if fi, err := os.Lstat(existing); err == nil && fi.IsDir() { + break + } + toCreate = append(toCreate, filepath.Base(existing)) + parent := filepath.Dir(existing) + if parent == existing { + break + } + existing = parent + } + + // Create each missing component and set the xattr + current := existing + for i := len(toCreate) - 1; i >= 0; i-- { + current = filepath.Join(current, toCreate[i]) + if err := os.Mkdir(current, mode); err != nil && !os.IsExist(err) { + return err + } + if err := setDirmetaDelegateXattr(current); err != nil { + return err + } + } + return nil +} + +// mkdirAllAndChownWithDirmetaDelegate creates all directories in path that don't +// exist, chowns newly-created directories, and sets the overlay dirmeta_delegate +// xattr on each newly-created directory. +func mkdirAllAndChownWithDirmetaDelegate(path string, mode os.FileMode, ids idtools.IDPair) error { + // Find the deepest existing ancestor + existing := path + var toCreate []string + for { + if fi, err := os.Lstat(existing); err == nil && fi.IsDir() { + break + } + toCreate = append(toCreate, filepath.Base(existing)) + parent := filepath.Dir(existing) + if parent == existing { + break + } + existing = parent + } + + // Create each missing component, chown it, and set the xattr + current := existing + for i := len(toCreate) - 1; i >= 0; i-- { + current = filepath.Join(current, toCreate[i]) + if err := os.Mkdir(current, mode); err != nil && !os.IsExist(err) { + return err + } + if err := idtools.SafeChown(current, ids.UID, ids.GID); err != nil { + return err + } + if err := setDirmetaDelegateXattr(current); err != nil { + return err + } + } + return nil +} + func GetFileOwner(path string) (uint32, uint32, uint32, error) { f, err := os.Stat(path) if err != nil { diff --git a/storage/pkg/archive/archive_linux_test.go b/storage/pkg/archive/archive_linux_test.go index 5c244b87ce..71abb60e3c 100644 --- a/storage/pkg/archive/archive_linux_test.go +++ b/storage/pkg/archive/archive_linux_test.go @@ -2,12 +2,15 @@ package archive import ( "archive/tar" + "bytes" "io" "os" "path/filepath" "syscall" "testing" + "time" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "go.podman.io/storage/pkg/system" "golang.org/x/sys/unix" @@ -201,3 +204,127 @@ func TestNestedOverlayWhiteouts(t *testing.T) { require.NoError(t, err) checkFileMode(t, filepath.Join(dst, "foo"), os.ModeDevice|os.ModeCharDevice) } + +func checkDirmetaDelegate(t *testing.T, path string, expected string) { + t.Helper() + xattrName := GetOverlayXattrName("dirmeta_delegate") + val, err := system.Lgetxattr(path, xattrName) + require.NoError(t, err) + assert.Equal(t, expected, string(val), "unexpected dirmeta_delegate xattr value for %s", path) +} + +// makeTarBuf creates a tar archive in memory from a list of headers and +// optional content. For TypeReg entries, content is the file data; for +// other types content is ignored. +func makeTarBuf(t *testing.T, entries []tar.Header, contents map[string][]byte) *bytes.Buffer { + t.Helper() + var buf bytes.Buffer + tw := tar.NewWriter(&buf) + for _, hdr := range entries { + hdr := hdr + if hdr.Typeflag == tar.TypeReg { + data := contents[hdr.Name] + hdr.Size = int64(len(data)) + require.NoError(t, tw.WriteHeader(&hdr)) + if len(data) > 0 { + _, err := tw.Write(data) + require.NoError(t, err) + } + } else { + require.NoError(t, tw.WriteHeader(&hdr)) + } + } + require.NoError(t, tw.Close()) + return &buf +} + +func TestDirmetaDelegate(t *testing.T) { + epoch := time.Unix(0, 0) + + t.Run("implicit dirs get xattr", func(t *testing.T) { + // Tar contains only a file at foo/bar/file with no directory + // entries. UnpackLayer must create foo/ and foo/bar/ implicitly. + buf := makeTarBuf(t, []tar.Header{ + {Typeflag: tar.TypeReg, Name: "foo/bar/file", Mode: 0o644}, + }, map[string][]byte{ + "foo/bar/file": []byte("hello"), + }) + + dst := t.TempDir() + _, err := UnpackLayer(dst, buf, &TarOptions{DirmetaDelegate: true, IgnoreChownErrors: true}) + require.NoError(t, err) + + // The file should exist. + _, err = os.Lstat(filepath.Join(dst, "foo/bar/file")) + require.NoError(t, err) + + // Both implicit parent dirs should have the xattr. + checkDirmetaDelegate(t, filepath.Join(dst, "foo"), "y") + checkDirmetaDelegate(t, filepath.Join(dst, "foo/bar"), "y") + }) + + t.Run("explicit dirs do not get xattr", func(t *testing.T) { + // Tar contains an explicit directory entry for foo/ with a + // specific mtime, followed by a file foo/file. + buf := makeTarBuf(t, []tar.Header{ + {Typeflag: tar.TypeDir, Name: "foo/", Mode: 0o755, ModTime: epoch}, + {Typeflag: tar.TypeReg, Name: "foo/file", Mode: 0o644}, + }, map[string][]byte{ + "foo/file": []byte("world"), + }) + + dst := t.TempDir() + _, err := UnpackLayer(dst, buf, &TarOptions{DirmetaDelegate: true, IgnoreChownErrors: true}) + require.NoError(t, err) + + // The file should exist. + _, err = os.Lstat(filepath.Join(dst, "foo/file")) + require.NoError(t, err) + + // The explicit directory should NOT have the xattr. + checkDirmetaDelegate(t, filepath.Join(dst, "foo"), "") + }) + + t.Run("mixed explicit and implicit", func(t *testing.T) { + // Tar contains an explicit dir explicit/, but the file beneath + // it is at explicit/implicit-child/file — so implicit-child/ + // must be created implicitly. + buf := makeTarBuf(t, []tar.Header{ + {Typeflag: tar.TypeDir, Name: "explicit/", Mode: 0o755, ModTime: epoch}, + {Typeflag: tar.TypeReg, Name: "explicit/implicit-child/file", Mode: 0o644}, + }, map[string][]byte{ + "explicit/implicit-child/file": []byte("data"), + }) + + dst := t.TempDir() + _, err := UnpackLayer(dst, buf, &TarOptions{DirmetaDelegate: true, IgnoreChownErrors: true}) + require.NoError(t, err) + + // explicit/ was in the tar stream — no xattr. + checkDirmetaDelegate(t, filepath.Join(dst, "explicit"), "") + + // implicit-child/ was NOT in the tar stream — should have xattr. + checkDirmetaDelegate(t, filepath.Join(dst, "explicit/implicit-child"), "y") + }) + + t.Run("disabled does not set xattr", func(t *testing.T) { + // Same tar as "implicit dirs get xattr" but with DirmetaDelegate + // disabled. + buf := makeTarBuf(t, []tar.Header{ + {Typeflag: tar.TypeReg, Name: "foo/file", Mode: 0o644}, + }, map[string][]byte{ + "foo/file": []byte("hello"), + }) + + dst := t.TempDir() + _, err := UnpackLayer(dst, buf, &TarOptions{DirmetaDelegate: false, IgnoreChownErrors: true}) + require.NoError(t, err) + + // The file should exist. + _, err = os.Lstat(filepath.Join(dst, "foo/file")) + require.NoError(t, err) + + // No xattr should be set when the feature is off. + checkDirmetaDelegate(t, filepath.Join(dst, "foo"), "") + }) +} diff --git a/storage/pkg/archive/archive_other.go b/storage/pkg/archive/archive_other.go index 1943c09187..4d8de7d8b3 100644 --- a/storage/pkg/archive/archive_other.go +++ b/storage/pkg/archive/archive_other.go @@ -2,6 +2,12 @@ package archive +import ( + "os" + + "go.podman.io/storage/pkg/idtools" +) + func GetWhiteoutConverter(_ WhiteoutFormat, _ any) TarWhiteoutConverter { return nil } @@ -9,3 +15,11 @@ func GetWhiteoutConverter(_ WhiteoutFormat, _ any) TarWhiteoutConverter { func GetFileOwner(path string) (uint32, uint32, uint32, error) { return 0, 0, 0, nil } + +func mkdirAllWithDirmetaDelegate(path string, mode os.FileMode) error { + return os.MkdirAll(path, mode) +} + +func mkdirAllAndChownWithDirmetaDelegate(path string, mode os.FileMode, ids idtools.IDPair) error { + return idtools.MkdirAllAndChownNew(path, mode, ids) +} diff --git a/storage/pkg/archive/diff.go b/storage/pkg/archive/diff.go index 355d65f212..8c473890a6 100644 --- a/storage/pkg/archive/diff.go +++ b/storage/pkg/archive/diff.go @@ -83,7 +83,11 @@ func UnpackLayer(dest string, layer io.Reader, options *TarOptions) (size int64, parentPath := filepath.Join(dest, parent) if err := fileutils.Lexists(parentPath); err != nil && os.IsNotExist(err) { - err = os.MkdirAll(parentPath, 0o755) + if options.DirmetaDelegate { + err = mkdirAllWithDirmetaDelegate(parentPath, 0o755) + } else { + err = os.MkdirAll(parentPath, 0o755) + } if err != nil { return 0, err }