From 2f2b0210a357516898e4cebaf31f287aa3c3e949 Mon Sep 17 00:00:00 2001 From: Paul Holzinger Date: Mon, 15 Sep 2025 15:53:58 +0200 Subject: [PATCH 01/37] storage/internal/tempdir: add StageAddition() Add a new function to stage additions. This should be used to extract the layer content into a temp directory without holding the storage lock and then under the lock just rename the directory into the final location to reduce the lock contention. Signed-off-by: Paul Holzinger (cherry picked from commit de050ac1d1125f483d8e08c991dd4e2a9ed5e175) Signed-off-by: Paul Holzinger --- storage/internal/tempdir/tempdir.go | 38 +++++++++++++++ storage/internal/tempdir/tempdir_test.go | 60 ++++++++++++++++++++++++ 2 files changed, 98 insertions(+) diff --git a/storage/internal/tempdir/tempdir.go b/storage/internal/tempdir/tempdir.go index 6522c45d18..bd852688c8 100644 --- a/storage/internal/tempdir/tempdir.go +++ b/storage/internal/tempdir/tempdir.go @@ -6,6 +6,7 @@ import ( "io/fs" "os" "path/filepath" + "strconv" "strings" "github.com/sirupsen/logrus" @@ -91,6 +92,26 @@ type TempDir struct { counter uint64 } +// StagedAddition is a temporary object which holds the information of where to +// put the data into and then use Commit() to move the data into the final location. +type StagedAddition struct { + // Path is the temporary path. The path is not created so caller must create + // a file or directory on it in order to use Commit(). The path is only valid + // until Commit() is called or until the TempDir instance Cleanup() method is used. + Path string +} + +// Commit the staged content into its final destination by using os.Rename(). +// That means the dest must be on the same on the same fs as the root directory +// that was given to NewTempDir() and the dest must not exist yet. +// Commit must only be called once per instance returned from the +// StagedAddition() call. +func (s *StagedAddition) Commit(destination string) error { + err := os.Rename(s.Path, destination) + s.Path = "" // invalidate Path to avoid reuse + return err +} + // CleanupTempDirFunc is a function type that can be returned by operations // which need to perform cleanup actions later. type CleanupTempDirFunc func() error @@ -190,6 +211,23 @@ func NewTempDir(rootDir string) (*TempDir, error) { return td, nil } +// StageAddition creates a new temporary path that is returned as field in the StagedAddition +// struct. The returned type StagedAddition has a Commit() function to move the content from +// the temporary location to the final one. +// +// The caller MUST call Commit() before Cleanup() is called on the TempDir, otherwise the +// staged content will be deleted and the Commit() will fail. +// If the TempDir has been cleaned up already, this method will return an error. +func (td *TempDir) StageAddition() (*StagedAddition, error) { + if td.tempDirLock == nil { + return nil, fmt.Errorf("temp dir instance not initialized or already cleaned up") + } + fileName := strconv.FormatUint(td.counter, 10) + "-addition" + tmpAddPath := filepath.Join(td.tempDirPath, fileName) + td.counter++ + return &StagedAddition{Path: tmpAddPath}, nil +} + // StageDeletion moves the specified file into the instance's temporary directory. // The temporary directory must already exist (created during NewTempDir). // Files are renamed with a counter-based prefix (e.g., "0-filename", "1-filename") to ensure uniqueness. diff --git a/storage/internal/tempdir/tempdir_test.go b/storage/internal/tempdir/tempdir_test.go index a556950524..8298321b1e 100644 --- a/storage/internal/tempdir/tempdir_test.go +++ b/storage/internal/tempdir/tempdir_test.go @@ -271,3 +271,63 @@ func TestTempDirFileNaming(t *testing.T) { assert.True(t, found, "Expected file %s not found", expectedName) } } + +func TestStageAddition(t *testing.T) { + rootDir := t.TempDir() + td, err := NewTempDir(rootDir) + require.NoError(t, err) + t.Cleanup(func() { + assert.NoError(t, td.Cleanup()) + }) + + sa1, err := td.StageAddition() + require.NoError(t, err) + // Path should not be created by StagedAddition. + assert.NoFileExists(t, sa1.Path) + + // ensure we can create file + f, err := os.Create(sa1.Path) + require.NoError(t, err) + require.NoError(t, f.Close()) + + // need to use a dest file which does not exist yet + dest := filepath.Join(t.TempDir(), "file1") + + err = sa1.Commit(dest) + require.NoError(t, err) + assert.FileExists(t, dest) + assert.NoFileExists(t, sa1.Path) + + // now test the same with a directory + sa2, err := td.StageAddition() + require.NoError(t, err) + // Path should not be created by StagedAddition. + assert.NoDirExists(t, sa2.Path) + + // ensure we can create a directory + err = os.Mkdir(sa2.Path, 0o755) + require.NoError(t, err) + + // need to use a dest which does not exist yet + dest = filepath.Join(t.TempDir(), "dir") + + err = sa2.Commit(dest) + require.NoError(t, err) + assert.DirExists(t, dest) + assert.NoDirExists(t, sa2.Path) + + // Commit the same stage addition struct again should error + err = sa2.Commit(dest) + require.Error(t, err) + + // Cleanup() should cleanup the temp paths from StagedAddition + sa3, err := td.StageAddition() + require.NoError(t, err) + + err = os.Mkdir(sa3.Path, 0o755) + require.NoError(t, err) + + err = td.Cleanup() + require.NoError(t, err) + assert.NoDirExists(t, sa3.Path) +} From 7f23d54ee69134a3978c384d1f1a8bb3fb204b2c Mon Sep 17 00:00:00 2001 From: Paul Holzinger Date: Wed, 8 Oct 2025 12:40:56 +0200 Subject: [PATCH 02/37] storage: simplify ApplyDiff() in overlay driver It is not clear to me when it will hit the code path there, by normal layer creation we always pass a valid parent so this branch is never reached AFAICT. Let's remove it and see if all tests still pass in podman, buildah and others... Signed-off-by: Paul Holzinger (cherry picked from commit 1b00a691de836601cd062fac64f8c1dd52cf87d9) Signed-off-by: Paul Holzinger --- storage/drivers/overlay/overlay.go | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/storage/drivers/overlay/overlay.go b/storage/drivers/overlay/overlay.go index c08e060466..5e894216c1 100644 --- a/storage/drivers/overlay/overlay.go +++ b/storage/drivers/overlay/overlay.go @@ -2371,16 +2371,6 @@ func (d *Driver) DifferTarget(id string) (string, error) { // ApplyDiff applies the new layer into a root func (d *Driver) ApplyDiff(id, parent string, options graphdriver.ApplyDiffOpts) (size int64, err error) { - if !d.isParent(id, parent) { - if d.options.ignoreChownErrors { - options.IgnoreChownErrors = d.options.ignoreChownErrors - } - if d.options.forceMask != nil { - options.ForceMask = d.options.forceMask - } - return d.naiveDiff.ApplyDiff(id, parent, options) - } - idMappings := options.Mappings if idMappings == nil { idMappings = &idtools.IDMappings{} From 10ab058d151a65ac4a8f3a4fe07806fb82e451f2 Mon Sep 17 00:00:00 2001 From: Paul Holzinger Date: Mon, 10 Nov 2025 18:45:09 +0100 Subject: [PATCH 03/37] overlay: add getLayerPermissions() Split out the layer permission gathering from the main create() function so it can be reused elsehwere, see the next commit. Signed-off-by: Paul Holzinger (cherry picked from commit 3517ebaaa46281d464c8e4756b5a9abbfb0b7320) Signed-off-by: Paul Holzinger --- storage/drivers/overlay/overlay.go | 74 ++++++++++++++++++------------ 1 file changed, 44 insertions(+), 30 deletions(-) diff --git a/storage/drivers/overlay/overlay.go b/storage/drivers/overlay/overlay.go index 5e894216c1..f30a1957dc 100644 --- a/storage/drivers/overlay/overlay.go +++ b/storage/drivers/overlay/overlay.go @@ -995,27 +995,15 @@ func (d *Driver) Create(id, parent string, opts *graphdriver.CreateOpts) (retErr return d.create(id, parent, opts, true) } -func (d *Driver) create(id, parent string, opts *graphdriver.CreateOpts, readOnly bool) (retErr error) { - dir, homedir, _ := d.dir2(id, readOnly) - - disableQuota := readOnly - - var uidMaps []idtools.IDMap - var gidMaps []idtools.IDMap - - if opts != nil && opts.IDMappings != nil { - uidMaps = opts.IDMappings.UIDs() - gidMaps = opts.IDMappings.GIDs() - } - - // Make the link directory if it does not exist - if err := idtools.MkdirAllAs(path.Join(homedir, linkDir), 0o755, 0, 0); err != nil { - return err - } - +// getLayerPermissions returns the base permissions to use for the layer directories. +// The first return value is the idPair to create the possible parent directories with. +// The second return value is the mode how it should be stored on disk. +// The third return value is the mode the layer expects to have which may be stored +// in an xattr when using forceMask, without forceMask both values are the same. +func (d *Driver) getLayerPermissions(parent string, uidMaps, gidMaps []idtools.IDMap) (idtools.IDPair, idtools.Stat, idtools.Stat, error) { rootUID, rootGID, err := idtools.GetRootUIDGID(uidMaps, gidMaps) if err != nil { - return err + return idtools.IDPair{}, idtools.Stat{}, idtools.Stat{}, err } idPair := idtools.IDPair{ @@ -1023,10 +1011,6 @@ func (d *Driver) create(id, parent string, opts *graphdriver.CreateOpts, readOnl GID: rootGID, } - if err := idtools.MkdirAllAndChownNew(path.Dir(dir), 0o755, idPair); err != nil { - return err - } - st := idtools.Stat{IDs: idPair, Mode: defaultPerms} if parent != "" { @@ -1037,7 +1021,7 @@ func (d *Driver) create(id, parent string, opts *graphdriver.CreateOpts, readOnl } else { systemSt, err := system.Stat(parentDiff) if err != nil { - return err + return idtools.IDPair{}, idtools.Stat{}, idtools.Stat{}, err } st.IDs.UID = int(systemSt.UID()) st.IDs.GID = int(systemSt.GID()) @@ -1045,6 +1029,42 @@ func (d *Driver) create(id, parent string, opts *graphdriver.CreateOpts, readOnl } } + forcedSt := st + if d.options.forceMask != nil { + forcedSt.IDs = idPair + forcedSt.Mode = *d.options.forceMask + } + + return idPair, forcedSt, st, nil +} + +func (d *Driver) create(id, parent string, opts *graphdriver.CreateOpts, readOnly bool) (retErr error) { + dir, homedir, _ := d.dir2(id, readOnly) + + disableQuota := readOnly + + var uidMaps []idtools.IDMap + var gidMaps []idtools.IDMap + + if opts != nil && opts.IDMappings != nil { + uidMaps = opts.IDMappings.UIDs() + gidMaps = opts.IDMappings.GIDs() + } + + // Make the link directory if it does not exist + if err := idtools.MkdirAllAs(path.Join(homedir, linkDir), 0o755, 0, 0); err != nil { + return err + } + + idPair, forcedSt, st, err := d.getLayerPermissions(parent, uidMaps, gidMaps) + if err != nil { + return err + } + + if err := idtools.MkdirAllAndChownNew(path.Dir(dir), 0o755, idPair); err != nil { + return err + } + if err := fileutils.Lexists(dir); err == nil { logrus.Warnf("Trying to create a layer %#v while directory %q already exists; removing it first", id, dir) // Don’t just os.RemoveAll(dir) here; d.Remove also removes the link in linkDir, @@ -1088,12 +1108,6 @@ func (d *Driver) create(id, parent string, opts *graphdriver.CreateOpts, readOnl } } - forcedSt := st - if d.options.forceMask != nil { - forcedSt.IDs = idPair - forcedSt.Mode = *d.options.forceMask - } - diff := path.Join(dir, "diff") if err := idtools.MkdirAs(diff, forcedSt.Mode, forcedSt.IDs.UID, forcedSt.IDs.GID); err != nil { return err From 35cbb6fff6263aa028c9d0aedd9362a0e4857883 Mon Sep 17 00:00:00 2001 From: Paul Holzinger Date: Fri, 10 Oct 2025 14:34:40 +0200 Subject: [PATCH 04/37] overlay: add StartStagingDiffToApply() Add a function to apply the diff into a tmporary directory so we can do that unlocked and only rename under the lock. Signed-off-by: Paul Holzinger (cherry picked from commit d1cc083c6cd601b8255e2a01af3755369af76389) Signed-off-by: Paul Holzinger --- storage/drivers/driver.go | 13 ++++ storage/drivers/overlay/overlay.go | 97 +++++++++++++++++++++++-- storage/drivers/overlay/overlay_test.go | 3 + 3 files changed, 105 insertions(+), 8 deletions(-) diff --git a/storage/drivers/driver.go b/storage/drivers/driver.go index fed80751b5..e43e1176de 100644 --- a/storage/drivers/driver.go +++ b/storage/drivers/driver.go @@ -299,6 +299,19 @@ type DriverWithDiffer interface { DifferTarget(id string) (string, error) } +// ApplyDiffStaging is an interface for driver who can apply the diff without holding the main storage lock. +// This API is experimental and can be changed without bumping the major version number. +type ApplyDiffStaging interface { + // StartStagingDiffToApply applies the new layer into a temporary directory. + // It returns a CleanupTempDirFunc which can be nil or set regardless if the function return an error or not. + // StagedAddition is only set when there is no error returned and the int64 value returns the size of the layer. + // This can be done without holding the storage lock, if a parent is given the caller must check for existence + // beforehand while holding a lock. + StartStagingDiffToApply(parent string, options ApplyDiffOpts) (tempdir.CleanupTempDirFunc, *tempdir.StagedAddition, int64, error) + // CommitStagedLayer commits the staged layer from StartStagingDiffToApply(). This must be done while holding the storage lock. + CommitStagedLayer(id string, commit *tempdir.StagedAddition) error +} + // Capabilities defines a list of capabilities a driver may implement. // These capabilities are not required; however, they do determine how a // graphdriver can be used. diff --git a/storage/drivers/overlay/overlay.go b/storage/drivers/overlay/overlay.go index f30a1957dc..b621c35c47 100644 --- a/storage/drivers/overlay/overlay.go +++ b/storage/drivers/overlay/overlay.go @@ -1370,6 +1370,14 @@ func (d *Driver) getTempDirRoot(id string) string { return filepath.Join(d.home, tempDirName) } +// getTempDirRootForNewLayer returns the correct temp directory root based on where +// the layer should be created. +// +// This must be kept in sync with GetTempDirRootDirs(). +func (d *Driver) getTempDirRootForNewLayer() string { + return filepath.Join(d.homeDirForImageStore(), tempDirName) +} + func (d *Driver) DeferredRemove(id string) (tempdir.CleanupTempDirFunc, error) { tempDirRoot := d.getTempDirRoot(id) t, err := tempdir.NewTempDir(tempDirRoot) @@ -2383,21 +2391,94 @@ func (d *Driver) DifferTarget(id string) (string, error) { return d.getDiffPath(id) } -// ApplyDiff applies the new layer into a root -func (d *Driver) ApplyDiff(id, parent string, options graphdriver.ApplyDiffOpts) (size int64, err error) { - idMappings := options.Mappings - if idMappings == nil { - idMappings = &idtools.IDMappings{} +// StartStagingDiffToApply applies the new layer into a temporary directory. +// It returns a CleanupTempDirFunc which can be nil or set regardless if the function return an error or not. +// StagedAddition is only set when there is no error returned and the int64 value returns the size of the layer. +// This can be done without holding the storage lock, if a parent is given the caller must check for existence +// beforehand while holding a lock. +// +// This API is experimental and can be changed without bumping the major version number. +func (d *Driver) StartStagingDiffToApply(parent string, options graphdriver.ApplyDiffOpts) (tempdir.CleanupTempDirFunc, *tempdir.StagedAddition, int64, error) { + tempDirRoot := d.getTempDirRootForNewLayer() + t, err := tempdir.NewTempDir(tempDirRoot) + if err != nil { + return nil, nil, -1, err + } + + sa, err := t.StageAddition() + if err != nil { + return t.Cleanup, nil, -1, err + } + + _, forcedSt, st, err := d.getLayerPermissions(parent, options.Mappings.UIDs(), options.Mappings.GIDs()) + if err != nil { + // If we have a ENOENT it means the parent was removed which can happen as we are unlocked here. + // In this case also wrap ErrLayerUnknown which some callers can handle to retry after recreating the parent. + if errors.Is(err, fs.ErrNotExist) { + err = fmt.Errorf("parent layer %q: %w: %w", parent, graphdriver.ErrLayerUnknown, err) + } + return t.Cleanup, nil, -1, err + } + + if err := idtools.MkdirAs(sa.Path, forcedSt.Mode, forcedSt.IDs.UID, forcedSt.IDs.GID); err != nil { + return t.Cleanup, nil, -1, err + } + + if d.options.forceMask != nil { + st.Mode |= os.ModeDir + if err := idtools.SetContainersOverrideXattr(sa.Path, st); err != nil { + return t.Cleanup, nil, -1, err + } } + size, err := d.applyDiff(sa.Path, options) + if err != nil { + return t.Cleanup, nil, -1, err + } + + return t.Cleanup, sa, size, nil +} + +// CommitStagedLayer that was created with StartStagingDiffToApply(). +// +// This API is experimental and can be changed without bumping the major version number. +func (d *Driver) CommitStagedLayer(id string, sa *tempdir.StagedAddition) error { + applyDir, err := d.getDiffPath(id) + if err != nil { + return err + } + + // The os.Rename() function used by CommitFunc errors when the target directory already + // exists, as such delete the dir. The create() function creates it and it would be more + // complicated to code in a way that it didn't create it. + if err := os.Remove(applyDir); err != nil { + return err + } + + return sa.Commit(applyDir) +} + +// ApplyDiff applies the new layer into a root +func (d *Driver) ApplyDiff(id, parent string, options graphdriver.ApplyDiffOpts) (size int64, err error) { applyDir, err := d.getDiffPath(id) if err != nil { return 0, err } + return d.applyDiff(applyDir, options) +} + +// ApplyDiff applies the new layer into a root. +// This can run concurrently with any other driver operations, as such it is the +// callers responsibility to ensure the target path passed is safe to use if that is the case. +func (d *Driver) applyDiff(target string, options graphdriver.ApplyDiffOpts) (size int64, err error) { + idMappings := options.Mappings + if idMappings == nil { + idMappings = &idtools.IDMappings{} + } - logrus.Debugf("Applying tar in %s", applyDir) + logrus.Debugf("Applying tar in %s", target) // Overlay doesn't need the parent id to apply the diff - if err := untar(options.Diff, applyDir, &archive.TarOptions{ + if err := untar(options.Diff, target, &archive.TarOptions{ UIDMaps: idMappings.UIDs(), GIDMaps: idMappings.GIDs(), IgnoreChownErrors: d.options.ignoreChownErrors, @@ -2408,7 +2489,7 @@ func (d *Driver) ApplyDiff(id, parent string, options graphdriver.ApplyDiffOpts) return 0, err } - return directory.Size(applyDir) + return directory.Size(target) } func (d *Driver) getComposefsData(id string) string { diff --git a/storage/drivers/overlay/overlay_test.go b/storage/drivers/overlay/overlay_test.go index dea86bb5b6..d1e96e7f14 100644 --- a/storage/drivers/overlay/overlay_test.go +++ b/storage/drivers/overlay/overlay_test.go @@ -17,6 +17,9 @@ import ( const driverName = "overlay" +// check that Driver correctly implements the ApplyDiffTemporary interface +var _ graphdriver.ApplyDiffStaging = &Driver{} + func init() { // Do not sure chroot to speed run time and allow archive // errors or hangs to be debugged directly from the test process. From c2357f921d286c8e89ae3048f075bee38acffa02 Mon Sep 17 00:00:00 2001 From: Paul Holzinger Date: Tue, 21 Oct 2025 15:48:39 +0200 Subject: [PATCH 05/37] storage: don't buffer tar split file in memory I cannot see any reason why we should buffer the full tar split content in memory before writing it. That layer is still mark partial at this point and the store is locked so there is no concurrent access either thus we do not need the atomic rename here. Signed-off-by: Paul Holzinger (cherry picked from commit 8620f7eeecf0bda25f8ba1c8cb71ad19de502a0f) Signed-off-by: Paul Holzinger --- storage/layers.go | 50 +++++++++++++++++++++++++++++++++++------------ 1 file changed, 37 insertions(+), 13 deletions(-) diff --git a/storage/layers.go b/storage/layers.go index 64d3f5c72c..093f67b8c8 100644 --- a/storage/layers.go +++ b/storage/layers.go @@ -31,6 +31,7 @@ import ( "go.podman.io/storage/pkg/ioutils" "go.podman.io/storage/pkg/lockfile" "go.podman.io/storage/pkg/mount" + "go.podman.io/storage/pkg/pools" "go.podman.io/storage/pkg/stringid" "go.podman.io/storage/pkg/system" "go.podman.io/storage/pkg/tarlog" @@ -2398,6 +2399,13 @@ func (r *layerStore) ApplyDiff(to string, diff io.Reader) (size int64, err error return r.applyDiffWithOptions(to, nil, diff) } +func createTarSplitFile(r *layerStore, layerID string) (*os.File, error) { + if err := os.MkdirAll(filepath.Dir(r.tspath(layerID)), 0o700); err != nil { + return nil, err + } + return os.OpenFile(r.tspath(layerID), os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0o600) +} + // Requires startWriting. func (r *layerStore) applyDiffWithOptions(to string, layerOptions *LayerOptions, diff io.Reader) (size int64, err error) { if !r.lockfile.IsReadWrite() { @@ -2442,13 +2450,20 @@ func (r *layerStore) applyDiffWithOptions(to string, layerOptions *LayerOptions, compressedCounter := ioutils.NewWriteCounter(compressedWriter) defragmented = io.TeeReader(defragmented, compressedCounter) - tsdata := bytes.Buffer{} + tarSplitFile, err := createTarSplitFile(r, layer.ID) + if err != nil { + return -1, err + } + defer tarSplitFile.Close() + tarSplitWriter := pools.BufioWriter32KPool.Get(tarSplitFile) + defer pools.BufioWriter32KPool.Put(tarSplitWriter) + uidLog := make(map[uint32]struct{}) gidLog := make(map[uint32]struct{}) var uncompressedCounter *ioutils.WriteCounter size, err = func() (int64, error) { // A scope for defer - compressor, err := pgzip.NewWriterLevel(&tsdata, pgzip.BestSpeed) + compressor, err := pgzip.NewWriterLevel(tarSplitWriter, pgzip.BestSpeed) if err != nil { return -1, err } @@ -2496,12 +2511,13 @@ func (r *layerStore) applyDiffWithOptions(to string, layerOptions *LayerOptions, return -1, err } - if err := os.MkdirAll(filepath.Dir(r.tspath(layer.ID)), 0o700); err != nil { - return -1, err + if err := tarSplitWriter.Flush(); err != nil { + return -1, fmt.Errorf("failed to flush tar-split writer buffer: %w", err) } - if err := ioutils.AtomicWriteFile(r.tspath(layer.ID), tsdata.Bytes(), 0o600); err != nil { - return -1, err + if err := tarSplitFile.Sync(); err != nil { + return -1, fmt.Errorf("sync tar-split file: %w", err) } + if compressedDigester != nil { compressedDigest = compressedDigester.Digest() } @@ -2597,10 +2613,17 @@ func (r *layerStore) applyDiffFromStagingDirectory(id string, diffOutput *driver } if diffOutput.TarSplit != nil { - tsdata := bytes.Buffer{} - compressor, err := pgzip.NewWriterLevel(&tsdata, pgzip.BestSpeed) + tarSplitFile, err := createTarSplitFile(r, layer.ID) if err != nil { - compressor = pgzip.NewWriter(&tsdata) + return err + } + defer tarSplitFile.Close() + tarSplitWriter := pools.BufioWriter32KPool.Get(tarSplitFile) + defer pools.BufioWriter32KPool.Put(tarSplitWriter) + + compressor, err := pgzip.NewWriterLevel(tarSplitWriter, pgzip.BestSpeed) + if err != nil { + compressor = pgzip.NewWriter(tarSplitWriter) } if _, err := diffOutput.TarSplit.Seek(0, io.SeekStart); err != nil { return err @@ -2614,11 +2637,12 @@ func (r *layerStore) applyDiffFromStagingDirectory(id string, diffOutput *driver return err } compressor.Close() - if err := os.MkdirAll(filepath.Dir(r.tspath(layer.ID)), 0o700); err != nil { - return err + + if err := tarSplitWriter.Flush(); err != nil { + return fmt.Errorf("failed to flush tar-split writer buffer: %w", err) } - if err := ioutils.AtomicWriteFile(r.tspath(layer.ID), tsdata.Bytes(), 0o600); err != nil { - return err + if err := tarSplitFile.Sync(); err != nil { + return fmt.Errorf("sync tar-split file: %w", err) } } for k, v := range diffOutput.BigData { From f6e07a413fbf6cc51913fe8731248304d32d0fa3 Mon Sep 17 00:00:00 2001 From: Paul Holzinger Date: Wed, 22 Oct 2025 18:21:04 +0200 Subject: [PATCH 06/37] storage: rework applyDiffWithOptions() Split it into multiple function to make it reusable without having a layer and so that it can be used unlocked see the following commits. Signed-off-by: Paul Holzinger (cherry picked from commit 5403dfc060c460a71e5571fa8a7cd4826f4efe1c) Signed-off-by: Paul Holzinger --- storage/layers.go | 153 +++++++++++++++++++++++++++++----------------- 1 file changed, 97 insertions(+), 56 deletions(-) diff --git a/storage/layers.go b/storage/layers.go index 093f67b8c8..1b293b2fab 100644 --- a/storage/layers.go +++ b/storage/layers.go @@ -203,6 +203,21 @@ type stagedLayerOptions struct { DiffOptions *drivers.ApplyDiffWithDifferOpts } +type applyDiffResult struct { + compressedDigest digest.Digest + compressedSize int64 + compressionType archive.Compression + uncompressedDigest digest.Digest + uncompressedSize int64 + // size of the data, including the full size of sparse files, and excluding all metadata + // It is neither compressedSize nor uncompressedSize. + // The use case for this seems unclear, it gets returned in PutLayer() but in the Podman + // stack at least that value is never used so maybe we can look into removing this. + size int64 + uids []uint32 + gids []uint32 +} + // roLayerStore wraps a graph driver, adding the ability to refer to layers by // name, and keeping track of parent-child relationships, along with a list of // all known layers. @@ -2406,37 +2421,29 @@ func createTarSplitFile(r *layerStore, layerID string) (*os.File, error) { return os.OpenFile(r.tspath(layerID), os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0o600) } -// Requires startWriting. -func (r *layerStore) applyDiffWithOptions(to string, layerOptions *LayerOptions, diff io.Reader) (size int64, err error) { - if !r.lockfile.IsReadWrite() { - return -1, fmt.Errorf("not allowed to modify layer contents at %q: %w", r.layerdir, ErrStoreIsReadOnly) - } - - layer, ok := r.lookup(to) - if !ok { - return -1, ErrLayerUnknown - } - +// applyDiff can be called without holding any store locks so if the supplied +// applyDriverFunc requires locking the caller must ensure proper locking. +func applyDiff(layerOptions *LayerOptions, diff io.Reader, tarSplitFile *os.File, applyDriverFunc func(io.Reader) (int64, error)) (*applyDiffResult, error) { header := make([]byte, 10240) n, err := diff.Read(header) if err != nil && err != io.EOF { - return -1, err + return nil, err } compression := archive.DetectCompression(header[:n]) defragmented := io.MultiReader(bytes.NewReader(header[:n]), diff) - // Decide if we need to compute digests - var compressedDigest, uncompressedDigest digest.Digest // = "" + result := applyDiffResult{} + var compressedDigester, uncompressedDigester digest.Digester // = nil if layerOptions != nil && layerOptions.OriginalDigest != "" && layerOptions.OriginalDigest.Algorithm() == digest.Canonical { - compressedDigest = layerOptions.OriginalDigest + result.compressedDigest = layerOptions.OriginalDigest } else { compressedDigester = digest.Canonical.Digester() } if layerOptions != nil && layerOptions.UncompressedDigest != "" && layerOptions.UncompressedDigest.Algorithm() == digest.Canonical { - uncompressedDigest = layerOptions.UncompressedDigest + result.uncompressedDigest = layerOptions.UncompressedDigest } else if compression != archive.Uncompressed { uncompressedDigester = digest.Canonical.Digester() } @@ -2450,11 +2457,6 @@ func (r *layerStore) applyDiffWithOptions(to string, layerOptions *LayerOptions, compressedCounter := ioutils.NewWriteCounter(compressedWriter) defragmented = io.TeeReader(defragmented, compressedCounter) - tarSplitFile, err := createTarSplitFile(r, layer.ID) - if err != nil { - return -1, err - } - defer tarSplitFile.Close() tarSplitWriter := pools.BufioWriter32KPool.Get(tarSplitFile) defer pools.BufioWriter32KPool.Put(tarSplitWriter) @@ -2462,7 +2464,7 @@ func (r *layerStore) applyDiffWithOptions(to string, layerOptions *LayerOptions, gidLog := make(map[uint32]struct{}) var uncompressedCounter *ioutils.WriteCounter - size, err = func() (int64, error) { // A scope for defer + size, err := func() (int64, error) { // A scope for defer compressor, err := pgzip.NewWriterLevel(tarSplitWriter, pgzip.BestSpeed) if err != nil { return -1, err @@ -2496,63 +2498,102 @@ func (r *layerStore) applyDiffWithOptions(to string, layerOptions *LayerOptions, if err != nil { return -1, err } - options := drivers.ApplyDiffOpts{ - Diff: payload, - Mappings: r.layerMappings(layer), - MountLabel: layer.MountLabel, - } - size, err := r.driver.ApplyDiff(layer.ID, layer.Parent, options) - if err != nil { - return -1, err - } - return size, err + + return applyDriverFunc(payload) }() if err != nil { - return -1, err + return nil, err } if err := tarSplitWriter.Flush(); err != nil { - return -1, fmt.Errorf("failed to flush tar-split writer buffer: %w", err) - } - if err := tarSplitFile.Sync(); err != nil { - return -1, fmt.Errorf("sync tar-split file: %w", err) + return nil, fmt.Errorf("failed to flush tar-split writer buffer: %w", err) } if compressedDigester != nil { - compressedDigest = compressedDigester.Digest() + result.compressedDigest = compressedDigester.Digest() } if uncompressedDigester != nil { - uncompressedDigest = uncompressedDigester.Digest() + result.uncompressedDigest = uncompressedDigester.Digest() } - if uncompressedDigest == "" && compression == archive.Uncompressed { - uncompressedDigest = compressedDigest + if result.uncompressedDigest == "" && compression == archive.Uncompressed { + result.uncompressedDigest = result.compressedDigest } - updateDigestMap(&r.bycompressedsum, layer.CompressedDigest, compressedDigest, layer.ID) - layer.CompressedDigest = compressedDigest if layerOptions != nil && layerOptions.OriginalDigest != "" && layerOptions.OriginalSize != nil { - layer.CompressedSize = *layerOptions.OriginalSize + result.compressedSize = *layerOptions.OriginalSize } else { - layer.CompressedSize = compressedCounter.Count + result.compressedSize = compressedCounter.Count } - updateDigestMap(&r.byuncompressedsum, layer.UncompressedDigest, uncompressedDigest, layer.ID) - layer.UncompressedDigest = uncompressedDigest - layer.UncompressedSize = uncompressedCounter.Count - layer.CompressionType = compression - layer.UIDs = make([]uint32, 0, len(uidLog)) + result.uncompressedSize = uncompressedCounter.Count + result.compressionType = compression + + result.uids = make([]uint32, 0, len(uidLog)) for uid := range uidLog { - layer.UIDs = append(layer.UIDs, uid) + result.uids = append(result.uids, uid) } - slices.Sort(layer.UIDs) - layer.GIDs = make([]uint32, 0, len(gidLog)) + slices.Sort(result.uids) + result.gids = make([]uint32, 0, len(gidLog)) for gid := range gidLog { - layer.GIDs = append(layer.GIDs, gid) + result.gids = append(result.gids, gid) + } + slices.Sort(result.gids) + + result.size = size + + return &result, err +} + +// Requires startWriting. +func (r *layerStore) applyDiffWithOptions(to string, layerOptions *LayerOptions, diff io.Reader) (int64, error) { + if !r.lockfile.IsReadWrite() { + return -1, fmt.Errorf("not allowed to modify layer contents at %q: %w", r.layerdir, ErrStoreIsReadOnly) + } + + layer, ok := r.lookup(to) + if !ok { + return -1, ErrLayerUnknown + } + + tarSplitFile, err := createTarSplitFile(r, layer.ID) + if err != nil { + return -1, err + } + defer tarSplitFile.Close() + + result, err := applyDiff(layerOptions, diff, tarSplitFile, func(payload io.Reader) (int64, error) { + options := drivers.ApplyDiffOpts{ + Diff: payload, + Mappings: r.layerMappings(layer), + MountLabel: layer.MountLabel, + } + return r.driver.ApplyDiff(layer.ID, layer.Parent, options) + }) + if err != nil { + return -1, err + } + + if err := tarSplitFile.Sync(); err != nil { + return -1, fmt.Errorf("sync tar-split file: %w", err) } - slices.Sort(layer.GIDs) + + r.applyDiffResultToLayer(layer, result) err = r.saveFor(layer) - return size, err + return result.size, err +} + +// Requires startWriting. +func (r *layerStore) applyDiffResultToLayer(layer *Layer, result *applyDiffResult) { + updateDigestMap(&r.bycompressedsum, layer.CompressedDigest, result.compressedDigest, layer.ID) + layer.CompressedDigest = result.compressedDigest + layer.CompressedSize = result.compressedSize + updateDigestMap(&r.byuncompressedsum, layer.UncompressedDigest, result.uncompressedDigest, layer.ID) + layer.UncompressedDigest = result.uncompressedDigest + layer.UncompressedSize = result.uncompressedSize + layer.CompressionType = result.compressionType + layer.UIDs = result.uids + layer.GIDs = result.gids } // Requires (startReading or?) startWriting. From 82a7f1502d17418a4ff7dd03ae20dc6a6a9cdc1e Mon Sep 17 00:00:00 2001 From: Paul Holzinger Date: Wed, 22 Oct 2025 20:13:54 +0200 Subject: [PATCH 07/37] storage: support staged addition of layers The extracting of the tar under the store lock is a bottleneck as many concurrent processes might hold the locks for a long time on big layers. To address this move the layer extraction before we take the locks if possible. Currently this only work when using the overlay driver as the implementation requires driver specifc details in order for a rename() to work. Signed-off-by: Paul Holzinger (cherry picked from commit c4432727b1b977f3b4f13ede5fe2bce2e68aae6d) Signed-off-by: Paul Holzinger --- storage/layers.go | 143 ++++++++++++++++++++++++++++++++++++++++++---- storage/store.go | 31 ++++++++-- storage/userns.go | 2 +- 3 files changed, 158 insertions(+), 18 deletions(-) diff --git a/storage/layers.go b/storage/layers.go index 1b293b2fab..def929aed5 100644 --- a/storage/layers.go +++ b/storage/layers.go @@ -199,8 +199,35 @@ type DiffOptions struct { // stagedLayerOptions are the options passed to .create to populate a staged // layer type stagedLayerOptions struct { + // These are used via the zstd:chunked pull paths DiffOutput *drivers.DriverWithDifferOutput DiffOptions *drivers.ApplyDiffWithDifferOpts + + // stagedLayerExtraction is used by the normal tar layer extraction. + stagedLayerExtraction *maybeStagedLayerExtraction +} + +// maybeStagedLayerExtraction is a helper to encapsulate details around extracting +// a layer potentially before we even take a look if the driver implements the +// ApplyDiffStaging interface. +// This should be initialized with layerStore.newMaybeStagedLayerExtraction() +type maybeStagedLayerExtraction struct { + // diff contains the tar archive, can be compressed, must be non nil, but can be at EOF when the content was already staged + diff io.Reader + // staging interface of the storage driver, set when the driver supports staging and nil otherwise + staging drivers.ApplyDiffStaging + // result is a placeholder for the applyDiff() result so we can pass that down the stack easily. + // If result is not nil the layer was staged successfully, if this is set stagedTarSplit and + // stagedLayer must be set as well. + result *applyDiffResult + + // stagedTarSplit is the temp file where we staged the tar split file + stagedTarSplit *tempdir.StagedAddition + // stagedLayer is the temp directory where we staged the extracted layer content + stagedLayer *tempdir.StagedAddition + + // cleanupFuncs contains the set of tempdir cleanup function that get executed in cleanup() + cleanupFuncs []tempdir.CleanupTempDirFunc } type applyDiffResult struct { @@ -304,7 +331,7 @@ type rwLayerStore interface { // underlying drivers do not themselves distinguish between writeable // and read-only layers. Returns the new layer structure and the size of the // diff which was applied to its parent to initialize its contents. - create(id string, parent *Layer, names []string, mountLabel string, options map[string]string, moreOptions *LayerOptions, writeable bool, diff io.Reader, slo *stagedLayerOptions) (*Layer, int64, error) + create(id string, parent *Layer, names []string, mountLabel string, options map[string]string, moreOptions *LayerOptions, writeable bool, slo *stagedLayerOptions) (*Layer, int64, error) // updateNames modifies names associated with a layer based on (op, names). updateNames(id string, names []string, op updateNameOperation) error @@ -370,6 +397,14 @@ type rwLayerStore interface { // Dedup deduplicates layers in the store. dedup(drivers.DedupArgs) (drivers.DedupResult, error) + + // newMaybeStagedLayerExtraction initializes a new maybeStagedLayerExtraction. The caller + // must call maybeStagedLayerExtraction.cleanup() to remove any temporary files. + newMaybeStagedLayerExtraction(diff io.Reader) *maybeStagedLayerExtraction + + // stageWithUnlockedStore stages the layer content without needing the store locked. + // If the driver does not support stage addition then this is a NOP and does nothing. + stageWithUnlockedStore(m *maybeStagedLayerExtraction, parent string, options *LayerOptions) error } type multipleLockFile struct { @@ -1395,7 +1430,7 @@ func (r *layerStore) pickStoreLocation(volatile, writeable bool) layerLocations } // Requires startWriting. -func (r *layerStore) create(id string, parentLayer *Layer, names []string, mountLabel string, options map[string]string, moreOptions *LayerOptions, writeable bool, diff io.Reader, slo *stagedLayerOptions) (layer *Layer, size int64, err error) { +func (r *layerStore) create(id string, parentLayer *Layer, names []string, mountLabel string, options map[string]string, moreOptions *LayerOptions, writeable bool, slo *stagedLayerOptions) (layer *Layer, size int64, err error) { if moreOptions == nil { moreOptions = &LayerOptions{} } @@ -1584,15 +1619,28 @@ func (r *layerStore) create(id string, parentLayer *Layer, names []string, mount } size = -1 - if diff != nil { - if size, err = r.applyDiffWithOptions(layer.ID, moreOptions, diff); err != nil { - cleanupFailureContext = "applying layer diff" - return nil, -1, err - } - } else if slo != nil { - if err := r.applyDiffFromStagingDirectory(layer.ID, slo.DiffOutput, slo.DiffOptions); err != nil { - cleanupFailureContext = "applying staged directory diff" - return nil, -1, err + if slo != nil { + if slo.stagedLayerExtraction != nil { + if slo.stagedLayerExtraction.result != nil { + // The layer is staged, just commit it and update the metadata. + if err := slo.stagedLayerExtraction.commitLayer(r, layer.ID); err != nil { + cleanupFailureContext = "committing staged layer diff" + return nil, -1, err + } + r.applyDiffResultToLayer(layer, slo.stagedLayerExtraction.result) + } else { + // The diff was not staged, apply it now here instead. + if size, err = r.applyDiffWithOptions(layer.ID, moreOptions, slo.stagedLayerExtraction.diff); err != nil { + cleanupFailureContext = "applying layer diff" + return nil, -1, err + } + } + } else { + // staging logic for the chunked pull path + if err := r.applyDiffFromStagingDirectory(layer.ID, slo.DiffOutput, slo.DiffOptions); err != nil { + cleanupFailureContext = "applying staged directory diff" + return nil, -1, err + } } } else { // applyDiffWithOptions() would have updated r.bycompressedsum @@ -2421,6 +2469,79 @@ func createTarSplitFile(r *layerStore, layerID string) (*os.File, error) { return os.OpenFile(r.tspath(layerID), os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0o600) } +// newMaybeStagedLayerExtraction initializes a new maybeStagedLayerExtraction. The caller +// must call maybeStagedLayerExtraction.cleanup() to remove any temporary files. +func (r *layerStore) newMaybeStagedLayerExtraction(diff io.Reader) *maybeStagedLayerExtraction { + m := &maybeStagedLayerExtraction{ + diff: diff, + } + if d, ok := r.driver.(drivers.ApplyDiffStaging); ok { + m.staging = d + } + return m +} + +func (sl *maybeStagedLayerExtraction) cleanup() error { + return tempdir.CleanupTemporaryDirectories(sl.cleanupFuncs...) +} + +// stageWithUnlockedStore stages the layer content without needing the store locked. +// If the driver does not support stage addition then this is a NOP and does nothing. +// This should be done without holding the storage lock, if a parent is given the caller +// must check for existence beforehand while holding a lock. +func (r *layerStore) stageWithUnlockedStore(sl *maybeStagedLayerExtraction, parent string, layerOptions *LayerOptions) error { + if sl.staging == nil { + return nil + } + td, err := tempdir.NewTempDir(filepath.Join(r.layerdir, tempDirPath)) + if err != nil { + return err + } + sl.cleanupFuncs = append(sl.cleanupFuncs, td.Cleanup) + + stagedTarSplit, err := td.StageAddition() + if err != nil { + return err + } + sl.stagedTarSplit = stagedTarSplit + + f, err := os.OpenFile(stagedTarSplit.Path, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0o600) + if err != nil { + return err + } + defer f.Close() + + result, err := applyDiff(layerOptions, sl.diff, f, func(payload io.Reader) (int64, error) { + cleanup, stagedLayer, size, err := sl.staging.StartStagingDiffToApply(parent, drivers.ApplyDiffOpts{ + Diff: payload, + Mappings: idtools.NewIDMappingsFromMaps(layerOptions.UIDMap, layerOptions.GIDMap), + // FIXME: What to do here? We have no lock and assigned label yet. + // Overlayfs should not need it anyway so this seems fine for now. + MountLabel: "", + }) + sl.cleanupFuncs = append(sl.cleanupFuncs, cleanup) + sl.stagedLayer = stagedLayer + return size, err + }) + if err != nil { + return err + } + + sl.result = result + return nil +} + +// commitLayer() commits the content that was staged in stageWithUnlockedStore() +// +// Requires startWriting. +func (sl *maybeStagedLayerExtraction) commitLayer(r *layerStore, layerID string) error { + err := sl.stagedTarSplit.Commit(r.tspath(layerID)) + if err != nil { + return err + } + return sl.staging.CommitStagedLayer(layerID, sl.stagedLayer) +} + // applyDiff can be called without holding any store locks so if the supplied // applyDriverFunc requires locking the caller must ensure proper locking. func applyDiff(layerOptions *LayerOptions, diff io.Reader, tarSplitFile *os.File, applyDriverFunc func(io.Reader) (int64, error)) (*applyDiffResult, error) { diff --git a/storage/store.go b/storage/store.go index 84019a4947..abdf08abd3 100644 --- a/storage/store.go +++ b/storage/store.go @@ -1452,7 +1452,7 @@ func (s *store) canUseShifting(uidmap, gidmap []idtools.IDMap) bool { // On entry: // - rlstore must be locked for writing // - rlstores MUST NOT be locked -func (s *store) putLayer(rlstore rwLayerStore, rlstores []roLayerStore, id, parent string, names []string, mountLabel string, writeable bool, lOptions *LayerOptions, diff io.Reader, slo *stagedLayerOptions) (*Layer, int64, error) { +func (s *store) putLayer(rlstore rwLayerStore, rlstores []roLayerStore, id, parent string, names []string, mountLabel string, writeable bool, lOptions *LayerOptions, slo *stagedLayerOptions) (*Layer, int64, error) { var parentLayer *Layer var options LayerOptions if lOptions != nil { @@ -1533,7 +1533,7 @@ func (s *store) putLayer(rlstore rwLayerStore, rlstores []roLayerStore, id, pare GIDMap: copySlicePreferringNil(gidMap), } } - return rlstore.create(id, parentLayer, names, mountLabel, nil, &options, writeable, diff, slo) + return rlstore.create(id, parentLayer, names, mountLabel, nil, &options, writeable, slo) } func (s *store) PutLayer(id, parent string, names []string, mountLabel string, writeable bool, lOptions *LayerOptions, diff io.Reader) (*Layer, int64, error) { @@ -1541,11 +1541,30 @@ func (s *store) PutLayer(id, parent string, names []string, mountLabel string, w if err != nil { return nil, -1, err } + + var slo *stagedLayerOptions + + if diff != nil { + m := rlstore.newMaybeStagedLayerExtraction(diff) + defer func() { + if err := m.cleanup(); err != nil { + logrus.Errorf("Error cleaning up temporary directories: %v", err) + } + }() + err = rlstore.stageWithUnlockedStore(m, parent, lOptions) + if err != nil { + return nil, -1, err + } + slo = &stagedLayerOptions{ + stagedLayerExtraction: m, + } + } + if err := rlstore.startWriting(); err != nil { return nil, -1, err } defer rlstore.stopWriting() - return s.putLayer(rlstore, rlstores, id, parent, names, mountLabel, writeable, lOptions, diff, nil) + return s.putLayer(rlstore, rlstores, id, parent, names, mountLabel, writeable, lOptions, slo) } func (s *store) CreateLayer(id, parent string, names []string, mountLabel string, writeable bool, options *LayerOptions) (*Layer, error) { @@ -1753,7 +1772,7 @@ func (s *store) imageTopLayerForMapping(image *Image, ristore roImageStore, rlst } } layerOptions.TemplateLayer = layer.ID - mappedLayer, _, err := rlstore.create("", parentLayer, nil, layer.MountLabel, nil, &layerOptions, false, nil, nil) + mappedLayer, _, err := rlstore.create("", parentLayer, nil, layer.MountLabel, nil, &layerOptions, false, nil) if err != nil { return nil, fmt.Errorf("creating an ID-mapped copy of layer %q: %w", layer.ID, err) } @@ -1924,7 +1943,7 @@ func (s *store) CreateContainer(id string, names []string, image, layer, metadat options.Flags[mountLabelFlag] = mountLabel } - clayer, _, err := rlstore.create(layer, imageTopLayer, nil, mlabel, options.StorageOpt, layerOptions, true, nil, nil) + clayer, _, err := rlstore.create(layer, imageTopLayer, nil, mlabel, options.StorageOpt, layerOptions, true, nil) if err != nil { return nil, err } @@ -3186,7 +3205,7 @@ func (s *store) ApplyStagedLayer(args ApplyStagedLayerOptions) (*Layer, error) { DiffOutput: args.DiffOutput, DiffOptions: args.DiffOptions, } - layer, _, err = s.putLayer(rlstore, rlstores, args.ID, args.ParentLayer, args.Names, args.MountLabel, args.Writeable, args.LayerOptions, nil, &slo) + layer, _, err = s.putLayer(rlstore, rlstores, args.ID, args.ParentLayer, args.Names, args.MountLabel, args.Writeable, args.LayerOptions, &slo) return layer, err } diff --git a/storage/userns.go b/storage/userns.go index b3d76a31bf..91bfc27c90 100644 --- a/storage/userns.go +++ b/storage/userns.go @@ -197,7 +197,7 @@ outer: // We need to create a temporary layer so we can mount it and lookup the // maximum IDs used. - clayer, _, err := rlstore.create("", topLayer, nil, "", nil, layerOptions, false, nil, nil) + clayer, _, err := rlstore.create("", topLayer, nil, "", nil, layerOptions, false, nil) if err != nil { return 0, err } From fea087c0ae0cce6dd7b824ae68fde392986b9f52 Mon Sep 17 00:00:00 2001 From: Paul Holzinger Date: Tue, 28 Oct 2025 17:48:11 +0100 Subject: [PATCH 08/37] storage: don't lock container store in putLayer It doesn't seem needed here so don't take it. Signed-off-by: Paul Holzinger (cherry picked from commit 1cecfee0968792b745eeea7f0578a81afe528ff6) Signed-off-by: Paul Holzinger --- storage/store.go | 7 ------- 1 file changed, 7 deletions(-) diff --git a/storage/store.go b/storage/store.go index abdf08abd3..61d8707626 100644 --- a/storage/store.go +++ b/storage/store.go @@ -1509,13 +1509,6 @@ func (s *store) putLayer(rlstore rwLayerStore, rlstores []roLayerStore, id, pare gidMap = ilayer.GIDMap } } else { - // FIXME? It’s unclear why we are holding containerStore locked here at all - // (and because we are not modifying it, why it is a write lock, not a read lock). - if err := s.containerStore.startWriting(); err != nil { - return nil, -1, err - } - defer s.containerStore.stopWriting() - if !options.HostUIDMapping && len(options.UIDMap) == 0 { uidMap = s.uidMap } From 2404422e319eed355cd867269a53390b84135c8f Mon Sep 17 00:00:00 2001 From: Paul Holzinger Date: Tue, 28 Oct 2025 19:12:00 +0100 Subject: [PATCH 09/37] storage: extract layer mapping into separate function A minor rework to enable more changes in following commits. Note the caller still must hold the layer store locks so ensure we return the layer locked and let the caller unlock instead. Signed-off-by: Paul Holzinger (cherry picked from commit ba7580b9cf34f647abd60b3bd3f03c15b7d85994) Signed-off-by: Paul Holzinger --- storage/store.go | 50 ++++++++++++++++++++++++++++++++++++++---------- 1 file changed, 40 insertions(+), 10 deletions(-) diff --git a/storage/store.go b/storage/store.go index 61d8707626..6c568d804a 100644 --- a/storage/store.go +++ b/storage/store.go @@ -1452,7 +1452,26 @@ func (s *store) canUseShifting(uidmap, gidmap []idtools.IDMap) bool { // On entry: // - rlstore must be locked for writing // - rlstores MUST NOT be locked -func (s *store) putLayer(rlstore rwLayerStore, rlstores []roLayerStore, id, parent string, names []string, mountLabel string, writeable bool, lOptions *LayerOptions, slo *stagedLayerOptions) (*Layer, int64, error) { +func (s *store) putLayer(rlstore rwLayerStore, id string, parentLayer *Layer, names []string, mountLabel string, writeable bool, options *LayerOptions, slo *stagedLayerOptions) (*Layer, int64, error) { + return rlstore.create(id, parentLayer, names, mountLabel, nil, options, writeable, slo) +} + +// On entry: +// - rlstore must be locked for writing +// - rlstores MUST NOT be locked +// +// Returns the new copied LayerOptions with mappings set, the parent Layer and +// an extra unlock function to unlock any potentially read locked rlstores by this function. +// The unlock function is always set and thus must always be called. +func populateLayerOptions(s *store, rlstore rwLayerStore, rlstores []roLayerStore, parent string, lOptions *LayerOptions) (*LayerOptions, *Layer, func(), error) { + // function we return to the caller so the caller gets the right stores locked and can unlock at the proper time themselves + var lockedLayerStores []roLayerStore + unlock := func() { + for _, i := range lockedLayerStores { + i.stopReading() + } + } + var parentLayer *Layer var options LayerOptions if lOptions != nil { @@ -1474,9 +1493,9 @@ func (s *store) putLayer(rlstore rwLayerStore, rlstores []roLayerStore, id, pare lstore := l if lstore != rlstore { if err := lstore.startReading(); err != nil { - return nil, -1, err + return nil, nil, unlock, err } - defer lstore.stopReading() + lockedLayerStores = append(lockedLayerStores, lstore) } if l, err := lstore.Get(parent); err == nil && l != nil { ilayer = l @@ -1485,21 +1504,21 @@ func (s *store) putLayer(rlstore rwLayerStore, rlstores []roLayerStore, id, pare } } if ilayer == nil { - return nil, -1, ErrLayerUnknown + return nil, nil, unlock, ErrLayerUnknown } parentLayer = ilayer if err := s.containerStore.startWriting(); err != nil { - return nil, -1, err + return nil, nil, unlock, err } defer s.containerStore.stopWriting() containers, err := s.containerStore.Containers() if err != nil { - return nil, -1, err + return nil, nil, unlock, err } for _, container := range containers { if container.LayerID == parent { - return nil, -1, ErrParentIsContainer + return nil, nil, unlock, ErrParentIsContainer } } if !options.HostUIDMapping && len(options.UIDMap) == 0 { @@ -1526,7 +1545,7 @@ func (s *store) putLayer(rlstore rwLayerStore, rlstores []roLayerStore, id, pare GIDMap: copySlicePreferringNil(gidMap), } } - return rlstore.create(id, parentLayer, names, mountLabel, nil, &options, writeable, slo) + return &options, parentLayer, unlock, nil } func (s *store) PutLayer(id, parent string, names []string, mountLabel string, writeable bool, lOptions *LayerOptions, diff io.Reader) (*Layer, int64, error) { @@ -1557,7 +1576,13 @@ func (s *store) PutLayer(id, parent string, names []string, mountLabel string, w return nil, -1, err } defer rlstore.stopWriting() - return s.putLayer(rlstore, rlstores, id, parent, names, mountLabel, writeable, lOptions, slo) + + options, parentLayer, unlockLayerStores, err := populateLayerOptions(s, rlstore, rlstores, parent, lOptions) + defer unlockLayerStores() + if err != nil { + return nil, -1, err + } + return s.putLayer(rlstore, id, parentLayer, names, mountLabel, writeable, options, slo) } func (s *store) CreateLayer(id, parent string, names []string, mountLabel string, writeable bool, options *LayerOptions) (*Layer, error) { @@ -3198,7 +3223,12 @@ func (s *store) ApplyStagedLayer(args ApplyStagedLayerOptions) (*Layer, error) { DiffOutput: args.DiffOutput, DiffOptions: args.DiffOptions, } - layer, _, err = s.putLayer(rlstore, rlstores, args.ID, args.ParentLayer, args.Names, args.MountLabel, args.Writeable, args.LayerOptions, &slo) + options, parentLayer, unlockLayerStores, err := populateLayerOptions(s, rlstore, rlstores, args.ParentLayer, args.LayerOptions) + defer unlockLayerStores() + if err != nil { + return nil, err + } + layer, _, err = s.putLayer(rlstore, args.ID, parentLayer, args.Names, args.MountLabel, args.Writeable, options, &slo) return layer, err } From acae81f168d358a5049007b2c8fc35025afdc69b Mon Sep 17 00:00:00 2001 From: Paul Holzinger Date: Thu, 30 Oct 2025 19:26:50 +0100 Subject: [PATCH 10/37] storage: extract id or name conflict check into function Make it reusable for other callers, see next commit. Also while at it remove the dedupeStrings() call, as pointed out by Miloslav the work it is doing is more expensive than just checking the same name several times as it does a O(1) map lookup. Also most callers won't pass duplicated names to begin with. Signed-off-by: Paul Holzinger (cherry picked from commit f782a73efcf06a314ae4d28ecf9543a5cd55dd82) Signed-off-by: Paul Holzinger --- storage/layers.go | 36 +++++++++++++++++++++--------------- 1 file changed, 21 insertions(+), 15 deletions(-) diff --git a/storage/layers.go b/storage/layers.go index def929aed5..aaa4c821ae 100644 --- a/storage/layers.go +++ b/storage/layers.go @@ -1358,13 +1358,8 @@ func (r *layerStore) Status() ([][2]string, error) { // Requires startWriting. func (r *layerStore) PutAdditionalLayer(id string, parentLayer *Layer, names []string, aLayer drivers.AdditionalLayer) (layer *Layer, err error) { - if duplicateLayer, idInUse := r.byid[id]; idInUse { - return duplicateLayer, ErrDuplicateID - } - for _, name := range names { - if _, nameInUse := r.byname[name]; nameInUse { - return nil, ErrDuplicateName - } + if layer, err := r.checkIdOrNameConflict(id, names); err != nil { + return layer, err } parent := "" @@ -1429,6 +1424,23 @@ func (r *layerStore) pickStoreLocation(volatile, writeable bool) layerLocations } } +// checkIdOrNameConflict checks if the id or names are already in use and returns an +// error in that case. As Special case if the layer already exists it returns it as +// well together with the error. +// +// Requires startReading or startWriting. +func (r *layerStore) checkIdOrNameConflict(id string, names []string) (*Layer, error) { + if duplicateLayer, idInUse := r.byid[id]; idInUse { + return duplicateLayer, ErrDuplicateID + } + for _, name := range names { + if _, nameInUse := r.byname[name]; nameInUse { + return nil, ErrDuplicateName + } + } + return nil, nil +} + // Requires startWriting. func (r *layerStore) create(id string, parentLayer *Layer, names []string, mountLabel string, options map[string]string, moreOptions *LayerOptions, writeable bool, slo *stagedLayerOptions) (layer *Layer, size int64, err error) { if moreOptions == nil { @@ -1451,14 +1463,8 @@ func (r *layerStore) create(id string, parentLayer *Layer, names []string, mount _, idInUse = r.byid[id] } } - if duplicateLayer, idInUse := r.byid[id]; idInUse { - return duplicateLayer, -1, ErrDuplicateID - } - names = dedupeStrings(names) - for _, name := range names { - if _, nameInUse := r.byname[name]; nameInUse { - return nil, -1, ErrDuplicateName - } + if layer, err := r.checkIdOrNameConflict(id, names); err != nil { + return layer, -1, err } parent := "" if parentLayer != nil { From 79616ad8f1ec35ffe57c3925b557529debca2159 Mon Sep 17 00:00:00 2001 From: Paul Holzinger Date: Thu, 30 Oct 2025 19:54:41 +0100 Subject: [PATCH 11/37] storage: do conflict checks before staging untar The untar can be quite expensive so check for id, name conflicts right away. Also we must populate the idmappings so we extract with the right uids/gids. Signed-off-by: Paul Holzinger (cherry picked from commit f524d1419163a3a7a657f8937c381461756b86cf) Signed-off-by: Paul Holzinger --- storage/layers.go | 5 ++ storage/store.go | 127 ++++++++++++++++++++++++++++++++++------------ 2 files changed, 99 insertions(+), 33 deletions(-) diff --git a/storage/layers.go b/storage/layers.go index aaa4c821ae..f5db4e5b5a 100644 --- a/storage/layers.go +++ b/storage/layers.go @@ -259,6 +259,11 @@ type roLayerStore interface { // stopReading releases locks obtained by startReading. stopReading() + // checkIdOrNameConflict checks if the id or names are already in use and returns an + // error in that case. As Special case if the layer already exists it returns it as + // well together with the error. + checkIdOrNameConflict(id string, names []string) (*Layer, error) + // Exists checks if a layer with the specified name or ID is known. Exists(id string) bool diff --git a/storage/store.go b/storage/store.go index 6c568d804a..66bdd807db 100644 --- a/storage/store.go +++ b/storage/store.go @@ -1457,13 +1457,11 @@ func (s *store) putLayer(rlstore rwLayerStore, id string, parentLayer *Layer, na } // On entry: -// - rlstore must be locked for writing +// - rlstore must be locked for reading or writing // - rlstores MUST NOT be locked -// -// Returns the new copied LayerOptions with mappings set, the parent Layer and -// an extra unlock function to unlock any potentially read locked rlstores by this function. +// Returns an extra unlock function to unlock any potentially read locked rlstores by this function. // The unlock function is always set and thus must always be called. -func populateLayerOptions(s *store, rlstore rwLayerStore, rlstores []roLayerStore, parent string, lOptions *LayerOptions) (*LayerOptions, *Layer, func(), error) { +func getParentLayer(rlstore roLayerStore, rlstores []roLayerStore, parent string) (*Layer, func(), error) { // function we return to the caller so the caller gets the right stores locked and can unlock at the proper time themselves var lockedLayerStores []roLayerStore unlock := func() { @@ -1471,9 +1469,35 @@ func populateLayerOptions(s *store, rlstore rwLayerStore, rlstores []roLayerStor i.stopReading() } } + for _, l := range append([]roLayerStore{rlstore}, rlstores...) { + lstore := l + if lstore != rlstore { + if err := lstore.startReading(); err != nil { + return nil, unlock, err + } + lockedLayerStores = append(lockedLayerStores, lstore) + } + if l, err := lstore.Get(parent); err == nil && l != nil { + return l, unlock, nil + } + } + + return nil, unlock, ErrLayerUnknown +} +// On entry: +// - rlstore must be locked for writing +// - rlstores MUST NOT be locked +// +// Returns the new copied LayerOptions with mappings set, the parent Layer and +// an extra unlock function to unlock any potentially read locked rlstores by this function. +// The unlock function is always set and thus must always be called. +func populateLayerOptions(s *store, rlstore rwLayerStore, rlstores []roLayerStore, parent string, lOptions *LayerOptions) (*LayerOptions, *Layer, func(), error) { + // WARNING: Update also the freshLayer checks in store.PutLayer if adding more logic here. var parentLayer *Layer var options LayerOptions + // make sure we always return a valid func instead of nil so the caller can call it without checking + unlock := func() {} if lOptions != nil { options = *lOptions options.BigData = slices.Clone(lOptions.BigData) @@ -1488,25 +1512,11 @@ func populateLayerOptions(s *store, rlstore rwLayerStore, rlstores []roLayerStor uidMap := options.UIDMap gidMap := options.GIDMap if parent != "" { - var ilayer *Layer - for _, l := range append([]roLayerStore{rlstore}, rlstores...) { - lstore := l - if lstore != rlstore { - if err := lstore.startReading(); err != nil { - return nil, nil, unlock, err - } - lockedLayerStores = append(lockedLayerStores, lstore) - } - if l, err := lstore.Get(parent); err == nil && l != nil { - ilayer = l - parent = ilayer.ID - break - } - } - if ilayer == nil { - return nil, nil, unlock, ErrLayerUnknown + var err error + parentLayer, unlock, err = getParentLayer(rlstore, rlstores, parent) + if err != nil { + return nil, nil, unlock, err } - parentLayer = ilayer if err := s.containerStore.startWriting(); err != nil { return nil, nil, unlock, err @@ -1522,10 +1532,10 @@ func populateLayerOptions(s *store, rlstore rwLayerStore, rlstores []roLayerStor } } if !options.HostUIDMapping && len(options.UIDMap) == 0 { - uidMap = ilayer.UIDMap + uidMap = parentLayer.UIDMap } if !options.HostGIDMapping && len(options.GIDMap) == 0 { - gidMap = ilayer.GIDMap + gidMap = parentLayer.GIDMap } } else { if !options.HostUIDMapping && len(options.UIDMap) == 0 { @@ -1554,7 +1564,11 @@ func (s *store) PutLayer(id, parent string, names []string, mountLabel string, w return nil, -1, err } - var slo *stagedLayerOptions + var ( + slo *stagedLayerOptions + options *LayerOptions + parentLayer *Layer + ) if diff != nil { m := rlstore.newMaybeStagedLayerExtraction(diff) @@ -1563,10 +1577,38 @@ func (s *store) PutLayer(id, parent string, names []string, mountLabel string, w logrus.Errorf("Error cleaning up temporary directories: %v", err) } }() - err = rlstore.stageWithUnlockedStore(m, parent, lOptions) - if err != nil { - return nil, -1, err + // driver can do unlocked staging so do that without holding the layer lock + if m.staging != nil { + // func so we have a scope for defer, we don't want to hold the lock for stageWithUnlockedStore() + layer, err := func() (*Layer, error) { + if err := rlstore.startWriting(); err != nil { + return nil, err + } + defer rlstore.stopWriting() + + if layer, err := rlstore.checkIdOrNameConflict(id, names); err != nil { + return layer, err + } + + var unlockLayerStores func() + options, parentLayer, unlockLayerStores, err = populateLayerOptions(s, rlstore, rlstores, parent, lOptions) + unlockLayerStores() + return nil, err + }() + if err != nil { + return layer, -1, err + } + + // make sure to use the resolved full ID if there is a parent + if parentLayer != nil { + parent = parentLayer.ID + } + + if err := rlstore.stageWithUnlockedStore(m, parent, options); err != nil { + return nil, -1, err + } } + slo = &stagedLayerOptions{ stagedLayerExtraction: m, } @@ -1577,10 +1619,29 @@ func (s *store) PutLayer(id, parent string, names []string, mountLabel string, w } defer rlstore.stopWriting() - options, parentLayer, unlockLayerStores, err := populateLayerOptions(s, rlstore, rlstores, parent, lOptions) - defer unlockLayerStores() - if err != nil { - return nil, -1, err + if options == nil { + var unlockLayerStores func() + options, parentLayer, unlockLayerStores, err = populateLayerOptions(s, rlstore, rlstores, parent, lOptions) + defer unlockLayerStores() + if err != nil { + return nil, -1, err + } + } else if parent != "" { + // We used the staged extraction without holding the lock. + // Check again that the parent layer is still valid and exists. + freshLayer, unlockLayerStores, err := getParentLayer(rlstore, rlstores, parent) + defer unlockLayerStores() + if err != nil { + return nil, -1, err + } + // In populateLayerOptions() we get the ID mappings in order to extract correctly, ensure the freshly + // looked up parent Layer still has the same mappings to prevent silent UID/GID corruption. + if !slices.Equal(freshLayer.UIDMap, parentLayer.UIDMap) || !slices.Equal(freshLayer.GIDMap, parentLayer.GIDMap) { + // Fatal problem. Mappings changed so the parent must be considered different now. + // Since we consumed the diff there is no we to recover, return error to caller. The caller would need to retry. + // How likely is that and would need to return a special error so c/image could do the retries? + return nil, -1, fmt.Errorf("error during staged layer apply, parent layer %q changed id mappings while the content was extracted, must retry layer creation", parent) + } } return s.putLayer(rlstore, id, parentLayer, names, mountLabel, writeable, options, slo) } From 2883b09e80f74726b218537cbd32e8631d0fcd26 Mon Sep 17 00:00:00 2001 From: Paul Holzinger Date: Tue, 4 Nov 2025 15:23:59 +0100 Subject: [PATCH 12/37] storage: remove NaiveCreateFromTemplate() This function was added in commit c577a81 and used by older drivers we no longer suppor, such as aufs and windows. As such this is dead code and can be removed. Signed-off-by: Paul Holzinger (cherry picked from commit af19fb644c26a9a9f47872723521907dedfe5b8e) Signed-off-by: Paul Holzinger --- storage/drivers/driver.go | 1 - storage/drivers/template.go | 52 ------------------------------------- 2 files changed, 53 deletions(-) delete mode 100644 storage/drivers/template.go diff --git a/storage/drivers/driver.go b/storage/drivers/driver.go index e43e1176de..c0a37a677b 100644 --- a/storage/drivers/driver.go +++ b/storage/drivers/driver.go @@ -47,7 +47,6 @@ type CreateOpts struct { MountLabel string StorageOpt map[string]string *idtools.IDMappings - ignoreChownErrors bool } // MountOpts contains optional arguments for Driver.Get() methods. diff --git a/storage/drivers/template.go b/storage/drivers/template.go deleted file mode 100644 index 03da13c5bb..0000000000 --- a/storage/drivers/template.go +++ /dev/null @@ -1,52 +0,0 @@ -package graphdriver - -import ( - "github.com/sirupsen/logrus" - "go.podman.io/storage/pkg/idtools" -) - -// TemplateDriver is just barely enough of a driver that we can implement a -// naive version of CreateFromTemplate on top of it. -type TemplateDriver interface { - DiffDriver - CreateReadWrite(id, parent string, opts *CreateOpts) error - Create(id, parent string, opts *CreateOpts) error - Remove(id string) error -} - -// CreateFromTemplate creates a layer with the same contents and parent as -// another layer. Internally, it may even depend on that other layer -// continuing to exist, as if it were actually a child of the child layer. -func NaiveCreateFromTemplate(d TemplateDriver, id, template string, templateIDMappings *idtools.IDMappings, parent string, parentIDMappings *idtools.IDMappings, opts *CreateOpts, readWrite bool) error { - var err error - if readWrite { - err = d.CreateReadWrite(id, parent, opts) - } else { - err = d.Create(id, parent, opts) - } - if err != nil { - return err - } - diff, err := d.Diff(template, templateIDMappings, parent, parentIDMappings, opts.MountLabel) - if err != nil { - if err2 := d.Remove(id); err2 != nil { - logrus.Errorf("Removing layer %q: %v", id, err2) - } - return err - } - defer diff.Close() - - applyOptions := ApplyDiffOpts{ - Diff: diff, - Mappings: templateIDMappings, - MountLabel: opts.MountLabel, - IgnoreChownErrors: opts.ignoreChownErrors, - } - if _, err = d.ApplyDiff(id, parent, applyOptions); err != nil { - if err2 := d.Remove(id); err2 != nil { - logrus.Errorf("Removing layer %q: %v", id, err2) - } - return err - } - return nil -} From 54161d3a4272b8debea02599b5b1a608d592cfe2 Mon Sep 17 00:00:00 2001 From: Paul Holzinger Date: Tue, 4 Nov 2025 15:16:49 +0100 Subject: [PATCH 13/37] storage: remove parent param from ApplyDiff() It is unused in all drivers now, so it can be removed. Signed-off-by: Paul Holzinger (cherry picked from commit b37aa042f501c2339142f67d3fcd29cc37f902d9) Signed-off-by: Paul Holzinger --- storage/drivers/driver.go | 2 +- storage/drivers/fsdiff.go | 2 +- storage/drivers/graphtest/graphbench_unix.go | 2 +- storage/drivers/graphtest/graphtest_unix.go | 2 +- storage/drivers/overlay/overlay.go | 2 +- storage/drivers/vfs/driver.go | 4 ++-- storage/layers.go | 2 +- 7 files changed, 8 insertions(+), 8 deletions(-) diff --git a/storage/drivers/driver.go b/storage/drivers/driver.go index c0a37a677b..38706dc999 100644 --- a/storage/drivers/driver.go +++ b/storage/drivers/driver.go @@ -183,7 +183,7 @@ type DiffDriver interface { // layer with the specified id and parent, returning the size of the // new layer in bytes. // The io.Reader must be an uncompressed stream. - ApplyDiff(id string, parent string, options ApplyDiffOpts) (size int64, err error) + ApplyDiff(id string, options ApplyDiffOpts) (size int64, err error) // DiffSize calculates the changes between the specified id // and its parent and returns the size in bytes of the changes // relative to its base filesystem directory. diff --git a/storage/drivers/fsdiff.go b/storage/drivers/fsdiff.go index 77f98d49da..89d18e6d35 100644 --- a/storage/drivers/fsdiff.go +++ b/storage/drivers/fsdiff.go @@ -151,7 +151,7 @@ func (gdw *NaiveDiffDriver) Changes(id string, idMappings *idtools.IDMappings, p // ApplyDiff extracts the changeset from the given diff into the // layer with the specified id and parent, returning the size of the // new layer in bytes. -func (gdw *NaiveDiffDriver) ApplyDiff(id, parent string, options ApplyDiffOpts) (int64, error) { +func (gdw *NaiveDiffDriver) ApplyDiff(id string, options ApplyDiffOpts) (int64, error) { driver := gdw.ProtoDriver if options.Mappings == nil { diff --git a/storage/drivers/graphtest/graphbench_unix.go b/storage/drivers/graphtest/graphbench_unix.go index aca7728407..e6a8ffd0ec 100644 --- a/storage/drivers/graphtest/graphbench_unix.go +++ b/storage/drivers/graphtest/graphbench_unix.go @@ -155,7 +155,7 @@ func DriverBenchDiffApplyN(b *testing.B, fileCount int, drivername string, drive b.Fatal(err) } - applyDiffSize, err := driver.ApplyDiff(diff, "", graphdriver.ApplyDiffOpts{}) + applyDiffSize, err := driver.ApplyDiff(diff, graphdriver.ApplyDiffOpts{}) if err != nil { b.Fatal(err) } diff --git a/storage/drivers/graphtest/graphtest_unix.go b/storage/drivers/graphtest/graphtest_unix.go index 475c66460f..c40c6a0559 100644 --- a/storage/drivers/graphtest/graphtest_unix.go +++ b/storage/drivers/graphtest/graphtest_unix.go @@ -362,7 +362,7 @@ func DriverTestDiffApply(t testing.TB, fileCount int, drivername string, driverO t.Fatal(err) } - applyDiffSize, err := driver.ApplyDiff(diff, base, graphdriver.ApplyDiffOpts{Diff: bytes.NewReader(buf.Bytes())}) + applyDiffSize, err := driver.ApplyDiff(diff, graphdriver.ApplyDiffOpts{Diff: bytes.NewReader(buf.Bytes())}) if err != nil { t.Fatal(err) } diff --git a/storage/drivers/overlay/overlay.go b/storage/drivers/overlay/overlay.go index b621c35c47..2eb720c188 100644 --- a/storage/drivers/overlay/overlay.go +++ b/storage/drivers/overlay/overlay.go @@ -2459,7 +2459,7 @@ func (d *Driver) CommitStagedLayer(id string, sa *tempdir.StagedAddition) error } // ApplyDiff applies the new layer into a root -func (d *Driver) ApplyDiff(id, parent string, options graphdriver.ApplyDiffOpts) (size int64, err error) { +func (d *Driver) ApplyDiff(id string, options graphdriver.ApplyDiffOpts) (size int64, err error) { applyDir, err := d.getDiffPath(id) if err != nil { return 0, err diff --git a/storage/drivers/vfs/driver.go b/storage/drivers/vfs/driver.go index ffd3bd24ea..b90c2046cf 100644 --- a/storage/drivers/vfs/driver.go +++ b/storage/drivers/vfs/driver.go @@ -132,11 +132,11 @@ func (d *Driver) CreateFromTemplate(id, template string, templateIDMappings *idt } // ApplyDiff applies the new layer into a root -func (d *Driver) ApplyDiff(id, parent string, options graphdriver.ApplyDiffOpts) (size int64, err error) { +func (d *Driver) ApplyDiff(id string, options graphdriver.ApplyDiffOpts) (size int64, err error) { if d.ignoreChownErrors { options.IgnoreChownErrors = d.ignoreChownErrors } - return d.naiveDiff.ApplyDiff(id, parent, options) + return d.naiveDiff.ApplyDiff(id, options) } // CreateReadWrite creates a layer that is writable for use as a container diff --git a/storage/layers.go b/storage/layers.go index f5db4e5b5a..09b0426110 100644 --- a/storage/layers.go +++ b/storage/layers.go @@ -2698,7 +2698,7 @@ func (r *layerStore) applyDiffWithOptions(to string, layerOptions *LayerOptions, Mappings: r.layerMappings(layer), MountLabel: layer.MountLabel, } - return r.driver.ApplyDiff(layer.ID, layer.Parent, options) + return r.driver.ApplyDiff(layer.ID, options) }) if err != nil { return -1, err From 03d17147da9896fde0cb0ac4f47540e5264ca3e3 Mon Sep 17 00:00:00 2001 From: Paul Holzinger Date: Mon, 17 Nov 2025 18:19:20 +0100 Subject: [PATCH 14/37] storage: rename stagedLayerOptions to layerCreationContents We use this this typo all the time now so make the naming a bit more clear. Signed-off-by: Paul Holzinger (cherry picked from commit b93f966eedb907e4420a8d0cf8333fbee5d6f046) Signed-off-by: Paul Holzinger --- storage/layers.go | 24 ++++++++++++------------ storage/store.go | 14 +++++++------- 2 files changed, 19 insertions(+), 19 deletions(-) diff --git a/storage/layers.go b/storage/layers.go index 09b0426110..785f84933d 100644 --- a/storage/layers.go +++ b/storage/layers.go @@ -196,9 +196,9 @@ type DiffOptions struct { Compression *archive.Compression } -// stagedLayerOptions are the options passed to .create to populate a staged +// layerCreationContents are the options passed to .create to populate a staged // layer -type stagedLayerOptions struct { +type layerCreationContents struct { // These are used via the zstd:chunked pull paths DiffOutput *drivers.DriverWithDifferOutput DiffOptions *drivers.ApplyDiffWithDifferOpts @@ -336,7 +336,7 @@ type rwLayerStore interface { // underlying drivers do not themselves distinguish between writeable // and read-only layers. Returns the new layer structure and the size of the // diff which was applied to its parent to initialize its contents. - create(id string, parent *Layer, names []string, mountLabel string, options map[string]string, moreOptions *LayerOptions, writeable bool, slo *stagedLayerOptions) (*Layer, int64, error) + create(id string, parent *Layer, names []string, mountLabel string, options map[string]string, moreOptions *LayerOptions, writeable bool, contents *layerCreationContents) (*Layer, int64, error) // updateNames modifies names associated with a layer based on (op, names). updateNames(id string, names []string, op updateNameOperation) error @@ -1447,7 +1447,7 @@ func (r *layerStore) checkIdOrNameConflict(id string, names []string) (*Layer, e } // Requires startWriting. -func (r *layerStore) create(id string, parentLayer *Layer, names []string, mountLabel string, options map[string]string, moreOptions *LayerOptions, writeable bool, slo *stagedLayerOptions) (layer *Layer, size int64, err error) { +func (r *layerStore) create(id string, parentLayer *Layer, names []string, mountLabel string, options map[string]string, moreOptions *LayerOptions, writeable bool, contents *layerCreationContents) (layer *Layer, size int64, err error) { if moreOptions == nil { moreOptions = &LayerOptions{} } @@ -1630,31 +1630,31 @@ func (r *layerStore) create(id string, parentLayer *Layer, names []string, mount } size = -1 - if slo != nil { - if slo.stagedLayerExtraction != nil { - if slo.stagedLayerExtraction.result != nil { + if contents != nil { + if contents.stagedLayerExtraction != nil { + if contents.stagedLayerExtraction.result != nil { // The layer is staged, just commit it and update the metadata. - if err := slo.stagedLayerExtraction.commitLayer(r, layer.ID); err != nil { + if err := contents.stagedLayerExtraction.commitLayer(r, layer.ID); err != nil { cleanupFailureContext = "committing staged layer diff" return nil, -1, err } - r.applyDiffResultToLayer(layer, slo.stagedLayerExtraction.result) + r.applyDiffResultToLayer(layer, contents.stagedLayerExtraction.result) } else { // The diff was not staged, apply it now here instead. - if size, err = r.applyDiffWithOptions(layer.ID, moreOptions, slo.stagedLayerExtraction.diff); err != nil { + if size, err = r.applyDiffWithOptions(layer.ID, moreOptions, contents.stagedLayerExtraction.diff); err != nil { cleanupFailureContext = "applying layer diff" return nil, -1, err } } } else { // staging logic for the chunked pull path - if err := r.applyDiffFromStagingDirectory(layer.ID, slo.DiffOutput, slo.DiffOptions); err != nil { + if err := r.applyDiffFromStagingDirectory(layer.ID, contents.DiffOutput, contents.DiffOptions); err != nil { cleanupFailureContext = "applying staged directory diff" return nil, -1, err } } } else { - // applyDiffWithOptions() would have updated r.bycompressedsum + // The layer creation content above would have updated r.bycompressedsum // and r.byuncompressedsum for us, but if we used a template // layer, we didn't call it, so add the new layer as candidates // for searches for layers by checksum diff --git a/storage/store.go b/storage/store.go index 66bdd807db..1eb6bfbfb2 100644 --- a/storage/store.go +++ b/storage/store.go @@ -1452,8 +1452,8 @@ func (s *store) canUseShifting(uidmap, gidmap []idtools.IDMap) bool { // On entry: // - rlstore must be locked for writing // - rlstores MUST NOT be locked -func (s *store) putLayer(rlstore rwLayerStore, id string, parentLayer *Layer, names []string, mountLabel string, writeable bool, options *LayerOptions, slo *stagedLayerOptions) (*Layer, int64, error) { - return rlstore.create(id, parentLayer, names, mountLabel, nil, options, writeable, slo) +func (s *store) putLayer(rlstore rwLayerStore, id string, parentLayer *Layer, names []string, mountLabel string, writeable bool, options *LayerOptions, contents *layerCreationContents) (*Layer, int64, error) { + return rlstore.create(id, parentLayer, names, mountLabel, nil, options, writeable, contents) } // On entry: @@ -1565,7 +1565,7 @@ func (s *store) PutLayer(id, parent string, names []string, mountLabel string, w } var ( - slo *stagedLayerOptions + contents *layerCreationContents options *LayerOptions parentLayer *Layer ) @@ -1609,7 +1609,7 @@ func (s *store) PutLayer(id, parent string, names []string, mountLabel string, w } } - slo = &stagedLayerOptions{ + contents = &layerCreationContents{ stagedLayerExtraction: m, } } @@ -1643,7 +1643,7 @@ func (s *store) PutLayer(id, parent string, names []string, mountLabel string, w return nil, -1, fmt.Errorf("error during staged layer apply, parent layer %q changed id mappings while the content was extracted, must retry layer creation", parent) } } - return s.putLayer(rlstore, id, parentLayer, names, mountLabel, writeable, options, slo) + return s.putLayer(rlstore, id, parentLayer, names, mountLabel, writeable, options, contents) } func (s *store) CreateLayer(id, parent string, names []string, mountLabel string, writeable bool, options *LayerOptions) (*Layer, error) { @@ -3280,7 +3280,7 @@ func (s *store) ApplyStagedLayer(args ApplyStagedLayerOptions) (*Layer, error) { // if the layer doesn't exist yet, try to create it. - slo := stagedLayerOptions{ + contents := layerCreationContents{ DiffOutput: args.DiffOutput, DiffOptions: args.DiffOptions, } @@ -3289,7 +3289,7 @@ func (s *store) ApplyStagedLayer(args ApplyStagedLayerOptions) (*Layer, error) { if err != nil { return nil, err } - layer, _, err = s.putLayer(rlstore, args.ID, parentLayer, args.Names, args.MountLabel, args.Writeable, options, &slo) + layer, _, err = s.putLayer(rlstore, args.ID, parentLayer, args.Names, args.MountLabel, args.Writeable, options, &contents) return layer, err } From fed6fe197d07aaef60ffb3f601c3fc7ffbd06051 Mon Sep 17 00:00:00 2001 From: Paul Holzinger Date: Mon, 17 Nov 2025 18:27:29 +0100 Subject: [PATCH 15/37] storage: don't use stagedLayerExtract when mountLabel is set Just be safe based on the review feedback from the PR. https://github.com/containers/container-libs/pull/378 Signed-off-by: Paul Holzinger (cherry picked from commit 3bfe961f1d2cb4a7d2cf87142ba1e96a71b687c3) Signed-off-by: Paul Holzinger --- storage/layers.go | 3 +-- storage/store.go | 7 ++++++- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/storage/layers.go b/storage/layers.go index 785f84933d..abfb1c0370 100644 --- a/storage/layers.go +++ b/storage/layers.go @@ -2526,8 +2526,7 @@ func (r *layerStore) stageWithUnlockedStore(sl *maybeStagedLayerExtraction, pare cleanup, stagedLayer, size, err := sl.staging.StartStagingDiffToApply(parent, drivers.ApplyDiffOpts{ Diff: payload, Mappings: idtools.NewIDMappingsFromMaps(layerOptions.UIDMap, layerOptions.GIDMap), - // FIXME: What to do here? We have no lock and assigned label yet. - // Overlayfs should not need it anyway so this seems fine for now. + // MountLabel is not supported for the unlocked extraction, see the comment in (*store).PutLayer() MountLabel: "", }) sl.cleanupFuncs = append(sl.cleanupFuncs, cleanup) diff --git a/storage/store.go b/storage/store.go index 1eb6bfbfb2..09afaa322b 100644 --- a/storage/store.go +++ b/storage/store.go @@ -1578,7 +1578,12 @@ func (s *store) PutLayer(id, parent string, names []string, mountLabel string, w } }() // driver can do unlocked staging so do that without holding the layer lock - if m.staging != nil { + // Special case we only support it when no mount label is used. c/image doesn't set it for layers + // and the overlay driver doesn't use it for extract today so it would be safe even when set but + // that is not exactly obvious and if someone would implement the ApplyDiffStaging interface for + // another driver that may be no longer true. So for now simply fall back to the locked extract path + // to ensure we don't cause any weird issues here. + if m.staging != nil && mountLabel == "" { // func so we have a scope for defer, we don't want to hold the lock for stageWithUnlockedStore() layer, err := func() (*Layer, error) { if err := rlstore.startWriting(); err != nil { From c420709f00761af2e7049fd9080bd2038250feac Mon Sep 17 00:00:00 2001 From: Paul Holzinger Date: Wed, 26 Nov 2025 15:11:22 +0100 Subject: [PATCH 16/37] storage: inline putLayer The function is just a redirection to another one so inline it directly as we do not gain anything from the extra indirection. Signed-off-by: Paul Holzinger (cherry picked from commit e58297d2cf4ad1e0805ccc542dded8783d959b2c) Signed-off-by: Paul Holzinger --- storage/store.go | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/storage/store.go b/storage/store.go index 09afaa322b..3d8ea50759 100644 --- a/storage/store.go +++ b/storage/store.go @@ -1449,13 +1449,6 @@ func (s *store) canUseShifting(uidmap, gidmap []idtools.IDMap) bool { return s.graphDriver.SupportsShifting(uidmap, gidmap) } -// On entry: -// - rlstore must be locked for writing -// - rlstores MUST NOT be locked -func (s *store) putLayer(rlstore rwLayerStore, id string, parentLayer *Layer, names []string, mountLabel string, writeable bool, options *LayerOptions, contents *layerCreationContents) (*Layer, int64, error) { - return rlstore.create(id, parentLayer, names, mountLabel, nil, options, writeable, contents) -} - // On entry: // - rlstore must be locked for reading or writing // - rlstores MUST NOT be locked @@ -1648,7 +1641,7 @@ func (s *store) PutLayer(id, parent string, names []string, mountLabel string, w return nil, -1, fmt.Errorf("error during staged layer apply, parent layer %q changed id mappings while the content was extracted, must retry layer creation", parent) } } - return s.putLayer(rlstore, id, parentLayer, names, mountLabel, writeable, options, contents) + return rlstore.create(id, parentLayer, names, mountLabel, nil, options, writeable, contents) } func (s *store) CreateLayer(id, parent string, names []string, mountLabel string, writeable bool, options *LayerOptions) (*Layer, error) { @@ -3294,7 +3287,7 @@ func (s *store) ApplyStagedLayer(args ApplyStagedLayerOptions) (*Layer, error) { if err != nil { return nil, err } - layer, _, err = s.putLayer(rlstore, args.ID, parentLayer, args.Names, args.MountLabel, args.Writeable, options, &contents) + layer, _, err = rlstore.create(args.ID, parentLayer, args.Names, args.MountLabel, nil, options, args.Writeable, &contents) return layer, err } From 8f14b0abb1625a2fdb8bda705b952dd9f93fed61 Mon Sep 17 00:00:00 2001 From: Paul Holzinger Date: Wed, 26 Nov 2025 15:23:38 +0100 Subject: [PATCH 17/37] storage: check for close error on the tar split file Also add a missing sync when we stage to ensure the content was flushed to disk. Signed-off-by: Paul Holzinger (cherry picked from commit 83dd0eb61809c5d4e6a10eb6c11e673652dc5c74) Signed-off-by: Paul Holzinger --- storage/layers.go | 34 ++++++++++++++++++++++++++++------ 1 file changed, 28 insertions(+), 6 deletions(-) diff --git a/storage/layers.go b/storage/layers.go index abfb1c0370..d485c9b4fa 100644 --- a/storage/layers.go +++ b/storage/layers.go @@ -2500,7 +2500,7 @@ func (sl *maybeStagedLayerExtraction) cleanup() error { // If the driver does not support stage addition then this is a NOP and does nothing. // This should be done without holding the storage lock, if a parent is given the caller // must check for existence beforehand while holding a lock. -func (r *layerStore) stageWithUnlockedStore(sl *maybeStagedLayerExtraction, parent string, layerOptions *LayerOptions) error { +func (r *layerStore) stageWithUnlockedStore(sl *maybeStagedLayerExtraction, parent string, layerOptions *LayerOptions) (retErr error) { if sl.staging == nil { return nil } @@ -2520,7 +2520,13 @@ func (r *layerStore) stageWithUnlockedStore(sl *maybeStagedLayerExtraction, pare if err != nil { return err } - defer f.Close() + // make sure to check for errors on close and return that one. + defer func() { + closeErr := f.Close() + if retErr == nil { + retErr = closeErr + } + }() result, err := applyDiff(layerOptions, sl.diff, f, func(payload io.Reader) (int64, error) { cleanup, stagedLayer, size, err := sl.staging.StartStagingDiffToApply(parent, drivers.ApplyDiffOpts{ @@ -2537,6 +2543,10 @@ func (r *layerStore) stageWithUnlockedStore(sl *maybeStagedLayerExtraction, pare return err } + if err := f.Sync(); err != nil { + return fmt.Errorf("sync staged tar-split file: %w", err) + } + sl.result = result return nil } @@ -2675,7 +2685,7 @@ func applyDiff(layerOptions *LayerOptions, diff io.Reader, tarSplitFile *os.File } // Requires startWriting. -func (r *layerStore) applyDiffWithOptions(to string, layerOptions *LayerOptions, diff io.Reader) (int64, error) { +func (r *layerStore) applyDiffWithOptions(to string, layerOptions *LayerOptions, diff io.Reader) (_ int64, retErr error) { if !r.lockfile.IsReadWrite() { return -1, fmt.Errorf("not allowed to modify layer contents at %q: %w", r.layerdir, ErrStoreIsReadOnly) } @@ -2689,7 +2699,13 @@ func (r *layerStore) applyDiffWithOptions(to string, layerOptions *LayerOptions, if err != nil { return -1, err } - defer tarSplitFile.Close() + // make sure to check for errors on close and return that one. + defer func() { + closeErr := tarSplitFile.Close() + if retErr == nil { + retErr = closeErr + } + }() result, err := applyDiff(layerOptions, diff, tarSplitFile, func(payload io.Reader) (int64, error) { options := drivers.ApplyDiffOpts{ @@ -2741,7 +2757,7 @@ func (r *layerStore) DifferTarget(id string) (string, error) { } // Requires startWriting. -func (r *layerStore) applyDiffFromStagingDirectory(id string, diffOutput *drivers.DriverWithDifferOutput, options *drivers.ApplyDiffWithDifferOpts) error { +func (r *layerStore) applyDiffFromStagingDirectory(id string, diffOutput *drivers.DriverWithDifferOutput, options *drivers.ApplyDiffWithDifferOpts) (retErr error) { ddriver, ok := r.driver.(drivers.DriverWithDiffer) if !ok { return ErrNotSupported @@ -2789,7 +2805,13 @@ func (r *layerStore) applyDiffFromStagingDirectory(id string, diffOutput *driver if err != nil { return err } - defer tarSplitFile.Close() + // make sure to check for errors on close and return that one. + defer func() { + closeErr := tarSplitFile.Close() + if retErr == nil { + retErr = closeErr + } + }() tarSplitWriter := pools.BufioWriter32KPool.Get(tarSplitFile) defer pools.BufioWriter32KPool.Put(tarSplitWriter) From 2c58dded9d6e0bed0b0bbd9d9f536a86c8a9164c Mon Sep 17 00:00:00 2001 From: Paul Holzinger Date: Tue, 20 Jan 2026 16:55:16 +0100 Subject: [PATCH 18/37] cirrus: test against released skopeo skopeo main moved already forward on much newer dependency versions which are not compatibile with the current version on this container-libs branch. Signed-off-by: Paul Holzinger --- .cirrus.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.cirrus.yml b/.cirrus.yml index 3370294919..77ed2a7265 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -243,7 +243,7 @@ image_test_skopeo_task: HOME: "/root" # default unset, needed by golangci-lint. GOSRC: "${CIRRUS_WORKING_DIR}" GOPATH: "/var/tmp/go" - SKOPEO_CI_BRANCH: "main" + SKOPEO_CI_BRANCH: "release-1.21" SCRIPT_BASE: "./contrib/cirrus" # Built along with the standard PR-based workflow in c/automation_images SKOPEO_CIDEV_CONTAINER_FQIN: "quay.io/libpod/skopeo_cidev:${IMAGE_SUFFIX}" From 22d9d9cde6271d7bbbe7e5a05823030d9ae08a85 Mon Sep 17 00:00:00 2001 From: Kyounghoon Jang Date: Tue, 25 Nov 2025 10:33:48 +0900 Subject: [PATCH 19/37] Add a DefaultNetwork field to NetworkInfo Signed-off-by: Kyounghoon Jang (cherry picked from commit 905f456046384e376f319fc1b966e862e26f7fbe) Signed-off-by: Paul Holzinger --- common/libnetwork/cni/network.go | 7 ++++--- common/libnetwork/netavark/network.go | 9 +++++---- common/libnetwork/types/network.go | 11 ++++++----- 3 files changed, 15 insertions(+), 12 deletions(-) diff --git a/common/libnetwork/cni/network.go b/common/libnetwork/cni/network.go index 8a22773388..33220a2827 100644 --- a/common/libnetwork/cni/network.go +++ b/common/libnetwork/cni/network.go @@ -327,9 +327,10 @@ func (n *cniNetwork) NetworkInfo() types.NetworkInfo { } info := types.NetworkInfo{ - Backend: types.CNI, - Package: packageVersion, - Path: path, + Backend: types.CNI, + Package: packageVersion, + Path: path, + DefaultNetwork: n.defaultNetwork, } dnsPath := filepath.Join(path, "dnsname") diff --git a/common/libnetwork/netavark/network.go b/common/libnetwork/netavark/network.go index 855023db7d..9dbe3b452d 100644 --- a/common/libnetwork/netavark/network.go +++ b/common/libnetwork/netavark/network.go @@ -369,10 +369,11 @@ func (n *netavarkNetwork) NetworkInfo() types.NetworkInfo { logrus.Infof("Failed to get the netavark version: %v", err) } info := types.NetworkInfo{ - Backend: types.Netavark, - Version: programVersion, - Package: packageVersion, - Path: path, + Backend: types.Netavark, + Version: programVersion, + Package: packageVersion, + Path: path, + DefaultNetwork: n.defaultNetwork, } dnsPath := n.aardvarkBinary diff --git a/common/libnetwork/types/network.go b/common/libnetwork/types/network.go index df6a5ee446..d6c5d00d15 100644 --- a/common/libnetwork/types/network.go +++ b/common/libnetwork/types/network.go @@ -97,11 +97,12 @@ type NetworkUpdateOptions struct { // NetworkInfo contains the network information. type NetworkInfo struct { - Backend NetworkBackend `json:"backend"` - Version string `json:"version,omitempty"` - Package string `json:"package,omitempty"` - Path string `json:"path,omitempty"` - DNS DNSNetworkInfo `json:"dns,omitempty"` + Backend NetworkBackend `json:"backend"` + Version string `json:"version,omitempty"` + Package string `json:"package,omitempty"` + Path string `json:"path,omitempty"` + DNS DNSNetworkInfo `json:"dns,omitempty"` + DefaultNetwork string `json:"default_network,omitempty"` } // DNSNetworkInfo contains the DNS information. From 47de5a4b857e86cbebb8a6d9aa3b80c084b89f2b Mon Sep 17 00:00:00 2001 From: Kyounghoon Jang Date: Sat, 29 Nov 2025 23:07:39 +0900 Subject: [PATCH 20/37] Convert default_network from snake_case to camelCase Signed-off-by: Kyounghoon Jang (cherry picked from commit 97de24ca47af99bef02a96edc25cc8ebf1327d8b) Signed-off-by: Paul Holzinger --- common/libnetwork/types/network.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common/libnetwork/types/network.go b/common/libnetwork/types/network.go index d6c5d00d15..bc383a5aaa 100644 --- a/common/libnetwork/types/network.go +++ b/common/libnetwork/types/network.go @@ -102,7 +102,7 @@ type NetworkInfo struct { Package string `json:"package,omitempty"` Path string `json:"path,omitempty"` DNS DNSNetworkInfo `json:"dns,omitempty"` - DefaultNetwork string `json:"default_network,omitempty"` + DefaultNetwork string `json:"defaultNetwork,omitempty"` } // DNSNetworkInfo contains the DNS information. From 4ca2b096bc1c7bef62d4699cf4a65f496dab06e7 Mon Sep 17 00:00:00 2001 From: Konstantin Khlebnikov Date: Wed, 19 Nov 2025 19:46:27 +0100 Subject: [PATCH 21/37] Do not block try lock on state mutex TryLock should not block when file lock is held by other process and state lock is held by other coroutine which waits for file lock. Signed-off-by: Konstantin Khlebnikov (cherry picked from commit 0b25f83aee5290c1299a263b389343f384fc1d7f) Signed-off-by: Paul Holzinger --- storage/pkg/lockfile/lockfile.go | 5 ++- storage/pkg/lockfile/lockfile_test.go | 49 +++++++++++++++++++++++++++ 2 files changed, 53 insertions(+), 1 deletion(-) diff --git a/storage/pkg/lockfile/lockfile.go b/storage/pkg/lockfile/lockfile.go index 52b4fe597e..3a8a4bc398 100644 --- a/storage/pkg/lockfile/lockfile.go +++ b/storage/pkg/lockfile/lockfile.go @@ -420,7 +420,10 @@ func (l *LockFile) tryLock(lType rawfilelock.LockType) error { if !success { return fmt.Errorf("resource temporarily unavailable") } - l.stateMutex.Lock() + if !l.stateMutex.TryLock() { + rwMutexUnlocker() + return fmt.Errorf("resource temporarily unavailable") + } defer l.stateMutex.Unlock() if l.counter == 0 { // If we're the first reference on the lock, we need to open the file again. diff --git a/storage/pkg/lockfile/lockfile_test.go b/storage/pkg/lockfile/lockfile_test.go index 41b45f48ae..9c6fc2f068 100644 --- a/storage/pkg/lockfile/lockfile_test.go +++ b/storage/pkg/lockfile/lockfile_test.go @@ -277,6 +277,55 @@ func TestTryReadLockFile(t *testing.T) { assert.Nil(t, <-errChan) } +func TestTryLockState(t *testing.T) { + l, err := getTempLockfile() + require.NoError(t, err, "creating lock") + defer os.Remove(l.name) + + // Take a write lock somewhere. + cmd, wc, rc, err := subLock(l) + require.NoError(t, err) + _, err = io.Copy(io.Discard, rc) + require.NoError(t, err) + + err = l.TryRLock() + assert.NotNil(t, err) + + // Lock and hold state mutex. + locked := make(chan bool) + go func() { + locked <- false + l.RLock() + locked <- true + l.Unlock() + locked <- false + }() + + assert.False(t, <-locked) + + // Wait state mutex is locked. + for l.stateMutex.TryLock() { + l.stateMutex.Unlock() + time.Sleep(100 * time.Millisecond) + } + + // Try locks should fail without blocking. + errChan := make(chan error) + go func() { + errChan <- l.TryRLock() + errChan <- l.TryLock() + }() + assert.NotNil(t, <-errChan) + assert.NotNil(t, <-errChan) + + wc.Close() + err = cmd.Wait() + require.NoError(t, err) + + assert.True(t, <-locked) + assert.False(t, <-locked) +} + func TestLockfileRead(t *testing.T) { l, err := getTempLockfile() require.Nil(t, err, "error getting temporary lock file") From 52ca8fb38e2b6f0fbfd18bf0f6b1e29e2ac3edb3 Mon Sep 17 00:00:00 2001 From: Reinhard Tartler Date: Thu, 27 Nov 2025 18:28:11 -0500 Subject: [PATCH 22/37] Cleanup manpage formatting Also address most troff issues. Not all of them are easy, in particular those manpages with long urls and examples that include digests are difficult to render in traditional manpages. Add troff renderering to the Manpage to surface those troff issues more easily Signed-off-by: Reinhard Tartler (cherry picked from commit 9c9c17f4b4b056097b6c560b75d807e6c8e0d343) Signed-off-by: Paul Holzinger --- storage/docs/containers-storage-add-names.md | 10 ++++++++-- ...tainers-storage-applydiff-using-staging-dir.md | 6 ++++-- storage/docs/containers-storage-applydiff.md | 5 +++-- storage/docs/containers-storage-changes.md | 5 +++-- storage/docs/containers-storage-check.md | 5 +++-- storage/docs/containers-storage-composefs.md | 15 +++++++++------ storage/docs/containers-storage-config.md | 6 ++++-- storage/docs/containers-storage-container.md | 7 ++++--- storage/docs/containers-storage-containers.md | 5 +++-- storage/docs/containers-storage-copy.md | 5 +++-- .../docs/containers-storage-create-container.md | 5 +++-- storage/docs/containers-storage-create-image.md | 5 +++-- storage/docs/containers-storage-create-layer.md | 5 +++-- .../containers-storage-create-storage-layer.md | 5 +++-- storage/docs/containers-storage-dedup.md | 5 +++-- .../docs/containers-storage-delete-container.md | 5 +++-- storage/docs/containers-storage-delete-image.md | 5 +++-- storage/docs/containers-storage-delete-layer.md | 5 +++-- storage/docs/containers-storage-delete.md | 5 +++-- storage/docs/containers-storage-diff.md | 5 +++-- storage/docs/containers-storage-diffsize.md | 5 +++-- storage/docs/containers-storage-exists.md | 5 +++-- storage/docs/containers-storage-gc.md | 5 +++-- ...ontainers-storage-get-container-data-digest.md | 5 +++-- .../containers-storage-get-container-data-size.md | 5 +++-- .../docs/containers-storage-get-container-data.md | 5 +++-- .../docs/containers-storage-get-container-dir.md | 5 +++-- .../containers-storage-get-container-run-dir.md | 5 +++-- .../containers-storage-get-image-data-digest.md | 5 +++-- .../containers-storage-get-image-data-size.md | 5 +++-- storage/docs/containers-storage-get-image-data.md | 5 +++-- storage/docs/containers-storage-get-image-dir.md | 5 +++-- .../docs/containers-storage-get-image-run-dir.md | 5 +++-- storage/docs/containers-storage-get-layer-data.md | 9 +++++++-- storage/docs/containers-storage-get-names.md | 8 ++++++-- storage/docs/containers-storage-image.md | 8 +++++--- .../docs/containers-storage-images-by-digest.md | 5 +++-- storage/docs/containers-storage-images.md | 5 +++-- storage/docs/containers-storage-import-layer.md | 5 +++-- storage/docs/containers-storage-layer.md | 7 ++++--- storage/docs/containers-storage-layers.md | 7 ++++--- .../containers-storage-list-container-data.md | 5 +++-- .../docs/containers-storage-list-image-data.md | 5 +++-- .../docs/containers-storage-list-layer-data.md | 5 +++-- storage/docs/containers-storage-metadata.md | 5 +++-- storage/docs/containers-storage-mount.md | 5 +++-- storage/docs/containers-storage-mounted.md | 5 +++-- storage/docs/containers-storage-remove-names.md | 5 +++-- .../docs/containers-storage-set-container-data.md | 5 +++-- storage/docs/containers-storage-set-image-data.md | 5 +++-- storage/docs/containers-storage-set-layer-data.md | 8 ++++++-- storage/docs/containers-storage-set-metadata.md | 5 +++-- storage/docs/containers-storage-set-names.md | 7 +++++-- storage/docs/containers-storage-shutdown.md | 5 +++-- storage/docs/containers-storage-status.md | 5 +++-- storage/docs/containers-storage-unmount.md | 7 ++++--- storage/docs/containers-storage-unshare.md | 5 +++-- storage/docs/containers-storage-version.md | 5 +++-- storage/docs/containers-storage-wipe.md | 5 +++-- storage/docs/containers-storage-zstd-chunked.md | 1 - storage/docs/containers-storage.md | 4 +++- 61 files changed, 211 insertions(+), 129 deletions(-) diff --git a/storage/docs/containers-storage-add-names.md b/storage/docs/containers-storage-add-names.md index d39c6a8ced..6746fca01b 100644 --- a/storage/docs/containers-storage-add-names.md +++ b/storage/docs/containers-storage-add-names.md @@ -1,7 +1,7 @@ ## containers-storage-add-names 1 "August 2016" ## NAME -containers-storage add-names - Add names to a layer/image/container +containers-storage-add-names - Add names to a layer/image/container ## SYNOPSIS **containers-storage** **add-names** [*options* [...]] *layerOrImageOrContainerNameOrID* @@ -19,7 +19,13 @@ is already used by another layer, image, or container, it is removed from that other layer, image, or container. ## EXAMPLE -**containers-storage add-names -n my-awesome-container -n my-for-realsies-awesome-container f3be6c6134d0d980936b4c894f1613b69a62b79588fdeda744d0be3693bde8ec** + +``` +containers-storage add-names \ + -n my-awesome-container \ + -n my-for-realsies-awesome-container \ + f3be6c6134d0d980936b4c894f1613b69a62b79588fdeda744d0be3693bde8ec +``` ## SEE ALSO containers-storage-get-names(1) diff --git a/storage/docs/containers-storage-applydiff-using-staging-dir.md b/storage/docs/containers-storage-applydiff-using-staging-dir.md index c7fcc4c1f0..3613f64688 100644 --- a/storage/docs/containers-storage-applydiff-using-staging-dir.md +++ b/storage/docs/containers-storage-applydiff-using-staging-dir.md @@ -1,7 +1,7 @@ ## containers-storage-applydiff-using-staging-dir 1 "September 2023" ## NAME -containers-storage applydiff-using-staging-dir - Apply a layer diff to a layer using a staging directory +containers-storage-applydiff-using-staging-dir - Apply a layer diff to a layer using a staging directory ## SYNOPSIS **containers-storage** **applydiff-using-staging-dir** *layerNameOrID* *source* @@ -18,7 +18,9 @@ Differently than **apply-diff**, the command **applydiff-using-staging-dir** first creates a staging directory and then moves the final result to the destination. ## EXAMPLE -**containers-storage applydiff-using-staging-dir 5891b5b522 /path/to/diff** +``` +containers-storage applydiff-using-staging-dir 5891b5b522 /path/to/diff +``` ## SEE ALSO containers-storage-apply-diff(1) diff --git a/storage/docs/containers-storage-applydiff.md b/storage/docs/containers-storage-applydiff.md index bfe9ed25d7..7a98e819d3 100644 --- a/storage/docs/containers-storage-applydiff.md +++ b/storage/docs/containers-storage-applydiff.md @@ -1,7 +1,7 @@ ## containers-storage-apply-diff 1 "August 2016" ## NAME -containers-storage apply-diff - Apply a layer diff to a layer +containers-storage-apply-diff - Apply a layer diff to a layer ## SYNOPSIS **containers-storage** **apply-diff** [*options* [...]] *layerNameOrID* [*referenceLayerNameOrID*] @@ -24,7 +24,8 @@ Specifies the name of a file from which the diff should be read. If this option is not used, the diff is read from standard input. ## EXAMPLE -**containers-storage apply-diff -f 71841c97e320d6cde.tar.gz layer1** + + containers-storage apply-diff -f 71841c97e320d6cde.tar.gz layer1 ## SEE ALSO containers-storage-changes(1) diff --git a/storage/docs/containers-storage-changes.md b/storage/docs/containers-storage-changes.md index 87fa18c4fc..4909ef9448 100644 --- a/storage/docs/containers-storage-changes.md +++ b/storage/docs/containers-storage-changes.md @@ -1,7 +1,7 @@ ## containers-storage-changes 1 "August 2016" ## NAME -containers-storage changes - Produce a list of changes in a layer +containers-storage-changes - Produce a list of changes in a layer ## SYNOPSIS **containers-storage** **changes** *layerNameOrID* [*referenceLayerNameOrID*] @@ -13,7 +13,8 @@ obtain a summary of which files have been added, deleted, or modified in the layer. ## EXAMPLE -**containers-storage changes f3be6c6134d0d980936b4c894f1613b69a62b79588fdeda744d0be3693bde8ec** + + containers-storage changes f3be6c6134d0d980936b4c894f1613b69a62b79588fdeda744d0be3693bde8ec ## SEE ALSO containers-storage-applydiff(1) diff --git a/storage/docs/containers-storage-check.md b/storage/docs/containers-storage-check.md index 6cd6c694d1..327dffb3d3 100644 --- a/storage/docs/containers-storage-check.md +++ b/storage/docs/containers-storage-check.md @@ -1,7 +1,7 @@ ## containers-storage-check 1 "September 2022" ## NAME -containers-storage check - Check for and remove damaged layers/images/containers +containers-storage-check - Check for and remove damaged layers/images/containers ## SYNOPSIS **containers-storage** **check** [-q] [-r [-f]] @@ -28,7 +28,8 @@ currently skips verifying that a layer which was initialized using a diff can reproduce that diff if asked to. ## EXAMPLE -**containers-storage check -r -f + + containers-storage check -r -f ## SEE ALSO containers-storage(1) diff --git a/storage/docs/containers-storage-composefs.md b/storage/docs/containers-storage-composefs.md index fdfb8ed14c..9e31dc24e4 100644 --- a/storage/docs/containers-storage-composefs.md +++ b/storage/docs/containers-storage-composefs.md @@ -15,7 +15,8 @@ use_composefs = "true" This value must be a "string bool", it cannot be a native TOML boolean. However at the current time, composefs requires zstd:chunked images, so first -you must be sure that zstd:chunked is enabled. For more, see [zstd:chunked](containers-storage-zstd-chunked.md). +you must be sure that zstd:chunked is enabled. +For more, see [zstd:chunked](containers-storage-zstd-chunked.md). Additionally, not many images are in zstd:chunked format. In order to bridge this gap, `convert_images = "true"` can be specified which does a dynamic conversion; this adds @@ -37,10 +38,12 @@ is implemented as an "option" for the `overlay` driver. Some file formats remain unchanged and are inherited from the overlay driver, even when composefs is in use. The primary differences are enumerated below. -The `diff/` directory for each layer is no longer a plain unpacking of the tarball, -but becomes an "object hash directory", where each filename is the sha256 of its contents. This `diff/` -directory is the backing store for a `composefs-data/composefs.blob` created for -each layer which is the composefs "superblock" containing all the non-regular-file content (i.e. metadata) from the tarball. +The `diff/` directory for each layer is no longer a plain unpacking of the +tarball, but becomes an "object hash directory", where each filename is the +sha256 of its contents. This `diff/` directory is the backing store for a +`composefs-data/composefs.blob` created for each layer which is the composefs +"superblock" containing all the non-regular-file content (i.e. metadata) from +the tarball. As with `zstd:chunked`, existing layers are scanned for matching objects, and reused (via hardlink or reflink as configured) if objects with a matching "full sha256" are @@ -59,7 +62,7 @@ in the "default overlay" (unpacked) format will be reused as is. ## BUGS -- https://github.com/containers/storage/issues?q=is%3Aissue+is%3Aopen+label%3Aarea%2Fcomposefs +https://github.com/containers/storage/issues?q=is%3Aissue+is%3Aopen+label%3Aarea%2Fcomposefs ## FOOTNOTES The Containers Storage project is committed to inclusivity, a core value of open source. diff --git a/storage/docs/containers-storage-config.md b/storage/docs/containers-storage-config.md index 5b452f3f65..572bc70950 100644 --- a/storage/docs/containers-storage-config.md +++ b/storage/docs/containers-storage-config.md @@ -1,7 +1,7 @@ ## containers-storage-config 1 "November 2024" ## NAME -containers-storage config - Output the configuration for the storage library +containers-storage-config - Output the configuration for the storage library ## SYNOPSIS **containers-storage** **config** [configurationFile] @@ -12,7 +12,9 @@ current configuration with the contents of a specified configuration file loaded in, in a JSON format. ## EXAMPLE -**containers-storage config** + + containers-storage config + ## SEE ALSO containers-storage-version(1) diff --git a/storage/docs/containers-storage-container.md b/storage/docs/containers-storage-container.md index 29b2edf469..1f20c3a939 100644 --- a/storage/docs/containers-storage-container.md +++ b/storage/docs/containers-storage-container.md @@ -1,7 +1,7 @@ ## containers-storage-container 1 "August 2016" ## NAME -containers-storage container - Examine a single container +containers-storage-container - Examine a single container ## SYNOPSIS **containers-storage** **container** *containerNameOrID* @@ -11,8 +11,9 @@ Retrieve information about a container: any names it has, which image was used to create it, any names that image has, and the ID of the container's layer. ## EXAMPLE -**containers-storage container f3be6c6134d0d980936b4c894f1613b69a62b79588fdeda744d0be3693bde8ec** -**containers-storage container my-awesome-container** + + containers-storage container f3be6c6134d0d980936b4c894f1613b69a62b79588fdeda744d0be3693bde8ec + containers-storage container my-awesome-container ## SEE ALSO containers-storage-containers(1) diff --git a/storage/docs/containers-storage-containers.md b/storage/docs/containers-storage-containers.md index 7d394ef805..94b0cf772e 100644 --- a/storage/docs/containers-storage-containers.md +++ b/storage/docs/containers-storage-containers.md @@ -1,7 +1,7 @@ ## containers-storage-containers 1 "August 2016" ## NAME -containers-storage containers - List known containers +containers-storage-containers - List known containers ## SYNOPSIS **containers-storage** **containers** @@ -10,7 +10,8 @@ containers-storage containers - List known containers Retrieves information about all known containers and lists their IDs and names. ## EXAMPLE -**containers-storage containers** + + containers-storage containers ## SEE ALSO containers-storage-container(1) diff --git a/storage/docs/containers-storage-copy.md b/storage/docs/containers-storage-copy.md index 121cdc8142..c84e914de8 100644 --- a/storage/docs/containers-storage-copy.md +++ b/storage/docs/containers-storage-copy.md @@ -1,7 +1,7 @@ ## containers-storage-copy 1 "April 2018" ## NAME -containers-storage copy - Copy content into a layer +containers-storage-copy - Copy content into a layer ## SYNOPSIS **containers-storage** **copy** [--chown UID[:GID]] [*sourceLayerNameOrID*:]/path [...] *targetLayerNameOrID*:/path @@ -17,7 +17,8 @@ specified, the UID is used. The owner IDs are mapped as appropriate for the target layer. ## EXAMPLE -**containers-storage copy layer-with-mapping:/root/config.txt layer-with-different-mapping:/root/config.txt** + + containers-storage copy layer-with-mapping:/root/config.txt layer-with-different-mapping:/root/config.txt ## SEE ALSO containers-storage(1) diff --git a/storage/docs/containers-storage-create-container.md b/storage/docs/containers-storage-create-container.md index 465777792f..a6299c49ff 100644 --- a/storage/docs/containers-storage-create-container.md +++ b/storage/docs/containers-storage-create-container.md @@ -1,7 +1,7 @@ ## containers-storage-create-container 1 "August 2016" ## NAME -containers-storage create-container - Create a container +containers-storage-create-container - Create a container ## SYNOPSIS **containers-storage** **create-container** [*options*...] *imageNameOrID* @@ -29,7 +29,8 @@ Sets the metadata for the container to the specified value. Sets the metadata for the container to the contents of the specified file. ## EXAMPLE -**containers-storage create-container -f manifest.json -n new-container goodimage** + + containers-storage create-container -f manifest.json -n new-container goodimage ## SEE ALSO containers-storage-create-image(1) diff --git a/storage/docs/containers-storage-create-image.md b/storage/docs/containers-storage-create-image.md index 00110b2513..730693b5e2 100644 --- a/storage/docs/containers-storage-create-image.md +++ b/storage/docs/containers-storage-create-image.md @@ -1,7 +1,7 @@ ## containers-storage-create-image 1 "August 2016" ## NAME -containers-storage create-image - Create an image +containers-storage-create-image - Create an image ## SYNOPSIS **containers-storage** **create-image** [*options*...] *topLayerNameOrID* @@ -29,7 +29,8 @@ Sets the metadata for the image to the specified value. Sets the metadata for the image to the contents of the specified file. ## EXAMPLE -**containers-storage create-image -f manifest.json -n new-image somelayer** + + containers-storage create-image -f manifest.json -n new-image somelayer ## SEE ALSO containers-storage-create-container(1) diff --git a/storage/docs/containers-storage-create-layer.md b/storage/docs/containers-storage-create-layer.md index 4f97dcff97..e0467cfa65 100644 --- a/storage/docs/containers-storage-create-layer.md +++ b/storage/docs/containers-storage-create-layer.md @@ -1,7 +1,7 @@ ## containers-storage-create-layer 1 "August 2016" ## NAME -containers-storage create-layer - Create a layer +containers-storage-create-layer - Create a layer ## SYNOPSIS **containers-storage** **create-layer** [*options* [...]] [*parentLayerNameOrID*] @@ -26,7 +26,8 @@ Sets the label which should be assigned as an SELinux context when mounting the layer. ## EXAMPLE -**containers-storage create-layer -f manifest.json -n new-layer somelayer** + + containers-storage create-layer -f manifest.json -n new-layer somelayer ## SEE ALSO containers-storage-create-container(1) diff --git a/storage/docs/containers-storage-create-storage-layer.md b/storage/docs/containers-storage-create-storage-layer.md index ed29af2be8..7ef96565f8 100644 --- a/storage/docs/containers-storage-create-storage-layer.md +++ b/storage/docs/containers-storage-create-storage-layer.md @@ -1,7 +1,7 @@ ## containers-storage-create-storage-layer 1 "September 2022" ## NAME -containers-storage create-storage-layer - Create a layer in a lower-level storage driver +containers-storage-create-storage-layer - Create a layer in a lower-level storage driver ## SYNOPSIS **containers-storage** **create-storage-layer** [*options* [...]] [*parentLayerNameOrID*] @@ -21,7 +21,8 @@ Sets the label which should be assigned as an SELinux context when mounting the layer. ## EXAMPLE -**containers-storage create-storage-layer somelayer** + + containers-storage create-storage-layer somelayer ## SEE ALSO containers-storage-create-container(1) diff --git a/storage/docs/containers-storage-dedup.md b/storage/docs/containers-storage-dedup.md index eb58ccab44..acba6857d7 100644 --- a/storage/docs/containers-storage-dedup.md +++ b/storage/docs/containers-storage-dedup.md @@ -1,7 +1,7 @@ ## containers-storage-dedup 1 "November 2024" ## NAME -containers-storage dedup - Deduplicate similar files in the images +containers-storage-dedup - Deduplicate similar files in the images ## SYNOPSIS **containers-storage** **dedup** @@ -15,4 +15,5 @@ Find similar files in the images and deduplicate them. It requires reflink supp Specify the function to use to calculate the hash for a file. It can be one of: *size*, *crc*, *sha256sum*. ## EXAMPLE -**containers-storage dedup** + + containers-storage dedup diff --git a/storage/docs/containers-storage-delete-container.md b/storage/docs/containers-storage-delete-container.md index 4afa6d9121..57e788e60f 100644 --- a/storage/docs/containers-storage-delete-container.md +++ b/storage/docs/containers-storage-delete-container.md @@ -1,7 +1,7 @@ ## containers-storage-delete-container 1 "August 2016" ## NAME -containers-storage delete-container - Delete a container +containers-storage-delete-container - Delete a container ## SYNOPSIS **containers-storage** **delete-container** *containerNameOrID* @@ -10,7 +10,8 @@ containers-storage delete-container - Delete a container Deletes a container and its layer. ## EXAMPLE -**containers-storage delete-container my-awesome-container** + + containers-storage delete-container my-awesome-container ## SEE ALSO containers-storage-create-container(1) diff --git a/storage/docs/containers-storage-delete-image.md b/storage/docs/containers-storage-delete-image.md index 9e7fb124dc..da4c770732 100644 --- a/storage/docs/containers-storage-delete-image.md +++ b/storage/docs/containers-storage-delete-image.md @@ -1,7 +1,7 @@ ## containers-storage-delete-image 1 "August 2016" ## NAME -containers-storage delete-image - Delete an image +containers-storage-delete-image - Delete an image ## SYNOPSIS **containers-storage** **delete-image** *imageNameOrID* @@ -13,7 +13,8 @@ If that image's parent is then not being used by other images, it, too, will be removed, and this will be repeated for each parent's parent. ## EXAMPLE -**containers-storage delete-image my-base-image** + + containers-storage delete-image my-base-image ## SEE ALSO containers-storage-create-image(1) diff --git a/storage/docs/containers-storage-delete-layer.md b/storage/docs/containers-storage-delete-layer.md index a96d47cdae..b0e20e4203 100644 --- a/storage/docs/containers-storage-delete-layer.md +++ b/storage/docs/containers-storage-delete-layer.md @@ -1,7 +1,7 @@ ## containers-storage-delete-layer 1 "August 2016" ## NAME -containers-storage delete-layer - Delete a layer +containers-storage-delete-layer - Delete a layer ## SYNOPSIS **containers-storage** **delete-layer** *layerNameOrID* @@ -11,7 +11,8 @@ Deletes a layer if it is not currently being used by any images or containers, and is not the parent of any other layers. ## EXAMPLE -**containers-storage delete-layer my-base-layer** + + containers-storage delete-layer my-base-layer ## SEE ALSO containers-storage-create-layer(1) diff --git a/storage/docs/containers-storage-delete.md b/storage/docs/containers-storage-delete.md index 42ae68654c..5b4dc0c285 100644 --- a/storage/docs/containers-storage-delete.md +++ b/storage/docs/containers-storage-delete.md @@ -1,7 +1,7 @@ ## containers-storage-delete 1 "August 2016" ## NAME -containers-storage delete - Force deletion of a layer, image, or container +containers-storage-delete - Force deletion of a layer, image, or container ## SYNOPSIS **containers-storage** **delete** *layerOrImageOrContainerNameOrID* @@ -11,7 +11,8 @@ Deletes a specified layer, image, or container, with no safety checking. This can corrupt data, and may be removed. ## EXAMPLE -**containers-storage delete my-base-layer** + + containers-storage delete my-base-layer ## SEE ALSO containers-storage-delete-container(1) diff --git a/storage/docs/containers-storage-diff.md b/storage/docs/containers-storage-diff.md index 8a1b2f243b..466c3de200 100644 --- a/storage/docs/containers-storage-diff.md +++ b/storage/docs/containers-storage-diff.md @@ -1,7 +1,7 @@ ## containers-storage-diff 1 "August 2016" ## NAME -containers-storage diff - Generate a layer diff +containers-storage-diff - Generate a layer diff ## SYNOPSIS **containers-storage** **diff** [*options* [...]] *layerNameOrID* @@ -29,7 +29,8 @@ Force the diff to be uncompressed. If the layer was populated by a layer diff, and that layer diff was compressed, it will be decompressed for output. ## EXAMPLE -**containers-storage diff my-base-layer** + + containers-storage diff my-base-layer ## SEE ALSO containers-storage-applydiff(1) diff --git a/storage/docs/containers-storage-diffsize.md b/storage/docs/containers-storage-diffsize.md index 77f19345f1..35eae92314 100644 --- a/storage/docs/containers-storage-diffsize.md +++ b/storage/docs/containers-storage-diffsize.md @@ -1,7 +1,7 @@ ## containers-storage-diffsize 1 "August 2016" ## NAME -containers-storage diffsize - Compute the size of a layer diff +containers-storage-diffsize - Compute the size of a layer diff ## SYNOPSIS **containers-storage** **diffsize** *layerNameOrID* @@ -11,7 +11,8 @@ Computes the expected size of the layer diff which would be generated for the specified layer. ## EXAMPLE -**containers-storage diffsize my-base-layer** + + containers-storage diffsize my-base-layer ## SEE ALSO containers-storage-applydiff(1) diff --git a/storage/docs/containers-storage-exists.md b/storage/docs/containers-storage-exists.md index dd8a4eb19a..3d652c1128 100644 --- a/storage/docs/containers-storage-exists.md +++ b/storage/docs/containers-storage-exists.md @@ -1,7 +1,7 @@ ## containers-storage-exists 1 "August 2016" ## NAME -containers-storage exists - Check if a layer, image, or container exists +containers-storage-exists - Check if a layer, image, or container exists ## SYNOPSIS **containers-storage** **exists** [*options* [...]] *layerOrImageOrContainerNameOrID* [...] @@ -28,4 +28,5 @@ Only succeed if the names or IDs are that of layers. Suppress output. ## EXAMPLE -**containers-storage exists my-base-layer** + + containers-storage exists my-base-layer diff --git a/storage/docs/containers-storage-gc.md b/storage/docs/containers-storage-gc.md index 81c51c599b..02004669f1 100644 --- a/storage/docs/containers-storage-gc.md +++ b/storage/docs/containers-storage-gc.md @@ -1,7 +1,7 @@ ## containers-storage-gc 1 "January 2023" ## NAME -containers-storage gc - Garbage collect leftovers from partial layers/images/containers +containers-storage-gc - Garbage collect leftovers from partial layers/images/containers ## SYNOPSIS **containers-storage** **gc** @@ -13,4 +13,5 @@ which may have been left on the filesystem after canceled attempts to create those layers, images, or containers. ## EXAMPLE -**containers-storage gc** + + containers-storage gc diff --git a/storage/docs/containers-storage-get-container-data-digest.md b/storage/docs/containers-storage-get-container-data-digest.md index 4a2e78306d..8c022d5b28 100644 --- a/storage/docs/containers-storage-get-container-data-digest.md +++ b/storage/docs/containers-storage-get-container-data-digest.md @@ -1,7 +1,7 @@ ## containers-storage-get-container-data-digest 1 "August 2017" ## NAME -containers-storage get-container-data-digest - Retrieve the digest of a lookaside data item +containers-storage-get-container-data-digest - Retrieve the digest of a lookaside data item ## SYNOPSIS **containers-storage** **get-container-data-digest** *containerNameOrID* *dataName* @@ -11,7 +11,8 @@ Prints the digest of the named data item which is associated with the specified container. ## EXAMPLE -**containers-storage get-container-data-digest my-container manifest.json** + + containers-storage get-container-data-digest my-container manifest.json ## SEE ALSO containers-storage-get-container-data(1) diff --git a/storage/docs/containers-storage-get-container-data-size.md b/storage/docs/containers-storage-get-container-data-size.md index b92dec26c5..3757ca685c 100644 --- a/storage/docs/containers-storage-get-container-data-size.md +++ b/storage/docs/containers-storage-get-container-data-size.md @@ -1,7 +1,7 @@ ## containers-storage-get-container-data-size 1 "August 2017" ## NAME -containers-storage get-container-data-size - Retrieve the size of a lookaside data item +containers-storage-get-container-data-size - Retrieve the size of a lookaside data item ## SYNOPSIS **containers-storage** **get-container-data-size** *containerNameOrID* *dataName* @@ -11,7 +11,8 @@ Prints the size of the named data item which is associated with the specified container. ## EXAMPLE -**containers-storage get-container-data-size my-container blah.foo** + + containers-storage get-container-data-size my-container blah.foo ## SEE ALSO containers-storage-get-container-data(1) diff --git a/storage/docs/containers-storage-get-container-data.md b/storage/docs/containers-storage-get-container-data.md index 0dbfdcca20..e2b7a5b0ee 100644 --- a/storage/docs/containers-storage-get-container-data.md +++ b/storage/docs/containers-storage-get-container-data.md @@ -1,7 +1,7 @@ ## containers-storage-get-container-data 1 "August 2016" ## NAME -containers-storage get-container-data - Retrieve lookaside data for a container +containers-storage-get-container-data - Retrieve lookaside data for a container ## SYNOPSIS **containers-storage** **get-container-data** [*options* [...]] *containerNameOrID* *dataName* @@ -15,7 +15,8 @@ Retrieves a piece of named data which is associated with a container. Write the data to a file instead of stdout. ## EXAMPLE -**containers-storage get-container-data -f config.json my-container configuration** + + containers-storage get-container-data -f config.json my-container configuration ## SEE ALSO containers-storage-list-container-data(1) diff --git a/storage/docs/containers-storage-get-container-dir.md b/storage/docs/containers-storage-get-container-dir.md index 1d90485ea8..ff7c61a2d5 100644 --- a/storage/docs/containers-storage-get-container-dir.md +++ b/storage/docs/containers-storage-get-container-dir.md @@ -1,7 +1,7 @@ ## containers-storage-get-container-dir 1 "September 2016" ## NAME -containers-storage get-container-dir - Find lookaside directory for a container +containers-storage-get-container-dir - Find lookaside directory for a container ## SYNOPSIS **containers-storage** **get-container-dir** [*options* [...]] *containerNameOrID* @@ -11,7 +11,8 @@ Prints the location of a directory which the caller can use to store lookaside information which should be cleaned up when the container is deleted. ## EXAMPLE -**containers-storage get-container-dir my-container** + + containers-storage get-container-dir my-container ## SEE ALSO containers-storage-get-container-run-dir(1) diff --git a/storage/docs/containers-storage-get-container-run-dir.md b/storage/docs/containers-storage-get-container-run-dir.md index 3c78999870..f5dbafe76b 100644 --- a/storage/docs/containers-storage-get-container-run-dir.md +++ b/storage/docs/containers-storage-get-container-run-dir.md @@ -1,7 +1,7 @@ ## containers-storage-get-container-run-dir 1 "September 2016" ## NAME -containers-storage get-container-run-dir - Find runtime lookaside directory for a container +containers-storage-get-container-run-dir - Find runtime lookaside directory for a container ## SYNOPSIS **containers-storage** **get-container-run-dir** [*options* [...]] *containerNameOrID* @@ -11,7 +11,8 @@ Prints the location of a directory which the caller can use to store lookaside information which should be cleaned up when the host is rebooted. ## EXAMPLE -**containers-storage get-container-run-dir my-container** + + containers-storage get-container-run-dir my-container ## SEE ALSO containers-storage-get-container-dir(1) diff --git a/storage/docs/containers-storage-get-image-data-digest.md b/storage/docs/containers-storage-get-image-data-digest.md index 2b5c89d527..8e98e82065 100644 --- a/storage/docs/containers-storage-get-image-data-digest.md +++ b/storage/docs/containers-storage-get-image-data-digest.md @@ -1,7 +1,7 @@ ## containers-storage-get-image-data-digest 1 "August 2017" ## NAME -containers-storage get-image-data-digest - Retrieve the digest of a lookaside data item +containers-storage-get-image-data-digest - Retrieve the digest of a lookaside data item ## SYNOPSIS **containers-storage** **get-image-data-digest** *imageNameOrID* *dataName* @@ -11,7 +11,8 @@ Prints the digest of the named data item which is associated with the specified image. ## EXAMPLE -**containers-storage get-image-data-digest my-image manifest.json** + + containers-storage get-image-data-digest my-image manifest.json ## SEE ALSO containers-storage-get-image-data(1) diff --git a/storage/docs/containers-storage-get-image-data-size.md b/storage/docs/containers-storage-get-image-data-size.md index e946508eb6..baadea0459 100644 --- a/storage/docs/containers-storage-get-image-data-size.md +++ b/storage/docs/containers-storage-get-image-data-size.md @@ -1,7 +1,7 @@ ## containers-storage-get-image-data-size 1 "August 2017" ## NAME -containers-storage get-image-data-size - Retrieve the size of a lookaside data item +containers-storage-get-image-data-size - Retrieve the size of a lookaside data item ## SYNOPSIS **containers-storage** **get-image-data-size** *imageNameOrID* *dataName* @@ -11,7 +11,8 @@ Prints the size of the named data item which is associated with the specified image. ## EXAMPLE -**containers-storage get-image-data-size my-image manifest.json** + + containers-storage get-image-data-size my-image manifest.json ## SEE ALSO containers-storage-get-image-data(1) diff --git a/storage/docs/containers-storage-get-image-data.md b/storage/docs/containers-storage-get-image-data.md index ec853d7258..8fd5f7c70c 100644 --- a/storage/docs/containers-storage-get-image-data.md +++ b/storage/docs/containers-storage-get-image-data.md @@ -1,7 +1,7 @@ ## containers-storage-get-image-data 1 "August 2016" ## NAME -containers-storage get-image-data - Retrieve lookaside data for an image +containers-storage-get-image-data - Retrieve lookaside data for an image ## SYNOPSIS **containers-storage** **get-image-data** [*options* [...]] *imageNameOrID* *dataName* @@ -15,7 +15,8 @@ Retrieves a piece of named data which is associated with an image. Write the data to a file instead of stdout. ## EXAMPLE -**containers-storage get-image-data -f manifest.json my-image manifest** + + containers-storage get-image-data -f manifest.json my-image manifest ## SEE ALSO containers-storage-list-image-data(1) diff --git a/storage/docs/containers-storage-get-image-dir.md b/storage/docs/containers-storage-get-image-dir.md index 3b5e10b0e1..df1e9d7d0e 100644 --- a/storage/docs/containers-storage-get-image-dir.md +++ b/storage/docs/containers-storage-get-image-dir.md @@ -1,7 +1,7 @@ ## containers-storage-get-image-dir 1 "January 2024" ## NAME -containers-storage get-image-dir - Find lookaside directory for an image +containers-storage-get-image-dir - Find lookaside directory for an image ## SYNOPSIS **containers-storage** **get-image-dir** [*options* [...]] *imageNameOrID* @@ -11,7 +11,8 @@ Prints the location of a directory which the caller can use to store lookaside information which should be cleaned up when the image is deleted. ## EXAMPLE -**containers-storage get-image-dir my-image** + + containers-storage get-image-dir my-image ## SEE ALSO containers-storage-get-image-run-dir(1) diff --git a/storage/docs/containers-storage-get-image-run-dir.md b/storage/docs/containers-storage-get-image-run-dir.md index 03196cbbbe..1df6b6cb3e 100644 --- a/storage/docs/containers-storage-get-image-run-dir.md +++ b/storage/docs/containers-storage-get-image-run-dir.md @@ -1,7 +1,7 @@ ## containers-storage-get-image-run-dir 1 "January 2024" ## NAME -containers-storage get-image-run-dir - Find runtime lookaside directory for an image +containers-storage-get-image-run-dir - Find runtime lookaside directory for an image ## SYNOPSIS **containers-storage** **get-image-run-dir** [*options* [...]] *imageNameOrID* @@ -11,7 +11,8 @@ Prints the location of a directory which the caller can use to store lookaside information which should be cleaned up when the host is rebooted. ## EXAMPLE -**containers-storage get-image-run-dir my-image** + + containers-storage get-image-run-dir my-image ## SEE ALSO containers-storage-get-image-dir(1) diff --git a/storage/docs/containers-storage-get-layer-data.md b/storage/docs/containers-storage-get-layer-data.md index b34ce3e866..edb42f3b97 100644 --- a/storage/docs/containers-storage-get-layer-data.md +++ b/storage/docs/containers-storage-get-layer-data.md @@ -1,7 +1,7 @@ ## containers-storage-get-layer-data 1 "December 2020" ## NAME -containers-storage get-layer-data - Retrieve lookaside data for a layer +containers-storage-get-layer-data - Retrieve lookaside data for a layer ## SYNOPSIS **containers-storage** **get-layer-data** [*options* [...]] *layerID* *dataName* @@ -15,7 +15,12 @@ Retrieves a piece of named data which is associated with a layer. Write the data to a file instead of stdout. ## EXAMPLE -**containers-storage get-layer-data -f config.json 2cf24dba5fb0a30e26e83b2ac5b9e29e1b161e5c1fa7425e73043362938b9824 configuration** + +``` +containers-storage get-layer-data -f config.json \ + 2cf24dba5fb0a30e26e83b2ac5b9e29e1b161e5c1fa7425e73043362938b9824 \ + configuration +``` ## SEE ALSO containers-storage-set-layer-data(1) diff --git a/storage/docs/containers-storage-get-names.md b/storage/docs/containers-storage-get-names.md index 83e5cc1c25..b6d6b8719e 100644 --- a/storage/docs/containers-storage-get-names.md +++ b/storage/docs/containers-storage-get-names.md @@ -1,7 +1,7 @@ ## containers-storage-get-names 1 "September 2017" ## NAME -containers-storage get-names - Get names of a layer/image/container +containers-storage-get-names - Get names of a layer/image/container ## SYNOPSIS **containers-storage** **get-names** *layerOrImageOrContainerNameOrID* @@ -14,7 +14,11 @@ command can be used to read the list of names for any of them. ## OPTIONS ## EXAMPLE -**containers-storage get-names f3be6c6134d0d980936b4c894f1613b69a62b79588fdeda744d0be3693bde8ec** + +``` +containers-storage get-names \ + f3be6c6134d0d980936b4c894f1613b69a62b79588fdeda744d0be3693bde8ec +``` ## SEE ALSO containers-storage-add-names(1) diff --git a/storage/docs/containers-storage-image.md b/storage/docs/containers-storage-image.md index a72eb5312f..900acea6b8 100644 --- a/storage/docs/containers-storage-image.md +++ b/storage/docs/containers-storage-image.md @@ -1,7 +1,7 @@ ## containers-storage-image 1 "August 2016" ## NAME -containers-storage image - Examine a single image +containers-storage-image - Examine a single image ## SYNOPSIS **containers-storage** **image** *imageNameOrID* @@ -11,8 +11,10 @@ Retrieve information about an image: its ID, any names it has, and the ID of its top layer. ## EXAMPLE -**containers-storage image 49bff34e4baf9378c01733d02276a731a4c4771ebeab305020c5303679f88bb8** -**containers-storage image my-favorite-image** + + containers-storage image 49bff34e4baf9378c01733d02276a731a4c4771ebeab305020c5303679f88bb8 + + containers-storage image my-favorite-image ## SEE ALSO containers-storage-images(1) diff --git a/storage/docs/containers-storage-images-by-digest.md b/storage/docs/containers-storage-images-by-digest.md index 2a03c8eac8..ef74d362e6 100644 --- a/storage/docs/containers-storage-images-by-digest.md +++ b/storage/docs/containers-storage-images-by-digest.md @@ -1,7 +1,7 @@ ## containers-storage-images-by-digest 1 "February 2019" ## NAME -containers-storage images-by-digest - List known images by digest +containers-storage-images-by-digest - List known images by digest ## SYNOPSIS **containers-storage** **images-by-digest** *digest* @@ -10,7 +10,8 @@ containers-storage images-by-digest - List known images by digest Retrieves information about images which match a specified digest ## EXAMPLE -**containers-storage images-by-digest sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855** + + containers-storage images-by-digest sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 ## SEE ALSO containers-storage-image(1) diff --git a/storage/docs/containers-storage-images.md b/storage/docs/containers-storage-images.md index 00a381ac88..2f6c75d6d9 100644 --- a/storage/docs/containers-storage-images.md +++ b/storage/docs/containers-storage-images.md @@ -1,7 +1,7 @@ ## containers-storage-images 1 "August 2016" ## NAME -containers-storage images - List known images +containers-storage-images - List known images ## SYNOPSIS **containers-storage** **images** @@ -10,7 +10,8 @@ containers-storage images - List known images Retrieves information about all known images and lists their IDs and names. ## EXAMPLE -**containers-storage images** + + containers-storage images ## SEE ALSO containers-storage-image(1) diff --git a/storage/docs/containers-storage-import-layer.md b/storage/docs/containers-storage-import-layer.md index 3cbc153e37..50d192c74f 100644 --- a/storage/docs/containers-storage-import-layer.md +++ b/storage/docs/containers-storage-import-layer.md @@ -1,7 +1,7 @@ ## containers-storage-import-layer 1 "April 2019" ## NAME -containers-storage import-layer - Import files to a new layer +containers-storage-import-layer - Import files to a new layer ## SYNOPSIS **containers-storage** **import-layer** [*options* [...]] [*parentLayerNameOrID*] @@ -69,7 +69,8 @@ Create UID map for username using the data from /etc/subuid. It cannot be specif Create GID map for group-name using the data from /etc/subgid. It cannot be specified simultaneously with *--hostuidmap*. ## EXAMPLE -**containers-storage import-layer -f 71841c97e320d6cde.tar.gz -n new-layer somelayer** + + containers-storage import-layer -f 71841c97e320d6cde.tar.gz -n new-layer somelayer ## SEE ALSO containers-storage-create-layer(1) diff --git a/storage/docs/containers-storage-layer.md b/storage/docs/containers-storage-layer.md index 2d4869dff8..f8bbcbba4f 100644 --- a/storage/docs/containers-storage-layer.md +++ b/storage/docs/containers-storage-layer.md @@ -1,7 +1,7 @@ ## containers-storage-layer 1 "September 2017" ## NAME -containers-storage layer - Examine a single layer +containers-storage-layer - Examine a single layer ## SYNOPSIS **containers-storage** **layer** *layerNameOrID* @@ -11,8 +11,9 @@ Retrieve information about a layer: its ID, any names it has, and the ID of its parent, if it has one. ## EXAMPLE -**containers-storage layer 49bff34e4baf9378c01733d02276a731a4c4771ebeab305020c5303679f88bb8** -**containers-storage layer my-favorite-layer** + + containers-storage layer 49bff34e4baf9378c01733d02276a731a4c4771ebeab305020c5303679f88bb8 + containers-storage layer my-favorite-layer ## SEE ALSO containers-storage-image(1) diff --git a/storage/docs/containers-storage-layers.md b/storage/docs/containers-storage-layers.md index 939916a2f4..30348d583f 100644 --- a/storage/docs/containers-storage-layers.md +++ b/storage/docs/containers-storage-layers.md @@ -1,7 +1,7 @@ ## containers-storage-layers 1 "August 2016" ## NAME -containers-storage layers - List known layers +containers-storage-layers - List known layers ## SYNOPSIS **containers-storage** [*options* [...]] **layers** @@ -19,5 +19,6 @@ Display results using a tree to show the hierarchy of parent-child relationships between layers. ## EXAMPLE -**containers-storage layers** -**containers-storage layers -t** + + containers-storage layers + containers-storage layers -t diff --git a/storage/docs/containers-storage-list-container-data.md b/storage/docs/containers-storage-list-container-data.md index 0c7ccbc957..5ca7275202 100644 --- a/storage/docs/containers-storage-list-container-data.md +++ b/storage/docs/containers-storage-list-container-data.md @@ -1,7 +1,7 @@ ## containers-storage-list-container-data 1 "August 2016" ## NAME -containers-storage list-container-data - List lookaside data for a container +containers-storage-list-container-data - List lookaside data for a container ## SYNOPSIS **containers-storage** **list-container-data** *containerNameOrID* @@ -10,7 +10,8 @@ containers-storage list-container-data - List lookaside data for a container List the pieces of named data which are associated with a container. ## EXAMPLE -**containers-storage list-container-data my-container** + + containers-storage list-container-data my-container ## SEE ALSO containers-storage-get-container-data(1) diff --git a/storage/docs/containers-storage-list-image-data.md b/storage/docs/containers-storage-list-image-data.md index 5dd8fc6f8e..77c74a45c2 100644 --- a/storage/docs/containers-storage-list-image-data.md +++ b/storage/docs/containers-storage-list-image-data.md @@ -1,7 +1,7 @@ ## containers-storage-list-image-data 1 "August 2016" ## NAME -containers-storage list-image-data - List lookaside data for an image +containers-storage-list-image-data - List lookaside data for an image ## SYNOPSIS **containers-storage** **list-image-data** *imageNameOrID* @@ -10,7 +10,8 @@ containers-storage list-image-data - List lookaside data for an image List the pieces of named data which are associated with an image. ## EXAMPLE -**containers-storage list-image-data my-image** + + containers-storage list-image-data my-image ## SEE ALSO containers-storage-get-image-data(1) diff --git a/storage/docs/containers-storage-list-layer-data.md b/storage/docs/containers-storage-list-layer-data.md index 41afe2797a..20cc020ffc 100644 --- a/storage/docs/containers-storage-list-layer-data.md +++ b/storage/docs/containers-storage-list-layer-data.md @@ -1,7 +1,7 @@ ## containers-storage-list-layer-data 1 "December 2020" ## NAME -containers-storage list-layer-data - List lookaside data for a layer +containers-storage-list-layer-data - List lookaside data for a layer ## SYNOPSIS **containers-storage** **list-layer-data** *layerID* @@ -10,7 +10,8 @@ containers-storage list-layer-data - List lookaside data for a layer List the pieces of named data which are associated with a layer. ## EXAMPLE -**containers-storage list-layer-data 2cf24dba5fb0a30e26e83b2ac5b9e29e1b161e5c1fa7425e73043362938b9824** + + containers-storage list-layer-data 2cf24dba5fb0a30e26e83b2ac5b9e29e1b161e5c1fa7425e73043362938b9824 ## SEE ALSO containers-storage-get-layer-data(1) diff --git a/storage/docs/containers-storage-metadata.md b/storage/docs/containers-storage-metadata.md index 0ac629fb84..90bb1fbf4b 100644 --- a/storage/docs/containers-storage-metadata.md +++ b/storage/docs/containers-storage-metadata.md @@ -1,7 +1,7 @@ ## containers-storage-metadata 1 "August 2016" ## NAME -containers-storage metadata - Retrieve metadata for a layer, image, or container +containers-storage-metadata - Retrieve metadata for a layer, image, or container ## SYNOPSIS **containers-storage** **metadata** [*options* [...]] *layerOrImageOrContainerNameOrID* @@ -16,7 +16,8 @@ intended to be small, and is expected to be cached in memory. Don't print the ID or name of the item with which the metadata is associated. ## EXAMPLE -**containers-storage metadata -q my-image > my-image.txt** + + containers-storage metadata -q my-image > my-image.txt ## SEE ALSO containers-storage-set-metadata(1) diff --git a/storage/docs/containers-storage-mount.md b/storage/docs/containers-storage-mount.md index 85559f27f8..7ff624fc1a 100644 --- a/storage/docs/containers-storage-mount.md +++ b/storage/docs/containers-storage-mount.md @@ -1,7 +1,7 @@ ## containers-storage-mount 1 "August 2016" ## NAME -containers-storage mount - Mount a layer or a container's layer for manipulation +containers-storage-mount - Mount a layer or a container's layer for manipulation ## SYNOPSIS **containers-storage** **mount** [*options* [...]] *layerOrContainerNameOrID* @@ -16,7 +16,8 @@ mountpoint. Specify an SELinux context for the mounted layer. ## EXAMPLE -**containers-storage mount my-container** + + containers-storage mount my-container ## SEE ALSO containers-storage-mounted(1) diff --git a/storage/docs/containers-storage-mounted.md b/storage/docs/containers-storage-mounted.md index 204ac28edb..079b5d6bbb 100644 --- a/storage/docs/containers-storage-mounted.md +++ b/storage/docs/containers-storage-mounted.md @@ -1,7 +1,7 @@ ## containers-storage-mounted 1 "November 2018" ## NAME -containers-storage mounted - Check if a file system is mounted +containers-storage-mounted - Check if a file system is mounted ## SYNOPSIS **containers-storage** **mounted** *LayerOrContainerNameOrID* @@ -10,7 +10,8 @@ containers-storage mounted - Check if a file system is mounted Check if a filesystem is mounted ## EXAMPLE -**containers-storage mounted my-container** + + containers-storage mounted my-container ## SEE ALSO containers-storage-mount(1) diff --git a/storage/docs/containers-storage-remove-names.md b/storage/docs/containers-storage-remove-names.md index 767517f902..08cd523408 100644 --- a/storage/docs/containers-storage-remove-names.md +++ b/storage/docs/containers-storage-remove-names.md @@ -1,7 +1,7 @@ ## containers-storage-remove-names 1 "January 2023" ## NAME -containers-storage remove-names - Remove names from a layer/image/container +containers-storage-remove-names - Remove names from a layer/image/container ## SYNOPSIS **containers-storage** **remove-names** [*options* [...]] *layerOrImageOrContainerNameOrID* @@ -17,7 +17,8 @@ command can be used to remove one or more names from them. Specifies a name to remove from the layer, image, or container. ## EXAMPLE -**containers-storage remove-names -n my-for-realsies-awesome-container f3be6c6134d0d980936b4c894f1613b69a62b79588fdeda744d0be3693bde8ec** + + containers-storage remove-names -n my-for-realsies-awesome-container f3be6c6134d0d980936b4c894f1613b69a62b79588fdeda744d0be3693bde8ec ## SEE ALSO containers-storage-add-names(1) diff --git a/storage/docs/containers-storage-set-container-data.md b/storage/docs/containers-storage-set-container-data.md index 4ee23730e6..b1e2a40e46 100644 --- a/storage/docs/containers-storage-set-container-data.md +++ b/storage/docs/containers-storage-set-container-data.md @@ -1,7 +1,7 @@ ## containers-storage-set-container-data 1 "August 2016" ## NAME -containers-storage set-container-data - Set lookaside data for a container +containers-storage-set-container-data - Set lookaside data for a container ## SYNOPSIS **containers-storage** **set-container-data** [*options* [...]] *containerNameOrID* *dataName* @@ -15,7 +15,8 @@ Sets a piece of named data which is associated with a container. Read the data contents from a file instead of stdin. ## EXAMPLE -**containers-storage set-container-data -f ./config.json my-container configuration** + + containers-storage set-container-data -f ./config.json my-container configuration ## SEE ALSO containers-storage-list-container-data(1) diff --git a/storage/docs/containers-storage-set-image-data.md b/storage/docs/containers-storage-set-image-data.md index dd0abe6638..f9bcad71f8 100644 --- a/storage/docs/containers-storage-set-image-data.md +++ b/storage/docs/containers-storage-set-image-data.md @@ -1,7 +1,7 @@ ## containers-storage-set-image-data 1 "August 2016" ## NAME -containers-storage set-image-data - Set lookaside data for an image +containers-storage-set-image-data - Set lookaside data for an image ## SYNOPSIS **containers-storage** **set-image-data** [*options* [...]] *imageNameOrID* *dataName* @@ -15,7 +15,8 @@ Sets a piece of named data which is associated with an image. Read the data contents from a file instead of stdin. ## EXAMPLE -**containers-storage set-image-data -f ./manifest.json my-image manifest** + + containers-storage set-image-data -f ./manifest.json my-image manifest ## SEE ALSO containers-storage-list-image-data(1) diff --git a/storage/docs/containers-storage-set-layer-data.md b/storage/docs/containers-storage-set-layer-data.md index b626dc0c30..4f0cd89b00 100644 --- a/storage/docs/containers-storage-set-layer-data.md +++ b/storage/docs/containers-storage-set-layer-data.md @@ -1,7 +1,7 @@ ## containers-storage-set-layer-data 1 "December 2020" ## NAME -containers-storage set-layer-data - Set lookaside data for a layer +containers-storage-set-layer-data - Set lookaside data for a layer ## SYNOPSIS **containers-storage** **set-layer-data** [*options* [...]] *layerID* *dataName* @@ -15,7 +15,11 @@ Sets a piece of named data which is associated with a layer. Read the data contents from a file instead of stdin. ## EXAMPLE -**containers-storage set-layer-data -f ./config.json 2cf24dba5fb0a30e26e83b2ac5b9e29e1b161e5c1fa7425e73043362938b9824 configuration** + + containers-storage set-layer-data \ + -f ./config.json \ + 2cf24dba5fb0a30e26e83b2ac5b9e29e1b161e5c1fa7425e73043362938b9824 \ + configuration ## SEE ALSO containers-storage-get-layer-data(1) diff --git a/storage/docs/containers-storage-set-metadata.md b/storage/docs/containers-storage-set-metadata.md index 1589ce6107..fdbc9ef7a0 100644 --- a/storage/docs/containers-storage-set-metadata.md +++ b/storage/docs/containers-storage-set-metadata.md @@ -1,7 +1,7 @@ ## containers-storage-set-metadata 1 "August 2016" ## NAME -containers-storage set-metadata - Set metadata for a layer, image, or container +containers-storage-set-metadata - Set metadata for a layer, image, or container ## SYNOPSIS **containers-storage** **set-metadata** [*options* [...]] *layerOrImageOrContainerNameOrID* @@ -20,7 +20,8 @@ Use the contents of the specified file as the metadata. Use the specified value as the metadata. ## EXAMPLE -**containers-storage set-metadata -m "compression: gzip" my-layer** + + containers-storage set-metadata -m "compression: gzip" my-layer ## SEE ALSO containers-storage-metadata(1) diff --git a/storage/docs/containers-storage-set-names.md b/storage/docs/containers-storage-set-names.md index bbc4afd32e..f4cea32a74 100644 --- a/storage/docs/containers-storage-set-names.md +++ b/storage/docs/containers-storage-set-names.md @@ -1,7 +1,7 @@ ## containers-storage-set-names 1 "August 2016" ## NAME -containers-storage set-names - Set names for a layer/image/container +containers-storage-set-names - Set names for a layer/image/container ## SYNOPSIS **containers-storage** **set-names** [**-n** *name* [...]] *layerOrImageOrContainerNameOrID* @@ -21,7 +21,10 @@ this layer, image, or container, and which are not specified using this option, will be removed from the layer, image, or container. ## EXAMPLE -**containers-storage set-names -n my-one-and-only-name f3be6c6134d0d980936b4c894f1613b69a62b79588fdeda744d0be3693bde8ec** + + containers-storage set-names \ + -n my-one-and-only-name \ + f3be6c6134d0d980936b4c894f1613b69a62b79588fdeda744d0be3693bde8ec ## SEE ALSO containers-storage-add-names(1) diff --git a/storage/docs/containers-storage-shutdown.md b/storage/docs/containers-storage-shutdown.md index 0ea6916cda..09e8a24e37 100644 --- a/storage/docs/containers-storage-shutdown.md +++ b/storage/docs/containers-storage-shutdown.md @@ -1,7 +1,7 @@ ## containers-storage-shutdown 1 "October 2016" ## NAME -containers-storage shutdown - Shut down layer storage +containers-storage-shutdown - Shut down layer storage ## SYNOPSIS **containers-storage** **shutdown** [*options* [...]] @@ -17,4 +17,5 @@ driver. If this option is not specified, if any layers are mounted, shutdown will not be attempted. ## EXAMPLE -**containers-storage shutdown** + + containers-storage shutdown diff --git a/storage/docs/containers-storage-status.md b/storage/docs/containers-storage-status.md index fc8a39c592..bac945b7ad 100644 --- a/storage/docs/containers-storage-status.md +++ b/storage/docs/containers-storage-status.md @@ -1,7 +1,7 @@ ## containers-storage-status 1 "August 2016" ## NAME -containers-storage status - Output status information from the storage library's driver +containers-storage-status - Output status information from the storage library's driver ## SYNOPSIS **containers-storage** **status** @@ -10,7 +10,8 @@ containers-storage status - Output status information from the storage library's Queries the storage library's driver for status information. ## EXAMPLE -**containers-storage status** + + containers-storage status ## SEE ALSO containers-storage-version(1) diff --git a/storage/docs/containers-storage-unmount.md b/storage/docs/containers-storage-unmount.md index ef427a5948..0c5b25c9fc 100644 --- a/storage/docs/containers-storage-unmount.md +++ b/storage/docs/containers-storage-unmount.md @@ -1,7 +1,7 @@ ## containers-storage-unmount 1 "August 2016" ## NAME -containers-storage unmount - Unmount a layer or a container's layer +containers-storage-unmount - Unmount a layer or a container's layer ## SYNOPSIS **containers-storage** **unmount** *layerOrContainerMountpointOrNameOrID* @@ -10,9 +10,10 @@ containers-storage unmount - Unmount a layer or a container's layer Unmounts a layer or a container's layer from the host's filesystem. ## EXAMPLE -**containers-storage unmount my-container** -**containers-storage unmount /var/lib/containers/storage/mounts/my-container** + containers-storage unmount my-container + + containers-storage unmount /var/lib/containers/storage/mounts/my-container ## SEE ALSO containers-storage-mount(1) diff --git a/storage/docs/containers-storage-unshare.md b/storage/docs/containers-storage-unshare.md index 97b9fe4e0f..e5a1a59151 100644 --- a/storage/docs/containers-storage-unshare.md +++ b/storage/docs/containers-storage-unshare.md @@ -1,7 +1,7 @@ ## containers-storage-unshare 1 "September 2022" ## NAME -containers-storage unshare - Run a command in a user namespace +containers-storage-unshare - Run a command in a user namespace ## SYNOPSIS **containers-storage** **unshare** [command [...]] @@ -11,7 +11,8 @@ Sets up a user namespace using mappings configured for the current user and runs either a specified command or the current user's shell. ## EXAMPLE -**containers-storage unshare id** + + containers-storage unshare id ## SEE ALSO containers-storage-status(1) diff --git a/storage/docs/containers-storage-version.md b/storage/docs/containers-storage-version.md index 8bc4147a17..b1c364304b 100644 --- a/storage/docs/containers-storage-version.md +++ b/storage/docs/containers-storage-version.md @@ -1,7 +1,7 @@ ## containers-storage-version 1 "August 2016" ## NAME -containers-storage version - Output version information about the storage library +containers-storage-version - Output version information about the storage library ## SYNOPSIS **containers-storage** **version** @@ -10,7 +10,8 @@ containers-storage version - Output version information about the storage librar Outputs version information about the storage library and *containers-storage*. ## EXAMPLE -**containers-storage version** + + containers-storage version ## SEE ALSO containers-storage-status(1) diff --git a/storage/docs/containers-storage-wipe.md b/storage/docs/containers-storage-wipe.md index 85b2edef42..1f7830b28d 100644 --- a/storage/docs/containers-storage-wipe.md +++ b/storage/docs/containers-storage-wipe.md @@ -1,7 +1,7 @@ ## containers-storage-wipe 1 "August 2016" ## NAME -containers-storage wipe - Delete all containers, images, and layers +containers-storage-wipe - Delete all containers, images, and layers ## SYNOPSIS **containers-storage** **wipe** @@ -11,4 +11,5 @@ Deletes all known containers, images, and layers. Depending on your use case, use with caution or abandon. ## EXAMPLE -**containers-storage wipe** + + containers-storage wipe diff --git a/storage/docs/containers-storage-zstd-chunked.md b/storage/docs/containers-storage-zstd-chunked.md index 5e1366e405..e6eb1ba479 100644 --- a/storage/docs/containers-storage-zstd-chunked.md +++ b/storage/docs/containers-storage-zstd-chunked.md @@ -3,7 +3,6 @@ ## NAME containers-storage-zstd-chunked - Information about zstd:chunked - ## DESCRIPTION The traditional format for container image layers is [application/vnd.oci.image.layer.v1.tar+gzip](https://github.com/opencontainers/image-spec/blob/main/layer.md#gzip-media-types). diff --git a/storage/docs/containers-storage.md b/storage/docs/containers-storage.md index 1e95fc4944..3764563694 100644 --- a/storage/docs/containers-storage.md +++ b/storage/docs/containers-storage.md @@ -172,8 +172,10 @@ using the system's default ID mappings for the non-root user. **CONTAINERS_STORAGE_CONF** If set will use the configuration file path provided in *$CONTAINERS_STORAGE_CONF* instead of the default `/etc/containers/storage.conf`. + ## EXAMPLES -**containers-storage layers -t** + + containers-storage layers -t ## BUGS This is still a work in progress, so some functionality may not yet be From ffb437512dd865f3e60abeaa0c18dd4d7fd41a9e Mon Sep 17 00:00:00 2001 From: "Z. Liu" Date: Wed, 17 Dec 2025 10:43:09 +0800 Subject: [PATCH 23/37] idtools: avoid direct use of C.stderr to fix musl cgo build failures MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit On musl-based systems, stderr is declared as FILE *const. Referencing stderr directly from Go code (via C.stderr) causes cgo to generate assignment code for a const-qualified pointer, which is invalid C and fails to compile. Both gcc and clang reject the generated code with error messages below: clang: > cgo-gcc-prolog:85:9: error: cannot assign to variable '_cgo_r' with const-qualified type 'typeof (_cgo_a->r)' (aka 'struct _IO_FILE *const') > cgo-gcc-prolog:83:24: note: variable '_cgo_r' declared const here > cgo-gcc-prolog:88:12: error: cannot assign to non-static data member 'r' with const-qualified type 'FILE *const' (aka 'struct _IO_FILE *const') > cgo-gcc-prolog:80:15: note: non-static data member 'r' declared const here gcc: > cgo-gcc-prolog:85:9: error: assignment of read-only variable '_cgo_r' > cgo-gcc-prolog:88:12: error: assignment of read-only member 'r' This patch avoids referencing C.stderr from Go code and instead returns stderr from a small C helper function. This keeps the usage entirely in C and avoids cgo’s broken handling for const-qualified global objects. Signed-off-by: Z. Liu (cherry picked from commit 9f3bf573f7b794e1cbe30f8194b813ffa9fdc14c) Signed-off-by: Paul Holzinger --- storage/pkg/idtools/idtools_supported.go | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/storage/pkg/idtools/idtools_supported.go b/storage/pkg/idtools/idtools_supported.go index 9a17f57014..8a3076a0f8 100644 --- a/storage/pkg/idtools/idtools_supported.go +++ b/storage/pkg/idtools/idtools_supported.go @@ -20,6 +20,12 @@ struct subid_range get_range(struct subid_range *ranges, int i) return ranges[i]; } +// helper for stderr to avoid referencing C.stderr from Go code, +// which breaks cgo on musl due to stderr being declared as FILE *const +static FILE *subid_stderr(void) { + return stderr; +} + #if !defined(SUBID_ABI_MAJOR) || (SUBID_ABI_MAJOR < 4) # define subid_init libsubid_init # define subid_get_uid_ranges get_subuid_ranges @@ -44,7 +50,7 @@ func readSubid(username string, isUser bool) (ranges, error) { } onceInit.Do(func() { - C.subid_init(C.CString("storage"), C.stderr) + C.subid_init(C.CString("storage"), C.subid_stderr()) }) cUsername := C.CString(username) From 857c346d1e6c5f37491a8403ce1b4f3307de4722 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miloslav=20Trma=C4=8D?= Date: Tue, 6 Jan 2026 18:21:55 +0100 Subject: [PATCH 24/37] Set the MIME type on created OCI indices/manifests MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Miloslav Trmač (cherry picked from commit cdc7019c4aab5721d9928ba44670a2e3c41d7814) Signed-off-by: Paul Holzinger --- image/oci/layout/oci_dest.go | 5 ++--- image/tarball/tarball_src.go | 5 ++--- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/image/oci/layout/oci_dest.go b/image/oci/layout/oci_dest.go index 48fe812df5..2e885bdc30 100644 --- a/image/oci/layout/oci_dest.go +++ b/image/oci/layout/oci_dest.go @@ -50,9 +50,8 @@ func newImageDestination(sys *types.SystemContext, ref ociReference) (private.Im } } else { index = &imgspecv1.Index{ - Versioned: imgspec.Versioned{ - SchemaVersion: 2, - }, + Versioned: imgspec.Versioned{SchemaVersion: 2}, + MediaType: imgspecv1.MediaTypeImageIndex, Annotations: make(map[string]string), } } diff --git a/image/tarball/tarball_src.go b/image/tarball/tarball_src.go index a8af4b3256..3194f95203 100644 --- a/image/tarball/tarball_src.go +++ b/image/tarball/tarball_src.go @@ -179,9 +179,8 @@ func (r *tarballReference) NewImageSource(ctx context.Context, sys *types.System // Populate a manifest with the configuration blob and the layers. manifest := imgspecv1.Manifest{ - Versioned: imgspecs.Versioned{ - SchemaVersion: 2, - }, + Versioned: imgspecs.Versioned{SchemaVersion: 2}, + MediaType: imgspecv1.MediaTypeImageManifest, Config: imgspecv1.Descriptor{ Digest: configID, Size: int64(len(configBytes)), From d9389803ca695608ecb03b4000d1e2b5ca650d4d Mon Sep 17 00:00:00 2001 From: Giuseppe Scrivano Date: Wed, 7 Jan 2026 16:43:07 +0100 Subject: [PATCH 25/37] seccomp: block AF_VSOCK sockets Block the socket() syscall with AF_VSOCK to prevent container escapes via VM sockets. Signed-off-by: Giuseppe Scrivano (cherry picked from commit eaec878beb81c25755c799ddf849e22591143db5) Signed-off-by: Paul Holzinger --- common/pkg/seccomp/default_linux.go | 33 ++++++++++++++-------- common/pkg/seccomp/seccomp.json | 43 ++++++++++++++++++----------- 2 files changed, 49 insertions(+), 27 deletions(-) diff --git a/common/pkg/seccomp/default_linux.go b/common/pkg/seccomp/default_linux.go index 3013765667..e2f5d60d22 100644 --- a/common/pkg/seccomp/default_linux.go +++ b/common/pkg/seccomp/default_linux.go @@ -816,6 +816,21 @@ func DefaultProfile() *Seccomp { Caps: []string{"CAP_SYS_TTY_CONFIG"}, }, }, + { + Names: []string{ + "socket", + }, + Action: ActErrno, + Errno: "EPERM", + ErrnoRet: &eperm, + Args: []*Arg{ + { + Index: 0, + Value: unix.AF_VSOCK, + Op: OpEqualTo, + }, + }, + }, { Names: []string{ "socket", @@ -845,6 +860,11 @@ func DefaultProfile() *Seccomp { }, Action: ActAllow, Args: []*Arg{ + { + Index: 0, + Value: unix.AF_NETLINK, + Op: OpEqualTo, + }, { Index: 2, Value: unix.NETLINK_AUDIT, @@ -878,20 +898,11 @@ func DefaultProfile() *Seccomp { Action: ActAllow, Args: []*Arg{ { - Index: 2, - Value: unix.NETLINK_AUDIT, + Index: 0, + Value: unix.AF_VSOCK, Op: OpNotEqual, }, }, - Excludes: Filter{ - Caps: []string{"CAP_AUDIT_WRITE"}, - }, - }, - { - Names: []string{ - "socket", - }, - Action: ActAllow, Includes: Filter{ Caps: []string{"CAP_AUDIT_WRITE"}, }, diff --git a/common/pkg/seccomp/seccomp.json b/common/pkg/seccomp/seccomp.json index 92d882b5cc..8dc5d95434 100644 --- a/common/pkg/seccomp/seccomp.json +++ b/common/pkg/seccomp/seccomp.json @@ -957,6 +957,25 @@ "errnoRet": 1, "errno": "EPERM" }, + { + "names": [ + "socket" + ], + "action": "SCMP_ACT_ERRNO", + "args": [ + { + "index": 0, + "value": 40, + "valueTwo": 0, + "op": "SCMP_CMP_EQ" + } + ], + "comment": "", + "includes": {}, + "excludes": {}, + "errnoRet": 1, + "errno": "EPERM" + }, { "names": [ "socket" @@ -992,6 +1011,12 @@ ], "action": "SCMP_ACT_ALLOW", "args": [ + { + "index": 0, + "value": 16, + "valueTwo": 0, + "op": "SCMP_CMP_EQ" + }, { "index": 2, "value": 9, @@ -1035,27 +1060,13 @@ "action": "SCMP_ACT_ALLOW", "args": [ { - "index": 2, - "value": 9, + "index": 0, + "value": 40, "valueTwo": 0, "op": "SCMP_CMP_NE" } ], "comment": "", - "includes": {}, - "excludes": { - "caps": [ - "CAP_AUDIT_WRITE" - ] - } - }, - { - "names": [ - "socket" - ], - "action": "SCMP_ACT_ALLOW", - "args": null, - "comment": "", "includes": { "caps": [ "CAP_AUDIT_WRITE" From 32a83757469e85d51e2753829a209d65143a4d04 Mon Sep 17 00:00:00 2001 From: Pablo Rodriguez Nava Date: Fri, 9 Jan 2026 10:12:07 +0100 Subject: [PATCH 26/37] Add DockerProxy field for dynamic proxy configuration Add a new DockerProxy field to SystemContext that accepts a function for determining proxy URLs dynamically per request. This provides more flexibility than the static DockerProxyURL field, allowing for advanced proxy configurations such as those from httpproxy.Config.ProxyFunc(). Signed-off-by: Pablo Rodriguez Nava (cherry picked from commit 2cf5727a0a0247545ee529539c4015322c5aa4dc) Signed-off-by: Paul Holzinger --- image/docker/docker_client.go | 5 +++++ image/types/types.go | 4 ++++ 2 files changed, 9 insertions(+) diff --git a/image/docker/docker_client.go b/image/docker/docker_client.go index 1c0d67105e..30f338da78 100644 --- a/image/docker/docker_client.go +++ b/image/docker/docker_client.go @@ -916,6 +916,11 @@ func (c *dockerClient) detectPropertiesHelper(ctx context.Context) error { if c.sys != nil && c.sys.DockerProxyURL != nil { tr.Proxy = http.ProxyURL(c.sys.DockerProxyURL) } + if c.sys != nil && c.sys.DockerProxy != nil { + tr.Proxy = func(request *http.Request) (*url.URL, error) { + return c.sys.DockerProxy(request.URL) + } + } c.client = &http.Client{Transport: tr} ping := func(scheme string) error { diff --git a/image/types/types.go b/image/types/types.go index 41f1a632e2..de25dabcdc 100644 --- a/image/types/types.go +++ b/image/types/types.go @@ -668,6 +668,10 @@ type SystemContext struct { DockerRegistryPushPrecomputeDigests bool // DockerProxyURL specifies proxy configuration schema (like socks5://username:password@ip:port) DockerProxyURL *url.URL + // DockerProxy is a function that determines the proxy URL for a given request URL. + // If set, this takes precedence over DockerProxyURL. The function should return the proxy URL to use, + // or nil if no proxy should be used for the given request. + DockerProxy func(reqURL *url.URL) (*url.URL, error) // === docker/daemon.Transport overrides === // A directory containing a CA certificate (ending with ".crt"), From d1241f8bc422070205ce55cbebcbc68945b6b245 Mon Sep 17 00:00:00 2001 From: lyp256 Date: Mon, 19 Jan 2026 18:19:14 +0800 Subject: [PATCH 27/37] fix debug log for #579 Signed-off-by: lyp256 (cherry picked from commit 1b658a4e8f74a317c28b445e92cb5538ab557572) Signed-off-by: Paul Holzinger --- image/copy/single.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/image/copy/single.go b/image/copy/single.go index 5c81fd2d53..36d221133c 100644 --- a/image/copy/single.go +++ b/image/copy/single.go @@ -386,14 +386,14 @@ func (ic *imageCopier) compareImageDestinationManifestEqual(ctx context.Context, destImageSource, err := ic.c.dest.Reference().NewImageSource(ctx, ic.c.options.DestinationCtx) if err != nil { - logrus.Debugf("Unable to create destination image %s source: %v", ic.c.dest.Reference(), err) + logrus.Debugf("Unable to create destination image %s source: %v", transports.ImageName(ic.c.dest.Reference()), err) return nil, nil } defer destImageSource.Close() destManifest, destManifestType, err := destImageSource.GetManifest(ctx, targetInstance) if err != nil { - logrus.Debugf("Unable to get destination image %s/%s manifest: %v", destImageSource, targetInstance, err) + logrus.Debugf("Unable to get destination image %s/%s manifest: %v", transports.ImageName(destImageSource.Reference()), targetInstance, err) return nil, nil } From 90383df2587fae116f31f785115b25957e5c84cb Mon Sep 17 00:00:00 2001 From: Chawye Hsu Date: Wed, 28 Jan 2026 01:37:14 +0800 Subject: [PATCH 28/37] common: safer use of `filepath.EvalSymlinks()` in `findBindir()` This patch is directly ported from [1] to resolve the exact same issue mentioned in that pull-request. Why is this patch needed? Because in [2], `FindExecutablePeer()` was replaced with `FindHelperBinary()`, making the fix introduced in [1] no longer effective, resulting in a regression. To resolve the regression[3], `safeEvalSymlinks()` is added to `findBindir()`. The function call stack is now: ``` FindHelperBinary() -> findBindir() -> safeEvalSymlinks() ``` xref: - https://github.com/ScoopInstaller/Main/pull/6335#issuecomment-3769785863 [1]: https://github.com/containers/podman/pull/25151 [2]: https://github.com/containers/podman/pull/27612 [3]: https://github.com/containers/podman/issues/27763 Signed-off-by: Chawye Hsu --- common/pkg/config/config.go | 6 +- common/pkg/config/config_unix.go | 4 + common/pkg/config/config_windows.go | 28 ++++- common/pkg/config/config_windows_test.go | 149 +++++++++++++++++++++++ 4 files changed, 184 insertions(+), 3 deletions(-) create mode 100644 common/pkg/config/config_windows_test.go diff --git a/common/pkg/config/config.go b/common/pkg/config/config.go index 8bc23deba1..afc45bd7f6 100644 --- a/common/pkg/config/config.go +++ b/common/pkg/config/config.go @@ -1079,8 +1079,10 @@ func findBindir() string { } execPath, err := os.Executable() if err == nil { - // Resolve symbolic links to find the actual binary file path. - execPath, err = filepath.EvalSymlinks(execPath) + // Resolve symlinks for the binary path. + // On Windows, an additional symlink check is performed; + // on other platforms, this is equivalent to filepath.EvalSymlinks. + execPath, err = safeEvalSymlinks(execPath) } if err != nil { // If failed to find executable (unlikely to happen), warn about it. diff --git a/common/pkg/config/config_unix.go b/common/pkg/config/config_unix.go index 97befd1522..a67c418c04 100644 --- a/common/pkg/config/config_unix.go +++ b/common/pkg/config/config_unix.go @@ -33,3 +33,7 @@ func userConfigPath() (string, error) { func overrideContainersConfigPath() (string, error) { return overrideContainersConfig, nil } + +func safeEvalSymlinks(filePath string) (string, error) { + return filepath.EvalSymlinks(filePath) +} diff --git a/common/pkg/config/config_windows.go b/common/pkg/config/config_windows.go index 9caf89dc42..95c8ee235a 100644 --- a/common/pkg/config/config_windows.go +++ b/common/pkg/config/config_windows.go @@ -1,6 +1,10 @@ package config -import "os" +import ( + "io/fs" + "os" + "path/filepath" +) const ( // _configPath is the path to the containers/containers.conf @@ -35,3 +39,25 @@ var defaultHelperBinariesDir = []string{ // directory where the current process binary (i.e. podman) is located. "$BINDIR", } + +func safeEvalSymlinks(filePath string) (string, error) { + fileInfo, err := os.Lstat(filePath) + if err != nil { + return "", err + } + if fileInfo.Mode()&fs.ModeSymlink != 0 { + // Only call filepath.EvalSymlinks if it is a symlink. + // Starting with v1.23, EvalSymlinks returns an error for mount points. + // See https://go-review.googlesource.com/c/go/+/565136 for reference. + filePath, err = filepath.EvalSymlinks(filePath) + if err != nil { + return "", err + } + } else { + // Call filepath.Clean when filePath is not a symlink. That's for + // consistency with the symlink case (filepath.EvalSymlinks calls + // Clean after evaluating filePath). + filePath = filepath.Clean(filePath) + } + return filePath, nil +} diff --git a/common/pkg/config/config_windows_test.go b/common/pkg/config/config_windows_test.go new file mode 100644 index 0000000000..fc28794d8a --- /dev/null +++ b/common/pkg/config/config_windows_test.go @@ -0,0 +1,149 @@ +package config + +import ( + "fmt" + "os" + "os/exec" + "path/filepath" + "testing" + + "golang.org/x/sys/windows" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// shortPathToLongPath converts a Windows short path (C:\PROGRA~1) to its +// long path equivalent (C:\Program Files). It returns an error if shortPath +// doesn't exist. +func shortPathToLongPath(shortPath string) (string, error) { + shortPathPtr, err := windows.UTF16PtrFromString(shortPath) + if err != nil { + return "", err + } + len, err := windows.GetLongPathName(shortPathPtr, nil, 0) + if err != nil { + return "", err + } + if len == 0 { + return "", fmt.Errorf("failed to get buffer size for path: %s", shortPath) + } + longPathPtr := &(make([]uint16, len)[0]) + _, err = windows.GetLongPathName(shortPathPtr, longPathPtr, len) + if err != nil { + return "", err + } + return windows.UTF16PtrToString(longPathPtr), nil +} + +// CreateNewItemWithPowerShell creates a new item using PowerShell. +// It's an helper to easily create junctions on Windows (as well as other file types). +// It constructs a PowerShell command to create a new item at the specified path with the given item type. +// If a target is provided, it includes it in the command. +// +// Parameters: +// - t: The testing.T instance. +// - path: The path where the new item will be created. +// - itemType: The type of the item to be created (e.g., "File", "SymbolicLink", "Junction"). +// - target: The target for the new item, if applicable. +func CreateNewItemWithPowerShell(t *testing.T, path string, itemType string, target string) { + var pwshCmd, pwshPath string + // Look for Powershell 7 first as it allow Symlink creation for non-admins too + pwshPath, err := exec.LookPath("pwsh.exe") + if err != nil { + // Use Powershell 5 that is always present + pwshPath = "powershell.exe" + } + pwshCmd = "New-Item -Path " + path + " -ItemType " + itemType + if target != "" { + pwshCmd += " -Target " + target + } + cmd := exec.Command(pwshPath, "-Command", pwshCmd) + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + err = cmd.Run() + require.NoError(t, err) +} + +// TestSafeEvalSymlinks tests the safeEvalSymlinks function. +// In particular it verifies that safeEvalSymlinks behaves as +// filepath.EvalSymlink before Go 1.23 - with the exception of +// files under a mount point (juntion) that aren't resolved +// anymore. +// The old behavior of filepath.EvalSymlinks can be tested with +// the directive "//go:debug winsymlink=0" and replacing safeEvalSymlinks() +// with filepath.EvalSymlink(). +func TestSafeEvalSymlinks(t *testing.T) { + // Create a temporary directory to store the normal file + normalFileDir, err := shortPathToLongPath(t.TempDir()) + require.NoError(t, err) + + // Create a temporary directory to store the (hard/sym)link files + linkFilesDir, err := shortPathToLongPath(t.TempDir()) + require.NoError(t, err) + + // Create a temporary directory where the mount point will be created + mountPointDir, err := shortPathToLongPath(t.TempDir()) + require.NoError(t, err) + + // Create a normal file + normalFile := filepath.Join(normalFileDir, "testFile") + CreateNewItemWithPowerShell(t, normalFile, "File", "") + + // Create a symlink file + symlinkFile := filepath.Join(linkFilesDir, "testSymbolicLink") + CreateNewItemWithPowerShell(t, symlinkFile, "SymbolicLink", normalFile) + + // Create a hardlink file + hardlinkFile := filepath.Join(linkFilesDir, "testHardLink") + CreateNewItemWithPowerShell(t, hardlinkFile, "HardLink", normalFile) + + // Create a mount point file + mountPoint := filepath.Join(mountPointDir, "testJunction") + mountPointFile := filepath.Join(mountPoint, "testFile") + CreateNewItemWithPowerShell(t, mountPoint, "Junction", normalFileDir) + + // Replaces the backslashes with forward slashes in the normal file path + normalFileWithBadSeparators := filepath.ToSlash(normalFile) + + tests := []struct { + name string + filePath string + want string + }{ + { + name: "Normal file", + filePath: normalFile, + want: normalFile, + }, + { + name: "File under a mount point (juntion)", + filePath: mountPointFile, + want: mountPointFile, + }, + { + name: "Symbolic link", + filePath: symlinkFile, + want: normalFile, + }, + { + name: "Hard link", + filePath: hardlinkFile, + want: hardlinkFile, + }, + { + name: "Bad separators in path", + filePath: normalFileWithBadSeparators, + want: normalFile, + }, + } + + for _, tt := range tests { + assert := assert.New(t) + t.Run(tt.name, func(t *testing.T) { + got, err := safeEvalSymlinks(tt.filePath) + require.NoError(t, err) + assert.Equal(tt.want, got) + }) + } +} From bb290dc125b3e3ea2f18e7cf2f2ec4b8810265b6 Mon Sep 17 00:00:00 2001 From: Tom Sweeney Date: Wed, 4 Feb 2026 14:55:48 -0500 Subject: [PATCH 29/37] [podman-5.8] Bump storage to v1.62.0 Thus begins the vendor dance for Podman v5.8 Bump the version of c/storage in the podman-5.8 branch to v1.62.0 Signed-off-by: Tom Sweeney --- storage/VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/storage/VERSION b/storage/VERSION index 91951fd8ad..76d0536205 100644 --- a/storage/VERSION +++ b/storage/VERSION @@ -1 +1 @@ -1.61.0 +1.62.0 From d5d959a8faa860f260c8b05e84a33ac4e8d9ed31 Mon Sep 17 00:00:00 2001 From: Tom Sweeney Date: Thu, 5 Feb 2026 09:45:42 -0500 Subject: [PATCH 30/37] [podman-5.8] Bump storage to 1.62.0 in image Bump storage to v1.62.0 in preparaton for Podman v5.8 and image v5.39.0 Signed-off-by: Tom Sweeney --- image/go.mod | 2 +- image/go.sum | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/image/go.mod b/image/go.mod index a7fa219c3c..bc0d2361c8 100644 --- a/image/go.mod +++ b/image/go.mod @@ -36,7 +36,7 @@ require ( github.com/ulikunitz/xz v0.5.15 github.com/vbauerster/mpb/v8 v8.10.2 go.etcd.io/bbolt v1.4.3 - go.podman.io/storage v1.61.0 + go.podman.io/storage v1.62.0 golang.org/x/crypto v0.43.0 golang.org/x/oauth2 v0.32.0 golang.org/x/sync v0.17.0 diff --git a/image/go.sum b/image/go.sum index 54fa9408aa..0ea2201b73 100644 --- a/image/go.sum +++ b/image/go.sum @@ -302,6 +302,8 @@ go.opentelemetry.io/proto/otlp v1.5.0 h1:xJvq7gMzB31/d406fB8U5CBdyQGw4P399D1aQWU go.opentelemetry.io/proto/otlp v1.5.0/go.mod h1:keN8WnHxOy8PG0rQZjJJ5A2ebUoafqWp0eVQ4yIXvJ4= go.podman.io/storage v1.61.0 h1:5hD/oyRYt1f1gxgvect+8syZBQhGhV28dCw2+CZpx0Q= go.podman.io/storage v1.61.0/go.mod h1:A3UBK0XypjNZ6pghRhuxg62+2NIm5lcUGv/7XyMhMUI= +go.podman.io/storage v1.62.0 h1:0QjX1XlzVmbiaulb+aR/CG6p9+pzaqwIeZPe3tEjHbY= +go.podman.io/storage v1.62.0/go.mod h1:A3UBK0XypjNZ6pghRhuxg62+2NIm5lcUGv/7XyMhMUI= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= From a1da33bdfddae9f31cf436f30dd4d8712d76d922 Mon Sep 17 00:00:00 2001 From: Tom Sweeney Date: Thu, 5 Feb 2026 09:49:24 -0500 Subject: [PATCH 31/37] [podman-5.8] Bump image to v5.39.0 Bump image to v5.39.0 in preparation for Podman v5.8 Signed-off-by: Tom Sweeney --- image/version/version.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/image/version/version.go b/image/version/version.go index 71a957fc68..c9719d623f 100644 --- a/image/version/version.go +++ b/image/version/version.go @@ -6,7 +6,7 @@ const ( // VersionMajor is for an API incompatible changes VersionMajor = 5 // VersionMinor is for functionality in a backwards-compatible manner - VersionMinor = 38 + VersionMinor = 39 // VersionPatch is for backwards-compatible bug fixes VersionPatch = 0 From c41710e4e2fe11eb1716151f552f29d0f61df565 Mon Sep 17 00:00:00 2001 From: Tom Sweeney Date: Thu, 5 Feb 2026 11:36:41 -0500 Subject: [PATCH 32/37] [podman-5.8] Add missing image go.sum I stupidly neglected to update go.sum in my last PR, that corrects this. Signed-off-by: Tom Sweeney --- image/go.sum | 2 -- 1 file changed, 2 deletions(-) diff --git a/image/go.sum b/image/go.sum index 0ea2201b73..f107d021dd 100644 --- a/image/go.sum +++ b/image/go.sum @@ -300,8 +300,6 @@ go.opentelemetry.io/otel/trace v1.36.0 h1:ahxWNuqZjpdiFAyrIoQ4GIiAIhxAunQR6MUoKr go.opentelemetry.io/otel/trace v1.36.0/go.mod h1:gQ+OnDZzrybY4k4seLzPAWNwVBBVlF2szhehOBB/tGA= go.opentelemetry.io/proto/otlp v1.5.0 h1:xJvq7gMzB31/d406fB8U5CBdyQGw4P399D1aQWU/3i4= go.opentelemetry.io/proto/otlp v1.5.0/go.mod h1:keN8WnHxOy8PG0rQZjJJ5A2ebUoafqWp0eVQ4yIXvJ4= -go.podman.io/storage v1.61.0 h1:5hD/oyRYt1f1gxgvect+8syZBQhGhV28dCw2+CZpx0Q= -go.podman.io/storage v1.61.0/go.mod h1:A3UBK0XypjNZ6pghRhuxg62+2NIm5lcUGv/7XyMhMUI= go.podman.io/storage v1.62.0 h1:0QjX1XlzVmbiaulb+aR/CG6p9+pzaqwIeZPe3tEjHbY= go.podman.io/storage v1.62.0/go.mod h1:A3UBK0XypjNZ6pghRhuxg62+2NIm5lcUGv/7XyMhMUI= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= From 80fb329c24eb41f760488720a493946435196f31 Mon Sep 17 00:00:00 2001 From: Tom Sweeney Date: Thu, 5 Feb 2026 11:38:14 -0500 Subject: [PATCH 33/37] [podman-5.8] Bump to image 5.39.1 Touch up the last bump of image. I stupidly forgot to run `go mod vendor`. The go.sum file was not included in my last PR, this will create a new tag to include it. Signed-off-by: Tom Sweeney --- image/version/version.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/image/version/version.go b/image/version/version.go index c9719d623f..8dc2af7d81 100644 --- a/image/version/version.go +++ b/image/version/version.go @@ -8,7 +8,7 @@ const ( // VersionMinor is for functionality in a backwards-compatible manner VersionMinor = 39 // VersionPatch is for backwards-compatible bug fixes - VersionPatch = 0 + VersionPatch = 1 // VersionDev indicates development branch. Releases will be empty string. VersionDev = "" From a5fe5444d3d5a85826929576af06f18b558569bd Mon Sep 17 00:00:00 2001 From: Paul Holzinger Date: Thu, 5 Feb 2026 18:06:38 +0100 Subject: [PATCH 34/37] github: run validation workflow also on release branches We call our release branches podman-x.y now so make sure we cover them as well. Signed-off-by: Paul Holzinger (cherry picked from commit c028ad0b024ff1c822eb754a6ee34ba6e3a9595e) Signed-off-by: Paul Holzinger --- .github/workflows/validate.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/validate.yml b/.github/workflows/validate.yml index 55d76c557a..231caa1986 100644 --- a/.github/workflows/validate.yml +++ b/.github/workflows/validate.yml @@ -4,9 +4,11 @@ on: push: branches: - main + - podman-* pull_request: branches: - main + - podman-* permissions: read-all From 6d3663dade0bdd9eb7126a8f40678f7e4670ec3d Mon Sep 17 00:00:00 2001 From: Tom Sweeney Date: Thu, 5 Feb 2026 13:08:52 -0500 Subject: [PATCH 35/37] [podman-5.8] Bump image to v5.39.1, storage to v1.62.0 Bump c/storage to v1.62.0, c/image to v5.39.1 in preparation for Podman v5.8. Note, there was no vendor directory in the podman-5.8 branch prior, after consulting with @Luap99, I'm leaving it that way. Signed-off-by: Tom Sweeney --- common/go.mod | 4 ++-- common/go.sum | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/common/go.mod b/common/go.mod index 494703b159..249e5bd5ba 100644 --- a/common/go.mod +++ b/common/go.mod @@ -43,8 +43,8 @@ require ( github.com/stretchr/testify v1.11.1 github.com/vishvananda/netlink v1.3.1 go.etcd.io/bbolt v1.4.3 - go.podman.io/image/v5 v5.38.0 - go.podman.io/storage v1.61.0 + go.podman.io/image/v5 v5.39.1 + go.podman.io/storage v1.62.0 golang.org/x/crypto v0.43.0 golang.org/x/sync v0.17.0 golang.org/x/sys v0.37.0 diff --git a/common/go.sum b/common/go.sum index 8c21a4c118..fd321afdae 100644 --- a/common/go.sum +++ b/common/go.sum @@ -323,10 +323,10 @@ go.opentelemetry.io/otel/trace v1.36.0 h1:ahxWNuqZjpdiFAyrIoQ4GIiAIhxAunQR6MUoKr go.opentelemetry.io/otel/trace v1.36.0/go.mod h1:gQ+OnDZzrybY4k4seLzPAWNwVBBVlF2szhehOBB/tGA= go.opentelemetry.io/proto/otlp v1.5.0 h1:xJvq7gMzB31/d406fB8U5CBdyQGw4P399D1aQWU/3i4= go.opentelemetry.io/proto/otlp v1.5.0/go.mod h1:keN8WnHxOy8PG0rQZjJJ5A2ebUoafqWp0eVQ4yIXvJ4= -go.podman.io/image/v5 v5.38.0 h1:aUKrCANkPvze1bnhLJsaubcfz0d9v/bSDLnwsXJm6G4= -go.podman.io/image/v5 v5.38.0/go.mod h1:hSIoIUzgBnmc4DjoIdzk63aloqVbD7QXDMkSE/cvG90= -go.podman.io/storage v1.61.0 h1:5hD/oyRYt1f1gxgvect+8syZBQhGhV28dCw2+CZpx0Q= -go.podman.io/storage v1.61.0/go.mod h1:A3UBK0XypjNZ6pghRhuxg62+2NIm5lcUGv/7XyMhMUI= +go.podman.io/image/v5 v5.39.1 h1:loIw4qHzZzBlUguYZau40u8HbR5MrTPQhwT4Hy6sCm0= +go.podman.io/image/v5 v5.39.1/go.mod h1:SlaR6Pra1ATIx4BcuZ16oafb3QcCHISaKcJbtlN/G/0= +go.podman.io/storage v1.62.0 h1:0QjX1XlzVmbiaulb+aR/CG6p9+pzaqwIeZPe3tEjHbY= +go.podman.io/storage v1.62.0/go.mod h1:A3UBK0XypjNZ6pghRhuxg62+2NIm5lcUGv/7XyMhMUI= go.uber.org/automaxprocs v1.6.0 h1:O3y2/QNTOdbF+e/dpXNNW7Rx2hZ4sTIPyybbxyNqTUs= go.uber.org/automaxprocs v1.6.0/go.mod h1:ifeIMSnPZuznNm6jmdzmU3/bfk01Fe2fotchwEFJ8r8= go.yaml.in/yaml/v2 v2.4.2 h1:DzmwEr2rDGHl7lsFgAHxmNz/1NlQ7xLIrlN2h5d1eGI= From 078c746c8158d7bdb98377c91ee8ecdf5639fbf2 Mon Sep 17 00:00:00 2001 From: Tom Sweeney Date: Thu, 5 Feb 2026 13:11:12 -0500 Subject: [PATCH 36/37] [podman-5.8] Common to v0.67.0 Bump common to v0.67.0 to be included in Podman v5.8 Signed-off-by: Tom Sweeney --- common/version/version.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common/version/version.go b/common/version/version.go index 69b597ef70..874a374a0f 100644 --- a/common/version/version.go +++ b/common/version/version.go @@ -1,4 +1,4 @@ package version // Version is the version of the build. -const Version = "0.66.1" +const Version = "0.67.0" From d92a871b9e1ef6c59a49d2ce1e84428e805a209c Mon Sep 17 00:00:00 2001 From: Paul Holzinger Date: Fri, 6 Feb 2026 14:02:22 +0100 Subject: [PATCH 37/37] validate: fetch all commits The aprse checkout seems to be causing trouble when we merge back the release branch into main, it seems like it is getting the wrong list of commits in the range as it does not have the full history. Signed-off-by: Paul Holzinger --- .github/workflows/validate.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/validate.yml b/.github/workflows/validate.yml index 9f06800979..fc89341a6b 100644 --- a/.github/workflows/validate.yml +++ b/.github/workflows/validate.yml @@ -81,7 +81,8 @@ jobs: # By default github actions creates a merge commit which fails the validation, # we only must validate the actual commits of the author. ref: ${{ github.event.pull_request.head.sha }} - fetch-depth: ${{ github.event.pull_request.commits }} + # Fetch all commits, a sparse checkout with only the commits count in the PR will not result in the right range. + fetch-depth: 0 - uses: actions/setup-go@v6 with: go-version: 1.25.x