From d7e4d0d9451c0e65cc4c291f5fe2a21d90c35a64 Mon Sep 17 00:00:00 2001 From: folbrich Date: Sat, 5 Apr 2025 18:55:01 +0200 Subject: [PATCH 1/6] Support OCI registries for chunk storage --- README.md | 9 +++++++++ cmd/desync/store.go | 5 +++++ go.mod | 3 +++ go.sum | 6 ++++++ 4 files changed, 23 insertions(+) diff --git a/README.md b/README.md index bef0d46..8b47d36 100644 --- a/README.md +++ b/README.md @@ -23,6 +23,7 @@ Among the distinguishing factors: - While casync supports very small min chunk sizes, optimizations in desync require min chunk sizes larger than the window size of the rolling hash used (currently 48 bytes). The tool's default chunk sizes match the defaults used in casync, min 16k, avg 64k, max 256k. - Allows FUSE mounting of blob indexes - S3/GC protocol support to access chunk stores for read operations and some some commands that write chunks +- OCI Registries for chunk storage using [ORAS](https://oras.land/docs/) - Stores and retrieves index files from remote index stores such as HTTP, SFTP, Google Storage and S3 - Built-in HTTP(S) index server to read/write indexes - Reflinking matching blocks (rather than copying) from seed files if supported by the filesystem (currently only Btrfs and XFS) @@ -221,6 +222,14 @@ s3+https://s3.internal.company/bucket/prefix?lookup=dns s3+https://example.com/bucket/prefix?lookup=auto ``` +### OCI Registries as chunk stores + +OCI Registries can be used to store chunks. Use the `oci+http` or `oci+https` schema when pointing at OCI stores. + +```text +oci+http://127.0.0.1:5000/myrepo +``` + ### Compressed vs Uncompressed chunk stores By default, desync reads and writes chunks in compressed form to all supported stores. This is in line with upstream casync's goal of storing in the most efficient way. It is however possible to change this behavior by providing desync with a config file (see Configuration section below). Disabling compression and store chunks uncompressed may reduce latency in some use-cases and improve performance. desync supports reading and writing uncompressed chunks to SFTP, S3, HTTP and local stores and caches. If more than one store is used, each of those can be configured independently, for example it's possible to read compressed chunks from S3 while using a local uncompressed cache for best performance. However, care needs to be taken when using the `chunk-server` command and building chains of chunk store proxies to avoid shifting the decompression load onto the server (it's possible this is actually desirable). diff --git a/cmd/desync/store.go b/cmd/desync/store.go index 5cbbd7d..5c028d0 100644 --- a/cmd/desync/store.go +++ b/cmd/desync/store.go @@ -149,6 +149,11 @@ func storeFromLocation(location string, cmdOpt cmdStoreOptions) (desync.Store, e if err != nil { return nil, err } + case "oci+http", "oci+https": + s, err = desync.NewOCIStore(loc, opt) + if err != nil { + return nil, err + } default: local, err := desync.NewLocalStore(location, opt) if err != nil { diff --git a/go.mod b/go.mod index 89c5c24..6f27974 100644 --- a/go.mod +++ b/go.mod @@ -53,6 +53,8 @@ require ( github.com/mitchellh/go-homedir v1.1.0 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect + github.com/opencontainers/go-digest v1.0.0 // indirect + github.com/opencontainers/image-spec v1.1.0 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/rivo/uniseg v0.2.0 // indirect github.com/russross/blackfriday/v2 v2.1.0 // indirect @@ -68,4 +70,5 @@ require ( google.golang.org/protobuf v1.33.0 // indirect gopkg.in/ini.v1 v1.67.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect + oras.land/oras-go/v2 v2.5.0 // indirect ) diff --git a/go.sum b/go.sum index 6f7719b..70b9704 100644 --- a/go.sum +++ b/go.sum @@ -117,6 +117,10 @@ github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJ github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= +github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= +github.com/opencontainers/image-spec v1.1.0 h1:8SG7/vwALn54lVB/0yZ/MMwhFrPYtpEHQb2IpWsCzug= +github.com/opencontainers/image-spec v1.1.0/go.mod h1:W4s4sFTMaBeK1BQLXbG4AdM2szdn85PY75RI83NrTrM= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/sftp v1.13.5 h1:a3RLUqkyjYRtBTZJZ1VRrKbN3zhuPLlUc3sphVz81go= @@ -257,3 +261,5 @@ gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +oras.land/oras-go/v2 v2.5.0 h1:o8Me9kLY74Vp5uw07QXPiitjsw7qNXi8Twd+19Zf02c= +oras.land/oras-go/v2 v2.5.0/go.mod h1:z4eisnLP530vwIOUOJeBIj0aGI0L1C3d53atvCBqZHg= From 6cedf319e084cc0c7309bb574d924dfa7e8209e9 Mon Sep 17 00:00:00 2001 From: folbrich Date: Sat, 5 Apr 2025 18:58:45 +0200 Subject: [PATCH 2/6] go mod tidy --- go.mod | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 6f27974..88b255a 100644 --- a/go.mod +++ b/go.mod @@ -28,6 +28,12 @@ require ( gopkg.in/cheggaaa/pb.v1 v1.0.28 ) +require ( + github.com/opencontainers/go-digest v1.0.0 + github.com/opencontainers/image-spec v1.1.0 + oras.land/oras-go/v2 v2.5.0 +) + require ( cloud.google.com/go v0.110.0 // indirect cloud.google.com/go/compute v1.19.1 // indirect @@ -53,8 +59,6 @@ require ( github.com/mitchellh/go-homedir v1.1.0 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect - github.com/opencontainers/go-digest v1.0.0 // indirect - github.com/opencontainers/image-spec v1.1.0 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/rivo/uniseg v0.2.0 // indirect github.com/russross/blackfriday/v2 v2.1.0 // indirect @@ -70,5 +74,4 @@ require ( google.golang.org/protobuf v1.33.0 // indirect gopkg.in/ini.v1 v1.67.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect - oras.land/oras-go/v2 v2.5.0 // indirect ) From eda713cad5612ab70b1f6895147d2252d85600ea Mon Sep 17 00:00:00 2001 From: folbrich Date: Sat, 5 Apr 2025 19:00:13 +0200 Subject: [PATCH 3/6] missing file --- oci.go | 91 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 91 insertions(+) create mode 100644 oci.go diff --git a/oci.go b/oci.go new file mode 100644 index 0000000..996b13f --- /dev/null +++ b/oci.go @@ -0,0 +1,91 @@ +package desync + +import ( + "bytes" + "context" + "fmt" + "io" + "net/url" + "strings" + + "github.com/opencontainers/go-digest" + ocispec "github.com/opencontainers/image-spec/specs-go/v1" + "oras.land/oras-go/v2/registry/remote" +) + +var _ WriteStore = OCIStore{} + +// OCIStore operates on chunks in an Open Container Image registry. +type OCIStore struct { + repo *remote.Repository + location string + opt StoreOptions + converters Converters +} + +// NewOCIStore initializes a new Open Registry As Storage backend. +func NewOCIStore(u *url.URL, opt StoreOptions) (OCIStore, error) { + repo, err := remote.NewRepository(u.Host + u.Path) + if err != nil { + return OCIStore{}, fmt.Errorf("failed to connect to oci registry: %w", err) + } + s := OCIStore{ + repo: repo, + location: u.String(), + opt: opt, + } + return s, nil +} + +func (s OCIStore) String() string { + return s.location +} + +// Close the store. NOP operation but needed to implement the store interface. +func (s OCIStore) Close() error { return nil } + +// GetChunk reads and returns one chunk from the store +func (s OCIStore) GetChunk(id ChunkID) (*Chunk, error) { + r, err := s.repo.Fetch(context.Background(), ociDescriptorForChunk(id)) + if err != nil { + return nil, err + } + defer r.Close() + b, err := io.ReadAll(r) + if err != nil { + return nil, err + } + return NewChunkFromStorage(id, b, s.converters, s.opt.SkipVerify) +} + +// StoreChunk adds a new chunk to the store. +func (s OCIStore) StoreChunk(chunk *Chunk) error { + b, err := chunk.Data() + if err != nil { + return err + } + b, err = s.converters.toStorage(b) + if err != nil { + return err + } + return s.repo.Push(context.Background(), ociDescriptorForChunk(chunk.ID()), bytes.NewReader(b)) +} + +// HasChunk returns true if the chunk is in the store. +func (s OCIStore) HasChunk(id ChunkID) (bool, error) { + return s.repo.Exists(context.Background(), ociDescriptorForChunk(id)) +} + +// RemoveChunk deletes a chunk, typically an invalid one, from the store. +// Used when verifying and repairing caches. +func (s OCIStore) RemoveChunk(id ChunkID) error { + return s.repo.Delete(context.Background(), ociDescriptorForChunk(id)) +} + +func ociDescriptorForChunk(id ChunkID) ocispec.Descriptor { + return ocispec.Descriptor{ + // TODO: this may only work for SHA256 stores + Digest: digest.Digest(strings.ToLower(Digest.Algorithm().String()) + ":" + id.String()), + MediaType: "application/vnd.oci.image.layer.v1.tar+zstd", + } +} From caeebc2814150ade95753337dcf200a0adfd7ad6 Mon Sep 17 00:00:00 2001 From: folbrich Date: Mon, 7 Apr 2025 11:30:28 +0200 Subject: [PATCH 4/6] simplify oci store schema --- README.md | 4 ++-- cmd/desync/store.go | 2 +- oci.go | 6 ++++-- 3 files changed, 7 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 8b47d36..8b439e0 100644 --- a/README.md +++ b/README.md @@ -224,10 +224,10 @@ s3+https://example.com/bucket/prefix?lookup=auto ### OCI Registries as chunk stores -OCI Registries can be used to store chunks. Use the `oci+http` or `oci+https` schema when pointing at OCI stores. +OCI Registries can be used to store chunks. Use the `oci` schema when pointing at OCI stores. ```text -oci+http://127.0.0.1:5000/myrepo +oci://127.0.0.1:5000/myrepo ``` ### Compressed vs Uncompressed chunk stores diff --git a/cmd/desync/store.go b/cmd/desync/store.go index 5c028d0..3740fb6 100644 --- a/cmd/desync/store.go +++ b/cmd/desync/store.go @@ -149,7 +149,7 @@ func storeFromLocation(location string, cmdOpt cmdStoreOptions) (desync.Store, e if err != nil { return nil, err } - case "oci+http", "oci+https": + case "oci": s, err = desync.NewOCIStore(loc, opt) if err != nil { return nil, err diff --git a/oci.go b/oci.go index 996b13f..7e3c1b2 100644 --- a/oci.go +++ b/oci.go @@ -6,7 +6,6 @@ import ( "fmt" "io" "net/url" - "strings" "github.com/opencontainers/go-digest" ocispec "github.com/opencontainers/image-spec/specs-go/v1" @@ -48,6 +47,7 @@ func (s OCIStore) Close() error { return nil } func (s OCIStore) GetChunk(id ChunkID) (*Chunk, error) { r, err := s.repo.Fetch(context.Background(), ociDescriptorForChunk(id)) if err != nil { + // TODO: figure out what a 404 looks like and return ChunkMissing{id} if so return nil, err } defer r.Close() @@ -80,12 +80,14 @@ func (s OCIStore) HasChunk(id ChunkID) (bool, error) { // Used when verifying and repairing caches. func (s OCIStore) RemoveChunk(id ChunkID) error { return s.repo.Delete(context.Background(), ociDescriptorForChunk(id)) + // TODO: figure out what a 404 looks like and return ChunkMissing{id} if so } func ociDescriptorForChunk(id ChunkID) ocispec.Descriptor { return ocispec.Descriptor{ // TODO: this may only work for SHA256 stores - Digest: digest.Digest(strings.ToLower(Digest.Algorithm().String()) + ":" + id.String()), + Digest: digest.Digest("sha256:" + id.String()), + // Digest: digest.Digest(strings.ToLower(Digest.Algorithm().String()) + ":" + id.String()), MediaType: "application/vnd.oci.image.layer.v1.tar+zstd", } } From 2a95d4b4bae2a593d118a5f666d7fcd38cf68a02 Mon Sep 17 00:00:00 2001 From: folbrich Date: Sat, 12 Apr 2025 13:05:29 +0200 Subject: [PATCH 5/6] implement auth --- README.md | 11 +++++++++-- cmd/desync/config.go | 30 ++++++++++++++++++++++++++++-- cmd/desync/store.go | 5 +++-- oci.go | 42 ++++++++++++++++++++++++++++++++++-------- 4 files changed, 74 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index 8b439e0..6df7050 100644 --- a/README.md +++ b/README.md @@ -224,10 +224,10 @@ s3+https://example.com/bucket/prefix?lookup=auto ### OCI Registries as chunk stores -OCI Registries can be used to store chunks. Use the `oci` schema when pointing at OCI stores. +OCI Registries can be used to store chunks. Use the `oci+https` schema when pointing at OCI stores. If the store does not support TLS, use `oci+http` instead. ```text -oci://127.0.0.1:5000/myrepo +oci+https://ghcr.io/myrepo ``` ### Compressed vs Uncompressed chunk stores @@ -251,6 +251,7 @@ For most use cases, it is sufficient to use the tool's default configuration not Available configuration values: - `s3-credentials` - Defines credentials for use with S3 stores. Especially useful if more than one S3 store is used. The key in the config needs to be the URL scheme and host used for the store, excluding the path, but including the port number if used in the store URL. The key can also contain glob patterns, and the available wildcards are `*`, `?` and `[…]`. Please refer to the [filepath.Match](https://pkg.go.dev/path/filepath#Match) documentation for additional information. It is also possible to use a [standard aws credentials file](https://docs.aws.amazon.com/cli/latest/userguide/cli-config-files.html) in order to store s3 credentials. +- `oci-credentials` - Defines credentials for use with Open Container Registry stores. - `store-options` - Allows customization of chunk and index stores, for example compression settings, timeouts, retry behavior and keys. Not all options are applicable to every store, some of these like `timeout` are ignored for local stores. Some of these options, such as the client certificates are overwritten with any values set in the command line. Note that the store location used in the command line needs to match the key under `store-options` exactly for these options to be used. As for the `s3-credentials`, glob patterns are also supported. A configuration file where more than one key matches a single store location, is considered invalid. - `timeout` - Time limit for chunk read or write operation in nanoseconds. Default: 1 minute. If set to a negative value, timeout is infinite. - `error-retry` - Number of times to retry failed chunk requests. Default: 0. @@ -286,6 +287,12 @@ Available configuration values: "aws-profile": "profile_refreshable" } }, + "oci-credentials": { + "ghcr.io/myuser/repo": { + "username": "myuser", + "secret": "MYSECRET" + }, + }, "store-options": { "https://192.168.1.1/store": { "client-cert": "/path/to/crt", diff --git a/cmd/desync/config.go b/cmd/desync/config.go index 448f137..b6ab69d 100644 --- a/cmd/desync/config.go +++ b/cmd/desync/config.go @@ -17,6 +17,7 @@ import ( "github.com/pkg/errors" "github.com/sirupsen/logrus" "github.com/spf13/cobra" + "oras.land/oras-go/v2/registry/remote/auth" ) // S3Creds holds credentials or references to an S3 credentials file. @@ -32,8 +33,18 @@ type S3Creds struct { // Config is used to hold the global tool configuration. It's used to customize // store features and provide credentials where needed. type Config struct { - S3Credentials map[string]S3Creds `json:"s3-credentials"` - StoreOptions map[string]desync.StoreOptions `json:"store-options"` + S3Credentials map[string]S3Creds `json:"s3-credentials"` + OCICredentials map[string]OCICreds `json:"oci-credentials"` + StoreOptions map[string]desync.StoreOptions `json:"store-options"` +} + +// OCICreds holds OCI credentials for a container registry store. +type OCICreds struct { + // Username for OCI store authentication. + Username string `json:"username,omitempty"` + + // Secret (password or token) for OCI store authentication. + Secret string `json:"secret,omitempty"` } // GetS3CredentialsFor attempts to find creds and region for an S3 location in the @@ -73,6 +84,21 @@ func (c Config) GetS3CredentialsFor(u *url.URL) (*credentials.Credentials, strin return creds, region } +// GetOCICredentialsFor attempts to find creds and region for an OCI location in the config. +func (c Config) GetOCICredentialsFor(u *url.URL) auth.CredentialFunc { + key := u.Host + u.Path + credsConfig, ok := c.OCICredentials[key] + if !ok { + return nil + } + return func(ctx context.Context, hostport string) (auth.Credential, error) { + return auth.Credential{ + Username: credsConfig.Username, + Password: credsConfig.Secret, + }, nil + } +} + // GetStoreOptionsFor returns optional config options for a specific store. Note that // an error will be returned if the location string matches multiple entries in the // config file. diff --git a/cmd/desync/store.go b/cmd/desync/store.go index 3740fb6..a4e07ad 100644 --- a/cmd/desync/store.go +++ b/cmd/desync/store.go @@ -149,8 +149,9 @@ func storeFromLocation(location string, cmdOpt cmdStoreOptions) (desync.Store, e if err != nil { return nil, err } - case "oci": - s, err = desync.NewOCIStore(loc, opt) + case "oci+https", "oci+http": + creds := cfg.GetOCICredentialsFor(loc) + s, err = desync.NewOCIStore(loc, creds, opt) if err != nil { return nil, err } diff --git a/oci.go b/oci.go index 7e3c1b2..2364872 100644 --- a/oci.go +++ b/oci.go @@ -3,13 +3,20 @@ package desync import ( "bytes" "context" + "crypto/tls" + "errors" "fmt" "io" + "net/http" "net/url" + "strings" "github.com/opencontainers/go-digest" ocispec "github.com/opencontainers/image-spec/specs-go/v1" + "oras.land/oras-go/v2/errdef" "oras.land/oras-go/v2/registry/remote" + "oras.land/oras-go/v2/registry/remote/auth" + "oras.land/oras-go/v2/registry/remote/retry" ) var _ WriteStore = OCIStore{} @@ -23,11 +30,24 @@ type OCIStore struct { } // NewOCIStore initializes a new Open Registry As Storage backend. -func NewOCIStore(u *url.URL, opt StoreOptions) (OCIStore, error) { +func NewOCIStore(u *url.URL, creds auth.CredentialFunc, opt StoreOptions) (OCIStore, error) { repo, err := remote.NewRepository(u.Host + u.Path) if err != nil { - return OCIStore{}, fmt.Errorf("failed to connect to oci registry: %w", err) + return OCIStore{}, fmt.Errorf("failed to initialize oci registry store: %w", err) } + baseTransport := http.DefaultTransport.(*http.Transport).Clone() + baseTransport.TLSClientConfig = &tls.Config{ + InsecureSkipVerify: opt.TrustInsecure, + } + client := &auth.Client{ + Client: &http.Client{ + Transport: retry.NewTransport(baseTransport), + }, + Credential: creds, + } + client.SetUserAgent("desync") + repo.Client = client + repo.PlainHTTP = strings.HasSuffix(u.Scheme, "-http") s := OCIStore{ repo: repo, location: u.String(), @@ -47,7 +67,9 @@ func (s OCIStore) Close() error { return nil } func (s OCIStore) GetChunk(id ChunkID) (*Chunk, error) { r, err := s.repo.Fetch(context.Background(), ociDescriptorForChunk(id)) if err != nil { - // TODO: figure out what a 404 looks like and return ChunkMissing{id} if so + if errors.Is(err, errdef.ErrNotFound) { + return nil, ChunkMissing{id} + } return nil, err } defer r.Close() @@ -68,7 +90,9 @@ func (s OCIStore) StoreChunk(chunk *Chunk) error { if err != nil { return err } - return s.repo.Push(context.Background(), ociDescriptorForChunk(chunk.ID()), bytes.NewReader(b)) + descriptor := ociDescriptorForChunk(chunk.ID()) + descriptor.Size = int64(len(b)) + return s.repo.Push(context.Background(), descriptor, bytes.NewReader(b)) } // HasChunk returns true if the chunk is in the store. @@ -79,15 +103,17 @@ func (s OCIStore) HasChunk(id ChunkID) (bool, error) { // RemoveChunk deletes a chunk, typically an invalid one, from the store. // Used when verifying and repairing caches. func (s OCIStore) RemoveChunk(id ChunkID) error { - return s.repo.Delete(context.Background(), ociDescriptorForChunk(id)) - // TODO: figure out what a 404 looks like and return ChunkMissing{id} if so + err := s.repo.Delete(context.Background(), ociDescriptorForChunk(id)) + if errors.Is(err, errdef.ErrNotFound) { + return ChunkMissing{id} + } + return err } func ociDescriptorForChunk(id ChunkID) ocispec.Descriptor { return ocispec.Descriptor{ // TODO: this may only work for SHA256 stores - Digest: digest.Digest("sha256:" + id.String()), - // Digest: digest.Digest(strings.ToLower(Digest.Algorithm().String()) + ":" + id.String()), + Digest: digest.Digest("sha256:" + id.String()), MediaType: "application/vnd.oci.image.layer.v1.tar+zstd", } } From e5f88692538a1919efa610cb7a425e0f6e132a6f Mon Sep 17 00:00:00 2001 From: folbrich Date: Sun, 13 Apr 2025 10:12:20 +0200 Subject: [PATCH 6/6] Fix blob fetch --- oci.go | 23 ++++++++++++++++++++--- 1 file changed, 20 insertions(+), 3 deletions(-) diff --git a/oci.go b/oci.go index 2364872..056e9d6 100644 --- a/oci.go +++ b/oci.go @@ -3,6 +3,7 @@ package desync import ( "bytes" "context" + "crypto" "crypto/tls" "errors" "fmt" @@ -31,6 +32,12 @@ type OCIStore struct { // NewOCIStore initializes a new Open Registry As Storage backend. func NewOCIStore(u *url.URL, creds auth.CredentialFunc, opt StoreOptions) (OCIStore, error) { + // The OCI spec does not support desync's default hash algorithm (SHA512/256), so we must + // be using SHA256 only. + if Digest.Algorithm() != crypto.SHA256 { + return OCIStore{}, errors.New("OCI stores only support SHA256, use --digest=sha256") + } + repo, err := remote.NewRepository(u.Host + u.Path) if err != nil { return OCIStore{}, fmt.Errorf("failed to initialize oci registry store: %w", err) @@ -65,7 +72,14 @@ func (s OCIStore) Close() error { return nil } // GetChunk reads and returns one chunk from the store func (s OCIStore) GetChunk(id ChunkID) (*Chunk, error) { - r, err := s.repo.Fetch(context.Background(), ociDescriptorForChunk(id)) + descriptor, err := s.repo.Blobs().Resolve(context.Background(), ociReference(id)) + if err != nil { + if errors.Is(err, errdef.ErrNotFound) { + return nil, ChunkMissing{id} + } + return nil, err + } + r, err := s.repo.Fetch(context.Background(), descriptor) if err != nil { if errors.Is(err, errdef.ErrNotFound) { return nil, ChunkMissing{id} @@ -112,8 +126,11 @@ func (s OCIStore) RemoveChunk(id ChunkID) error { func ociDescriptorForChunk(id ChunkID) ocispec.Descriptor { return ocispec.Descriptor{ - // TODO: this may only work for SHA256 stores - Digest: digest.Digest("sha256:" + id.String()), + Digest: digest.Digest(ociReference(id)), MediaType: "application/vnd.oci.image.layer.v1.tar+zstd", } } + +func ociReference(id ChunkID) string { + return "sha256:" + id.String() +}