From 3cefe4eefb0cdd15ab3b0361f0784c20e099dd89 Mon Sep 17 00:00:00 2001 From: Brian Myers Date: Fri, 12 Sep 2025 15:55:15 -0400 Subject: [PATCH] Add ability to add raw tarballs to images --- docs/docs.md | 3 +- go/cmd/ocitool/appendlayer_cmd.go | 63 +++++++++++++++++++++++++++++-- go/cmd/ocitool/main.go | 3 ++ go/pkg/ociutil/BUILD.bazel | 1 + go/pkg/ociutil/compression.go | 42 +++++++++++++++++++++ oci/image.bzl | 18 ++++++++- 6 files changed, 124 insertions(+), 6 deletions(-) create mode 100644 go/pkg/ociutil/compression.go diff --git a/docs/docs.md b/docs/docs.md index 42285ab..2aaedd4 100644 --- a/docs/docs.md +++ b/docs/docs.md @@ -9,7 +9,7 @@ public API 
 load("@rules_oci//oci:defs.bzl", "oci_image")
 
-oci_image(name, annotations, arch, base, entrypoint, env, labels, layers, os, stamp)
+oci_image(name, annotations, arch, base, entrypoint, env, labels, layers, os, stamp, tars)
Creates a new image manifest and config by appending the `layers` to an existing image @@ -31,6 +31,7 @@ be used to extract the image manifest. | layers | A list of layers defined by oci_image_layer | List of labels | optional | `[]` | | os | Used to extract a manifest from base if base is an index | String | optional | `""` | | stamp | Whether to encode build information into the output. Possible values:

- `stamp = 1`: Always stamp the build information into the output, even in [--nostamp](https://docs.bazel.build/versions/main/user-manual.html#flag--stamp) builds. This setting should be avoided, since it is non-deterministic. It potentially causes remote cache misses for the target and any downstream actions that depend on the result. - `stamp = 0`: Never stamp, instead replace build information by constant values. This gives good build result caching. - `stamp = -1`: Embedding of build information is controlled by the [--[no]stamp](https://docs.bazel.build/versions/main/user-manual.html#flag--stamp) flag. Stamped targets are not rebuilt unless their dependencies change. | Integer | optional | `-1` | +| tars | A list of tars to add as layers | List of labels | optional | `[]` | diff --git a/go/cmd/ocitool/appendlayer_cmd.go b/go/cmd/ocitool/appendlayer_cmd.go index f9adc7d..cb1f4d3 100644 --- a/go/cmd/ocitool/appendlayer_cmd.go +++ b/go/cmd/ocitool/appendlayer_cmd.go @@ -2,6 +2,7 @@ package main import ( "bufio" + "crypto/sha256" "fmt" "io" "os" @@ -131,13 +132,64 @@ func AppendLayersCmd(c *cli.Context) error { log.WithField("base_desc", baseManifestDesc).Debugf("using as base") + tarPaths := c.StringSlice("tar") + tarDescriptors := []ocispec.Descriptor{} + for _, tarPath := range tarPaths { + // Determine compression type of tarball + compression, err := ociutil.DetectCompression(tarPath) + if err != nil { + return fmt.Errorf("failed to determine compression type for %s: %w", tarPath, err) + } + + // Determine media type of tarball + var mediaType string + switch compression { + case ociutil.CompressionGzip: + mediaType = ocispec.MediaTypeImageLayerGzip + case ociutil.CompressionZstd: + mediaType = ocispec.MediaTypeImageLayerZstd + case ociutil.CompressionNone: + mediaType = ocispec.MediaTypeImageLayer + } + + // Compute the sha256 of the tarball + f, err := os.Open(tarPath) + if err != nil { + return fmt.Errorf("failed to open %s: %w", tarPath, err) + } + defer f.Close() + + hasher := sha256.New() + wc := ociutil.NewWriterCounter(hasher) + if _, err := io.Copy(wc, f); err != nil { + return fmt.Errorf("failed to read %s: %w", tarPath, err) + } + + hash := hasher.Sum(nil) + + s := fmt.Sprintf("sha256:%x", hash) + d, err := digest.Parse(s) + if err != nil { + return fmt.Errorf("failed to parse '%s' into a valid Digest: %w", s, err) + } + + // Create descriptor + desc := ocispec.Descriptor{ + Digest: d, + MediaType: mediaType, + Size: int64(wc.Count()), + } + + tarDescriptors = append(tarDescriptors, desc) + } + layerAndDescriptorPaths := c.Generic("layer").(*flagutil.KeyValueFlag).List layerProvider := &blob.Index{ Blobs: make(map[digest.Digest]string), } - layerDescs := make([]ocispec.Descriptor, 0, len(layerAndDescriptorPaths)) + layerDescs := []ocispec.Descriptor{} for _, layerAndDescriptorPath := range layerAndDescriptorPaths { layerDesc, err := ociutil.ReadDescriptorFromFile(layerAndDescriptorPath.Value) if err != nil { @@ -148,6 +200,13 @@ func AppendLayersCmd(c *cli.Context) error { layerDescs = append(layerDescs, layerDesc) } + // Treat raw tar files as if they were any other layer + for i, tarPath := range tarPaths { + tarDesc := tarDescriptors[i] + layerProvider.Blobs[tarDesc.Digest] = tarPath + layerDescs = append(layerDescs, tarDesc) + } + var entrypoint []string if entrypoint_file := c.String("entrypoint"); entrypoint_file != "" { var entrypointStruct struct { @@ -162,8 +221,6 @@ func AppendLayersCmd(c *cli.Context) error { entrypoint = entrypointStruct.Entrypoint } - log.Debugf("created descriptors for layers(n=%v): %#v", len(layerAndDescriptorPaths), layerDescs) - outIngestor := layer.NewAppendIngester(c.String("out-manifest"), c.String("out-config")) newManifest, newConfig, err := layer.AppendLayers( diff --git a/go/cmd/ocitool/main.go b/go/cmd/ocitool/main.go index 57af879..1aa90ae 100644 --- a/go/cmd/ocitool/main.go +++ b/go/cmd/ocitool/main.go @@ -98,6 +98,9 @@ var app = &cli.App{ Name: "layer", Value: &flagutil.KeyValueFlag{}, }, + &cli.StringSliceFlag{ + Name: "tar", + }, &cli.StringFlag{ Name: "outd", }, diff --git a/go/pkg/ociutil/BUILD.bazel b/go/pkg/ociutil/BUILD.bazel index 0629b60..0b580de 100644 --- a/go/pkg/ociutil/BUILD.bazel +++ b/go/pkg/ociutil/BUILD.bazel @@ -4,6 +4,7 @@ go_library( name = "go_default_library", srcs = [ "bazel.go", + "compression.go", "desc.go", "diff.go", "fetch.go", diff --git a/go/pkg/ociutil/compression.go b/go/pkg/ociutil/compression.go new file mode 100644 index 0000000..8722b72 --- /dev/null +++ b/go/pkg/ociutil/compression.go @@ -0,0 +1,42 @@ +package ociutil + +import ( + "fmt" + "os" +) + +type Compression int + +const ( + CompressionNone Compression = iota + CompressionGzip + CompressionZstd +) + +func DetectCompression(path string) (Compression, error) { + f, err := os.Open(path) + if err != nil { + return CompressionNone, fmt.Errorf("failed to open %s: %w", path, err) + } + defer f.Close() + + // Read up to 4 bytes for magic numbers + var hdr [4]byte + n, err := f.Read(hdr[:]) + if err != nil { + return CompressionNone, fmt.Errorf("failed to read %s: %w", path, err) + } + + // gzip: 1F 8B + if n >= 2 && hdr[0] == 0x1F && hdr[1] == 0x8B { + return CompressionGzip, nil + } + + // zstd: 28 B5 2F FD + if n >= 4 && hdr[0] == 0x28 && hdr[1] == 0xB5 && + hdr[2] == 0x2F && hdr[3] == 0xFD { + return CompressionZstd, nil + } + + return CompressionNone, nil +} diff --git a/oci/image.bzl b/oci/image.bzl index 2fb1496..faa4f0c 100755 --- a/oci/image.bzl +++ b/oci/image.bzl @@ -120,6 +120,14 @@ def _oci_image_impl(ctx): [f.path for f in layer_descriptor_files], ) + tars = [] + for tar in ctx.attr.tars: + tmp = tar.files.to_list() + if len(tmp) != 1: + fail("tar must contain exactly one file") + tar = tmp[0] + tars.append(tar) + stamp_args = [] if maybe_stamp(ctx): stamp_args.append("--bazel-version-file={}".format(ctx.version_file.path)) @@ -142,6 +150,7 @@ def _oci_image_impl(ctx): "--layer={}={}".format(layer, descriptor) for layer, descriptor in layer_and_descriptor_paths ] + + ["--tar={}".format(tar.path) for tar in tars] + ["--annotations={}={}".format(k, v) for k, v in annotations.items()] + ["--labels={}={}".format(k, v) for k, v in labels.items()] + ["--env={}".format(env) for env in ctx.attr.env] + @@ -153,7 +162,8 @@ def _oci_image_impl(ctx): entrypoint_config_file, ] + ctx.files.layers + layer_descriptor_files + - base_layout.files.to_list(), + base_layout.files.to_list() + + tars, mnemonic = "OCIImageAppendLayers", outputs = [ manifest_file, @@ -170,7 +180,7 @@ def _oci_image_impl(ctx): OCILayout( blob_index = layout_file, files = depset( - ctx.files.layers + [manifest_file, config_file, layout_file], + ctx.files.layers + ctx.files.tars + [manifest_file, config_file, layout_file], transitive = [base_layout.files], ), ), @@ -219,6 +229,10 @@ oci_image = rule( OCIDescriptor, ], ), + "tars": attr.label_list( + doc = "A list of tars to add as layers", + allow_files = [".tar", ".tar.gz", ".tgz", ".tar.zst"], + ), "annotations": attr.string_dict( doc = """[OCI Annotations](https://github.com/opencontainers/image-spec/blob/main/annotations.md) to add to the manifest.""",