Add ability to add raw tarballs to images

docs/docs.md

@@ -9,7 +9,7 @@ public API
 <pre>
 load("@rules_oci//oci:defs.bzl", "oci_image")
 
-oci_image(<a href="#oci_image-name">name</a>, <a href="#oci_image-annotations">annotations</a>, <a href="#oci_image-arch">arch</a>, <a href="#oci_image-base">base</a>, <a href="#oci_image-entrypoint">entrypoint</a>, <a href="#oci_image-env">env</a>, <a href="#oci_image-labels">labels</a>, <a href="#oci_image-layers">layers</a>, <a href="#oci_image-os">os</a>, <a href="#oci_image-stamp">stamp</a>)
+oci_image(<a href="#oci_image-name">name</a>, <a href="#oci_image-annotations">annotations</a>, <a href="#oci_image-arch">arch</a>, <a href="#oci_image-base">base</a>, <a href="#oci_image-entrypoint">entrypoint</a>, <a href="#oci_image-env">env</a>, <a href="#oci_image-labels">labels</a>, <a href="#oci_image-layers">layers</a>, <a href="#oci_image-os">os</a>, <a href="#oci_image-stamp">stamp</a>, <a href="#oci_image-tars">tars</a>)
 </pre>
 
 Creates a new image manifest and config by appending the `layers` to an existing image
@@ -31,6 +31,7 @@ be used to extract the image manifest.
 | <a id="oci_image-layers"></a>layers |  A list of layers defined by oci_image_layer   | <a href="https://bazel.build/concepts/labels">List of labels</a> | optional |  `[]`  |
 | <a id="oci_image-os"></a>os |  Used to extract a manifest from base if base is an index   | String | optional |  `""`  |
 | <a id="oci_image-stamp"></a>stamp |  Whether to encode build information into the output. Possible values:<br><br>- `stamp = 1`: Always stamp the build information into the output, even in     [--nostamp](https://docs.bazel.build/versions/main/user-manual.html#flag--stamp) builds.     This setting should be avoided, since it is non-deterministic.     It potentially causes remote cache misses for the target and     any downstream actions that depend on the result. - `stamp = 0`: Never stamp, instead replace build information by constant values.     This gives good build result caching. - `stamp = -1`: Embedding of build information is controlled by the     [--[no]stamp](https://docs.bazel.build/versions/main/user-manual.html#flag--stamp) flag.     Stamped targets are not rebuilt unless their dependencies change.   | Integer | optional |  `-1`  |
+| <a id="oci_image-tars"></a>tars |  A list of tars to add as layers   | <a href="https://bazel.build/concepts/labels">List of labels</a> | optional |  `[]`  |
 
 
 <a id="oci_image_config"></a>

go/cmd/ocitool/appendlayer_cmd.go

@@ -2,6 +2,7 @@ package main
 
 import (
 	"bufio"
+	"crypto/sha256"
 	"fmt"
 	"io"
 	"os"
@@ -131,13 +132,64 @@ func AppendLayersCmd(c *cli.Context) error {
 
 	log.WithField("base_desc", baseManifestDesc).Debugf("using as base")
 
+	tarPaths := c.StringSlice("tar")
+	tarDescriptors := []ocispec.Descriptor{}
+	for _, tarPath := range tarPaths {
+		// Determine compression type of tarball
+		compression, err := ociutil.DetectCompression(tarPath)
+		if err != nil {
+			return fmt.Errorf("failed to determine compression type for %s: %w", tarPath, err)
+		}
+
+		// Determine media type of tarball
+		var mediaType string
+		switch compression {
+		case ociutil.CompressionGzip:
+			mediaType = ocispec.MediaTypeImageLayerGzip
+		case ociutil.CompressionZstd:
+			mediaType = ocispec.MediaTypeImageLayerZstd
+		case ociutil.CompressionNone:
+			mediaType = ocispec.MediaTypeImageLayer
+		}
+
+		// Compute the sha256 of the tarball
+		f, err := os.Open(tarPath)
+		if err != nil {
+			return fmt.Errorf("failed to open %s: %w", tarPath, err)
+		}
+		defer f.Close()
+
+		hasher := sha256.New()
+		wc := ociutil.NewWriterCounter(hasher)
+		if _, err := io.Copy(wc, f); err != nil {
+			return fmt.Errorf("failed to read %s: %w", tarPath, err)
+		}
+
+		hash := hasher.Sum(nil)
+
+		s := fmt.Sprintf("sha256:%x", hash)
+		d, err := digest.Parse(s)
+		if err != nil {
+			return fmt.Errorf("failed to parse '%s' into a valid Digest: %w", s, err)
+		}
+
+		// Create descriptor
+		desc := ocispec.Descriptor{
+			Digest:    d,
+			MediaType: mediaType,
+			Size:      int64(wc.Count()),
+		}
+
+		tarDescriptors = append(tarDescriptors, desc)
+	}
+
 	layerAndDescriptorPaths := c.Generic("layer").(*flagutil.KeyValueFlag).List
 
 	layerProvider := &blob.Index{
 		Blobs: make(map[digest.Digest]string),
 	}
 
-	layerDescs := make([]ocispec.Descriptor, 0, len(layerAndDescriptorPaths))
+	layerDescs := []ocispec.Descriptor{}
 	for _, layerAndDescriptorPath := range layerAndDescriptorPaths {
 		layerDesc, err := ociutil.ReadDescriptorFromFile(layerAndDescriptorPath.Value)
 		if err != nil {
@@ -148,6 +200,13 @@ func AppendLayersCmd(c *cli.Context) error {
 		layerDescs = append(layerDescs, layerDesc)
 	}
 
+	// Treat raw tar files as if they were any other layer
+	for i, tarPath := range tarPaths {
+		tarDesc := tarDescriptors[i]
+		layerProvider.Blobs[tarDesc.Digest] = tarPath
+		layerDescs = append(layerDescs, tarDesc)
+	}
+
 	var entrypoint []string
 	if entrypoint_file := c.String("entrypoint"); entrypoint_file != "" {
 		var entrypointStruct struct {
@@ -162,8 +221,6 @@ func AppendLayersCmd(c *cli.Context) error {
 		entrypoint = entrypointStruct.Entrypoint
 	}
 
-	log.Debugf("created descriptors for layers(n=%v): %#v", len(layerAndDescriptorPaths), layerDescs)
-
 	outIngestor := layer.NewAppendIngester(c.String("out-manifest"), c.String("out-config"))
 
 	newManifest, newConfig, err := layer.AppendLayers(

go/cmd/ocitool/main.go

@@ -98,6 +98,9 @@ var app = &cli.App{
 					Name:  "layer",
 					Value: &flagutil.KeyValueFlag{},
 				},
+				&cli.StringSliceFlag{
+					Name: "tar",
+				},
 				&cli.StringFlag{
 					Name: "outd",
 				},

go/pkg/ociutil/BUILD.bazel

@@ -4,6 +4,7 @@ go_library(
     name = "go_default_library",
     srcs = [
         "bazel.go",
+        "compression.go",
         "desc.go",
         "diff.go",
         "fetch.go",

go/pkg/ociutil/compression.go

@@ -0,0 +1,42 @@
+package ociutil
+
+import (
+	"fmt"
+	"os"
+)
+
+type Compression int
+
+const (
+	CompressionNone Compression = iota
+	CompressionGzip
+	CompressionZstd
+)
+
+func DetectCompression(path string) (Compression, error) {
+	f, err := os.Open(path)
+	if err != nil {
+		return CompressionNone, fmt.Errorf("failed to open %s: %w", path, err)
+	}
+	defer f.Close()
+
+	// Read up to 4 bytes for magic numbers
+	var hdr [4]byte
+	n, err := f.Read(hdr[:])
+	if err != nil {
+		return CompressionNone, fmt.Errorf("failed to read %s: %w", path, err)
+	}
+
+	// gzip: 1F 8B
+	if n >= 2 && hdr[0] == 0x1F && hdr[1] == 0x8B {
+		return CompressionGzip, nil
+	}
+
+	// zstd: 28 B5 2F FD
+	if n >= 4 && hdr[0] == 0x28 && hdr[1] == 0xB5 &&
+		hdr[2] == 0x2F && hdr[3] == 0xFD {
+		return CompressionZstd, nil
+	}
+
+	return CompressionNone, nil
+}

oci/image.bzl

@@ -120,6 +120,14 @@ def _oci_image_impl(ctx):
         [f.path for f in layer_descriptor_files],
     )
 
+    tars = []
+    for tar in ctx.attr.tars:
+        tmp = tar.files.to_list()
+        if len(tmp) != 1:
+            fail("tar must contain exactly one file")
+        tar = tmp[0]
+        tars.append(tar)
+
     stamp_args = []
     if maybe_stamp(ctx):
         stamp_args.append("--bazel-version-file={}".format(ctx.version_file.path))
@@ -142,6 +150,7 @@ def _oci_image_impl(ctx):
                         "--layer={}={}".format(layer, descriptor)
                         for layer, descriptor in layer_and_descriptor_paths
                     ] +
+                    ["--tar={}".format(tar.path) for tar in tars] +
                     ["--annotations={}={}".format(k, v) for k, v in annotations.items()] +
                     ["--labels={}={}".format(k, v) for k, v in labels.items()] +
                     ["--env={}".format(env) for env in ctx.attr.env] +
@@ -153,7 +162,8 @@ def _oci_image_impl(ctx):
                      entrypoint_config_file,
                  ] + ctx.files.layers +
                  layer_descriptor_files +
-                 base_layout.files.to_list(),
+                 base_layout.files.to_list() +
+                 tars,
         mnemonic = "OCIImageAppendLayers",
         outputs = [
             manifest_file,
@@ -170,7 +180,7 @@ def _oci_image_impl(ctx):
         OCILayout(
             blob_index = layout_file,
             files = depset(
-                ctx.files.layers + [manifest_file, config_file, layout_file],
+                ctx.files.layers + ctx.files.tars + [manifest_file, config_file, layout_file],
                 transitive = [base_layout.files],
             ),
         ),
@@ -219,6 +229,10 @@ oci_image = rule(
                 OCIDescriptor,
             ],
         ),
+        "tars": attr.label_list(
+            doc = "A list of tars to add as layers",
+            allow_files = [".tar", ".tar.gz", ".tgz", ".tar.zst"],
+        ),
         "annotations": attr.string_dict(
             doc = """[OCI Annotations](https://github.com/opencontainers/image-spec/blob/main/annotations.md)
             to add to the manifest.""",