From 5715256e2cf2caa5cccbfe2306a88b6c5243dfe6 Mon Sep 17 00:00:00 2001 From: Ian Pascoe Date: Mon, 2 Mar 2026 09:21:43 -0500 Subject: [PATCH 1/2] feat(registry): adopt packages/releases snapshot layout --- crates/crosspack-cli/src/tests.rs | 57 ++- crates/crosspack-registry/AGENTS.md | 6 +- crates/crosspack-registry/src/fs_ops.rs | 21 +- .../crosspack-registry/src/registry_index.rs | 163 +++++-- crates/crosspack-registry/src/source_sync.rs | 75 +++- crates/crosspack-registry/src/tests.rs | 424 ++++++++++++++++-- docs/manifest-spec.md | 247 ++++------ docs/registry-spec.md | 59 ++- docs/release-checklist.md | 2 +- docs/source-management-spec.md | 31 +- docs/trust/core-registry-fingerprint.txt | 8 +- registry | 2 +- scripts/sync-crosspack-registry-release.sh | 79 +++- 13 files changed, 808 insertions(+), 366 deletions(-) diff --git a/crates/crosspack-cli/src/tests.rs b/crates/crosspack-cli/src/tests.rs index a893179..c1f6247 100644 --- a/crates/crosspack-cli/src/tests.rs +++ b/crates/crosspack-cli/src/tests.rs @@ -6298,8 +6298,10 @@ description = " \n\t" fn search_uses_configured_sources_without_registry_root() { let layout = test_layout(); let state_root = registry_state_root(&layout); - std::fs::create_dir_all(state_root.join("cache/official/index/ripgrep")) + std::fs::create_dir_all(state_root.join("cache/official/releases/ripgrep")) .expect("must create source cache structure"); + std::fs::create_dir_all(state_root.join("cache/official/packages")) + .expect("must create package template cache structure"); std::fs::write( state_root.join("sources.toml"), concat!( @@ -6439,7 +6441,7 @@ sha256 = "abc" registry_state_root(&layout) .join("cache") .join("official") - .join("index") + .join("releases") .join("ripgrep"), ) .expect("must create package directory"); @@ -8286,7 +8288,8 @@ sha256 = "abc" let mut path = std::env::temp_dir(); let nanos = current_unix_nanos(); path.push(format!("crosspack-cli-test-registry-{name}-{nanos}")); - std::fs::create_dir_all(path.join("index")).expect("must create index dir"); + std::fs::create_dir_all(path.join("packages")).expect("must create packages dir"); + std::fs::create_dir_all(path.join("releases")).expect("must create releases dir"); if with_registry_pub { std::fs::write(path.join("registry.pub"), "test-key\n") .expect("must write registry key"); @@ -8351,8 +8354,13 @@ sha256 = "abc" let cache_root = registry_state_root(layout) .join("cache") .join(spec.source_name); - let package_dir = cache_root.join("index").join(spec.package_name); + let package_template_path = cache_root + .join("packages") + .join(format!("{}.toml", spec.package_name)); + let package_dir = cache_root.join("releases").join(spec.package_name); std::fs::create_dir_all(&package_dir).expect("must create package directory"); + std::fs::create_dir_all(cache_root.join("packages")) + .expect("must create package template directory"); let signing_key = test_signing_key(); std::fs::write( @@ -8360,6 +8368,15 @@ sha256 = "abc" public_key_hex(&signing_key), ) .expect("must write registry key"); + let package_template = format!("name = \"{}\"\n", spec.package_name); + std::fs::write(&package_template_path, package_template.as_bytes()) + .expect("must write package template"); + let package_signature = signing_key.sign(package_template.as_bytes()); + std::fs::write( + package_template_path.with_extension("toml.sig"), + hex::encode(package_signature.to_bytes()), + ) + .expect("must write package template signature"); let manifest = manifest_toml( spec.package_name, @@ -8388,8 +8405,13 @@ sha256 = "abc" artifact_target: &str, ) { let cache_root = registry_state_root(layout).join("cache").join(source_name); - let package_dir = cache_root.join("index").join(package_name); + let package_template_path = cache_root + .join("packages") + .join(format!("{package_name}.toml")); + let package_dir = cache_root.join("releases").join(package_name); std::fs::create_dir_all(&package_dir).expect("must create package directory"); + std::fs::create_dir_all(cache_root.join("packages")) + .expect("must create package template directory"); let signing_key = test_signing_key(); std::fs::write( @@ -8397,6 +8419,15 @@ sha256 = "abc" public_key_hex(&signing_key), ) .expect("must write registry key"); + let package_template = format!("name = \"{package_name}\"\n"); + std::fs::write(&package_template_path, package_template.as_bytes()) + .expect("must write package template"); + let package_signature = signing_key.sign(package_template.as_bytes()); + std::fs::write( + package_template_path.with_extension("toml.sig"), + hex::encode(package_signature.to_bytes()), + ) + .expect("must write package template signature"); let manifest = format!( r#" @@ -8493,8 +8524,13 @@ install_commands = ["sh", "-c", "{install_script}"] ); let cache_root = registry_state_root(layout).join("cache").join(source_name); - let package_dir = cache_root.join("index").join(package_name); + let package_template_path = cache_root + .join("packages") + .join(format!("{package_name}.toml")); + let package_dir = cache_root.join("releases").join(package_name); std::fs::create_dir_all(&package_dir).expect("must create package directory"); + std::fs::create_dir_all(cache_root.join("packages")) + .expect("must create package template directory"); let signing_key = test_signing_key(); std::fs::write( @@ -8502,6 +8538,15 @@ install_commands = ["sh", "-c", "{install_script}"] public_key_hex(&signing_key), ) .expect("must write registry key"); + let package_template = format!("name = \"{package_name}\"\n"); + std::fs::write(&package_template_path, package_template.as_bytes()) + .expect("must write package template"); + let package_signature = signing_key.sign(package_template.as_bytes()); + std::fs::write( + package_template_path.with_extension("toml.sig"), + hex::encode(package_signature.to_bytes()), + ) + .expect("must write package template signature"); let manifest_path = package_dir.join(format!("{version}.toml")); std::fs::write(&manifest_path, manifest.as_bytes()).expect("must write manifest"); diff --git a/crates/crosspack-registry/AGENTS.md b/crates/crosspack-registry/AGENTS.md index 2114844..86bef07 100644 --- a/crates/crosspack-registry/AGENTS.md +++ b/crates/crosspack-registry/AGENTS.md @@ -8,15 +8,15 @@ Registry source state, snapshot lifecycle, and manifest signature enforcement li - `RegistrySourceRecord`: source contract (`name`, `kind`, `location`, `fingerprint_sha256`, `enabled`, `priority`). - `update_sources` + `finalize_staged_source_update`: stage, validate, atomically swap cache, then write `snapshot.json`. - `RegistrySourceSnapshotState`: per-source cache health (`None`, `Ready`, `Error` with reason code). -- `RegistryIndex`: reads `index//*.toml`, verifies `*.toml.sig`, parses manifests. +- `RegistryIndex`: reads `packages/.toml` and `releases//*.toml`, verifies `*.toml.sig`, merges/parses manifests. - `ConfiguredRegistryIndex`: loads enabled sources with ready snapshots only; resolves in source priority order. ## TRUST MODEL - Trust root is per-source `registry.pub`; expected key fingerprint is pinned in `sources.toml`. - Updates fail closed if computed key fingerprint mismatches `fingerprint_sha256`. -- Metadata validity requires both signed manifest bytes and parseable TOML payload. +- Metadata validity requires signed package/release bytes and parseable merged TOML payload. - Signature verification uses `verify_ed25519_signature_hex` from `crosspack-security`. -- Layout is mandatory: staged source must contain `registry.pub` and `index/` before acceptance. +- Layout is mandatory: staged source must contain `registry.pub`, `packages/`, and `releases/` before acceptance. - Snapshot readiness gate is explicit: only `snapshot.json` with `status == "ready"` is eligible for configured reads. ## CHANGE IMPACT diff --git a/crates/crosspack-registry/src/fs_ops.rs b/crates/crosspack-registry/src/fs_ops.rs index 6f46373..02ec883 100644 --- a/crates/crosspack-registry/src/fs_ops.rs +++ b/crates/crosspack-registry/src/fs_ops.rs @@ -91,10 +91,19 @@ pub(crate) fn validate_staged_registry_layout(staged_root: &Path, source_name: & ); } - let index_root = staged_root.join("index"); - if !index_root.is_dir() { + let packages_root = staged_root.join("packages"); + if !packages_root.is_dir() { anyhow::bail!( - "source-snapshot-missing: source '{}' missing index/ in {}", + "source-snapshot-missing: source '{}' missing packages/ in {}", + source_name, + staged_root.display() + ); + } + + let releases_root = staged_root.join("releases"); + if !releases_root.is_dir() { + anyhow::bail!( + "source-snapshot-missing: source '{}' missing releases/ in {}", source_name, staged_root.display() ); @@ -103,14 +112,14 @@ pub(crate) fn validate_staged_registry_layout(staged_root: &Path, source_name: & Ok(()) } -pub(crate) fn count_manifest_files(index_root: &Path) -> Result { +pub(crate) fn count_manifest_files(releases_root: &Path) -> Result { let mut count = 0_u64; let mut queue: VecDeque = VecDeque::new(); - queue.push_back(index_root.to_path_buf()); + queue.push_back(releases_root.to_path_buf()); while let Some(dir) = queue.pop_front() { for entry in fs::read_dir(&dir) - .with_context(|| format!("failed reading index directory {}", dir.display()))? + .with_context(|| format!("failed reading release directory {}", dir.display()))? { let entry = entry?; let path = entry.path(); diff --git a/crates/crosspack-registry/src/registry_index.rs b/crates/crosspack-registry/src/registry_index.rs index b38d216..42c07ac 100644 --- a/crates/crosspack-registry/src/registry_index.rs +++ b/crates/crosspack-registry/src/registry_index.rs @@ -5,6 +5,8 @@ use std::path::{Path, PathBuf}; use anyhow::{Context, Result}; use crosspack_core::PackageManifest; use crosspack_security::verify_ed25519_signature_hex; +use toml::value::Table; +use toml::Value; use crate::{ parse_source_state_file, sort_sources, source_has_ready_snapshot, @@ -37,13 +39,13 @@ impl RegistryIndex { } pub fn search_names(&self, needle: &str) -> Result> { - let index_root = self.root.join("index"); - if !index_root.exists() { + let releases_root = self.root.join("releases"); + if !releases_root.exists() { return Ok(Vec::new()); } let mut names = Vec::new(); - for entry in fs::read_dir(index_root).context("failed to read registry index")? { + for entry in fs::read_dir(releases_root).context("failed to read registry releases")? { let entry = entry?; if entry.file_type()?.is_dir() { let name = entry.file_name().to_string_lossy().to_string(); @@ -61,8 +63,11 @@ impl RegistryIndex { } pub fn package_versions(&self, package: &str) -> Result> { - let package_dir = self.root.join("index").join(package); - if !package_dir.exists() { + let release_dir = self.root.join("releases").join(package); + let package_template_path = self.root.join("packages").join(format!("{package}.toml")); + let has_release_dir = release_dir.exists(); + let has_package_template = package_template_path.exists(); + if !has_release_dir && !has_package_template { return Ok(Vec::new()); } @@ -76,9 +81,35 @@ impl RegistryIndex { let trusted_public_key_hex = trusted_public_key_hex.trim(); let key_identifier: String = trusted_public_key_hex.chars().take(16).collect(); + let package_template_bytes = fs::read(&package_template_path).with_context(|| { + format!( + "failed reading package template: {}", + package_template_path.display() + ) + })?; + verify_signed_toml_document( + &package_template_path, + &package_template_bytes, + trusted_public_key_hex, + &key_identifier, + )?; + let package_template = parse_toml_table( + &package_template_bytes, + &package_template_path, + "package template", + )?; + + if !has_release_dir { + anyhow::bail!( + "orphaned package template without releases directory: package template {}, expected release directory {}", + package_template_path.display(), + release_dir.display() + ); + } + let mut manifests = Vec::new(); - for entry in fs::read_dir(&package_dir) - .with_context(|| format!("failed to read package directory: {package}"))? + for entry in fs::read_dir(&release_dir) + .with_context(|| format!("failed to read release directory: {package}"))? { let entry = entry?; if !entry.file_type()?.is_file() { @@ -90,49 +121,105 @@ impl RegistryIndex { continue; } - let manifest_bytes = fs::read(&path) - .with_context(|| format!("failed reading manifest: {}", path.display()))?; - - let signature_path = path.with_extension("toml.sig"); - let signature_hex = fs::read_to_string(&signature_path).with_context(|| { + let release_bytes = fs::read(&path) + .with_context(|| format!("failed reading release file: {}", path.display()))?; + verify_signed_toml_document( + &path, + &release_bytes, + trusted_public_key_hex, + &key_identifier, + )?; + let release_document = parse_toml_table(&release_bytes, &path, "release metadata")?; + let merged_document = merge_manifest_documents(&package_template, &release_document); + let merged_manifest = + toml::to_string(&Value::Table(merged_document)).with_context(|| { + format!( + "failed serializing merged manifest: package template {}, release {}", + package_template_path.display(), + path.display() + ) + })?; + let manifest = PackageManifest::from_toml_str(&merged_manifest).with_context(|| { format!( - "failed reading manifest signature for key {}: {}", - key_identifier, - signature_path.display() + "failed parsing merged manifest: package template {}, release {}", + package_template_path.display(), + path.display() ) })?; - let signature_hex = signature_hex.trim(); + manifests.push(manifest); + } - let signature_is_valid = verify_ed25519_signature_hex( - &manifest_bytes, - trusted_public_key_hex, - signature_hex, - ) + manifests.sort_by(|a, b| b.version.cmp(&a.version)); + Ok(manifests) + } +} + +fn verify_signed_toml_document( + document_path: &Path, + document_bytes: &[u8], + trusted_public_key_hex: &str, + key_identifier: &str, +) -> Result<()> { + let signature_path = document_path.with_extension("toml.sig"); + let signature_hex = fs::read_to_string(&signature_path).with_context(|| { + format!( + "failed reading metadata signature for trusted key {}: {}", + key_identifier, + signature_path.display() + ) + })?; + let signature_hex = signature_hex.trim(); + + let signature_is_valid = + verify_ed25519_signature_hex(document_bytes, trusted_public_key_hex, signature_hex) .with_context(|| { format!( - "failed verifying manifest signature for key {}: {}", + "failed verifying metadata signature for trusted key {}: {}", key_identifier, signature_path.display() ) })?; - if !signature_is_valid { - anyhow::bail!( - "invalid manifest signature for key {}: manifest {}, signature {}", - key_identifier, - path.display(), - signature_path.display() - ); - } + if !signature_is_valid { + anyhow::bail!( + "invalid metadata signature for trusted key {}: document {}, signature {}", + key_identifier, + document_path.display(), + signature_path.display() + ); + } - let content = String::from_utf8(manifest_bytes) - .with_context(|| format!("manifest is not valid UTF-8: {}", path.display()))?; - let manifest = PackageManifest::from_toml_str(&content) - .with_context(|| format!("failed parsing manifest: {}", path.display()))?; - manifests.push(manifest); - } + Ok(()) +} - manifests.sort_by(|a, b| b.version.cmp(&a.version)); - Ok(manifests) +fn parse_toml_table(document_bytes: &[u8], document_path: &Path, kind: &str) -> Result { + let content = String::from_utf8(document_bytes.to_vec()) + .with_context(|| format!("{kind} is not valid UTF-8: {}", document_path.display()))?; + let value: Value = toml::from_str(&content) + .with_context(|| format!("failed parsing {kind}: {}", document_path.display()))?; + let Value::Table(table) = value else { + anyhow::bail!( + "failed parsing {kind}: expected TOML table at root in {}", + document_path.display() + ); + }; + Ok(table) +} + +fn merge_manifest_documents(package_template: &Table, release_document: &Table) -> Table { + let mut merged = package_template.clone(); + merge_tables(&mut merged, release_document); + merged +} + +fn merge_tables(base: &mut Table, overlay: &Table) { + for (key, overlay_value) in overlay { + if let Some(Value::Table(base_table)) = base.get_mut(key) { + if let Value::Table(overlay_table) = overlay_value { + merge_tables(base_table, overlay_table); + continue; + } + } + base.insert(key.clone(), overlay_value.clone()); } } diff --git a/crates/crosspack-registry/src/source_sync.rs b/crates/crosspack-registry/src/source_sync.rs index 85a5612..8c96849 100644 --- a/crates/crosspack-registry/src/source_sync.rs +++ b/crates/crosspack-registry/src/source_sync.rs @@ -1,3 +1,4 @@ +use std::collections::BTreeSet; use std::fs; use std::path::{Path, PathBuf}; @@ -151,7 +152,7 @@ fn finalize_staged_source_update( verify_metadata_signature_policy(&staged_root, &source.name)?; verify_community_recipe_catalog_policy(&staged_root, source)?; - let manifest_count = count_manifest_files(&staged_root.join("index"))?; + let manifest_count = count_manifest_files(&staged_root.join("releases"))?; let existing_snapshot_id = read_snapshot_id( &store .state_root @@ -339,13 +340,13 @@ pub(crate) fn verify_community_recipe_catalog_policy( ); } - let package_index_dir = staged_root.join("index").join(&entry.package); - if !package_index_dir.is_dir() { + let package_release_dir = staged_root.join("releases").join(&entry.package); + if !package_release_dir.is_dir() { anyhow::bail!( - "source-metadata-invalid: source '{}' community recipe '{}' missing index directory {}", + "source-metadata-invalid: source '{}' community recipe '{}' missing release directory {}", source.name, entry.package, - package_index_dir.display() + package_release_dir.display() ); } @@ -367,12 +368,29 @@ pub(crate) fn verify_metadata_signature_policy( staged_root: &Path, source_name: &str, ) -> Result<()> { - let index_root = staged_root.join("index"); - for entry in fs::read_dir(&index_root).with_context(|| { + let index = RegistryIndex::open(staged_root); + let package_names = collect_metadata_packages(staged_root, source_name)?; + + for package in package_names { + index.package_versions(&package).with_context(|| { + format!( + "source-metadata-invalid: source '{}' package '{}' failed metadata signature validation", + source_name, package + ) + })?; + } + + Ok(()) +} + +fn collect_metadata_packages(staged_root: &Path, source_name: &str) -> Result> { + let mut packages = BTreeSet::new(); + let releases_root = staged_root.join("releases"); + for entry in fs::read_dir(&releases_root).with_context(|| { format!( - "source-metadata-invalid: source '{}' failed reading index {}", + "source-metadata-invalid: source '{}' failed reading releases {}", source_name, - index_root.display() + releases_root.display() ) })? { let entry = entry?; @@ -380,17 +398,42 @@ pub(crate) fn verify_metadata_signature_policy( continue; } let package = entry.file_name().to_string_lossy().to_string(); - RegistryIndex::open(staged_root) - .package_versions(&package) - .with_context(|| { - format!( - "source-metadata-invalid: source '{}' package '{}' failed signature validation", - source_name, package + packages.insert(package); + } + + let packages_root = staged_root.join("packages"); + for entry in fs::read_dir(&packages_root).with_context(|| { + format!( + "source-metadata-invalid: source '{}' failed reading packages {}", + source_name, + packages_root.display() + ) + })? { + let entry = entry?; + if !entry.file_type()?.is_file() { + continue; + } + + let path = entry.path(); + if path.extension().and_then(|value| value.to_str()) != Some("toml") { + continue; + } + + let package = path + .file_stem() + .and_then(|value| value.to_str()) + .ok_or_else(|| { + anyhow::anyhow!( + "source-metadata-invalid: source '{}' has non-utf8 package template filename {}", + source_name, + path.display() ) })?; + + packages.insert(package.to_string()); } - Ok(()) + Ok(packages) } pub(crate) fn combine_replace_restore_errors( diff --git a/crates/crosspack-registry/src/tests.rs b/crates/crosspack-registry/src/tests.rs index 0fc6c3a..be5f7a6 100644 --- a/crates/crosspack-registry/src/tests.rs +++ b/crates/crosspack-registry/src/tests.rs @@ -641,7 +641,8 @@ fn update_filesystem_source_writes_ready_snapshot() { let cache_root = root.join("cache").join("local"); assert!(cache_root.join("registry.pub").exists()); - assert!(cache_root.join("index").exists()); + assert!(cache_root.join("packages").exists()); + assert!(cache_root.join("releases").exists()); let snapshot_path = cache_root.join("snapshot.json"); let snapshot_content = fs::read_to_string(&snapshot_path).expect("must write snapshot.json"); @@ -715,6 +716,183 @@ fn update_filesystem_source_accepts_uppercase_configured_fingerprint() { let _ = fs::remove_dir_all(&root); } +#[test] +fn update_filesystem_source_fails_when_packages_root_is_missing() { + let root = test_registry_root(); + let source_root = test_registry_root(); + fs::create_dir_all(source_root.join("releases")).expect("must create releases root"); + + let signing_key = signing_key(); + fs::write( + source_root.join("registry.pub"), + public_key_hex(&signing_key), + ) + .expect("must write registry public key"); + + let store = RegistrySourceStore::new(&root); + let registry_pub = fs::read(source_root.join("registry.pub")).expect("must read registry pub"); + store + .add_source(filesystem_source_record( + "local", + source_root + .to_str() + .expect("filesystem source path must be valid UTF-8"), + sha256_hex_bytes(®istry_pub), + 0, + )) + .expect("must add source"); + + let results = store + .update_sources(&[]) + .expect("update API must report per-source failure"); + assert_eq!(results.len(), 1); + assert_eq!(results[0].status, SourceUpdateStatus::Failed); + assert!( + results[0] + .error + .as_deref() + .expect("must include error message") + .contains("missing packages/"), + "expected missing packages root error, got: {:?}", + results[0].error + ); + + let _ = fs::remove_dir_all(&source_root); + let _ = fs::remove_dir_all(&root); +} + +#[test] +fn update_filesystem_source_fails_when_releases_root_is_missing() { + let root = test_registry_root(); + let source_root = test_registry_root(); + fs::create_dir_all(source_root.join("packages")).expect("must create packages root"); + + let signing_key = signing_key(); + fs::write( + source_root.join("registry.pub"), + public_key_hex(&signing_key), + ) + .expect("must write registry public key"); + + let store = RegistrySourceStore::new(&root); + let registry_pub = fs::read(source_root.join("registry.pub")).expect("must read registry pub"); + store + .add_source(filesystem_source_record( + "local", + source_root + .to_str() + .expect("filesystem source path must be valid UTF-8"), + sha256_hex_bytes(®istry_pub), + 0, + )) + .expect("must add source"); + + let results = store + .update_sources(&[]) + .expect("update API must report per-source failure"); + assert_eq!(results.len(), 1); + assert_eq!(results[0].status, SourceUpdateStatus::Failed); + assert!( + results[0] + .error + .as_deref() + .expect("must include error message") + .contains("missing releases/"), + "expected missing releases root error, got: {:?}", + results[0].error + ); + + let _ = fs::remove_dir_all(&source_root); + let _ = fs::remove_dir_all(&root); +} + +#[test] +fn update_filesystem_source_fails_when_orphaned_package_template_exists() { + let root = test_registry_root(); + let source_root = filesystem_source_fixture(); + let store = RegistrySourceStore::new(&root); + + write_signed_package_template( + &source_root, + &signing_key(), + "zsh", + &package_template_toml("zsh"), + ); + + let registry_pub = fs::read(source_root.join("registry.pub")).expect("must read registry pub"); + store + .add_source(filesystem_source_record( + "local", + source_root + .to_str() + .expect("filesystem source path must be valid UTF-8"), + sha256_hex_bytes(®istry_pub), + 0, + )) + .expect("must add source"); + + let results = store + .update_sources(&[]) + .expect("update API must report per-source failure"); + assert_eq!(results.len(), 1); + assert_eq!(results[0].status, SourceUpdateStatus::Failed); + assert!( + results[0] + .error + .as_deref() + .expect("must include error message") + .contains("orphaned package template"), + "expected orphaned package template failure, got: {:?}", + results[0].error + ); + + let _ = fs::remove_dir_all(&source_root); + let _ = fs::remove_dir_all(&root); +} + +#[test] +fn update_filesystem_source_fails_when_orphaned_package_template_signature_is_missing() { + let root = test_registry_root(); + let source_root = filesystem_source_fixture(); + let store = RegistrySourceStore::new(&root); + + fs::create_dir_all(source_root.join("packages")).expect("must create packages root"); + fs::write( + source_root.join("packages").join("zsh.toml"), + package_template_toml("zsh"), + ) + .expect("must write orphaned package template without signature"); + + let registry_pub = fs::read(source_root.join("registry.pub")).expect("must read registry pub"); + store + .add_source(filesystem_source_record( + "local", + source_root + .to_str() + .expect("filesystem source path must be valid UTF-8"), + sha256_hex_bytes(®istry_pub), + 0, + )) + .expect("must add source"); + + let results = store + .update_sources(&[]) + .expect("update API must report per-source failure"); + assert_eq!(results.len(), 1); + assert_eq!(results[0].status, SourceUpdateStatus::Failed); + let rendered = results[0] + .error + .as_deref() + .expect("must include error message"); + assert!( + rendered.contains("packages") && rendered.contains(".sig"), + "expected package signature verification failure, got: {rendered}" + ); + + let _ = fs::remove_dir_all(&source_root); + let _ = fs::remove_dir_all(&root); +} + #[test] fn update_filesystem_source_fails_when_community_recipe_catalog_signature_is_missing() { let root = test_registry_root(); @@ -788,7 +966,7 @@ fn update_filesystem_source_preserves_existing_cache_on_failure() { fs::remove_file( source_root - .join("index") + .join("releases") .join("ripgrep") .join("14.1.0.toml.sig"), ) @@ -802,7 +980,7 @@ fn update_filesystem_source_preserves_existing_cache_on_failure() { let cached_signature = root .join("cache") .join("local") - .join("index") + .join("releases") .join("ripgrep") .join("14.1.0.toml.sig"); assert!( @@ -840,7 +1018,7 @@ fn update_filesystem_source_reports_updated_when_manifest_changes_with_same_key( rewrite_signed_manifest_with_extra_field( &source_root - .join("index") + .join("releases") .join("ripgrep") .join("14.1.0.toml"), &signing_key(), @@ -881,7 +1059,8 @@ fn update_git_source_clones_and_records_snapshot_id() { let cache_root = root.join("cache").join("origin"); assert!(cache_root.join("registry.pub").exists()); - assert!(cache_root.join("index").exists()); + assert!(cache_root.join("packages").exists()); + assert!(cache_root.join("releases").exists()); assert_eq!(git_head_short(&cache_root), expected_snapshot_id); let _ = fs::remove_dir_all(&source_root); @@ -912,7 +1091,7 @@ fn update_git_source_fetches_new_commit() { rewrite_signed_manifest_with_extra_field( &source_root - .join("index") + .join("releases") .join("ripgrep") .join("14.1.0.toml"), &signing_key(), @@ -930,7 +1109,7 @@ fn update_git_source_fetches_new_commit() { let cached_manifest = fs::read_to_string( root.join("cache") .join("origin") - .join("index") + .join("releases") .join("ripgrep") .join("14.1.0.toml"), ) @@ -1038,13 +1217,10 @@ fn update_unknown_source_returns_source_not_found() { #[test] fn search_names_fails_when_registry_public_key_is_missing() { let root = test_registry_root(); - let package_dir = root.join("index").join("ripgrep"); + let package_dir = root.join("releases").join("ripgrep"); fs::create_dir_all(&package_dir).expect("must create package dir"); - fs::write( - package_dir.join("14.1.0.toml"), - manifest_toml("ripgrep", "14.1.0"), - ) - .expect("must write manifest"); + fs::write(package_dir.join("14.1.0.toml"), release_toml("14.1.0")) + .expect("must write manifest"); let index = RegistryIndex::open(&root); let err = index @@ -1237,8 +1413,8 @@ fn community_recipe_catalog_rejects_unsorted_and_duplicate_entries() { let staged_root = filesystem_source_fixture(); let signing_key = signing_key(); let catalog_path = staged_root.join("community").join("recipes.toml"); - fs::create_dir_all(staged_root.join("index").join("zsh")) - .expect("must create zsh package index directory"); + fs::create_dir_all(staged_root.join("releases").join("zsh")) + .expect("must create zsh package release directory"); write_signed_community_recipe_catalog_raw( &catalog_path, &signing_key, @@ -1334,7 +1510,7 @@ fn community_recipe_catalog_rejects_empty_package_entry() { } #[test] -fn community_recipe_catalog_rejects_missing_index_directory_for_listed_package() { +fn community_recipe_catalog_rejects_missing_release_directory_for_listed_package() { let staged_root = filesystem_source_fixture(); let signing_key = signing_key(); let catalog_path = staged_root.join("community").join("recipes.toml"); @@ -1359,14 +1535,14 @@ fn community_recipe_catalog_rejects_missing_index_directory_for_listed_package() }); let err = verify_community_recipe_catalog_policy(&staged_root, &source) - .expect_err("must reject catalog entries missing index directories"); + .expect_err("must reject catalog entries missing release directories"); let rendered = format!("{err:#}"); assert!( - rendered.contains("missing index directory"), - "expected missing index directory error, got: {rendered}" + rendered.contains("missing release directory"), + "expected missing release directory error, got: {rendered}" ); assert!( - rendered.contains(&format!("index{}zsh", std::path::MAIN_SEPARATOR)), + rendered.contains(&format!("releases{}zsh", std::path::MAIN_SEPARATOR)), "expected deterministic missing directory path details, got: {rendered}" ); @@ -1379,8 +1555,8 @@ fn community_recipe_catalog_rejects_invalid_package_tokens() { let signing_key = signing_key(); let catalog_path = staged_root.join("community").join("recipes.toml"); - fs::create_dir_all(staged_root.join("index").join("ripgrep").join("plugins")) - .expect("must create nested index path to ensure grammar check is enforced"); + fs::create_dir_all(staged_root.join("releases").join("ripgrep").join("plugins")) + .expect("must create nested release path to ensure grammar check is enforced"); let mut source = filesystem_source_record( "local", @@ -1439,7 +1615,7 @@ fn configured_index_open_fails_when_sources_file_is_unreadable() { #[test] fn search_names_returns_matching_package_with_valid_signed_manifests() { let root = test_registry_root(); - let package_dir = root.join("index").join("ripgrep"); + let package_dir = root.join("releases").join("ripgrep"); fs::create_dir_all(&package_dir).expect("must create package dir"); let signing_key = signing_key(); @@ -1459,10 +1635,22 @@ fn search_names_returns_matching_package_with_valid_signed_manifests() { #[test] fn package_versions_fails_when_registry_public_key_is_missing() { let root = test_registry_root(); - let package_dir = root.join("index").join("ripgrep"); + let package_dir = root.join("releases").join("ripgrep"); fs::create_dir_all(&package_dir).expect("must create package dir"); - let manifest_path = package_dir.join("14.1.0.toml"); - fs::write(&manifest_path, manifest_toml("ripgrep", "14.1.0")).expect("must write manifest"); + + let signing_key = signing_key(); + write_signed_package_template( + &root, + &signing_key, + "ripgrep", + &package_template_toml("ripgrep"), + ); + write_signed_release_manifest( + &package_dir, + &signing_key, + "14.1.0", + &release_toml("14.1.0"), + ); let index = RegistryIndex::open(&root); let err = index @@ -1474,39 +1662,120 @@ fn package_versions_fails_when_registry_public_key_is_missing() { } #[test] -fn package_versions_fails_when_manifest_signature_is_missing() { +fn package_versions_fails_when_package_signature_is_missing() { let root = test_registry_root(); - let package_dir = root.join("index").join("ripgrep"); + let package_dir = root.join("releases").join("ripgrep"); fs::create_dir_all(&package_dir).expect("must create package dir"); - let manifest_path = package_dir.join("14.1.0.toml"); - fs::write(&manifest_path, manifest_toml("ripgrep", "14.1.0")).expect("must write manifest"); let signing_key = signing_key(); fs::write(root.join("registry.pub"), public_key_hex(&signing_key)) .expect("must write registry public key"); + let package_manifest_path = root.join("packages").join("ripgrep.toml"); + fs::create_dir_all(root.join("packages")).expect("must create packages dir"); + fs::write( + &package_manifest_path, + package_template_toml("ripgrep").as_bytes(), + ) + .expect("must write unsigned package template"); + write_signed_release_manifest( + &package_dir, + &signing_key, + "14.1.0", + &release_toml("14.1.0"), + ); let index = RegistryIndex::open(&root); let err = index .package_versions("ripgrep") - .expect_err("must fail when signature sidecar is missing"); + .expect_err("must fail when package signature sidecar is missing"); + assert!(err.to_string().contains("packages")); assert!(err.to_string().contains(".sig")); let _ = fs::remove_dir_all(&root); } #[test] -fn package_versions_fails_when_manifest_signature_is_invalid() { +fn package_versions_fails_when_package_signature_is_invalid() { let root = test_registry_root(); - let package_dir = root.join("index").join("ripgrep"); + let package_dir = root.join("releases").join("ripgrep"); fs::create_dir_all(&package_dir).expect("must create package dir"); - let manifest = manifest_toml("ripgrep", "14.1.0"); + let signing_key = signing_key(); + fs::write(root.join("registry.pub"), public_key_hex(&signing_key)) + .expect("must write registry public key"); + let package_manifest_path = root.join("packages").join("ripgrep.toml"); + fs::create_dir_all(root.join("packages")).expect("must create packages dir"); + fs::write( + &package_manifest_path, + package_template_toml("ripgrep").as_bytes(), + ) + .expect("must write package template"); + fs::write(package_manifest_path.with_extension("toml.sig"), "00") + .expect("must write invalid package signature"); + write_signed_release_manifest( + &package_dir, + &signing_key, + "14.1.0", + &release_toml("14.1.0"), + ); + + let index = RegistryIndex::open(&root); + let err = index + .package_versions("ripgrep") + .expect_err("must fail when package signature is invalid"); + assert!(err.to_string().contains("packages")); + assert!(err.to_string().contains("signature")); + + let _ = fs::remove_dir_all(&root); +} + +#[test] +fn package_versions_fails_when_release_signature_is_missing() { + let root = test_registry_root(); + let package_dir = root.join("releases").join("ripgrep"); + fs::create_dir_all(&package_dir).expect("must create package dir"); + + let release_manifest_path = package_dir.join("14.1.0.toml"); + fs::write(&release_manifest_path, release_toml("14.1.0")).expect("must write manifest"); + + let signing_key = signing_key(); + fs::write(root.join("registry.pub"), public_key_hex(&signing_key)) + .expect("must write registry public key"); + write_signed_package_template( + &root, + &signing_key, + "ripgrep", + &package_template_toml("ripgrep"), + ); + + let index = RegistryIndex::open(&root); + let err = index + .package_versions("ripgrep") + .expect_err("must fail when release signature sidecar is missing"); + assert!(err.to_string().contains(".sig")); + + let _ = fs::remove_dir_all(&root); +} + +#[test] +fn package_versions_fails_when_release_signature_is_invalid() { + let root = test_registry_root(); + let package_dir = root.join("releases").join("ripgrep"); + fs::create_dir_all(&package_dir).expect("must create package dir"); + + let manifest = release_toml("14.1.0"); let manifest_path = package_dir.join("14.1.0.toml"); fs::write(&manifest_path, manifest.as_bytes()).expect("must write manifest"); let signing_key = signing_key(); fs::write(root.join("registry.pub"), public_key_hex(&signing_key)) .expect("must write registry public key"); + write_signed_package_template( + &root, + &signing_key, + "ripgrep", + &package_template_toml("ripgrep"), + ); fs::write(manifest_path.with_extension("toml.sig"), "00") .expect("must write invalid signature"); @@ -1523,12 +1792,18 @@ fn package_versions_fails_when_manifest_signature_is_invalid() { #[test] fn package_versions_succeeds_with_valid_signatures_and_descending_sort() { let root = test_registry_root(); - let package_dir = root.join("index").join("ripgrep"); + let package_dir = root.join("releases").join("ripgrep"); fs::create_dir_all(&package_dir).expect("must create package dir"); let signing_key = signing_key(); fs::write(root.join("registry.pub"), public_key_hex(&signing_key)) .expect("must write registry public key"); + write_signed_package_template( + &root, + &signing_key, + "ripgrep", + &package_template_toml_with_license("ripgrep", "MIT"), + ); write_signed_manifest(&package_dir, &signing_key, "14.0.0"); write_signed_manifest(&package_dir, &signing_key, "14.1.0"); @@ -1543,23 +1818,82 @@ fn package_versions_succeeds_with_valid_signatures_and_descending_sort() { .map(|manifest| manifest.version.to_string()) .collect(); assert_eq!(versions, vec!["14.1.0", "14.0.0"]); + assert_eq!(manifests[0].name, "ripgrep"); + assert_eq!(manifests[0].license.as_deref(), Some("MIT")); let _ = fs::remove_dir_all(&root); } fn write_signed_manifest(package_dir: &std::path::Path, signing_key: &SigningKey, version: &str) { + let package_name = package_dir + .file_name() + .and_then(|value| value.to_str()) + .expect("release package directory must end with package name"); + let registry_root = package_dir + .parent() + .and_then(|parent| parent.parent()) + .expect("release package directory must be nested under /releases/"); + + let package_template_path = registry_root + .join("packages") + .join(format!("{package_name}.toml")); + if !package_template_path.exists() { + write_signed_package_template( + registry_root, + signing_key, + package_name, + &package_template_toml(package_name), + ); + } + + write_signed_release_manifest(package_dir, signing_key, version, &release_toml(version)); +} + +fn write_signed_package_template( + registry_root: &Path, + signing_key: &SigningKey, + package_name: &str, + template: &str, +) { + let packages_root = registry_root.join("packages"); + fs::create_dir_all(&packages_root).expect("must create packages dir"); + let package_path = packages_root.join(format!("{package_name}.toml")); + write_signed_toml_file(&package_path, signing_key, template); +} + +fn write_signed_release_manifest( + package_dir: &Path, + signing_key: &SigningKey, + version: &str, + release_manifest: &str, +) { + fs::create_dir_all(package_dir).expect("must create release package dir"); let manifest_path = package_dir.join(format!("{version}.toml")); - let manifest = manifest_toml("ripgrep", version); - fs::write(&manifest_path, manifest.as_bytes()).expect("must write manifest"); + write_signed_toml_file(&manifest_path, signing_key, release_manifest); +} - let signature = signing_key.sign(manifest.as_bytes()); +fn write_signed_toml_file(path: &Path, signing_key: &SigningKey, content: &str) { + fs::write(path, content.as_bytes()).expect("must write manifest"); + let signature = signing_key.sign(content.as_bytes()); fs::write( - manifest_path.with_extension("toml.sig"), + path.with_extension("toml.sig"), hex::encode(signature.to_bytes()), ) .expect("must write signature sidecar"); } +fn package_template_toml(name: &str) -> String { + format!("name = \"{name}\"\n") +} + +fn package_template_toml_with_license(name: &str, license: &str) -> String { + format!("name = \"{name}\"\nlicense = \"{license}\"\n") +} + +fn release_toml(version: &str) -> String { + format!("version = \"{version}\"\n") +} + fn write_signed_community_recipe_catalog( catalog_path: &Path, signing_key: &SigningKey, @@ -1619,14 +1953,6 @@ fn rewrite_signed_manifest_with_extra_field( .expect("must rewrite signature sidecar"); } -fn manifest_toml(name: &str, version: &str) -> String { - format!( - r#"name = "{name}" -version = "{version}" -"# - ) -} - fn signing_key() -> SigningKey { SigningKey::from_bytes(&[7u8; 32]) } @@ -1684,7 +2010,7 @@ fn filesystem_source_record( fn filesystem_source_fixture() -> PathBuf { let root = test_registry_root(); - let package_dir = root.join("index").join("ripgrep"); + let package_dir = root.join("releases").join("ripgrep"); fs::create_dir_all(&package_dir).expect("must create package dir"); let signing_key = signing_key(); @@ -1778,7 +2104,7 @@ fn write_ready_snapshot_cache( versions: &[&str], ) { let cache_root = state_root.join("cache").join(source_name); - let package_dir = cache_root.join("index").join("ripgrep"); + let package_dir = cache_root.join("releases").join("ripgrep"); fs::create_dir_all(&package_dir).expect("must create package directory in cache"); fs::write(cache_root.join("registry.pub"), public_key_hex(signing_key)) .expect("must write registry key for cache source"); diff --git a/docs/manifest-spec.md b/docs/manifest-spec.md index 4702857..ed1efd0 100644 --- a/docs/manifest-spec.md +++ b/docs/manifest-spec.md @@ -1,206 +1,111 @@ -# Manifest Specification (Draft v0.2) +# Manifest Specification (Draft v0.4) -Each package version is represented by a TOML manifest stored in the registry index. +Crosspack registry metadata is split across two TOML document types: -## Required Fields +- package template document: `packages/.toml` +- release document: `releases//.toml` -- `name`: package identifier. -- `version`: semantic version. -- `artifacts`: list of downloadable package artifacts. +At read time, Crosspack merges package + release data into runtime `PackageManifest` values. -## Optional Fields +## Package Template Document (`packages/.toml`) -- `description` -- `license` -- `homepage` -- `dependencies`: map of package name to semver constraint. -- `source_build`: optional source-build metadata block used when `--build-from-source` is requested. -- `services`: optional list of service declarations consumed by `crosspack services` commands. +Package template docs store shared metadata and artifact templates. -`crosspack info ` prints `Description: ` when `description` is present and non-empty. -For deterministic plain output, tab/newline/carriage-return characters in `description` are normalized to spaces. +### Required Fields -### Source Build Metadata (`source_build`) +- `name`: package identifier +- `license`: package license string +- `homepage`: HTTPS homepage URL +- `source`: upstream release source metadata +- `artifacts`: non-empty array of artifact templates -`source_build` is parsed, validated, and used by source-build install flows. - -- `url`: source archive or source tree URL. -- `archive_sha256`: expected SHA-256 digest of downloaded source archive bytes (required). -- `build_system`: build-system token (`cargo`, `cmake`, etc.). -- `build_commands`: deterministic command-token array used for build steps (must be non-empty). -- `install_commands`: deterministic command-token array used for install steps (must be non-empty). - -Source-build constraints: - -- source builds run only when `--build-from-source` is set, -- source URL must infer to a supported archive type (`zip`, `tar.gz`, `tar.zst`), -- source archive checksum must be a 64-character hexadecimal SHA-256 digest, -- command tokens must be non-empty, -- metadata or command validation failures fail closed. - -### Service Declarations (`services`) - -`services` is parsed and validated strictly. - -- `name`: service token exposed to `crosspack services `. -- `native_id` (optional): host-native service identifier; defaults to `name` when omitted. - -Service declaration constraints: - -- service names must follow package-token grammar (`[a-z0-9][a-z0-9._+-]{0,63}`), -- `native_id`, when present, must follow package-token grammar with optional `@` segments, -- service names must be unique per manifest, -- unknown fields fail closed. - -## Artifact Fields - -- `target`: Rust-style target triple (for example `x86_64-pc-windows-msvc`). -- `url`: artifact download location. -- `sha256`: expected SHA-256 digest of artifact bytes. -- `size` (optional): expected size in bytes. -- `signature` (optional in v0.1): artifact-level detached signature reference. -- `archive` (optional): artifact kind override (`zip`, `tar.gz`, `tar.zst`, `bin`, `msi`, `dmg`, `appimage`, `exe`, `pkg`, `msix`, `appx`). If omitted, inferred from URL suffix (or inferred as `bin` for extensionless final URL path segments). -- `strip_components` (optional): number of leading path components to strip during extraction. -- `artifact_root` (optional): expected top-level extracted path (validation hint). -- `binaries` (optional): list of exposed commands for this artifact. -- `completions` (optional): list of shell completion files exposed for this artifact. -- `gui_apps` (optional): list of GUI application integrations exposed for this artifact. - -### Artifact Kind Policy - -- Artifact ingestion is deterministic and fail-closed. -- Pre-1.0 scope reset: `deb` and `rpm` are removed from the supported artifact-kind contract. -- Install mode defaults by artifact kind: - - managed: `zip`, `tar.gz`, `tar.zst`, `bin`, `dmg`, `appimage`. - - native: `pkg`, `exe`, `msi`, `msix`, `appx`. -- Host constraints are fail-closed: - - Windows-only native kinds: `exe`, `msi`, `msix`, `appx`. - - macOS-only native kind: `pkg`. - - macOS-only managed kind: `dmg`. - - Linux-only managed kind: `appimage`. -- Installer/package formats are deterministic and extraction-oriented; Crosspack does not run vendor installer UI/execution fallback flows. -- `appimage` artifacts are staged as direct payload files and require `strip_components = 0` with no `artifact_root` override. -- `bin` artifacts are staged as direct payload files using the downloaded file name and require `strip_components = 0` with no `artifact_root` override. -- `pkg` maintainer scripts are not executed; script-dependent installs fail closed. - -### Native GUI Registration Policy - -- GUI metadata may be projected into native platform registration locations. -- Native registration is best-effort and warning-driven (install success does not require adapter success). -- Known current limits: - - Linux refresh depends on `update-desktop-database` availability. - - Windows protocol/file-association registration is scoped to HKCU only. - - macOS `.app` registration prefers bundle-copy deployment into `/Applications/.app` and falls back to `~/Applications/.app` when system destination prepare/write steps fail. - - macOS registration refuses to overwrite unmanaged existing app bundles at either destination and emits warnings instead. - - macOS LaunchServices refresh remains best-effort and warning-driven. - -## Registry Metadata Signing +### `source` Fields -- Registry metadata signing is strict and enabled by default. -- The trusted registry key file is `registry.pub` at the registry root. -- Every manifest file `.toml` must have a detached sidecar signature file `.toml.sig`. -- Sidecar signatures are stored as hex-encoded detached signature bytes. -- Commands that rely on registry metadata fail closed on missing/invalid key or signature material. -- `artifact.signature` is separate from registry metadata sidecar signatures and applies only to downloaded artifacts. -- As of current GA behavior, registry metadata sidecar signatures are required and enforced; `artifact.signature` remains optional metadata and is not a prerequisite for install success. +- `provider`: currently `github` +- `repo`: `owner/name` +- `tag_prefix` (optional) +- `include_prereleases` (optional boolean) -## Planned Policy Extensions +### Artifact Template Fields (`[[artifacts]]`) -The following schema and policy additions are planned but not part of the v0.2 baseline: +- `target`: Rust-style target triple +- `asset`: release asset-name template (usually includes `{version}`) +- `archive` (optional): extraction hint +- `strip_components` (optional): extraction hint +- `binaries`: required non-empty array of executable mappings +- `completions` (optional): shell completion mappings +- `gui_apps` (optional): GUI integration metadata -- v0.3 source management is an index/snapshot workflow change and does not add or modify manifest fields. +`asset` is template metadata only; resolved download URLs/checksums live in release docs. -- v0.4 dependency policy fields (`provides`, `conflicts`, `replaces`): `docs/dependency-policy-spec.md`. -- Optional artifact signature enforcement policy details are planned for a future manifest/security spec update (non-GA; no enforcement in current release). - -Until these milestones land, manifests should use the current v0.2 field set documented above. - -## Related Docs +## Release Document (`releases//.toml`) -- Current install/runtime behavior: `docs/install-flow.md` -- Current architecture/module boundaries: `docs/architecture.md` -- Roadmap dependency policy (non-GA): `docs/dependency-policy-spec.md` -- Roadmap transaction policy (non-GA): `docs/transaction-rollback-spec.md` +Release docs store version-specific resolved artifact data. -### Artifact Binary Fields +### Required Fields -- `name`: exposed command name placed into `/bin/`. -- `path`: relative path inside extracted package content. +- `name`: package identifier (must match `` directory) +- `version`: semantic version (must match `` filename) +- `artifacts`: non-empty array -### Artifact Completion Fields +### Release Artifact Fields (`[[artifacts]]`) -- `shell`: one of `bash`, `zsh`, `fish`, `powershell`. -- `path`: relative path inside extracted package content. +- `target`: Rust-style target triple +- `url`: HTTPS download URL +- `sha256`: expected SHA-256 digest of artifact bytes -### Artifact GUI App Fields +## Runtime Manifest Fields (Merged Output) -- `app_id`: stable GUI application identifier (unique per artifact target). -- `display_name`: user-facing name for launcher metadata. -- `exec`: relative executable path inside extracted package content. -- `icon` (optional): relative icon path inside extracted package content. -- `categories` (optional): category labels for launcher metadata. -- `file_associations` (optional): list of file association declarations. -- `protocols` (optional): list of URL protocol handler declarations. +After merge, Crosspack expects runtime manifest semantics equivalent to: -#### GUI File Association Fields +- `name`, `version`, `description` (optional), `license` (optional), `homepage` (optional) +- `dependencies` (optional map) +- `source_build` (optional) +- `services` (optional) +- `artifacts` with executable/completion/GUI metadata -- `mime_type`: MIME type identifier. -- `extensions` (optional): list of file extensions (for example `.txt`, `.md`). +The merge model allows package templates to carry stable metadata while release docs carry per-version URL/checksum data. -#### GUI Protocol Fields +## Source Build Metadata (`source_build`) -- `scheme`: URL scheme token (for example `zed`, `myapp+internal`). - -## Example +`source_build` is parsed, validated, and used by source-build install flows. -```toml -name = "ripgrep" -version = "14.1.0" -license = "MIT" -homepage = "https://github.com/BurntSushi/ripgrep" +- `url`: source archive or source tree URL +- `archive_sha256`: expected SHA-256 digest of downloaded source archive bytes +- `build_system`: build-system token (`cargo`, `cmake`, etc.) +- `build_commands`: deterministic command-token array (non-empty) +- `install_commands`: deterministic command-token array (non-empty) -[dependencies] -pcre2 = ">=10.0, <11.0" +## Service Declarations (`services`) -[[artifacts]] -target = "x86_64-unknown-linux-gnu" -url = "https://packages.example/ripgrep-14.1.0-x86_64-unknown-linux-gnu.tar.zst" -sha256 = "..." -strip_components = 1 +- `name`: service token exposed to `crosspack services` +- `native_id` (optional): host-native service identifier -[[artifacts.binaries]] -name = "rg" -path = "rg" +Constraints: -[[artifacts.gui_apps]] -app_id = "org.example.ripgrep-viewer" -display_name = "Ripgrep Viewer" -exec = "tools/rg-viewer" -categories = ["Utility", "Development"] +- service tokens use package-token grammar (`[a-z0-9][a-z0-9._+-]{0,63}`) +- names must be unique per manifest +- invalid declarations fail closed -[[artifacts.gui_apps.file_associations]] -mime_type = "text/plain" -extensions = [".txt"] +## Artifact Kind Policy -[[artifacts.gui_apps.protocols]] -scheme = "rgview" +- Artifact ingestion is deterministic and fail-closed. +- Supported kinds: `zip`, `tar.gz`, `tar.zst`, `bin`, `msi`, `dmg`, `appimage`, `exe`, `pkg`, `msix`, `appx`. +- Pre-1.0 scope reset: `deb` and `rpm` are out of scope. +- Install mode defaults by kind: + - managed: `zip`, `tar.gz`, `tar.zst`, `bin`, `dmg`, `appimage` + - native: `pkg`, `exe`, `msi`, `msix`, `appx` -[[artifacts.completions]] -shell = "bash" -path = "completions/rg.bash" +## Registry Metadata Signing -[[artifacts.completions]] -shell = "zsh" -path = "completions/_rg" +- Registry metadata signing is strict and enabled by default. +- Trusted key file is `registry.pub` at registry root. +- Every package and release TOML file must have a detached `.sig` sidecar. +- Sidecars are hex-encoded detached signature bytes. +- Metadata-dependent operations fail closed on missing/invalid key or signatures. -[[artifacts]] -target = "x86_64-pc-windows-msvc" -url = "https://packages.example/ripgrep-14.1.0-x86_64-pc-windows-msvc.zip" -sha256 = "..." -archive = "zip" -artifact_root = "ripgrep-14.1.0-x86_64-pc-windows-msvc" +## Related Docs -[[artifacts.binaries]] -name = "rg" -path = "rg.exe" -``` +- `docs/registry-spec.md` +- `docs/source-management-spec.md` +- `docs/install-flow.md` diff --git a/docs/registry-spec.md b/docs/registry-spec.md index ec81663..44885f3 100644 --- a/docs/registry-spec.md +++ b/docs/registry-spec.md @@ -1,4 +1,4 @@ -# Registry Specification (Draft v0.3) +# Registry Specification (Draft v0.4) Crosspack uses configured registry sources with verified local snapshots. @@ -10,56 +10,52 @@ Crosspack uses configured registry sources with verified local snapshots. cache/ / registry.pub - index/ - / + packages/ + .toml + .toml.sig + releases/ + / .toml .toml.sig snapshot.json ``` -Legacy compatibility path when `--registry-root` is provided: - -```text -/ - registry.pub - index/ - / - .toml - .toml.sig -``` +When `--registry-root` is provided, the pointed registry root must expose the same `registry.pub` + `packages/` + `releases/` contract. ## Sync Strategy - Configure sources via `crosspack registry add`. -- Refresh snapshots via `crosspack update` (all sources by default, or selected via repeated `--registry `). -- Read manifests from local verified snapshots on disk. +- Refresh snapshots via `crosspack update`. +- Read manifests from local verified snapshots only. - Keep cached snapshots for deterministic resolution and source precedence. - If a source defines optional community metadata in `sources.toml`, verify the configured recipe catalog path and signature before snapshot acceptance. -## Version Discovery +## Version Discovery and Merge Model -- Package versions are discovered by listing TOML files in snapshot `index//`. -- Manifests are parsed and sorted by semantic version. -- If the same package exists in multiple sources, source precedence is deterministic: lowest priority wins, then lexical source name tie-break. +- Package names are discovered from `releases//` directories. +- Versions are discovered from `releases//.toml` files. +- For every version lookup: + 1. verify `packages/.toml(.sig)`, + 2. verify `releases//.toml(.sig)`, + 3. merge package template + release document into runtime manifest data. +- If the same package exists in multiple sources, precedence is deterministic: lowest `priority` first, then lexical source name tie-break. ## Security Baseline -- Artifacts must include SHA-256 digests. - Registry metadata signing is strict and enabled by default. -- `registry.pub` at the registry root is the local trust anchor for that registry snapshot or mirror. -- Each manifest must have a detached signature sidecar at `.toml.sig`. -- The sidecar format is hex-encoded detached signature bytes. -- Operations that rely on registry metadata fail closed on signature or key errors. -- Optional community recipe metadata is signed and validated with the same source trust root (`registry.pub`) and fails closed on missing/invalid signatures or invalid catalog content. -- If the entire registry root content is compromised (including `registry.pub`), this model does not provide authenticity guarantees for that compromised root. +- `registry.pub` at the source root is the trust anchor. +- Both package and release TOML files require detached `.sig` sidecars. +- Sidecar format is hex-encoded detached signature bytes. +- Metadata-dependent operations fail closed on key or signature errors. +- Optional community recipe metadata is signed and validated against the same source trust root. ## Optional Community Recipe Metadata - Source records may include an optional `community` block in `sources.toml`. - `community.recipe_catalog_path` points to a relative `.toml` file within the source snapshot (for example: `community/recipes.toml`). -- The recipe catalog requires a detached signature at `.sig` and must verify against the source `registry.pub` key. -- Catalog schema currently supports `version = 1` and `[[recipes]] package = ""` entries. -- Recipe entries must be strictly sorted by package name and each package must exist under `index//`. +- The catalog requires a detached signature at `.sig` and must verify against the source `registry.pub` key. +- Catalog schema supports `version = 1` and `[[recipes]] package = ""` entries. +- Recipe entries must be strictly sorted by package name and each package must exist under `releases//`. ## Source Management Commands @@ -67,8 +63,3 @@ Legacy compatibility path when `--registry-root` is provided: - `crosspack registry list` - `crosspack registry remove [--purge-cache]` - `crosspack update [--registry ]...` - -Metadata command fallback behavior: - -- `--registry-root` provided: use that root directly (legacy mode). -- `--registry-root` omitted: require at least one configured source with a ready snapshot. diff --git a/docs/release-checklist.md b/docs/release-checklist.md index 735546d..6fe8ad3 100644 --- a/docs/release-checklist.md +++ b/docs/release-checklist.md @@ -12,7 +12,7 @@ 1. Confirm repository variable `CROSSPACK_BOT_APP_ID` and repository secret `CROSSPACK_BOT_APP_PRIVATE_KEY` are configured for `.github/workflows/release-please.yml`. 2. Confirm registry sync configuration for `.github/workflows/registry-sync.yml`: - repository variable `CROSSPACK_REGISTRY_REPOSITORY` (default `spiritledsoftware/crosspack-registry`) or `CROSSPACK_REGISTRY_REPOSITORY_NAME` - - repository secret `CROSSPACK_REGISTRY_SIGNING_PRIVATE_KEY_PEM` (Ed25519 private key PEM used to sign `index/crosspack/.toml`) + - repository secret `CROSSPACK_REGISTRY_SIGNING_PRIVATE_KEY_PEM` (Ed25519 private key PEM used to sign `packages/crosspack.toml` and `releases/crosspack/.toml`) - GitHub App installation has `contents:write` on the registry repository 3. Verify merged commits on `main` follow Conventional Commits: - `fix:` -> patch bump diff --git a/docs/source-management-spec.md b/docs/source-management-spec.md index b2ea479..236484a 100644 --- a/docs/source-management-spec.md +++ b/docs/source-management-spec.md @@ -1,6 +1,6 @@ -# Source Management and Metadata Update Specification (Draft v0.3) +# Source Management and Metadata Update Specification (Draft v0.4) -This document defines the v0.3 source-management feature set for Crosspack. It introduces a multi-registry model with explicit trust pinning, snapshot updates, and deterministic source precedence. +This document defines the v0.4 source-management feature set for Crosspack. It introduces a multi-registry model with explicit trust pinning, snapshot updates, and deterministic source precedence. ## Scope @@ -10,7 +10,7 @@ This spec covers: - CLI commands for adding, listing, removing, and updating sources. - Snapshot fetch and trust verification rules. - How query and install commands read from multiple sources. -- Required tests and backward compatibility constraints. +- Required tests and fail-closed migration constraints. This spec does not cover: @@ -157,7 +157,8 @@ All source state is under the Crosspack prefix. cache/ / registry.pub - index/ + packages/ + releases/ snapshot.json ``` @@ -171,7 +172,7 @@ name = "core" kind = "git" location = "https://github.com/spiritledsoftware/crosspack-registry.git" priority = 100 -fingerprint_sha256 = "65149d198a39db9ecfea6f63d098858ed3b06c118c1f455f84ab571106b830c2" +fingerprint_sha256 = "87085672ad174b59ec6e8cfac8cfffebf84568ba08917426fd9e82b310780a52" enabled = true [sources.community] @@ -210,11 +211,12 @@ For each targeted source, `crosspack update` performs: 1. Sync source into a temporary directory. 2. Validate required files: - `registry.pub` - - `index/` + - `packages/` + - `releases/` 3. Compute fingerprint from fetched `registry.pub` and compare against `sources.toml`. 4. Verify metadata signature policy can be enforced (sidecar files must be present for manifests that are read by registry APIs). 5. If source `community` metadata is configured, verify `recipe_catalog_path` and detached signature (`.toml.sig`) using the same pinned source trust root. -6. Parse and validate the community recipe catalog (supported schema version, strictly sorted package names, package directories present). +6. Parse and validate the community recipe catalog (supported schema version, strictly sorted package names, `releases//` directories present). 7. Atomically replace `/state/registries/cache//`. 8. Write `snapshot.json`. @@ -228,7 +230,7 @@ Rules: - Commands never read directly from remote sources. - If no verified snapshot exists for any enabled source, metadata-dependent commands fail. -- Each manifest still requires detached signature verification (`.toml.sig`) and trusted key (`registry.pub`) in the source snapshot. +- Package and release documents each require detached signature verification and trusted key (`registry.pub`) in the source snapshot. ## Source Precedence and Package Selection @@ -260,7 +262,7 @@ Errors must include source name and actionable context. Rotation is explicit and fail-closed. Operators must complete all steps in order: -1. Generate and publish new `registry.pub` in the source index at the target cutover revision. +1. Generate and publish new `registry.pub` in the source root at the target cutover revision. 2. Compute new SHA-256 fingerprint from raw `registry.pub` bytes. 3. Update `docs/trust/core-registry-fingerprint.txt` with: - `fingerprint_sha256` @@ -280,13 +282,12 @@ crosspack update If local cache is suspected stale, users may use `crosspack registry remove core --purge-cache` before re-adding. -## Backward Compatibility +## Backward Compatibility and Migration -- Existing single-root usage via `--registry_root` remains valid for development and tests. -- If `sources.toml` is absent, commands behave as follows: - - with `--registry_root`: current behavior. - - without `--registry_root`: fail with guidance to run the official `core` bootstrap flow and then `crosspack update`. -- Receipt format remains backward compatible in v0.3, but new optional fields may be added in later versions. +- Legacy `index/` layout support is removed in v0.4. +- `--registry-root` requires `registry.pub`, `packages/`, and `releases/`. +- If `sources.toml` is absent and `--registry-root` is not provided, metadata-dependent commands fail with bootstrap guidance. +- Receipt format remains backward compatible; registry metadata layout is intentionally non-backward-compatible pre-1.0. ## Testing Requirements diff --git a/docs/trust/core-registry-fingerprint.txt b/docs/trust/core-registry-fingerprint.txt index 7c1ecf3..f33f103 100644 --- a/docs/trust/core-registry-fingerprint.txt +++ b/docs/trust/core-registry-fingerprint.txt @@ -1,10 +1,10 @@ source = core kind = git url = https://github.com/spiritledsoftware/crosspack-registry.git -fingerprint_sha256 = 65149d198a39db9ecfea6f63d098858ed3b06c118c1f455f84ab571106b830c2 -updated_at = 2026-02-23T16:15:00Z -key_id = core-registry-ed25519-2026-02-23 -release = https://github.com/spiritledsoftware/crosspack-registry/releases/tag/trust-core-2026-02-23 +fingerprint_sha256 = 87085672ad174b59ec6e8cfac8cfffebf84568ba08917426fd9e82b310780a52 +updated_at = 2026-03-02T12:00:00Z +key_id = core-registry-ed25519-2026-03-02 +release = https://github.com/spiritledsoftware/crosspack-registry/releases/tag/trust-core-2026-03-02 Verification: - Cross-check these exact values against the matching GitHub Release note entry. diff --git a/registry b/registry index d1b91dc..a43a33c 160000 --- a/registry +++ b/registry @@ -1 +1 @@ -Subproject commit d1b91dc094f4f3438c885e589c1e5ba3f68ef868 +Subproject commit a43a33c54f91a669447d621a28fb2c7090625335 diff --git a/scripts/sync-crosspack-registry-release.sh b/scripts/sync-crosspack-registry-release.sh index f02e4d6..8402789 100755 --- a/scripts/sync-crosspack-registry-release.sh +++ b/scripts/sync-crosspack-registry-release.sh @@ -93,16 +93,25 @@ registry_dir="$workdir/registry" echo "cloning registry repository: ${REGISTRY_REPOSITORY}" gh repo clone "$REGISTRY_REPOSITORY" "$registry_dir" -- --depth 1 -manifest_dir="$registry_dir/index/crosspack" -mkdir -p "$manifest_dir" -manifest_path="$manifest_dir/${VERSION}.toml" -sig_path="$manifest_path.sig" +package_dir="$registry_dir/packages" +release_dir="$registry_dir/releases/crosspack" +mkdir -p "$package_dir" "$release_dir" + +package_path="$package_dir/crosspack.toml" +package_sig_path="$package_path.sig" +release_path="$release_dir/${VERSION}.toml" +release_sig_path="$release_path.sig" { echo 'name = "crosspack"' - echo "version = \"${VERSION}\"" echo 'license = "MIT"' echo "homepage = \"${HOME_URL}\"" + echo + echo '[source]' + echo 'provider = "github"' + echo "repo = \"${RELEASE_REPOSITORY}\"" + echo 'tag_prefix = "v"' + echo 'include_prereleases = false' for target in "${targets[@]}"; do if [[ "$target" == "x86_64-pc-windows-msvc" ]]; then @@ -113,20 +122,12 @@ sig_path="$manifest_path.sig" binary_path="crosspack" fi - asset="crosspack-${RELEASE_TAG}-${target}.${archive}" - sha256="$(checksum_for_asset "$asset")" - if [ -z "$sha256" ]; then - echo "checksum not found for release asset: ${asset}" >&2 - exit 1 - fi - - url="${HOME_URL}/releases/download/${RELEASE_TAG}/${asset}" + asset_template="crosspack-v{version}-${target}.${archive}" echo echo '[[artifacts]]' echo "target = \"${target}\"" - echo "url = \"${url}\"" - echo "sha256 = \"${sha256}\"" + echo "asset = \"${asset_template}\"" echo "archive = \"${archive}\"" echo 'strip_components = 0' echo @@ -141,19 +142,53 @@ sig_path="$manifest_path.sig" echo "path = \"${binary_path}\"" fi done -} > "$manifest_path" +} > "$package_path" + +{ + echo 'name = "crosspack"' + echo "version = \"${VERSION}\"" + + for target in "${targets[@]}"; do + if [[ "$target" == "x86_64-pc-windows-msvc" ]]; then + archive="zip" + else + archive="tar.gz" + fi + + asset="crosspack-${RELEASE_TAG}-${target}.${archive}" + sha256="$(checksum_for_asset "$asset")" + if [ -z "$sha256" ]; then + echo "checksum not found for release asset: ${asset}" >&2 + exit 1 + fi + url="${HOME_URL}/releases/download/${RELEASE_TAG}/${asset}" + + echo + echo '[[artifacts]]' + echo "target = \"${target}\"" + echo "url = \"${url}\"" + echo "sha256 = \"${sha256}\"" + done +} > "$release_path" key_path="$workdir/registry-signing.key" printf '%s' "$REGISTRY_SIGNING_PRIVATE_KEY_PEM" > "$key_path" chmod 600 "$key_path" -sig_bin_path="$workdir/signature.bin" -openssl pkeyutl -sign -rawin -inkey "$key_path" -in "$manifest_path" -out "$sig_bin_path" -xxd -p -c 9999 "$sig_bin_path" | tr -d '\n' > "$sig_path" -printf '\n' >> "$sig_path" +sign_manifest() { + local manifest_path="$1" + local signature_path="$2" + local sig_bin_path="$workdir/signature.bin" + openssl pkeyutl -sign -rawin -inkey "$key_path" -in "$manifest_path" -out "$sig_bin_path" + xxd -p -c 9999 "$sig_bin_path" | tr -d '\n' > "$signature_path" + printf '\n' >> "$signature_path" +} + +sign_manifest "$package_path" "$package_sig_path" +sign_manifest "$release_path" "$release_sig_path" pushd "$registry_dir" >/dev/null -if [ -z "$(git status --porcelain -- "index/crosspack/${VERSION}.toml" "index/crosspack/${VERSION}.toml.sig")" ]; then +if [ -z "$(git status --porcelain -- "packages/crosspack.toml" "packages/crosspack.toml.sig" "releases/crosspack/${VERSION}.toml" "releases/crosspack/${VERSION}.toml.sig")" ]; then echo "registry metadata already up to date for crosspack@${VERSION}" exit 0 fi @@ -165,7 +200,7 @@ if [ -n "${GH_TOKEN:-}" ]; then git remote set-url origin "https://x-access-token:${GH_TOKEN}@github.com/${REGISTRY_REPOSITORY}.git" fi -git add "index/crosspack/${VERSION}.toml" "index/crosspack/${VERSION}.toml.sig" +git add "packages/crosspack.toml" "packages/crosspack.toml.sig" "releases/crosspack/${VERSION}.toml" "releases/crosspack/${VERSION}.toml.sig" git commit -m "chore(registry): add crosspack@${VERSION}" git push origin HEAD:main popd >/dev/null From 10e0c94431cab8d3eda512cc2954f328ce2cd1e8 Mon Sep 17 00:00:00 2001 From: Ian Pascoe Date: Mon, 2 Mar 2026 10:02:44 -0500 Subject: [PATCH 2/2] fix(registry): merge package template artifact fields by target --- .../crosspack-registry/src/registry_index.rs | 42 +++++++++++++ crates/crosspack-registry/src/tests.rs | 60 +++++++++++++++++++ 2 files changed, 102 insertions(+) diff --git a/crates/crosspack-registry/src/registry_index.rs b/crates/crosspack-registry/src/registry_index.rs index 42c07ac..f50f982 100644 --- a/crates/crosspack-registry/src/registry_index.rs +++ b/crates/crosspack-registry/src/registry_index.rs @@ -213,6 +213,18 @@ fn merge_manifest_documents(package_template: &Table, release_document: &Table) fn merge_tables(base: &mut Table, overlay: &Table) { for (key, overlay_value) in overlay { + if key == "artifacts" { + if let (Some(Value::Array(base_array)), Value::Array(overlay_array)) = + (base.get(key), overlay_value) + { + base.insert( + key.clone(), + Value::Array(merge_artifacts(base_array, overlay_array)), + ); + continue; + } + } + if let Some(Value::Table(base_table)) = base.get_mut(key) { if let Value::Table(overlay_table) = overlay_value { merge_tables(base_table, overlay_table); @@ -223,6 +235,36 @@ fn merge_tables(base: &mut Table, overlay: &Table) { } } +fn merge_artifacts(base: &[Value], overlay: &[Value]) -> Vec { + overlay + .iter() + .map(|overlay_value| { + let Some(overlay_table) = overlay_value.as_table() else { + return overlay_value.clone(); + }; + let Some(target) = overlay_table.get("target").and_then(Value::as_str) else { + return overlay_value.clone(); + }; + + let Some(base_table) = base.iter().find_map(|base_value| { + let base_table = base_value.as_table()?; + let base_target = base_table.get("target")?.as_str()?; + if base_target == target { + Some(base_table) + } else { + None + } + }) else { + return overlay_value.clone(); + }; + + let mut merged = base_table.clone(); + merge_tables(&mut merged, overlay_table); + Value::Table(merged) + }) + .collect() +} + impl ConfiguredRegistryIndex { pub fn open(state_root: impl Into) -> Result { let state_root = state_root.into(); diff --git a/crates/crosspack-registry/src/tests.rs b/crates/crosspack-registry/src/tests.rs index be5f7a6..b5f7e83 100644 --- a/crates/crosspack-registry/src/tests.rs +++ b/crates/crosspack-registry/src/tests.rs @@ -1824,6 +1824,66 @@ fn package_versions_succeeds_with_valid_signatures_and_descending_sort() { let _ = fs::remove_dir_all(&root); } +#[test] +fn package_versions_merges_release_artifacts_with_package_template_fields() { + let root = test_registry_root(); + let package_dir = root.join("releases").join("ripgrep"); + fs::create_dir_all(&package_dir).expect("must create package dir"); + + let signing_key = signing_key(); + fs::write(root.join("registry.pub"), public_key_hex(&signing_key)) + .expect("must write registry public key"); + write_signed_package_template( + &root, + &signing_key, + "ripgrep", + r#"name = "ripgrep" +license = "MIT" + +[[artifacts]] +target = "x86_64-unknown-linux-gnu" +archive = "tar.gz" +strip_components = 1 + +[[artifacts.binaries]] +name = "rg" +path = "rg" +"#, + ); + write_signed_release_manifest( + &package_dir, + &signing_key, + "14.1.0", + r#"version = "14.1.0" + +[[artifacts]] +target = "x86_64-unknown-linux-gnu" +url = "https://example.invalid/ripgrep-14.1.0-linux.tar.gz" +sha256 = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" +"#, + ); + + let index = RegistryIndex::open(&root); + let manifests = index + .package_versions("ripgrep") + .expect("must load merged manifest"); + + assert_eq!(manifests.len(), 1); + assert_eq!(manifests[0].version.to_string(), "14.1.0"); + assert_eq!(manifests[0].artifacts.len(), 1); + assert_eq!( + manifests[0].artifacts[0].url, + "https://example.invalid/ripgrep-14.1.0-linux.tar.gz" + ); + assert_eq!(manifests[0].artifacts[0].archive.as_deref(), Some("tar.gz")); + assert_eq!(manifests[0].artifacts[0].strip_components, Some(1)); + assert_eq!(manifests[0].artifacts[0].binaries.len(), 1); + assert_eq!(manifests[0].artifacts[0].binaries[0].name, "rg"); + assert_eq!(manifests[0].artifacts[0].binaries[0].path, "rg"); + + let _ = fs::remove_dir_all(&root); +} + fn write_signed_manifest(package_dir: &std::path::Path, signing_key: &SigningKey, version: &str) { let package_name = package_dir .file_name()