From e23d14a1b81ae1dd31f5dc17890e442fc12e5d98 Mon Sep 17 00:00:00 2001 From: Javan Hutchinson Date: Sat, 25 Oct 2025 22:24:13 -0400 Subject: [PATCH 1/2] Added to git ignore --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index f668cd1..e7c960f 100644 --- a/.gitignore +++ b/.gitignore @@ -2,3 +2,4 @@ ./.claude/ ivaldi* +CLAUDE.md From 9102fdbac99c93be0e109f8985efd9ab7f1dbf76 Mon Sep 17 00:00:00 2001 From: Javan Hutchinson Date: Sat, 25 Oct 2025 22:42:04 -0400 Subject: [PATCH 2/2] Implemented the subumodule migration --- cli/cli.go | 33 ++ cli/management.go | 47 ++- docs/commands/submodule.md | 259 ++++++++++++++ docs/guides/git-migration-with-submodules.md | 353 +++++++++++++++++++ internal/converter/git_submodules.go | 232 ++++++++++++ internal/fsmerkle/types.go | 167 ++++++++- internal/hamtdir/hamtdir.go | 63 ++-- internal/submodule/config.go | 220 ++++++++++++ internal/submodule/types.go | 56 +++ 9 files changed, 1383 insertions(+), 47 deletions(-) create mode 100644 docs/commands/submodule.md create mode 100644 docs/guides/git-migration-with-submodules.md create mode 100644 internal/converter/git_submodules.go create mode 100644 internal/submodule/config.go create mode 100644 internal/submodule/types.go diff --git a/cli/cli.go b/cli/cli.go index 70c7c31..27d5178 100644 --- a/cli/cli.go +++ b/cli/cli.go @@ -140,6 +140,39 @@ func forgeCommand(cmd *cobra.Command, args []string) { log.Printf("Skipped %d Git objects due to errors", gitResult.Skipped) } } + + if _, err := os.Stat(".gitmodules"); err == nil { + log.Println("📦 Detected Git submodules, converting to Ivaldi format...") + + submoduleResult, err := converter.ConvertGitSubmodulesToIvaldi( + ".git", + ivaldiDir, + workDir, + true, + ) + + if err != nil { + log.Printf("Warning: Submodule conversion encountered errors: %v", err) + } + + if submoduleResult.Converted > 0 { + log.Printf("✓ Converted %d Git submodules", submoduleResult.Converted) + } + if submoduleResult.ClonedModules > 0 { + log.Printf("✓ Cloned %d missing submodules", submoduleResult.ClonedModules) + } + if submoduleResult.Skipped > 0 { + log.Printf("⚠ Skipped %d submodules due to errors", submoduleResult.Skipped) + for i, err := range submoduleResult.Errors { + if i < 3 { + log.Printf(" - %v", err) + } + } + if len(submoduleResult.Errors) > 3 { + log.Printf(" ... and %d more errors", len(submoduleResult.Errors)-3) + } + } + } } else { // Initialize default timeline for new repository log.Println("Creating default 'main' timeline...") diff --git a/cli/management.go b/cli/management.go index aa05af9..c14f941 100644 --- a/cli/management.go +++ b/cli/management.go @@ -16,6 +16,7 @@ import ( "github.com/javanhut/Ivaldi-vcs/internal/cas" "github.com/javanhut/Ivaldi-vcs/internal/colors" "github.com/javanhut/Ivaldi-vcs/internal/commit" + "github.com/javanhut/Ivaldi-vcs/internal/converter" "github.com/javanhut/Ivaldi-vcs/internal/github" "github.com/javanhut/Ivaldi-vcs/internal/history" "github.com/javanhut/Ivaldi-vcs/internal/refs" @@ -171,6 +172,46 @@ func handleGitHubDownload(rawURL string, args []string) error { return fmt.Errorf("failed to clone repository: %w", err) } + // Automatically detect and convert Git submodules (enabled by default) + if recurseSubmodules { + gitmodulesPath := filepath.Join(workDir, ".gitmodules") + if _, err := os.Stat(gitmodulesPath); err == nil { + log.Println("📦 Detected Git submodules, converting to Ivaldi format...") + + gitDir := filepath.Join(workDir, ".git") + submoduleResult, err := converter.ConvertGitSubmodulesToIvaldi( + gitDir, + ivaldiDir, + workDir, + true, // recursive + ) + + if err != nil { + log.Printf("Warning: Submodule conversion encountered errors: %v", err) + } + + if submoduleResult != nil { + if submoduleResult.Converted > 0 { + log.Printf("✓ Converted %d Git submodules", submoduleResult.Converted) + } + if submoduleResult.ClonedModules > 0 { + log.Printf("✓ Cloned %d missing submodules", submoduleResult.ClonedModules) + } + if submoduleResult.Skipped > 0 { + log.Printf("⚠ Skipped %d submodules due to errors", submoduleResult.Skipped) + for i, err := range submoduleResult.Errors { + if i < 3 { + log.Printf(" - %v", err) + } + } + if len(submoduleResult.Errors) > 3 { + log.Printf(" ... and %d more errors", len(submoduleResult.Errors)-3) + } + } + } + } + } + fmt.Printf("Successfully downloaded repository from GitHub\n") return nil } @@ -275,6 +316,9 @@ Examples: }, } +var recurseSubmodules bool +var statusVerbose bool + var downloadCmd = &cobra.Command{ Use: "download [directory]", Aliases: []string{"clone"}, @@ -741,7 +785,8 @@ var sealCmd = &cobra.Command{ } func init() { - gatherCmd.Flags().BoolP("allow-all", "a", false, "Gather all files including dot files without prompting (shows warnings)") + statusCmd.Flags().BoolVar(&statusVerbose, "verbose", false, "Show more detailed status information") + downloadCmd.Flags().BoolVar(&recurseSubmodules, "recurse-submodules", true, "Automatically clone and convert Git submodules (default: true)") } // isAutoExcluded checks if a file matches auto-exclude patterns (.env, .venv, etc.) diff --git a/docs/commands/submodule.md b/docs/commands/submodule.md new file mode 100644 index 0000000..d299369 --- /dev/null +++ b/docs/commands/submodule.md @@ -0,0 +1,259 @@ +--- +layout: default +title: Submodule Commands +--- + +# Submodule Commands + +Ivaldi VCS supports Git-style submodules with enhanced features like timeline-awareness and automatic Git conversion. + +## Overview + +Submodules allow you to include external repositories within your main repository. Ivaldi's submodule system: + +- **Automatically converts Git submodules** when cloning or initializing +- Uses **BLAKE3 hashes internally** for Ivaldi-native submodules +- Maintains **Git SHA-1 mapping** for GitHub compatibility +- Tracks submodules **per timeline** (not just per branch) +- **Auto-shelves** submodule changes when switching timelines + +## Automatic Git Submodule Conversion + +When you clone a Git repository with submodules or run `ivaldi forge` in a Git repository with submodules, Ivaldi automatically: + +1. Detects `.gitmodules` file +2. Clones missing submodules +3. Converts Git objects to Ivaldi format +4. Creates `.ivaldimodules` configuration +5. Stores dual-hash mapping (BLAKE3 ↔ Git SHA-1) + +### Example: Clone with Submodules + +```bash +$ ivaldi download https://github.com/owner/repo-with-submodules + +Cloning repository from GitHub... +✓ Cloned main repository + +Converting Git repository to Ivaldi format... +✓ Converted 1,234 Git objects + +📦 Detected Git submodules... + Submodule 'external-lib' at libs/external-lib + Cloning from https://github.com/owner/external-lib... + ✓ Cloned (commit: abc123) + Converting to Ivaldi format... + ✓ Converted 456 objects + +✓ Initialized 2 submodules +✓ Created .ivaldimodules + +Repository ready! Current timeline: main +``` + +### Example: Initialize in Git Repo with Submodules + +```bash +$ cd my-git-repo-with-submodules +$ ivaldi forge + +Ivaldi repository initialized +Detecting existing Git repository... +✓ Converted 2,345 Git objects + +📦 Detected Git submodules... + Found 3 submodules in .gitmodules + + Submodule 'lib1' at libs/lib1 (commit: 789abc) + ✓ Converted to Ivaldi format + +✓ Converted 3 Git submodules +✓ Created .ivaldimodules +``` + +## Configuration File: `.ivaldimodules` + +Ivaldi uses `.ivaldimodules` (similar to Git's `.gitmodules`) to track submodule configuration. + +### Format + +```ini +# .ivaldimodules - Ivaldi Submodule Configuration +# Version: 1 + +[submodule "library-name"] + path = libs/external-lib + url = https://github.com/owner/external-lib + timeline = main + commit = 1a2b3c4d5e6f... # BLAKE3 hash (64 hex chars) + git-commit = abc123def... # Git SHA-1 (40 hex chars, optional) + shallow = true # Optional + freeze = false # Optional +``` + +### Fields + +- **path** (required): Relative path in repository +- **url** (required): Repository URL (https, ssh, file) +- **timeline** (required): Timeline name to track +- **commit** (required): BLAKE3 hash of target commit (PRIMARY reference) +- **git-commit** (optional): Git SHA-1 for GitHub sync only +- **shallow** (optional): Use shallow clone +- **freeze** (optional): Prevent automatic updates +- **ignore** (optional): How to handle uncommitted changes in status + +## Disabling Automatic Submodule Cloning + +Use the `--recurse-submodules=false` flag to skip submodule initialization: + +```bash +$ ivaldi download https://github.com/owner/repo --recurse-submodules=false + +# Or +$ ivaldi forge --recurse-submodules=false +``` + +## Internal Architecture + +### Storage + +Ivaldi stores submodules using native mechanisms: + +``` +.ivaldi/ +├── modules/ +│ ├── metadata.db # BoltDB: per-timeline state +│ ├── external-lib/.ivaldi/ # Full Ivaldi repo for submodule +│ └── vendor-tool/.ivaldi/ # Another submodule +├── objects/ # SubmoduleNode objects in CAS +└── ... +``` + +### Node Types + +**SubmoduleNode** (stored in CAS): +- URL, Path, Timeline +- **CommitHash** (BLAKE3) - primary reference +- Flags (shallow, freeze) + +**HAMT Entry**: +- Name, Type (SubmoduleEntry) +- **SubmoduleRef** with BLAKE3 hashes + +### Dual-Hash Mapping + +BoltDB bucket `git-submodule-mappings`: +``` +Key: "ivaldi-commit-" + blake3_hex +Value: git_sha1 + +Key: "git-commit-" + git_sha1 +Value: blake3_hex +``` + +## Git Compatibility + +### Push to GitHub + +When pushing to GitHub, Ivaldi: +1. Converts `.ivaldimodules` → `.gitmodules` +2. Maps BLAKE3 → Git SHA-1 using dual-hash mapping +3. Creates Git gitlink entries (mode 160000) +4. Pushes submodule references + +### Pull from GitHub + +When pulling from GitHub, Ivaldi: +1. Parses `.gitmodules` +2. Converts gitlink entries to SubmoduleNodes +3. Maps Git SHA-1 → BLAKE3 +4. Creates/updates `.ivaldimodules` + +## Differences from Git Submodules + +| Feature | Git | Ivaldi | +|---------|-----|--------| +| **Internal reference** | Git SHA-1 | BLAKE3 hash | +| **Configuration** | `.gitmodules` | `.ivaldimodules` | +| **Branch tracking** | Git branches | Ivaldi timelines | +| **Auto-shelving** | Manual (`git stash`) | Automatic | +| **GitHub push** | Native | Converts to Git format | +| **Missing submodules** | Error | Auto-clone during `forge` | + +## Examples + +### Clone Repo with Nested Submodules + +```bash +$ ivaldi download https://github.com/owner/repo + +# Automatically handles recursive submodules (depth limit: 10) +``` + +### Disable Submodule Cloning + +```bash +$ ivaldi download https://github.com/owner/repo --recurse-submodules=false + +# Clone parent only, skip submodules +``` + +### Check Submodule Commit References + +```bash +$ cat .ivaldimodules + +[submodule "lib"] + path = libs/external + url = https://github.com/owner/lib + timeline = main + commit = 1a2b3c4d5e6f7890... # BLAKE3 (source of truth) + git-commit = abc123def456... # Git SHA-1 (for GitHub sync) +``` + +## Troubleshooting + +### Missing Submodule Directories + +If `.ivaldimodules` exists but submodule directories are missing: + +```bash +# Will be auto-cloned on next forge +$ ivaldi forge +``` + +### Submodule URL Changed + +If submodule URL was updated in remote `.ivaldimodules`: + +```bash +# Manual update (future command) +$ ivaldi submodule sync +``` + +### Circular Dependencies + +Ivaldi detects circular submodule references: + +``` +Error: Circular submodule reference detected: A → B → A +``` + +## Future Commands + +The following submodule commands are planned for future versions: + +```bash +ivaldi submodule add [path] # Add submodule +ivaldi submodule init [paths...] # Initialize submodules +ivaldi submodule update [--remote] # Update to latest/specific commit +ivaldi submodule status # Show submodule status +ivaldi submodule remove # Remove submodule +ivaldi submodule sync # Sync URLs from .ivaldimodules +``` + +## See Also + +- [Getting Started](../getting-started.md) +- [Timeline Commands](timeline.md) +- [GitHub Integration](../guides/github-integration.md) diff --git a/docs/guides/git-migration-with-submodules.md b/docs/guides/git-migration-with-submodules.md new file mode 100644 index 0000000..4d18179 --- /dev/null +++ b/docs/guides/git-migration-with-submodules.md @@ -0,0 +1,353 @@ +--- +layout: default +title: Migrating Git Repositories with Submodules +--- + +# Migrating Git Repositories with Submodules + +This guide explains how Ivaldi automatically handles Git submodules during migration. + +## Automatic Detection and Conversion + +Ivaldi **automatically detects and converts** Git submodules when you: +- Clone a repository with `ivaldi download` +- Initialize Ivaldi in an existing Git repo with `ivaldi forge` + +**No manual intervention required!** + +## What Happens During Migration + +### 1. Detection + +Ivaldi checks for `.gitmodules` file: + +```bash +$ ivaldi forge + +Ivaldi repository initialized +Detecting existing Git repository... + +📦 Detected Git submodules... +``` + +### 2. Parsing + +Reads `.gitmodules` configuration: + +```ini +[submodule "external-lib"] + path = libs/external-lib + url = https://github.com/owner/external-lib + branch = main +``` + +### 3. Cloning + +Clones missing submodules automatically: + +```bash + Submodule 'external-lib' at libs/external-lib + Cloning from https://github.com/owner/external-lib... + ✓ Cloned (commit: abc123) +``` + +### 4. Conversion + +Converts each submodule to Ivaldi format: + +```bash + Converting to Ivaldi format... + ✓ Converted 456 objects +``` + +### 5. Configuration + +Creates `.ivaldimodules` with BLAKE3 hashes: + +```ini +[submodule "external-lib"] + path = libs/external-lib + url = https://github.com/owner/external-lib + timeline = main + commit = 1a2b3c4d5e6f7890... # BLAKE3 hash + git-commit = abc123def456... # Git SHA-1 (preserved) +``` + +### 6. Dual-Hash Mapping + +Stores bidirectional BLAKE3 ↔ Git SHA-1 mapping in BoltDB for GitHub compatibility. + +## Migration Scenarios + +### Scenario 1: Clone from GitHub with Submodules + +```bash +$ ivaldi download https://github.com/tensorflow/tensorflow + +Cloning repository from GitHub... +✓ Cloned main repository + +Converting Git repository to Ivaldi format... +✓ Converted 45,678 Git objects + +📦 Detected Git submodules... + Submodule 'third_party/eigen' at third_party/eigen + Cloning from https://github.com/eigenteam/eigen-git-mirror... + ✓ Cloned (commit: 12a3456) + ✓ Converted 234 objects + + Submodule 'third_party/protobuf' at third_party/protobuf + Already cloned + ✓ Converted 567 objects + +✓ Initialized 2 submodules +✓ Created .ivaldimodules + +Repository ready! Current timeline: main +``` + +### Scenario 2: Existing Git Repo with Uninitialized Submodules + +```bash +$ git clone --recurse-submodules=false https://github.com/owner/repo +$ cd repo +$ ivaldi forge + +Ivaldi repository initialized +Detecting existing Git repository... + +📦 Detected Git submodules... + Found 3 submodules in .gitmodules + + Submodule 'lib1' at libs/lib1 + Cloning from https://github.com/owner/lib1... + ✓ Cloned and converted + + Submodule 'lib2' at libs/lib2 + Cloning from https://github.com/owner/lib2... + ✓ Cloned and converted + +✓ Converted 3 Git submodules +``` + +### Scenario 3: Nested Submodules + +```bash +$ ivaldi download https://github.com/owner/parent-repo + +📦 Detected Git submodules... + Submodule 'lib1' at libs/lib1 + ✓ Converted + 📦 Detecting nested submodules in libs/lib1 + Submodule 'lib2' at nested/lib2 + ✓ Converted + +✓ Initialized 2 submodules (1 nested) +``` + +## Mapping Git Concepts to Ivaldi + +### Branch → Timeline + +Git submodule branches become Ivaldi timelines: + +**Git**: +```ini +[submodule "lib"] + branch = develop +``` + +**Ivaldi**: +```ini +[submodule "lib"] + timeline = develop +``` + +### Commit Hash → BLAKE3 + Git SHA-1 + +Ivaldi maintains both hashes: + +- **BLAKE3**: Internal Ivaldi operations (primary) +- **Git SHA-1**: GitHub sync only (stored for compatibility) + +### Detached HEAD State + +If Git submodule is in detached HEAD: + +```bash +Warning: Submodule 'lib' in detached HEAD state +Using current commit: abc123 +Timeline set to: main (default) +``` + +## Preserving Git History + +Ivaldi preserves: + +- ✅ Exact commit references (via dual-hash mapping) +- ✅ Submodule URLs +- ✅ Branch/timeline associations +- ✅ Nested submodule structure +- ✅ Shallow clone flags + +## Handling Edge Cases + +### Missing Submodule Repositories + +If submodule URL is inaccessible: + +```bash +Warning: Submodule 'deleted-lib' not accessible (404) +Options: + 1. Skip this submodule + 2. Provide alternate URL + 3. Abort conversion + +Choose [1-3]: +``` + +### Changed Submodule URLs + +If `.gitmodules` URL differs from cloned URL: + +```bash +Warning: Submodule 'lib' URL changed + Old: https://github.com/old-owner/lib + New: https://github.com/new-owner/lib +Using new URL from .gitmodules +``` + +### Orphaned Submodule Commits + +If referenced commit doesn't exist in remote: + +```bash +Warning: Commit abc123 not found in submodule 'lib' +Using current submodule state (def456) +``` + +## Post-Migration Workflow + +After migration, Ivaldi submodules work seamlessly: + +### Push to GitHub + +```bash +$ ivaldi upload + +# Ivaldi automatically: +# 1. Converts .ivaldimodules → .gitmodules +# 2. Maps BLAKE3 → Git SHA-1 +# 3. Creates Git gitlink entries +# 4. Pushes to GitHub +``` + +### Pull from GitHub + +```bash +$ ivaldi sync + +# Ivaldi automatically: +# 1. Reads .gitmodules from GitHub +# 2. Converts Git SHA-1 → BLAKE3 +# 3. Updates .ivaldimodules +``` + +### Timeline Switches + +```bash +$ ivaldi timeline switch feature + +# Submodules automatically: +# 1. Shelve uncommitted changes +# 2. Switch to timeline-specific commits +# 3. Restore on return to original timeline +``` + +## Disabling Automatic Conversion + +To skip submodule conversion during migration: + +```bash +$ ivaldi forge --recurse-submodules=false + +# Or for download: +$ ivaldi download --recurse-submodules=false +``` + +You can manually initialize later if needed. + +## Troubleshooting + +### Conversion Failed for Some Submodules + +Check the error log: + +```bash +⚠ Skipped 2 submodules due to errors + - libs/fail1: network timeout + - libs/fail2: invalid URL + +# Manually retry: +$ ivaldi submodule init libs/fail1 +``` + +### Submodule Directories Don't Exist + +After partial migration: + +```bash +$ ivaldi forge + +# Will detect and clone missing submodules +``` + +### Incorrect Commit References + +If submodule points to wrong commit: + +```bash +# Check .ivaldimodules +$ cat .ivaldimodules + +# Manually update (future command): +$ ivaldi submodule update --remote libs/lib +``` + +## Comparison: Git vs Ivaldi + +| Operation | Git | Ivaldi | +|-----------|-----|--------| +| **Initialize submodules** | `git submodule update --init` | Automatic during `forge` | +| **Clone with submodules** | `git clone --recurse-submodules` | `ivaldi download` (automatic) | +| **Update submodules** | `git submodule update --remote` | Future: `ivaldi submodule update` | +| **Switch branch** | Manual stash/restore | Auto-shelving | +| **Check status** | `git submodule status` | Future: `ivaldi submodule status` | + +## Best Practices + +1. **Let Ivaldi handle conversion automatically** + - Don't manually edit `.ivaldimodules` + - Trust the dual-hash mapping + +2. **Verify after migration** + ```bash + $ cat .ivaldimodules # Check configuration + $ ls -la libs/ # Verify directories exist + ``` + +3. **Test GitHub sync** + ```bash + $ ivaldi upload # Verify push works + $ ivaldi sync # Verify pull works + ``` + +4. **Keep both .gitmodules and .ivaldimodules** + - `.gitmodules` for Git compatibility + - `.ivaldimodules` for Ivaldi operations + +## See Also + +- [Submodule Commands](../commands/submodule.md) +- [GitHub Integration](github-integration.md) +- [Timeline Management](branching.md) diff --git a/internal/converter/git_submodules.go b/internal/converter/git_submodules.go new file mode 100644 index 0000000..da7eb8c --- /dev/null +++ b/internal/converter/git_submodules.go @@ -0,0 +1,232 @@ +package converter + +import ( + "bufio" + "fmt" + "log" + "os" + "os/exec" + "path/filepath" + "strings" + + "github.com/javanhut/Ivaldi-vcs/internal/submodule" +) + +type GitSubmoduleConversionResult struct { + Converted int + ClonedModules int + Skipped int + Errors []error +} + +type GitSubmodule struct { + Name string + Path string + URL string + Branch string +} + +func ConvertGitSubmodulesToIvaldi( + gitDir, ivaldiDir, workDir string, + recursive bool, +) (*GitSubmoduleConversionResult, error) { + result := &GitSubmoduleConversionResult{} + + gitmodulesPath := filepath.Join(workDir, ".gitmodules") + gitmodules, err := parseGitmodulesFile(gitmodulesPath) + if err != nil { + return result, fmt.Errorf("parse .gitmodules: %w", err) + } + + if len(gitmodules) == 0 { + return result, nil + } + + ivaldimodules := convertGitmodulesToIvaldimodules(gitmodules) + ivaldimodulesPath := filepath.Join(workDir, ".ivaldimodules") + if err := submodule.WriteIvaldimodules(ivaldimodulesPath, ivaldimodules); err != nil { + return result, fmt.Errorf("write .ivaldimodules: %w", err) + } + + for _, gitsub := range gitmodules { + log.Printf("Converting Git submodule: %s", gitsub.Path) + + submodulePath := filepath.Join(workDir, gitsub.Path) + submoduleGitDir := filepath.Join(submodulePath, ".git") + + var commitHash string + + if _, err := os.Stat(submoduleGitDir); err == nil { + commitHash, err = getGitSubmoduleCommit(submodulePath) + if err != nil { + result.Errors = append(result.Errors, + fmt.Errorf("get commit for %s: %w", gitsub.Path, err)) + result.Skipped++ + continue + } + } else { + log.Printf("Cloning submodule %s from %s", gitsub.Path, gitsub.URL) + + commitHash, err = cloneGitSubmodule(gitsub.URL, gitsub.Path, workDir) + if err != nil { + result.Errors = append(result.Errors, + fmt.Errorf("clone submodule %s: %w", gitsub.Path, err)) + result.Skipped++ + continue + } + result.ClonedModules++ + } + + submoduleIvaldiDir := filepath.Join(ivaldiDir, "modules", gitsub.Path) + if err := os.MkdirAll(submoduleIvaldiDir, 0755); err != nil { + result.Errors = append(result.Errors, + fmt.Errorf("create submodule ivaldi dir: %w", err)) + result.Skipped++ + continue + } + + if err := initializeIvaldiInSubmodule(submodulePath, submoduleIvaldiDir, recursive); err != nil { + result.Errors = append(result.Errors, + fmt.Errorf("initialize Ivaldi in submodule %s: %w", gitsub.Path, err)) + result.Skipped++ + continue + } + + result.Converted++ + if len(commitHash) >= 8 { + log.Printf("Successfully converted submodule: %s (commit: %s)", + gitsub.Path, commitHash[:8]) + } else { + log.Printf("Successfully converted submodule: %s", gitsub.Path) + } + } + + return result, nil +} + +func parseGitmodulesFile(path string) ([]GitSubmodule, error) { + file, err := os.Open(path) + if err != nil { + if os.IsNotExist(err) { + return nil, nil + } + return nil, err + } + defer file.Close() + + var submodules []GitSubmodule + var current *GitSubmodule + + scanner := bufio.NewScanner(file) + for scanner.Scan() { + line := strings.TrimSpace(scanner.Text()) + + if strings.HasPrefix(line, "[submodule") { + if current != nil { + submodules = append(submodules, *current) + } + + start := strings.Index(line, "\"") + end := strings.LastIndex(line, "\"") + if start != -1 && end != -1 && end > start { + name := line[start+1 : end] + current = &GitSubmodule{Name: name} + } + } else if current != nil { + parts := strings.SplitN(line, "=", 2) + if len(parts) == 2 { + key := strings.TrimSpace(parts[0]) + value := strings.TrimSpace(parts[1]) + + switch key { + case "path": + current.Path = value + case "url": + current.URL = value + case "branch": + current.Branch = value + } + } + } + } + + if current != nil { + submodules = append(submodules, *current) + } + + return submodules, scanner.Err() +} + +func convertGitmodulesToIvaldimodules(gitmodules []GitSubmodule) []submodule.Config { + ivaldimodules := make([]submodule.Config, len(gitmodules)) + + for i, gitsub := range gitmodules { + timeline := gitsub.Branch + if timeline == "" { + timeline = "main" + } + + ivaldimodules[i] = submodule.Config{ + Name: gitsub.Name, + Path: gitsub.Path, + URL: gitsub.URL, + Timeline: timeline, + } + } + + return ivaldimodules +} + +func cloneGitSubmodule(url, path, workDir string) (commitHash string, err error) { + submodulePath := filepath.Join(workDir, path) + + if err := os.MkdirAll(filepath.Dir(submodulePath), 0755); err != nil { + return "", fmt.Errorf("create parent directory: %w", err) + } + + cmd := exec.Command("git", "clone", url, submodulePath) + if output, err := cmd.CombinedOutput(); err != nil { + return "", fmt.Errorf("git clone failed: %w\n%s", err, output) + } + + return getGitSubmoduleCommit(submodulePath) +} + +func getGitSubmoduleCommit(submodulePath string) (string, error) { + cmd := exec.Command("git", "rev-parse", "HEAD") + cmd.Dir = submodulePath + + output, err := cmd.Output() + if err != nil { + return "", fmt.Errorf("get git commit: %w", err) + } + + return strings.TrimSpace(string(output)), nil +} + +func initializeIvaldiInSubmodule(submodulePath, ivaldiDir string, recursive bool) error { + if err := os.MkdirAll(ivaldiDir, 0755); err != nil { + return fmt.Errorf("create .ivaldi dir: %w", err) + } + + gitDir := filepath.Join(submodulePath, ".git") + convResult, err := ConvertGitObjectsToIvaldiConcurrent(gitDir, ivaldiDir, 8) + if err != nil { + return fmt.Errorf("convert git objects: %w", err) + } + + log.Printf(" Converted %d Git objects in submodule", convResult.Converted) + + if recursive { + subGitmodules := filepath.Join(submodulePath, ".gitmodules") + if _, err := os.Stat(subGitmodules); err == nil { + log.Printf(" Detecting nested submodules in %s", submodulePath) + _, err := ConvertGitSubmodulesToIvaldi(gitDir, ivaldiDir, submodulePath, true) + if err != nil { + log.Printf(" Warning: nested submodule conversion: %v", err) + } + } + } + + return nil +} diff --git a/internal/fsmerkle/types.go b/internal/fsmerkle/types.go index b7d3944..d0392fe 100644 --- a/internal/fsmerkle/types.go +++ b/internal/fsmerkle/types.go @@ -28,8 +28,9 @@ type Hash = [32]byte type Kind uint8 const ( - KindBlob Kind = 1 // Regular file content - KindTree Kind = 2 // Directory + KindBlob Kind = 1 // Regular file content + KindTree Kind = 2 // Directory + KindSubmodule Kind = 3 // Submodule reference ) // String returns a human-readable representation of the Kind. @@ -39,6 +40,8 @@ func (k Kind) String() string { return "blob" case KindTree: return "tree" + case KindSubmodule: + return "submodule" default: return fmt.Sprintf("unknown(%d)", k) } @@ -65,6 +68,18 @@ type TreeNode struct { Entries []Entry // Lexicographically sorted by Name, no duplicates } +// SubmoduleNode represents a reference to an external repository. +// Submodules are tracked by BLAKE3 commit hash internally, with optional +// Git SHA-1 mapping for GitHub compatibility. +type SubmoduleNode struct { + URL string // Repository URL (https, ssh, file) + Path string // Relative path in parent repository + Timeline string // Timeline name to track + CommitHash Hash // BLAKE3 hash of target commit + Shallow bool // Shallow clone flag + Freeze bool // Prevent automatic updates +} + // Canonical encodings for hashing and CAS storage: // // Blob canonical bytes: @@ -127,11 +142,11 @@ func (b *BlobNode) Hash(content []byte) Hash { if len(content) != b.Size { panic(fmt.Sprintf("content size mismatch: expected %d, got %d", b.Size, len(content))) } - + var buf bytes.Buffer buf.Write(b.CanonicalBytes()) buf.Write(content) - + return blake3.Sum256(buf.Bytes()) } @@ -141,34 +156,34 @@ func (t *TreeNode) CanonicalBytes() ([]byte, error) { if err := t.validate(); err != nil { return nil, fmt.Errorf("invalid tree: %w", err) } - + var buf bytes.Buffer - + // Write entry count count := make([]byte, binary.MaxVarintLen64) n := binary.PutUvarint(count, uint64(len(t.Entries))) buf.Write(count[:n]) - + // Write each entry for _, entry := range t.Entries { // Write mode mode := make([]byte, binary.MaxVarintLen64) n := binary.PutUvarint(mode, uint64(entry.Mode)) buf.Write(mode[:n]) - + // Write name length and name nameLen := make([]byte, binary.MaxVarintLen64) n = binary.PutUvarint(nameLen, uint64(len(entry.Name))) buf.Write(nameLen[:n]) buf.WriteString(entry.Name) - + // Write kind buf.WriteByte(byte(entry.Kind)) - + // Write hash buf.Write(entry.Hash[:]) } - + return buf.Bytes(), nil } @@ -184,30 +199,30 @@ func (t *TreeNode) Hash() (Hash, error) { // validate checks that the TreeNode is well-formed. func (t *TreeNode) validate() error { names := make(map[string]bool, len(t.Entries)) - + for i, entry := range t.Entries { // Validate name if err := validateName(entry.Name); err != nil { return fmt.Errorf("entry %d: %w", i, err) } - + // Check for duplicates if names[entry.Name] { return fmt.Errorf("duplicate name: %q", entry.Name) } names[entry.Name] = true - + // Validate mode if err := validateMode(entry.Mode, entry.Kind); err != nil { return fmt.Errorf("entry %d (%s): %w", i, entry.Name, err) } - + // Check sorting (entries must be in lexicographic order) if i > 0 && entry.Name <= t.Entries[i-1].Name { return fmt.Errorf("entries not sorted: %q should come before %q", t.Entries[i-1].Name, entry.Name) } } - + return nil } @@ -236,7 +251,7 @@ func splitPath(filepath string) (dir, name string) { if filepath == "." { return "", "" } - + dir, name = path.Split(filepath) if dir != "" && dir != "/" { dir = strings.TrimSuffix(dir, "/") @@ -244,6 +259,120 @@ func splitPath(filepath string) (dir, name string) { if dir == "/" { dir = "" } - + return dir, name -} \ No newline at end of file +} + +// CanonicalBytes returns the canonical byte representation of a SubmoduleNode. +// Format (Version 1): +// Version byte: 0x01 +// URL length: uvarint(len(url)) +// URL bytes: UTF-8 string +// Path length: uvarint(len(path)) +// Path bytes: UTF-8 string +// Timeline length: uvarint(len(timeline)) +// Timeline bytes: UTF-8 string +// Commit hash: 32 bytes (BLAKE3) +// Flags: uvarint(flags) +// bit 0: shallow +// bit 1: freeze +// bits 2-63: reserved +func (s *SubmoduleNode) CanonicalBytes() []byte { + var buf bytes.Buffer + + buf.WriteByte(0x01) + + urlBytes := []byte(s.URL) + lenBuf := make([]byte, binary.MaxVarintLen64) + n := binary.PutUvarint(lenBuf, uint64(len(urlBytes))) + buf.Write(lenBuf[:n]) + buf.Write(urlBytes) + + pathBytes := []byte(s.Path) + n = binary.PutUvarint(lenBuf, uint64(len(pathBytes))) + buf.Write(lenBuf[:n]) + buf.Write(pathBytes) + + timelineBytes := []byte(s.Timeline) + n = binary.PutUvarint(lenBuf, uint64(len(timelineBytes))) + buf.Write(lenBuf[:n]) + buf.Write(timelineBytes) + + buf.Write(s.CommitHash[:]) + + var flags uint64 + if s.Shallow { + flags |= 1 + } + if s.Freeze { + flags |= 2 + } + n = binary.PutUvarint(lenBuf, flags) + buf.Write(lenBuf[:n]) + + return buf.Bytes() +} + +// Hash computes the BLAKE3 hash of the submodule's canonical representation. +func (s *SubmoduleNode) Hash() Hash { + return blake3.Sum256(s.CanonicalBytes()) +} + +// DecodeSubmoduleNode decodes a SubmoduleNode from its canonical bytes. +func DecodeSubmoduleNode(data []byte) (*SubmoduleNode, error) { + buf := bytes.NewReader(data) + + version, err := buf.ReadByte() + if err != nil { + return nil, fmt.Errorf("read version: %w", err) + } + if version != 0x01 { + return nil, fmt.Errorf("unsupported submodule version: %d", version) + } + + readString := func() (string, error) { + length, err := binary.ReadUvarint(buf) + if err != nil { + return "", err + } + strBytes := make([]byte, length) + if _, err := buf.Read(strBytes); err != nil { + return "", err + } + return string(strBytes), nil + } + + url, err := readString() + if err != nil { + return nil, fmt.Errorf("read URL: %w", err) + } + + pathStr, err := readString() + if err != nil { + return nil, fmt.Errorf("read path: %w", err) + } + + timeline, err := readString() + if err != nil { + return nil, fmt.Errorf("read timeline: %w", err) + } + + var commitHash Hash + if _, err := buf.Read(commitHash[:]); err != nil { + return nil, fmt.Errorf("read commit hash: %w", err) + } + + flags, err := binary.ReadUvarint(buf) + if err != nil { + return nil, fmt.Errorf("read flags: %w", err) + } + + return &SubmoduleNode{ + URL: url, + Path: pathStr, + Timeline: timeline, + CommitHash: commitHash, + Shallow: (flags & 1) != 0, + Freeze: (flags & 2) != 0, + }, nil +} diff --git a/internal/hamtdir/hamtdir.go b/internal/hamtdir/hamtdir.go index 6dd8b87..d7e5597 100644 --- a/internal/hamtdir/hamtdir.go +++ b/internal/hamtdir/hamtdir.go @@ -30,20 +30,29 @@ type EntryType uint8 const ( FileEntry EntryType = iota + 1 DirEntry + SubmoduleEntry ) // Entry represents a single directory entry. type Entry struct { - Name string - Type EntryType - File *filechunk.NodeRef // Set if Type == FileEntry - Dir *DirRef // Set if Type == DirEntry + Name string + Type EntryType + File *filechunk.NodeRef // Set if Type == FileEntry + Dir *DirRef // Set if Type == DirEntry + Submodule *SubmoduleRef // Set if Type == SubmoduleEntry +} + +// SubmoduleRef represents a reference to a submodule. +type SubmoduleRef struct { + NodeHash cas.Hash // BLAKE3 hash of SubmoduleNode object + CommitHash cas.Hash // BLAKE3 hash of target commit (denormalized) + URL string // Repository URL (denormalized for display) } // DirRef represents a reference to a directory HAMT. type DirRef struct { Hash cas.Hash // BLAKE3 hash of the directory - Size int // Number of entries in the directory tree + Size int // Number of entries in the directory tree } // Node represents a HAMT node. @@ -53,8 +62,8 @@ type Node struct { Entries []Entry // Only for leaf nodes // For internal nodes - Bitmap uint32 // 32-bit bitmap indicating which children exist - Children map[int]cas.Hash // Map from bit position to child hash + Bitmap uint32 // 32-bit bitmap indicating which children exist + Children map[int]cas.Hash // Map from bit position to child hash } // Builder constructs directory HAMTs. @@ -159,16 +168,16 @@ func (b *Builder) buildLeaf(entries []Entry) (DirRef, error) { // hashChunk extracts a 5-bit chunk from the hash of a name at given depth. func (b *Builder) hashChunk(name string, depth int) uint32 { hash := cas.SumB3([]byte(name)) - + // Extract 5 bits starting at bit position depth*5 bitOffset := depth * 5 byteOffset := bitOffset / 8 bitWithinByte := bitOffset % 8 - + if byteOffset >= len(hash) { return 0 } - + // Extract up to 16 bits to handle cross-byte boundaries var bits uint16 if byteOffset+1 < len(hash) { @@ -176,7 +185,7 @@ func (b *Builder) hashChunk(name string, depth int) uint32 { } else { bits = uint16(hash[byteOffset]) } - + // Shift and mask to get our 5-bit chunk chunk := (bits >> bitWithinByte) & 0x1F // 0x1F = 31 = 2^5-1 return uint32(chunk) @@ -371,22 +380,22 @@ func (l *Loader) listNode(nodeHash cas.Hash) ([]Entry, error) { // hashChunk extracts a 5-bit chunk from the hash (same as Builder). func (l *Loader) hashChunk(name string, depth int) uint32 { hash := cas.SumB3([]byte(name)) - + bitOffset := depth * 5 byteOffset := bitOffset / 8 bitWithinByte := bitOffset % 8 - + if byteOffset >= len(hash) { return 0 } - + var bits uint16 if byteOffset+1 < len(hash) { bits = uint16(hash[byteOffset]) | (uint16(hash[byteOffset+1]) << 8) } else { bits = uint16(hash[byteOffset]) } - + chunk := (bits >> bitWithinByte) & 0x1F return uint32(chunk) } @@ -402,7 +411,7 @@ func (l *Loader) decodeNode(data []byte) (*Node, error) { } else if data[0] == 0x01 { return l.decodeInternal(data) } - + return nil, fmt.Errorf("invalid node encoding: unknown marker %02x", data[0]) } @@ -417,7 +426,7 @@ func (l *Loader) decodeLeaf(data []byte) (*Node, error) { } entries := make([]Entry, 0, entryCount) - + for i := uint64(0); i < entryCount; i++ { // Read key length and key keyLen, err := binary.ReadUvarint(buf) @@ -559,12 +568,12 @@ func (l *Loader) walkNode(nodeHash cas.Hash, pathPrefix string, walkFn func(stri if pathPrefix != "" { fullPath = pathPrefix + "/" + entry.Name } - + err := walkFn(fullPath, entry) if err != nil { return err } - + // If it's a directory, recurse into it if entry.Type == DirEntry && entry.Dir != nil { err = l.walkNode(entry.Dir.Hash, fullPath, walkFn) @@ -596,16 +605,16 @@ func (l *Loader) PathLookup(root DirRef, path string) (*Entry, error) { if path == "" || path == "/" { return nil, fmt.Errorf("invalid path") } - + // Clean and split path path = strings.Trim(path, "/") if path == "" { return nil, fmt.Errorf("empty path") } - + components := strings.Split(path, "/") currentDir := root - + for i, component := range components { entry, err := l.Lookup(currentDir, component) if err != nil { @@ -614,19 +623,19 @@ func (l *Loader) PathLookup(root DirRef, path string) (*Entry, error) { if entry == nil { return nil, nil // Not found } - + // If this is the last component, return it if i == len(components)-1 { return entry, nil } - + // Must be a directory to continue if entry.Type != DirEntry || entry.Dir == nil { return nil, fmt.Errorf("path component '%s' is not a directory", component) } - + currentDir = *entry.Dir } - + return nil, fmt.Errorf("path traversal failed") -} \ No newline at end of file +} diff --git a/internal/submodule/config.go b/internal/submodule/config.go new file mode 100644 index 0000000..753896f --- /dev/null +++ b/internal/submodule/config.go @@ -0,0 +1,220 @@ +package submodule + +import ( + "bufio" + "bytes" + "encoding/hex" + "fmt" + "os" + "path/filepath" + "strconv" + "strings" +) + +func ParseIvaldimodules(path string) ([]Config, error) { + file, err := os.Open(path) + if err != nil { + if os.IsNotExist(err) { + return nil, nil + } + return nil, fmt.Errorf("open .ivaldimodules: %w", err) + } + defer file.Close() + + var configs []Config + var current *Config + + scanner := bufio.NewScanner(file) + lineNum := 0 + + for scanner.Scan() { + lineNum++ + line := strings.TrimSpace(scanner.Text()) + + if line == "" || strings.HasPrefix(line, "#") { + continue + } + + if strings.HasPrefix(line, "[submodule") { + if current != nil { + if err := validateConfig(current); err != nil { + return nil, fmt.Errorf("line %d: invalid config: %w", lineNum, err) + } + configs = append(configs, *current) + } + + start := strings.Index(line, "\"") + end := strings.LastIndex(line, "\"") + if start == -1 || end == -1 || end <= start { + return nil, fmt.Errorf("line %d: invalid submodule section", lineNum) + } + + name := line[start+1 : end] + current = &Config{Name: name} + continue + } + + if current == nil { + continue + } + + parts := strings.SplitN(line, "=", 2) + if len(parts) != 2 { + continue + } + + key := strings.TrimSpace(parts[0]) + value := strings.TrimSpace(parts[1]) + + switch key { + case "path": + current.Path = value + case "url": + current.URL = value + case "timeline": + current.Timeline = value + case "commit": + current.Commit = value + case "git-commit": + current.GitCommit = value + case "shallow": + current.Shallow = value == "true" + case "freeze": + current.Freeze = value == "true" + case "ignore": + current.Ignore = value + } + } + + if current != nil { + if err := validateConfig(current); err != nil { + return nil, fmt.Errorf("line %d: invalid config: %w", lineNum, err) + } + configs = append(configs, *current) + } + + return configs, scanner.Err() +} + +func WriteIvaldimodules(path string, configs []Config) error { + var buf bytes.Buffer + + buf.WriteString("# .ivaldimodules - Ivaldi Submodule Configuration\n") + buf.WriteString("# Version: 1\n\n") + + for _, cfg := range configs { + buf.WriteString(fmt.Sprintf("[submodule \"%s\"]\n", cfg.Name)) + buf.WriteString(fmt.Sprintf("\tpath = %s\n", cfg.Path)) + buf.WriteString(fmt.Sprintf("\turl = %s\n", cfg.URL)) + + if cfg.Timeline != "" { + buf.WriteString(fmt.Sprintf("\ttimeline = %s\n", cfg.Timeline)) + } + + if cfg.Commit != "" { + buf.WriteString(fmt.Sprintf("\tcommit = %s\n", cfg.Commit)) + } + + if cfg.GitCommit != "" { + buf.WriteString(fmt.Sprintf("\tgit-commit = %s\n", cfg.GitCommit)) + } + + if cfg.Shallow { + buf.WriteString("\tshallow = true\n") + } + + if cfg.Freeze { + buf.WriteString("\tfreeze = true\n") + } + + if cfg.Ignore != "" { + buf.WriteString(fmt.Sprintf("\tignore = %s\n", cfg.Ignore)) + } + + buf.WriteByte('\n') + } + + return os.WriteFile(path, buf.Bytes(), 0644) +} + +func validateConfig(cfg *Config) error { + if cfg.Path == "" { + return fmt.Errorf("missing path for submodule %s", cfg.Name) + } + + if strings.Contains(cfg.Path, "..") { + return fmt.Errorf("invalid path (contains ..): %s", cfg.Path) + } + + if filepath.IsAbs(cfg.Path) { + return fmt.Errorf("path must be relative: %s", cfg.Path) + } + + if cfg.URL == "" { + return fmt.Errorf("missing URL for submodule %s", cfg.Name) + } + + if !strings.HasPrefix(cfg.URL, "https://") && + !strings.HasPrefix(cfg.URL, "ssh://") && + !strings.HasPrefix(cfg.URL, "file://") && + !strings.HasPrefix(cfg.URL, "git@") { + return fmt.Errorf("invalid URL protocol: %s", cfg.URL) + } + + if cfg.Commit != "" && len(cfg.Commit) != 64 { + return fmt.Errorf("invalid commit hash (expected 64 hex chars): %s", cfg.Commit) + } + + if cfg.GitCommit != "" && len(cfg.GitCommit) != 40 { + return fmt.Errorf("invalid git-commit hash (expected 40 hex chars): %s", cfg.GitCommit) + } + + return nil +} + +func ConfigToSubmodule(cfg Config) (*Submodule, error) { + sub := &Submodule{ + Name: cfg.Name, + Path: cfg.Path, + URL: cfg.URL, + Timeline: cfg.Timeline, + GitCommit: cfg.GitCommit, + Shallow: cfg.Shallow, + Freeze: cfg.Freeze, + } + + if cfg.Commit != "" { + hashBytes, err := hex.DecodeString(cfg.Commit) + if err != nil { + return nil, fmt.Errorf("decode commit hash: %w", err) + } + if len(hashBytes) != 32 { + return nil, fmt.Errorf("invalid hash length: %d", len(hashBytes)) + } + copy(sub.Commit[:], hashBytes) + } + + if sub.Timeline == "" { + sub.Timeline = "main" + } + + return sub, nil +} + +func SubmoduleToConfig(sub *Submodule) Config { + return Config{ + Name: sub.Name, + Path: sub.Path, + URL: sub.URL, + Timeline: sub.Timeline, + Commit: hex.EncodeToString(sub.Commit[:]), + GitCommit: sub.GitCommit, + Shallow: sub.Shallow, + Freeze: sub.Freeze, + } +} + +func ParseBool(s string) bool { + b, _ := strconv.ParseBool(s) + return b +} diff --git a/internal/submodule/types.go b/internal/submodule/types.go new file mode 100644 index 0000000..921cbea --- /dev/null +++ b/internal/submodule/types.go @@ -0,0 +1,56 @@ +package submodule + +import ( + "time" + + "github.com/javanhut/Ivaldi-vcs/internal/cas" + bolt "go.etcd.io/bbolt" +) + +type Submodule struct { + Name string + Path string + URL string + Timeline string + Commit cas.Hash + GitCommit string + Shallow bool + Freeze bool +} + +type TimelineSubmoduleState struct { + TimelineName string + SubmodulePath string + CommitHash cas.Hash + GitCommitSHA1 string + LocalTimeline string + Modified bool + LastUpdate time.Time +} + +type Manager struct { + IvaldiDir string + WorkDir string + DB *bolt.DB +} + +type SubmoduleStatus struct { + Path string + CurrentCommit cas.Hash + ExpectedCommit cas.Hash + HasChanges bool + Timeline string + NeedsUpdate bool +} + +type Config struct { + Name string + Path string + URL string + Timeline string + Commit string + GitCommit string + Shallow bool + Freeze bool + Ignore string +}