diff --git a/Makefile b/Makefile index 651e5f2..ea35ab5 100644 --- a/Makefile +++ b/Makefile @@ -42,6 +42,29 @@ build\:all: GOOS=windows GOARCH=amd64 go build $(LDFLAGS) -o $(BINARY_DIR)/$(BINARY_NAME)-windows-amd64.exe ./$(MAIN_PATH) @$(call success,"Built all") +.PHONY: benchmark +benchmark: build + @echo "Running Benchmarks..." + go test ./benchmark -bench=. \ + -benchmem \ + -count=3 \ + -benchtime=2s \ + -cpu=1,4 \ + -timeout=30m \ + | tee ./benchmark/benchmark_results.txt + @$(call success,"Standard benchmarks complete.") + +benchmark\:full: build + @echo "Running Benchmarks..." + go test ./benchmark -bench=. \ + -benchmem \ + -count=5 \ + -benchtime=5s \ + -cpu=1,6,12 \ + -timeout=45m \ + | tee ./benchmark/benchmark_results_full.txt + @$(call success,"Full benchmarks complete.") + # Run tests .PHONY: test test: diff --git a/README.md b/README.md index c79b795..7ac147a 100644 --- a/README.md +++ b/README.md @@ -245,6 +245,32 @@ seed --format json -f path/to/structure.json - 🌲 Supports standard tree format - 📁 Creates both files and directories +## Benchmarks + +### Overview + +#### Time performance + +| nodes | ascii (ms) | json (ms) | difference | +|-------|------------|-----------|------------| +| 100 | 8.62 | 9.00 | +4.4% | +| 500 | 35.55 | 36.76 | +3.4% | +| 1000 | 64.48 | 66.23 | +2.7% | +| 5000 | 428.16 | 438.29 | +2.4% | + +#### Memory Usage + +| Nodes | ASCII (KB) | JSON (KB) | Difference | +|-------|------------|-----------|------------| +| 100 | 13.89 | 13.95 | +0.4% | +| 500 | 17.09 | 17.14 | +0.3% | +| 1000 | 24.82 | 25.11 | +1.2% | +| 5000 | 235.83 | 235.89 | +0.03% | + +### In Depth + +A more in depth analysis and breakdown of the benchmarks can be found [here](./benchmark/README.md) + ## Contributing Contributions are welcome! Please feel free to submit a Pull Request. For major changes, please open an issue first to discuss what you would like to change. diff --git a/benchmark/README.md b/benchmark/README.md new file mode 100644 index 0000000..e90cbd3 --- /dev/null +++ b/benchmark/README.md @@ -0,0 +1,108 @@ +# Benchmark Results + +This document details the performance characteristics of the tree parsing implementations, comparing ASCII tree and JSON parsing methods across various node counts and input methods. + +The benchmark results are in `./benchmark_results.txt` + +## Environment + +- **OS**: Darwin (macOS) +- **Architecture**: ARM64 +- **CPU**: Apple M3 Pro + +## Methodology + +Benchmarks were conducted using Go's built-in testing framework with the following parameters: + +- 100, 500, 1000, and 5000 nodes (files and dirs) +- 2 second runs +- 3 runs per test +- Single core and quad core +- File and String input +- Metrics measured: + - Time (ns/op) + - Memory allocation (B/op) + - Allocation count (allocs/op) + +## Key Findings + +### Performance Comparison + +#### time performance +| nodes | ascii (ms) | json (ms) | difference | +|-------|------------|-----------|------------| +| 100 | 8.62 | 9.00 | +4.4% | +| 500 | 35.55 | 36.76 | +3.4% | +| 1000 | 64.48 | 66.23 | +2.7% | +| 5000 | 428.16 | 438.29 | +2.4% | + +#### Memory Usage +| Nodes | ASCII (KB) | JSON (KB) | Difference | +|-------|------------|-----------|------------| +| 100 | 13.89 | 13.95 | +0.4% | +| 500 | 17.09 | 17.14 | +0.3% | +| 1000 | 24.82 | 25.11 | +1.2% | +| 5000 | 235.83 | 235.89 | +0.03% | + +### Input Method Comparison (500 nodes) + +- **String Input** + - Time: 35.82ms + - Memory: 40.62KB + - Allocations: 77/op + +- **File Input** + - Time: 35.57ms + - Memory: 16.17KB + - Allocations: 77/op + +## Analysis + +1. **Parsing Performance** + - ASCII tree parsing consistently outperforms JSON parsing + - Performance gap decreases with larger node counts + - Both methods show linear scaling with node count + +2. **Memory Efficiency** + - Both parsers show similar memory patterns + - JSON consistently uses marginally more memory + - Differences in memory usage become negligible at larger node counts + +3. **Input Methods** + - File input shows significant memory advantages (~60% reduction) + - No performance penalty for file-based input + - Consistent allocation patterns across both input methods + +## Practical Implications + +1. **For Performance-Critical Applications** + - ASCII tree parsing offers a slight but consistent performance advantage + - Benefits are most noticeable with smaller node counts + - Consider using file-based input for better memory efficiency + +2. **For Memory-Constrained Systems** + - File input method is strongly recommended + - Both parsing methods show similar memory characteristics + - Memory usage scales linearly with node count + +3. **For General Usage** + - Both methods are viable for typical use cases + - Performance differences are minimal for most applications + - Choose based on your specific needs for data format and interoperability + +## Running the Benchmarks + +To run the benchmarks yourself: + +```bash +# Standard +make benchmark +# More intensive (time consuming) +make benchmark:full +``` +## Notes + +- All benchmarks were run with Go's default settings +- Results may vary based on hardware and system load +- Memory statistics include both heap and stack allocations +- Each benchmark includes multiple runs to account for variance diff --git a/benchmark/benchmark_results.txt b/benchmark/benchmark_results.txt new file mode 100644 index 0000000..f6a7e57 --- /dev/null +++ b/benchmark/benchmark_results.txt @@ -0,0 +1,66 @@ +goos: darwin +goarch: arm64 +pkg: github.com/jpwallace22/seed/benchmark +cpu: Apple M3 Pro +BenchmarkParseASCIITree/100_Nodes 279 8564668 ns/op 13872 B/op 47 allocs/op +BenchmarkParseASCIITree/100_Nodes 276 8583771 ns/op 13873 B/op 47 allocs/op +BenchmarkParseASCIITree/100_Nodes 276 8509018 ns/op 13869 B/op 47 allocs/op +BenchmarkParseASCIITree/100_Nodes-4 274 8841260 ns/op 13971 B/op 47 allocs/op +BenchmarkParseASCIITree/100_Nodes-4 274 8661431 ns/op 13887 B/op 47 allocs/op +BenchmarkParseASCIITree/100_Nodes-4 271 8579817 ns/op 13873 B/op 47 allocs/op +BenchmarkParseASCIITree/500_Nodes 61 35318035 ns/op 17105 B/op 87 allocs/op +BenchmarkParseASCIITree/500_Nodes 62 35815971 ns/op 17051 B/op 87 allocs/op +BenchmarkParseASCIITree/500_Nodes 60 35705947 ns/op 17163 B/op 88 allocs/op +BenchmarkParseASCIITree/500_Nodes-4 62 35331921 ns/op 17048 B/op 87 allocs/op +BenchmarkParseASCIITree/500_Nodes-4 61 35478822 ns/op 17106 B/op 87 allocs/op +BenchmarkParseASCIITree/500_Nodes-4 62 35632753 ns/op 17048 B/op 87 allocs/op +BenchmarkParseASCIITree/1000_Nodes 32 64032953 ns/op 25009 B/op 188 allocs/op +BenchmarkParseASCIITree/1000_Nodes 31 64733102 ns/op 25380 B/op 193 allocs/op +BenchmarkParseASCIITree/1000_Nodes 37 63564145 ns/op 23481 B/op 169 allocs/op +BenchmarkParseASCIITree/1000_Nodes-4 32 64366712 ns/op 25009 B/op 188 allocs/op +BenchmarkParseASCIITree/1000_Nodes-4 32 65188727 ns/op 25009 B/op 188 allocs/op +BenchmarkParseASCIITree/1000_Nodes-4 32 65007410 ns/op 25014 B/op 188 allocs/op +BenchmarkParseASCIITree/5000_Nodes 5 428583600 ns/op 235820 B/op 3482 allocs/op +BenchmarkParseASCIITree/5000_Nodes 5 431194442 ns/op 235820 B/op 3482 allocs/op +BenchmarkParseASCIITree/5000_Nodes 5 425311117 ns/op 235820 B/op 3482 allocs/op +BenchmarkParseASCIITree/5000_Nodes-4 5 423967017 ns/op 235856 B/op 3482 allocs/op +BenchmarkParseASCIITree/5000_Nodes-4 5 427780992 ns/op 235824 B/op 3482 allocs/op +BenchmarkParseASCIITree/5000_Nodes-4 5 432100433 ns/op 235824 B/op 3482 allocs/op +BenchmarkParseJSON/100_Nodes 266 8805398 ns/op 13941 B/op 47 allocs/op +BenchmarkParseJSON/100_Nodes 266 8795410 ns/op 13941 B/op 47 allocs/op +BenchmarkParseJSON/100_Nodes 266 8882962 ns/op 13940 B/op 47 allocs/op +BenchmarkParseJSON/100_Nodes-4 259 8968533 ns/op 13947 B/op 47 allocs/op +BenchmarkParseJSON/100_Nodes-4 264 8830172 ns/op 13942 B/op 47 allocs/op +BenchmarkParseJSON/100_Nodes-4 265 9740763 ns/op 13982 B/op 47 allocs/op +BenchmarkParseJSON/500_Nodes 60 38419985 ns/op 17227 B/op 88 allocs/op +BenchmarkParseJSON/500_Nodes 64 36303060 ns/op 17009 B/op 85 allocs/op +BenchmarkParseJSON/500_Nodes 62 36148765 ns/op 17112 B/op 87 allocs/op +BenchmarkParseJSON/500_Nodes-4 61 36262189 ns/op 17170 B/op 87 allocs/op +BenchmarkParseJSON/500_Nodes-4 61 36725622 ns/op 17173 B/op 87 allocs/op +BenchmarkParseJSON/500_Nodes-4 61 36688264 ns/op 17174 B/op 87 allocs/op +BenchmarkParseJSON/1000_Nodes 36 65342146 ns/op 23813 B/op 172 allocs/op +BenchmarkParseJSON/1000_Nodes 31 66417485 ns/op 25438 B/op 193 allocs/op +BenchmarkParseJSON/1000_Nodes 31 68197324 ns/op 25444 B/op 193 allocs/op +BenchmarkParseJSON/1000_Nodes-4 31 65345148 ns/op 25444 B/op 193 allocs/op +BenchmarkParseJSON/1000_Nodes-4 32 66411648 ns/op 25073 B/op 188 allocs/op +BenchmarkParseJSON/1000_Nodes-4 31 65663575 ns/op 25444 B/op 193 allocs/op +BenchmarkParseJSON/5000_Nodes 5 440192250 ns/op 235888 B/op 3482 allocs/op +BenchmarkParseJSON/5000_Nodes 5 435061700 ns/op 235920 B/op 3482 allocs/op +BenchmarkParseJSON/5000_Nodes 5 432037625 ns/op 235888 B/op 3482 allocs/op +BenchmarkParseJSON/5000_Nodes-4 5 430941733 ns/op 235888 B/op 3482 allocs/op +BenchmarkParseJSON/5000_Nodes-4 5 452172008 ns/op 235891 B/op 3482 allocs/op +BenchmarkParseJSON/5000_Nodes-4 5 439331125 ns/op 235891 B/op 3482 allocs/op +BenchmarkInputMethods/StringInput_-_500_Nodes 61 35929608 ns/op 40638 B/op 77 allocs/op +BenchmarkInputMethods/StringInput_-_500_Nodes 61 35659387 ns/op 40646 B/op 77 allocs/op +BenchmarkInputMethods/StringInput_-_500_Nodes 61 36299533 ns/op 40644 B/op 77 allocs/op +BenchmarkInputMethods/StringInput_-_500_Nodes-4 61 35834596 ns/op 40638 B/op 77 allocs/op +BenchmarkInputMethods/StringInput_-_500_Nodes-4 62 35751481 ns/op 40583 B/op 77 allocs/op +BenchmarkInputMethods/StringInput_-_500_Nodes-4 62 35456389 ns/op 40583 B/op 77 allocs/op +BenchmarkInputMethods/FileInput_-_500_nodes 61 35676591 ns/op 16216 B/op 77 allocs/op +BenchmarkInputMethods/FileInput_-_500_nodes 62 35672579 ns/op 16163 B/op 77 allocs/op +BenchmarkInputMethods/FileInput_-_500_nodes 62 35439092 ns/op 16163 B/op 77 allocs/op +BenchmarkInputMethods/FileInput_-_500_nodes-4 62 35490837 ns/op 16163 B/op 77 allocs/op +BenchmarkInputMethods/FileInput_-_500_nodes-4 62 35664337 ns/op 16165 B/op 77 allocs/op +BenchmarkInputMethods/FileInput_-_500_nodes-4 62 35499835 ns/op 16163 B/op 77 allocs/op +PASS +ok github.com/jpwallace22/seed/benchmark 250.933s diff --git a/benchmark/data_generators.go b/benchmark/data_generators.go new file mode 100644 index 0000000..1cc1a03 --- /dev/null +++ b/benchmark/data_generators.go @@ -0,0 +1,201 @@ +package benchmark + +import "fmt" + +type ProjectSize struct { + numAreas int + numModules int + numComponents int + numFiles int +} + +var ( + SmallSize = ProjectSize{ + numAreas: 2, + numModules: 5, + numComponents: 2, + numFiles: 5, + } // 100 nodes (80 files + 20 dirs) + + MediumSize = ProjectSize{ + numAreas: 2, + numModules: 10, + numComponents: 5, + numFiles: 5, + } // 500 nodes (400 files + 100 dirs) + + LargeSize = ProjectSize{ + numAreas: 3, + numModules: 10, + numComponents: 6, + numFiles: 5, + } // 1000 nodes (810 files + 190 dirs) + + ExtraLargeSize = ProjectSize{ + numAreas: 4, + numModules: 15, + numComponents: 8, + numFiles: 10, + } // 5000 nodes (4320 files + 680 dirs) +) + +// generateTreeStructure creates an ASCII tree with the given parameters +func generateTreeStructure(size ProjectSize) string { + tree := "monorepo\n" + mainAreas := []string{"frontend", "backend", "infrastructure", "tools", "docs"} + mainAreas = mainAreas[:size.numAreas] // Limit areas based on size + + for areaIndex, area := range mainAreas { + isLast := areaIndex == len(mainAreas)-1 + prefix := "├──" + if isLast { + prefix = "└──" + } + + tree += fmt.Sprintf("%s %s\n", prefix, area) + + // Add modules for each area + for i := 0; i < size.numModules; i++ { + subPrefix := "│ ├──" + if isLast { + subPrefix = " ├──" + } + if i == size.numModules-1 { + subPrefix = "│ └──" + if isLast { + subPrefix = " └──" + } + } + + tree += fmt.Sprintf("%s module%d\n", subPrefix, i+1) + + // Add components for each module + for j := 0; j < size.numComponents; j++ { + componentPrefix := "│ │ ├──" + if isLast { + componentPrefix = " │ ├──" + } + if i == size.numModules-1 { + componentPrefix = " │ ├──" + } + if j == size.numComponents-1 { + componentPrefix = "│ │ └──" + if isLast || i == size.numModules-1 { + componentPrefix = " │ └──" + } + } + + tree += fmt.Sprintf("%s component%d\n", componentPrefix, j+1) + + // Add files for each component + for k := 0; k < size.numFiles; k++ { + filePrefix := "│ │ │ ├──" + if isLast || i == size.numModules-1 { + filePrefix = " │ │ ├──" + } + if j == size.numComponents-1 { + filePrefix = "│ │ │ ├──" + if isLast || i == size.numModules-1 { + filePrefix = " │ │ ├──" + } + } + if k == size.numFiles-1 { + filePrefix = "│ │ │ └──" + if isLast || i == size.numModules-1 { + filePrefix = " │ │ └──" + } + } + + extension := getExtensionForArea(areaIndex) + tree += fmt.Sprintf("%s file%d%s\n", filePrefix, k+1, extension) + } + } + } + } + + return tree +} + +// generateJSONStructure creates a JSON structure with the given parameters +func generateJSONStructure(size ProjectSize) []interface{} { + mainAreas := []string{"frontend", "backend", "infrastructure", "tools", "docs"} + mainAreas = mainAreas[:size.numAreas] + + // Calculate total counts for the report + totalDirs := 1 + // root + size.numAreas + // main areas + (size.numAreas * size.numModules) + // modules + (size.numAreas * size.numModules * size.numComponents) // components + + totalFiles := size.numAreas * size.numModules * size.numComponents * size.numFiles + + // Generate structure + areaContents := make([]interface{}, len(mainAreas)) + + for areaIdx, areaName := range mainAreas { + moduleContents := make([]interface{}, size.numModules) + + for moduleIdx := 0; moduleIdx < size.numModules; moduleIdx++ { + componentContents := make([]interface{}, size.numComponents) + + for compIdx := 0; compIdx < size.numComponents; compIdx++ { + fileContents := make([]interface{}, size.numFiles) + + extension := getExtensionForArea(areaIdx) + + for fileIdx := 0; fileIdx < size.numFiles; fileIdx++ { + fileContents[fileIdx] = map[string]interface{}{ + "type": "file", + "name": fmt.Sprintf("file%d%s", fileIdx+1, extension), + } + } + + componentContents[compIdx] = map[string]interface{}{ + "type": "directory", + "name": fmt.Sprintf("component%d", compIdx+1), + "contents": fileContents, + } + } + + moduleContents[moduleIdx] = map[string]interface{}{ + "type": "directory", + "name": fmt.Sprintf("module%d", moduleIdx+1), + "contents": componentContents, + } + } + + areaContents[areaIdx] = map[string]interface{}{ + "type": "directory", + "name": areaName, + "contents": moduleContents, + } + } + + return []interface{}{ + map[string]interface{}{ + "type": "directory", + "name": "monorepo", + "contents": areaContents, + }, + map[string]interface{}{ + "type": "report", + "directories": totalDirs, + "files": totalFiles, + }, + } +} + +func getExtensionForArea(areaIndex int) string { + switch areaIndex { + case 0: + return ".tsx" + case 1: + return ".go" + case 2: + return ".tf" + case 3: + return ".sh" + default: + return ".md" + } +} diff --git a/benchmark/seed_test.go b/benchmark/seed_test.go new file mode 100644 index 0000000..9be962f --- /dev/null +++ b/benchmark/seed_test.go @@ -0,0 +1,198 @@ +package benchmark + +import ( + "encoding/json" + "fmt" + "os" + "os/exec" + "path/filepath" + "runtime" + "testing" +) + +// getBinaryPath returns the path to the seed binary +func getBinaryPath() (string, error) { + // Get the directory where the test file is located + _, filename, _, _ := runtime.Caller(0) + benchmarkDir := filepath.Dir(filename) + + // Navigate up to project root and then to bin/seed + binPath := filepath.Join(benchmarkDir, "..", "bin", "seed") + + // Get absolute path and verify binary exists + absPath, err := filepath.Abs(binPath) + if err != nil { + return "", fmt.Errorf("failed to get absolute path: %v", err) + } + + if _, err := os.Stat(absPath); err != nil { + return "", fmt.Errorf("binary not found at %s: %v", absPath, err) + } + + return absPath, nil +} + +func generateLargeTree() string { + // Generate a large tree structure programmatically + tree := "large-project\n" + for i := 0; i < 10; i++ { + tree += fmt.Sprintf("├── module%d\n", i) + for j := 0; j < 5; j++ { + tree += fmt.Sprintf("│ ├── submodule%d\n", j) + for k := 0; k < 4; k++ { + tree += fmt.Sprintf("│ │ ├── file%d.ts\n", k) + } + } + } + return tree +} + +func TestMain(m *testing.M) { + // Ensure we're in the benchmark directory + if err := os.Chdir(filepath.Dir(os.Args[0])); err != nil { + fmt.Printf("Failed to change directory: %v\n", err) + os.Exit(1) + } + os.Exit(m.Run()) +} + +// Benchmark ASCII tree parsing with different sizes +func BenchmarkParseASCIITree(b *testing.B) { + binaryPath, err := getBinaryPath() + if err != nil { + b.Fatal(err) + } + + tests := []struct { + name string + tree string + }{ + {"100 Nodes", generateTreeStructure(SmallSize)}, + {"500 Nodes", generateTreeStructure(MediumSize)}, + {"1000 Nodes", generateTreeStructure(LargeSize)}, + {"5000 Nodes", generateTreeStructure(ExtraLargeSize)}, + } + + for _, tt := range tests { + b.Run(tt.name, func(b *testing.B) { + tmpDir, err := os.MkdirTemp("", "seed-benchmark-*") + if err != nil { + b.Fatal(err) + } + defer os.RemoveAll(tmpDir) + + tmpFile := filepath.Join(tmpDir, "tree.txt") + if err := os.WriteFile(tmpFile, []byte(tt.tree), 0644); err != nil { + b.Fatal(err) + } + + b.ResetTimer() + for i := 0; i < b.N; i++ { + cmd := exec.Command(binaryPath, "-f", tmpFile) + cmd.Dir = tmpDir + output, err := cmd.CombinedOutput() + if err != nil { + b.Fatalf("command failed: %v\nOutput: %s", err, output) + } + } + }) + } +} + +// Benchmark JSON parsing with different sizes +func BenchmarkParseJSON(b *testing.B) { + binaryPath, err := getBinaryPath() + if err != nil { + b.Fatal(err) + } + + tests := []struct { + name string + json []interface{} + }{ + {"100 Nodes", generateJSONStructure(SmallSize)}, + {"500 Nodes", generateJSONStructure(MediumSize)}, + {"1000 Nodes", generateJSONStructure(LargeSize)}, + {"5000 Nodes", generateJSONStructure(ExtraLargeSize)}, + } + + for _, tt := range tests { + name := tt.name + b.Run(name, func(b *testing.B) { + tmpDir, err := os.MkdirTemp("", "seed-benchmark-*") + if err != nil { + b.Fatal(err) + } + defer os.RemoveAll(tmpDir) + + jsonData, err := json.Marshal(tt.json) + if err != nil { + b.Fatal(err) + } + + tmpFile := filepath.Join(tmpDir, "structure.json") + if err := os.WriteFile(tmpFile, jsonData, 0644); err != nil { + b.Fatal(err) + } + + b.ResetTimer() + for i := 0; i < b.N; i++ { + cmd := exec.Command(binaryPath, "-F", "json", "-f", tmpFile) + cmd.Dir = tmpDir + output, err := cmd.CombinedOutput() + if err != nil { + b.Fatalf("command failed: %v\nOutput: %s", err, string(output)) + } + } + }) + } +} + +// Benchmark string input vs file input +func BenchmarkInputMethods(b *testing.B) { + binaryPath, err := getBinaryPath() + if err != nil { + b.Fatal(err) + } + + tree := generateTreeStructure(MediumSize) + + b.Run("StringInput - 500 Nodes", func(b *testing.B) { + tmpDir, err := os.MkdirTemp("", "seed-benchmark-*") + if err != nil { + b.Fatal(err) + } + defer os.RemoveAll(tmpDir) + + b.ResetTimer() + for i := 0; i < b.N; i++ { + cmd := exec.Command(binaryPath, tree) + cmd.Dir = tmpDir + if err := cmd.Run(); err != nil { + b.Fatal(err) + } + } + }) + + b.Run("FileInput - 500 nodes", func(b *testing.B) { + tmpDir, err := os.MkdirTemp("", "seed-benchmark-*") + if err != nil { + b.Fatal(err) + } + defer os.RemoveAll(tmpDir) + + tmpFile := filepath.Join(tmpDir, "tree.txt") + if err := os.WriteFile(tmpFile, []byte(tree), 0644); err != nil { + b.Fatal(err) + } + + b.ResetTimer() + for i := 0; i < b.N; i++ { + cmd := exec.Command(binaryPath, "-f", tmpFile) + cmd.Dir = tmpDir + if err := cmd.Run(); err != nil { + b.Fatal(err) + } + } + }) +}