diff --git a/configs/examples/snapshot.yml b/configs/examples/snapshot.yml index b5b73dc..d802945 100644 --- a/configs/examples/snapshot.yml +++ b/configs/examples/snapshot.yml @@ -1,15 +1,10 @@ -name: Two-Tier Snapshot Functionality Test +name: Snapshot Functionality Test description: | - IMPORTANT: This feature is currently in development, and the results are not yet reliable. + Snapshot Functionality Test - Tests client snapshot creation and loading capabilities to validate state snapshot performance for fast sync operations. - Two-Tier Snapshot Functionality Test - Demonstrates the new optimized snapshot system with initial snapshots and per-test copying. + This benchmark suite tests snapshot functionality with both Sepolia Alpha and development network data, including snapshot creation, loading, and validation processes. Features skip-if-nonempty optimization for development efficiency and tests multiple gas limit configurations. - This benchmark suite uses a two-tier snapshot approach: - 1. Initial snapshots are downloaded once at benchmark startup and stored persistently - 2. Per-test snapshots are copied from initial snapshots for each test run using rsync - 3. Test-specific copies are cleaned up after each test while preserving initial snapshots - - Use Case: Optimized snapshot performance for fast test execution, reduced network overhead, and efficient storage management across multiple node types. + Use Case: Validate state snapshot performance for fast sync operations, test snapshot creation and loading capabilities across different environments, and ensure snapshot performance remains consistent in development workflows. payloads: - name: Transfer-only @@ -17,24 +12,33 @@ payloads: type: transfer-only benchmarks: - - initial_snapshots: - - node_type: reth - # Download an initial reth snapshot that can be copied for each test - command: ./scripts/setup-base-snapshot.sh --network=sepolia --node-type=reth --destination=/data/snapshots/reth/initial - destination: /data/snapshots/reth/initial - superchain_chain_id: 84532 - - node_type: geth - # Download an initial geth snapshot that can be copied for each test - command: ./scripts/setup-base-snapshot.sh --network=sepolia --node-type=geth --destination=/data/snapshots/geth/initial - destination: /data/snapshots/geth/initial - superchain_chain_id: 84532 + - snapshot: + # skip non-empty for testing so we don't copy every time we run this + # just delete the snapshot directory to force a full copy + command: ./scripts/copy-local-snapshot.sh --skip-if-nonempty + genesis_file: ../../sepolia-alpha/sepolia-alpha-genesis.json + # force_clean is true by default to ensure consistency, but we can skip it for testing + force_clean: false variables: - type: payload value: transfer-only - type: node_type values: - reth - - geth + - type: num_blocks + value: 10 + - type: gas_limit + values: + - 15000000 + - 30000000 + - 60000000 + - 90000000 + - variables: + - type: payload + value: transfer-only + - type: node_type + values: + - reth - type: num_blocks value: 10 - type: gas_limit diff --git a/runner/benchmark/definition.go b/runner/benchmark/definition.go index d720996..767705e 100644 --- a/runner/benchmark/definition.go +++ b/runner/benchmark/definition.go @@ -2,6 +2,11 @@ package benchmark import ( "errors" + "fmt" + "os" + "os/exec" + "path" + "strings" "github.com/base/base-bench/runner/payload" ) @@ -31,6 +36,54 @@ type ProofProgramOptions struct { Type string `yaml:"type"` } +// SnapshotDefinition is the user-facing YAML configuration for specifying +// a snapshot to be restored before running a benchmark. +type SnapshotDefinition struct { + Command string `yaml:"command"` + GenesisFile *string `yaml:"genesis_file"` + SuperchainChainID *uint64 `yaml:"superchain_chain_id"` + ForceClean *bool `yaml:"force_clean"` +} + +// CreateSnapshot copies the snapshot to the output directory for the given +// node type. +func (s SnapshotDefinition) CreateSnapshot(nodeType string, outputDir string) error { + // default to true if not set + forceClean := s.ForceClean == nil || *s.ForceClean + if _, err := os.Stat(outputDir); err == nil && forceClean { + // TODO: we could reuse it here potentially + if err := os.RemoveAll(outputDir); err != nil { + return fmt.Errorf("failed to remove existing snapshot: %w", err) + } + } + + // get absolute path of outputDir + currentDir, err := os.Getwd() + if err != nil { + return fmt.Errorf("failed to get absolute path of outputDir: %w", err) + } + + outputDir = path.Join(currentDir, outputDir) + + var cmdBin string + var args []string + // split out default args from command + parts := strings.SplitN(s.Command, " ", 2) + if len(parts) < 2 { + cmdBin = parts[0] + } else { + cmdBin = parts[0] + args = strings.Split(parts[1], " ") + } + + args = append(args, nodeType, outputDir) + + cmd := exec.Command(cmdBin, args...) + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + return cmd.Run() +} + type BenchmarkConfig struct { Name string `yaml:"name"` Description *string `yaml:"description"` @@ -41,11 +94,11 @@ type BenchmarkConfig struct { // TestDefinition is the user-facing YAML configuration for specifying a // matrix of benchmark runs. type TestDefinition struct { - InitialSnapshots []SnapshotDefinition `yaml:"initial_snapshots"` - Metrics *ThresholdConfig `yaml:"metrics"` - Tags *map[string]string `yaml:"tags"` - Variables []Param `yaml:"variables"` - ProofProgram *ProofProgramOptions `yaml:"proof_program"` + Snapshot *SnapshotDefinition `yaml:"snapshot"` + Metrics *ThresholdConfig `yaml:"metrics"` + Tags *map[string]string `yaml:"tags"` + Variables []Param `yaml:"variables"` + ProofProgram *ProofProgramOptions `yaml:"proof_program"` } func (bc *TestDefinition) Check() error { diff --git a/runner/benchmark/matrix.go b/runner/benchmark/matrix.go index d8a80fc..769d447 100644 --- a/runner/benchmark/matrix.go +++ b/runner/benchmark/matrix.go @@ -12,10 +12,10 @@ type ThresholdConfig struct { // TestPlan represents a list of test runs to be executed. type TestPlan struct { - Runs []TestRun - InitialSnapshots []SnapshotDefinition - ProofProgram *ProofProgramOptions - Thresholds *ThresholdConfig + Runs []TestRun + Snapshot *SnapshotDefinition + ProofProgram *ProofProgramOptions + Thresholds *ThresholdConfig } func NewTestPlanFromConfig(c TestDefinition, testFileName string, config *BenchmarkConfig) (*TestPlan, error) { @@ -36,24 +36,13 @@ func NewTestPlanFromConfig(c TestDefinition, testFileName string, config *Benchm } return &TestPlan{ - Runs: testRuns, - InitialSnapshots: c.InitialSnapshots, - ProofProgram: proofProgram, - Thresholds: c.Metrics, + Runs: testRuns, + Snapshot: c.Snapshot, + ProofProgram: proofProgram, + Thresholds: c.Metrics, }, nil } -// GetInitialSnapshotForNodeType returns the initial snapshot definition for the given node type. -// Returns nil if no initial snapshot is found for the node type. -func (tp *TestPlan) GetInitialSnapshotForNodeType(nodeType string) *SnapshotDefinition { - for _, snapshot := range tp.InitialSnapshots { - if snapshot.NodeType == nodeType { - return &snapshot - } - } - return nil -} - // ResolveTestRunsFromMatrix constructs a new ParamsMatrix from a config. func ResolveTestRunsFromMatrix(c TestDefinition, testFileName string, config *BenchmarkConfig) ([]TestRun, error) { seenParams := make(map[string]bool) @@ -128,7 +117,7 @@ func ResolveTestRunsFromMatrix(c TestDefinition, testFileName string, config *Be testParams[i] = TestRun{ ID: id, Params: *params, - OutputDir: fmt.Sprintf("%s/%s-%d", id, id, i), + OutputDir: fmt.Sprintf("%s-%d", id, i), Name: params.Name, Description: params.Description, TestFile: testFileName, diff --git a/runner/benchmark/snapshots.go b/runner/benchmark/snapshots.go index 4c0f9d6..4073d8f 100644 --- a/runner/benchmark/snapshots.go +++ b/runner/benchmark/snapshots.go @@ -3,103 +3,12 @@ package benchmark import ( "crypto/sha256" "fmt" - "os" - "os/exec" - "path" "path/filepath" - "strings" ) -const ( - // SnapshotMethodChainCopy performs a full copy of the snapshot data (default behavior) - SnapshotMethodChainCopy = "chain_copy" - // SnapshotMethodHeadRollback uses debug.setHead to rollback to a specific block - SnapshotMethodHeadRollback = "head_rollback" - // SnapshotMethodReuseExisting continues using the same snapshot without modifications - SnapshotMethodReuseExisting = "reuse_existing" -) - -// SnapshotDefinition is the user-facing YAML configuration for specifying -// a snapshot to be restored before running a benchmark. -type SnapshotDefinition struct { - NodeType string `yaml:"node_type"` - Command string `yaml:"command"` - Destination *string `yaml:"destination"` - GenesisFile *string `yaml:"genesis_file"` - SuperchainChainID *uint64 `yaml:"superchain_chain_id"` - ForceClean *bool `yaml:"force_clean"` - SnapshotMethod *string `yaml:"snapshot_method"` // "chain_copy" (default), "head_rollback", or "reuse_existing" - RollbackBlock *uint64 `yaml:"rollback_block"` // Block number to rollback to (only used with head_rollback) -} - -// GetSnapshotMethod returns the snapshot method, defaulting to chain_copy if not specified -func (s SnapshotDefinition) GetSnapshotMethod() string { - if s.SnapshotMethod == nil || *s.SnapshotMethod == "" { - return SnapshotMethodChainCopy - } - return *s.SnapshotMethod -} - -// CreateSnapshot copies the snapshot to the output directory for the given -// node type. -func (s SnapshotDefinition) CreateSnapshot(nodeType string, outputDir string) error { - // default to true if not set - forceClean := s.ForceClean == nil || *s.ForceClean - if _, err := os.Stat(outputDir); err == nil && forceClean { - // TODO: we could reuse it here potentially - if err := os.RemoveAll(outputDir); err != nil { - return fmt.Errorf("failed to remove existing snapshot: %w", err) - } - } - - // get absolute path of outputDir - if !filepath.IsAbs(outputDir) { - currentDir, err := os.Getwd() - if err != nil { - return fmt.Errorf("failed to get absolute path of outputDir: %w", err) - } - outputDir = path.Join(currentDir, outputDir) - } - - var cmdBin string - var args []string - // split out default args from command - parts := strings.SplitN(s.Command, " ", 2) - if len(parts) < 2 { - cmdBin = parts[0] - } else { - cmdBin = parts[0] - args = strings.Split(parts[1], " ") - } - - args = append(args, nodeType, outputDir) - - cmd := exec.Command(cmdBin, args...) - cmd.Stdout = os.Stdout - cmd.Stderr = os.Stderr - return cmd.Run() -} - // SnapshotManager is an interface that manages snapshots for different node types // and roles. type SnapshotManager interface { - // EnsureInitialSnapshot ensures that an initial snapshot exists for the given node type. - // If it does not exist, it will create it using the given snapshot definition. - // Returns the path to the initial snapshot. - EnsureInitialSnapshot(definition SnapshotDefinition) (string, error) - - // GetInitialSnapshotPath returns the path to the initial snapshot for the given node type. - // Returns empty string if no initial snapshot exists for the node type. - GetInitialSnapshotPath(nodeType string) string - - // GetInitialSnapshotHeadBlock returns the head block number of the initial snapshot for the given node type. - // Returns 0 if no head block is recorded for the node type. - GetInitialSnapshotHeadBlock(nodeType string) uint64 - - // CopyFromInitialSnapshot copies data from an initial snapshot to a test-specific directory. - // This is used for per-test snapshots that need to be isolated from each other. - CopyFromInitialSnapshot(initialSnapshotPath, testSnapshotPath string) error - // EnsureSnapshot ensures that a snapshot exists for the given node type and // role. If it does not exist, it will create it using the given snapshot // definition. It returns the path to the snapshot. @@ -137,12 +46,6 @@ type benchmarkDatadirState struct { // done. currentDataDirs map[snapshotStoragePath]string - // initialSnapshots tracks the paths to initial snapshots by node type - initialSnapshots map[string]string - - // snapshotHeadBlocks tracks the head block number of initial snapshots by node type - snapshotHeadBlocks map[string]uint64 - // snapshotsDir is the directory where all the snapshots are stored. Each // file will have the format __. snapshotsDir string @@ -150,114 +53,9 @@ type benchmarkDatadirState struct { func NewSnapshotManager(snapshotsDir string) SnapshotManager { return &benchmarkDatadirState{ - currentDataDirs: make(map[snapshotStoragePath]string), - initialSnapshots: make(map[string]string), - snapshotHeadBlocks: make(map[string]uint64), - snapshotsDir: snapshotsDir, - } -} - -func (b *benchmarkDatadirState) EnsureInitialSnapshot(definition SnapshotDefinition) (string, error) { - // Check if we already have this initial snapshot - if path, exists := b.initialSnapshots[definition.NodeType]; exists { - // Validate that the existing snapshot still exists - if _, err := os.Stat(path); err == nil { - return path, nil - } - // If it doesn't exist, remove and recreate the snapshot - delete(b.initialSnapshots, definition.NodeType) - } - - // Use the destination path if provided, otherwise fall back to hashed path - var initialSnapshotPath string - if definition.Destination != nil && *definition.Destination != "" { - initialSnapshotPath = *definition.Destination - } else { - return "", fmt.Errorf("destination path is required for initial snapshots") - } - - // Create the initial snapshot - err := definition.CreateSnapshot(definition.NodeType, initialSnapshotPath) - if err != nil { - return "", fmt.Errorf("failed to create initial snapshot: %w", err) + currentDataDirs: make(map[snapshotStoragePath]string), + snapshotsDir: snapshotsDir, } - - // Validate that the snapshot was actually created - if _, err := os.Stat(initialSnapshotPath); err != nil { - return "", fmt.Errorf("initial snapshot was not created successfully at path %s: %w", initialSnapshotPath, err) - } - - b.initialSnapshots[definition.NodeType] = initialSnapshotPath - return initialSnapshotPath, nil -} - -func (b *benchmarkDatadirState) GetInitialSnapshotPath(nodeType string) string { - if path, exists := b.initialSnapshots[nodeType]; exists { - return path - } - return "" -} - -func (b *benchmarkDatadirState) GetInitialSnapshotHeadBlock(nodeType string) uint64 { - if headBlock, exists := b.snapshotHeadBlocks[nodeType]; exists { - return headBlock - } - return 0 -} - -func (b *benchmarkDatadirState) CopyFromInitialSnapshot(initialSnapshotPath, testSnapshotPath string) error { - // Validate that the initial snapshot path exists - if _, err := os.Stat(initialSnapshotPath); os.IsNotExist(err) { - return fmt.Errorf("initial snapshot path does not exist: %s", initialSnapshotPath) - } else if err != nil { - return fmt.Errorf("failed to check initial snapshot path: %w", err) - } - - // Remove existing test snapshot directory if it exists - if _, err := os.Stat(testSnapshotPath); err == nil { - if err := os.RemoveAll(testSnapshotPath); err != nil { - return fmt.Errorf("failed to remove existing test snapshot: %w", err) - } - } - - // Create the test snapshot directory - if err := os.MkdirAll(testSnapshotPath, 0755); err != nil { - return fmt.Errorf("failed to create test snapshot directory: %w", err) - } - - // Fallback to optimized rsync - if err := b.copyWithOptimizedRsync(initialSnapshotPath, testSnapshotPath); err == nil { - return nil - } - - // Final fallback to standard rsync - return b.copyWithStandardRsync(initialSnapshotPath, testSnapshotPath) -} - -// copyWithOptimizedRsync uses rsync with optimized flags for better performance -func (b *benchmarkDatadirState) copyWithOptimizedRsync(src, dst string) error { - cmd := exec.Command("rsync", "-aHAX", "--numeric-ids", "--inplace", "--no-whole-file", "--info=progress2", src+"/", dst+"/") - cmd.Stdout = os.Stdout - cmd.Stderr = os.Stderr - - if err := cmd.Run(); err != nil { - return fmt.Errorf("failed to copy with optimized rsync: %w", err) - } - - return nil -} - -// copyWithStandardRsync uses the original rsync approach as final fallback -func (b *benchmarkDatadirState) copyWithStandardRsync(src, dst string) error { - cmd := exec.Command("rsync", "-a", src+"/", dst+"/") - cmd.Stdout = os.Stdout - cmd.Stderr = os.Stderr - - if err := cmd.Run(); err != nil { - return fmt.Errorf("failed to copy with standard rsync: %w", err) - } - - return nil } func (b *benchmarkDatadirState) EnsureSnapshot(definition SnapshotDefinition, nodeType string, role string) (string, error) { diff --git a/runner/network/network_benchmark.go b/runner/network/network_benchmark.go index ca38ba8..d20ba74 100644 --- a/runner/network/network_benchmark.go +++ b/runner/network/network_benchmark.go @@ -41,9 +41,6 @@ type NetworkBenchmark struct { transactionPayload payload.Definition ports portmanager.PortManager - - clientVersion string - snapshotConfig *benchmark.SnapshotDefinition } // NewNetworkBenchmark creates a new network benchmark and initializes the payload worker and consensus client @@ -59,11 +56,6 @@ func NewNetworkBenchmark(config *benchtypes.TestConfig, log log.Logger, sequence }, nil } -// SetSnapshotConfig sets the snapshot configuration for head rollback -func (nb *NetworkBenchmark) SetSnapshotConfig(config benchmark.SnapshotDefinition) { - nb.snapshotConfig = &config -} - // Run executes the benchmark test func (nb *NetworkBenchmark) Run(ctx context.Context) error { // Create an L1 chain if needed for fault proof benchmark @@ -91,19 +83,11 @@ func (nb *NetworkBenchmark) Run(ctx context.Context) error { } func (nb *NetworkBenchmark) benchmarkSequencer(ctx context.Context, l1Chain *l1Chain) ([]engine.ExecutableData, uint64, error) { - sequencerClient, clientVersion, err := setupNode(ctx, nb.log, nb.testConfig.Params, nb.sequencerOptions, nb.ports) + sequencerClient, err := setupNode(ctx, nb.log, nb.testConfig.Params, nb.sequencerOptions, nb.ports) if err != nil { return nil, 0, fmt.Errorf("failed to setup sequencer node: %w", err) } - // Store client version from sequencer (both sequencer and validator use same client type) - nb.clientVersion = clientVersion - - // Perform head rollback if configured - if err := nb.performHeadRollbackIfNeeded(ctx, sequencerClient); err != nil { - return nil, 0, fmt.Errorf("failed to perform head rollback on sequencer: %w", err) - } - // Ensure client is stopped even if benchmark fails defer func() { currentHeader, err := sequencerClient.Client().HeaderByNumber(ctx, nil) @@ -135,16 +119,11 @@ func (nb *NetworkBenchmark) benchmarkSequencer(ctx context.Context, l1Chain *l1C } func (nb *NetworkBenchmark) benchmarkValidator(ctx context.Context, payloads []engine.ExecutableData, firstTestBlock uint64, l1Chain *l1Chain) error { - validatorClient, _, err := setupNode(ctx, nb.log, nb.testConfig.Params, nb.validatorOptions, nb.ports) + validatorClient, err := setupNode(ctx, nb.log, nb.testConfig.Params, nb.validatorOptions, nb.ports) if err != nil { return fmt.Errorf("failed to setup validator node: %w", err) } - // Perform head rollback if configured - if err := nb.performHeadRollbackIfNeeded(ctx, validatorClient); err != nil { - return fmt.Errorf("failed to perform head rollback on validator: %w", err) - } - defer func() { currentHeader, err := validatorClient.Client().HeaderByNumber(ctx, nil) if err != nil { @@ -184,13 +163,12 @@ func (nb *NetworkBenchmark) GetResult() (*benchmark.RunResult, error) { ValidatorMetrics: *nb.collectedValidatorMetrics, Success: true, Complete: true, - ClientVersion: nb.clientVersion, }, nil } -func setupNode(ctx context.Context, l log.Logger, params benchtypes.RunParams, options *config.InternalClientOptions, portManager portmanager.PortManager) (types.ExecutionClient, string, error) { +func setupNode(ctx context.Context, l log.Logger, params benchtypes.RunParams, options *config.InternalClientOptions, portManager portmanager.PortManager) (types.ExecutionClient, error) { if options == nil { - return nil, "", errors.New("client options cannot be nil") + return nil, errors.New("client options cannot be nil") } var nodeType clients.Client @@ -202,24 +180,16 @@ func setupNode(ctx context.Context, l log.Logger, params benchtypes.RunParams, o case "rbuilder": nodeType = clients.Rbuilder default: - return nil, "", fmt.Errorf("unsupported node type: %s", params.NodeType) + return nil, fmt.Errorf("unsupported node type: %s", params.NodeType) } clientLogger := l.With("nodeType", params.NodeType) client := clients.NewClient(nodeType, clientLogger, options, portManager) - // Get client version before starting - clientVersion, err := client.GetVersion(ctx) - if err != nil { - l.Warn("Failed to get client version", "error", err) - clientVersion = "unknown" - } - l.Info("Client version detected", "version", clientVersion, "nodeType", params.NodeType) - logPath := path.Join(options.TestDirPath, ExecutionLayerLogFileName) fileWriter, err := os.OpenFile(logPath, os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0644) if err != nil { - return nil, "", fmt.Errorf("failed to open log file at %s: %w", logPath, err) + return nil, fmt.Errorf("failed to open log file at %s: %w", logPath, err) } stdoutLogger := logger.NewMultiWriterCloser(logger.NewLogWriter(clientLogger), fileWriter) @@ -232,42 +202,8 @@ func setupNode(ctx context.Context, l log.Logger, params benchtypes.RunParams, o } if err := client.Run(ctx, runtimeConfig); err != nil { - return nil, "", fmt.Errorf("failed to run execution layer client: %w", err) + return nil, fmt.Errorf("failed to run execution layer client: %w", err) } - return client, clientVersion, nil -} - -// SetupNodeForHeadDetection creates a temporary client for head block detection -func SetupNodeForHeadDetection(ctx context.Context, l log.Logger, nodeType string, options *config.InternalClientOptions, portManager portmanager.PortManager) (types.ExecutionClient, string, error) { - // This is similar to setupNode but simplified for head detection only - return setupNode(ctx, l, benchtypes.RunParams{NodeType: nodeType}, options, portManager) -} - -// performHeadRollbackIfNeeded performs head rollback if the snapshot method is head_rollback -func (nb *NetworkBenchmark) performHeadRollbackIfNeeded(ctx context.Context, client types.ExecutionClient) error { - if nb.snapshotConfig == nil { - return nil - } - - snapshotMethod := nb.snapshotConfig.GetSnapshotMethod() - if snapshotMethod != benchmark.SnapshotMethodHeadRollback { - return nil - } - - // At this point, the rollback block should be set (either by user or auto-detected) - if nb.snapshotConfig.RollbackBlock == nil { - return fmt.Errorf("rollback_block not set for head_rollback method - this should have been auto-detected") - } - - blockNumber := *nb.snapshotConfig.RollbackBlock - nb.log.Info("Performing head rollback", "blockNumber", blockNumber, "nodeType", nb.testConfig.Params.NodeType) - - // Perform the rollback - if err := client.SetHead(ctx, blockNumber); err != nil { - return fmt.Errorf("failed to perform head rollback to block %d: %w", blockNumber, err) - } - - nb.log.Info("Head rollback completed successfully", "blockNumber", blockNumber) - return nil + return client, nil } diff --git a/runner/service.go b/runner/service.go index dcaf325..8fe38eb 100644 --- a/runner/service.go +++ b/runner/service.go @@ -52,7 +52,7 @@ type service struct { func NewService(version string, cfg config.Config, log log.Logger) Service { metadataPath := path.Join(cfg.OutputDir(), "metadata.json") - s := &service{ + return &service{ metadataPath: metadataPath, portState: portmanager.NewPortManager(), dataDirState: benchmark.NewSnapshotManager(path.Join(cfg.DataDir(), "snapshots")), @@ -60,13 +60,6 @@ func NewService(version string, cfg config.Config, log log.Logger) Service { version: version, log: log, } - - return s -} - -func (s *service) fileExists(path string) bool { - _, err := os.Stat(path) - return err == nil } func readBenchmarkConfig(path string) (*benchmark.BenchmarkConfig, error) { @@ -99,6 +92,7 @@ func (s *service) setupInternalDirectories(testDir string, params types.RunParam return nil, errors.Wrap(err, "failed to open chain config file") } + // write chain cfg err = json.NewEncoder(chainCfgFile).Encode(genesis) if err != nil { return nil, errors.Wrap(err, "failed to write chain config") @@ -107,44 +101,13 @@ func (s *service) setupInternalDirectories(testDir string, params types.RunParam var dataDirPath string isSnapshot := snapshot != nil && snapshot.Command != "" if isSnapshot { - dataDirPath = path.Join(testDir, "data") - - initialSnapshotPath := s.dataDirState.GetInitialSnapshotPath(params.NodeType) - - if initialSnapshotPath != "" && s.fileExists(initialSnapshotPath) { - snapshotMethod := snapshot.GetSnapshotMethod() - - switch snapshotMethod { - case benchmark.SnapshotMethodReuseExisting: - dataDirPath = initialSnapshotPath - s.log.Info("Reusing existing snapshot", "snapshotPath", initialSnapshotPath, "method", snapshotMethod) - case benchmark.SnapshotMethodHeadRollback: - // For head_rollback, copy the snapshot but mark it for rollback later - err := s.dataDirState.CopyFromInitialSnapshot(initialSnapshotPath, dataDirPath) - if err != nil { - return nil, errors.Wrap(err, "failed to copy from initial snapshot for head rollback") - } - s.log.Info("Copied from initial snapshot for head rollback", "initialSnapshotPath", initialSnapshotPath, "dataDirPath", dataDirPath, "method", snapshotMethod) - default: - // Default chain_copy behavior - err := s.dataDirState.CopyFromInitialSnapshot(initialSnapshotPath, dataDirPath) - if err != nil { - return nil, errors.Wrap(err, "failed to copy from initial snapshot") - } - s.log.Info("Copied from initial snapshot", "initialSnapshotPath", initialSnapshotPath, "dataDirPath", dataDirPath) - } - } else { - // Fallback to direct snapshot creation - if initialSnapshotPath != "" { - s.log.Warn("Initial snapshot path registered but doesn't exist, falling back to direct snapshot creation", - "path", initialSnapshotPath, "nodeType", params.NodeType) - } - snapshotDir, err := s.dataDirState.EnsureSnapshot(*snapshot, params.NodeType, role) - if err != nil { - return nil, errors.Wrap(err, "failed to ensure snapshot") - } - dataDirPath = snapshotDir + // if we have a snapshot, restore it if needed or reuse from a previous test + snapshotDir, err := s.dataDirState.EnsureSnapshot(*snapshot, params.NodeType, role) + if err != nil { + return nil, errors.Wrap(err, "failed to ensure snapshot") } + + dataDirPath = snapshotDir } else { // if no snapshot, just create a new datadir dataDirPath = path.Join(testDir, "data") @@ -201,6 +164,14 @@ type TestRunMetadata struct { func (s *service) exportOutput(testName string, returnedError error, testDirs *config.InternalClientOptions, testOutputDir string, nodeType string) error { // package up logs from the EL client and write them to the output dir + // outputDir/ + // ├── + // │ ├── result-.json + // │ ├── logs-.gz + // │ ├── metrics-.json + + // create output directory + // copy metrics.json to output dir metricsPath := path.Join(testDirs.MetricsPath, metrics.MetricsFileName) metricsOutputPath := path.Join(testOutputDir, fmt.Sprintf("metrics-%s.json", nodeType)) @@ -349,202 +320,10 @@ func (s *service) setupBlobsDir(workingDir string) error { return nil } -func (s *service) setupInitialSnapshots(testPlans []benchmark.TestPlan) error { - // Collect all unique initial snapshots across all test plans - initialSnapshotsMap := make(map[string]benchmark.SnapshotDefinition) - - for _, testPlan := range testPlans { - for _, snapshot := range testPlan.InitialSnapshots { - // Use node_type as the key to avoid duplicates - initialSnapshotsMap[snapshot.NodeType] = snapshot - } - } - - // Setup each unique initial snapshot - for nodeType, snapshot := range initialSnapshotsMap { - s.log.Info("Setting up initial snapshot from remote source", "node_type", nodeType, "command", snapshot.Command) - _, err := s.dataDirState.EnsureInitialSnapshot(snapshot) - if err != nil { - return errors.Wrap(err, fmt.Sprintf("failed to setup initial snapshot for node type %s", nodeType)) - } - s.log.Info("Initial snapshot setup completed", "node_type", nodeType) - } - - return nil -} - -func (s *service) detectSnapshotHeadBlocks(testPlans []benchmark.TestPlan) error { - // Collect all unique snapshots that use head_rollback method - snapshotsToDetect := make(map[string]benchmark.SnapshotDefinition) - - for _, testPlan := range testPlans { - for _, snapshot := range testPlan.InitialSnapshots { - if snapshot.GetSnapshotMethod() == benchmark.SnapshotMethodHeadRollback && snapshot.RollbackBlock == nil { - // Only detect head block if rollback_block is not specified - snapshotsToDetect[snapshot.NodeType] = snapshot - } - } - } - - if len(snapshotsToDetect) == 0 { - return nil // No head block detection needed - } - - s.log.Info("Detecting head blocks for snapshots", "count", len(snapshotsToDetect)) - - // Detect head block for each snapshot that needs it - for nodeType, snapshot := range snapshotsToDetect { - headBlock, err := s.detectHeadBlockForSnapshot(nodeType, snapshot) - if err != nil { - s.log.Warn("Failed to detect head block for snapshot", "nodeType", nodeType, "error", err) - continue - } - - s.log.Info("Detected head block for snapshot", "nodeType", nodeType, "headBlock", headBlock) - // Store the detected head block back into the snapshot manager - // We'll need to update the test plans to use this detected block - s.updateTestPlansWithDetectedHeadBlock(testPlans, nodeType, headBlock) - } - - return nil -} - -func (s *service) detectHeadBlockForSnapshot(nodeType string, snapshot benchmark.SnapshotDefinition) (uint64, error) { - // Get the snapshot path - snapshotPath := s.dataDirState.GetInitialSnapshotPath(nodeType) - if snapshotPath == "" { - return 0, fmt.Errorf("no initial snapshot path found for node type %s", nodeType) - } - - // Create a temporary working directory for head block detection - tempDir, err := os.MkdirTemp(s.config.DataDir(), fmt.Sprintf("head-detect-%s-*", nodeType)) - if err != nil { - return 0, fmt.Errorf("failed to create temp directory: %w", err) - } - defer func() { - if err := os.RemoveAll(tempDir); err != nil { - s.log.Warn("Failed to remove temp directory", "path", tempDir, "error", err) - } - }() - - // Copy the snapshot to the temp directory - tempSnapshotPath := path.Join(tempDir, "data") - err = s.dataDirState.CopyFromInitialSnapshot(snapshotPath, tempSnapshotPath) - if err != nil { - return 0, fmt.Errorf("failed to copy snapshot for head detection: %w", err) - } - - // Create a temporary client configuration to detect the head block - ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute) - defer cancel() - - // Setup temporary client options - clientOptions := s.config.ClientOptions() - // Set SkipInit for geth since we're using a snapshot - clientOptions.SkipInit = true - - tempOptions := &config.InternalClientOptions{ - ClientOptions: clientOptions, - DataDirPath: tempSnapshotPath, - TestDirPath: tempDir, - } - - // Set the appropriate binary path based on node type - switch nodeType { - case "geth": - tempOptions.GethBin = s.config.ClientOptions().GethBin - case "reth": - tempOptions.RethBin = s.config.ClientOptions().RethBin - case "rbuilder": - tempOptions.RbuilderBin = s.config.ClientOptions().RbuilderBin - } - - // Create JWT secret for the temporary client - var jwtSecret [32]byte - _, err = rand.Read(jwtSecret[:]) - if err != nil { - return 0, fmt.Errorf("failed to generate jwt secret: %w", err) - } - - jwtSecretPath := path.Join(tempDir, "jwt_secret") - jwtSecretFile, err := os.OpenFile(jwtSecretPath, os.O_WRONLY|os.O_CREATE, 0644) - if err != nil { - return 0, fmt.Errorf("failed to create jwt secret file: %w", err) - } - - _, err = jwtSecretFile.Write([]byte(hex.EncodeToString(jwtSecret[:]))) - if err != nil { - if closeErr := jwtSecretFile.Close(); closeErr != nil { - s.log.Warn("Failed to close jwt secret file after write error", "error", closeErr) - } - return 0, fmt.Errorf("failed to write jwt secret: %w", err) - } - if err := jwtSecretFile.Close(); err != nil { - return 0, fmt.Errorf("failed to close jwt secret file: %w", err) - } - - tempOptions.JWTSecretPath = jwtSecretPath - tempOptions.JWTSecret = hex.EncodeToString(jwtSecret[:]) - - // Create a minimal genesis for the temp client - genesis := benchmark.DefaultDevnetGenesis() - chainCfgPath := path.Join(tempDir, "chain.json") - chainCfgFile, err := os.OpenFile(chainCfgPath, os.O_WRONLY|os.O_CREATE, 0644) - if err != nil { - return 0, fmt.Errorf("failed to create chain config file: %w", err) - } - - err = json.NewEncoder(chainCfgFile).Encode(genesis) - if closeErr := chainCfgFile.Close(); closeErr != nil { - s.log.Warn("Failed to close chain config file", "error", closeErr) - } - if err != nil { - return 0, fmt.Errorf("failed to write chain config: %w", err) - } - - tempOptions.ChainCfgPath = chainCfgPath - - // Start the temporary client to query head block - tempClient, _, err := network.SetupNodeForHeadDetection(ctx, s.log, nodeType, tempOptions, s.portState) - if err != nil { - return 0, fmt.Errorf("failed to setup temporary client for head detection: %w", err) - } - defer tempClient.Stop() - - // Query the head block - currentHeader, err := tempClient.Client().HeaderByNumber(ctx, nil) - if err != nil { - return 0, fmt.Errorf("failed to query head block: %w", err) - } - - return currentHeader.Number.Uint64(), nil -} - -func (s *service) updateTestPlansWithDetectedHeadBlock(testPlans []benchmark.TestPlan, nodeType string, headBlock uint64) { - for i := range testPlans { - for j := range testPlans[i].InitialSnapshots { - snapshot := &testPlans[i].InitialSnapshots[j] - if snapshot.NodeType == nodeType && snapshot.GetSnapshotMethod() == benchmark.SnapshotMethodHeadRollback && snapshot.RollbackBlock == nil { - snapshot.RollbackBlock = &headBlock - s.log.Info("Updated snapshot with detected head block", "nodeType", nodeType, "headBlock", headBlock) - } - } - } -} - -func (s *service) runTest(ctx context.Context, params types.RunParams, workingDir string, outputDir string, snapshots []benchmark.SnapshotDefinition, proofConfig *benchmark.ProofProgramOptions, transactionPayload payload.Definition) (*benchmark.RunResult, error) { +func (s *service) runTest(ctx context.Context, params types.RunParams, workingDir string, outputDir string, snapshotConfig *benchmark.SnapshotDefinition, proofConfig *benchmark.ProofProgramOptions, transactionPayload payload.Definition) (*benchmark.RunResult, error) { s.log.Info(fmt.Sprintf("Running benchmark with params: %+v", params)) - // Find the appropriate snapshot for this node type - var snapshotConfig *benchmark.SnapshotDefinition - for _, s := range snapshots { - if s.NodeType == params.NodeType { - snapshotConfig = &s - break - } - } - // get genesis block genesis, err := s.getGenesisForSnapshotConfig(snapshotConfig) if err != nil { @@ -609,11 +388,6 @@ func (s *service) runTest(ctx context.Context, params types.RunParams, workingDi if err != nil { return nil, errors.Wrap(err, "failed to create network benchmark") } - - // Set snapshot config for head rollback if needed - if snapshotConfig != nil { - benchmark.SetSnapshotConfig(*snapshotConfig) - } err = benchmark.Run(ctx) if err != nil { return nil, errors.Wrap(err, "failed to run benchmark") @@ -734,18 +508,6 @@ func (s *service) Run(ctx context.Context) error { } } - // Setup initial snapshots for all test plans before running any tests - err = s.setupInitialSnapshots(testPlans) - if err != nil { - return errors.Wrap(err, "failed to setup initial snapshots") - } - - // Detect head blocks for snapshots that use head_rollback method - err = s.detectSnapshotHeadBlocks(testPlans) - if err != nil { - return errors.Wrap(err, "failed to detect snapshot head blocks") - } - // Create machine info from config var machineInfo *benchmark.MachineInfo s.log.Info("Machine info config values", @@ -804,7 +566,7 @@ outerLoop: return errors.Wrap(err, "failed to create output directory") } - metricSummary, err := s.runTest(ctx, c.Params, s.config.DataDir(), outputDir, testPlan.InitialSnapshots, testPlan.ProofProgram, transactionPayloads[c.Params.PayloadID]) + metricSummary, err := s.runTest(ctx, c.Params, s.config.DataDir(), outputDir, testPlan.Snapshot, testPlan.ProofProgram, transactionPayloads[c.Params.PayloadID]) if err != nil { log.Error("Failed to run test", "err", err) metricSummary = &benchmark.RunResult{ diff --git a/scripts/copy-local-snapshot.sh b/scripts/copy-local-snapshot.sh new file mode 100755 index 0000000..990bfc4 --- /dev/null +++ b/scripts/copy-local-snapshot.sh @@ -0,0 +1,53 @@ +#!/bin/bash + +set -e + +# copy-local-snapshot.sh [--skip-if-nonempty] +# Copies a snapshot to the destination if it does not exist. + +# Usage: ./copy-local-snapshot.sh + +POSITIONAL_ARGS=() +for arg in "$@"; do + case $arg in + --skip-if-nonempty) + SKIP_IF_NONEMPTY=true + shift # Remove --skip-if-nonempty from processing + ;; + *) + POSITIONAL_ARGS+=("$arg") # Save positional argument + ;; + esac +done +set -- "${POSITIONAL_ARGS[@]}" # Restore positional parameters +# Check if the correct number of arguments is provided + +if [ "$#" -ne 2 ]; then + echo "Usage: $0 [--skip-if-nonempty] " + exit 1 +fi + +NODE_TYPE=$1 +DESTINATION=$2 + +# TODO: inject with env var +RETH_SNAPSHOT_LOCATION="/data/snapshots" + +case $NODE_TYPE in +reth) + echo "Copying reth snapshot to $DESTINATION" + + if [[ -f "$DESTINATION/db/mdbx.dat" ]]; then + echo "Destination is not empty, skipping copy." + exit 0 + else + echo "Does not exist, copying snapshot." + mkdir -p "$DESTINATION" + rsync -q -r "$RETH_SNAPSHOT_LOCATION/" "$DESTINATION/" + fi + ;; +*) + echo "Unknown node type: $NODE_TYPE" + exit 1 + ;; +esac diff --git a/scripts/setup-base-snapshot.sh b/scripts/download-snapshot.sh similarity index 74% rename from scripts/setup-base-snapshot.sh rename to scripts/download-snapshot.sh index f310567..88fb77f 100755 --- a/scripts/setup-base-snapshot.sh +++ b/scripts/download-snapshot.sh @@ -2,65 +2,60 @@ set -e -# setup-base-snapshot.sh - Downloads and extracts Base network snapshots +# download-snapshot.sh - Downloads and extracts Base network snapshots # # Downloads the latest snapshot from Base's official snapshot servers and extracts # it to the specified destination directory. Supports both mainnet and testnet (sepolia). # # Requirements: curl, tar, zstd (for .tar.zst files) # -# Usage: ./setup-base-snapshot.sh --network --node-type --destination [--skip-if-nonempty] +# Usage: ./download-snapshot.sh --network [--skip-if-nonempty] # # Networks: mainnet, sepolia (testnet) # Node types: geth (full snapshots), reth (archive snapshots) # # Examples: -# ./setup-base-snapshot.sh --network mainnet --node-type geth --destination ./geth-data -# ./setup-base-snapshot.sh --network sepolia --node-type reth --destination ./reth-data --skip-if-nonempty +# ./download-snapshot.sh --network mainnet geth ./geth-data +# ./download-snapshot.sh --network sepolia --skip-if-nonempty reth ./reth-data POSITIONAL_ARGS=() -for arg in "$@"; do - case $arg in +while [[ $# -gt 0 ]]; do + case $1 in --skip-if-nonempty) SKIP_IF_NONEMPTY=true - shift # Remove --skip-if-nonempty from processing + shift ;; --network) NETWORK=$2 shift 2 ;; - --node-type) - NODE_TYPE=$2 - shift 2 - ;; - --destination) - DESTINATION=$2 - shift 2 - ;; *) - POSITIONAL_ARGS+=("$arg") # Save positional argument + POSITIONAL_ARGS+=("$1") + shift ;; esac done -set -- "${POSITIONAL_ARGS[@]}" # Restore positional parameters -# Check if the correct number of arguments is provided + +# Extract positional arguments +NODE_TYPE="${POSITIONAL_ARGS[0]}" +DESTINATION="${POSITIONAL_ARGS[1]}" if [ -z "$NETWORK" ] || [ -z "$NODE_TYPE" ] || [ -z "$DESTINATION" ]; then echo "Error: Missing required parameters" echo "" - echo "Usage: $0 --network --node-type --destination [--skip-if-nonempty]" + echo "Usage: $0 --network [--skip-if-nonempty] " echo "" echo "Required parameters:" - echo " --network Network to download snapshot for (mainnet, sepolia)" - echo " --node-type Node type (geth, reth)" - echo " --destination Directory to extract snapshot to" + echo " --network Network to download snapshot for (mainnet, sepolia)" + echo " Node type (geth, reth)" + echo " Directory to extract snapshot to" echo "" echo "Optional parameters:" - echo " --skip-if-nonempty Skip download if destination already contains data" + echo " --skip-if-nonempty Skip download if destination already contains data" echo "" echo "Examples:" - echo " $0 --network mainnet --node-type geth --destination ./geth-data" - echo " $0 --network sepolia --node-type reth --destination ./reth-data --skip-if-nonempty" + echo " $0 --network mainnet geth ./geth-data" + echo " $0 --network sepolia --skip-if-nonempty reth ./reth-data" exit 1 fi