diff --git a/README.md b/README.md index fd88e7e..7e902e7 100644 --- a/README.md +++ b/README.md @@ -68,6 +68,7 @@ Key and Value pairs are specified in a specific format documented in the [Tutori | `epcc runbooks show ` | Show a specific runbook (script) | | `epcc runbooks validate` | Validates all runbooks (built in and user supplied, outputting any errors) | | `epcc runbooks run ` | Run a specific runbook (script) | +| `epcc runbooks exec-script ` | Execute a standalone YAML script file containing epcc commands | #### Tuning Runbooks diff --git a/cmd/runbooks.go b/cmd/runbooks.go index 8640695..2877882 100644 --- a/cmd/runbooks.go +++ b/cmd/runbooks.go @@ -3,6 +3,7 @@ package cmd import ( "context" "fmt" + "os" "strconv" "strings" "sync/atomic" @@ -36,6 +37,7 @@ func initRunbookCommands() { runbookGlobalCmd.AddCommand(initRunbookShowCommands()) runbookGlobalCmd.AddCommand(initRunbookRunCommands()) + runbookGlobalCmd.AddCommand(initRunbookRunScriptCmd()) runbookGlobalCmd.AddCommand(initRunbookDevCommands()) runbookGlobalCmd.AddCommand(initRunbookValidateCommands()) } @@ -262,185 +264,230 @@ func initRunbookRunCommands() *cobra.Command { Long: runbookAction.Description.Long, Short: runbookAction.Description.Short, RunE: func(cmd *cobra.Command, args []string) error { - numSteps := len(runbookAction.RawCommands) + return processRunBookCommands(runbook.Name, runbookStringArguments, runbookAction, maxConcurrency, execTimeoutInSeconds) + }, + } + processRunbookVariablesOnCommand(runbookActionRunActionCommand, runbookStringArguments, runbookAction.Variables, true) - parentCtx := clictx.Ctx + runbookRunRunbookCmd.AddCommand(runbookActionRunActionCommand) + } + } - ctx, cancelFunc := context.WithCancel(parentCtx) + return runbookRunCommand +} - concurrentRunSemaphore := semaphore.NewWeighted(int64(*maxConcurrency)) - factory := pool.NewPooledObjectFactorySimple( - func(ctx2 context.Context) (interface{}, error) { - return generateRunbookCmd(), nil - }) +func initRunbookRunScriptCmd() *cobra.Command { + cmd := &cobra.Command{ + Use: "exec-script", + Short: "Execute a YAML script file containing epcc commands", + Args: cobra.ExactArgs(1), + SilenceUsage: true, + } - objectPool := pool.NewObjectPool(ctx, factory, &pool.ObjectPoolConfig{ - MaxTotal: *maxConcurrency, - MaxIdle: *maxConcurrency, - }) + execTimeoutInSeconds := cmd.Flags().Int64("execution-timeout", 900, "How long should the script take to execute before timing out") + maxConcurrency := cmd.Flags().Int("max-concurrency", 20, "Maximum number of commands that can run simultaneously") - rawCmds := runbookAction.RawCommands - for stepIdx := 0; stepIdx < len(rawCmds); stepIdx++ { + cmd.ValidArgsFunction = func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { + return []string{"yaml", "yml"}, cobra.ShellCompDirectiveFilterFileExt + } - origIndex := &stepIdx - // Create a copy of loop variables - stepIdx := stepIdx - rawCmd := rawCmds[stepIdx] + cmd.PersistentPreRunE = func(cmd *cobra.Command, args []string) error { + return RootCmd.PersistentPreRunE(RootCmd, args) + } - templateName := fmt.Sprintf("Runbook: %s Action: %s Step: %d", runbook.Name, runbookAction.Name, stepIdx) - rawCmdLines, err := runbooks.RenderTemplates(templateName, rawCmd, runbookStringArguments, runbookAction.Variables) + cmd.RunE = func(cmd *cobra.Command, args []string) error { + data, err := os.ReadFile(args[0]) + if err != nil { + return fmt.Errorf("could not read file %s: %v", args[0], err) + } - if err != nil { - cancelFunc() - return err - } + var commands []string + err = yaml.Unmarshal(data, &commands) + if err != nil { + return fmt.Errorf("could not parse YAML file %s: %v", args[0], err) + } - joinedString := strings.Join(rawCmdLines, "\n") - renderedCmd := []string{} + runbookAction := &runbooks.RunbookAction{ + RawCommands: commands, + } - err = yaml.Unmarshal([]byte(joinedString), &renderedCmd) + return processRunBookCommands("exec-script", map[string]*string{}, runbookAction, maxConcurrency, execTimeoutInSeconds) + } - if err == nil { - log.Tracef("Line %d is a Yaml array %s, inserting into stack", stepIdx, joinedString) - newCmds := make([]string, 0, len(rawCmds)+len(renderedCmd)-1) - newCmds = append(newCmds, rawCmds[0:stepIdx]...) - newCmds = append(newCmds, renderedCmd...) - newCmds = append(newCmds, rawCmds[stepIdx+1:]...) - rawCmds = newCmds - *origIndex-- - continue - } + return cmd +} - log.Infof("Executing> %s", rawCmd) - resultChan := make(chan *commandResult, *maxConcurrency*2) - funcs := make([]func(), 0, len(rawCmdLines)) +func processRunBookCommands(runbookName string, runbookStringArguments map[string]*string, runbookAction *runbooks.RunbookAction, maxConcurrency *int, execTimeoutInSeconds *int64) error { + numSteps := len(runbookAction.RawCommands) - for commandIdx, rawCmdLine := range rawCmdLines { + parentCtx := clictx.Ctx - commandIdx := commandIdx - rawCmdLine := strings.Trim(rawCmdLine, " \n") + ctx, cancelFunc := context.WithCancel(parentCtx) - if rawCmdLine == "" { - // Allow blank lines - continue - } + concurrentRunSemaphore := semaphore.NewWeighted(int64(*maxConcurrency)) + factory := pool.NewPooledObjectFactorySimple( + func(ctx2 context.Context) (interface{}, error) { + return generateRunbookCmd(), nil + }) - if !strings.HasPrefix(rawCmdLine, "epcc ") { - // Some commands like sleep don't have prefix - // This hack allows them to run - rawCmdLine = "epcc " + rawCmdLine - } - rawCmdArguments, err := shellwords.SplitPosix(strings.Trim(rawCmdLine, " \n")) + objectPool := pool.NewObjectPool(ctx, factory, &pool.ObjectPoolConfig{ + MaxTotal: *maxConcurrency, + MaxIdle: *maxConcurrency, + }) - if err != nil { - cancelFunc() - return err - } + rawCmds := runbookAction.RawCommands + for stepIdx := 0; stepIdx < len(rawCmds); stepIdx++ { - funcs = append(funcs, func() { + origIndex := &stepIdx + // Create a copy of loop variables + stepIdx := stepIdx + rawCmd := rawCmds[stepIdx] - log.Tracef("(Step %d/%d Command %d/%d) Building Commmand", stepIdx+1, numSteps, commandIdx+1, len(funcs)) + templateName := fmt.Sprintf("Runbook: %s Action: %s Step: %d", runbookName, runbookAction.Name, stepIdx) + rawCmdLines, err := runbooks.RenderTemplates(templateName, rawCmd, runbookStringArguments, runbookAction.Variables) - stepCmdObject, err := objectPool.BorrowObject(ctx) - defer objectPool.ReturnObject(ctx, stepCmdObject) + if err != nil { + cancelFunc() + return err + } - if err == nil { - commandAndResetFunc := stepCmdObject.(*CommandAndReset) - commandAndResetFunc.reset() - stepCmd := commandAndResetFunc.cmd + joinedString := strings.Join(rawCmdLines, "\n") + renderedCmd := []string{} - tweakedArguments := misc.AddImplicitDoubleDash(rawCmdArguments) - stepCmd.SetArgs(tweakedArguments[1:]) + err = yaml.Unmarshal([]byte(joinedString), &renderedCmd) - stepCmd.SilenceErrors = true - log.Tracef("(Step %d/%d Command %d/%d) Starting Command", stepIdx+1, numSteps, commandIdx+1, len(funcs)) + if err == nil { + log.Tracef("Line %d is a Yaml array %s, inserting into stack", stepIdx, joinedString) + newCmds := make([]string, 0, len(rawCmds)+len(renderedCmd)-1) + newCmds = append(newCmds, rawCmds[0:stepIdx]...) + newCmds = append(newCmds, renderedCmd...) + newCmds = append(newCmds, rawCmds[stepIdx+1:]...) + rawCmds = newCmds + *origIndex-- + continue + } - stepCmd.ResetFlags() - err = stepCmd.ExecuteContext(ctx) - log.Tracef("(Step %d/%d Command %d/%d) Complete Command", stepIdx+1, numSteps, commandIdx+1, len(funcs)) - } + log.Infof("Executing> %s", rawCmd) + resultChan := make(chan *commandResult, *maxConcurrency*2) + funcs := make([]func(), 0, len(rawCmdLines)) - commandResult := &commandResult{ - stepIdx: stepIdx, - commandIdx: commandIdx, - commandLine: rawCmdLine, - error: err, - } + for commandIdx, rawCmdLine := range rawCmdLines { - resultChan <- commandResult + commandIdx := commandIdx + rawCmdLine := strings.Trim(rawCmdLine, " \n") - }) + if rawCmdLine == "" { + // Allow blank lines + continue + } - } + if !strings.HasPrefix(rawCmdLine, "epcc ") { + // Some commands like sleep don't have prefix + // This hack allows them to run + rawCmdLine = "epcc " + rawCmdLine + } + rawCmdArguments, err := shellwords.SplitPosix(strings.Trim(rawCmdLine, " \n")) - if len(funcs) > 1 { - log.Debugf("Running %d commands", len(funcs)) - } + if err != nil { + cancelFunc() + return err + } - // Start processing all the functions - go func() { - for idx, fn := range funcs { - idx := idx - if shutdown.ShutdownFlag.Load() { - log.Infof("Aborting runbook execution, after %d scheduled executions", idx) - cancelFunc() - break - } + funcs = append(funcs, func() { - fn := fn - log.Tracef("Run %d is waiting on semaphore", idx) - if err := concurrentRunSemaphore.Acquire(ctx, 1); err == nil { - go func() { - log.Tracef("Run %d is starting", idx) - defer concurrentRunSemaphore.Release(1) - fn() - }() - } else { - log.Warnf("Run %d failed to get semaphore %v", idx, err) - } - } - }() - - errorCount := 0 - for i := 0; i < len(funcs); i++ { - select { - case result := <-resultChan: - if !shutdown.ShutdownFlag.Load() { - if result.error != nil { - log.Warnf("(Step %d/%d Command %d/%d) %v", result.stepIdx+1, numSteps, result.commandIdx+1, len(funcs), fmt.Errorf("error processing command [%s], %w", result.commandLine, result.error)) - errorCount++ - } else { - log.Debugf("(Step %d/%d Command %d/%d) finished successfully ", result.stepIdx+1, numSteps, result.commandIdx+1, len(funcs)) - } - } else { - log.Tracef("Shutdown flag enabled, completion result %v", result) - cancelFunc() - } - case <-time.After(time.Duration(*execTimeoutInSeconds) * time.Second): - return fmt.Errorf("timeout of %d seconds reached, only %d of %d commands finished of step %d/%d", *execTimeoutInSeconds, i+1, len(funcs), stepIdx+1, numSteps) + log.Tracef("(Step %d/%d Command %d/%d) Building Commmand", stepIdx+1, numSteps, commandIdx+1, len(funcs)) - } - } + stepCmdObject, err := objectPool.BorrowObject(ctx) + defer objectPool.ReturnObject(ctx, stepCmdObject) - if len(funcs) > 1 { - log.Debugf("Running %d commands complete", len(funcs)) - } + if err == nil { + commandAndResetFunc := stepCmdObject.(*CommandAndReset) + commandAndResetFunc.reset() + stepCmd := commandAndResetFunc.cmd - if !runbookAction.IgnoreErrors && errorCount > 0 { - return fmt.Errorf("error occurred while processing script aborting") - } + tweakedArguments := misc.AddImplicitDoubleDash(rawCmdArguments) + stepCmd.SetArgs(tweakedArguments[1:]) + + stepCmd.SilenceErrors = true + log.Tracef("(Step %d/%d Command %d/%d) Starting Command", stepIdx+1, numSteps, commandIdx+1, len(funcs)) + + stepCmd.ResetFlags() + err = stepCmd.ExecuteContext(ctx) + log.Tracef("(Step %d/%d Command %d/%d) Complete Command", stepIdx+1, numSteps, commandIdx+1, len(funcs)) + } + + commandResult := &commandResult{ + stepIdx: stepIdx, + commandIdx: commandIdx, + commandLine: rawCmdLine, + error: err, + } + + resultChan <- commandResult + + }) + + } + + if len(funcs) > 1 { + log.Debugf("Running %d commands", len(funcs)) + } + + // Start processing all the functions + go func() { + for idx, fn := range funcs { + idx := idx + if shutdown.ShutdownFlag.Load() { + log.Infof("Aborting runbook execution, after %d scheduled executions", idx) + cancelFunc() + break + } + + fn := fn + log.Tracef("Run %d is waiting on semaphore", idx) + if err := concurrentRunSemaphore.Acquire(ctx, 1); err == nil { + go func() { + log.Tracef("Run %d is starting", idx) + defer concurrentRunSemaphore.Release(1) + fn() + }() + } else { + log.Warnf("Run %d failed to get semaphore %v", idx, err) + } + } + }() + + errorCount := 0 + for i := 0; i < len(funcs); i++ { + select { + case result := <-resultChan: + if !shutdown.ShutdownFlag.Load() { + if result.error != nil { + log.Warnf("(Step %d/%d Command %d/%d) %v", result.stepIdx+1, numSteps, result.commandIdx+1, len(funcs), fmt.Errorf("error processing command [%s], %w", result.commandLine, result.error)) + errorCount++ + } else { + log.Debugf("(Step %d/%d Command %d/%d) finished successfully ", result.stepIdx+1, numSteps, result.commandIdx+1, len(funcs)) } - defer cancelFunc() - return nil - }, + } else { + log.Tracef("Shutdown flag enabled, completion result %v", result) + cancelFunc() + } + case <-time.After(time.Duration(*execTimeoutInSeconds) * time.Second): + return fmt.Errorf("timeout of %d seconds reached, only %d of %d commands finished of step %d/%d", *execTimeoutInSeconds, i+1, len(funcs), stepIdx+1, numSteps) + } - processRunbookVariablesOnCommand(runbookActionRunActionCommand, runbookStringArguments, runbookAction.Variables, true) + } - runbookRunRunbookCmd.AddCommand(runbookActionRunActionCommand) + if len(funcs) > 1 { + log.Debugf("Running %d commands complete", len(funcs)) } - } - return runbookRunCommand + if !runbookAction.IgnoreErrors && errorCount > 0 { + return fmt.Errorf("error occurred while processing script aborting") + } + } + defer cancelFunc() + return nil } func processRunbookVariablesOnCommand(runbookActionRunActionCommand *cobra.Command, runbookStringArguments map[string]*string, variables map[string]runbooks.Variable, enableRequiredVars bool) { diff --git a/docs/runbook-development.md b/docs/runbook-development.md index e15e2f6..da69160 100644 --- a/docs/runbook-development.md +++ b/docs/runbook-development.md @@ -278,6 +278,39 @@ actions: +### Standalone Script Files + +If you want to run a set of epcc commands without defining a full runbook, you can use `exec-script` to execute a standalone YAML file: + +```bash +epcc runbooks exec-script my-script.yml +``` + +The YAML file is simply a list of commands, using the same syntax as the `commands` block in a runbook action. Sequential execution uses one command per list element: + +```yaml +- epcc create -s account name Account1 --auto-fill +- epcc create -s account name Account2 --auto-fill +- epcc create -s account-address name=Account1 --auto-fill +- epcc create -s account-address name=Account2 --auto-fill +``` + +Parallel execution uses YAML block scalars (`|`) so multiple commands share a single list element: + +```yaml +- | + epcc create -s account name Account1 --auto-fill + epcc create -s account name Account2 --auto-fill + +- | + epcc create -s account-address name=Account1 --auto-fill + epcc create -s account-address name=Account2 --auto-fill +``` + +The `--execution-timeout` and `--max-concurrency` flags are supported, just like `runbooks run`. + +Example script files can be found in `external/runbooks/scripts/`. + ### Output JQ You will need to escape \ when using the `--output-jq` argument. diff --git a/external/runbooks/run-all-runbooks.sh b/external/runbooks/run-all-runbooks.sh index cbb074d..cc019d9 100755 --- a/external/runbooks/run-all-runbooks.sh +++ b/external/runbooks/run-all-runbooks.sh @@ -116,6 +116,11 @@ epcc runbooks run extend-customer-resources-how-to update-example-customer epcc runbooks run extend-customer-resources-how-to reset +echo "Starting exec-script Tests" +epcc reset-store .+ +epcc runbooks exec-script "$SCRIPT_DIR/scripts/account-sequential-script.yml" +epcc runbooks exec-script "$SCRIPT_DIR/scripts/account-parallel-script.yml" + echo "Starting Account Management Runbook" epcc reset-store .+ diff --git a/external/runbooks/scripts/account-parallel-script.yml b/external/runbooks/scripts/account-parallel-script.yml new file mode 100644 index 0000000..a970213 --- /dev/null +++ b/external/runbooks/scripts/account-parallel-script.yml @@ -0,0 +1,7 @@ +- | + epcc create -s account name Account1 --auto-fill + epcc create -s account name Account2 --auto-fill + +- | + epcc create -s account-address name=Account1 --auto-fill + epcc create -s account-address name=Account2 --auto-fill diff --git a/external/runbooks/scripts/account-sequential-script.yml b/external/runbooks/scripts/account-sequential-script.yml new file mode 100644 index 0000000..1d78b16 --- /dev/null +++ b/external/runbooks/scripts/account-sequential-script.yml @@ -0,0 +1,4 @@ +- epcc create -s account name Account1 --auto-fill +- epcc create -s account name Account2 --auto-fill +- epcc create -s account-address name=Account1 --auto-fill +- epcc create -s account-address name=Account2 --auto-fill