diff --git a/cmd/argoexec/commands/artifact_plugin.go b/cmd/argoexec/commands/artifact_plugin.go index 9577d67ea235..d848546f8f33 100644 --- a/cmd/argoexec/commands/artifact_plugin.go +++ b/cmd/argoexec/commands/artifact_plugin.go @@ -51,11 +51,8 @@ func NewArtifactPluginCommand() *cobra.Command { _ = osspecific.Kill(command.Process.Pid, s.(syscall.Signal)) } }() - // pid := command.Process.Pid - // ctx, cancel := context.WithCancel(ctx) - // defer cancel() }() - err := loadArtifactPlugin(cmd.Context(), artifactPlugin) + err := loadArtifactPlugin(cmd.Context(), wfv1.ArtifactPluginName(artifactPlugin)) if err != nil { return fmt.Errorf("%+v", err) } @@ -66,7 +63,10 @@ func NewArtifactPluginCommand() *cobra.Command { return &command } -func loadArtifactPlugin(ctx context.Context, pluginName string) error { +func loadArtifactPlugin(ctx context.Context, pluginName wfv1.ArtifactPluginName) error { + if err := os.MkdirAll(pluginName.SocketDir(), 0755); err != nil { + return err + } wfExecutor := initExecutor(ctx) defer wfExecutor.HandleError(ctx) defer stats.LogStats() diff --git a/docs/artifact-plugin.md b/docs/artifact-plugin.md index 44cd9996f3f5..cb68cf27230a 100644 --- a/docs/artifact-plugin.md +++ b/docs/artifact-plugin.md @@ -11,30 +11,19 @@ By implementing a plugin, you can integrate with proprietary storage systems, ad To create an artifact plugin, you need to: -### 1. Create and Distribute a Docker Image - -Your plugin must be packaged as a Docker image that contains: - -- Your plugin implementation -- All necessary dependencies and runtime requirements -- The GRPC server that implements the artifact interface - -The Docker image will be deployed alongside workflow pods as sidecars and init containers. - -### 2. Implement a GRPC Server +### 1. Implement a GRPC Server Your plugin's entrypoint must run a GRPC server that: -- Listens on the socket path provided as the first and only command-line parameter +- Listens on the Unix socket path provided as the first and only command-line parameter. The `unix://` prefix should not be necessary. - Implements the artifact service interface - Handles artifact operations (load, save, delete, etc.) +- Shuts down gracefully when given a SIGTERM signal The GRPC interface is defined in **[`artifact.proto`](https://github.com/argoproj/argo-workflows/blob/main/pkg/apiclient/artifact/artifact.proto)**. This contains the main `ArtifactService` interface and all request/response message types your plugin must implement. -### 3. Language Flexibility - -You can implement your plugin in any programming language that supports GRPC, including: +The plugin can be implemented in any programming language that supports GRPC, including: - Go - Python @@ -46,6 +35,16 @@ You can implement your plugin in any programming language that supports GRPC, in Choose the language that best fits your team's expertise and your storage backend's SDK requirements. +### 2. Create and Distribute a Docker Image + +Your plugin must be packaged as a Docker image that contains: + +- Your plugin implementation +- All necessary dependencies and runtime requirements +- The GRPC server that implements the artifact interface + +The Docker image will be deployed alongside workflow pods as sidecars and init containers. + ## Implementation Steps Follow these steps to implement an artifact plugin in your chosen language: @@ -56,7 +55,8 @@ Download the [`artifact.proto`](https://github.com/argoproj/argo-workflows/blob/ ### 2. Generate GRPC Server Code -Use your language's GRPC tooling to generate server stubs from the protocol definition: +Use your language's GRPC tooling to generate server stubs from the protocol definition. +You will need the `googleapis` protocol definitions from . #### Go @@ -72,11 +72,17 @@ protoc --go_out=. --go-grpc_out=. artifact.proto #### Python ```bash -# Install grpcio-tools -pip install grpcio-tools +# Create a project and venv +python -m venv .venv && source .venv/bin/activate + +# Install grpcio-tools and protobuf +pip install grpcio-tools protobuf + +# Create a module +mkdir my_artifact_plugin # Generate Python code -python -m grpc_tools.protoc --python_out=. --grpc_python_out=. artifact.proto +python -m grpc_tools.protoc --python_out=artifact_plugin --pyi_out=artifact_plugin --grpc_python_out=artifact_plugin -I googleapis -I . artifact.proto ``` #### Java @@ -149,11 +155,12 @@ Your GRPC server must implement these six methods from the `ArtifactService` int #### Implementation Notes -- Parse the plugin configuration from `artifact.configuration` field in each request -- Use the `artifact.key` field to identify the specific artifact location in your storage +- Parse the plugin configuration from `artifact.plugin.configuration` field in each request +- Use the `artifact.plugin.key` field to identify the specific artifact location in your storage - Handle errors gracefully and return appropriate error messages - Support both file and directory artifacts where applicable - Consider implementing timeouts and retry logic for storage operations +- IMPORTANT: SIGTERM signals must be handled gracefully to exit with a `0` exit code ### 4. Build and Package @@ -305,6 +312,6 @@ if __name__ == "__main__": Once local testing passes, test with the full workflow controller: 1. Build and push your Docker image -2. Configure it in the workflow controller ConfigMap +2. Configure it in the workflow controller ConfigMap, as shown under [Configuring Your Artifact Repository](configure-artifact-repository.md#plugin-configuration) 3. Create workflows that use your plugin for artifacts 4. Verify all artifact operations work correctly by using artifacts as inputs, outputs and performing garbage collection diff --git a/workflow/controller/workflowpod.go b/workflow/controller/workflowpod.go index 9e24a9f931a8..6fedc3f5f46f 100644 --- a/workflow/controller/workflowpod.go +++ b/workflow/controller/workflowpod.go @@ -664,6 +664,7 @@ func (woc *wfOperationCtx) newInitContainers(ctx context.Context, tmpl *wfv1.Tem } ctr.Command = append([]string{common.VarRunArgoPath + "/argoexec", "artifact-plugin", "--plugin-name", string(driver.Name)}, woc.getExecutorLogOpts(ctx)...) + ctr.Command = append(ctr.Command, "--") ctr.Command = append(ctr.Command, x.Entrypoint...) ctr.Command = append(ctr.Command, driver.Name.SocketPath()) initContainers[i+1] = *ctr