Add pants support for pytest

.buildkite/Dockerfile

@@ -12,6 +12,7 @@ RUN gem install rspec
 RUN yarn global add jest
 RUN pip install pytest
 RUN pip install buildkite-test-collector==0.2.0
+RUN curl --proto '=https' --tlsv1.2 -fsSL https://static.pantsbuild.org/setup/get-pants.sh | bash -s -- --bin-dir /usr/local/bin
 
 # Install curl, download bktec binary, make it executable, place it, and cleanup
 RUN apt-get update && \

README.md

@@ -4,13 +4,13 @@ Buildkite Test Engine Client (bktec) is an open source tool to orchestrate your
 
 bktec supports multiple test runners and offers various features to enhance your testing workflow. Below is a comparison of the features supported by each test runner:
 
-| Feature                                            | RSpec | Jest | Playwright | Cypress | pytest | Go test |
-| -------------------------------------------------- | :---: | :--: | :--------: | :-----: | :----: | :--: |
-| Filter test files                                  |   ✅  |   ✅  |    ✅      |    ✅   |    ✅   |  ❌  |
-| Automatically retry failed test                    |   ✅  |   ✅  |    ✅      |    ❌   |    ✅   |  ✅  |
-| Split slow files by individual test example        |   ✅  |   ❌  |    ❌      |    ❌   |    ❌   |  ❌  |
-| Mute tests (ignore test failures)                  |   ✅  |   ✅  |    ✅      |    ❌   |    ✅   |  ✅  |
-| Skip tests                                         |   ✅  |   ❌  |    ❌      |    ❌   |    ❌   |  ❌  |
+| Feature                                            | RSpec | Jest | Playwright  | Cypress | pytest  | pants (pytest) | Go test |
+| -------------------------------------------------- | :---: | :--: | :---------: | :-----: | :-----: | :------------: | :-----: |
+| Filter test files                                  |   ✅  |   ✅  |    ✅      |    ✅   |    ✅   |       ❌       |    ❌   |
+| Automatically retry failed test                    |   ✅  |   ✅  |    ✅      |    ❌   |    ✅   |       ✅       |    ✅   |
+| Split slow files by individual test example        |   ✅  |   ❌  |    ❌      |    ❌   |    ❌   |       ❌       |    ❌   |
+| Mute tests (ignore test failures)                  |   ✅  |   ✅  |    ✅      |    ❌   |    ✅   |       ✅       |    ✅   |
+| Skip tests                                         |   ✅  |   ❌  |    ❌      |    ❌   |    ❌   |       ❌       |    ❌   |
 
 ## Installation
 The latest version of bktec can be downloaded from https://github.com/buildkite/test-engine-client/releases
@@ -60,6 +60,7 @@ To configure the test runner for bktec, please refer to the detailed guides prov
 - [Playwright](./docs/playwright.md)
 - [Cypress](./docs/cypress.md)
 - [pytest](./docs/pytest.md)
+- [pytest pants](./docs/pytest-pants.md)
 - [go test](./docs/gotest.md)
 - [RSpec](./docs/rspec.md)
 

docs/pytest-pants.md

@@ -0,0 +1,82 @@
+# Using bktec with pants (Experimental)
+
+> [!WARNING]
+> Pants support is currently experimental and has limited feature support. Only the following features are supported:
+>
+> - Automatically retry failed tests
+> - Mute tests (ignore test failures)
+>
+> The following features are not supported:
+>
+> - Filter test files
+> - Split slow files by individual test example
+> - Skip tests
+
+To integrate bktec with pants, you need to [install and configure Buildkite Test Collector for pytest](https://buildkite.com/docs/test-engine/python-collectors#pytest-collector) first. Then set the `BUILDKITE_TEST_ENGINE_TEST_RUNNER` environment variable to `pytest-pants`.
+
+Look at the example configuration files in the [pytest_pants testdata directory](../internal/runner/testdata/pytest_pants) for an example of how to add buildkite-test-collector to the pants resolve used by pytest. Specifically:
+
+- [pants.toml](../internal/runner/testdata/pytest_pants/pants.toml) - pants configuration
+- [3rdparty/python/BUILD](../internal/runner/testdata/pytest_pants/3rdparty/python/BUILD) - python_requirement targets
+- [3rdparty/python/pytest-requirements.txt](../internal/runner/testdata/pytest_pants/3rdparty/python/pytest-requirements.txt) - Python requirements.txt
+
+In the example in the repository, you would need to generate a lockfile next, i.e.
+
+```sh
+pants generate-lockfiles --resolve=pytest
+```
+
+Only running `pants test` with `python_test` targets is supported at this time.
+
+```sh
+export BUILDKITE_TEST_ENGINE_TEST_RUNNER=pytest-pants
+export BUILDKITE_TEST_ENGINE_TEST_CMD="pants --filter-target-type=python_test --changed-since=HEAD~1 test -- --json={{resultPath}} --merge-json"
+bktec
+```
+
+## Configure test command
+
+While pants support is experimental there is no default command. That means it is required to set `BUILDKITE_TEST_ENGINE_TEST_CMD`.
+Below are a few recommendations for specific scenarios:
+
+---
+
+```sh
+export BUILDKITE_TEST_ENGINE_TEST_CMD="pants --filter-target-type=python_test test //:: -- --json={{resultPath}} --merge-json""
+```
+
+This command is a good option if you want to run all python tests in your repository.
+
+---
+
+```sh
+export BUILDKITE_TEST_ENGINE_TEST_CMD="pants --filter-target-type=python_test --changed-since=HEAD~1 test -- --json={{resultPath}} --merge-json"
+```
+
+This command is a good option if you want to only run the python tests that were
+impacted by any changes made since `HEAD~1`. Checkout [pants Advanced target
+selection doc][pants-advanced-target-selection] for more information on
+`--changed-since`.
+
+---
+
+In both commands, `{{resultPath}}` is replaced with a unique temporary path created by bktec. `--json` option is a custom pytest option added by Buildkite Test Collector to save the result into a JSON file at given path. You can further customize the test command for your specific use case.
+
+> [!IMPORTANT]
+> Make sure to append `-- --json={{resultPath}} --merge-json` in your custom pants test command, as bktec requires these options to read the test results for retries and verification purposes.
+
+## Filter test files
+
+There is not support for filtering test files at this time.
+
+## Automatically retry failed tests
+
+You can configure bktec to automatically retry failed tests using the `BUILDKITE_TEST_ENGINE_RETRY_COUNT` environment variable. When this variable is set to a number greater than `0`, bktec will retry each failed test up to the specified number of times, using either the default test command or the command specified in `BUILDKITE_TEST_ENGINE_TEST_CMD`. Because pants caches test results, only failed tests will be retried.
+
+To enable automatic retry, set the following environment variable:
+
+```sh
+export BUILDKITE_TEST_ENGINE_RETRY_COUNT=2
+```
+
+[pants-advanced-target-selection]: https://www.pantsbuild.org/stable/docs/using-pants/advanced-target-selection

internal/config/validate.go

@@ -60,7 +60,7 @@ func (c *Config) validate() error {
 		c.errs.appendFieldError("BUILDKITE_TEST_ENGINE_SUITE_SLUG", "must not be blank")
 	}
 
-	if c.ResultPath == "" && c.TestRunner != "cypress" && c.TestRunner != "pytest" {
+	if c.ResultPath == "" && c.TestRunner != "cypress" && c.TestRunner != "pytest" && c.TestRunner != "pytest-pants" {
 		c.errs.appendFieldError("BUILDKITE_TEST_ENGINE_RESULT_PATH", "must not be blank")
 	}
 

internal/runner/detector.go

@@ -47,10 +47,12 @@ func DetectRunner(cfg config.Config) (TestRunner, error) {
 		return NewPlaywright(runnerConfig), nil
 	case "pytest":
 		return NewPytest(runnerConfig), nil
+	case "pytest-pants":
+		return NewPytestPants(runnerConfig), nil
 	case "gotest":
 		return NewGoTest(runnerConfig), nil
 	default:
 		// Update the error message to include the new runner
-		return nil, errors.New("runner value is invalid, possible values are 'rspec', 'jest', 'cypress', 'playwright', 'pytest', or 'gotest'")
+		return nil, errors.New("runner value is invalid, possible values are 'rspec', 'jest', 'cypress', 'playwright', 'pytest', 'pytest-pants', or 'gotest'")
 	}
 }

internal/runner/pytest_pants.go

@@ -0,0 +1,141 @@
+package runner
+
+import (
+	"errors"
+	"fmt"
+	"os"
+	"os/exec"
+	"strings"
+
+	"github.com/buildkite/test-engine-client/internal/plan"
+	"github.com/kballard/go-shellquote"
+)
+
+type PytestPants struct {
+	RunnerConfig
+}
+
+func (p PytestPants) Name() string {
+	return "pytest-pants"
+}
+
+func NewPytestPants(c RunnerConfig) PytestPants {
+	fmt.Fprintln(os.Stderr, "Info: Python package 'buildkite-test-collector' is required and will not be verified by bktec. Please ensure it is added to the pants resolve used by pytest. See https://github.com/buildkite/test-engine-client/blob/main/docs/pytest-pants.md for more information.")
+
+	if c.TestCommand == "" {
+		fmt.Fprintln(os.Stderr, "Error: The test command must be set via BUILDKITE_TEST_ENGINE_TEST_CMD.")
+		os.Exit(1)
+	}
+
+	if c.TestFilePattern != "" || c.TestFileExcludePattern != "" {
+		fmt.Fprintln(os.Stderr, "Warning: Pants test runner variant does not support discovering test files. Please ensure the test command is set correctly via BUILDKITE_TEST_ENGINE_TEST_CMD and do *not* set either:")
+		fmt.Fprintf(os.Stderr, "  BUILDKITE_TEST_ENGINE_TEST_FILE_PATTERN=%q\n", c.TestFilePattern)
+		fmt.Fprintf(os.Stderr, "  BUILDKITE_TEST_ENGINE_TEST_FILE_EXCLUDE_PATTERN=%q\n", c.TestFileExcludePattern)
+	}
+
+	if c.TestFilePattern == "" {
+		c.TestFilePattern = "**/{*_test,test_*}.py"
+	}
+
+	if c.RetryTestCommand == "" {
+		c.RetryTestCommand = c.TestCommand
+	}
+
+	if c.ResultPath == "" {
+		c.ResultPath = getRandomTempFilename()
+	}
+
+	return PytestPants{
+		RunnerConfig: c,
+	}
+}
+
+func (p PytestPants) Run(result *RunResult, testCases []plan.TestCase, retry bool) error {
+	testPaths := make([]string, len(testCases))
+	for i, tc := range testCases {
+		testPaths[i] = tc.Path
+	}
+
+	command := p.TestCommand
+
+	if retry {
+		command = p.RetryTestCommand
+	}
+
+	cmdName, cmdArgs, err := p.commandNameAndArgs(command, testPaths)
+	if err != nil {
+		return fmt.Errorf("failed to build command: %w", err)
+	}
+
+	cmd := exec.Command(cmdName, cmdArgs...)
+
+	err = runAndForwardSignal(cmd)
+
+	// Only rescue exit code 1 because it indicates a test failures.
+	// Ref: https://docs.pytest.org/en/7.1.x/reference/exit-codes.html
+	if exitError := new(exec.ExitError); errors.As(err, &exitError) && exitError.ExitCode() != 1 {
+		return err
+	}
+
+	tests, parseErr := ParsePytestCollectorResult(p.ResultPath)
+
+	if parseErr != nil {
+		fmt.Println("Buildkite Test Engine Client: Failed to read json output, failed tests will not be retried.")
+		return err
+	}
+
+	for _, test := range tests {
+
+		result.RecordTestResult(plan.TestCase{
+			Identifier: test.Id,
+			Format:     plan.TestCaseFormatExample,
+			Scope:      test.Scope,
+			Name:       test.Name,
+			// pytest can execute individual test using node id, which is a filename, classname (if any), and function, separated by `::`.
+			// Ref: https://docs.pytest.org/en/6.2.x/usage.html#nodeids
+			Path: fmt.Sprintf("%s::%s", test.Scope, test.Name),
+		}, test.Result)
+	}
+
+	return nil
+}
+
+func (p PytestPants) GetFiles() ([]string, error) {
+	return []string{}, nil
+}
+
+func (p PytestPants) GetExamples(files []string) ([]plan.TestCase, error) {
+	return nil, fmt.Errorf("not supported in pytest pants")
+}
+
+func (p PytestPants) commandNameAndArgs(cmd string, testCases []string) (string, []string, error) {
+	if strings.Contains(cmd, "{{testExamples}}") {
+		return "", []string{}, fmt.Errorf("currently, bktec does not support dynamically injecting {{testExamples}}. Please ensure the test command in BUILDKITE_TEST_ENGINE_TEST_CMD does *not* include {{testExamples}}")
+	}
+
+	// Split command into parts before and after the first --
+	parts := strings.SplitN(cmd, "--", 2)
+	if len(parts) != 2 {
+		return "", []string{}, fmt.Errorf("please ensure the test command in BUILDKITE_TEST_ENGINE_TEST_CMD includes a -- separator")
+	}
+
+	// Check that both required flags are after the --
+	afterDash := parts[1]
+	if !strings.Contains(afterDash, "--json={{resultPath}}") {
+		return "", []string{}, fmt.Errorf("please ensure the test command in BUILDKITE_TEST_ENGINE_TEST_CMD includes --json={{resultPath}} after the -- separator")
+	}
+
+	if !strings.Contains(afterDash, "--merge-json") {
+		return "", []string{}, fmt.Errorf("please ensure the test command in BUILDKITE_TEST_ENGINE_TEST_CMD includes --merge-json after the -- separator")
+	}
+
+	cmd = strings.Replace(cmd, "{{resultPath}}", p.ResultPath, 1)
+
+	args, err := shellquote.Split(cmd)
+
+	if err != nil {
+		return "", []string{}, err
+	}
+
+	return args[0], args[1:], nil
+}