From 97bc055bd2315992981756557a32dd98fa92f880 Mon Sep 17 00:00:00 2001 From: William Woodruff Date: Fri, 13 Jan 2023 17:37:48 -0500 Subject: [PATCH 01/19] [WIP] README, workflows: experimentation Signed-off-by: William Woodruff --- .github/workflows/lint.yml | 28 ++++++++++++++++++++++++++++ README.md | 2 +- 2 files changed, 29 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/lint.yml diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml new file mode 100644 index 0000000..cbbeeb6 --- /dev/null +++ b/.github/workflows/lint.yml @@ -0,0 +1,28 @@ +on: + workflow_call: + inputs: + type: + description: "the expected lint entrypoint" + default: "make" + required: false + type: string + +jobs: + lint: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v3 + + - name: run lints + env: + LINT_TYPE: "${{ inputs.type }}" + run: | + if [[ "${LINT_TYPE}" == "make" ]]; then + make lint + elif [[ "${LINT_TYPE}" == "rust" ]]; then + cargo fmt + else + >&2 echo "Fatal: unknown linting type: ${LINT_TYPE}" + exit 1 + fi diff --git a/README.md b/README.md index 0b8dfca..1e2b51f 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,5 @@ .github ======= -Public shared workflow templates for Trail of Bits. +Public shared workflow templates and reusable workflows for Trail of Bits. From 2b7f871d3eeb78a35c67051d4f74c9c43e112b45 Mon Sep 17 00:00:00 2001 From: William Woodruff Date: Fri, 13 Jan 2023 17:43:56 -0500 Subject: [PATCH 02/19] workflows/lint: configure python optionally Signed-off-by: William Woodruff --- .github/workflows/lint.yml | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index cbbeeb6..0f6f7ef 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -6,6 +6,11 @@ on: default: "make" required: false type: string + python-version: + description: "the version of Python to configure, if any (can be SemVer-formatted)" + default: "" + required: false + type: string jobs: lint: @@ -14,15 +19,21 @@ jobs: steps: - uses: actions/checkout@v3 + - name: configure python, if required + if: inputs.python-version != '' + uses: actions/setup-python@v4 + with: + python-version: "${{ inputs.python-version }}" + - name: run lints env: - LINT_TYPE: "${{ inputs.type }}" + TOB_LINT_TYPE: "${{ inputs.type }}" run: | - if [[ "${LINT_TYPE}" == "make" ]]; then + if [[ "${TOB_LINT_TYPE}" == "make" ]]; then make lint - elif [[ "${LINT_TYPE}" == "rust" ]]; then + elif [[ "${TOB_LINT_TYPE}" == "rust" ]]; then cargo fmt else - >&2 echo "Fatal: unknown linting type: ${LINT_TYPE}" + >&2 echo "Fatal: unknown linting type: ${TOB_LINT_TYPE}" exit 1 fi From 218350a31830c4017938f7d8438c0df7d8406a78 Mon Sep 17 00:00:00 2001 From: William Woodruff Date: Fri, 13 Jan 2023 18:01:49 -0500 Subject: [PATCH 03/19] lint: be even more normative about Python ...we use `pyproject.toml` and `pip`, so we can safely set these caching settings. Signed-off-by: William Woodruff --- .github/workflows/lint.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 0f6f7ef..38eb05f 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -24,6 +24,8 @@ jobs: uses: actions/setup-python@v4 with: python-version: "${{ inputs.python-version }}" + cache: "pip" + cache-dependency-path: pyproject.toml - name: run lints env: From 7d43b79824202c428ab5a3cb93b39b6bd10a8278 Mon Sep 17 00:00:00 2001 From: William Woodruff Date: Fri, 13 Jan 2023 18:06:39 -0500 Subject: [PATCH 04/19] workflows/lint: add a TODO Signed-off-by: William Woodruff --- .github/workflows/lint.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 38eb05f..6ccb957 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -27,7 +27,7 @@ jobs: cache: "pip" cache-dependency-path: pyproject.toml - - name: run lints + - name: run baseline lint env: TOB_LINT_TYPE: "${{ inputs.type }}" run: | @@ -39,3 +39,6 @@ jobs: >&2 echo "Fatal: unknown linting type: ${TOB_LINT_TYPE}" exit 1 fi + + # TODO: other lints would go here, as configured: + # - LICENSE conformance (checking headers of all source files) From 3020cc3009ae296421e17401b60f53e0cd7b488e Mon Sep 17 00:00:00 2001 From: William Woodruff Date: Tue, 17 Jan 2023 17:09:28 -0500 Subject: [PATCH 05/19] lint: remove TODO Signed-off-by: William Woodruff --- .github/workflows/lint.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 6ccb957..6b88d40 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -39,6 +39,3 @@ jobs: >&2 echo "Fatal: unknown linting type: ${TOB_LINT_TYPE}" exit 1 fi - - # TODO: other lints would go here, as configured: - # - LICENSE conformance (checking headers of all source files) From ede8697012d20a65f4c1db5e16859bb2444390d2 Mon Sep 17 00:00:00 2001 From: William Woodruff Date: Tue, 17 Jan 2023 17:16:22 -0500 Subject: [PATCH 06/19] workflows/lint: `cargo fmt -> cargo clippy` `cargo fmt` should go in a separate workflow. Signed-off-by: William Woodruff --- .github/workflows/lint.yml | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 6b88d40..6581639 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -27,6 +27,13 @@ jobs: cache: "pip" cache-dependency-path: pyproject.toml + - nane: configure clippy, if required + if: inputs.type == 'rust' + run: | + # we always run the latest stable Rust and Clippy for Rust linting. + rustup update + rustup component add clippy + - name: run baseline lint env: TOB_LINT_TYPE: "${{ inputs.type }}" @@ -34,7 +41,7 @@ jobs: if [[ "${TOB_LINT_TYPE}" == "make" ]]; then make lint elif [[ "${TOB_LINT_TYPE}" == "rust" ]]; then - cargo fmt + cargo clippy else >&2 echo "Fatal: unknown linting type: ${TOB_LINT_TYPE}" exit 1 From 5705c90a8ad43271762ac75ae37b8ca088a609bc Mon Sep 17 00:00:00 2001 From: William Woodruff Date: Tue, 17 Jan 2023 17:39:28 -0500 Subject: [PATCH 07/19] lint: reuse clippy-check for clippy This has support for annotations, which is the kind of thing we want! Signed-off-by: William Woodruff --- .github/workflows/lint.yml | 21 ++++++++++----------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 6581639..1cb632d 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -34,15 +34,14 @@ jobs: rustup update rustup component add clippy - - name: run baseline lint - env: - TOB_LINT_TYPE: "${{ inputs.type }}" + - name: run lint (rust) + uses: actions-rs/clippy-check@v1 + if: inputs.type == 'rust' + with: + token: ${{ secrets.GITHUB_TOKEN }} + args: --all-features + + - name: run lint (make) + if: inputs.type == 'make' run: | - if [[ "${TOB_LINT_TYPE}" == "make" ]]; then - make lint - elif [[ "${TOB_LINT_TYPE}" == "rust" ]]; then - cargo clippy - else - >&2 echo "Fatal: unknown linting type: ${TOB_LINT_TYPE}" - exit 1 - fi + make lint From a2f849b1c4eed7eefc14a64bbb0ffa328ca15ed5 Mon Sep 17 00:00:00 2001 From: William Woodruff Date: Wed, 18 Jan 2023 14:43:32 -0500 Subject: [PATCH 08/19] workflows/lint: avoid unmaintained clippy-check Signed-off-by: William Woodruff --- .github/workflows/lint.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 1cb632d..a356a8a 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -35,7 +35,7 @@ jobs: rustup component add clippy - name: run lint (rust) - uses: actions-rs/clippy-check@v1 + uses: actions-rs-plus/clippy-check@5eb300cdebee2681ff8513b9348b0ca793d8a293 if: inputs.type == 'rust' with: token: ${{ secrets.GITHUB_TOKEN }} From 676675658a00005c19d27f2c2cb2f416fbe00820 Mon Sep 17 00:00:00 2001 From: William Woodruff Date: Wed, 18 Jan 2023 14:45:22 -0500 Subject: [PATCH 09/19] workflows/lint: note Signed-off-by: William Woodruff --- .github/workflows/lint.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index a356a8a..7fc9ad5 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -35,6 +35,9 @@ jobs: rustup component add clippy - name: run lint (rust) + # NOTE: This is a fork of the original `actions-rs/clippy-check`, + # which is no longer maintained; the commit is pinned since no release + # has been made and its long term maintenance status/ownership is unclear. uses: actions-rs-plus/clippy-check@5eb300cdebee2681ff8513b9348b0ca793d8a293 if: inputs.type == 'rust' with: From 90ae18cfbac35298e4fb40a39ddad971dd6c0ea7 Mon Sep 17 00:00:00 2001 From: William Woodruff Date: Wed, 18 Jan 2023 14:59:24 -0500 Subject: [PATCH 10/19] lint: cargo-sort support Signed-off-by: William Woodruff --- .github/workflows/lint.yml | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 7fc9ad5..59cc6ab 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -11,6 +11,11 @@ on: default: "" required: false type: string + cargo-sort: + description: "run cargo-sort as part of Rust linting" + default: false + required: false + type: boolean jobs: lint: @@ -27,14 +32,14 @@ jobs: cache: "pip" cache-dependency-path: pyproject.toml - - nane: configure clippy, if required + - nane: configure rust and clippy, if required if: inputs.type == 'rust' run: | # we always run the latest stable Rust and Clippy for Rust linting. rustup update rustup component add clippy - - name: run lint (rust) + - name: run clippy (rust) # NOTE: This is a fork of the original `actions-rs/clippy-check`, # which is no longer maintained; the commit is pinned since no release # has been made and its long term maintenance status/ownership is unclear. @@ -44,6 +49,12 @@ jobs: token: ${{ secrets.GITHUB_TOKEN }} args: --all-features + - name: run cargo sort (rust) + if: inputs.type == 'rust' && inputs.cargo-sort == 'true' + run: | + cargo install cargo-sort + cargo sort -c + - name: run lint (make) if: inputs.type == 'make' run: | From 7c04c8fce398a21af5184ce7e3a3fb8904c20e63 Mon Sep 17 00:00:00 2001 From: William Woodruff Date: Wed, 18 Jan 2023 15:01:39 -0500 Subject: [PATCH 11/19] lint: typo Signed-off-by: William Woodruff --- .github/workflows/lint.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 59cc6ab..dd47051 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -32,7 +32,7 @@ jobs: cache: "pip" cache-dependency-path: pyproject.toml - - nane: configure rust and clippy, if required + - name: configure rust and clippy, if required if: inputs.type == 'rust' run: | # we always run the latest stable Rust and Clippy for Rust linting. From 6d069e463fdb94b52de42cb31f6a65462cf90ab5 Mon Sep 17 00:00:00 2001 From: William Woodruff Date: Wed, 18 Jan 2023 15:08:19 -0500 Subject: [PATCH 12/19] lint: testing Signed-off-by: William Woodruff --- .github/workflows/lint.yml | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index dd47051..26b0b1c 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -40,10 +40,7 @@ jobs: rustup component add clippy - name: run clippy (rust) - # NOTE: This is a fork of the original `actions-rs/clippy-check`, - # which is no longer maintained; the commit is pinned since no release - # has been made and its long term maintenance status/ownership is unclear. - uses: actions-rs-plus/clippy-check@5eb300cdebee2681ff8513b9348b0ca793d8a293 + uses: actions-rs/clippy-check@v1.0.7 if: inputs.type == 'rust' with: token: ${{ secrets.GITHUB_TOKEN }} From 34bb1a925835a14bd40c0d7274337f2c72e002a4 Mon Sep 17 00:00:00 2001 From: William Woodruff Date: Wed, 18 Jan 2023 15:13:26 -0500 Subject: [PATCH 13/19] Revert "lint: testing" This reverts commit 6d069e463fdb94b52de42cb31f6a65462cf90ab5. --- .github/workflows/lint.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 26b0b1c..dd47051 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -40,7 +40,10 @@ jobs: rustup component add clippy - name: run clippy (rust) - uses: actions-rs/clippy-check@v1.0.7 + # NOTE: This is a fork of the original `actions-rs/clippy-check`, + # which is no longer maintained; the commit is pinned since no release + # has been made and its long term maintenance status/ownership is unclear. + uses: actions-rs-plus/clippy-check@5eb300cdebee2681ff8513b9348b0ca793d8a293 if: inputs.type == 'rust' with: token: ${{ secrets.GITHUB_TOKEN }} From ada2eb532a136e32589092243c9b66190a7f7acc Mon Sep 17 00:00:00 2001 From: William Woodruff Date: Wed, 18 Jan 2023 15:14:08 -0500 Subject: [PATCH 14/19] workflows/lint: fix evaluation Signed-off-by: William Woodruff --- .github/workflows/lint.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index dd47051..14ce913 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -50,7 +50,7 @@ jobs: args: --all-features - name: run cargo sort (rust) - if: inputs.type == 'rust' && inputs.cargo-sort == 'true' + if: inputs.type == 'rust' && inputs.cargo-sort run: | cargo install cargo-sort cargo sort -c From e48d2d19c54916fa4f18f0b440fee081423b7e5d Mon Sep 17 00:00:00 2001 From: William Woodruff Date: Wed, 18 Jan 2023 15:22:39 -0500 Subject: [PATCH 15/19] lint: make clippy fail on warnings Signed-off-by: William Woodruff --- .github/workflows/lint.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 14ce913..e099161 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -47,7 +47,7 @@ jobs: if: inputs.type == 'rust' with: token: ${{ secrets.GITHUB_TOKEN }} - args: --all-features + args: --all-features -- -D warnings - name: run cargo sort (rust) if: inputs.type == 'rust' && inputs.cargo-sort From 3c66e09c2736cc9856e48063582c7ec62732ee4d Mon Sep 17 00:00:00 2001 From: William Woodruff Date: Wed, 18 Jan 2023 15:31:40 -0500 Subject: [PATCH 16/19] lint: add rustfmt, remove redundant token Signed-off-by: William Woodruff --- .github/workflows/lint.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index e099161..be17be8 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -39,6 +39,10 @@ jobs: rustup update rustup component add clippy + - name: run rustfmt (rust) + if: inputs.type == 'rust' + run: cargo fmt --check + - name: run clippy (rust) # NOTE: This is a fork of the original `actions-rs/clippy-check`, # which is no longer maintained; the commit is pinned since no release @@ -46,7 +50,6 @@ jobs: uses: actions-rs-plus/clippy-check@5eb300cdebee2681ff8513b9348b0ca793d8a293 if: inputs.type == 'rust' with: - token: ${{ secrets.GITHUB_TOKEN }} args: --all-features -- -D warnings - name: run cargo sort (rust) From f72f9a28358624a48fbc91cfe6f4ed048604d6c2 Mon Sep 17 00:00:00 2001 From: James Olds Date: Wed, 18 Jan 2023 16:13:52 -0500 Subject: [PATCH 17/19] add go steps --- .github/workflows/lint.yml | 27 ++++++++++++++++++++++++++- 1 file changed, 26 insertions(+), 1 deletion(-) diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index be17be8..169f7d5 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -2,7 +2,7 @@ on: workflow_call: inputs: type: - description: "the expected lint entrypoint" + description: "the expected lint entrypoint: a language name or either 'make' or 'just'" default: "make" required: false type: string @@ -11,6 +11,16 @@ on: default: "" required: false type: string + go-version-file: + description: "the file to get the go version from" + default: "go.mod" + required: false + type: string + golangci-lint-version: + description: "the golangci-lint version for Go linting" + default: "v1.50.1" + required: false + type: string cargo-sort: description: "run cargo-sort as part of Rust linting" default: false @@ -24,6 +34,7 @@ jobs: steps: - uses: actions/checkout@v3 + # Python - name: configure python, if required if: inputs.python-version != '' uses: actions/setup-python@v4 @@ -32,6 +43,7 @@ jobs: cache: "pip" cache-dependency-path: pyproject.toml + # Rust - name: configure rust and clippy, if required if: inputs.type == 'rust' run: | @@ -58,6 +70,19 @@ jobs: cargo install cargo-sort cargo sort -c + # Go + - name: setup go + if: inputs.type == 'go' + uses: actions/setup-go@v3 + with: + go-version-file: "${{ inputs.go-version-file }}" + - name: run golangci-lint (go) + if: inputs.type == 'go' + uses: golangci/golangci-lint-action@0ad9a0988b3973e851ab0a07adf248ec2e100376 # @v3.3.1 + with: + version: "${{ inputs.golangci-lint-version }}" + + # Make - name: run lint (make) if: inputs.type == 'make' run: | From 2aa1af1d16c0f5ac5de553af23f93de6022e80f5 Mon Sep 17 00:00:00 2001 From: William Woodruff Date: Wed, 18 Jan 2023 16:20:52 -0500 Subject: [PATCH 18/19] lint: add RUFF_FORMAT Signed-off-by: William Woodruff --- .github/workflows/lint.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 169f7d5..70df326 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -76,6 +76,7 @@ jobs: uses: actions/setup-go@v3 with: go-version-file: "${{ inputs.go-version-file }}" + - name: run golangci-lint (go) if: inputs.type == 'go' uses: golangci/golangci-lint-action@0ad9a0988b3973e851ab0a07adf248ec2e100376 # @v3.3.1 @@ -85,5 +86,9 @@ jobs: # Make - name: run lint (make) if: inputs.type == 'make' + env: + # NOTE: If ruff is an underlying linter, this makes it generate + # GitHub-style annotations. + RUFF_FORMAT: github run: | make lint From 9283d515e467ff4b474098f2d0aab79e1c202274 Mon Sep 17 00:00:00 2001 From: William Woodruff Date: Wed, 18 Jan 2023 16:39:39 -0500 Subject: [PATCH 19/19] workflows/lint: workflow-level env Signed-off-by: William Woodruff --- .github/workflows/lint.yml | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 70df326..4d12980 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -27,6 +27,13 @@ on: required: false type: boolean +env: + # Rust: GitHub Actions supports color codes, so always enable them. + CARGO_TERM_COLOR: always + + # Python: Tell ruff that we're in GitHub Actions, so that it emits annotations. + RUFF_FORMAT: github + jobs: lint: runs-on: ubuntu-latest @@ -86,9 +93,4 @@ jobs: # Make - name: run lint (make) if: inputs.type == 'make' - env: - # NOTE: If ruff is an underlying linter, this makes it generate - # GitHub-style annotations. - RUFF_FORMAT: github - run: | - make lint + run: make lint