From 7a6de423367f381a9604646e5e9a073e88b2603b Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Fri, 12 Dec 2025 16:36:51 +0000 Subject: [PATCH 1/4] Initial plan From 6ae448ad3c3db676cdc77cb117c2b2901573244d Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Fri, 12 Dec 2025 16:40:30 +0000 Subject: [PATCH 2/4] Copy composite actions from opencb/java-common-libs TASK-8067 branch Co-authored-by: juanfeSanahuja <85166064+juanfeSanahuja@users.noreply.github.com> --- .github/actions/setup-java-maven/action.yml | 130 ++++++++++--- .github/actions/test-summary/action.yml | 205 +++++++++++++++++--- 2 files changed, 291 insertions(+), 44 deletions(-) diff --git a/.github/actions/setup-java-maven/action.yml b/.github/actions/setup-java-maven/action.yml index 123760c..694b2e8 100644 --- a/.github/actions/setup-java-maven/action.yml +++ b/.github/actions/setup-java-maven/action.yml @@ -1,26 +1,112 @@ -name: 'Setup Java and Maven' -description: 'Composite action to set up Java and Maven environment for OpenCB/Xetabase projects' - -# TODO: Define inputs for Java version, Maven version, cache configuration, etc. -# inputs: -# java-version: -# description: 'Java version to set up' -# required: false -# default: '11' - -# TODO: Define outputs if needed -# outputs: -# java-version: -# description: 'Installed Java version' -# value: ${{ steps.setup.outputs.java-version }} +name: "Setup Java, dependencies and Maven cache" +description: "Set up JDK, clone and optionally compile dependencies, and configure Maven cache using a key that includes dependencies_sha" + +inputs: + java_version: + description: "Java version to use" + required: false + default: "11" + storage_hadoop: + description: "Hadoop flavour, used as part of the Maven cache key" + required: false + default: "hdi5.1" + dependency_repos: + description: "Comma-separated list of dependency repositories (e.g. 'java-common-libs,biodata,cellbase')" + required: false + default: "" + require_cache_hit: + description: "If true, fail the job when the Maven cache is not found for the current dependencies_sha" + required: false + default: "false" +outputs: + dependencies_sha: + description: "Hash that represents dependency commits (OpenCGA/java-common-libs/biodata/cellbase, etc.)" + value: ${{ steps.clone_dependencies.outputs.dependencies_sha }} + cache-hit: + description: "True if the Maven cache key with this dependencies_sha was already present" + value: ${{ steps.maven_cache.outputs.cache-hit }} runs: - using: 'composite' + using: "composite" steps: - # TODO: Add steps to set up Java - # TODO: Add steps to set up Maven - # TODO: Add caching for Maven dependencies - # TODO: Configure Maven settings if needed - - name: Placeholder + - name: Set up JDK + # This step installs the requested JDK version for the job + uses: actions/setup-java@v4 + with: + distribution: "temurin" + java-version: ${{ inputs.java_version }} + + - name: Clone dependencies and compute dependencies_sha + id: clone_dependencies + if: ${{ inputs.dependency_repos != '' }} + shell: bash + run: | + # This step clones the dependency repositories using the helper script + # and computes a stable dependencies_sha based on all dependency commits. + # + # It expects get_same_branch.sh to: + # - clone or checkout each repo on the same branch as the caller + # - update DEPENDENCIES_SHA in-place + # - finally write `dependencies_sha=` into $GITHUB_OUTPUT + + if [ -f "java-common-libs/.github/workflows/scripts/get_same_branch.sh" ]; then + chmod +x java-common-libs/.github/workflows/scripts/get_same_branch.sh + + # Initialize DEPENDENCIES_SHA with the current repo SHA so that + # main project changes also affect the final cache key. + export DEPENDENCIES_SHA="${{ github.sha }}" + + java-common-libs/.github/workflows/scripts/get_same_branch.sh \ + "${{ github.ref_name }}" \ + "${{ inputs.dependency_repos }}" + else + echo "get_same_branch.sh script not found. Skipping dependency checkout." + # Fallback: use only the current repo SHA as dependencies_sha + DEPENDENCIES_SHA="${{ github.sha }}" + echo "dependencies_sha=${DEPENDENCIES_SHA}" >> "$GITHUB_OUTPUT" + fi + + - name: Cache local Maven repository + id: maven_cache + # This step restores and saves the Maven local repository cache, using + # storage_hadoop and dependencies_sha as part of the cache key. + uses: actions/cache@v4 + with: + path: ~/.m2/repository + key: ${{ runner.os }}-maven-${{ inputs.storage_hadoop }}-${{ steps.clone_dependencies.outputs.dependencies_sha }} + restore-keys: | + ${{ runner.os }}-maven-${{ inputs.storage_hadoop }}- + ## Force cache hit to avoid analyzing with incomplete dependencies + fail-on-cache-miss: ${{ inputs.require_cache_hit }} + - name: Compile dependencies (only if cache did not hit) + if: ${{ inputs.dependency_repos != '' && steps.maven_cache.outputs.cache-hit != 'true' }} + shell: bash + run: | + # This step compiles dependency repositories only when the Maven cache + # for the current dependencies_sha was not found. + # + # It expects compile_same_branch.sh to: + # - iterate over the comma-separated dependency_repos list + # - run `mvn clean install -DskipTests` on each dependency + # - populate ~/.m2/repository with the latest built artifacts + + if [ -f "java-common-libs/.github/workflows/scripts/compile_same_branch.sh" ]; then + chmod +x java-common-libs/.github/workflows/scripts/compile_same_branch.sh + java-common-libs/.github/workflows/scripts/compile_same_branch.sh "${{ inputs.dependency_repos }}" + else + echo "compile_same_branch.sh script not found. Skipping dependency compilation." + fi + + - name: Cache summary + # This step writes a small summary into the GitHub job summary, so it is + # easy to inspect which dependencies_sha and cache key were used. shell: bash - run: echo "Setup Java and Maven action - to be implemented" + run: | + { + echo "## Java / Maven cache summary" + echo "" + echo "- Java version: \`${{ inputs.java_version }}\`" + echo "- Dependencies sha: \`${{ steps.clone_dependencies.outputs.dependencies_sha || 'No dependencies' }}\`" + echo "- Maven cache key: \`${{ runner.os }}-maven-${{ inputs.storage_hadoop }}-${{ steps.clone_dependencies.outputs.dependencies_sha }}\`" + echo "- Maven cache hit: \`${{ steps.maven_cache.outputs.cache-hit || 'false' }}\`" + } >> "$GITHUB_STEP_SUMMARY" diff --git a/.github/actions/test-summary/action.yml b/.github/actions/test-summary/action.yml index 94517f0..d1c47b1 100644 --- a/.github/actions/test-summary/action.yml +++ b/.github/actions/test-summary/action.yml @@ -1,26 +1,187 @@ -name: 'Test Summary' -description: 'Composite action to generate and publish test summaries for OpenCB/Xetabase projects' - -# TODO: Define inputs for test results paths, formats, etc. -# inputs: -# test-results-path: -# description: 'Path to test results' -# required: false -# default: '**/target/surefire-reports/*.xml' - -# TODO: Define outputs if needed -# outputs: -# summary: -# description: 'Test summary' -# value: ${{ steps.summary.outputs.summary }} +name: "Test summary" +description: "Generate a Markdown summary of Surefire test results and write it to GITHUB_STEP_SUMMARY" + +inputs: + report_paths: + description: "Glob pattern for Surefire XML reports" + required: false + default: "./**/surefire-reports/TEST-*.xml" + title: + description: "Title for the test summary section" + required: false + default: "Test summary" + include_module_table: + description: "Whether to include the per-module breakdown table" + required: false + default: "true" + +outputs: + total_success: + description: "Total number of successful tests" + value: ${{ steps.generate_summary.outputs.total_success }} + total_failed: + description: "Total number of failed tests" + value: ${{ steps.generate_summary.outputs.total_failed }} + total_errors: + description: "Total number of tests with errors" + value: ${{ steps.generate_summary.outputs.total_errors }} + total_skipped: + description: "Total number of skipped tests" + value: ${{ steps.generate_summary.outputs.total_skipped }} + total_tests: + description: "Total number of tests" + value: ${{ steps.generate_summary.outputs.total_tests }} runs: - using: 'composite' + using: "composite" steps: - # TODO: Add steps to collect test results - # TODO: Add steps to parse test results - # TODO: Add steps to generate summary - # TODO: Add steps to publish summary to GitHub Actions UI - - name: Placeholder + - name: Generate test summary + id: generate_summary shell: bash - run: echo "Test summary action - to be implemented" + env: + REPORT_GLOB: ${{ inputs.report_paths }} + SUMMARY_TITLE: ${{ inputs.title }} + INCLUDE_MODULE_TABLE: ${{ inputs.include_module_table }} + run: | + # Use Python to parse Surefire XML reports and generate a Markdown summary + python - << 'PY' + import glob + import os + import xml.etree.ElementTree as ET + import sys + + report_glob = os.environ.get("REPORT_GLOB", "./**/surefire-reports/TEST-*.xml") + title = os.environ.get("SUMMARY_TITLE", "Test summary") + include_module_table = os.environ.get("INCLUDE_MODULE_TABLE", "true").lower() == "true" + github_step_summary = os.environ.get("GITHUB_STEP_SUMMARY") + github_output = os.environ.get("GITHUB_OUTPUT") + + # Collect all report files + report_files = glob.glob(report_glob, recursive=True) + + # Data structures for totals and per-module aggregation + total = { + "tests": 0, + "failures": 0, + "errors": 0, + "skipped": 0, + } + modules = {} + + def update_counts(target, tests, failures, errors, skipped): + target["tests"] += tests + target["failures"] += failures + target["errors"] += errors + target["skipped"] += skipped + + for path in report_files: + try: + tree = ET.parse(path) + root = tree.getroot() + except Exception as e: + # Skip malformed XML files + continue + + # Surefire can be or root + suites = [] + if root.tag == "testsuite": + suites = [root] + elif root.tag == "testsuites": + suites = list(root.findall("testsuite")) + + if not suites: + continue + + # Deduce module name from path (dir before 'target') + # Example: some-module/target/surefire-reports/TEST-*.xml -> module = 'some-module' + parts = path.split(os.sep) + module_name = "root" + if "target" in parts: + idx = parts.index("target") + if idx > 0: + module_name = parts[idx - 1] + + if module_name not in modules: + modules[module_name] = { + "tests": 0, + "failures": 0, + "errors": 0, + "skipped": 0, + } + + for suite in suites: + def get_int(attr_name): + value = suite.attrib.get(attr_name, "0") + try: + return int(value) + except ValueError: + return 0 + + tests = get_int("tests") + failures = get_int("failures") + errors = get_int("errors") + skipped = get_int("skipped") + + update_counts(total, tests, failures, errors, skipped) + update_counts(modules[module_name], tests, failures, errors, skipped) + + # Compute derived values + def compute_success(data): + return max(0, data["tests"] - data["failures"] - data["errors"] - data["skipped"]) + + total_success = compute_success(total) + + # Build Markdown summary + lines = [] + + lines.append(f"## {title}") + lines.append("") + if not report_files: + lines.append("_No test reports were found with pattern:_") + lines.append(f"`{report_glob}`") + else: + # Overall table + lines.append("### Overall") + lines.append("") + lines.append("| Success | Failed | Errors | Skipped | Total |") + lines.append("| --- | --- | --- | --- | --- |") + lines.append( + f"| {total_success} | {total['failures']} | {total['errors']} | {total['skipped']} | {total['tests']} |" + ) + lines.append("") + + if include_module_table and modules: + lines.append("### By module") + lines.append("") + lines.append("| Module | Success | Failed | Errors | Skipped | Total |") + lines.append("| --- | --- | --- | --- | --- | --- |") + for module_name in sorted(modules.keys()): + data = modules[module_name] + success = compute_success(data) + lines.append( + f"| {module_name} | {success} | {data['failures']} | {data['errors']} | {data['skipped']} | {data['tests']} |" + ) + lines.append("") + + # Optional detail: number of XML report files found + lines.append(f"_Processed {len(report_files)} Surefire report file(s)._") + + markdown = "\n".join(lines) + + # Write to step summary if available + if github_step_summary: + with open(github_step_summary, "a", encoding="utf-8") as f: + f.write(markdown + "\n") + + # Write outputs for reuse + if github_output: + def write_output(name, value): + with open(github_output, "a", encoding="utf-8") as f: + f.write(f"{name}={value}\n") + + write_output("total_success", total_success) + write_output("total_failed", total["failures"]) + write_output("total_errors", total["errors"]) + write_output("total_skipped", total["skipped"]) + write_output("total_tests", total["tests"]) + PY From ca11f416e4fea883c37907e2f10852d7fce6d596 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Fri, 12 Dec 2025 16:46:49 +0000 Subject: [PATCH 3/4] Add documentation and patch file for java-common-libs migration Co-authored-by: juanfeSanahuja <85166064+juanfeSanahuja@users.noreply.github.com> --- MIGRATION_INSTRUCTIONS.md | 86 ++++++++++++++++++++++++++++++++++ README.md | 64 ++++++++++++++++++++++++- java-common-libs-updates.patch | 63 +++++++++++++++++++++++++ 3 files changed, 212 insertions(+), 1 deletion(-) create mode 100644 MIGRATION_INSTRUCTIONS.md create mode 100644 java-common-libs-updates.patch diff --git a/MIGRATION_INSTRUCTIONS.md b/MIGRATION_INSTRUCTIONS.md new file mode 100644 index 0000000..17019b6 --- /dev/null +++ b/MIGRATION_INSTRUCTIONS.md @@ -0,0 +1,86 @@ +# Migration Instructions for Composite GitHub Actions + +This document provides instructions for completing the migration of composite GitHub Actions from `opencb/java-common-libs` to `opencb/ci-core`. + +## What Has Been Done + +### 1. Actions Migrated to ci-core Repository + +The following composite actions have been successfully copied from the `TASK-8067` branch of `opencb/java-common-libs` to this repository: + +- `.github/actions/setup-java-maven/action.yml` - Sets up Java, dependencies, and Maven cache +- `.github/actions/test-summary/action.yml` - Generates and publishes test summaries + +These actions are now available in the `opencb/ci-core` repository and can be referenced by other repositories. + +### 2. Patch File Created for java-common-libs + +A patch file (`java-common-libs-updates.patch`) has been created that contains all the necessary changes to update the workflow files in the `opencb/java-common-libs` repository (branch `TASK-8067`) to reference the migrated actions in `opencb/ci-core`. + +## What Needs to Be Done + +### Apply the Patch to opencb/java-common-libs + +The patch file needs to be applied to the `TASK-8067` branch of the `opencb/java-common-libs` repository. This can be done using one of the following methods: + +#### Method 1: Using git apply (Recommended) + +```bash +# Clone the repository and checkout the TASK-8067 branch +git clone https://github.com/opencb/java-common-libs.git +cd java-common-libs +git checkout TASK-8067 + +# Apply the patch +git apply /path/to/java-common-libs-updates.patch + +# Review the changes +git diff + +# Commit and push the changes +git add . +git commit -m "Update action references to use opencb/ci-core repository" +git push origin TASK-8067 +``` + +#### Method 2: Manual Updates + +Alternatively, the following workflow files can be manually updated in the `TASK-8067` branch: + +1. `.github/workflows/build-java-app-workflow.yml` - Line 43 + - Change: `uses: ./.github/actions/setup-java-maven` + - To: `uses: opencb/ci-core/.github/actions/setup-java-maven@main` + +2. `.github/workflows/test-analysis.yml` - Lines 24 and 39 + - Change: `uses: ./.github/actions/setup-java-maven` + - To: `uses: opencb/ci-core/.github/actions/setup-java-maven@main` + - Change: `uses: ./.github/actions/test-summary` + - To: `uses: opencb/ci-core/.github/actions/test-summary@main` + +3. `.github/workflows/test-xetabase-workflow.yml` - Line 126 + - Change: `uses: ./.github/actions/test-summary` + - To: `uses: opencb/ci-core/.github/actions/test-summary@main` + +## Summary of Changes + +The patch updates 3 workflow files to use the centralized actions: + +- **build-java-app-workflow.yml**: 1 reference updated +- **test-analysis.yml**: 2 references updated +- **test-xetabase-workflow.yml**: 1 reference updated + +Total: 4 action references now point to `opencb/ci-core/.github/actions/*@main` + +## Verification + +After applying the patch to the `opencb/java-common-libs` repository: + +1. Verify that all workflow files reference the correct actions +2. Test the workflows to ensure they work correctly with the centralized actions +3. Monitor the first few workflow runs to catch any issues early + +## Notes + +- The actions use `@main` to reference the main branch of the ci-core repository +- If you need to reference a specific version or tag, update the `@main` suffix accordingly +- The actions in ci-core preserve all the original functionality and comments from java-common-libs diff --git a/README.md b/README.md index 3cfcd93..2a18238 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,64 @@ # ci-core -Core CI/CD components for OpenCB and Xetabase projects. Centralized reusable GitHub Actions, composite actions, and internal workflows implementing our organization’s opinionated build, test, and release rules. + +Core CI/CD components for OpenCB and Xetabase projects. Centralized reusable GitHub Actions, composite actions, and internal workflows implementing our organization's opinionated build, test, and release rules. + +## Available Composite Actions + +### setup-java-maven + +**Location:** `.github/actions/setup-java-maven/` + +Sets up Java, dependencies, and Maven cache for OpenCB projects. + +**Usage:** +```yaml +- name: Setup Java and Maven + uses: opencb/ci-core/.github/actions/setup-java-maven@main + with: + java_version: '11' # Optional, default: '11' + storage_hadoop: 'hdi5.1' # Optional, default: 'hdi5.1' + dependency_repos: 'repo1,repo2' # Optional, default: '' + require_cache_hit: 'false' # Optional, default: 'false' +``` + +**Inputs:** +- `java_version`: Java version to use (default: "11") +- `storage_hadoop`: Hadoop flavour, used as part of the Maven cache key (default: "hdi5.1") +- `dependency_repos`: Comma-separated list of dependency repositories to clone and compile +- `require_cache_hit`: If true, fail the job when the Maven cache is not found + +**Outputs:** +- `dependencies_sha`: Hash representing dependency commits +- `cache-hit`: True if the Maven cache was found + +### test-summary + +**Location:** `.github/actions/test-summary/` + +Generates a Markdown summary of Surefire test results and writes it to GITHUB_STEP_SUMMARY. + +**Usage:** +```yaml +- name: Generate Test Summary + uses: opencb/ci-core/.github/actions/test-summary@main + with: + report_paths: './**/surefire-reports/TEST-*.xml' # Optional + title: 'Test Results' # Optional + include_module_table: 'true' # Optional +``` + +**Inputs:** +- `report_paths`: Glob pattern for Surefire XML reports (default: "./**/surefire-reports/TEST-*.xml") +- `title`: Title for the test summary section (default: "Test summary") +- `include_module_table`: Whether to include the per-module breakdown table (default: "true") + +**Outputs:** +- `total_success`: Total number of successful tests +- `total_failed`: Total number of failed tests +- `total_errors`: Total number of tests with errors +- `total_skipped`: Total number of skipped tests +- `total_tests`: Total number of tests + +## Migration + +See [MIGRATION_INSTRUCTIONS.md](MIGRATION_INSTRUCTIONS.md) for details on migrating actions from other repositories. diff --git a/java-common-libs-updates.patch b/java-common-libs-updates.patch new file mode 100644 index 0000000..3033966 --- /dev/null +++ b/java-common-libs-updates.patch @@ -0,0 +1,63 @@ +From 71467d072104c55bf7a1585f4a83253f077ae75f Mon Sep 17 00:00:00 2001 +From: "github-actions[bot]" +Date: Fri, 12 Dec 2025 16:41:30 +0000 +Subject: [PATCH] Update action references to use opencb/ci-core repository + +--- + .github/workflows/build-java-app-workflow.yml | 2 +- + .github/workflows/test-analysis.yml | 4 ++-- + .github/workflows/test-xetabase-workflow.yml | 2 +- + 3 files changed, 4 insertions(+), 4 deletions(-) + +diff --git a/.github/workflows/build-java-app-workflow.yml b/.github/workflows/build-java-app-workflow.yml +index cf7774b..1461fca 100644 +--- a/.github/workflows/build-java-app-workflow.yml ++++ b/.github/workflows/build-java-app-workflow.yml +@@ -40,7 +40,7 @@ jobs: + # This step sets up JDK, clones and optionally compiles dependencies, + # and configures the Maven cache for the whole job. + id: setup_java_maven +- uses: ./.github/actions/setup-java-maven ++ uses: opencb/ci-core/.github/actions/setup-java-maven@main + with: + java_version: ${{ inputs.java_version }} + storage_hadoop: ${{ inputs.hadoop_flavour || vars.HADOOP_FLAVOUR }} +diff --git a/.github/workflows/test-analysis.yml b/.github/workflows/test-analysis.yml +index a02a907..4b4fbe9 100644 +--- a/.github/workflows/test-analysis.yml ++++ b/.github/workflows/test-analysis.yml +@@ -21,7 +21,7 @@ jobs: + - name: Setup Java and Maven cache + # Reuse the composite action to configure Java and Maven cache + id: setup_java_maven +- uses: ./.github/actions/setup-java-maven ++ uses: opencb/ci-core/.github/actions/setup-java-maven@main + with: + require_cache_hit: true + - name: Start MongoDB v6.0 +@@ -36,7 +36,7 @@ jobs: + run: mvn -B verify surefire-report:report --fail-never org.sonarsource.scanner.maven:sonar-maven-plugin:sonar -Dsonar.projectKey=opencb_java-common-libs --no-transfer-progress + - name: Append test summary to job summary + if: success() || failure() +- uses: ./.github/actions/test-summary ++ uses: opencb/ci-core/.github/actions/test-summary@main + with: + report_paths: './**/surefire-reports/TEST-*.xml' + title: "Java tests summary" +\ No newline at end of file +diff --git a/.github/workflows/test-xetabase-workflow.yml b/.github/workflows/test-xetabase-workflow.yml +index d8d573a..1949a1a 100644 +--- a/.github/workflows/test-xetabase-workflow.yml ++++ b/.github/workflows/test-xetabase-workflow.yml +@@ -123,7 +123,7 @@ jobs: + cat ./opencga-enterprise/build.log | tee -a $GITHUB_STEP_SUMMARY + - name: Append test summary to job summary + if: success() || failure() +- uses: ./.github/actions/test-summary ++ uses: opencb/ci-core/.github/actions/test-summary@main + with: + report_paths: './**/surefire-reports/TEST-*.xml' + title: "Xetabase tests summary" +-- +2.52.0 + From 16dc24c3582c10a5fd5c8b57e0dc3c7c4d06c4d6 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Fri, 12 Dec 2025 16:48:35 +0000 Subject: [PATCH 4/4] Fix cache key to fallback to github.sha when no dependencies provided Co-authored-by: juanfeSanahuja <85166064+juanfeSanahuja@users.noreply.github.com> --- .github/actions/setup-java-maven/action.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/actions/setup-java-maven/action.yml b/.github/actions/setup-java-maven/action.yml index 694b2e8..14fdee2 100644 --- a/.github/actions/setup-java-maven/action.yml +++ b/.github/actions/setup-java-maven/action.yml @@ -73,7 +73,7 @@ runs: uses: actions/cache@v4 with: path: ~/.m2/repository - key: ${{ runner.os }}-maven-${{ inputs.storage_hadoop }}-${{ steps.clone_dependencies.outputs.dependencies_sha }} + key: ${{ runner.os }}-maven-${{ inputs.storage_hadoop }}-${{ steps.clone_dependencies.outputs.dependencies_sha || github.sha }} restore-keys: | ${{ runner.os }}-maven-${{ inputs.storage_hadoop }}- ## Force cache hit to avoid analyzing with incomplete dependencies