From 610f1d47d4be3140086d529937f7e8aa9de1e2de Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?David=20Gonz=C3=A1lez?= Date: Fri, 19 Dec 2025 16:30:16 +0100 Subject: [PATCH 1/3] Maestro: Add detail to error flows --- .github/workflows/ads-end-to-end.yml | 79 ------------- .../design-system-composable-pr-test.yml | 11 ++ .github/workflows/duckplayer.yml | 11 ++ .../e2e-nightly-android-design-system.yml | 111 ++++++++++++++++++ .github/workflows/e2e-nightly-autofill.yml | 62 +++++++++- ...ightly.yml => e2e-nightly-custom-tabs.yml} | 27 ++++- .github/workflows/end-to-end-robintest.yml | 99 ++++++++++++++++ .github/workflows/input-screen-e2e-tests.yml | 11 ++ .../workflows/omnibar-nightly-e2e-tests.yml | 11 ++ .../privacy-dashboard-end-to-end.yml | 22 ++++ .github/workflows/release_tests.yml | 57 ++------- .github/workflows/security-e2e-tests.yml | 11 ++ .../workflows/security-internal-e2e-tests.yml | 11 ++ .github/workflows/sync-critical-path.yml | 11 ++ 14 files changed, 399 insertions(+), 135 deletions(-) delete mode 100644 .github/workflows/ads-end-to-end.yml create mode 100644 .github/workflows/e2e-nightly-android-design-system.yml rename .github/workflows/{custom-tabs-nightly.yml => e2e-nightly-custom-tabs.yml} (74%) diff --git a/.github/workflows/ads-end-to-end.yml b/.github/workflows/ads-end-to-end.yml deleted file mode 100644 index f59a55bb8147..000000000000 --- a/.github/workflows/ads-end-to-end.yml +++ /dev/null @@ -1,79 +0,0 @@ -name: Android Design System End-to-End tests - -on: - schedule: - - cron: '0 5 * * *' # run at 5 AM UTC - workflow_dispatch: - -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true - -jobs: - instrumentation_tests: - runs-on: ubuntu-latest - name: End-to-End tests - - steps: - - name: Checkout repository - uses: actions/checkout@v4 - with: - submodules: recursive - - - name: Set up JDK version - uses: actions/setup-java@v4 - with: - java-version-file: .github/.java-version - distribution: 'adopt' - - - name: Create folder - if: always() - run: mkdir apk - - - name: Decode keys - uses: davidSchuppa/base64Secret-toFile-action@v2 - with: - secret: ${{ secrets.FAKE_RELEASE_PROPERTIES }} - fileName: ddg_android_build.properties - destination-path: $HOME/jenkins_static/com.duckduckgo.mobile.android/ - - - name: Decode key file - uses: davidSchuppa/base64Secret-toFile-action@v2 - with: - secret: ${{ secrets.FAKE_RELEASE_KEY }} - fileName: android - destination-path: $HOME/jenkins_static/com.duckduckgo.mobile.android/ - - - name: Setup Gradle - uses: gradle/actions/setup-gradle@v3 - - - name: Assemble the project - run: ./gradlew assembleInternalRelease -Pforce-default-variant - - - name: Move APK to new folder - if: always() - run: find . -name "*.apk" -exec mv '{}' apk/release.apk \; - - - name: ADS Preview Flows - uses: mobile-dev-inc/action-maestro-cloud@v1.9.8 - timeout-minutes: 120 - with: - api-key: ${{ secrets.ROBIN_API_KEY }} - project-id: ${{ vars.ROBIN_ANDROID_PROJECT_ID }} - name: androidDesignSystem_${{ github.sha }} - timeout: ${{ vars.ROBIN_TIMEOUT_MINUTES }} - app-file: apk/release.apk - android-api-level: 30 - workspace: .maestro - include-tags: androidDesignSystemTest - - - name: Create Asana task when workflow failed - if: ${{ failure() }} - uses: honeycombio/gha-create-asana-task@main - with: - asana-secret: ${{ secrets.ASANA_ACCESS_TOKEN }} - asana-workspace-id: ${{ secrets.GH_ASANA_WORKSPACE_ID }} - asana-project-id: ${{ secrets.GH_ASANA_AOR_PROJECT_ID }} - asana-section-id: ${{ secrets.GH_ASANA_INCOMING_ID }} - asana-task-name: GH Workflow Failure - ADS Preview test (Robin) - asana-task-description: The ADS Preview end to end workflow has failed. See https://github.com/duckduckgo/Android/actions/runs/${{ github.run_id }} \ No newline at end of file diff --git a/.github/workflows/design-system-composable-pr-test.yml b/.github/workflows/design-system-composable-pr-test.yml index 242dd43fa078..e37f85a61cbc 100644 --- a/.github/workflows/design-system-composable-pr-test.yml +++ b/.github/workflows/design-system-composable-pr-test.yml @@ -55,6 +55,7 @@ jobs: run: find . -name "*internal-release*.apk" -exec mv '{}' apk/internal.apk \; - name: Run Design System Maestro Tests + id: maestro-run uses: mobile-dev-inc/action-maestro-cloud@v1.9.8 timeout-minutes: 30 with: @@ -67,6 +68,16 @@ jobs: workspace: .maestro include-tags: androidDesignSystemTest + - name: Analyze Maestro Flow Results + if: always() + id: analyze-maestro + uses: ./.github/actions/maestro-flow-analyzer + with: + flow_results_json: ${{ steps.maestro-run.outputs.MAESTRO_CLOUD_FLOW_RESULTS }} + console_url: ${{ steps.maestro-run.outputs.MAESTRO_CLOUD_CONSOLE_URL }} + upload_status: ${{ steps.maestro-run.outputs.MAESTRO_CLOUD_UPLOAD_STATUS }} + app_binary_id: ${{ steps.maestro-run.outputs.MAESTRO_CLOUD_APP_BINARY_ID }} + - name: Comment on PR if: failure() uses: actions/github-script@v8 diff --git a/.github/workflows/duckplayer.yml b/.github/workflows/duckplayer.yml index 4fe256060f01..499bdf98e5fe 100644 --- a/.github/workflows/duckplayer.yml +++ b/.github/workflows/duckplayer.yml @@ -66,6 +66,7 @@ jobs: - name: Duck Player Tests if: always() + id: maestro-run uses: mobile-dev-inc/action-maestro-cloud@v1.9.7 timeout-minutes: 120 with: @@ -78,6 +79,16 @@ jobs: workspace: .maestro include-tags: duckplayer + - name: Analyze Maestro Flow Results + if: always() + id: analyze-maestro + uses: ./.github/actions/maestro-flow-analyzer + with: + flow_results_json: ${{ steps.maestro-run.outputs.MAESTRO_CLOUD_FLOW_RESULTS }} + console_url: ${{ steps.maestro-run.outputs.MAESTRO_CLOUD_CONSOLE_URL }} + upload_status: ${{ steps.maestro-run.outputs.MAESTRO_CLOUD_UPLOAD_STATUS }} + app_binary_id: ${{ steps.maestro-run.outputs.MAESTRO_CLOUD_APP_BINARY_ID }} + - name: Create Asana task when workflow failed if: ${{ failure() && github.event_name != 'workflow_dispatch' }} uses: honeycombio/gha-create-asana-task@main diff --git a/.github/workflows/e2e-nightly-android-design-system.yml b/.github/workflows/e2e-nightly-android-design-system.yml new file mode 100644 index 000000000000..f791eb77479b --- /dev/null +++ b/.github/workflows/e2e-nightly-android-design-system.yml @@ -0,0 +1,111 @@ +name: 'End to End Tests - Android Design System (Nightly)' + +on: + schedule: + - cron: '0 5 * * *' # run at 5 AM UTC + workflow_dispatch: + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + instrumentation_tests: + runs-on: ubuntu-latest + name: End-to-End tests + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + submodules: recursive + + - name: Set up JDK version + uses: actions/setup-java@v4 + with: + java-version-file: .github/.java-version + distribution: 'adopt' + + - name: Create folder + if: always() + run: mkdir apk + + - name: Decode keys + uses: davidSchuppa/base64Secret-toFile-action@v2 + with: + secret: ${{ secrets.FAKE_RELEASE_PROPERTIES }} + fileName: ddg_android_build.properties + destination-path: $HOME/jenkins_static/com.duckduckgo.mobile.android/ + + - name: Decode key file + uses: davidSchuppa/base64Secret-toFile-action@v2 + with: + secret: ${{ secrets.FAKE_RELEASE_KEY }} + fileName: android + destination-path: $HOME/jenkins_static/com.duckduckgo.mobile.android/ + + - name: Setup Gradle + uses: gradle/actions/setup-gradle@v3 + + - name: Assemble the project + run: ./gradlew assembleInternalRelease -Pforce-default-variant + + - name: Move APK to new folder + if: always() + run: find . -name "*.apk" -exec mv '{}' apk/release.apk \; + + - name: ADS Preview Flows + id: maestro-run + uses: mobile-dev-inc/action-maestro-cloud@v1.9.8 + timeout-minutes: 120 + with: + api-key: ${{ secrets.ROBIN_API_KEY }} + project-id: ${{ vars.ROBIN_ANDROID_PROJECT_ID }} + name: androidDesignSystem_${{ github.sha }} + timeout: ${{ vars.ROBIN_TIMEOUT_MINUTES }} + app-file: apk/release.apk + android-api-level: 30 + workspace: .maestro + include-tags: androidDesignSystemTest + + - name: Analyze Maestro Flow Results + if: always() + id: analyze-maestro + uses: ./.github/actions/maestro-flow-analyzer + with: + flow_results_json: ${{ steps.maestro-run.outputs.MAESTRO_CLOUD_FLOW_RESULTS }} + console_url: ${{ steps.maestro-run.outputs.MAESTRO_CLOUD_CONSOLE_URL }} + upload_status: ${{ steps.maestro-run.outputs.MAESTRO_CLOUD_UPLOAD_STATUS }} + app_binary_id: ${{ steps.maestro-run.outputs.MAESTRO_CLOUD_APP_BINARY_ID }} + + - name: Show analyzer output + if: always() + run: echo "flow_summary_status=${{ steps.analyze-maestro.outputs.flow_summary_status }}" + + - name: Create Asana task when Maestro tests failed + if: always() && steps.analyze-maestro.outputs.flow_summary_status == 'failure' + id: create-maestro-web-failure-task + uses: duckduckgo/native-github-asana-sync@v2.0 + with: + asana-pat: ${{ secrets.ASANA_ACCESS_TOKEN }} + asana-project: ${{ vars.GH_ANDROID_APP_PROJECT_ID }} + asana-section: ${{ vars.GH_ANDROID_APP_INCOMING_SECTION_ID }} + asana-task-name: E2E Tests Failure - Android Design System + asana-task-description: | + Run: https://github.com/duckduckgo/Android/actions/runs/${{ github.run_id }} + Console: ${{ github.event.inputs.console_url }} + Summary: + ${{ steps.analyze-maestro.outputs.flow_summary_message }} + action: 'create-asana-task' + + - name: Create Asana task when workflow failed + if: ${{ failure() && steps.analyze-maestro-critical-path.outputs.flow_summary_status != 'failure' && steps.analyze-maestro-web-flows.outputs.flow_summary_status != 'failure' }} + id: create-failure-task + uses: duckduckgo/native-github-asana-sync@v2.0 + with: + asana-pat: ${{ secrets.ASANA_ACCESS_TOKEN }} + asana-project: ${{ vars.GH_ANDROID_APP_PROJECT_ID }} + asana-section: ${{ vars.GH_ANDROID_APP_INCOMING_SECTION_ID }} + asana-task-name: GH Workflow Failure - Autofill Critical Path (Nightly) + asana-task-description: Workflow execution has failed. See https://github.com/duckduckgo/Android/actions/runs/${{ github.run_id }} + action: 'create-asana-task' \ No newline at end of file diff --git a/.github/workflows/e2e-nightly-autofill.yml b/.github/workflows/e2e-nightly-autofill.yml index 3856db2bc127..8ff25a561e46 100644 --- a/.github/workflows/e2e-nightly-autofill.yml +++ b/.github/workflows/e2e-nightly-autofill.yml @@ -1,4 +1,4 @@ -name: Autofill Feature Critical Path End-to-End tests +name: 'End to End Tests - Autofill Critical Path (Nightly)' on: schedule: @@ -55,6 +55,7 @@ jobs: run: find . -name "*.apk" -exec mv '{}' apk/release.apk \; - name: Autofill Critical Path E2E Flows + id: maestro-critical-path uses: mobile-dev-inc/action-maestro-cloud@v1.9.8 timeout-minutes: 120 with: @@ -67,7 +68,34 @@ jobs: workspace: .maestro include-tags: autofillNoAuthTests + - name: Analyze Maestro Flow Results (Critical Path) + if: always() + id: analyze-maestro-critical-path + uses: ./.github/actions/maestro-flow-analyzer + with: + flow_results_json: ${{ steps.maestro-critical-path.outputs.MAESTRO_CLOUD_FLOW_RESULTS }} + console_url: ${{ steps.maestro-critical-path.outputs.MAESTRO_CLOUD_CONSOLE_URL }} + upload_status: ${{ steps.maestro-critical-path.outputs.MAESTRO_CLOUD_UPLOAD_STATUS }} + app_binary_id: ${{ steps.maestro-critical-path.outputs.MAESTRO_CLOUD_APP_BINARY_ID }} + + - name: Show analyzer output + if: always() + run: echo "flow_summary_status=${{ steps.analyze-maestro-critical-path.outputs.flow_summary_status }}" + + - name: Create Asana task when Maestro tests failed (Critical Path) + if: always() && steps.analyze-maestro-critical-path.outputs.flow_summary_status == 'failure' + id: create-maestro-critical-failure-task + uses: duckduckgo/native-github-asana-sync@v2.0 + with: + asana-pat: ${{ secrets.ASANA_ACCESS_TOKEN }} + asana-project: ${{ vars.GH_ANDROID_APP_PROJECT_ID }} + asana-section: ${{ vars.GH_ANDROID_APP_INCOMING_SECTION_ID }} + asana-task-name: E2E Tests Failure - Autofill Critical Path + asana-task-description: Autofill critical path Maestro tests failed. See https://github.com/duckduckgo/Android/actions/runs/${{ github.run_id }} + action: 'create-asana-task' + - name: Autofill web flows + id: maestro-web-flows uses: mobile-dev-inc/action-maestro-cloud@v1.9.8 timeout-minutes: 120 with: @@ -80,14 +108,40 @@ jobs: workspace: .maestro include-tags: autofillBackfillingUsername,autofillPasswordGeneration + - name: Analyze Maestro Flow Results (Web Flows) + if: always() + id: analyze-maestro-web-flows + uses: ./.github/actions/maestro-flow-analyzer + with: + flow_results_json: ${{ steps.maestro-web-flows.outputs.MAESTRO_CLOUD_FLOW_RESULTS }} + console_url: ${{ steps.maestro-web-flows.outputs.MAESTRO_CLOUD_CONSOLE_URL }} + upload_status: ${{ steps.maestro-web-flows.outputs.MAESTRO_CLOUD_UPLOAD_STATUS }} + app_binary_id: ${{ steps.maestro-web-flows.outputs.MAESTRO_CLOUD_APP_BINARY_ID }} + + - name: Show analyzer output + if: always() + run: echo "flow_summary_status=${{ steps.analyze-maestro-web-flows.outputs.flow_summary_status }}" + + - name: Create Asana task when Maestro tests failed (Web Flows) + if: always() && steps.analyze-maestro-web-flows.outputs.flow_summary_status == 'failure' + id: create-maestro-web-failure-task + uses: duckduckgo/native-github-asana-sync@v2.0 + with: + asana-pat: ${{ secrets.ASANA_ACCESS_TOKEN }} + asana-project: ${{ vars.GH_ANDROID_APP_PROJECT_ID }} + asana-section: ${{ vars.GH_ANDROID_APP_INCOMING_SECTION_ID }} + asana-task-name: E2E Tests Failure - Autofill Web Flows + asana-task-description: Autofill web flows Maestro tests failed. See https://github.com/duckduckgo/Android/actions/runs/${{ github.run_id }} + action: 'create-asana-task' + - name: Create Asana task when workflow failed - if: ${{ failure() }} + if: ${{ failure() && steps.analyze-maestro-critical-path.outputs.flow_summary_status != 'failure' && steps.analyze-maestro-web-flows.outputs.flow_summary_status != 'failure' }} id: create-failure-task uses: duckduckgo/native-github-asana-sync@v2.0 with: asana-pat: ${{ secrets.ASANA_ACCESS_TOKEN }} asana-project: ${{ vars.GH_ANDROID_APP_PROJECT_ID }} asana-section: ${{ vars.GH_ANDROID_APP_INCOMING_SECTION_ID }} - asana-task-name: GH Workflow Failure - Autofill E2E Flows (Robin) - asana-task-description: Autofill tests have failed. See https://github.com/duckduckgo/Android/actions/runs/${{ github.run_id }} + asana-task-name: GH Workflow Failure - Autofill Critical Path (Nightly) + asana-task-description: Workflow execution has failed. See https://github.com/duckduckgo/Android/actions/runs/${{ github.run_id }} action: 'create-asana-task' \ No newline at end of file diff --git a/.github/workflows/custom-tabs-nightly.yml b/.github/workflows/e2e-nightly-custom-tabs.yml similarity index 74% rename from .github/workflows/custom-tabs-nightly.yml rename to .github/workflows/e2e-nightly-custom-tabs.yml index 72d4644a9cf4..86cfdfef68fb 100644 --- a/.github/workflows/custom-tabs-nightly.yml +++ b/.github/workflows/e2e-nightly-custom-tabs.yml @@ -1,4 +1,4 @@ -name: Custom Tabs nightly tests +name: 'End to End Tests - Custom Tabs (Nightly)' on: schedule: @@ -55,6 +55,7 @@ jobs: run: find . -name "*.apk" -exec mv '{}' apk/release.apk \; - name: Custom Tabs Flows + id: maestro-run uses: mobile-dev-inc/action-maestro-cloud@v1.9.8 timeout-minutes: 120 with: @@ -67,17 +68,35 @@ jobs: workspace: .maestro include-tags: customTabsTest + - name: Analyze Maestro Flow Results + if: always() + id: analyze-maestro + uses: ./.github/actions/maestro-flow-analyzer + with: + flow_results_json: ${{ steps.maestro-run.outputs.MAESTRO_CLOUD_FLOW_RESULTS }} + console_url: ${{ steps.maestro-run.outputs.MAESTRO_CLOUD_CONSOLE_URL }} + upload_status: ${{ steps.maestro-run.outputs.MAESTRO_CLOUD_UPLOAD_STATUS }} + app_binary_id: ${{ steps.maestro-run.outputs.MAESTRO_CLOUD_APP_BINARY_ID }} + + - name: Show analyzer output + if: always() + run: echo "flow_summary_status=${{ steps.analyze-maestro.outputs.flow_summary_status }}" + - name: Create Asana task when workflow failed if: ${{ failure() }} id: create-failure-task uses: duckduckgo/native-github-asana-sync@v2.0 with: asana-pat: ${{ secrets.ASANA_ACCESS_TOKEN }} + action: 'create-asana-task' asana-project: ${{ vars.GH_ANDROID_APP_PROJECT_ID }} asana-section: ${{ vars.GH_ANDROID_APP_INCOMING_SECTION_ID }} - asana-task-name: GH Workflow Failure - Custom Tabs Flows (Robin) - asana-task-description: The Custom Tabs nightly workflow has failed. See https://github.com/duckduckgo/Android/actions/runs/${{ github.run_id }} - action: 'create-asana-task' + asana-task-name: GH Workflow Failure - Custom Tabs Flows + asana-task-description: | + Run: https://github.com/duckduckgo/Android/actions/runs/${{ github.run_id }} + Console: ${{ github.event.inputs.console_url }} + Summary: + ${{ steps.analyze-maestro.outputs.flow_summary_message }} - name: Adding as a release blocker if: ${{ failure() && steps.create-failure-task.outputs.taskId != '' }} diff --git a/.github/workflows/end-to-end-robintest.yml b/.github/workflows/end-to-end-robintest.yml index 87cb20e6bd3b..32a67c25da7a 100644 --- a/.github/workflows/end-to-end-robintest.yml +++ b/.github/workflows/end-to-end-robintest.yml @@ -62,6 +62,7 @@ jobs: run: find . -name "*internal-release*.apk" -exec mv '{}' apk/internal.apk \; - name: Onboarding flows + id: maestro-onboarding uses: mobile-dev-inc/action-maestro-cloud@v1.9.8 timeout-minutes: 120 with: @@ -74,7 +75,18 @@ jobs: workspace: .maestro include-tags: onboardingTest + - name: Analyze Maestro Flow Results (Onboarding) + if: always() + id: analyze-maestro-onboarding + uses: ./.github/actions/maestro-flow-analyzer + with: + flow_results_json: ${{ steps.maestro-onboarding.outputs.MAESTRO_CLOUD_FLOW_RESULTS }} + console_url: ${{ steps.maestro-onboarding.outputs.MAESTRO_CLOUD_CONSOLE_URL }} + upload_status: ${{ steps.maestro-onboarding.outputs.MAESTRO_CLOUD_UPLOAD_STATUS }} + app_binary_id: ${{ steps.maestro-onboarding.outputs.MAESTRO_CLOUD_APP_BINARY_ID }} + - name: Deeplink tests + id: maestro-deeplink uses: mobile-dev-inc/action-maestro-cloud@v1.9.8 timeout-minutes: 120 with: @@ -87,7 +99,18 @@ jobs: workspace: .maestro include-tags: navigationTest + - name: Analyze Maestro Flow Results (Deeplink) + if: always() + id: analyze-maestro-deeplink + uses: ./.github/actions/maestro-flow-analyzer + with: + flow_results_json: ${{ steps.maestro-deeplink.outputs.MAESTRO_CLOUD_FLOW_RESULTS }} + console_url: ${{ steps.maestro-deeplink.outputs.MAESTRO_CLOUD_CONSOLE_URL }} + upload_status: ${{ steps.maestro-deeplink.outputs.MAESTRO_CLOUD_UPLOAD_STATUS }} + app_binary_id: ${{ steps.maestro-deeplink.outputs.MAESTRO_CLOUD_APP_BINARY_ID }} + - name: Ad click detection flows + id: maestro-ad-click uses: mobile-dev-inc/action-maestro-cloud@v1.9.8 timeout-minutes: 120 with: @@ -100,8 +123,19 @@ jobs: workspace: .maestro include-tags: adClickTest + - name: Analyze Maestro Flow Results (Ad Click) + if: always() + id: analyze-maestro-ad-click + uses: ./.github/actions/maestro-flow-analyzer + with: + flow_results_json: ${{ steps.maestro-ad-click.outputs.MAESTRO_CLOUD_FLOW_RESULTS }} + console_url: ${{ steps.maestro-ad-click.outputs.MAESTRO_CLOUD_CONSOLE_URL }} + upload_status: ${{ steps.maestro-ad-click.outputs.MAESTRO_CLOUD_UPLOAD_STATUS }} + app_binary_id: ${{ steps.maestro-ad-click.outputs.MAESTRO_CLOUD_APP_BINARY_ID }} + - name: Privacy Tests if: always() + id: maestro-privacy uses: mobile-dev-inc/action-maestro-cloud@v1.9.8 timeout-minutes: 120 with: @@ -114,8 +148,19 @@ jobs: workspace: .maestro include-tags: privacyTest + - name: Analyze Maestro Flow Results (Privacy) + if: always() + id: analyze-maestro-privacy + uses: ./.github/actions/maestro-flow-analyzer + with: + flow_results_json: ${{ steps.maestro-privacy.outputs.MAESTRO_CLOUD_FLOW_RESULTS }} + console_url: ${{ steps.maestro-privacy.outputs.MAESTRO_CLOUD_CONSOLE_URL }} + upload_status: ${{ steps.maestro-privacy.outputs.MAESTRO_CLOUD_UPLOAD_STATUS }} + app_binary_id: ${{ steps.maestro-privacy.outputs.MAESTRO_CLOUD_APP_BINARY_ID }} + - name: Internal Privacy Tests if: always() + id: maestro-internal-privacy uses: mobile-dev-inc/action-maestro-cloud@v1.9.7 timeout-minutes: 120 with: @@ -128,8 +173,19 @@ jobs: workspace: .maestro include-tags: privacyTestInternal + - name: Analyze Maestro Flow Results (Internal Privacy) + if: always() + id: analyze-maestro-internal-privacy + uses: ./.github/actions/maestro-flow-analyzer + with: + flow_results_json: ${{ steps.maestro-internal-privacy.outputs.MAESTRO_CLOUD_FLOW_RESULTS }} + console_url: ${{ steps.maestro-internal-privacy.outputs.MAESTRO_CLOUD_CONSOLE_URL }} + upload_status: ${{ steps.maestro-internal-privacy.outputs.MAESTRO_CLOUD_UPLOAD_STATUS }} + app_binary_id: ${{ steps.maestro-internal-privacy.outputs.MAESTRO_CLOUD_APP_BINARY_ID }} + - name: Security Tests if: always() + id: maestro-security uses: mobile-dev-inc/action-maestro-cloud@v1.9.8 timeout-minutes: 120 with: @@ -142,9 +198,20 @@ jobs: workspace: .maestro include-tags: securityTest + - name: Analyze Maestro Flow Results (Security) + if: always() + id: analyze-maestro-security + uses: ./.github/actions/maestro-flow-analyzer + with: + flow_results_json: ${{ steps.maestro-security.outputs.MAESTRO_CLOUD_FLOW_RESULTS }} + console_url: ${{ steps.maestro-security.outputs.MAESTRO_CLOUD_CONSOLE_URL }} + upload_status: ${{ steps.maestro-security.outputs.MAESTRO_CLOUD_UPLOAD_STATUS }} + app_binary_id: ${{ steps.maestro-security.outputs.MAESTRO_CLOUD_APP_BINARY_ID }} + ## This workflow will be deleted once we close the https://app.asana.com/1/137249556945/project/1211724162604201/task/1212015085457815?focus=true task - name: Internal Security Tests if: always() + id: maestro-internal-security uses: mobile-dev-inc/action-maestro-cloud@v1.9.8 timeout-minutes: 120 with: @@ -157,8 +224,19 @@ jobs: workspace: .maestro include-tags: securityTestInternal + - name: Analyze Maestro Flow Results (Internal Security) + if: always() + id: analyze-maestro-internal-security + uses: ./.github/actions/maestro-flow-analyzer + with: + flow_results_json: ${{ steps.maestro-internal-security.outputs.MAESTRO_CLOUD_FLOW_RESULTS }} + console_url: ${{ steps.maestro-internal-security.outputs.MAESTRO_CLOUD_CONSOLE_URL }} + upload_status: ${{ steps.maestro-internal-security.outputs.MAESTRO_CLOUD_UPLOAD_STATUS }} + app_binary_id: ${{ steps.maestro-internal-security.outputs.MAESTRO_CLOUD_APP_BINARY_ID }} + - name: Release Tests if: always() + id: maestro-release uses: mobile-dev-inc/action-maestro-cloud@v1.9.8 timeout-minutes: 120 with: @@ -171,8 +249,19 @@ jobs: workspace: .maestro include-tags: releaseTest + - name: Analyze Maestro Flow Results (Release Tests) + if: always() + id: analyze-maestro-release + uses: ./.github/actions/maestro-flow-analyzer + with: + flow_results_json: ${{ steps.maestro-release.outputs.MAESTRO_CLOUD_FLOW_RESULTS }} + console_url: ${{ steps.maestro-release.outputs.MAESTRO_CLOUD_CONSOLE_URL }} + upload_status: ${{ steps.maestro-release.outputs.MAESTRO_CLOUD_UPLOAD_STATUS }} + app_binary_id: ${{ steps.maestro-release.outputs.MAESTRO_CLOUD_APP_BINARY_ID }} + - name: Notifications permissions Android 13+ if: always() + id: maestro-notification-permissions uses: mobile-dev-inc/action-maestro-cloud@v1.9.8 timeout-minutes: 120 with: @@ -185,6 +274,16 @@ jobs: workspace: .maestro include-tags: permissionsTest + - name: Analyze Maestro Flow Results (Notification Permissions) + if: always() + id: analyze-maestro-notification-permissions + uses: ./.github/actions/maestro-flow-analyzer + with: + flow_results_json: ${{ steps.maestro-notification-permissions.outputs.MAESTRO_CLOUD_FLOW_RESULTS }} + console_url: ${{ steps.maestro-notification-permissions.outputs.MAESTRO_CLOUD_CONSOLE_URL }} + upload_status: ${{ steps.maestro-notification-permissions.outputs.MAESTRO_CLOUD_UPLOAD_STATUS }} + app_binary_id: ${{ steps.maestro-notification-permissions.outputs.MAESTRO_CLOUD_APP_BINARY_ID }} + - name: Create Asana task when workflow failed if: ${{ failure() }} id: create-failure-task diff --git a/.github/workflows/input-screen-e2e-tests.yml b/.github/workflows/input-screen-e2e-tests.yml index 28f2eb29c785..af299c8a1d2f 100644 --- a/.github/workflows/input-screen-e2e-tests.yml +++ b/.github/workflows/input-screen-e2e-tests.yml @@ -53,6 +53,7 @@ jobs: run: find . -name "*.apk" -exec mv '{}' apk/playRelease.apk \; - name: Input Screen Maestro run + id: maestro-run uses: mobile-dev-inc/action-maestro-cloud@v1.9.8 timeout-minutes: 120 with: @@ -65,6 +66,16 @@ jobs: workspace: .maestro include-tags: inputScreenTest + - name: Analyze Maestro Flow Results + if: always() + id: analyze-maestro + uses: ./.github/actions/maestro-flow-analyzer + with: + flow_results_json: ${{ steps.maestro-run.outputs.MAESTRO_CLOUD_FLOW_RESULTS }} + console_url: ${{ steps.maestro-run.outputs.MAESTRO_CLOUD_CONSOLE_URL }} + upload_status: ${{ steps.maestro-run.outputs.MAESTRO_CLOUD_UPLOAD_STATUS }} + app_binary_id: ${{ steps.maestro-run.outputs.MAESTRO_CLOUD_APP_BINARY_ID }} + - name: Create Asana task when workflow failed if: ${{ failure() }} uses: honeycombio/gha-create-asana-task@main diff --git a/.github/workflows/omnibar-nightly-e2e-tests.yml b/.github/workflows/omnibar-nightly-e2e-tests.yml index 045b6e4100cb..187eccac3785 100644 --- a/.github/workflows/omnibar-nightly-e2e-tests.yml +++ b/.github/workflows/omnibar-nightly-e2e-tests.yml @@ -55,6 +55,7 @@ jobs: run: find . -name "*.apk" -exec mv '{}' apk/playRelease.apk \; - name: Omnibar Maestro run + id: maestro-run uses: mobile-dev-inc/action-maestro-cloud@v1.9.8 timeout-minutes: 120 with: @@ -67,6 +68,16 @@ jobs: workspace: .maestro include-tags: omnibarTest + - name: Analyze Maestro Flow Results + if: always() + id: analyze-maestro + uses: ./.github/actions/maestro-flow-analyzer + with: + flow_results_json: ${{ steps.maestro-run.outputs.MAESTRO_CLOUD_FLOW_RESULTS }} + console_url: ${{ steps.maestro-run.outputs.MAESTRO_CLOUD_CONSOLE_URL }} + upload_status: ${{ steps.maestro-run.outputs.MAESTRO_CLOUD_UPLOAD_STATUS }} + app_binary_id: ${{ steps.maestro-run.outputs.MAESTRO_CLOUD_APP_BINARY_ID }} + - name: Create Asana task when workflow failed if: ${{ failure() }} uses: honeycombio/gha-create-asana-task@main diff --git a/.github/workflows/privacy-dashboard-end-to-end.yml b/.github/workflows/privacy-dashboard-end-to-end.yml index c4cf886b2f5b..a879b1349f10 100644 --- a/.github/workflows/privacy-dashboard-end-to-end.yml +++ b/.github/workflows/privacy-dashboard-end-to-end.yml @@ -58,6 +58,7 @@ jobs: run: find . -name "*.apk" -exec mv '{}' apk/release.apk \; - name: Ad click detection flows + id: maestro-ad-click uses: mobile-dev-inc/action-maestro-cloud@v1.9.8 timeout-minutes: 120 with: @@ -70,8 +71,19 @@ jobs: workspace: .maestro include-tags: adClickTest + - name: Analyze Maestro Flow Results (Ad Click) + if: always() + id: analyze-maestro-ad-click + uses: ./.github/actions/maestro-flow-analyzer + with: + flow_results_json: ${{ steps.maestro-ad-click.outputs.MAESTRO_CLOUD_FLOW_RESULTS }} + console_url: ${{ steps.maestro-ad-click.outputs.MAESTRO_CLOUD_CONSOLE_URL }} + upload_status: ${{ steps.maestro-ad-click.outputs.MAESTRO_CLOUD_UPLOAD_STATUS }} + app_binary_id: ${{ steps.maestro-ad-click.outputs.MAESTRO_CLOUD_APP_BINARY_ID }} + - name: Privacy Tests if: always() + id: maestro-privacy-tests uses: mobile-dev-inc/action-maestro-cloud@v1.9.8 timeout-minutes: 120 with: @@ -82,3 +94,13 @@ jobs: app-file: apk/release.apk workspace: .maestro include-tags: privacyTest + + - name: Analyze Maestro Flow Results (Privacy Tests) + if: always() + id: analyze-maestro-privacy-tests + uses: ./.github/actions/maestro-flow-analyzer + with: + flow_results_json: ${{ steps.maestro-privacy-tests.outputs.MAESTRO_CLOUD_FLOW_RESULTS }} + console_url: ${{ steps.maestro-privacy-tests.outputs.MAESTRO_CLOUD_CONSOLE_URL }} + upload_status: ${{ steps.maestro-privacy-tests.outputs.MAESTRO_CLOUD_UPLOAD_STATUS }} + app_binary_id: ${{ steps.maestro-privacy-tests.outputs.MAESTRO_CLOUD_APP_BINARY_ID }} diff --git a/.github/workflows/release_tests.yml b/.github/workflows/release_tests.yml index 1e952a54f4a0..12cbac6191f0 100644 --- a/.github/workflows/release_tests.yml +++ b/.github/workflows/release_tests.yml @@ -87,54 +87,14 @@ jobs: include-tags: ${{ github.event.inputs.test-tag }} - name: Analyze Maestro Flow Results - id: analyze-flow-results if: always() - run: | - echo "Console URL: ${{ steps.release-tests.outputs.MAESTRO_CLOUD_CONSOLE_URL }}" - echo "Upload Status: ${{ steps.release-tests.outputs.MAESTRO_CLOUD_UPLOAD_STATUS }}" - echo "App Binary ID: ${{ steps.release-tests.outputs.MAESTRO_CLOUD_APP_BINARY_ID }}" - - flow_results_json='${{ steps.release-tests.outputs.MAESTRO_CLOUD_FLOW_RESULTS }}' - echo "Raw Flow Results JSON: $flow_results_json" - - # Default to success, change if issues are found - final_status="success" - - # Check for empty or invalid JSON (though Maestro action should provide valid JSON) - if ! echo "$flow_results_json" | jq -e . > /dev/null 2>&1; then - echo "::warning::MAESTRO_CLOUD_FLOW_RESULTS is not valid JSON or is empty." - final_status="unknown_format" - else - # Check for any flow with status "ERROR" - if echo "$flow_results_json" | jq -e '.[] | select(.status=="ERROR")' > /dev/null; then - echo "::error::At least one Maestro flow has status: ERROR." - final_status="failure" - fi - - # Check for any flow with status "CANCELED" - # You might decide if CANCELED flows also mean the overall status is a failure for your release - if echo "$flow_results_json" | jq -e '.[] | select(.status=="CANCELED")' > /dev/null; then - echo "::warning::At least one Maestro flow has status: CANCELED." - # If any canceled flow makes the whole thing a failure: - if [ "$final_status" != "failure" ]; then # Don't override if already a critical failure - final_status="canceled_present" # Or treat as "failure" if preferred - fi - fi - - # If after all checks, final_status is still "success", it means no "ERROR" or "CANCELED" - if [ "$final_status" == "success" ]; then - # Additional check: ensure there's at least one flow and it's not empty array if that's a concern - if echo "$flow_results_json" | jq -e '. | length > 0' > /dev/null; then - echo "All flows appear to be successful (no ERROR or CANCELED statuses found that are treated as errors)." - else - echo "::warning::MAESTRO_CLOUD_FLOW_RESULTS is an empty array. No flows reported." - final_status="empty_results" # Or "success" if empty results are acceptable - fi - fi - fi - - echo "Final determined status: $final_status" - echo "flow_summary_status=$final_status" >> $GITHUB_OUTPUT + id: analyze-flow-results + uses: ./.github/actions/maestro-flow-analyzer + with: + flow_results_json: ${{ steps.release-tests.outputs.MAESTRO_CLOUD_FLOW_RESULTS }} + console_url: ${{ steps.release-tests.outputs.MAESTRO_CLOUD_CONSOLE_URL }} + upload_status: ${{ steps.release-tests.outputs.MAESTRO_CLOUD_UPLOAD_STATUS }} + app_binary_id: ${{ steps.release-tests.outputs.MAESTRO_CLOUD_APP_BINARY_ID }} - name: Access Outputs (for debugging) if: always() @@ -142,7 +102,8 @@ jobs: echo "Console URL: ${{ steps.release-tests.outputs.MAESTRO_CLOUD_CONSOLE_URL }}" echo 'Flow Results (JSON): ${{ steps.release-tests.outputs.MAESTRO_CLOUD_FLOW_RESULTS }}' echo "Release Tests Step Conclusion: ${{ steps.release-tests.conclusion }}" # From Maestro action itself - echo "Analyzed Flow Summary Status: ${{ steps.analyze-flow-results.outputs.flow_summary_status }}" # From our script + echo "Analyzed Flow Summary Status: ${{ steps.analyze-flow-results.outputs.flow_summary_status }}" # From analyzer action + printf "Analyzed Flow Summary Message:\n%s\n" "${{ steps.analyze-flow-results.outputs.flow_summary_message }}" - name: Notify Mattermost - Maestro Tests ALL SUCCEEDED # Condition 1: Our script says success diff --git a/.github/workflows/security-e2e-tests.yml b/.github/workflows/security-e2e-tests.yml index 75f99cc3d86e..b892ec629a7e 100644 --- a/.github/workflows/security-e2e-tests.yml +++ b/.github/workflows/security-e2e-tests.yml @@ -53,6 +53,7 @@ jobs: run: find . -name "*internal-release*.apk" -exec mv '{}' apk/internal.apk \; - name: Security Maestro run + id: maestro-run uses: mobile-dev-inc/action-maestro-cloud@v1.9.8 timeout-minutes: 120 with: @@ -65,6 +66,16 @@ jobs: workspace: .maestro include-tags: securityTest + - name: Analyze Maestro Flow Results + if: always() + id: analyze-maestro + uses: ./.github/actions/maestro-flow-analyzer + with: + flow_results_json: ${{ steps.maestro-run.outputs.MAESTRO_CLOUD_FLOW_RESULTS }} + console_url: ${{ steps.maestro-run.outputs.MAESTRO_CLOUD_CONSOLE_URL }} + upload_status: ${{ steps.maestro-run.outputs.MAESTRO_CLOUD_UPLOAD_STATUS }} + app_binary_id: ${{ steps.maestro-run.outputs.MAESTRO_CLOUD_APP_BINARY_ID }} + - name: Create Asana task when workflow failed if: ${{ failure() }} uses: honeycombio/gha-create-asana-task@main diff --git a/.github/workflows/security-internal-e2e-tests.yml b/.github/workflows/security-internal-e2e-tests.yml index 2313f8b01432..2bfd067a3481 100644 --- a/.github/workflows/security-internal-e2e-tests.yml +++ b/.github/workflows/security-internal-e2e-tests.yml @@ -53,6 +53,7 @@ jobs: run: find . -name "*internal-release*.apk" -exec mv '{}' apk/internal.apk \; - name: Security Internal Maestro run + id: maestro-run uses: mobile-dev-inc/action-maestro-cloud@v1.9.8 timeout-minutes: 120 with: @@ -65,6 +66,16 @@ jobs: workspace: .maestro include-tags: securityTestInternal + - name: Analyze Maestro Flow Results + if: always() + id: analyze-maestro + uses: ./.github/actions/maestro-flow-analyzer + with: + flow_results_json: ${{ steps.maestro-run.outputs.MAESTRO_CLOUD_FLOW_RESULTS }} + console_url: ${{ steps.maestro-run.outputs.MAESTRO_CLOUD_CONSOLE_URL }} + upload_status: ${{ steps.maestro-run.outputs.MAESTRO_CLOUD_UPLOAD_STATUS }} + app_binary_id: ${{ steps.maestro-run.outputs.MAESTRO_CLOUD_APP_BINARY_ID }} + - name: Create Asana task when workflow failed if: ${{ failure() }} uses: honeycombio/gha-create-asana-task@main diff --git a/.github/workflows/sync-critical-path.yml b/.github/workflows/sync-critical-path.yml index a362f0038f6d..562f05269aa3 100644 --- a/.github/workflows/sync-critical-path.yml +++ b/.github/workflows/sync-critical-path.yml @@ -62,6 +62,7 @@ jobs: run: find . -name "*.apk" -exec mv '{}' apk/release.apk \; - name: Sync Flows + id: maestro-run uses: mobile-dev-inc/action-maestro-cloud@v1.9.8 timeout-minutes: 120 with: @@ -76,6 +77,16 @@ jobs: env: | CODE=${{ steps.sync-recovery-code.outputs.recovery-code }} + - name: Analyze Maestro Flow Results + if: always() + id: analyze-maestro + uses: ./.github/actions/maestro-flow-analyzer + with: + flow_results_json: ${{ steps.maestro-run.outputs.MAESTRO_CLOUD_FLOW_RESULTS }} + console_url: ${{ steps.maestro-run.outputs.MAESTRO_CLOUD_CONSOLE_URL }} + upload_status: ${{ steps.maestro-run.outputs.MAESTRO_CLOUD_UPLOAD_STATUS }} + app_binary_id: ${{ steps.maestro-run.outputs.MAESTRO_CLOUD_APP_BINARY_ID }} + - name: Create Asana task when workflow failed if: ${{ failure() }} id: create-failure-task From 415c429632ab8f2cc6d22c4ac6d662b20a425bfa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?David=20Gonz=C3=A1lez?= Date: Fri, 19 Dec 2025 23:07:06 +0100 Subject: [PATCH 2/3] added task creation for all flows --- .../e2e-nightly-android-design-system.yml | 6 +- .github/workflows/e2e-nightly-autofill.yml | 14 +- .github/workflows/e2e-nightly-custom-tabs.yml | 38 +- ...kplayer.yml => e2e-nightly-duckplayer.yml} | 36 +- .github/workflows/e2e-nightly-full-suite.yml | 531 ++++++++++++++++++ ...tests.yml => e2e-nightly-input-screen.yml} | 38 +- ...-e2e-tests.yml => e2e-nightly-omnibar.yml} | 38 +- ...yml => e2e-nightly-sync-critical-path.yml} | 37 +- ...d-to-end.yml => e2e-privacy-dashboard.yml} | 42 +- ...2e-tests.yml => e2e-security-internal.yml} | 38 +- ...ecurity-e2e-tests.yml => e2e-security.yml} | 38 +- .github/workflows/end-to-end-robintest.yml | 307 ---------- 12 files changed, 786 insertions(+), 377 deletions(-) rename .github/workflows/{duckplayer.yml => e2e-nightly-duckplayer.yml} (67%) create mode 100644 .github/workflows/e2e-nightly-full-suite.yml rename .github/workflows/{input-screen-e2e-tests.yml => e2e-nightly-input-screen.yml} (63%) rename .github/workflows/{omnibar-nightly-e2e-tests.yml => e2e-nightly-omnibar.yml} (63%) rename .github/workflows/{sync-critical-path.yml => e2e-nightly-sync-critical-path.yml} (68%) rename .github/workflows/{privacy-dashboard-end-to-end.yml => e2e-privacy-dashboard.yml} (62%) rename .github/workflows/{security-internal-e2e-tests.yml => e2e-security-internal.yml} (61%) rename .github/workflows/{security-e2e-tests.yml => e2e-security.yml} (60%) delete mode 100644 .github/workflows/end-to-end-robintest.yml diff --git a/.github/workflows/e2e-nightly-android-design-system.yml b/.github/workflows/e2e-nightly-android-design-system.yml index f791eb77479b..0bc535bad245 100644 --- a/.github/workflows/e2e-nightly-android-design-system.yml +++ b/.github/workflows/e2e-nightly-android-design-system.yml @@ -93,19 +93,19 @@ jobs: asana-task-name: E2E Tests Failure - Android Design System asana-task-description: | Run: https://github.com/duckduckgo/Android/actions/runs/${{ github.run_id }} - Console: ${{ github.event.inputs.console_url }} + Console: ${{ steps.maestro-run.outputs.MAESTRO_CLOUD_CONSOLE_URL }} Summary: ${{ steps.analyze-maestro.outputs.flow_summary_message }} action: 'create-asana-task' - name: Create Asana task when workflow failed - if: ${{ failure() && steps.analyze-maestro-critical-path.outputs.flow_summary_status != 'failure' && steps.analyze-maestro-web-flows.outputs.flow_summary_status != 'failure' }} + if: ${{ failure() && steps.analyze-maestro.outputs.flow_summary_status != 'failure' }} id: create-failure-task uses: duckduckgo/native-github-asana-sync@v2.0 with: asana-pat: ${{ secrets.ASANA_ACCESS_TOKEN }} asana-project: ${{ vars.GH_ANDROID_APP_PROJECT_ID }} asana-section: ${{ vars.GH_ANDROID_APP_INCOMING_SECTION_ID }} - asana-task-name: GH Workflow Failure - Autofill Critical Path (Nightly) + asana-task-name: GH Workflow Failure - Android Design System End to End Tests (Nightly) asana-task-description: Workflow execution has failed. See https://github.com/duckduckgo/Android/actions/runs/${{ github.run_id }} action: 'create-asana-task' \ No newline at end of file diff --git a/.github/workflows/e2e-nightly-autofill.yml b/.github/workflows/e2e-nightly-autofill.yml index 8ff25a561e46..5aa9db527f9e 100644 --- a/.github/workflows/e2e-nightly-autofill.yml +++ b/.github/workflows/e2e-nightly-autofill.yml @@ -91,7 +91,11 @@ jobs: asana-project: ${{ vars.GH_ANDROID_APP_PROJECT_ID }} asana-section: ${{ vars.GH_ANDROID_APP_INCOMING_SECTION_ID }} asana-task-name: E2E Tests Failure - Autofill Critical Path - asana-task-description: Autofill critical path Maestro tests failed. See https://github.com/duckduckgo/Android/actions/runs/${{ github.run_id }} + asana-task-description: | + Run: https://github.com/duckduckgo/Android/actions/runs/${{ github.run_id }} + Console: ${{ steps.maestro-critical-path.outputs.MAESTRO_CLOUD_CONSOLE_URL }} + Summary: + ${{ steps.analyze-maestro-critical-path.outputs.flow_summary_message }} action: 'create-asana-task' - name: Autofill web flows @@ -131,7 +135,11 @@ jobs: asana-project: ${{ vars.GH_ANDROID_APP_PROJECT_ID }} asana-section: ${{ vars.GH_ANDROID_APP_INCOMING_SECTION_ID }} asana-task-name: E2E Tests Failure - Autofill Web Flows - asana-task-description: Autofill web flows Maestro tests failed. See https://github.com/duckduckgo/Android/actions/runs/${{ github.run_id }} + asana-task-description: | + Run: https://github.com/duckduckgo/Android/actions/runs/${{ github.run_id }} + Console: ${{ steps.maestro-web-flows.outputs.MAESTRO_CLOUD_CONSOLE_URL }} + Summary: + ${{ steps.analyze-maestro-web-flows.outputs.flow_summary_message }} action: 'create-asana-task' - name: Create Asana task when workflow failed @@ -142,6 +150,6 @@ jobs: asana-pat: ${{ secrets.ASANA_ACCESS_TOKEN }} asana-project: ${{ vars.GH_ANDROID_APP_PROJECT_ID }} asana-section: ${{ vars.GH_ANDROID_APP_INCOMING_SECTION_ID }} - asana-task-name: GH Workflow Failure - Autofill Critical Path (Nightly) + asana-task-name: GH Workflow Failure - Autofill Critical Path End to End Tests (Nightly) asana-task-description: Workflow execution has failed. See https://github.com/duckduckgo/Android/actions/runs/${{ github.run_id }} action: 'create-asana-task' \ No newline at end of file diff --git a/.github/workflows/e2e-nightly-custom-tabs.yml b/.github/workflows/e2e-nightly-custom-tabs.yml index 86cfdfef68fb..2de9debea3ee 100644 --- a/.github/workflows/e2e-nightly-custom-tabs.yml +++ b/.github/workflows/e2e-nightly-custom-tabs.yml @@ -78,25 +78,43 @@ jobs: upload_status: ${{ steps.maestro-run.outputs.MAESTRO_CLOUD_UPLOAD_STATUS }} app_binary_id: ${{ steps.maestro-run.outputs.MAESTRO_CLOUD_APP_BINARY_ID }} - - name: Show analyzer output - if: always() - run: echo "flow_summary_status=${{ steps.analyze-maestro.outputs.flow_summary_status }}" - - - name: Create Asana task when workflow failed - if: ${{ failure() }} - id: create-failure-task + - name: Create Asana task when Maestro tests failed + if: always() && steps.analyze-maestro.outputs.flow_summary_status == 'failure' + id: create-maestro-failure-task uses: duckduckgo/native-github-asana-sync@v2.0 with: asana-pat: ${{ secrets.ASANA_ACCESS_TOKEN }} - action: 'create-asana-task' asana-project: ${{ vars.GH_ANDROID_APP_PROJECT_ID }} asana-section: ${{ vars.GH_ANDROID_APP_INCOMING_SECTION_ID }} - asana-task-name: GH Workflow Failure - Custom Tabs Flows + asana-task-name: E2E Tests Failure - Custom Tabs asana-task-description: | Run: https://github.com/duckduckgo/Android/actions/runs/${{ github.run_id }} - Console: ${{ github.event.inputs.console_url }} + Console: ${{ steps.maestro-run.outputs.MAESTRO_CLOUD_CONSOLE_URL }} Summary: ${{ steps.analyze-maestro.outputs.flow_summary_message }} + action: 'create-asana-task' + + - name: Adding as a release blocker + if: ${{ steps.create-maestro-failure-task.outputs.taskId != '' }} + uses: duckduckgo/native-github-asana-sync@v2.0 + with: + asana-pat: ${{ secrets.ASANA_ACCESS_TOKEN }} + asana-project: ${{ vars.GH_ANDROID_APP_RELEASES_PROJECT_ID }} + asana-section: ${{ vars.GH_ANDROID_APP_RELEASES_PROJECT_BLOCKER_SECTION_ID }} + asana-task-id: ${{ steps.create-maestro-failure-task.outputs.taskId }} + action: 'add-task-asana-project' + + - name: Create Asana task when workflow failed + if: ${{ failure() && steps.analyze-maestro.outputs.flow_summary_status != 'failure' }} + id: create-failure-task + uses: duckduckgo/native-github-asana-sync@v2.0 + with: + asana-pat: ${{ secrets.ASANA_ACCESS_TOKEN }} + asana-project: ${{ vars.GH_ANDROID_APP_PROJECT_ID }} + asana-section: ${{ vars.GH_ANDROID_APP_INCOMING_SECTION_ID }} + asana-task-name: GH Workflow Failure - Custom Tabs End to End Tests (Nightly) + asana-task-description: Workflow execution has failed. See https://github.com/duckduckgo/Android/actions/runs/${{ github.run_id }} + action: 'create-asana-task' - name: Adding as a release blocker if: ${{ failure() && steps.create-failure-task.outputs.taskId != '' }} diff --git a/.github/workflows/duckplayer.yml b/.github/workflows/e2e-nightly-duckplayer.yml similarity index 67% rename from .github/workflows/duckplayer.yml rename to .github/workflows/e2e-nightly-duckplayer.yml index 499bdf98e5fe..53730bff756f 100644 --- a/.github/workflows/duckplayer.yml +++ b/.github/workflows/e2e-nightly-duckplayer.yml @@ -1,4 +1,4 @@ -name: Duck Player Tests +name: 'End to End Tests - Duck Player (Nightly)' on: pull_request: @@ -89,13 +89,33 @@ jobs: upload_status: ${{ steps.maestro-run.outputs.MAESTRO_CLOUD_UPLOAD_STATUS }} app_binary_id: ${{ steps.maestro-run.outputs.MAESTRO_CLOUD_APP_BINARY_ID }} + - name: Show analyzer output + if: always() + run: echo "flow_summary_status=${{ steps.analyze-maestro.outputs.flow_summary_status }}" + + - name: Create Asana task when Maestro tests failed + if: always() && steps.analyze-maestro.outputs.flow_summary_status == 'failure' + id: create-maestro-failure-task + uses: duckduckgo/native-github-asana-sync@v2.0 + with: + asana-pat: ${{ secrets.ASANA_ACCESS_TOKEN }} + asana-project: ${{ vars.GH_ANDROID_APP_PROJECT_ID }} + asana-section: ${{ vars.GH_ANDROID_APP_INCOMING_SECTION_ID }} + asana-task-name: E2E Tests Failure - Duck Player + asana-task-description: | + Run: https://github.com/duckduckgo/Android/actions/runs/${{ github.run_id }} + Console: ${{ steps.maestro-run.outputs.MAESTRO_CLOUD_CONSOLE_URL }} + Summary: + ${{ steps.analyze-maestro.outputs.flow_summary_message }} + action: 'create-asana-task' + - name: Create Asana task when workflow failed - if: ${{ failure() && github.event_name != 'workflow_dispatch' }} - uses: honeycombio/gha-create-asana-task@main + if: ${{ failure() && github.event_name != 'workflow_dispatch' && steps.analyze-maestro.outputs.flow_summary_status != 'failure'}} + uses: duckduckgo/native-github-asana-sync@v2.0 with: - asana-secret: ${{ secrets.ASANA_ACCESS_TOKEN }} - asana-workspace-id: ${{ secrets.GH_ASANA_WORKSPACE_ID }} - asana-project-id: ${{ secrets.DUCK_PLAYER_AOR_PROJECT_ID }} - asana-section-id: ${{ secrets.DUCK_PLAYER_AOR_INCOMING_ID }} - asana-task-name: GH Workflow Failure - Duck Player tests + asana-pat: ${{ secrets.ASANA_ACCESS_TOKEN }} + asana-project: ${{ vars.GH_ANDROID_APP_PROJECT_ID }} + asana-section: ${{ vars.GH_ANDROID_APP_INCOMING_SECTION_ID }} + asana-task-name: GH Workflow Failure - Duck Player End to End Tests asana-task-description: The Duck Player workflow has failed. See https://github.com/duckduckgo/Android/actions/runs/${{ github.run_id }} + action: 'create-asana-task' \ No newline at end of file diff --git a/.github/workflows/e2e-nightly-full-suite.yml b/.github/workflows/e2e-nightly-full-suite.yml new file mode 100644 index 000000000000..8e5a28ca79ae --- /dev/null +++ b/.github/workflows/e2e-nightly-full-suite.yml @@ -0,0 +1,531 @@ +name: 'End to End Tests - Full Suite (Nightly)' + +on: + schedule: + - cron: '0 3 * * *' # run at 3 AM UTC + workflow_dispatch: + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + instrumentation_tests: + runs-on: ubuntu-latest + name: End-to-End tests + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + submodules: recursive + + - name: Set up JDK version + uses: actions/setup-java@v4 + with: + java-version-file: .github/.java-version + distribution: 'adopt' + + - name: Create folder + if: always() + run: mkdir apk + + - name: Decode keys + uses: davidSchuppa/base64Secret-toFile-action@v2 + with: + secret: ${{ secrets.FAKE_RELEASE_PROPERTIES }} + fileName: ddg_android_build.properties + destination-path: $HOME/jenkins_static/com.duckduckgo.mobile.android/ + + - name: Decode key file + uses: davidSchuppa/base64Secret-toFile-action@v2 + with: + secret: ${{ secrets.FAKE_RELEASE_KEY }} + fileName: android + destination-path: $HOME/jenkins_static/com.duckduckgo.mobile.android/ + + - name: Setup Gradle + uses: gradle/actions/setup-gradle@v3 + + - name: Assemble release APK + run: ./gradlew assemblePlayRelease -Pforce-default-variant -Pskip-onboarding + + - name: Move APK to new folder + if: always() + run: find . -name "*play-release*.apk" -exec mv '{}' apk/release.apk \; + + - name: Assemble internal release APK + run: ./gradlew assembleInternalRelease -Pforce-default-variant -Pskip-onboarding + + - name: Move APK to new folder + if: always() + run: find . -name "*internal-release*.apk" -exec mv '{}' apk/internal.apk \; + + - name: Onboarding flows + id: maestro-onboarding + uses: mobile-dev-inc/action-maestro-cloud@v1.9.8 + timeout-minutes: 120 + with: + api-key: ${{ secrets.ROBIN_API_KEY }} + project-id: ${{ vars.ROBIN_ANDROID_PROJECT_ID }} + name: onboardingTest_${{ github.sha }} + timeout: ${{ vars.ROBIN_TIMEOUT_MINUTES }} + app-file: apk/release.apk + android-api-level: 30 + workspace: .maestro + include-tags: onboardingTest + + - name: Analyze Maestro Flow Results (Onboarding) + if: always() + id: analyze-maestro-onboarding + uses: ./.github/actions/maestro-flow-analyzer + with: + flow_results_json: ${{ steps.maestro-onboarding.outputs.MAESTRO_CLOUD_FLOW_RESULTS }} + console_url: ${{ steps.maestro-onboarding.outputs.MAESTRO_CLOUD_CONSOLE_URL }} + upload_status: ${{ steps.maestro-onboarding.outputs.MAESTRO_CLOUD_UPLOAD_STATUS }} + app_binary_id: ${{ steps.maestro-onboarding.outputs.MAESTRO_CLOUD_APP_BINARY_ID }} + + - name: Create Asana task when Maestro tests failed + if: always() && steps.analyze-maestro-onboarding.outputs.flow_summary_status == 'failure' + id: create-maestro-onboarding-failure-task + uses: duckduckgo/native-github-asana-sync@v2.0 + with: + asana-pat: ${{ secrets.ASANA_ACCESS_TOKEN }} + asana-project: ${{ vars.GH_ANDROID_APP_PROJECT_ID }} + asana-section: ${{ vars.GH_ANDROID_APP_INCOMING_SECTION_ID }} + asana-task-name: E2E Tests Failure - Onboarding (Nightly) + asana-task-description: | + Run: https://github.com/duckduckgo/Android/actions/runs/${{ github.run_id }} + Console: ${{ steps.maestro-onboarding.outputs.MAESTRO_CLOUD_CONSOLE_URL }} + Summary: + ${{ steps.analyze-maestro-onboarding.outputs.flow_summary_message }} + action: 'create-asana-task' + + - name: Adding as a release blocker + if: ${{ failure() && steps.create-maestro-onboarding-failure-task.outputs.taskId != '' }} + uses: duckduckgo/native-github-asana-sync@v2.0 + with: + asana-pat: ${{ secrets.ASANA_ACCESS_TOKEN }} + asana-project: ${{ vars.GH_ANDROID_APP_RELEASES_PROJECT_ID }} + asana-section: ${{ vars.GH_ANDROID_APP_RELEASES_PROJECT_BLOCKER_SECTION_ID }} + asana-task-id: ${{ steps.create-maestro-onboarding-failure-task.outputs.taskId }} + action: 'add-task-asana-project' + + - name: Deeplink tests + id: maestro-deeplink + uses: mobile-dev-inc/action-maestro-cloud@v1.9.8 + timeout-minutes: 120 + with: + api-key: ${{ secrets.ROBIN_API_KEY }} + project-id: ${{ vars.ROBIN_ANDROID_PROJECT_ID }} + name: navigationTest_${{ github.sha }} + timeout: ${{ vars.ROBIN_TIMEOUT_MINUTES }} + app-file: apk/release.apk + android-api-level: 30 + workspace: .maestro + include-tags: navigationTest + + - name: Analyze Maestro Flow Results (Deeplink) + if: always() + id: analyze-maestro-deeplink + uses: ./.github/actions/maestro-flow-analyzer + with: + flow_results_json: ${{ steps.maestro-deeplink.outputs.MAESTRO_CLOUD_FLOW_RESULTS }} + console_url: ${{ steps.maestro-deeplink.outputs.MAESTRO_CLOUD_CONSOLE_URL }} + upload_status: ${{ steps.maestro-deeplink.outputs.MAESTRO_CLOUD_UPLOAD_STATUS }} + app_binary_id: ${{ steps.maestro-deeplink.outputs.MAESTRO_CLOUD_APP_BINARY_ID }} + + - name: Create Asana task when Maestro tests failed + if: always() && steps.analyze-maestro-deeplink.outputs.flow_summary_status == 'failure' + id: create-maestro-deeplink-failure-task + uses: duckduckgo/native-github-asana-sync@v2.0 + with: + asana-pat: ${{ secrets.ASANA_ACCESS_TOKEN }} + asana-project: ${{ vars.GH_ANDROID_APP_PROJECT_ID }} + asana-section: ${{ vars.GH_ANDROID_APP_INCOMING_SECTION_ID }} + asana-task-name: E2E Tests Failure - Deeplink (Nightly) + asana-task-description: | + Run: https://github.com/duckduckgo/Android/actions/runs/${{ github.run_id }} + Console: ${{ steps.maestro-deeplink.outputs.MAESTRO_CLOUD_CONSOLE_URL }} + Summary: + ${{ steps.analyze-maestro-deeplink.outputs.flow_summary_message }} + action: 'create-asana-task' + + - name: Adding as a release blocker + if: ${{ failure() && steps.create-maestro-deeplink-failure-task.outputs.taskId != '' }} + uses: duckduckgo/native-github-asana-sync@v2.0 + with: + asana-pat: ${{ secrets.ASANA_ACCESS_TOKEN }} + asana-project: ${{ vars.GH_ANDROID_APP_RELEASES_PROJECT_ID }} + asana-section: ${{ vars.GH_ANDROID_APP_RELEASES_PROJECT_BLOCKER_SECTION_ID }} + asana-task-id: ${{ steps.create-maestro-deeplink-failure-task.outputs.taskId }} + action: 'add-task-asana-project' + + - name: Ad click detection flows + id: maestro-ad-click + uses: mobile-dev-inc/action-maestro-cloud@v1.9.8 + timeout-minutes: 120 + with: + api-key: ${{ secrets.ROBIN_API_KEY }} + project-id: ${{ vars.ROBIN_ANDROID_PROJECT_ID }} + name: adClickTest_${{ github.sha }} + timeout: ${{ vars.ROBIN_TIMEOUT_MINUTES }} + app-file: apk/release.apk + android-api-level: 30 + workspace: .maestro + include-tags: adClickTest + + - name: Analyze Maestro Flow Results (Ad Click) + if: always() + id: analyze-maestro-ad-click + uses: ./.github/actions/maestro-flow-analyzer + with: + flow_results_json: ${{ steps.maestro-ad-click.outputs.MAESTRO_CLOUD_FLOW_RESULTS }} + console_url: ${{ steps.maestro-ad-click.outputs.MAESTRO_CLOUD_CONSOLE_URL }} + upload_status: ${{ steps.maestro-ad-click.outputs.MAESTRO_CLOUD_UPLOAD_STATUS }} + app_binary_id: ${{ steps.maestro-ad-click.outputs.MAESTRO_CLOUD_APP_BINARY_ID }} + + - name: Create Asana task when Maestro tests failed + if: always() && steps.analyze-maestro-ad-click.outputs.flow_summary_status == 'failure' + id: create-maestro-ad-click-failure-task + uses: duckduckgo/native-github-asana-sync@v2.0 + with: + asana-pat: ${{ secrets.ASANA_ACCESS_TOKEN }} + asana-project: ${{ vars.GH_ANDROID_APP_PROJECT_ID }} + asana-section: ${{ vars.GH_ANDROID_APP_INCOMING_SECTION_ID }} + asana-task-name: E2E Tests Failure - Ad Click (Nightly) + asana-task-description: | + Run: https://github.com/duckduckgo/Android/actions/runs/${{ github.run_id }} + Console: ${{ steps.maestro-ad-click.outputs.MAESTRO_CLOUD_CONSOLE_URL }} + Summary: + ${{ steps.analyze-maestro-ad-click.outputs.flow_summary_message }} + action: 'create-asana-task' + + - name: Adding as a release blocker + if: ${{ failure() && steps.create-maestro-ad-click-failure-task.outputs.taskId != '' }} + uses: duckduckgo/native-github-asana-sync@v2.0 + with: + asana-pat: ${{ secrets.ASANA_ACCESS_TOKEN }} + asana-project: ${{ vars.GH_ANDROID_APP_RELEASES_PROJECT_ID }} + asana-section: ${{ vars.GH_ANDROID_APP_RELEASES_PROJECT_BLOCKER_SECTION_ID }} + asana-task-id: ${{ steps.create-maestro-ad-click-failure-task.outputs.taskId }} + action: 'add-task-asana-project' + + - name: Privacy Tests + if: always() + id: maestro-privacy + uses: mobile-dev-inc/action-maestro-cloud@v1.9.8 + timeout-minutes: 120 + with: + api-key: ${{ secrets.ROBIN_API_KEY }} + project-id: ${{ vars.ROBIN_ANDROID_PROJECT_ID }} + name: privacyTest_${{ github.sha }} + timeout: ${{ vars.ROBIN_TIMEOUT_MINUTES }} + app-file: apk/release.apk + android-api-level: 30 + workspace: .maestro + include-tags: privacyTest + + - name: Analyze Maestro Flow Results (Privacy) + if: always() + id: analyze-maestro-privacy + uses: ./.github/actions/maestro-flow-analyzer + with: + flow_results_json: ${{ steps.maestro-privacy.outputs.MAESTRO_CLOUD_FLOW_RESULTS }} + console_url: ${{ steps.maestro-privacy.outputs.MAESTRO_CLOUD_CONSOLE_URL }} + upload_status: ${{ steps.maestro-privacy.outputs.MAESTRO_CLOUD_UPLOAD_STATUS }} + app_binary_id: ${{ steps.maestro-privacy.outputs.MAESTRO_CLOUD_APP_BINARY_ID }} + + - name: Create Asana task when Maestro tests failed + if: always() && steps.analyze-maestro-privacy.outputs.flow_summary_status == 'failure' + id: create-maestro-privacy-failure-task + uses: duckduckgo/native-github-asana-sync@v2.0 + with: + asana-pat: ${{ secrets.ASANA_ACCESS_TOKEN }} + asana-project: ${{ vars.GH_ANDROID_APP_PROJECT_ID }} + asana-section: ${{ vars.GH_ANDROID_APP_INCOMING_SECTION_ID }} + asana-task-name: E2E Tests Failure - Privacy (Nightly) + asana-task-description: | + Run: https://github.com/duckduckgo/Android/actions/runs/${{ github.run_id }} + Console: ${{ steps.maestro-privacy.outputs.MAESTRO_CLOUD_CONSOLE_URL }} + Summary: + ${{ steps.analyze-maestro-privacy.outputs.flow_summary_message }} + action: 'create-asana-task' + + - name: Adding as a release blocker + if: ${{ failure() && steps.create-maestro-privacy-failure-task.outputs.taskId != '' }} + uses: duckduckgo/native-github-asana-sync@v2.0 + with: + asana-pat: ${{ secrets.ASANA_ACCESS_TOKEN }} + asana-project: ${{ vars.GH_ANDROID_APP_RELEASES_PROJECT_ID }} + asana-section: ${{ vars.GH_ANDROID_APP_RELEASES_PROJECT_BLOCKER_SECTION_ID }} + asana-task-id: ${{ steps.create-maestro-privacy-failure-task.outputs.taskId }} + action: 'add-task-asana-project' + + - name: Internal Privacy Tests + if: always() + id: maestro-internal-privacy + uses: mobile-dev-inc/action-maestro-cloud@v1.9.7 + timeout-minutes: 120 + with: + api-key: ${{ secrets.ROBIN_API_KEY }} + project-id: ${{ vars.ROBIN_ANDROID_PROJECT_ID }} + name: internalPrivacyTest_${{ github.sha }} + timeout: ${{ vars.ROBIN_TIMEOUT_MINUTES }} + app-file: apk/internal.apk + android-api-level: 34 + workspace: .maestro + include-tags: privacyTestInternal + + - name: Analyze Maestro Flow Results (Internal Privacy) + if: always() + id: analyze-maestro-internal-privacy + uses: ./.github/actions/maestro-flow-analyzer + with: + flow_results_json: ${{ steps.maestro-internal-privacy.outputs.MAESTRO_CLOUD_FLOW_RESULTS }} + console_url: ${{ steps.maestro-internal-privacy.outputs.MAESTRO_CLOUD_CONSOLE_URL }} + upload_status: ${{ steps.maestro-internal-privacy.outputs.MAESTRO_CLOUD_UPLOAD_STATUS }} + app_binary_id: ${{ steps.maestro-internal-privacy.outputs.MAESTRO_CLOUD_APP_BINARY_ID }} + + - name: Create Asana task when Maestro tests failed + if: always() && steps.analyze-maestro-internal-privacy.outputs.flow_summary_status == 'failure' + id: create-maestro-internal-privacy-failure-task + uses: duckduckgo/native-github-asana-sync@v2.0 + with: + asana-pat: ${{ secrets.ASANA_ACCESS_TOKEN }} + asana-project: ${{ vars.GH_ANDROID_APP_PROJECT_ID }} + asana-section: ${{ vars.GH_ANDROID_APP_INCOMING_SECTION_ID }} + asana-task-name: E2E Tests Failure - Internal Privacy (Nightly) + asana-task-description: | + Run: https://github.com/duckduckgo/Android/actions/runs/${{ github.run_id }} + Console: ${{ steps.maestro-internal-privacy.outputs.MAESTRO_CLOUD_CONSOLE_URL }} + Summary: + ${{ steps.analyze-maestro-internal-privacy.outputs.flow_summary_message }} + action: 'create-asana-task' + + - name: Adding as a release blocker + if: ${{ failure() && steps.create-maestro-internal-privacy-failure-task.outputs.taskId != '' }} + uses: duckduckgo/native-github-asana-sync@v2.0 + with: + asana-pat: ${{ secrets.ASANA_ACCESS_TOKEN }} + asana-project: ${{ vars.GH_ANDROID_APP_RELEASES_PROJECT_ID }} + asana-section: ${{ vars.GH_ANDROID_APP_RELEASES_PROJECT_BLOCKER_SECTION_ID }} + asana-task-id: ${{ steps.create-maestro-internal-privacy-failure-task.outputs.taskId }} + action: 'add-task-asana-project' + + - name: Security Tests + if: always() + id: maestro-security + uses: mobile-dev-inc/action-maestro-cloud@v1.9.8 + timeout-minutes: 120 + with: + api-key: ${{ secrets.ROBIN_API_KEY }} + project-id: ${{ vars.ROBIN_ANDROID_PROJECT_ID }} + name: securityTest_${{ github.sha }} + timeout: ${{ vars.ROBIN_TIMEOUT_MINUTES }} + app-file: apk/release.apk + android-api-level: 30 + workspace: .maestro + include-tags: securityTest + + - name: Analyze Maestro Flow Results (Security) + if: always() + id: analyze-maestro-security + uses: ./.github/actions/maestro-flow-analyzer + with: + flow_results_json: ${{ steps.maestro-security.outputs.MAESTRO_CLOUD_FLOW_RESULTS }} + console_url: ${{ steps.maestro-security.outputs.MAESTRO_CLOUD_CONSOLE_URL }} + upload_status: ${{ steps.maestro-security.outputs.MAESTRO_CLOUD_UPLOAD_STATUS }} + app_binary_id: ${{ steps.maestro-security.outputs.MAESTRO_CLOUD_APP_BINARY_ID }} + + - name: Create Asana task when Maestro tests failed + if: always() && steps.analyze-maestro-security.outputs.flow_summary_status == 'failure' + id: create-maestro-security-failure-task + uses: duckduckgo/native-github-asana-sync@v2.0 + with: + asana-pat: ${{ secrets.ASANA_ACCESS_TOKEN }} + asana-project: ${{ vars.GH_ANDROID_APP_PROJECT_ID }} + asana-section: ${{ vars.GH_ANDROID_APP_INCOMING_SECTION_ID }} + asana-task-name: E2E Tests Failure - Security (Nightly) + asana-task-description: | + Run: https://github.com/duckduckgo/Android/actions/runs/${{ github.run_id }} + Console: ${{ steps.maestro-security.outputs.MAESTRO_CLOUD_CONSOLE_URL }} + Summary: + ${{ steps.analyze-maestro-security.outputs.flow_summary_message }} + action: 'create-asana-task' + + - name: Adding as a release blocker + if: ${{ failure() && steps.create-maestro-security-failure-task.outputs.taskId != '' }} + uses: duckduckgo/native-github-asana-sync@v2.0 + with: + asana-pat: ${{ secrets.ASANA_ACCESS_TOKEN }} + asana-project: ${{ vars.GH_ANDROID_APP_RELEASES_PROJECT_ID }} + asana-section: ${{ vars.GH_ANDROID_APP_RELEASES_PROJECT_BLOCKER_SECTION_ID }} + asana-task-id: ${{ steps.create-maestro-security-failure-task.outputs.taskId }} + action: 'add-task-asana-project' + + ## This workflow will be deleted once we close the https://app.asana.com/1/137249556945/project/1211724162604201/task/1212015085457815?focus=true task + - name: Internal Security Tests + if: always() + id: maestro-internal-security + uses: mobile-dev-inc/action-maestro-cloud@v1.9.8 + timeout-minutes: 120 + with: + api-key: ${{ secrets.ROBIN_API_KEY }} + project-id: ${{ vars.ROBIN_ANDROID_PROJECT_ID }} + name: internalSecurityTest_${{ github.sha }} + timeout: ${{ vars.ROBIN_TIMEOUT_MINUTES }} + app-file: apk/internal.apk + android-api-level: 30 + workspace: .maestro + include-tags: securityTestInternal + + - name: Analyze Maestro Flow Results (Internal Security) + if: always() + id: analyze-maestro-internal-security + uses: ./.github/actions/maestro-flow-analyzer + with: + flow_results_json: ${{ steps.maestro-internal-security.outputs.MAESTRO_CLOUD_FLOW_RESULTS }} + console_url: ${{ steps.maestro-internal-security.outputs.MAESTRO_CLOUD_CONSOLE_URL }} + upload_status: ${{ steps.maestro-internal-security.outputs.MAESTRO_CLOUD_UPLOAD_STATUS }} + app_binary_id: ${{ steps.maestro-internal-security.outputs.MAESTRO_CLOUD_APP_BINARY_ID }} + + - name: Create Asana task when Maestro tests failed + if: always() && steps.analyze-maestro-internal-security.outputs.flow_summary_status == 'failure' + id: create-maestro-internal-security-failure-task + uses: duckduckgo/native-github-asana-sync@v2.0 + with: + asana-pat: ${{ secrets.ASANA_ACCESS_TOKEN }} + asana-project: ${{ vars.GH_ANDROID_APP_PROJECT_ID }} + asana-section: ${{ vars.GH_ANDROID_APP_INCOMING_SECTION_ID }} + asana-task-name: E2E Tests Failure - Internal Security (Nightly) + asana-task-description: | + Run: https://github.com/duckduckgo/Android/actions/runs/${{ github.run_id }} + Console: ${{ steps.maestro-internal-security.outputs.MAESTRO_CLOUD_CONSOLE_URL }} + Summary: + ${{ steps.analyze-maestro-internal-security.outputs.flow_summary_message }} + action: 'create-asana-task' + + - name: Adding as a release blocker + if: ${{ failure() && steps.create-maestro-internal-security-failure-task.outputs.taskId != '' }} + uses: duckduckgo/native-github-asana-sync@v2.0 + with: + asana-pat: ${{ secrets.ASANA_ACCESS_TOKEN }} + asana-project: ${{ vars.GH_ANDROID_APP_RELEASES_PROJECT_ID }} + asana-section: ${{ vars.GH_ANDROID_APP_RELEASES_PROJECT_BLOCKER_SECTION_ID }} + asana-task-id: ${{ steps.create-maestro-internal-security-failure-task.outputs.taskId }} + action: 'add-task-asana-project' + + - name: Release Tests + if: always() + id: maestro-release + uses: mobile-dev-inc/action-maestro-cloud@v1.9.8 + timeout-minutes: 120 + with: + api-key: ${{ secrets.ROBIN_API_KEY }} + project-id: ${{ vars.ROBIN_ANDROID_PROJECT_ID }} + name: releaseTest_${{ github.sha }} + timeout: ${{ vars.ROBIN_TIMEOUT_MINUTES }} + app-file: apk/release.apk + android-api-level: 30 + workspace: .maestro + include-tags: releaseTest + + - name: Analyze Maestro Flow Results (Release Tests) + if: always() + id: analyze-maestro-release + uses: ./.github/actions/maestro-flow-analyzer + with: + flow_results_json: ${{ steps.maestro-release.outputs.MAESTRO_CLOUD_FLOW_RESULTS }} + console_url: ${{ steps.maestro-release.outputs.MAESTRO_CLOUD_CONSOLE_URL }} + upload_status: ${{ steps.maestro-release.outputs.MAESTRO_CLOUD_UPLOAD_STATUS }} + app_binary_id: ${{ steps.maestro-release.outputs.MAESTRO_CLOUD_APP_BINARY_ID }} + + - name: Create Asana task when Maestro tests failed + if: always() && steps.analyze-maestro-release.outputs.flow_summary_status == 'failure' + id: create-maestro-release-tests-failure-task + uses: duckduckgo/native-github-asana-sync@v2.0 + with: + asana-pat: ${{ secrets.ASANA_ACCESS_TOKEN }} + asana-project: ${{ vars.GH_ANDROID_APP_PROJECT_ID }} + asana-section: ${{ vars.GH_ANDROID_APP_INCOMING_SECTION_ID }} + asana-task-name: E2E Tests Failure - Release Tests (Nightly) + asana-task-description: | + Run: https://github.com/duckduckgo/Android/actions/runs/${{ github.run_id }} + Console: ${{ steps.maestro-release.outputs.MAESTRO_CLOUD_CONSOLE_URL }} + Summary: + ${{ steps.analyze-maestro-release.outputs.flow_summary_message }} + action: 'create-asana-task' + + - name: Adding as a release blocker + if: ${{ failure() && steps.create-maestro-release-tests-failure-task.outputs.taskId != '' }} + uses: duckduckgo/native-github-asana-sync@v2.0 + with: + asana-pat: ${{ secrets.ASANA_ACCESS_TOKEN }} + asana-project: ${{ vars.GH_ANDROID_APP_RELEASES_PROJECT_ID }} + asana-section: ${{ vars.GH_ANDROID_APP_RELEASES_PROJECT_BLOCKER_SECTION_ID }} + asana-task-id: ${{ steps.create-maestro-release-tests-failure-task.outputs.taskId }} + action: 'add-task-asana-project' + + - name: Notifications permissions Android 13+ + if: always() + id: maestro-notification-permissions + uses: mobile-dev-inc/action-maestro-cloud@v1.9.8 + timeout-minutes: 120 + with: + api-key: ${{ secrets.ROBIN_API_KEY }} + project-id: ${{ vars.ROBIN_ANDROID_PROJECT_ID }} + name: notificationPermissionTest_${{ github.sha }} + timeout: ${{ vars.ROBIN_TIMEOUT_MINUTES }} + app-file: apk/release.apk + android-api-level: 33 + workspace: .maestro + include-tags: permissionsTest + + - name: Analyze Maestro Flow Results (Notification Permissions) + if: always() + id: analyze-maestro-notification-permissions + uses: ./.github/actions/maestro-flow-analyzer + with: + flow_results_json: ${{ steps.maestro-notification-permissions.outputs.MAESTRO_CLOUD_FLOW_RESULTS }} + console_url: ${{ steps.maestro-notification-permissions.outputs.MAESTRO_CLOUD_CONSOLE_URL }} + upload_status: ${{ steps.maestro-notification-permissions.outputs.MAESTRO_CLOUD_UPLOAD_STATUS }} + app_binary_id: ${{ steps.maestro-notification-permissions.outputs.MAESTRO_CLOUD_APP_BINARY_ID }} + + - name: Create Asana task when Maestro tests failed + if: always() && steps.analyze-maestro-notification-permissions.outputs.flow_summary_status == 'failure' + id: create-maestro-notification-permissions-tests-failure-task + uses: duckduckgo/native-github-asana-sync@v2.0 + with: + asana-pat: ${{ secrets.ASANA_ACCESS_TOKEN }} + asana-project: ${{ vars.GH_ANDROID_APP_PROJECT_ID }} + asana-section: ${{ vars.GH_ANDROID_APP_INCOMING_SECTION_ID }} + asana-task-name: E2E Tests Failure - Notification Permissions (Nightly) + asana-task-description: | + Run: https://github.com/duckduckgo/Android/actions/runs/${{ github.run_id }} + Console: ${{ steps.maestro-notification-permissions.outputs.MAESTRO_CLOUD_CONSOLE_URL }} + Summary: + ${{ steps.analyze-maestro-notification-permissions.outputs.flow_summary_message }} + action: 'create-asana-task' + + - name: Adding as a release blocker + if: ${{ failure() && steps.create-maestro-notification-permissions-tests-failure-task.outputs.taskId != '' }} + uses: duckduckgo/native-github-asana-sync@v2.0 + with: + asana-pat: ${{ secrets.ASANA_ACCESS_TOKEN }} + asana-project: ${{ vars.GH_ANDROID_APP_RELEASES_PROJECT_ID }} + asana-section: ${{ vars.GH_ANDROID_APP_RELEASES_PROJECT_BLOCKER_SECTION_ID }} + asana-task-id: ${{ steps.create-maestro-notification-permissions-tests-failure-task.outputs.taskId }} + action: 'add-task-asana-project' + + - name: Create Asana task when workflow failed + if: ${{ failure() }} + id: create-failure-task + uses: duckduckgo/native-github-asana-sync@v2.0 + with: + asana-pat: ${{ secrets.ASANA_ACCESS_TOKEN }} + asana-project: ${{ secrets.GH_ASANA_AOR_PROJECT_ID }} + asana-section: ${{ secrets.GH_ASANA_INCOMING_ID }} + asana-task-name: GH Workflow Failure - End to End tests (Full Suite) + asana-task-description: The end to end workflow has failed. See https://github.com/duckduckgo/Android/actions/runs/${{ github.run_id }} + action: 'create-asana-task' \ No newline at end of file diff --git a/.github/workflows/input-screen-e2e-tests.yml b/.github/workflows/e2e-nightly-input-screen.yml similarity index 63% rename from .github/workflows/input-screen-e2e-tests.yml rename to .github/workflows/e2e-nightly-input-screen.yml index af299c8a1d2f..6e6058f956cd 100644 --- a/.github/workflows/input-screen-e2e-tests.yml +++ b/.github/workflows/e2e-nightly-input-screen.yml @@ -1,4 +1,4 @@ -name: Input Screen End-to-End tests +name: 'End to End Tests - Input Screen (Nightly)' on: workflow_dispatch: @@ -76,13 +76,33 @@ jobs: upload_status: ${{ steps.maestro-run.outputs.MAESTRO_CLOUD_UPLOAD_STATUS }} app_binary_id: ${{ steps.maestro-run.outputs.MAESTRO_CLOUD_APP_BINARY_ID }} + - name: Show analyzer output + if: always() + run: echo "flow_summary_status=${{ steps.analyze-maestro.outputs.flow_summary_status }}" + + - name: Create Asana task when Maestro tests failed + if: always() && steps.analyze-maestro.outputs.flow_summary_status == 'failure' + id: create-maestro-failure-task + uses: duckduckgo/native-github-asana-sync@v2.0 + with: + asana-pat: ${{ secrets.ASANA_ACCESS_TOKEN }} + asana-project: ${{ vars.GH_ANDROID_APP_PROJECT_ID }} + asana-section: ${{ vars.GH_ANDROID_APP_INCOMING_SECTION_ID }} + asana-task-name: E2E Tests Failure - Input Screen + asana-task-description: | + Run: https://github.com/duckduckgo/Android/actions/runs/${{ github.run_id }} + Console: ${{ steps.maestro-run.outputs.MAESTRO_CLOUD_CONSOLE_URL }} + Summary: + ${{ steps.analyze-maestro.outputs.flow_summary_message }} + action: 'create-asana-task' + - name: Create Asana task when workflow failed - if: ${{ failure() }} - uses: honeycombio/gha-create-asana-task@main + if: ${{ failure() && steps.analyze-maestro.outputs.flow_summary_status != 'failure' }} + uses: duckduckgo/native-github-asana-sync@v2.0 with: - asana-secret: ${{ secrets.ASANA_ACCESS_TOKEN }} - asana-workspace-id: ${{ secrets.GH_ASANA_WORKSPACE_ID }} - asana-project-id: ${{ secrets.GH_ASANA_AOR_PROJECT_ID }} - asana-section-id: ${{ secrets.GH_ASANA_INCOMING_ID }} - asana-task-name: GH Workflow Failure - Input Screen E2E tests - asana-task-description: The Input Screen end-to-end workflow has failed. See https://github.com/duckduckgo/Android/actions/runs/${{ github.run_id }} \ No newline at end of file + asana-pat: ${{ secrets.ASANA_ACCESS_TOKEN }} + asana-project: ${{ vars.GH_ANDROID_APP_PROJECT_ID }} + asana-section: ${{ vars.GH_ANDROID_APP_INCOMING_SECTION_ID }} + asana-task-name: GH Workflow Failure - Input Screen End to End Tests (Nightly) + asana-task-description: The Input Screen end-to-end workflow has failed. See https://github.com/duckduckgo/Android/actions/runs/${{ github.run_id }} + action: 'create-asana-task' \ No newline at end of file diff --git a/.github/workflows/omnibar-nightly-e2e-tests.yml b/.github/workflows/e2e-nightly-omnibar.yml similarity index 63% rename from .github/workflows/omnibar-nightly-e2e-tests.yml rename to .github/workflows/e2e-nightly-omnibar.yml index 187eccac3785..6fba7e4be967 100644 --- a/.github/workflows/omnibar-nightly-e2e-tests.yml +++ b/.github/workflows/e2e-nightly-omnibar.yml @@ -1,4 +1,4 @@ -name: Omnibar End-to-End tests +name: 'End to End Tests - Omnibar (Nightly)' on: schedule: @@ -78,13 +78,33 @@ jobs: upload_status: ${{ steps.maestro-run.outputs.MAESTRO_CLOUD_UPLOAD_STATUS }} app_binary_id: ${{ steps.maestro-run.outputs.MAESTRO_CLOUD_APP_BINARY_ID }} + - name: Show analyzer output + if: always() + run: echo "flow_summary_status=${{ steps.analyze-maestro.outputs.flow_summary_status }}" + + - name: Create Asana task when Maestro tests failed + if: always() && steps.analyze-maestro.outputs.flow_summary_status == 'failure' + id: create-maestro-failure-task + uses: duckduckgo/native-github-asana-sync@v2.0 + with: + asana-pat: ${{ secrets.ASANA_ACCESS_TOKEN }} + asana-project: ${{ vars.GH_ANDROID_APP_PROJECT_ID }} + asana-section: ${{ vars.GH_ANDROID_APP_INCOMING_SECTION_ID }} + asana-task-name: E2E Tests Failure - Omnibar + asana-task-description: | + Run: https://github.com/duckduckgo/Android/actions/runs/${{ github.run_id }} + Console: ${{ steps.maestro-run.outputs.MAESTRO_CLOUD_CONSOLE_URL }} + Summary: + ${{ steps.analyze-maestro.outputs.flow_summary_message }} + action: 'create-asana-task' + - name: Create Asana task when workflow failed - if: ${{ failure() }} - uses: honeycombio/gha-create-asana-task@main + if: ${{ failure() && steps.analyze-maestro.outputs.flow_summary_status != 'failure' }} + uses: duckduckgo/native-github-asana-sync@v2.0 with: - asana-secret: ${{ secrets.ASANA_ACCESS_TOKEN }} - asana-workspace-id: ${{ secrets.GH_ASANA_WORKSPACE_ID }} - asana-project-id: ${{ secrets.GH_ASANA_AOR_PROJECT_ID }} - asana-section-id: ${{ secrets.GH_ASANA_INCOMING_ID }} - asana-task-name: GH Workflow Failure - Omnibar E2E tests - asana-task-description: The Omnibar end-to-end workflow has failed. See https://github.com/duckduckgo/Android/actions/runs/${{ github.run_id }} \ No newline at end of file + asana-pat: ${{ secrets.ASANA_ACCESS_TOKEN }} + asana-project: ${{ vars.GH_ANDROID_APP_PROJECT_ID }} + asana-section: ${{ vars.GH_ANDROID_APP_INCOMING_SECTION_ID }} + asana-task-name: GH Workflow Failure - Input Screen End to End Tests (Nightly) + asana-task-description: The Omnibar end-to-end workflow has failed. See https://github.com/duckduckgo/Android/actions/runs/${{ github.run_id }} + action: 'create-asana-task' \ No newline at end of file diff --git a/.github/workflows/sync-critical-path.yml b/.github/workflows/e2e-nightly-sync-critical-path.yml similarity index 68% rename from .github/workflows/sync-critical-path.yml rename to .github/workflows/e2e-nightly-sync-critical-path.yml index 562f05269aa3..d6080d667083 100644 --- a/.github/workflows/sync-critical-path.yml +++ b/.github/workflows/e2e-nightly-sync-critical-path.yml @@ -1,4 +1,4 @@ -name: Sync Feature Critical Path End-to-End tests +name: 'End to End Tests - Sync Critical Path (Nightly)' on: schedule: @@ -87,24 +87,43 @@ jobs: upload_status: ${{ steps.maestro-run.outputs.MAESTRO_CLOUD_UPLOAD_STATUS }} app_binary_id: ${{ steps.maestro-run.outputs.MAESTRO_CLOUD_APP_BINARY_ID }} - - name: Create Asana task when workflow failed - if: ${{ failure() }} - id: create-failure-task + - name: Show analyzer output + if: always() + run: echo "flow_summary_status=${{ steps.analyze-maestro.outputs.flow_summary_status }}" + + - name: Create Asana task when Maestro tests failed + if: always() && steps.analyze-maestro.outputs.flow_summary_status == 'failure' + id: create-maestro-failure-task uses: duckduckgo/native-github-asana-sync@v2.0 with: asana-pat: ${{ secrets.ASANA_ACCESS_TOKEN }} asana-project: ${{ vars.GH_ANDROID_APP_PROJECT_ID }} asana-section: ${{ vars.GH_ANDROID_APP_INCOMING_SECTION_ID }} - asana-task-name: GH Workflow Failure - Android Sync Critical Path E2E Flows (Robin) - asana-task-description: The Android Sync Critical Path Flows E2E workflow has failed. See https://github.com/duckduckgo/Android/actions/runs/${{ github.run_id }} + asana-task-name: E2E Tests Failure - Sync Critical Path + asana-task-description: | + Run: https://github.com/duckduckgo/Android/actions/runs/${{ github.run_id }} + Console: ${{ steps.maestro-run.outputs.MAESTRO_CLOUD_CONSOLE_URL }} + Summary: + ${{ steps.analyze-maestro.outputs.flow_summary_message }} action: 'create-asana-task' - name: Add Asana task to Browser Sync & Backup project - if: ${{ failure() }} + if: ${{ failure() && steps.create-failure-task.outputs.taskId != '' }} uses: duckduckgo/native-github-asana-sync@v2.0 with: asana-pat: ${{ secrets.ASANA_ACCESS_TOKEN }} asana-project: ${{ vars.GH_BROWSER_SYNC_BACKUP_PROJECT_ID }} asana-section: ${{ vars.GGH_BROWSER_SYNC_BACKUP_PROJECT_TRIAGE_SECTION_ID }} - asana-task-id: ${{ steps.create-failure-task.outputs.taskId }} - action: 'add-task-asana-project' \ No newline at end of file + asana-task-id: ${{ steps.create-maestro-failure-task.outputs.taskId }} + action: 'add-task-asana-project' + + - name: Create Asana task when workflow failed + if: ${{ failure() && github.event_name != 'workflow_dispatch' && steps.analyze-maestro.outputs.flow_summary_status != 'failure'}} + uses: duckduckgo/native-github-asana-sync@v2.0 + with: + asana-pat: ${{ secrets.ASANA_ACCESS_TOKEN }} + asana-project: ${{ vars.GH_ANDROID_APP_PROJECT_ID }} + asana-section: ${{ vars.GH_ANDROID_APP_INCOMING_SECTION_ID }} + asana-task-name: GH Workflow Failure - Sync Critical Path End to End Tests + asana-task-description: The Sync Critical Path End to End Testsworkflow has failed. See https://github.com/duckduckgo/Android/actions/runs/${{ github.run_id }} + action: 'create-asana-task' \ No newline at end of file diff --git a/.github/workflows/privacy-dashboard-end-to-end.yml b/.github/workflows/e2e-privacy-dashboard.yml similarity index 62% rename from .github/workflows/privacy-dashboard-end-to-end.yml rename to .github/workflows/e2e-privacy-dashboard.yml index a879b1349f10..67e5492864d5 100644 --- a/.github/workflows/privacy-dashboard-end-to-end.yml +++ b/.github/workflows/e2e-privacy-dashboard.yml @@ -1,4 +1,4 @@ -name: Privacy Dashboard End-to-End tests +name: 'End to End Tests - Privacy Dashboard (On Demand)' on: pull_request: @@ -81,6 +81,26 @@ jobs: upload_status: ${{ steps.maestro-ad-click.outputs.MAESTRO_CLOUD_UPLOAD_STATUS }} app_binary_id: ${{ steps.maestro-ad-click.outputs.MAESTRO_CLOUD_APP_BINARY_ID }} + - name: Show analyzer output + if: always() + run: echo "flow_summary_status=${{ steps.analyze-maestro-ad-click.outputs.flow_summary_status }}" + + - name: Create Asana task when Maestro tests failed (Ad Click) + if: always() && steps.analyze-maestro-ad-click.outputs.flow_summary_status == 'failure' + id: create-maestro-ad-click-failure-task + uses: duckduckgo/native-github-asana-sync@v2.0 + with: + asana-pat: ${{ secrets.ASANA_ACCESS_TOKEN }} + asana-project: ${{ vars.GH_ANDROID_APP_PROJECT_ID }} + asana-section: ${{ vars.GH_ANDROID_APP_INCOMING_SECTION_ID }} + asana-task-name: E2E Tests Failure - Ad Click (Privacy Dashboard) + asana-task-description: | + Run: https://github.com/duckduckgo/Android/actions/runs/${{ github.run_id }} + Console: ${{ steps.analyze-maestro-ad-click.outputs.MAESTRO_CLOUD_CONSOLE_URL }} + Summary: + ${{ steps.analyze-maestro-ad-click.outputs.flow_summary_message }} + action: 'create-asana-task' + - name: Privacy Tests if: always() id: maestro-privacy-tests @@ -104,3 +124,23 @@ jobs: console_url: ${{ steps.maestro-privacy-tests.outputs.MAESTRO_CLOUD_CONSOLE_URL }} upload_status: ${{ steps.maestro-privacy-tests.outputs.MAESTRO_CLOUD_UPLOAD_STATUS }} app_binary_id: ${{ steps.maestro-privacy-tests.outputs.MAESTRO_CLOUD_APP_BINARY_ID }} + + - name: Show analyzer output + if: always() + run: echo "flow_summary_status=${{ steps.analyze-maestro-privacy-tests.outputs.flow_summary_status }}" + + - name: Create Asana task when Maestro tests failed (Privacy Tests) + if: always() && steps.analyze-maestro-privacy-tests.outputs.flow_summary_status == 'failure' + id: create-maestro-privacy-tests-failure-task + uses: duckduckgo/native-github-asana-sync@v2.0 + with: + asana-pat: ${{ secrets.ASANA_ACCESS_TOKEN }} + asana-project: ${{ vars.GH_ANDROID_APP_PROJECT_ID }} + asana-section: ${{ vars.GH_ANDROID_APP_INCOMING_SECTION_ID }} + asana-task-name: E2E Tests Failure - Privacy Tests (Privacy Dashboard) + asana-task-description: | + Run: https://github.com/duckduckgo/Android/actions/runs/${{ github.run_id }} + Console: ${{ steps.maestro-privacy-tests.outputs.MAESTRO_CLOUD_CONSOLE_URL }} + Summary: + ${{ steps.analyze-maestro-privacy-tests.outputs.flow_summary_message }} + action: 'create-asana-task' diff --git a/.github/workflows/security-internal-e2e-tests.yml b/.github/workflows/e2e-security-internal.yml similarity index 61% rename from .github/workflows/security-internal-e2e-tests.yml rename to .github/workflows/e2e-security-internal.yml index 2bfd067a3481..922a5abc6aa2 100644 --- a/.github/workflows/security-internal-e2e-tests.yml +++ b/.github/workflows/e2e-security-internal.yml @@ -1,4 +1,4 @@ -name: Security Internal End-to-End tests +name: 'End to End Tests - Security Internal' on: workflow_dispatch: @@ -76,13 +76,33 @@ jobs: upload_status: ${{ steps.maestro-run.outputs.MAESTRO_CLOUD_UPLOAD_STATUS }} app_binary_id: ${{ steps.maestro-run.outputs.MAESTRO_CLOUD_APP_BINARY_ID }} + - name: Show analyzer output + if: always() + run: echo "flow_summary_status=${{ steps.analyze-maestro.outputs.flow_summary_status }}" + + - name: Create Asana task when Maestro tests failed + if: always() && steps.analyze-maestro.outputs.flow_summary_status == 'failure' + id: create-maestro-failure-task + uses: duckduckgo/native-github-asana-sync@v2.0 + with: + asana-pat: ${{ secrets.ASANA_ACCESS_TOKEN }} + asana-project: ${{ vars.GH_ANDROID_APP_PROJECT_ID }} + asana-section: ${{ vars.GH_ANDROID_APP_INCOMING_SECTION_ID }} + asana-task-name: E2E Tests Failure - Security + asana-task-description: | + Run: https://github.com/duckduckgo/Android/actions/runs/${{ github.run_id }} + Console: ${{ steps.maestro-run.outputs.MAESTRO_CLOUD_CONSOLE_URL }} + Summary: + ${{ steps.analyze-maestro.outputs.flow_summary_message }} + action: 'create-asana-task' + - name: Create Asana task when workflow failed - if: ${{ failure() }} - uses: honeycombio/gha-create-asana-task@main + if: ${{ failure() && steps.analyze-maestro.outputs.flow_summary_status != 'failure' }} + uses: duckduckgo/native-github-asana-sync@v2.0 with: - asana-secret: ${{ secrets.ASANA_ACCESS_TOKEN }} - asana-workspace-id: ${{ secrets.GH_ASANA_WORKSPACE_ID }} - asana-project-id: ${{ secrets.GH_ASANA_AOR_PROJECT_ID }} - asana-section-id: ${{ secrets.GH_ASANA_INCOMING_ID }} - asana-task-name: GH Workflow Failure - Security Internal E2E tests - asana-task-description: The Security Internal end-to-end workflow has failed. See https://github.com/duckduckgo/Android/actions/runs/${{ github.run_id }} \ No newline at end of file + asana-pat: ${{ secrets.ASANA_ACCESS_TOKEN }} + asana-project: ${{ vars.GH_ANDROID_APP_PROJECT_ID }} + asana-section: ${{ vars.GH_ANDROID_APP_INCOMING_SECTION_ID }} + asana-task-name: GH Workflow Failure - Security End to End Tests + asana-task-description: The Security end-to-end workflow has failed. See https://github.com/duckduckgo/Android/actions/runs/${{ github.run_id }} + action: 'create-asana-task' \ No newline at end of file diff --git a/.github/workflows/security-e2e-tests.yml b/.github/workflows/e2e-security.yml similarity index 60% rename from .github/workflows/security-e2e-tests.yml rename to .github/workflows/e2e-security.yml index b892ec629a7e..84829e84dd1b 100644 --- a/.github/workflows/security-e2e-tests.yml +++ b/.github/workflows/e2e-security.yml @@ -1,4 +1,4 @@ -name: Security End-to-End tests +name: 'End to End Tests - Security' on: workflow_dispatch: @@ -76,13 +76,33 @@ jobs: upload_status: ${{ steps.maestro-run.outputs.MAESTRO_CLOUD_UPLOAD_STATUS }} app_binary_id: ${{ steps.maestro-run.outputs.MAESTRO_CLOUD_APP_BINARY_ID }} + - name: Show analyzer output + if: always() + run: echo "flow_summary_status=${{ steps.analyze-maestro.outputs.flow_summary_status }}" + + - name: Create Asana task when Maestro tests failed + if: always() && steps.analyze-maestro.outputs.flow_summary_status == 'failure' + id: create-maestro-failure-task + uses: duckduckgo/native-github-asana-sync@v2.0 + with: + asana-pat: ${{ secrets.ASANA_ACCESS_TOKEN }} + asana-project: ${{ vars.GH_ANDROID_APP_PROJECT_ID }} + asana-section: ${{ vars.GH_ANDROID_APP_INCOMING_SECTION_ID }} + asana-task-name: E2E Tests Failure - Security + asana-task-description: | + Run: https://github.com/duckduckgo/Android/actions/runs/${{ github.run_id }} + Console: ${{ steps.maestro-run.outputs.MAESTRO_CLOUD_CONSOLE_URL }} + Summary: + ${{ steps.analyze-maestro.outputs.flow_summary_message }} + action: 'create-asana-task' + - name: Create Asana task when workflow failed - if: ${{ failure() }} - uses: honeycombio/gha-create-asana-task@main + if: ${{ failure() && steps.analyze-maestro.outputs.flow_summary_status != 'failure' }} + uses: duckduckgo/native-github-asana-sync@v2.0 with: - asana-secret: ${{ secrets.ASANA_ACCESS_TOKEN }} - asana-workspace-id: ${{ secrets.GH_ASANA_WORKSPACE_ID }} - asana-project-id: ${{ secrets.GH_ASANA_AOR_PROJECT_ID }} - asana-section-id: ${{ secrets.GH_ASANA_INCOMING_ID }} - asana-task-name: GH Workflow Failure - Security E2E tests - asana-task-description: The Security end-to-end workflow has failed. See https://github.com/duckduckgo/Android/actions/runs/${{ github.run_id }} \ No newline at end of file + asana-pat: ${{ secrets.ASANA_ACCESS_TOKEN }} + asana-project: ${{ vars.GH_ANDROID_APP_PROJECT_ID }} + asana-section: ${{ vars.GH_ANDROID_APP_INCOMING_SECTION_ID }} + asana-task-name: GH Workflow Failure - Security to End Tests + asana-task-description: Security End to End Tests workflow has failed. See https://github.com/duckduckgo/Android/actions/runs/${{ github.run_id }} + action: 'create-asana-task' \ No newline at end of file diff --git a/.github/workflows/end-to-end-robintest.yml b/.github/workflows/end-to-end-robintest.yml deleted file mode 100644 index 32a67c25da7a..000000000000 --- a/.github/workflows/end-to-end-robintest.yml +++ /dev/null @@ -1,307 +0,0 @@ -name: End-to-End tests (Robin) - -on: - schedule: - - cron: '0 3 * * *' # run at 3 AM UTC - workflow_dispatch: - -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true - -jobs: - instrumentation_tests: - runs-on: ubuntu-latest - name: End-to-End tests - - steps: - - name: Checkout repository - uses: actions/checkout@v4 - with: - submodules: recursive - - - name: Set up JDK version - uses: actions/setup-java@v4 - with: - java-version-file: .github/.java-version - distribution: 'adopt' - - - name: Create folder - if: always() - run: mkdir apk - - - name: Decode keys - uses: davidSchuppa/base64Secret-toFile-action@v2 - with: - secret: ${{ secrets.FAKE_RELEASE_PROPERTIES }} - fileName: ddg_android_build.properties - destination-path: $HOME/jenkins_static/com.duckduckgo.mobile.android/ - - - name: Decode key file - uses: davidSchuppa/base64Secret-toFile-action@v2 - with: - secret: ${{ secrets.FAKE_RELEASE_KEY }} - fileName: android - destination-path: $HOME/jenkins_static/com.duckduckgo.mobile.android/ - - - name: Setup Gradle - uses: gradle/actions/setup-gradle@v3 - - - name: Assemble release APK - run: ./gradlew assemblePlayRelease -Pforce-default-variant -Pskip-onboarding - - - name: Move APK to new folder - if: always() - run: find . -name "*play-release*.apk" -exec mv '{}' apk/release.apk \; - - - name: Assemble internal release APK - run: ./gradlew assembleInternalRelease -Pforce-default-variant -Pskip-onboarding - - - name: Move APK to new folder - if: always() - run: find . -name "*internal-release*.apk" -exec mv '{}' apk/internal.apk \; - - - name: Onboarding flows - id: maestro-onboarding - uses: mobile-dev-inc/action-maestro-cloud@v1.9.8 - timeout-minutes: 120 - with: - api-key: ${{ secrets.ROBIN_API_KEY }} - project-id: ${{ vars.ROBIN_ANDROID_PROJECT_ID }} - name: onboardingTest_${{ github.sha }} - timeout: ${{ vars.ROBIN_TIMEOUT_MINUTES }} - app-file: apk/release.apk - android-api-level: 30 - workspace: .maestro - include-tags: onboardingTest - - - name: Analyze Maestro Flow Results (Onboarding) - if: always() - id: analyze-maestro-onboarding - uses: ./.github/actions/maestro-flow-analyzer - with: - flow_results_json: ${{ steps.maestro-onboarding.outputs.MAESTRO_CLOUD_FLOW_RESULTS }} - console_url: ${{ steps.maestro-onboarding.outputs.MAESTRO_CLOUD_CONSOLE_URL }} - upload_status: ${{ steps.maestro-onboarding.outputs.MAESTRO_CLOUD_UPLOAD_STATUS }} - app_binary_id: ${{ steps.maestro-onboarding.outputs.MAESTRO_CLOUD_APP_BINARY_ID }} - - - name: Deeplink tests - id: maestro-deeplink - uses: mobile-dev-inc/action-maestro-cloud@v1.9.8 - timeout-minutes: 120 - with: - api-key: ${{ secrets.ROBIN_API_KEY }} - project-id: ${{ vars.ROBIN_ANDROID_PROJECT_ID }} - name: navigationTest_${{ github.sha }} - timeout: ${{ vars.ROBIN_TIMEOUT_MINUTES }} - app-file: apk/release.apk - android-api-level: 30 - workspace: .maestro - include-tags: navigationTest - - - name: Analyze Maestro Flow Results (Deeplink) - if: always() - id: analyze-maestro-deeplink - uses: ./.github/actions/maestro-flow-analyzer - with: - flow_results_json: ${{ steps.maestro-deeplink.outputs.MAESTRO_CLOUD_FLOW_RESULTS }} - console_url: ${{ steps.maestro-deeplink.outputs.MAESTRO_CLOUD_CONSOLE_URL }} - upload_status: ${{ steps.maestro-deeplink.outputs.MAESTRO_CLOUD_UPLOAD_STATUS }} - app_binary_id: ${{ steps.maestro-deeplink.outputs.MAESTRO_CLOUD_APP_BINARY_ID }} - - - name: Ad click detection flows - id: maestro-ad-click - uses: mobile-dev-inc/action-maestro-cloud@v1.9.8 - timeout-minutes: 120 - with: - api-key: ${{ secrets.ROBIN_API_KEY }} - project-id: ${{ vars.ROBIN_ANDROID_PROJECT_ID }} - name: adClickTest_${{ github.sha }} - timeout: ${{ vars.ROBIN_TIMEOUT_MINUTES }} - app-file: apk/release.apk - android-api-level: 30 - workspace: .maestro - include-tags: adClickTest - - - name: Analyze Maestro Flow Results (Ad Click) - if: always() - id: analyze-maestro-ad-click - uses: ./.github/actions/maestro-flow-analyzer - with: - flow_results_json: ${{ steps.maestro-ad-click.outputs.MAESTRO_CLOUD_FLOW_RESULTS }} - console_url: ${{ steps.maestro-ad-click.outputs.MAESTRO_CLOUD_CONSOLE_URL }} - upload_status: ${{ steps.maestro-ad-click.outputs.MAESTRO_CLOUD_UPLOAD_STATUS }} - app_binary_id: ${{ steps.maestro-ad-click.outputs.MAESTRO_CLOUD_APP_BINARY_ID }} - - - name: Privacy Tests - if: always() - id: maestro-privacy - uses: mobile-dev-inc/action-maestro-cloud@v1.9.8 - timeout-minutes: 120 - with: - api-key: ${{ secrets.ROBIN_API_KEY }} - project-id: ${{ vars.ROBIN_ANDROID_PROJECT_ID }} - name: privacyTest_${{ github.sha }} - timeout: ${{ vars.ROBIN_TIMEOUT_MINUTES }} - app-file: apk/release.apk - android-api-level: 30 - workspace: .maestro - include-tags: privacyTest - - - name: Analyze Maestro Flow Results (Privacy) - if: always() - id: analyze-maestro-privacy - uses: ./.github/actions/maestro-flow-analyzer - with: - flow_results_json: ${{ steps.maestro-privacy.outputs.MAESTRO_CLOUD_FLOW_RESULTS }} - console_url: ${{ steps.maestro-privacy.outputs.MAESTRO_CLOUD_CONSOLE_URL }} - upload_status: ${{ steps.maestro-privacy.outputs.MAESTRO_CLOUD_UPLOAD_STATUS }} - app_binary_id: ${{ steps.maestro-privacy.outputs.MAESTRO_CLOUD_APP_BINARY_ID }} - - - name: Internal Privacy Tests - if: always() - id: maestro-internal-privacy - uses: mobile-dev-inc/action-maestro-cloud@v1.9.7 - timeout-minutes: 120 - with: - api-key: ${{ secrets.ROBIN_API_KEY }} - project-id: ${{ vars.ROBIN_ANDROID_PROJECT_ID }} - name: internalPrivacyTest_${{ github.sha }} - timeout: ${{ vars.ROBIN_TIMEOUT_MINUTES }} - app-file: apk/internal.apk - android-api-level: 34 - workspace: .maestro - include-tags: privacyTestInternal - - - name: Analyze Maestro Flow Results (Internal Privacy) - if: always() - id: analyze-maestro-internal-privacy - uses: ./.github/actions/maestro-flow-analyzer - with: - flow_results_json: ${{ steps.maestro-internal-privacy.outputs.MAESTRO_CLOUD_FLOW_RESULTS }} - console_url: ${{ steps.maestro-internal-privacy.outputs.MAESTRO_CLOUD_CONSOLE_URL }} - upload_status: ${{ steps.maestro-internal-privacy.outputs.MAESTRO_CLOUD_UPLOAD_STATUS }} - app_binary_id: ${{ steps.maestro-internal-privacy.outputs.MAESTRO_CLOUD_APP_BINARY_ID }} - - - name: Security Tests - if: always() - id: maestro-security - uses: mobile-dev-inc/action-maestro-cloud@v1.9.8 - timeout-minutes: 120 - with: - api-key: ${{ secrets.ROBIN_API_KEY }} - project-id: ${{ vars.ROBIN_ANDROID_PROJECT_ID }} - name: securityTest_${{ github.sha }} - timeout: ${{ vars.ROBIN_TIMEOUT_MINUTES }} - app-file: apk/release.apk - android-api-level: 30 - workspace: .maestro - include-tags: securityTest - - - name: Analyze Maestro Flow Results (Security) - if: always() - id: analyze-maestro-security - uses: ./.github/actions/maestro-flow-analyzer - with: - flow_results_json: ${{ steps.maestro-security.outputs.MAESTRO_CLOUD_FLOW_RESULTS }} - console_url: ${{ steps.maestro-security.outputs.MAESTRO_CLOUD_CONSOLE_URL }} - upload_status: ${{ steps.maestro-security.outputs.MAESTRO_CLOUD_UPLOAD_STATUS }} - app_binary_id: ${{ steps.maestro-security.outputs.MAESTRO_CLOUD_APP_BINARY_ID }} - - ## This workflow will be deleted once we close the https://app.asana.com/1/137249556945/project/1211724162604201/task/1212015085457815?focus=true task - - name: Internal Security Tests - if: always() - id: maestro-internal-security - uses: mobile-dev-inc/action-maestro-cloud@v1.9.8 - timeout-minutes: 120 - with: - api-key: ${{ secrets.ROBIN_API_KEY }} - project-id: ${{ vars.ROBIN_ANDROID_PROJECT_ID }} - name: internalSecurityTest_${{ github.sha }} - timeout: ${{ vars.ROBIN_TIMEOUT_MINUTES }} - app-file: apk/internal.apk - android-api-level: 30 - workspace: .maestro - include-tags: securityTestInternal - - - name: Analyze Maestro Flow Results (Internal Security) - if: always() - id: analyze-maestro-internal-security - uses: ./.github/actions/maestro-flow-analyzer - with: - flow_results_json: ${{ steps.maestro-internal-security.outputs.MAESTRO_CLOUD_FLOW_RESULTS }} - console_url: ${{ steps.maestro-internal-security.outputs.MAESTRO_CLOUD_CONSOLE_URL }} - upload_status: ${{ steps.maestro-internal-security.outputs.MAESTRO_CLOUD_UPLOAD_STATUS }} - app_binary_id: ${{ steps.maestro-internal-security.outputs.MAESTRO_CLOUD_APP_BINARY_ID }} - - - name: Release Tests - if: always() - id: maestro-release - uses: mobile-dev-inc/action-maestro-cloud@v1.9.8 - timeout-minutes: 120 - with: - api-key: ${{ secrets.ROBIN_API_KEY }} - project-id: ${{ vars.ROBIN_ANDROID_PROJECT_ID }} - name: releaseTest_${{ github.sha }} - timeout: ${{ vars.ROBIN_TIMEOUT_MINUTES }} - app-file: apk/release.apk - android-api-level: 30 - workspace: .maestro - include-tags: releaseTest - - - name: Analyze Maestro Flow Results (Release Tests) - if: always() - id: analyze-maestro-release - uses: ./.github/actions/maestro-flow-analyzer - with: - flow_results_json: ${{ steps.maestro-release.outputs.MAESTRO_CLOUD_FLOW_RESULTS }} - console_url: ${{ steps.maestro-release.outputs.MAESTRO_CLOUD_CONSOLE_URL }} - upload_status: ${{ steps.maestro-release.outputs.MAESTRO_CLOUD_UPLOAD_STATUS }} - app_binary_id: ${{ steps.maestro-release.outputs.MAESTRO_CLOUD_APP_BINARY_ID }} - - - name: Notifications permissions Android 13+ - if: always() - id: maestro-notification-permissions - uses: mobile-dev-inc/action-maestro-cloud@v1.9.8 - timeout-minutes: 120 - with: - api-key: ${{ secrets.ROBIN_API_KEY }} - project-id: ${{ vars.ROBIN_ANDROID_PROJECT_ID }} - name: notificationPermissionTest_${{ github.sha }} - timeout: ${{ vars.ROBIN_TIMEOUT_MINUTES }} - app-file: apk/release.apk - android-api-level: 33 - workspace: .maestro - include-tags: permissionsTest - - - name: Analyze Maestro Flow Results (Notification Permissions) - if: always() - id: analyze-maestro-notification-permissions - uses: ./.github/actions/maestro-flow-analyzer - with: - flow_results_json: ${{ steps.maestro-notification-permissions.outputs.MAESTRO_CLOUD_FLOW_RESULTS }} - console_url: ${{ steps.maestro-notification-permissions.outputs.MAESTRO_CLOUD_CONSOLE_URL }} - upload_status: ${{ steps.maestro-notification-permissions.outputs.MAESTRO_CLOUD_UPLOAD_STATUS }} - app_binary_id: ${{ steps.maestro-notification-permissions.outputs.MAESTRO_CLOUD_APP_BINARY_ID }} - - - name: Create Asana task when workflow failed - if: ${{ failure() }} - id: create-failure-task - uses: duckduckgo/native-github-asana-sync@v2.0 - with: - asana-pat: ${{ secrets.ASANA_ACCESS_TOKEN }} - asana-project: ${{ secrets.GH_ASANA_AOR_PROJECT_ID }} - asana-section: ${{ secrets.GH_ASANA_INCOMING_ID }} - asana-task-name: GH Workflow Failure - End to end tests (Robin) - asana-task-description: The end to end workflow has failed. See https://github.com/duckduckgo/Android/actions/runs/${{ github.run_id }} - action: 'create-asana-task' - - - name: Adding as a release blocker - if: ${{ failure() && steps.create-failure-task.outputs.taskId != '' }} - uses: duckduckgo/native-github-asana-sync@v2.0 - with: - asana-pat: ${{ secrets.ASANA_ACCESS_TOKEN }} - asana-project: ${{ vars.GH_ANDROID_APP_RELEASES_PROJECT_ID }} - asana-section: ${{ vars.GH_ANDROID_APP_RELEASES_PROJECT_BLOCKER_SECTION_ID }} - asana-task-id: ${{ steps.create-failure-task.outputs.taskId }} - action: 'add-task-asana-project' \ No newline at end of file From 171057e3be517310acc5ec8d03229f4232c24662 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?David=20Gonz=C3=A1lez?= Date: Fri, 19 Dec 2025 23:11:49 +0100 Subject: [PATCH 3/3] lint issues --- .github/workflows/e2e-nightly-sync-critical-path.yml | 2 +- .github/workflows/e2e-privacy-dashboard.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/e2e-nightly-sync-critical-path.yml b/.github/workflows/e2e-nightly-sync-critical-path.yml index d6080d667083..938fa5f1362a 100644 --- a/.github/workflows/e2e-nightly-sync-critical-path.yml +++ b/.github/workflows/e2e-nightly-sync-critical-path.yml @@ -108,7 +108,7 @@ jobs: action: 'create-asana-task' - name: Add Asana task to Browser Sync & Backup project - if: ${{ failure() && steps.create-failure-task.outputs.taskId != '' }} + if: ${{ failure() && create-maestro-failure-task.outputs.taskId != '' }} uses: duckduckgo/native-github-asana-sync@v2.0 with: asana-pat: ${{ secrets.ASANA_ACCESS_TOKEN }} diff --git a/.github/workflows/e2e-privacy-dashboard.yml b/.github/workflows/e2e-privacy-dashboard.yml index 67e5492864d5..651c72e2537e 100644 --- a/.github/workflows/e2e-privacy-dashboard.yml +++ b/.github/workflows/e2e-privacy-dashboard.yml @@ -96,7 +96,7 @@ jobs: asana-task-name: E2E Tests Failure - Ad Click (Privacy Dashboard) asana-task-description: | Run: https://github.com/duckduckgo/Android/actions/runs/${{ github.run_id }} - Console: ${{ steps.analyze-maestro-ad-click.outputs.MAESTRO_CLOUD_CONSOLE_URL }} + Console: ${{ steps.maestro-ad-click.outputs.MAESTRO_CLOUD_CONSOLE_URL }} Summary: ${{ steps.analyze-maestro-ad-click.outputs.flow_summary_message }} action: 'create-asana-task'