diff --git a/.github/workflows/nightly.yaml b/.github/workflows/nightly.yaml index f66e2e0292..11ac989057 100644 --- a/.github/workflows/nightly.yaml +++ b/.github/workflows/nightly.yaml @@ -7,42 +7,42 @@ on: workflow_dispatch: jobs: - ios-nightly: - strategy: - fail-fast: false - matrix: - lib: [SalesforceSDKCommon, SalesforceAnalytics, SalesforceSDKCore, SmartStore, MobileSync] - ios: [^26, ^18, ^17] - include: - - ios: ^26 - xcode: ^26 - - ios: ^18 - xcode: ^16 - - ios: ^17 - xcode: ^16 - uses: ./.github/workflows/reusable-test-workflow.yaml - with: - lib: ${{ matrix.lib }} - ios: ${{ matrix.ios }} - xcode: ${{ matrix.xcode }} - secrets: inherit + ios-nightly: + strategy: + fail-fast: false + matrix: + lib: [SalesforceSDKCommon, SalesforceAnalytics, SalesforceSDKCore, SmartStore, MobileSync] + ios: [^26, ^18, ^17] + include: + - ios: ^26 + xcode: ^26 + - ios: ^18 + xcode: ^16 + - ios: ^17 + xcode: ^16 + uses: ./.github/workflows/reusable-test-workflow.yaml + with: + lib: ${{ matrix.lib }} + ios: ${{ matrix.ios }} + xcode: ${{ matrix.xcode }} + secrets: inherit - native-samples-nightly: - strategy: - fail-fast: false - matrix: - app: [RestAPIExplorer, MobileSyncExplorer, AuthFlowTester] - ios: [^26, ^18, ^17] - include: - - ios: ^26 - xcode: ^26 - - ios: ^18 - xcode: ^16 - - ios: ^17 - xcode: ^16 - uses: ./.github/workflows/reusable-build-workflow.yaml - with: - app: ${{ matrix.app }} - ios: ${{ matrix.ios }} - xcode: ${{ matrix.xcode }} - secrets: inherit \ No newline at end of file + native-samples-nightly: + strategy: + fail-fast: false + matrix: + app: [RestAPIExplorer, MobileSyncExplorer, AuthFlowTester] + ios: [^26, ^18, ^17] + include: + - ios: ^26 + xcode: ^26 + - ios: ^18 + xcode: ^16 + - ios: ^17 + xcode: ^16 + uses: ./.github/workflows/reusable-build-workflow.yaml + with: + app: ${{ matrix.app }} + ios: ${{ matrix.ios }} + xcode: ${{ matrix.xcode }} + secrets: inherit diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 932a046cd7..8fa2020dc9 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -142,4 +142,6 @@ jobs: with: is_pr: true pr_test: "AuthFlowTesterUITests/LegacyLoginTests/testCAOpaque_DefaultScopes_WebServerFlow" + short_timeout: "2" + long_timeout: "7" secrets: inherit \ No newline at end of file diff --git a/.github/workflows/reusable-ui-test-workflow.yaml b/.github/workflows/reusable-ui-test-workflow.yaml index 19c621d7ea..f29ff868b2 100644 --- a/.github/workflows/reusable-ui-test-workflow.yaml +++ b/.github/workflows/reusable-ui-test-workflow.yaml @@ -21,16 +21,39 @@ on: default: "" required: false type: string + test_suite: + description: "Test suite to run (e.g. 'AuthFlowTesterUITests/BeaconLoginTests'). Empty = run all tests. Used with matrix for parallel nightly runs." + default: "" + required: false + type: string destination: description: "xcodebuild -destination (e.g. 'platform=iOS Simulator,name=iPhone 17,OS=26.0'). Empty = use default for iOS 26." default: "" required: false type: string + short_timeout: + description: "UI test short timeout in seconds (env UI_TEST_SHORT_TIMEOUT). Empty = use code default (1)." + default: "" + required: false + type: string + long_timeout: + description: "UI test long timeout in seconds (env UI_TEST_LONG_TIMEOUT). Empty = use code default (3)." + default: "" + required: false + type: string jobs: test-ui: runs-on: ${{ inputs.macos }} steps: + - name: Set result suffix for matrix runs + id: result_suffix + run: | + if [ -n "${{ inputs.test_suite }}" ]; then + echo "suffix=$(echo '${{ inputs.test_suite }}' | tr '/' '-')" >> "$GITHUB_OUTPUT" + else + echo "suffix=all" >> "$GITHUB_OUTPUT" + fi - uses: actions/checkout@v4 if: ${{ inputs.is_pr }} with: @@ -62,18 +85,25 @@ jobs: if [ -n "${{ inputs.destination }}" ]; then echo "destination=${{ inputs.destination }}" >> "$GITHUB_OUTPUT" else + # Extract major iOS version from input (e.g. ^26 -> 26, 26.0 -> 26) + IOS_VERSION="${{ inputs.ios }}" + IOS_MAJOR=$(echo "$IOS_VERSION" | sed 's/^\^//' | cut -d. -f1) SIM_UDID=$(xcrun simctl list devices available -j | python3 -c ' import sys, json + major = sys.argv[1] d = json.load(sys.stdin) - for devices in d.get("devices", {}).values(): + runtime_prefix = "iOS-" + major + "-" + for runtime_key, devices in d.get("devices", {}).items(): + if runtime_prefix not in runtime_key: + continue for dev in devices: if dev.get("isAvailable", True) and "iPhone" in dev.get("name", ""): print(dev["udid"]) sys.exit(0) sys.exit(1) - ') + ' "$IOS_MAJOR") if [ -z "$SIM_UDID" ]; then - echo "::error::No available iPhone simulator found" + echo "::error::No available iPhone simulator found for iOS $IOS_MAJOR" exit 1 fi echo "destination=platform=iOS Simulator,id=$SIM_UDID" >> "$GITHUB_OUTPUT" @@ -85,7 +115,10 @@ jobs: ONLY_TESTING_ARGS=() if [ -n "${{ inputs.pr_test }}" ]; then ONLY_TESTING_ARGS+=(-only-testing "${{ inputs.pr_test }}") + elif [ -n "${{ inputs.test_suite }}" ]; then + ONLY_TESTING_ARGS+=(-only-testing "${{ inputs.test_suite }}") fi + set -o pipefail xcodebuild test \ -workspace SalesforceMobileSDK.xcworkspace \ -scheme AuthFlowTester \ @@ -97,13 +130,36 @@ jobs: | xcbeautify env: CODE_COVERAGE: YES - - name: Publish test results - uses: kishikawakatsumi/xcresulttool@v1 + # Xcode 15.3+ only forwards env vars prefixed with TEST_RUNNER_ to the test runner (prefix is stripped). + TEST_RUNNER_UI_TEST_SHORT_TIMEOUT: ${{ inputs.short_timeout }} + TEST_RUNNER_UI_TEST_LONG_TIMEOUT: ${{ inputs.long_timeout }} + - name: Verify xcresult bundle exists + if: success() || failure() + run: | + if [ ! -d "test.xcresult" ]; then + echo "::error::test.xcresult bundle was not created" + exit 1 + fi + echo "xcresult bundle exists and contains:" + ls -lh test.xcresult/ + - name: Parse test results + if: success() || failure() + run: | + brew install xcresultparser + xcresultparser -o junit test.xcresult > test-results-authflowtester-ui-ios${{ inputs.ios }}-${{ steps.result_suffix.outputs.suffix }}.xml + - name: Test Report + uses: mikepenz/action-junit-report@v5 if: success() || failure() with: - path: test.xcresult - show-passed-tests: true - title: AuthFlowTester UI Test Results + check_name: AuthFlowTester UI Test Results ${{ steps.result_suffix.outputs.suffix }} + job_name: AuthFlowTester UI Test Results ${{ steps.result_suffix.outputs.suffix }} + require_tests: true + include_empty_in_summary: false + simplified_summary: true + detailed_summary: true + comment: true + job_summary: true + report_paths: 'test-results-authflowtester-ui-ios${{ inputs.ios }}-${{ steps.result_suffix.outputs.suffix }}.xml' - uses: codecov/codecov-action@v4 if: success() || failure() with: @@ -114,6 +170,6 @@ jobs: if: success() || failure() uses: actions/upload-artifact@v4 with: - name: xcresult-authflowtester-ui-ios${{ inputs.ios }} + name: xcresult-authflowtester-ui-ios${{ inputs.ios }}-${{ steps.result_suffix.outputs.suffix }} path: test.xcresult/ retention-days: 30 diff --git a/.github/workflows/ui-test-nightly.yaml b/.github/workflows/ui-test-nightly.yaml index 81904cedfe..21e23da32a 100644 --- a/.github/workflows/ui-test-nightly.yaml +++ b/.github/workflows/ui-test-nightly.yaml @@ -2,7 +2,7 @@ name: UI Nightly Tests on: schedule: - - cron: "0 5 * * 3,5" # cron is UTC, this translates to 10 PM PST Tues and Thur. + - cron: "0 5 * * 2,4" # cron is UTC, this translates to 10 PM PST Mon and Wed. workflow_dispatch: jobs: @@ -11,4 +11,6 @@ jobs: with: ios: "^26" xcode: "^26" + short_timeout: "2" + long_timeout: "7" secrets: inherit diff --git a/native/SampleApps/AuthFlowTester/AuthFlowTesterUITests/PageObjects/AuthFlowTesterMainPageObject.swift b/native/SampleApps/AuthFlowTester/AuthFlowTesterUITests/PageObjects/AuthFlowTesterMainPageObject.swift index 2e69b1d332..fcdcbdcea8 100644 --- a/native/SampleApps/AuthFlowTester/AuthFlowTesterUITests/PageObjects/AuthFlowTesterMainPageObject.swift +++ b/native/SampleApps/AuthFlowTester/AuthFlowTesterUITests/PageObjects/AuthFlowTesterMainPageObject.swift @@ -213,14 +213,13 @@ struct JwtDetailsData { /// and extract data (user credentials, OAuth configuration, JWT details) from the UI. class AuthFlowTesterMainPageObject { let app: XCUIApplication - let timeout: double_t = 3 - + init(testApp: XCUIApplication) { app = testApp } func isShowing() -> Bool { - return navigationTitle().waitForExistence(timeout: 1) + return navigationTitle().waitForExistence(timeout: UITestTimeouts.long) } func performLogout() { @@ -231,7 +230,7 @@ class AuthFlowTesterMainPageObject { func makeRestRequest() -> Bool { tap(makeRestRequestButton()) let alert = app.alerts["Request Successful"] - if (alert.waitForExistence(timeout: timeout)) { + if (alert.waitForExistence(timeout: UITestTimeouts.long)) { alert.buttons["OK"].tap() return true } @@ -241,7 +240,7 @@ class AuthFlowTesterMainPageObject { func revokeAccessToken() -> Bool { tap(revokeButton()) let alert = app.alerts["Access Token Revoked"] - if (alert.waitForExistence(timeout: timeout)) { + if (alert.waitForExistence(timeout: UITestTimeouts.long)) { alert.buttons["OK"].tap() return true } @@ -282,7 +281,7 @@ class AuthFlowTesterMainPageObject { tapIfPresent(allowButton()) let alert = app.alerts["Migration Error"] - if (alert.waitForExistence(timeout: timeout)) { + if (alert.waitForExistence(timeout: UITestTimeouts.long)) { alert.buttons["OK"].tap() return false } @@ -309,7 +308,7 @@ class AuthFlowTesterMainPageObject { // Wait for alert to appear let alert = importConfigAlert() - _ = alert.waitForExistence(timeout: timeout) + _ = alert.waitForExistence(timeout: UITestTimeouts.long) // Type into the alert's text field let textField = importConfigTextField() @@ -438,25 +437,25 @@ class AuthFlowTesterMainPageObject { // MARK: - Actions private func tap(_ element: XCUIElement) { - _ = element.waitForExistence(timeout: timeout) + _ = element.waitForExistence(timeout: UITestTimeouts.long) element.tap() } private func tapIfPresent(_ element: XCUIElement) { - if (element.waitForExistence(timeout: timeout)) { + if (element.waitForExistence(timeout: UITestTimeouts.long)) { element.tap() } } private func setTextField(_ textField: XCUIElement, value: String) { - _ = textField.waitForExistence(timeout: timeout) + _ = textField.waitForExistence(timeout: UITestTimeouts.long) textField.tap() // Clear any existing text if let currentValue = textField.value as? String, !currentValue.isEmpty { textField.tap() let selectAll = app.menuItems["Select All"] - if selectAll.waitForExistence(timeout: 1) { + if selectAll.waitForExistence(timeout: UITestTimeouts.short) { selectAll.tap() textField.typeText(XCUIKeyboardKey.delete.rawValue) } @@ -535,7 +534,7 @@ class AuthFlowTesterMainPageObject { func getJwtDetails() -> JwtDetailsData? { // Check if JWT export button exists (indicates JWT token is available) - guard exportJwtTokenButton().waitForExistence(timeout: 1) else { + guard exportJwtTokenButton().waitForExistence(timeout: UITestTimeouts.short) else { return nil } @@ -571,7 +570,7 @@ class AuthFlowTesterMainPageObject { // Wait for and get the alert let alert = app.alerts[alertTitle] - guard alert.waitForExistence(timeout: timeout) else { + guard alert.waitForExistence(timeout: UITestTimeouts.long) else { return [:] } @@ -594,7 +593,7 @@ class AuthFlowTesterMainPageObject { private func hasStaticText(_ text: String) -> Bool { let staticText = app.staticTexts.containing(NSPredicate(format: "label CONTAINS '\(text)'")) - return staticText.firstMatch.waitForExistence(timeout: timeout) + return staticText.firstMatch.waitForExistence(timeout: UITestTimeouts.long) } } diff --git a/native/SampleApps/AuthFlowTester/AuthFlowTesterUITests/PageObjects/LoginOptionsPageObject.swift b/native/SampleApps/AuthFlowTester/AuthFlowTesterUITests/PageObjects/LoginOptionsPageObject.swift index ea5cd466e4..735b1b29ee 100644 --- a/native/SampleApps/AuthFlowTester/AuthFlowTesterUITests/PageObjects/LoginOptionsPageObject.swift +++ b/native/SampleApps/AuthFlowTester/AuthFlowTesterUITests/PageObjects/LoginOptionsPageObject.swift @@ -33,7 +33,6 @@ import SalesforceSDKCore /// Use after navigating to Login Options from the login screen (e.g. via Settings → Login Options). class LoginOptionsPageObject { let app: XCUIApplication - let timeout: double_t = 2 init(testApp: XCUIApplication) { app = testApp @@ -101,7 +100,7 @@ class LoginOptionsPageObject { // Wait for alert and enter JSON (text field is automatically focused) let alert = app.alerts["Import Configuration"] - _ = alert.waitForExistence(timeout: timeout) + _ = alert.waitForExistence(timeout: UITestTimeouts.long) let textField = alert.textFields.firstMatch textField.typeText(jsonString) @@ -116,7 +115,7 @@ class LoginOptionsPageObject { // Wait for alert and enter JSON (text field is automatically focused) let alert = app.alerts["Import Discovery Result"] - _ = alert.waitForExistence(timeout: timeout) + _ = alert.waitForExistence(timeout: UITestTimeouts.long) let textField = alert.textFields.firstMatch textField.typeText(jsonString) @@ -153,12 +152,12 @@ class LoginOptionsPageObject { // MARK: - Actions private func tap(_ element: XCUIElement) { - _ = element.waitForExistence(timeout: timeout) + _ = element.waitForExistence(timeout: UITestTimeouts.long) element.tap() } private func setSwitchField(_ switchField: XCUIElement, value: Bool) { - _ = switchField.waitForExistence(timeout: timeout) + _ = switchField.waitForExistence(timeout: UITestTimeouts.long) // Switch values are "0" (off) or "1" (on) in XCTest let currentValue = (switchField.value as? String) == "1" diff --git a/native/SampleApps/AuthFlowTester/AuthFlowTesterUITests/PageObjects/LoginPageObject.swift b/native/SampleApps/AuthFlowTester/AuthFlowTesterUITests/PageObjects/LoginPageObject.swift index 24a8142816..1d219a95d2 100644 --- a/native/SampleApps/AuthFlowTester/AuthFlowTesterUITests/PageObjects/LoginPageObject.swift +++ b/native/SampleApps/AuthFlowTester/AuthFlowTesterUITests/PageObjects/LoginPageObject.swift @@ -33,22 +33,21 @@ import SalesforceSDKCore /// Provides methods to configure login servers, login options and perform user authentication. class LoginPageObject { let app: XCUIApplication - let timeout: double_t = 3 - + init(testApp: XCUIApplication) { app = testApp } func isShowing() -> Bool { - return loginNavigationBar().waitForExistence(timeout: timeout) + return loginNavigationBar().waitForExistence(timeout: UITestTimeouts.long) } func hasFilledUsernameField(username: String) -> Bool { - return app.staticTexts[username].waitForExistence(timeout: timeout) + return app.staticTexts[username].waitForExistence(timeout: UITestTimeouts.long) } func isShowingAdvancedAuth() -> Bool { - return advancedAuthCloseButton().waitForExistence(timeout: 1) + return advancedAuthCloseButton().waitForExistence(timeout: UITestTimeouts.short) } func closeAdvancedAuth() -> Void { @@ -73,9 +72,9 @@ class LoginPageObject { func performLogin(username: String, password: String) { setTextField(usernameField(), value: username) - tap(toolbarDoneButton()) + dismissKeyboardAfterTyping() setTextField(passwordField(), value: password) - tap(toolbarDoneButton()) + dismissKeyboardAfterTyping() tap(loginButton()) tapIfPresent(allowButton()) } @@ -163,6 +162,15 @@ class LoginPageObject { return app.staticTexts[host].firstMatch } + /// Dismisses the keyboard after typing in a field. Tries the toolbar "Done" button first; + /// if not found (e.g. on some simulators it is exposed as a Key, not Button), taps the + /// password label to dismiss. + private func dismissKeyboardAfterTyping() { + if !tapIfPresent(toolbarDoneButton()) { + tap(passwordFieldLabel()) + } + } + private func toolbarDoneButton() -> XCUIElement { return app.toolbars["Toolbar"].buttons["Done"] } @@ -200,14 +208,17 @@ class LoginPageObject { // MARK: - Actions private func tap(_ element: XCUIElement) { - _ = element.waitForExistence(timeout: timeout) + _ = element.waitForExistence(timeout: UITestTimeouts.long) element.tap() } - private func tapIfPresent(_ element: XCUIElement) { - if (element.waitForExistence(timeout: timeout)) { + @discardableResult + private func tapIfPresent(_ element: XCUIElement) -> Bool { + if element.waitForExistence(timeout: UITestTimeouts.long) { element.tap() + return true } + return false } private func setTextField(_ textField: XCUIElement, value: String) { @@ -222,7 +233,7 @@ class LoginPageObject { if let currentValue = textField.value as? String, !currentValue.isEmpty { tap(textField) // second tap should bring up menu let selectAll = app.menuItems["Select All"] - if selectAll.waitForExistence(timeout: 1) { + if selectAll.waitForExistence(timeout: UITestTimeouts.short) { selectAll.tap() textField.typeText(XCUIKeyboardKey.delete.rawValue) } @@ -235,7 +246,7 @@ class LoginPageObject { private func hasHost(host: String) -> Bool { let row = hostRow(host: host) - return row.waitForExistence(timeout: timeout) + return row.waitForExistence(timeout: UITestTimeouts.long) } } diff --git a/native/SampleApps/AuthFlowTester/AuthFlowTesterUITests/Util/UITestTimeouts.swift b/native/SampleApps/AuthFlowTester/AuthFlowTesterUITests/Util/UITestTimeouts.swift new file mode 100644 index 0000000000..9afe2d5866 --- /dev/null +++ b/native/SampleApps/AuthFlowTester/AuthFlowTesterUITests/Util/UITestTimeouts.swift @@ -0,0 +1,56 @@ +/* + UITestTimeouts.swift + AuthFlowTesterUITests + + Copyright (c) 2026-present, salesforce.com, inc. All rights reserved. + + Redistribution and use of this software in source and binary forms, with or without modification, + are permitted provided that the following conditions are met: + * Redistributions of source code must retain the above copyright notice, this list of conditions + and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright notice, this list of + conditions and the following disclaimer in the documentation and/or other materials provided + with the distribution. + * Neither the name of salesforce.com, inc. nor the names of its contributors may be used to + endorse or promote products derived from this software without specific prior written + permission of salesforce.com, inc. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR + IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND + FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR + CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY + WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +import Foundation + +/// Provides timeout values for UI test element waits. +/// Values come from the environment (`UI_TEST_SHORT_TIMEOUT`, `UI_TEST_LONG_TIMEOUT`) when set, +/// otherwise from defaults. CI workflows can pass larger timeouts via these env vars. +enum UITestTimeouts { + /// Default short timeout in seconds (e.g. for quick UI state checks). + private static let defaultShort: TimeInterval = 1 + /// Default long timeout in seconds (e.g. for page load or alert appearance). + private static let defaultLong: TimeInterval = 3 + + private static func parseEnv(_ key: String) -> TimeInterval? { + guard let raw = ProcessInfo.processInfo.environment[key], + !raw.isEmpty, + let value = TimeInterval(raw), + value > 0 else { return nil } + return value + } + + /// Short timeout (seconds). Use for fast UI checks (e.g. menu items, close buttons). + static var short: TimeInterval { + parseEnv("UI_TEST_SHORT_TIMEOUT") ?? defaultShort + } + + /// Long timeout (seconds). Use for page visibility, alerts, and general element waits. + static var long: TimeInterval { + parseEnv("UI_TEST_LONG_TIMEOUT") ?? defaultLong + } +}