Add more registrations test stability

.github/workflows/ci-registrations.yml

@@ -10,6 +10,11 @@ on:
 
 jobs:
   mix-test:
+    name: Mix test (run ${{ matrix.repeat }})
+    strategy:
+      fail-fast: false
+      matrix:
+        repeat: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12]
     runs-on: ubuntu-latest
     services:
       db:

registrations/test/integration/admin_test.exs

@@ -214,6 +214,7 @@ defmodule Registrations.Integration.UnmnemonicDevices.Admin do
   alias Registrations.Pages.Login
   alias Registrations.Pages.Nav
   alias Wallaby.Query
+  require WaitForIt
 
   test "admin can create and update settings", %{session: session} do
     insert(:octavia, admin: true)
@@ -235,10 +236,9 @@ defmodule Registrations.Integration.UnmnemonicDevices.Admin do
 
     click(session, Query.css("button[type=submit]"))
 
-    assert current_path(session) == "/settings"
-
-    assert has?(session, Query.css("#settings_begun:checked"))
-    assert text(session, Query.css(".alert-info")) == "Settings updated successfully."
+    WaitForIt.wait!(current_path(session) == "/settings")
+    Nav.assert_info_text(session, "Settings updated successfully.")
+    WaitForIt.wait!(has?(session, Query.css("#settings_begun:checked")))
   end
 
   test "non-admins cannot access the user list or messages", %{session: session} do

registrations/test/integration/invitations_test.exs

@@ -29,7 +29,8 @@ defmodule Registrations.Integration.Invitations do
 
     Details.InviteButton.click(session)
 
-    refute Details.InviteButton.present?(session)
+    Nav.assert_info_text(session, "Invitation sent")
+    Details.InviteButton.assert_absent(session)
 
     wait_for_emails([invitation_email])
     assert invitation_email.to == [{"", "bedap@example.com"}]

registrations/test/support/pages/details.ex

@@ -16,15 +16,11 @@ defmodule Registrations.Pages.Details do
   end
 
   def proposers(session, opts \\ []) do
-    session
-    |> rows_with_wait("[data-test-proposers]", opts)
-    |> Enum.map(&email_and_text_row(&1))
+    map_rows_with_wait(session, "[data-test-proposers]", opts, &email_and_text_row/1)
   end
 
   def mutuals(session, opts \\ []) do
-    session
-    |> rows_with_wait("[data-test-mutuals]", opts)
-    |> Enum.map(fn row ->
+    map_rows_with_wait(session, "[data-test-mutuals]", opts, fn row ->
       proposed_team_name_element = Browser.find(row, Query.css(".proposed-team-name"))
       risk_aversion_element = Browser.find(row, Query.css(".risk-aversion"))
 
@@ -46,21 +42,20 @@ defmodule Registrations.Pages.Details do
   end
 
   def proposals_by_mutuals(session, opts \\ []) do
-    session
-    |> rows_with_wait("[data-test-proposals-by-mutuals]", opts)
-    |> Enum.map(&email_and_text_row(&1))
+    map_rows_with_wait(
+      session,
+      "[data-test-proposals-by-mutuals]",
+      opts,
+      &email_and_text_row/1
+    )
   end
 
   def invalids(session, opts \\ []) do
-    session
-    |> rows_with_wait("[data-test-invalids]", opts)
-    |> Enum.map(&email_and_text_row(&1))
+    map_rows_with_wait(session, "[data-test-invalids]", opts, &email_and_text_row/1)
   end
 
   def proposees(session, opts \\ []) do
-    session
-    |> rows_with_wait("[data-test-proposees]", opts)
-    |> Enum.map(&email_and_text_row(&1))
+    map_rows_with_wait(session, "[data-test-proposees]", opts, &email_and_text_row/1)
   end
 
   def fill_team_emails(session, team_emails) do
@@ -149,6 +144,7 @@ defmodule Registrations.Pages.Details do
     @moduledoc false
     alias Wallaby.Browser
     alias Wallaby.Query
+    require WaitForIt
 
     def present?(session) do
       Browser.has?(session, Query.css("[data-test-invite]"))
@@ -157,6 +153,14 @@ defmodule Registrations.Pages.Details do
     def click(session) do
       Browser.click(session, Query.css("[data-test-invite]"))
     end
+
+    def assert_absent(session, message \\ nil) do
+      WaitForIt.wait!(!present?(session))
+
+      if present?(session) do
+        raise(message || "Expected invite button to be absent")
+      end
+    end
   end
 
   defmodule Attending do
@@ -190,20 +194,38 @@ defmodule Registrations.Pages.Details do
       end
 
       def assert_present(session, message \\ nil) do
-        WaitForIt.wait!(present?(session))
+        WaitForIt.wait!(safe_present?(session) == {:ok, true})
 
-        if not present?(session) do
+        if safe_present?(session) != {:ok, true} do
           raise(message || "Expected attending error to be present")
         end
       end
 
       def assert_absent(session, message \\ nil) do
-        WaitForIt.wait!(!present?(session))
+        WaitForIt.wait!(safe_present?(session) == {:ok, false})
 
-        if present?(session) do
+        if safe_present?(session) != {:ok, false} do
           raise(message || "Expected attending error to be absent")
         end
       end
+
+      defp safe_present?(session) do
+        try do
+          {:ok, present?(session)}
+        rescue
+          Wallaby.StaleReferenceError -> :error
+          Wallaby.QueryError -> :error
+          error in RuntimeError -> handle_runtime_dom_error(error, __STACKTRACE__)
+        end
+      end
+
+      defp handle_runtime_dom_error(%RuntimeError{message: message} = error, stacktrace) do
+        if String.contains?(message, "does not belong to the document") do
+          :error
+        else
+          reraise error, stacktrace
+        end
+      end
     end
   end
 
@@ -237,14 +259,15 @@ defmodule Registrations.Pages.Details do
     Browser.click(session, Query.css("#submit"))
   end
 
-  defp rows_with_wait(session, selector, opts) do
+  defp map_rows_with_wait(session, selector, opts, mapper) do
     expected = Keyword.get(opts, :count)
 
-    if expected do
-      WaitForIt.wait!(length(Browser.all(session, Query.css(selector))) == expected)
-    end
+    WaitForIt.wait!(match?({:ok, _}, safe_map_rows(session, selector, expected, mapper)))
 
-    Browser.all(session, Query.css(selector))
+    case safe_map_rows(session, selector, expected, mapper) do
+      {:ok, rows} -> rows
+      :retry -> map_rows_with_wait(session, selector, opts, mapper)
+    end
   end
 
   defp email_and_text_row(row) do
@@ -264,4 +287,28 @@ defmodule Registrations.Pages.Details do
     |> Enum.member?(class_name)
   end
 
+  defp safe_map_rows(session, selector, expected, mapper) do
+    try do
+      rows = Browser.all(session, Query.css(selector))
+
+      if expected && length(rows) != expected do
+        :retry
+      else
+        {:ok, Enum.map(rows, mapper)}
+      end
+    rescue
+      Wallaby.StaleReferenceError -> :retry
+      Wallaby.QueryError -> :retry
+      error in RuntimeError -> handle_runtime_dom_error(error, __STACKTRACE__)
+    end
+  end
+
+  defp handle_runtime_dom_error(%RuntimeError{message: message} = error, stacktrace) do
+    if String.contains?(message, "does not belong to the document") do
+      :retry
+    else
+      reraise error, stacktrace
+    end
+  end
+
 end

registrations/test/support/pages/nav.ex

@@ -87,14 +87,42 @@ defmodule Registrations.Pages.Nav do
     end
 
     def click(session) do
-      WaitForIt.wait!(Browser.has?(session, Query.css(@selector)), timeout: @logout_timeout)
-      Browser.click(session, Query.css(@selector))
+      WaitForIt.wait!(safe_has?(session, @selector), timeout: @logout_timeout)
+      WaitForIt.wait!(
+        try do
+          Browser.click(session, Query.css(@selector))
+          true
+        rescue
+          Wallaby.StaleReferenceError -> false
+          Wallaby.QueryError -> false
+          error in RuntimeError -> handle_runtime_dom_error(error, __STACKTRACE__)
+        end,
+        timeout: @logout_timeout
+      )
       WaitForIt.wait!(
-        not Browser.has?(session, Query.css(@selector)) or
-          Browser.has?(session, Query.css("a.login")),
+        not safe_has?(session, @selector) or
+          safe_has?(session, "a.login"),
         timeout: @logout_timeout
       )
     end
+
+    defp safe_has?(session, selector) do
+      try do
+        Browser.has?(session, Query.css(selector))
+      rescue
+        Wallaby.StaleReferenceError -> false
+        Wallaby.QueryError -> false
+        error in RuntimeError -> handle_runtime_dom_error(error, __STACKTRACE__)
+      end
+    end
+
+    defp handle_runtime_dom_error(%RuntimeError{message: message} = error, stacktrace) do
+      if String.contains?(message, "does not belong to the document") do
+        false
+      else
+        reraise error, stacktrace
+      end
+    end
   end
 
   defmodule LoginLink do
@@ -198,6 +226,15 @@ defmodule Registrations.Pages.Nav do
     rescue
       Wallaby.StaleReferenceError -> :error
       Wallaby.QueryError -> :error
+      error in RuntimeError -> handle_runtime_dom_error(error, __STACKTRACE__)
+    end
+  end
+
+  defp handle_runtime_dom_error(%RuntimeError{message: message} = error, stacktrace) do
+    if String.contains?(message, "does not belong to the document") do
+      :error
+    else
+      reraise error, stacktrace
     end
   end