test: list each failed step of an ios flow with its error from maestro 2's command log

.maestro/features/07-move-delete-tag.yaml

@@ -81,23 +81,24 @@ tags:
     commands:
       - tapOn: "More actions.*"
 - tapOn: "Star|Unstar"
-# A bulk action ends selection mode by itself.
+# A bulk action ends selection mode by itself once the server has the
-- runFlow:
+# change, so the bar stays up for a moment: wait for it to go rather than
-    when:
+# tap Clear selection (the tap can land after the bar is gone).
-      visible: "Clear selection"
+- extendedWaitUntil:
-    commands:
+    notVisible: "Clear selection"
-      - tapOn: "Clear selection"
+    timeout: 15000
 - longPressOn:
     id: email-row
     text: ".*Grüße aus München.*"
+- extendedWaitUntil:
+    visible: "Clear selection"
+    timeout: 10000
 - runFlow:
     when:
       notVisible: "Unstar|Star"
     commands:
       - tapOn: "More actions.*"
 - tapOn: "Unstar|Star"
-- runFlow:
+- extendedWaitUntil:
-    when:
+    notVisible: "Clear selection"
-      visible: "Clear selection"
+    timeout: 15000
-    commands:
-      - tapOn: "Clear selection"

.maestro/features/13-settings-more.yaml

@@ -16,6 +16,9 @@ tags:
 # Each Q matches its page near the top. A PROOF the search lists under the
 # row would pass on the iPad without the page, so those differ. The Themes
 # cards are one element each, named by the theme: check the page's text.
+# "Storage" also finds Account and Files, which puts Storage & backup third,
+# under the keyboard in landscape (a tap there types on the keys): search
+# "Offline", which only that page has.
 - runFlow: {file: ../subflows/settings-page.yaml, env: {NAME: folders, ROW: "Folders, .*", Q: "New folder", PROOF: ".*Inbox.*"}}
 - runFlow: {file: ../subflows/settings-page.yaml, env: {NAME: tags, ROW: "Tags, .*", Q: "Tags", PROOF: "Email Tags"}}
 - runFlow: {file: ../subflows/settings-page.yaml, env: {NAME: downloads, ROW: "Downloads, .*", Q: "Downloads", PROOF: "Email .*eml.* filename"}}
@@ -26,7 +29,7 @@ tags:
 - runFlow: {file: ../subflows/settings-page.yaml, env: {NAME: files, ROW: "Files, .*", Q: "Files", PROOF: "Default View"}}
 - runFlow: {file: ../subflows/settings-page.yaml, env: {NAME: sidebar-apps, ROW: "Sidebar Apps, .*", Q: "Sidebar", PROOF: "Keep Apps Loaded"}}
 - runFlow: {file: ../subflows/settings-page.yaml, env: {NAME: themes, ROW: "Themes, .*", Q: "Themes", PROOF: "Pick a colou?r theme.*"}}
-- runFlow: {file: ../subflows/settings-page.yaml, env: {NAME: storage, ROW: "Storage & backup, .*", Q: "Storage", PROOF: "Offline mail.*"}}
+- runFlow: {file: ../subflows/settings-page.yaml, env: {NAME: storage, ROW: "Storage & backup, .*", Q: "Offline", PROOF: "Offline mail.*"}}
 - runFlow: {file: ../subflows/settings-page.yaml, env: {NAME: about, ROW: "About, .*", Q: "About", PROOF: "This device"}}
 - tapOn:
     id: tab-mail

.maestro/subflows/settings-page.yaml

@@ -18,8 +18,8 @@ appId: org.bulwarkmail.mobile
       - inputText: ${Q}
       - waitForAnimationToEnd:
           timeout: 3000
-      # Now and then the simulator drops or garbles a typed character
+      # Now and then the simulator drops or garbles a typed character:
-      # (seen: "Storagew,"): type it again, a few times if need be.
+      # type it again, a few times if need be.
       - repeat:
           while:
             notVisible: ${ROW}

scripts/e2e/failed-steps.mjs

@@ -43,14 +43,16 @@ function findHierarchy(obj, seen = new Set()) {
   return null;
 }
 
-for (const file of walkFiles(root).filter((f) => /commands-.*\.json$/.test(f))) {
+// Maestro 1 writes commands-<flow>.json, Maestro 2 <flow>/commands.json.
+for (const file of walkFiles(root).filter((f) => /(^|[\\/])commands(-[^\\/]*)?\.json$/.test(f))) {
   let data;
   try {
     data = JSON.parse(fs.readFileSync(file, 'utf8'));
   } catch {
     continue;
   }
-  out.push(`== ${path.basename(file)}`);
+  const base = path.basename(file);
+  out.push(`== ${base === 'commands.json' ? `${path.basename(path.dirname(file))} / ${base}` : base}`);
   for (const entry of Array.isArray(data) ? data : []) {
     const command = entry.command || {};
     const type = Object.keys(command).find((k) => command[k]) || '?';

scripts/e2e/run-flows.sh

@@ -8,8 +8,8 @@
 # $OUT/screenshots, $OUT/run.mp4, $OUT/app.log, $OUT/steps.txt.
 #
 # Each flow runs in its own Maestro process with a time cap: when the XCTest
-# driver dies (it does, now and then), only that flow fails and the next one
+# driver dies (it does, now and then), only that flow is affected: it runs
-# starts a fresh driver.
+# once more with a fresh driver, and the next flow starts a fresh one too.
 #
 # Secrets: Maestro's debug output holds every typed text, passwords included,
 # and artifacts of this public repo are public. Only the warm-up (which types
@@ -75,23 +75,43 @@ if [ $STATUS -eq 0 ]; then
   fi
 
   FAILED=()
+  RETRIED=()
   for f in "${FLOWS[@]}"; do
     name=$(basename "$f" .yaml)
-    started=$(date +%s)
+    # When the XCTest driver dies during a flow ("Device became unreachable"),
-    echo "::group::$name"
+    # the flow gets one more try in a fresh Maestro process, which starts a
-    (cd "$OUT/screenshots" && capped maestro --device "$UDID" test \
+    # new driver. Any other failure (an assertion, a missing element, the
-      --format junit --output "$OUT/junit/$name.xml" \
+    # time cap) stays a failure. The driver's death shows in Maestro's log in
-      --debug-output "$DEBUG_DIR/$name" \
+    # the flow's debug output (never uploaded; grep -q prints nothing). Piping
-      "${ARGS[@]}" "$f")
+    # Maestro's console output instead loses the failure reasons it prints.
-    rc=$?
+    for attempt in 1 2; do
-    echo "::endgroup::"
+      suffix=""
-    echo "$name: exit $rc after $(( $(date +%s) - started )) s"
+      [ $attempt -gt 1 ] && suffix="-retry"
+      started=$(date +%s)
+      echo "::group::$name$suffix"
+      (cd "$OUT/screenshots" && capped maestro --device "$UDID" test \
+        --format junit --output "$OUT/junit/$name.xml" \
+        --debug-output "$DEBUG_DIR/$name$suffix" \
+        "${ARGS[@]}" "$f")
+      rc=$?
+      echo "::endgroup::"
+      echo "$name$suffix: exit $rc after $(( $(date +%s) - started )) s"
+      [ $rc -eq 0 ] && break
+      [ $attempt -eq 2 ] && break
+      grep -rqE 'DeviceUnreachableException|Device became unreachable|Transport unreachable' \
+        "$DEBUG_DIR/$name$suffix" 2>/dev/null || break
+      echo "::warning::$name: the Maestro driver became unreachable, running the flow once more"
+      RETRIED+=("$name")
+    done
     if [ $rc -ne 0 ]; then
       FAILED+=("$name")
       # What the screen showed when it failed.
       xcrun simctl io "$UDID" screenshot "$OUT/screenshots/FAILED-$name.png" >/dev/null 2>&1 || true
     fi
   done
+  if [ ${#RETRIED[@]} -gt 0 ]; then
+    echo "::warning::Flows run again after the driver died: ${RETRIED[*]}"
+  fi
   if [ ${#FAILED[@]} -gt 0 ]; then
     echo "::error::Failed flows: ${FAILED[*]}"
     STATUS=1