From de414f7ed6bd19e98a26a940b1c4e884df53bd28 Mon Sep 17 00:00:00 2001 From: Linus Rath <139418639+rathlinus@users.noreply.github.com> Date: Thu, 22 Oct 2026 02:15:35 +0200 Subject: [PATCH] ci: run each ios flow in its own maestro process and add the feature flows --- .maestro/features/01-folders.yaml | 38 ++++++ .maestro/features/02-read-messages.yaml | 71 ++++++++++++ .maestro/features/03-compose-send.yaml | 62 ++++++++++ .../features/04-incoming-reply-forward.yaml | 69 +++++++++++ .maestro/features/05-search.yaml | 27 +++++ .maestro/features/06-swipe-undo.yaml | 36 ++++++ .maestro/features/07-move-delete-tag.yaml | 61 ++++++++++ .maestro/features/08-scheduled-send.yaml | 57 +++++++++ .maestro/features/09-calendar.yaml | 98 ++++++++++++++++ .maestro/features/10-contacts.yaml | 66 +++++++++++ .maestro/features/11-files.yaml | 58 ++++++++++ .maestro/features/12-settings.yaml | 38 ++++++ .maestro/launch.yaml | 1 + .maestro/signin/02-password-errors.yaml | 2 + .maestro/signin/16-add-password.yaml | 1 - .maestro/signin/17-add-oauth.yaml | 1 - .maestro/signin/18-add-webmail.yaml | 2 +- .maestro/signin/19-add-link.yaml | 1 - .maestro/subflows/assert-delivered.yaml | 18 +++ .maestro/subflows/check-active-account.yaml | 15 +-- .maestro/subflows/close-message.yaml | 8 ++ .maestro/subflows/compose-body.yaml | 19 +++ .maestro/subflows/go-to-folder.yaml | 8 ++ .maestro/subflows/go-to-inbox.yaml | 16 +++ .maestro/subflows/manual-server.yaml | 1 + .maestro/subflows/open-add-account.yaml | 8 +- .maestro/subflows/open-drawer.yaml | 12 ++ .maestro/subflows/open-message.yaml | 13 +++ .maestro/subflows/settings-page.yaml | 24 ++++ .maestro/subflows/sign-out.yaml | 4 +- .maestro/subflows/signed-in.yaml | 28 +++++ .maestro/subflows/type-secret.yaml | 12 +- .maestro/subflows/type.yaml | 51 +++++--- scripts/e2e/boot-sim.sh | 2 + scripts/e2e/run-flows.sh | 109 ++++++++++++------ 35 files changed, 962 insertions(+), 75 deletions(-) create mode 100644 .maestro/features/01-folders.yaml create mode 100644 .maestro/features/02-read-messages.yaml create mode 100644 .maestro/features/03-compose-send.yaml create mode 100644 .maestro/features/04-incoming-reply-forward.yaml create mode 100644 .maestro/features/05-search.yaml create mode 100644 .maestro/features/06-swipe-undo.yaml create mode 100644 .maestro/features/07-move-delete-tag.yaml create mode 100644 .maestro/features/08-scheduled-send.yaml create mode 100644 .maestro/features/09-calendar.yaml create mode 100644 .maestro/features/10-contacts.yaml create mode 100644 .maestro/features/11-files.yaml create mode 100644 .maestro/features/12-settings.yaml create mode 100644 .maestro/subflows/assert-delivered.yaml create mode 100644 .maestro/subflows/close-message.yaml create mode 100644 .maestro/subflows/compose-body.yaml create mode 100644 .maestro/subflows/go-to-folder.yaml create mode 100644 .maestro/subflows/go-to-inbox.yaml create mode 100644 .maestro/subflows/open-drawer.yaml create mode 100644 .maestro/subflows/open-message.yaml create mode 100644 .maestro/subflows/settings-page.yaml create mode 100644 .maestro/subflows/signed-in.yaml diff --git a/.maestro/features/01-folders.yaml b/.maestro/features/01-folders.yaml new file mode 100644 index 00000000..406d27ae --- /dev/null +++ b/.maestro/features/01-folders.yaml @@ -0,0 +1,38 @@ +# Mailbox list and folders: the drawer (or docked sidebar) lists the role +# folders and custom ones; opening a folder shows its messages. +appId: org.bulwarkmail.mobile +name: Folders +tags: + - features +--- +- runFlow: ../subflows/signed-in.yaml +- takeScreenshot: folders-01-inbox +- assertVisible: ".*Your verification code.*" +- runFlow: ../subflows/open-drawer.yaml +- takeScreenshot: folders-02-drawer +- assertVisible: "Inbox.*" +- assertVisible: "Sent.*" +- assertVisible: "Drafts.*" +- assertVisible: "Archive.*" +- assertVisible: "Projects.*" +- tapOn: "Archive.*" +- extendedWaitUntil: + visible: ".*Invoice 2025-12.*" + timeout: 20000 +- takeScreenshot: folders-03-archive +- runFlow: ../subflows/open-drawer.yaml +# Projects/Alpha holds the seeded thread. +- runFlow: + when: + notVisible: "Alpha.*" + commands: + - tapOn: + text: "Expand" + rightOf: "Projects.*" + optional: true +- tapOn: "Alpha.*" +- extendedWaitUntil: + visible: ".*Project Alpha kickoff.*" + timeout: 20000 +- takeScreenshot: folders-04-alpha +- runFlow: ../subflows/go-to-inbox.yaml diff --git a/.maestro/features/02-read-messages.yaml b/.maestro/features/02-read-messages.yaml new file mode 100644 index 00000000..7dd19dfe --- /dev/null +++ b/.maestro/features/02-read-messages.yaml @@ -0,0 +1,71 @@ +# Opening messages: plain text, HTML with blocked remote content, a thread, +# and a PDF attachment in the in-app preview. +appId: org.bulwarkmail.mobile +name: Read messages +tags: + - features +--- +- runFlow: ../subflows/signed-in.yaml + +# Plain text +- runFlow: + file: ../subflows/open-message.yaml + env: + SUBJECT: Grüße aus München +- extendedWaitUntil: + visible: ".*Jürgen.*" + timeout: 10000 +- takeScreenshot: read-01-plain +- runFlow: ../subflows/close-message.yaml + +# HTML newsletter +- runFlow: + file: ../subflows/open-message.yaml + env: + SUBJECT: Weekly digest +- extendedWaitUntil: + visible: ".*5 new articles.*|Weekly digest" + timeout: 20000 +- takeScreenshot: read-02-html +- runFlow: + when: + visible: "Load images" + commands: + - tapOn: "Load images" + - takeScreenshot: read-03-html-images +- runFlow: ../subflows/close-message.yaml + +# Attachment and its preview +- runFlow: + file: ../subflows/open-message.yaml + env: + SUBJECT: Q3 report +- extendedWaitUntil: + visible: ".*q3-report.pdf.*" + timeout: 20000 +- takeScreenshot: read-04-attachment +- tapOn: ".*q3-report.pdf.*" +- extendedWaitUntil: + visible: "Share" + timeout: 30000 +- waitForAnimationToEnd: + timeout: 5000 +- takeScreenshot: read-05-attachment-preview +- assertVisible: "q3-report.pdf" +- tapOn: "Close" +- extendedWaitUntil: + visible: "Reply All" + timeout: 10000 +- runFlow: ../subflows/close-message.yaml + +# A thread (Projects/Alpha) +- runFlow: + file: ../subflows/go-to-folder.yaml + env: + FOLDER: "Alpha.*" +- runFlow: + file: ../subflows/open-message.yaml + env: + SUBJECT: Project Alpha kickoff +- takeScreenshot: read-06-thread +- runFlow: ../subflows/go-to-inbox.yaml diff --git a/.maestro/features/03-compose-send.yaml b/.maestro/features/03-compose-send.yaml new file mode 100644 index 00000000..24810e57 --- /dev/null +++ b/.maestro/features/03-compose-send.yaml @@ -0,0 +1,62 @@ +# Compose a new message with a photo attached and send it to userb; userb's +# mailbox (checked over JMAP) must get it with the attachment. +appId: org.bulwarkmail.mobile +name: Compose and send with attachment +tags: + - features +--- +- runFlow: ../subflows/signed-in.yaml +- evalScript: ${output.subject = 'E2E compose ' + DEVICE + '-' + APPEARANCE + '-' + ORIENTATION + ' ' + Date.now()} +- tapOn: "Compose" +- extendedWaitUntil: + visible: "New Message" + timeout: 15000 +- tapOn: "Recipient email addresses" +- inputText: "userb@example.org" +- pressKey: Enter +- tapOn: + text: "Subject" + index: 1 +- inputText: ${output.subject} +- runFlow: + file: ../subflows/compose-body.yaml + env: + BODY: "Hello from the iOS simulator test." +- takeScreenshot: compose-01-filled + +# Attach a photo from the library (boot-sim.sh adds one). +- tapOn: "Attach" +- extendedWaitUntil: + visible: "Photos & Videos" + timeout: 10000 +- tapOn: "Photos & Videos" +- extendedWaitUntil: + visible: "Photos|Cancel" + timeout: 20000 +- waitForAnimationToEnd: + timeout: 5000 +- takeScreenshot: compose-02-photo-picker +- tapOn: + text: "Photo.*" + index: 0 +- runFlow: + when: + visible: "Add|Done" + commands: + - tapOn: "Add|Done" +- extendedWaitUntil: + visible: ".*(png|jpg|jpeg|heic|PNG|JPG|JPEG|HEIC).*" + timeout: 60000 +- takeScreenshot: compose-03-attached + +- tapOn: "Send" +- extendedWaitUntil: + visible: "Email sent successfully" + timeout: 60000 +- runFlow: + file: ../subflows/assert-delivered.yaml + env: + SUBJECT: ${output.subject} +- assertTrue: + condition: ${output.helper.attachments.length > 0} + label: "the sent message carries the photo" diff --git a/.maestro/features/04-incoming-reply-forward.yaml b/.maestro/features/04-incoming-reply-forward.yaml new file mode 100644 index 00000000..252276ae --- /dev/null +++ b/.maestro/features/04-incoming-reply-forward.yaml @@ -0,0 +1,69 @@ +# Incoming mail while the app is open (sent by userb over JMAP), then reply +# and forward it; userb receives both, the forward with the attachment. +appId: org.bulwarkmail.mobile +name: Incoming, reply, forward +tags: + - features +--- +- runFlow: ../subflows/signed-in.yaml +- evalScript: ${output.incoming = 'E2E incoming ' + DEVICE + '-' + APPEARANCE + '-' + ORIENTATION + ' ' + Date.now()} +- runScript: + file: ../scripts/helper.js + env: + CALL: /send?to=${ACCOUNT}&subject=${encodeURIComponent(output.incoming)}&attach=1&html=1 +# Arrives without a pull to refresh (push over EventSource). +- extendedWaitUntil: + visible: ".*${output.incoming}.*" + timeout: 90000 +- takeScreenshot: incoming-01-arrived +- runFlow: + file: ../subflows/open-message.yaml + env: + SUBJECT: ${output.incoming} +- takeScreenshot: incoming-02-open + +# Reply +- tapOn: + text: "Reply" + index: 0 +- extendedWaitUntil: + visible: "Re: ${output.incoming}" + timeout: 15000 +- runFlow: + file: ../subflows/compose-body.yaml + env: + BODY: "Reply from the iOS simulator test." +- takeScreenshot: incoming-03-reply +- tapOn: "Send" +- extendedWaitUntil: + visible: "Email sent successfully" + timeout: 60000 +- runFlow: + file: ../subflows/assert-delivered.yaml + env: + SUBJECT: "Re: ${output.incoming}" + +# Forward with the attachment +- extendedWaitUntil: + visible: "Forward" + timeout: 15000 +- tapOn: + text: "Forward" + index: 0 +- extendedWaitUntil: + visible: "Fwd: ${output.incoming}" + timeout: 15000 +- tapOn: "Recipient email addresses" +- inputText: "userb@example.org" +- pressKey: Enter +- takeScreenshot: incoming-04-forward +- tapOn: "Send" +- extendedWaitUntil: + visible: "Email sent successfully" + timeout: 60000 +- runFlow: + file: ../subflows/assert-delivered.yaml + env: + SUBJECT: "Fwd: ${output.incoming}" + ATTACHMENT: e2e-attachment.pdf +- runFlow: ../subflows/close-message.yaml diff --git a/.maestro/features/05-search.yaml b/.maestro/features/05-search.yaml new file mode 100644 index 00000000..309918c2 --- /dev/null +++ b/.maestro/features/05-search.yaml @@ -0,0 +1,27 @@ +# Search: a hit, a miss, clearing. +appId: org.bulwarkmail.mobile +name: Search +tags: + - features +--- +- runFlow: ../subflows/signed-in.yaml +- tapOn: "Search mail..." +- inputText: "verification" +- pressKey: Enter +- extendedWaitUntil: + visible: ".*Your verification code.*" + timeout: 30000 +- assertNotVisible: ".*Weekly digest.*" +- takeScreenshot: search-01-hit +- tapOn: "Clear search" +- inputText: "zzqx-no-such-mail" +- pressKey: Enter +- extendedWaitUntil: + visible: "No results found" + timeout: 30000 +- takeScreenshot: search-02-miss +- tapOn: "Clear" +- extendedWaitUntil: + visible: ".*Weekly digest.*" + timeout: 30000 +- hideKeyboard diff --git a/.maestro/features/06-swipe-undo.yaml b/.maestro/features/06-swipe-undo.yaml new file mode 100644 index 00000000..126bbd0d --- /dev/null +++ b/.maestro/features/06-swipe-undo.yaml @@ -0,0 +1,36 @@ +# Swipe actions with undo: swipe left archives (undo brings it back), swipe +# right toggles read. +appId: org.bulwarkmail.mobile +name: Swipe actions and undo +tags: + - features +--- +- runFlow: ../subflows/signed-in.yaml +- swipe: + from: ".*Grüße aus München.*" + direction: LEFT + duration: 400 +- extendedWaitUntil: + visible: "Email archived" + timeout: 10000 +- takeScreenshot: swipe-01-archived +- assertNotVisible: ".*Grüße aus München.*" +- tapOn: "UNDO" +- extendedWaitUntil: + visible: ".*Grüße aus München.*" + timeout: 15000 +- takeScreenshot: swipe-02-undone + +- swipe: + from: ".*Grüße aus München.*" + direction: RIGHT + duration: 400 +- waitForAnimationToEnd: + timeout: 3000 +- takeScreenshot: swipe-03-read-toggled +- swipe: + from: ".*Grüße aus München.*" + direction: RIGHT + duration: 400 +- waitForAnimationToEnd: + timeout: 3000 diff --git a/.maestro/features/07-move-delete-tag.yaml b/.maestro/features/07-move-delete-tag.yaml new file mode 100644 index 00000000..b0253bcd --- /dev/null +++ b/.maestro/features/07-move-delete-tag.yaml @@ -0,0 +1,61 @@ +# Message actions with undo: move to a folder, delete to Trash, select and +# star from the list. +appId: org.bulwarkmail.mobile +name: Move, delete, select +tags: + - features +--- +- runFlow: ../subflows/signed-in.yaml + +# Move via More → Move to... +- runFlow: + file: ../subflows/open-message.yaml + env: + SUBJECT: Weekly digest +- tapOn: "more" +- extendedWaitUntil: + visible: "More actions" + timeout: 10000 +- takeScreenshot: actions-01-more +- tapOn: "Move to..." +- extendedWaitUntil: + visible: "Move to..." + timeout: 10000 +- tapOn: "Projects" +- extendedWaitUntil: + visible: "Moved to Projects" + timeout: 15000 +- takeScreenshot: actions-02-moved +- tapOn: "UNDO" +- runFlow: ../subflows/close-message.yaml +- extendedWaitUntil: + visible: ".*Weekly digest.*" + timeout: 15000 + +# Delete (to Trash) and undo +- runFlow: + file: ../subflows/open-message.yaml + env: + SUBJECT: Weekly digest +- tapOn: "Delete" +- extendedWaitUntil: + visible: "Email moved to Trash" + timeout: 15000 +- takeScreenshot: actions-03-deleted +- tapOn: "UNDO" +- runFlow: ../subflows/close-message.yaml +- extendedWaitUntil: + visible: ".*Weekly digest.*" + timeout: 15000 + +# Select from the list and star, then unstar +- longPressOn: ".*Grüße aus München.*" +- extendedWaitUntil: + visible: "Clear selection" + timeout: 10000 +- takeScreenshot: actions-04-selected +- tapOn: "Star" +- tapOn: "Clear selection" +- longPressOn: ".*Grüße aus München.*" +- tapOn: "Unstar|Star" +- tapOn: "Clear selection" diff --git a/.maestro/features/08-scheduled-send.yaml b/.maestro/features/08-scheduled-send.yaml new file mode 100644 index 00000000..0f062ecd --- /dev/null +++ b/.maestro/features/08-scheduled-send.yaml @@ -0,0 +1,57 @@ +# Scheduled send: schedule a message for later, find it under Scheduled, +# cancel it. +appId: org.bulwarkmail.mobile +name: Scheduled send +tags: + - features +--- +- runFlow: ../subflows/signed-in.yaml +- evalScript: ${output.subject = 'E2E scheduled ' + DEVICE + '-' + APPEARANCE + '-' + ORIENTATION + ' ' + Date.now()} +- tapOn: "Compose" +- extendedWaitUntil: + visible: "New Message" + timeout: 15000 +- tapOn: "Recipient email addresses" +- inputText: "userb@example.org" +- pressKey: Enter +- tapOn: + text: "Subject" + index: 1 +- inputText: ${output.subject} +- runFlow: + file: ../subflows/compose-body.yaml + env: + BODY: "Scheduled from the iOS simulator test." +- tapOn: "Schedule send" +- extendedWaitUntil: + visible: "Tomorrow morning.*" + timeout: 10000 +- takeScreenshot: scheduled-01-options +- tapOn: "Tomorrow morning.*" +- extendedWaitUntil: + visible: "Scheduled" + timeout: 30000 +- takeScreenshot: scheduled-02-confirmed +- tapOn: "OK" + +- runFlow: + file: ../subflows/go-to-folder.yaml + env: + FOLDER: Scheduled +- extendedWaitUntil: + visible: ".*${output.subject}.*" + timeout: 30000 +- takeScreenshot: scheduled-03-list +- tapOn: ".*${output.subject}.*" +- tapOn: "Cancel send" +- extendedWaitUntil: + visible: "Keep scheduled" + timeout: 10000 +- tapOn: + text: "Cancel send" + below: "Keep scheduled|.*will not be delivered.*" +- extendedWaitUntil: + notVisible: ".*${output.subject}.*" + timeout: 30000 +- takeScreenshot: scheduled-04-cancelled +- runFlow: ../subflows/go-to-inbox.yaml diff --git a/.maestro/features/09-calendar.yaml b/.maestro/features/09-calendar.yaml new file mode 100644 index 00000000..c1c8de93 --- /dev/null +++ b/.maestro/features/09-calendar.yaml @@ -0,0 +1,98 @@ +# Calendar: the views, create an event with a reminder, edit it, delete it. +appId: org.bulwarkmail.mobile +name: Calendar +tags: + - features +--- +- runFlow: ../subflows/signed-in.yaml +- evalScript: ${output.event = 'E2E event ' + DEVICE + ' ' + Date.now() % 100000} +- tapOn: + id: tab-calendar +- extendedWaitUntil: + visible: "Go to today" + timeout: 30000 +- takeScreenshot: calendar-01-month + +- tapOn: "Open menu" +- tapOn: "Schedule" +- waitForAnimationToEnd: + timeout: 5000 +- takeScreenshot: calendar-02-schedule +- tapOn: "Open menu" +- tapOn: "Day" +- waitForAnimationToEnd: + timeout: 5000 +- takeScreenshot: calendar-03-day +- tapOn: "Open menu" +- tapOn: "Week" +- waitForAnimationToEnd: + timeout: 5000 +- takeScreenshot: calendar-04-week + +# Create +- tapOn: "New event" +- extendedWaitUntil: + visible: "Save" + timeout: 10000 +- tapOn: "Title" +- inputText: ${output.event} +- hideKeyboard +- scrollUntilVisible: + element: "Add reminder" + direction: DOWN + timeout: 10000 +- tapOn: "Add reminder" +- tapOn: "15 minutes before" +- takeScreenshot: calendar-05-editor +- tapOn: "Save" +- extendedWaitUntil: + notVisible: "New event" + timeout: 20000 + +# Find it in the schedule and open it +- tapOn: "Open menu" +- tapOn: "Schedule" +- tapOn: "Go to today" +- scrollUntilVisible: + element: ".*${output.event}.*" + direction: DOWN + timeout: 30000 +- tapOn: ".*${output.event}.*" +- extendedWaitUntil: + visible: "Edit" + timeout: 10000 +- takeScreenshot: calendar-06-detail +- assertVisible: ".*15 minutes before.*" + +# Edit the title +- tapOn: "Edit" +- extendedWaitUntil: + visible: "Edit event" + timeout: 10000 +- tapOn: ${output.event} +- inputText: " edited" +- hideKeyboard +- tapOn: "Save" +- extendedWaitUntil: + visible: ".*${output.event} edited.*" + timeout: 20000 +- takeScreenshot: calendar-07-edited + +# Delete +- tapOn: ".*${output.event} edited.*" +- extendedWaitUntil: + visible: "Delete" + timeout: 10000 +- tapOn: "Delete" +- extendedWaitUntil: + visible: "Delete this event.*" + timeout: 10000 +- tapOn: + text: "Delete" + below: "Delete this event.*" +- extendedWaitUntil: + notVisible: ".*${output.event} edited.*" + timeout: 20000 +- takeScreenshot: calendar-08-deleted +- tapOn: + id: tab-mail diff --git a/.maestro/features/10-contacts.yaml b/.maestro/features/10-contacts.yaml new file mode 100644 index 00000000..6d6c4779 --- /dev/null +++ b/.maestro/features/10-contacts.yaml @@ -0,0 +1,66 @@ +# Contacts: the list, a contact's detail, create one and delete it again. +appId: org.bulwarkmail.mobile +name: Contacts +tags: + - features +--- +- runFlow: ../subflows/signed-in.yaml +- evalScript: ${output.last = 'Tester' + Date.now() % 100000} +- tapOn: + id: tab-contacts +- extendedWaitUntil: + visible: "All Contacts" + timeout: 30000 +- takeScreenshot: contacts-01-list +- tapOn: ".*Alice.*" +- extendedWaitUntil: + visible: "Email" + timeout: 10000 +- takeScreenshot: contacts-02-detail +- runFlow: + when: + visible: "Back" + commands: + - tapOn: "Back" + +- tapOn: "New Contact" +- extendedWaitUntil: + visible: "Save" + timeout: 10000 +- tapOn: + text: "First name" + index: 1 +- inputText: "E2E" +- tapOn: + text: "Last name" + index: 1 +- inputText: ${output.last} +- tapOn: "email@example.com" +- inputText: "e2e.contact@example.net" +- hideKeyboard +- takeScreenshot: contacts-03-form +- tapOn: "Save" +- extendedWaitUntil: + visible: ".*${output.last}.*" + timeout: 20000 +- takeScreenshot: contacts-04-created + +- runFlow: + when: + notVisible: "More actions" + commands: + - tapOn: ".*E2E ${output.last}.*" +- tapOn: "More actions" +- tapOn: "Delete" +- extendedWaitUntil: + visible: "Delete contact" + timeout: 10000 +- tapOn: + text: "Delete" + below: "Are you sure.*" +- extendedWaitUntil: + notVisible: ".*E2E ${output.last}.*" + timeout: 20000 +- takeScreenshot: contacts-05-deleted +- tapOn: + id: tab-mail diff --git a/.maestro/features/11-files.yaml b/.maestro/features/11-files.yaml new file mode 100644 index 00000000..b5e4ad49 --- /dev/null +++ b/.maestro/features/11-files.yaml @@ -0,0 +1,58 @@ +# Files: open a folder, preview a text and a PDF file, create and delete a +# folder. +appId: org.bulwarkmail.mobile +name: Files +tags: + - features +--- +- runFlow: ../subflows/signed-in.yaml +- evalScript: ${output.folder = 'E2E folder ' + Date.now() % 100000} +- tapOn: + id: tab-files +- extendedWaitUntil: + visible: ".*Documents.*" + timeout: 30000 +- takeScreenshot: files-01-root +- tapOn: ".*Documents.*" +- extendedWaitUntil: + visible: ".*readme.txt.*" + timeout: 20000 +- tapOn: ".*readme.txt.*" +- extendedWaitUntil: + visible: "readme.txt" + timeout: 20000 +- waitForAnimationToEnd: + timeout: 5000 +- takeScreenshot: files-02-text-preview +- tapOn: "Close" +- tapOn: ".*q3-report.pdf.*" +- extendedWaitUntil: + visible: "q3-report.pdf" + timeout: 20000 +- waitForAnimationToEnd: + timeout: 5000 +- takeScreenshot: files-03-pdf-preview +- tapOn: "Close" +- tapOn: "Back" + +- tapOn: "New Folder" +- tapOn: "Folder name" +- inputText: ${output.folder} +- tapOn: "Create" +- extendedWaitUntil: + visible: ".*${output.folder}.*" + timeout: 20000 +- takeScreenshot: files-04-folder-created +- longPressOn: ".*${output.folder}.*" +- tapOn: "Delete" +- extendedWaitUntil: + visible: "Delete this item.*" + timeout: 10000 +- tapOn: + text: "Delete" + below: "Delete this item.*" +- extendedWaitUntil: + notVisible: ".*${output.folder}.*" + timeout: 20000 +- tapOn: + id: tab-mail diff --git a/.maestro/features/12-settings.yaml b/.maestro/features/12-settings.yaml new file mode 100644 index 00000000..5af6250a --- /dev/null +++ b/.maestro/features/12-settings.yaml @@ -0,0 +1,38 @@ +# Every settings page opens and shows its content. +appId: org.bulwarkmail.mobile +name: Settings pages +tags: + - features +--- +- runFlow: ../subflows/signed-in.yaml +- tapOn: + id: tab-settings +- extendedWaitUntil: + visible: "Search settings" + timeout: 20000 +- takeScreenshot: settings-00-list +- runFlow: {file: ../subflows/settings-page.yaml, env: {NAME: account, ROW: ".*@example.org, .*", PROOF: "Logged-in accounts|Display Name"}} +- runFlow: {file: ../subflows/settings-page.yaml, env: {NAME: language, ROW: "Language, Region & Time, .*", PROOF: "Date Format"}} +- runFlow: {file: ../subflows/settings-page.yaml, env: {NAME: notifications, ROW: "Notifications, .*", PROOF: "Email Notifications"}} +- runFlow: {file: ../subflows/settings-page.yaml, env: {NAME: appearance, ROW: "Appearance, .*", PROOF: "Choose your preferred color scheme"}} +- runFlow: {file: ../subflows/settings-page.yaml, env: {NAME: layout, ROW: "Layout, .*", PROOF: "Tune the email list interactions for mobile."}} +- runFlow: {file: ../subflows/settings-page.yaml, env: {NAME: reading, ROW: "Reading, .*", PROOF: "Mark as Read"}} +- runFlow: {file: ../subflows/settings-page.yaml, env: {NAME: composing, ROW: "Composing, .*", PROOF: "Composer"}} +- runFlow: {file: ../subflows/settings-page.yaml, env: {NAME: identities, ROW: "Identities, .*", PROOF: "Sending Identities"}} +- runFlow: {file: ../subflows/settings-page.yaml, env: {NAME: vacation, ROW: "Out of Office, .*", PROOF: "Reply automatically"}} +- runFlow: {file: ../subflows/settings-page.yaml, env: {NAME: filters, ROW: "Filters, .*", PROOF: "Email Filters"}} +- runFlow: {file: ../subflows/settings-page.yaml, env: {NAME: templates, ROW: "Templates, .*", PROOF: "Email Templates"}} +- runFlow: {file: ../subflows/settings-page.yaml, env: {NAME: folders, ROW: "Folders, .*", PROOF: ".*Inbox.*"}} +- runFlow: {file: ../subflows/settings-page.yaml, env: {NAME: tags, ROW: "Tags, .*", PROOF: "Email Tags"}} +- runFlow: {file: ../subflows/settings-page.yaml, env: {NAME: downloads, ROW: "Downloads, .*", PROOF: "Email .*eml.* filename"}} +- runFlow: {file: ../subflows/settings-page.yaml, env: {NAME: security, ROW: "Security, .*", PROOF: "Account Security|Detecting server capabilities.*"}} +- runFlow: {file: ../subflows/settings-page.yaml, env: {NAME: content, ROW: "Content & Senders, .*", PROOF: "External Content"}} +- runFlow: {file: ../subflows/settings-page.yaml, env: {NAME: calendar, ROW: "Calendar, .*", PROOF: "Default view"}} +- runFlow: {file: ../subflows/settings-page.yaml, env: {NAME: contacts, ROW: "Contacts, .*", PROOF: "Group by first letter"}} +- runFlow: {file: ../subflows/settings-page.yaml, env: {NAME: files, ROW: "Files, .*", PROOF: "Default View"}} +- runFlow: {file: ../subflows/settings-page.yaml, env: {NAME: sidebar-apps, ROW: "Sidebar Apps, .*", PROOF: "Keep Apps Loaded"}} +- runFlow: {file: ../subflows/settings-page.yaml, env: {NAME: themes, ROW: "Themes, .*", PROOF: "The default light and dark palettes."}} +- runFlow: {file: ../subflows/settings-page.yaml, env: {NAME: storage, ROW: "Storage & backup, .*", PROOF: "Offline mail.*"}} +- runFlow: {file: ../subflows/settings-page.yaml, env: {NAME: about, ROW: "About, .*", PROOF: "This device"}} +- tapOn: + id: tab-mail diff --git a/.maestro/launch.yaml b/.maestro/launch.yaml index 4dfcf872..cf402f84 100644 --- a/.maestro/launch.yaml +++ b/.maestro/launch.yaml @@ -25,5 +25,6 @@ tags: file: subflows/type.yaml env: FIELD: login-server-input + TAP: "no" TEXT: typing-probe.example.net - takeScreenshot: warmup-02-typed diff --git a/.maestro/signin/02-password-errors.yaml b/.maestro/signin/02-password-errors.yaml index 5bbbf2b5..bc0182bf 100644 --- a/.maestro/signin/02-password-errors.yaml +++ b/.maestro/signin/02-password-errors.yaml @@ -30,6 +30,8 @@ tags: file: ../subflows/type.yaml env: FIELD: login-server-input + TAP: "no" + ERASE: "yes" TEXT: mail.does-not-exist.invalid - tapOn: "Continue" - extendedWaitUntil: diff --git a/.maestro/signin/16-add-password.yaml b/.maestro/signin/16-add-password.yaml index 829c1093..6f11a60f 100644 --- a/.maestro/signin/16-add-password.yaml +++ b/.maestro/signin/16-add-password.yaml @@ -26,5 +26,4 @@ tags: file: ../subflows/check-active-account.yaml env: ACTIVE: ${ACCOUNT2}@example.org -- assertVisible: ${ACCOUNT}@example.org - takeScreenshot: add-password-02-two-accounts diff --git a/.maestro/signin/17-add-oauth.yaml b/.maestro/signin/17-add-oauth.yaml index f48ba879..024cc747 100644 --- a/.maestro/signin/17-add-oauth.yaml +++ b/.maestro/signin/17-add-oauth.yaml @@ -38,5 +38,4 @@ tags: file: ../subflows/check-active-account.yaml env: ACTIVE: ${ACCOUNT2}@example.org -- assertVisible: ${ACCOUNT}@example.org - takeScreenshot: add-oauth-01-two-accounts diff --git a/.maestro/signin/18-add-webmail.yaml b/.maestro/signin/18-add-webmail.yaml index 65b2e934..5ea8ebc1 100644 --- a/.maestro/signin/18-add-webmail.yaml +++ b/.maestro/signin/18-add-webmail.yaml @@ -19,6 +19,7 @@ tags: file: ../subflows/type.yaml env: FIELD: login-server-input + TAP: "no" TEXT: ${WEBMAIL} - tapOn: "Continue" - extendedWaitUntil: @@ -39,5 +40,4 @@ tags: file: ../subflows/check-active-account.yaml env: ACTIVE: ${ACCOUNT2}@example.org -- assertVisible: ${ACCOUNT}@example.org - takeScreenshot: add-webmail-01-two-accounts diff --git a/.maestro/signin/19-add-link.yaml b/.maestro/signin/19-add-link.yaml index a786d251..fc183e1b 100644 --- a/.maestro/signin/19-add-link.yaml +++ b/.maestro/signin/19-add-link.yaml @@ -26,5 +26,4 @@ tags: file: ../subflows/check-active-account.yaml env: ACTIVE: ${ACCOUNT2}@example.org -- assertVisible: ${ACCOUNT}@example.org - takeScreenshot: add-link-02-two-accounts diff --git a/.maestro/subflows/assert-delivered.yaml b/.maestro/subflows/assert-delivered.yaml new file mode 100644 index 00000000..d2bfc2fe --- /dev/null +++ b/.maestro/subflows/assert-delivered.yaml @@ -0,0 +1,18 @@ +# The helper checks over JMAP that AS (default userb) received SUBJECT; with +# ATTACHMENT set, that the message carries that file. +appId: org.bulwarkmail.mobile +--- +- runScript: + file: ../scripts/helper.js + env: + CALL: /find?as=${AS || 'userb'}&subject=${encodeURIComponent(SUBJECT)}&timeout=60 +- assertTrue: + condition: ${output.helper.found} + label: "${SUBJECT} delivered to ${AS || 'userb'}" +- runFlow: + when: + true: ${!!ATTACHMENT} + commands: + - assertTrue: + condition: ${output.helper.attachments.indexOf(ATTACHMENT) >= 0} + label: "${SUBJECT} carries ${ATTACHMENT}" diff --git a/.maestro/subflows/check-active-account.yaml b/.maestro/subflows/check-active-account.yaml index d8fd9de3..b0a1fabb 100644 --- a/.maestro/subflows/check-active-account.yaml +++ b/.maestro/subflows/check-active-account.yaml @@ -1,15 +1,16 @@ -# Settings → Account lists the accounts on this device; the active one is -# ACTIVE. Goes back to Mail afterwards. +# The Settings account card names the active account ACTIVE, and the Account +# page lists another account to switch to. Goes back to Mail afterwards. appId: org.bulwarkmail.mobile --- - tapOn: id: tab-settings - extendedWaitUntil: - visible: "Account" - timeout: 10000 -- tapOn: "Account" -- extendedWaitUntil: - visible: ${ACTIVE} + visible: ".*${ACTIVE}.*" + timeout: 20000 +- tapOn: ".*${ACTIVE}.*" +- scrollUntilVisible: + element: "Switch to this account" + direction: DOWN timeout: 20000 - takeScreenshot: accounts-${ACTIVE} - tapOn: diff --git a/.maestro/subflows/close-message.yaml b/.maestro/subflows/close-message.yaml new file mode 100644 index 00000000..1bedad8a --- /dev/null +++ b/.maestro/subflows/close-message.yaml @@ -0,0 +1,8 @@ +# Leave the message view: Back on a phone; the split layout keeps the list. +appId: org.bulwarkmail.mobile +--- +- runFlow: + when: + visible: "Back" + commands: + - tapOn: "Back" diff --git a/.maestro/subflows/compose-body.yaml b/.maestro/subflows/compose-body.yaml new file mode 100644 index 00000000..6c1a58c6 --- /dev/null +++ b/.maestro/subflows/compose-body.yaml @@ -0,0 +1,19 @@ +# In the composer: type BODY into the editor. The rich editor is a WebView +# whose placeholder is CSS, so it may not be exposed; then tap the middle of +# the screen, which is the body area below Subject (above the keyboard). +appId: org.bulwarkmail.mobile +--- +- runFlow: + when: + visible: "Write your message.*" + commands: + - tapOn: "Write your message.*" +- runFlow: + when: + notVisible: "Write your message.*" + commands: + - tapOn: + point: "50%,50%" +- waitForAnimationToEnd: + timeout: 3000 +- inputText: ${BODY} diff --git a/.maestro/subflows/go-to-folder.yaml b/.maestro/subflows/go-to-folder.yaml new file mode 100644 index 00000000..f660cf30 --- /dev/null +++ b/.maestro/subflows/go-to-folder.yaml @@ -0,0 +1,8 @@ +# Open folder FOLDER from the drawer or docked sidebar. +appId: org.bulwarkmail.mobile +--- +- runFlow: open-drawer.yaml +- tapOn: ${FOLDER} +- extendedWaitUntil: + visible: ${FOLDER} + timeout: 15000 diff --git a/.maestro/subflows/go-to-inbox.yaml b/.maestro/subflows/go-to-inbox.yaml new file mode 100644 index 00000000..4b1b84cd --- /dev/null +++ b/.maestro/subflows/go-to-inbox.yaml @@ -0,0 +1,16 @@ +# Back to the inbox list, from wherever in the Mail tab. +appId: org.bulwarkmail.mobile +--- +- runFlow: + when: + visible: "Back" + commands: + - tapOn: "Back" +- runFlow: + file: go-to-folder.yaml + env: + FOLDER: Inbox +- extendedWaitUntil: + visible: + id: email-row + timeout: 60000 diff --git a/.maestro/subflows/manual-server.yaml b/.maestro/subflows/manual-server.yaml index 4959e03b..f10e709c 100644 --- a/.maestro/subflows/manual-server.yaml +++ b/.maestro/subflows/manual-server.yaml @@ -11,6 +11,7 @@ appId: org.bulwarkmail.mobile file: type.yaml env: FIELD: login-server-input + TAP: "no" TEXT: ${HOST} - tapOn: "Continue" - extendedWaitUntil: diff --git a/.maestro/subflows/open-add-account.yaml b/.maestro/subflows/open-add-account.yaml index 92a22102..c6d86cd4 100644 --- a/.maestro/subflows/open-add-account.yaml +++ b/.maestro/subflows/open-add-account.yaml @@ -1,12 +1,14 @@ -# Signed in: Settings → Account → Add account, up to the add-mode choices. +# Signed in: Settings → account card → Account page → Add account, up to the +# add-mode choices. (On the iPad split layout the Account page is already +# open; tapping the card again is harmless.) appId: org.bulwarkmail.mobile --- - tapOn: id: tab-settings - extendedWaitUntil: - visible: "Account" + visible: ".*@example.org, .*" timeout: 10000 -- tapOn: "Account" +- tapOn: ".*@example.org, .*" - scrollUntilVisible: element: "Add account" direction: DOWN diff --git a/.maestro/subflows/open-drawer.yaml b/.maestro/subflows/open-drawer.yaml new file mode 100644 index 00000000..853414fc --- /dev/null +++ b/.maestro/subflows/open-drawer.yaml @@ -0,0 +1,12 @@ +# Show the folder list: the modal drawer, or nothing to do where the sidebar +# is docked (iPad landscape). +appId: org.bulwarkmail.mobile +--- +- runFlow: + when: + visible: "Toggle menu" + commands: + - tapOn: "Toggle menu" +- extendedWaitUntil: + visible: "Scheduled" + timeout: 10000 diff --git a/.maestro/subflows/open-message.yaml b/.maestro/subflows/open-message.yaml new file mode 100644 index 00000000..d5e27b35 --- /dev/null +++ b/.maestro/subflows/open-message.yaml @@ -0,0 +1,13 @@ +# From a message list: open the message whose row mentions SUBJECT. +appId: org.bulwarkmail.mobile +--- +- scrollUntilVisible: + element: ".*${SUBJECT}.*" + direction: DOWN + timeout: 30000 +- tapOn: ".*${SUBJECT}.*" +- extendedWaitUntil: + visible: "Reply All" + timeout: 20000 +- waitForAnimationToEnd: + timeout: 5000 diff --git a/.maestro/subflows/settings-page.yaml b/.maestro/subflows/settings-page.yaml new file mode 100644 index 00000000..c55a5502 --- /dev/null +++ b/.maestro/subflows/settings-page.yaml @@ -0,0 +1,24 @@ +# Settings list: scroll to the row ROW (regex on "Title, summary"), open it, +# check PROOF shows, screenshot, and go back on a phone. On the iPad the list +# is a left column, so scroll there instead of the middle of the screen. +appId: org.bulwarkmail.mobile +--- +- repeat: + while: + notVisible: ${ROW} + times: 10 + commands: + - swipe: + start: "${DEVICE == 'tablet' ? '18%,80%' : '50%,80%'}" + end: "${DEVICE == 'tablet' ? '18%,40%' : '50%,40%'}" + duration: 600 +- tapOn: ${ROW} +- extendedWaitUntil: + visible: ${PROOF} + timeout: 20000 +- takeScreenshot: settings-${NAME} +- runFlow: + when: + visible: "Back" + commands: + - tapOn: "Back" diff --git a/.maestro/subflows/sign-out.yaml b/.maestro/subflows/sign-out.yaml index 6f016d4f..e41c0aa4 100644 --- a/.maestro/subflows/sign-out.yaml +++ b/.maestro/subflows/sign-out.yaml @@ -4,10 +4,10 @@ appId: org.bulwarkmail.mobile - tapOn: id: tab-settings - scrollUntilVisible: - element: "Sign Out" + element: "Sign out" direction: DOWN timeout: 30000 -- tapOn: "Sign Out" +- tapOn: "Sign out" - extendedWaitUntil: visible: "The account is removed from this device.*" timeout: 10000 diff --git a/.maestro/subflows/signed-in.yaml b/.maestro/subflows/signed-in.yaml new file mode 100644 index 00000000..0d129495 --- /dev/null +++ b/.maestro/subflows/signed-in.yaml @@ -0,0 +1,28 @@ +# Feature flows start here: the app signed in to ACCOUNT, on the inbox. +# Keeps the app's data between flows; signs in only when needed. +appId: org.bulwarkmail.mobile +--- +- launchApp: + permissions: + notifications: allow + photos: allow + camera: allow +- runFlow: + when: + true: ${ORIENTATION == 'landscape'} + commands: + - setOrientation: LANDSCAPE_LEFT +- extendedWaitUntil: + visible: "Enter server details manually|Inbox" + timeout: 120000 +- runFlow: + when: + visible: "Enter server details manually" + commands: + - runFlow: + file: password-sign-in.yaml + env: + LOGIN: ${ACCOUNT}@example.org +- tapOn: + id: tab-mail +- runFlow: go-to-inbox.yaml diff --git a/.maestro/subflows/type-secret.yaml b/.maestro/subflows/type-secret.yaml index f4bd7a2f..bbde88fe 100644 --- a/.maestro/subflows/type-secret.yaml +++ b/.maestro/subflows/type-secret.yaml @@ -1,17 +1,15 @@ -# Type a secret into the field with id FIELD. Same focus handling as -# type.yaml, but the value can't be checked (a secure field shows dots), and -# it must never be shown, so a lost keystroke surfaces as a failed sign-in. +# Type a secret into the field with id FIELD (TAP=no for a field that focuses +# itself, see type.yaml). The value can't be checked (a secure field shows +# dots) and must never be shown, so a lost keystroke surfaces as a failed +# sign-in. appId: org.bulwarkmail.mobile --- - runFlow: when: - notVisible: - id: ${FIELD} - focused: true + true: ${TAP != 'no'} commands: - tapOn: id: ${FIELD} - waitForAnimationToEnd: timeout: 3000 -- eraseText: 80 - inputText: ${TEXT} diff --git a/.maestro/subflows/type.yaml b/.maestro/subflows/type.yaml index 5ca5d6e8..e7c7f5e4 100644 --- a/.maestro/subflows/type.yaml +++ b/.maestro/subflows/type.yaml @@ -1,25 +1,44 @@ # Type TEXT into the field with id FIELD and check it arrived in full. # # iOS drops keystrokes while the edit menu (the "AutoFill" bubble) opens, -# which happens when a flow taps a field that already has focus; an -# auto-focused field is left alone. Retyped up to twice if text got lost. +# which happens when a flow taps a field that already has focus. Maestro +# can't see the focus of a React Native field, so a caller passes TAP=no for +# a field that focuses itself. Retyped up to twice if text got lost (the +# retry taps: by then the field has focus and the menu is harmless). appId: org.bulwarkmail.mobile --- -- retry: - maxRetries: 2 +- runFlow: + when: + true: ${TAP != 'no'} commands: - - runFlow: - when: - notVisible: - id: ${FIELD} - focused: true + - tapOn: + id: ${FIELD} +- waitForAnimationToEnd: + timeout: 3000 +# ERASE=yes: the field already holds text. +- runFlow: + when: + true: ${ERASE == 'yes'} + commands: + - eraseText: 80 +- inputText: ${TEXT} +- runFlow: + when: + notVisible: + id: ${FIELD} + text: ${TEXT_PATTERN || TEXT} + commands: + - retry: + maxRetries: 2 commands: - tapOn: id: ${FIELD} - - waitForAnimationToEnd: - timeout: 3000 - - eraseText: 80 - - inputText: ${TEXT} - - assertVisible: - id: ${FIELD} - text: ${TEXT_PATTERN || TEXT} + - waitForAnimationToEnd: + timeout: 2000 + - eraseText: 80 + - waitForAnimationToEnd: + timeout: 1000 + - inputText: ${TEXT} + - assertVisible: + id: ${FIELD} + text: ${TEXT_PATTERN || TEXT} diff --git a/scripts/e2e/boot-sim.sh b/scripts/e2e/boot-sim.sh index 0547dd88..a96f12ac 100644 --- a/scripts/e2e/boot-sim.sh +++ b/scripts/e2e/boot-sim.sh @@ -44,6 +44,8 @@ xcrun simctl spawn "$UDID" defaults write com.apple.WebUI AutoFillPasswords -boo xcrun simctl ui "$UDID" appearance "$APPEARANCE" xcrun simctl install "$UDID" "$APP" +# A photo for the compose flow's attachment. +xcrun simctl addmedia "$UDID" "$(dirname "$0")/../../assets/icon.png" || true { echo "UDID=$UDID" diff --git a/scripts/e2e/run-flows.sh b/scripts/e2e/run-flows.sh index d11dddad..111d5695 100644 --- a/scripts/e2e/run-flows.sh +++ b/scripts/e2e/run-flows.sh @@ -5,18 +5,29 @@ # # Needs in the environment: UDID, OUT (output folder), and for the flows the # E2E_* values (see .github/workflows/e2e-ios.yml). Output: $OUT/report.xml, -# $OUT/screenshots, $OUT/run.mp4, $OUT/app.log. +# $OUT/screenshots, $OUT/run.mp4, $OUT/app.log, $OUT/steps.txt. # -# Secrets: Maestro's debug output and the JSON under ~/.maestro/tests hold -# every typed text, passwords included, and artifacts of this public repo are -# public. Only the warm-up (which types a made-up address) prints a debug -# log; afterwards only PNGs are copied, and every text file in $OUT is -# scrubbed of the secret values before upload. +# Each flow runs in its own Maestro process with a time cap: when the XCTest +# driver dies (it does, now and then), only that flow fails and the next one +# starts a fresh driver. +# +# Secrets: Maestro's debug output holds every typed text, passwords included, +# and artifacts of this public repo are public. Only the warm-up (which types +# a made-up address) prints a debug log; the debug output of the real flows +# stays on the runner, and only PNGs and an argument-free summary of the +# steps (failed-steps.mjs) are copied. Every text file in $OUT is scrubbed of +# the secret values before upload. set -uo pipefail SUITE=$1 ONLY=${2:-} ROOT=$(cd "$(dirname "$0")/../.." && pwd) -mkdir -p "$OUT/screenshots" +FLOW_TIMEOUT=${FLOW_TIMEOUT:-900} +mkdir -p "$OUT/screenshots" "$OUT/junit" +DEBUG_DIR="$RUNNER_TEMP/maestro-debug" +mkdir -p "$DEBUG_DIR" + +# perl's alarm stands in for coreutils timeout, which macOS lacks. +capped() { perl -e 'alarm shift; exec @ARGV' "$FLOW_TIMEOUT" "$@"; } maestro --version @@ -24,15 +35,15 @@ xcrun simctl io "$UDID" recordVideo --codec=h264 --force "$OUT/run.mp4" & RECORDER=$! # Warm-up: the XCTest driver takes minutes to start on a cold runner. -touch "$RUNNER_TEMP/flows-started" -(cd "$OUT/screenshots" && maestro --device "$UDID" test \ +WARMUP_START=$(date +%s) +(cd "$OUT/screenshots" && capped maestro --device "$UDID" test \ --debug-output "$RUNNER_TEMP/maestro-warmup" "$ROOT/.maestro/launch.yaml") STATUS=$? +echo "warm-up took $(( $(date +%s) - WARMUP_START )) s (status $STATUS)" if [ $STATUS -ne 0 ]; then echo "::group::Maestro warm-up log" find "$RUNNER_TEMP/maestro-warmup" -name 'maestro.log' -exec tail -n 300 {} \; echo "::endgroup::" - maestro --device "$UDID" hierarchy > "$OUT/hierarchy-warmup.json" 2>/dev/null || true fi if [ $STATUS -eq 0 ]; then @@ -48,48 +59,74 @@ if [ $STATUS -eq 0 ]; then -e "DEVICE=${DEVICE:-phone}" -e "APPEARANCE=${APPEARANCE:-light}" ) - TARGETS=() + FLOWS=() if [ -n "$ONLY" ]; then IFS=',' read -ra NAMES <<< "$ONLY" for n in "${NAMES[@]}"; do f="$ROOT/.maestro/$SUITE/${n%.yaml}.yaml" - [ -f "$f" ] && TARGETS+=("$f") || echo "::warning::no flow $SUITE/$n" + if [ -f "$f" ]; then FLOWS+=("$f"); else echo "::warning::no flow $SUITE/$n"; fi done else - TARGETS=("$ROOT/.maestro/$SUITE") + for f in "$ROOT/.maestro/$SUITE"/*.yaml; do FLOWS+=("$f"); done fi - if [ ${#TARGETS[@]} -gt 0 ]; then - (cd "$OUT/screenshots" && maestro --device "$UDID" test \ - --format junit --output "$OUT/report.xml" \ - "${ARGS[@]}" "${TARGETS[@]}") - STATUS=$? + + FAILED=() + for f in "${FLOWS[@]}"; do + name=$(basename "$f" .yaml) + started=$(date +%s) + echo "::group::$name" + (cd "$OUT/screenshots" && capped maestro --device "$UDID" test \ + --format junit --output "$OUT/junit/$name.xml" \ + --debug-output "$DEBUG_DIR/$name" \ + "${ARGS[@]}" "$f") + rc=$? + echo "::endgroup::" + echo "$name: exit $rc after $(( $(date +%s) - started )) s" + if [ $rc -ne 0 ]; then + FAILED+=("$name") + # What the screen showed when it failed. + xcrun simctl io "$UDID" screenshot "$OUT/screenshots/FAILED-$name.png" >/dev/null 2>&1 || true + fi + done + if [ ${#FAILED[@]} -gt 0 ]; then + echo "::error::Failed flows: ${FAILED[*]}" + STATUS=1 fi + + node -e ' + const fs = require("fs"), path = require("path"); + const dir = process.argv[1]; + const cases = fs.readdirSync(dir).filter((f) => f.endsWith(".xml")).sort() + .map((f) => (fs.readFileSync(path.join(dir, f), "utf8").match(//g) || []).join("\n")); + fs.writeFileSync(process.argv[2], `\n\n\n${cases.join("\n")}\n\n\n`); + ' "$OUT/junit" "$OUT/report.xml" "$SUITE" || true fi kill -INT "$RECORDER" 2>/dev/null || true wait "$RECORDER" 2>/dev/null || true -# Screenshots of failed steps land under ~/.maestro/tests. Images only, plus -# a summary of the steps without their arguments (see failed-steps.mjs). -find "$HOME/.maestro/tests" -name '*.png' -exec cp {} "$OUT/screenshots/" \; 2>/dev/null || true -node "$ROOT/scripts/e2e/failed-steps.mjs" "$HOME/.maestro/tests" > "$OUT/steps.txt" 2>&1 || true +# Images of failed steps, plus a summary of the steps without arguments. +find "$DEBUG_DIR" "$HOME/.maestro/tests" -name '*.png' -exec cp {} "$OUT/screenshots/" \; 2>/dev/null || true +node "$ROOT/scripts/e2e/failed-steps.mjs" "$DEBUG_DIR" > "$OUT/steps.txt" 2>&1 || true +find "$DEBUG_DIR" -type f | sed "s|$DEBUG_DIR/||" > "$OUT/debug-files.txt" 2>/dev/null || true -xcrun simctl spawn "$UDID" log show --last 60m --style compact \ +xcrun simctl spawn "$UDID" log show --last 90m --style compact \ --predicate 'process == "BulwarkMobile"' > "$OUT/app.log" 2>&1 || true # Scrub secrets from every text file that gets uploaded. -for f in $(find "$OUT" -type f \( -name '*.xml' -o -name '*.log' -o -name '*.json' -o -name '*.txt' \)); do - node -e ' - const fs = require("fs"); - const f = process.argv[1]; - let s = fs.readFileSync(f, "utf8"); - for (const k of ["E2E_PASSWORD", "E2E_USERB_PASSWORD", "E2E_TOTP_SECRET"]) { - const v = process.env[k]; - if (v && v.length >= 6) s = s.split(v).join("***"); - } - fs.writeFileSync(f, s); - ' "$f" -done +find "$OUT" -type f \( -name '*.xml' -o -name '*.log' -o -name '*.json' -o -name '*.txt' \) -print0 | + while IFS= read -r -d '' f; do + node -e ' + const fs = require("fs"); + const f = process.argv[1]; + let s = fs.readFileSync(f, "utf8"); + for (const k of ["E2E_PASSWORD", "E2E_USERB_PASSWORD", "E2E_TOTP_SECRET"]) { + const v = process.env[k]; + if (v && v.length >= 6) s = s.split(v).join("***"); + } + fs.writeFileSync(f, s); + ' "$f" + done -ls -la "$OUT/screenshots" | tail -n +2 | wc -l | xargs echo "screenshots:" +echo "screenshots: $(ls "$OUT/screenshots" | wc -l)" exit $STATUS