From 1f7acaf08abc58706b8c7c3e994d874efba56c07 Mon Sep 17 00:00:00 2001 From: Linus Rath <139418639+rathlinus@users.noreply.github.com> Date: Sun, 27 Sep 2026 01:19:17 +0200 Subject: [PATCH 01/87] refactor: start headless tasks for push and device sync the same way The code that starts React Native for a headless JS task, when the process was woken without any UI, lived inside BulwarkPushJobService. The device sync adapters need the same start, so it moves to a shared HeadlessJs object, with a small startTask helper next to it. Push behaviour does not change: - onStartJob still calls the start code on the main thread, and the listener-then-ready-check order is the same code, moved verbatim - the task still starts from `UiThreadUtil.runOnUiThread`, its id is still recorded in that same UI-thread turn, and a refusal because the app came to the foreground still logs and finishes the job - the start code now returns React Native's start task, so a caller can notice a start that fails (a debug build with no Metro); the push job ignores it The helper documents that it must run on the main thread: the ReactHost is created on first use without a lock, so a background caller racing MainActivity could create a second one. Checked with `gradlew assembleDebug -PreactNativeArchitectures=x86_64`. --- .../bulwarkmobile/BulwarkPushJobService.kt | 73 ++---------- .../com/anonymous/bulwarkmobile/HeadlessJs.kt | 106 ++++++++++++++++++ 2 files changed, 115 insertions(+), 64 deletions(-) create mode 100644 android/app/src/main/java/com/anonymous/bulwarkmobile/HeadlessJs.kt diff --git a/android/app/src/main/java/com/anonymous/bulwarkmobile/BulwarkPushJobService.kt b/android/app/src/main/java/com/anonymous/bulwarkmobile/BulwarkPushJobService.kt index cde4fd5f..60a7cc14 100644 --- a/android/app/src/main/java/com/anonymous/bulwarkmobile/BulwarkPushJobService.kt +++ b/android/app/src/main/java/com/anonymous/bulwarkmobile/BulwarkPushJobService.kt @@ -11,16 +11,11 @@ import android.os.Bundle import android.os.PersistableBundle import android.os.SystemClock import android.util.Log -import com.facebook.react.ReactApplication -import com.facebook.react.ReactInstanceEventListener import com.facebook.react.bridge.ReactContext -import com.facebook.react.bridge.UiThreadUtil -import com.facebook.react.internal.featureflags.ReactNativeNewArchitectureFeatureFlags import com.facebook.react.jstasks.HeadlessJsTaskConfig import com.facebook.react.jstasks.HeadlessJsTaskContext import com.facebook.react.jstasks.HeadlessJsTaskEventListener import java.util.concurrent.ConcurrentHashMap -import java.util.concurrent.atomic.AtomicBoolean /** * Runs the BulwarkPushTask headless JS task from a JobScheduler job. @@ -44,7 +39,8 @@ class BulwarkPushJobService : JobService(), HeadlessJsTaskEventListener { override fun onStartJob(params: JobParameters): Boolean { val config = BulwarkPushTaskService.taskConfig(Bundle(params.extras)) - withReadyReactContext { startTask(it, config, params) } + // JobService callbacks run on the main thread, where HeadlessJs wants to be called. + HeadlessJs.withReadyReactContext(application) { startTask(it, config, params) } return true } @@ -76,68 +72,17 @@ class BulwarkPushJobService : JobService(), HeadlessJsTaskEventListener { tasks.addTaskEventListener(this) taskContext = tasks } - UiThreadUtil.runOnUiThread { - try { - val taskId = tasks.startTask(config) - running[taskId] = RunningTask(params, SystemClock.elapsedRealtime()) - } catch (e: IllegalStateException) { + HeadlessJs.startTask( + tasks, + config, + onStarted = { taskId -> running[taskId] = RunningTask(params, SystemClock.elapsedRealtime()) }, + onRefused = { e -> // The app came to the foreground since the push arrived and // syncs on its own there. Log.i(TAG, "Push job ${params.jobId} skipped: ${e.message}") jobFinished(params, false) - } - } - } - - /** - * Calls [onReady] once with a React context whose JS instance is up, - * starting React Native if the process was woken for this job. - * - * A context exists before its instance is ready, and a task started on - * it then never reaches JS (HeadlessJsTaskContext only logs "CatalystInstance - * not available" and waits for the timeout), which happened when several - * pushes arrived while the app was starting. So the listener goes in - * first and the ready check after it; whichever sees the ready instance - * first runs the task. - */ - private fun withReadyReactContext(onReady: (ReactContext) -> Unit) { - val done = AtomicBoolean(false) - val runOnce = { context: ReactContext -> if (done.compareAndSet(false, true)) onReady(context) } - val app = application as ReactApplication - if (ReactNativeNewArchitectureFeatureFlags.enableBridgelessArchitecture()) { - val host = checkNotNull(app.reactHost) { "ReactHost is not initialized in New Architecture" } - val listener = object : ReactInstanceEventListener { - override fun onReactContextInitialized(context: ReactContext) { - host.removeReactInstanceEventListener(this) - runOnce(context) - } - } - host.addReactInstanceEventListener(listener) - val current = host.currentReactContext - if (current != null && current.hasActiveReactInstance()) { - host.removeReactInstanceEventListener(listener) - runOnce(current) - } else { - host.start() - } - } else { - @Suppress("DEPRECATION") - val manager = app.reactNativeHost.reactInstanceManager - val listener = object : ReactInstanceEventListener { - override fun onReactContextInitialized(context: ReactContext) { - manager.removeReactInstanceEventListener(this) - runOnce(context) - } - } - manager.addReactInstanceEventListener(listener) - val current = manager.currentReactContext - if (current != null && current.hasActiveReactInstance()) { - manager.removeReactInstanceEventListener(listener) - runOnce(current) - } else if (!manager.hasStartedCreatingInitialContext()) { - manager.createReactContextInBackground() - } - } + }, + ) } companion object { diff --git a/android/app/src/main/java/com/anonymous/bulwarkmobile/HeadlessJs.kt b/android/app/src/main/java/com/anonymous/bulwarkmobile/HeadlessJs.kt new file mode 100644 index 00000000..e8521c38 --- /dev/null +++ b/android/app/src/main/java/com/anonymous/bulwarkmobile/HeadlessJs.kt @@ -0,0 +1,106 @@ +package com.anonymous.bulwarkmobile + +import android.app.Application +import androidx.annotation.MainThread +import com.facebook.react.ReactApplication +import com.facebook.react.ReactInstanceEventListener +import com.facebook.react.bridge.ReactContext +import com.facebook.react.bridge.UiThreadUtil +import com.facebook.react.interfaces.TaskInterface +import com.facebook.react.internal.featureflags.ReactNativeNewArchitectureFeatureFlags +import com.facebook.react.jstasks.HeadlessJsTaskConfig +import com.facebook.react.jstasks.HeadlessJsTaskContext +import java.util.concurrent.atomic.AtomicBoolean + +/** + * Running a headless JS task without HeadlessJsTaskService: starting React + * Native when the process was woken without any UI, and starting the task on + * its context. Shared by the push job (BulwarkPushJobService) and the device + * sync adapters. + */ +object HeadlessJs { + /** + * Calls [onReady] once with a React context whose JS instance is up, + * starting React Native if the process was woken for this work. + * + * A context exists before its instance is ready, and a task started on + * it then never reaches JS (HeadlessJsTaskContext only logs "CatalystInstance + * not available" and waits for the timeout), which happened when several + * pushes arrived while the app was starting. So the listener goes in + * first and the ready check after it; whichever sees the ready instance + * first runs the task. + * + * Call it on the main thread; [onReady] runs there too. MainApplication + * creates the ReactHost on first use without a lock (ExpoReactHostFactory), + * so a background caller racing MainActivity could create a second one, + * with a second JS runtime. + * + * Returns React Native's start task when it had to be started, so a caller + * can notice a start that fails (a debug build with no Metro to load the + * bundle from), or null when the context was already up. + */ + @MainThread + fun withReadyReactContext(app: Application, onReady: (ReactContext) -> Unit): TaskInterface? { + val done = AtomicBoolean(false) + val runOnce = { context: ReactContext -> if (done.compareAndSet(false, true)) onReady(context) } + val reactApp = app as ReactApplication + if (ReactNativeNewArchitectureFeatureFlags.enableBridgelessArchitecture()) { + val host = checkNotNull(reactApp.reactHost) { "ReactHost is not initialized in New Architecture" } + val listener = object : ReactInstanceEventListener { + override fun onReactContextInitialized(context: ReactContext) { + host.removeReactInstanceEventListener(this) + runOnce(context) + } + } + host.addReactInstanceEventListener(listener) + val current = host.currentReactContext + if (current != null && current.hasActiveReactInstance()) { + host.removeReactInstanceEventListener(listener) + runOnce(current) + return null + } + return host.start() + } + @Suppress("DEPRECATION") + val manager = reactApp.reactNativeHost.reactInstanceManager + val listener = object : ReactInstanceEventListener { + override fun onReactContextInitialized(context: ReactContext) { + manager.removeReactInstanceEventListener(this) + runOnce(context) + } + } + manager.addReactInstanceEventListener(listener) + val current = manager.currentReactContext + if (current != null && current.hasActiveReactInstance()) { + manager.removeReactInstanceEventListener(listener) + runOnce(current) + } else if (!manager.hasStartedCreatingInitialContext()) { + manager.createReactContextInBackground() + } + return null + } + + /** + * Starts [config] on the UI thread, where HeadlessJsTaskContext wants it. + * [onStarted] gets the task id in that same UI-thread turn, so before any + * finish callback (HeadlessJsTaskContext posts those to the UI thread). + * [onRefused] gets the refusal when the context is resumed and [config] + * isn't allowed in the foreground. + */ + fun startTask( + tasks: HeadlessJsTaskContext, + config: HeadlessJsTaskConfig, + onStarted: (taskId: Int) -> Unit, + onRefused: (IllegalStateException) -> Unit, + ) { + UiThreadUtil.runOnUiThread { + val taskId = try { + tasks.startTask(config) + } catch (e: IllegalStateException) { + onRefused(e) + return@runOnUiThread + } + onStarted(taskId) + } + } +} From ed2525e5cb39b05bd54e3c8568e069e2b224b4d7 Mon Sep 17 00:00:00 2001 From: Linus Rath <139418639+rathlinus@users.noreply.github.com> Date: Sun, 27 Sep 2026 01:53:40 +0200 Subject: [PATCH 02/87] feat: register a bulwark account with android's contacts and calendar sync #34 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Android syncs the contacts and calendars of other services through accounts and sync adapters. This adds the Android side of device sync: a "Bulwark" account type (`${applicationId}.account`, set in build.gradle), an authenticator that sends Settings → Add account to the app instead of asking for credentials, and one sync adapter each for the contacts and the calendar provider. An adapter does no syncing itself. It starts the BulwarkDeviceSync headless JS task from the main thread, through the helper the push job uses, waits for the task's report and hands it to SyncManager as the SyncResult. The engine that will write the report runs in JS, so the existing client stays the only owner of token refresh, mTLS and session discovery; for now the task only reports that device sync is off. Nothing in the app creates an account yet, so the feature stays unreachable until its settings land. Debug builds get an adb hook for the device tests (DeviceSyncDebugReceiver, shell only). Hard errors are reported without progress: SyncManager retries a failed sync at once when it made progress, which would loop an auth failure. Checked on the API 34 and API 32 emulators with the debug build, from logcat and `dumpsys content`: - app in the foreground: report after 319-334 ms - app in the background: 320-327 ms - process killed: SyncManager starts it for the sync service, React Native starts headless, and the report arrives after 16.2 s (API 34) and 8.5 s (API 32), most of it the debug bundle download from Metro - an io report is retried with back-off and an auth report is not retried; too many deletions shows the system's "Attempted to delete too many Contacts." notification - with Metro unreachable the adapter gives up after 2 s instead of waiting for its deadline - `gradlew :app:testDebugUnitTest`: 10 JVM tests for the report mapping --- android/app/build.gradle | 10 + android/app/src/debug/AndroidManifest.xml | 5 +- .../sync/DeviceSyncDebugReceiver.kt | 58 +++ android/app/src/main/AndroidManifest.xml | 30 ++ .../bulwarkmobile/BulwarkFcmPackage.kt | 2 + .../sync/BulwarkAuthenticator.kt | 79 ++++ .../sync/BulwarkDeviceSyncModule.kt | 222 +++++++++++ .../bulwarkmobile/sync/DeviceSyncAccounts.kt | 111 ++++++ .../bulwarkmobile/sync/DeviceSyncAdapter.kt | 188 +++++++++ .../bulwarkmobile/sync/DeviceSyncRuns.kt | 82 ++++ .../bulwarkmobile/sync/SyncReports.kt | 90 +++++ android/app/src/main/res/values/strings.xml | 2 + .../app/src/main/res/xml/authenticator.xml | 7 + android/app/src/main/res/xml/contacts.xml | 29 ++ .../app/src/main/res/xml/sync_calendar.xml | 9 + .../app/src/main/res/xml/sync_contacts.xml | 9 + .../bulwarkmobile/sync/SyncReportsTest.kt | 84 ++++ index.ts | 8 + src/device-sync/task.ts | 41 ++ src/device-sync/types.ts | 369 ++++++++++++++++++ 20 files changed, 1434 insertions(+), 1 deletion(-) create mode 100644 android/app/src/debug/java/com/anonymous/bulwarkmobile/sync/DeviceSyncDebugReceiver.kt create mode 100644 android/app/src/main/java/com/anonymous/bulwarkmobile/sync/BulwarkAuthenticator.kt create mode 100644 android/app/src/main/java/com/anonymous/bulwarkmobile/sync/BulwarkDeviceSyncModule.kt create mode 100644 android/app/src/main/java/com/anonymous/bulwarkmobile/sync/DeviceSyncAccounts.kt create mode 100644 android/app/src/main/java/com/anonymous/bulwarkmobile/sync/DeviceSyncAdapter.kt create mode 100644 android/app/src/main/java/com/anonymous/bulwarkmobile/sync/DeviceSyncRuns.kt create mode 100644 android/app/src/main/java/com/anonymous/bulwarkmobile/sync/SyncReports.kt create mode 100644 android/app/src/main/res/xml/authenticator.xml create mode 100644 android/app/src/main/res/xml/contacts.xml create mode 100644 android/app/src/main/res/xml/sync_calendar.xml create mode 100644 android/app/src/main/res/xml/sync_contacts.xml create mode 100644 android/app/src/test/java/com/anonymous/bulwarkmobile/sync/SyncReportsTest.kt create mode 100644 src/device-sync/task.ts create mode 100644 src/device-sync/types.ts diff --git a/android/app/build.gradle b/android/app/build.gradle index d2ed16b4..ad476faf 100644 --- a/android/app/build.gradle +++ b/android/app/build.gradle @@ -154,6 +154,11 @@ android { versionCode appVersionCode.toInteger() versionName appVersionName + // Android account type of device sync (#34), derived from the package + // name so a later rename needs no other change. The XML in res/xml and + // the Kotlin in sync/ read it as @string/device_sync_account_type. + resValue "string", "device_sync_account_type", "${applicationId}.account" + buildConfigField "String", "REACT_NATIVE_RELEASE_LEVEL", "\"${findProperty('reactNativeReleaseLevel') ?: 'stable'}\"" } signingConfigs { @@ -280,6 +285,11 @@ dependencies { // Kotlin 2.1 toolchain; 3.3.5 went back to 2.2 which it can read. implementation("org.unifiedpush.android:connector:3.3.5") + // JVM unit tests for the logic in the Kotlin shell of device sync (src/test). + // android.jar only stubs org.json, so the tests use the real one. + testImplementation("junit:junit:4.13.2") + testImplementation("org.json:json:20240303") + def isGifEnabled = (findProperty('expo.gif.enabled') ?: "") == "true"; def isWebpEnabled = (findProperty('expo.webp.enabled') ?: "") == "true"; def isWebpAnimatedEnabled = (findProperty('expo.webp.animated') ?: "") == "true"; diff --git a/android/app/src/debug/AndroidManifest.xml b/android/app/src/debug/AndroidManifest.xml index 3ec2507b..1146debf 100644 --- a/android/app/src/debug/AndroidManifest.xml +++ b/android/app/src/debug/AndroidManifest.xml @@ -3,5 +3,8 @@ - + + + + diff --git a/android/app/src/debug/java/com/anonymous/bulwarkmobile/sync/DeviceSyncDebugReceiver.kt b/android/app/src/debug/java/com/anonymous/bulwarkmobile/sync/DeviceSyncDebugReceiver.kt new file mode 100644 index 00000000..b08f283b --- /dev/null +++ b/android/app/src/debug/java/com/anonymous/bulwarkmobile/sync/DeviceSyncDebugReceiver.kt @@ -0,0 +1,58 @@ +package com.anonymous.bulwarkmobile.sync + +import android.content.BroadcastReceiver +import android.content.Context +import android.content.Intent +import android.util.Log + +/** + * Debug builds only: lets adb set up device sync accounts without the settings + * UI, for the device tests in docs/device-sync.md ("Testing"). Protected by + * android.permission.DUMP, which only the shell and the system hold. Syncs + * themselves are triggered with `adb shell requestsync`. + * + * adb shell am broadcast -n com.anonymous.bulwarkmobile/.sync.DeviceSyncDebugReceiver \ + * --es cmd ensure --es name usera@example.org --es registry usera@example.org@10.0.2.2 \ + * --es enable contacts,calendar + * + * Commands: `ensure` (name, registry, optional enable), `enable` (name, + * authority, optional --ez on false), `remove` (name) and `list`. + */ +class DeviceSyncDebugReceiver : BroadcastReceiver() { + override fun onReceive(context: Context, intent: Intent) { + val result = try { + val name = intent.getStringExtra("name") + when (intent.getStringExtra("cmd")) { + "ensure" -> { + val created = DeviceSyncAccounts.ensure(context, name!!, intent.getStringExtra("registry")!!) + for (which in intent.getStringExtra("enable").orEmpty().split(',').filter { it.isNotBlank() }) { + DeviceSyncAccounts.setSyncEnabled(context, name, authority(which), true) + } + "created=$created" + } + "enable" -> { + val on = intent.getBooleanExtra("on", true) + DeviceSyncAccounts.setSyncEnabled(context, name!!, authority(intent.getStringExtra("authority")!!), on) + "enabled=$on" + } + "remove" -> "removed=${DeviceSyncAccounts.remove(context, name!!)}" + "list" -> DeviceSyncAccounts.list(context).joinToString(";") { (account, registryId) -> "${account.name}=$registryId" } + else -> "unknown cmd" + } + } catch (e: Exception) { + "error: ${e.javaClass.simpleName}: ${e.message}" + } + Log.i(TAG, "debug command: $result") + resultData = result + } + + private fun authority(which: String) = when (which) { + "contacts" -> DeviceSyncAccounts.CONTACTS_AUTHORITY + "calendar" -> DeviceSyncAccounts.CALENDAR_AUTHORITY + else -> which + } + + companion object { + private const val TAG = "BulwarkDeviceSync" + } +} diff --git a/android/app/src/main/AndroidManifest.xml b/android/app/src/main/AndroidManifest.xml index 1c2731e4..3d6513ce 100644 --- a/android/app/src/main/AndroidManifest.xml +++ b/android/app/src/main/AndroidManifest.xml @@ -6,6 +6,14 @@ + + + + + + + + @@ -66,6 +74,28 @@ + + + + + + + + + + + + + + + + + + + + diff --git a/android/app/src/main/java/com/anonymous/bulwarkmobile/BulwarkFcmPackage.kt b/android/app/src/main/java/com/anonymous/bulwarkmobile/BulwarkFcmPackage.kt index f384dd56..5df02288 100644 --- a/android/app/src/main/java/com/anonymous/bulwarkmobile/BulwarkFcmPackage.kt +++ b/android/app/src/main/java/com/anonymous/bulwarkmobile/BulwarkFcmPackage.kt @@ -1,5 +1,6 @@ package com.anonymous.bulwarkmobile +import com.anonymous.bulwarkmobile.sync.BulwarkDeviceSyncModule import com.facebook.react.ReactPackage import com.facebook.react.bridge.NativeModule import com.facebook.react.bridge.ReactApplicationContext @@ -12,6 +13,7 @@ class BulwarkFcmPackage : ReactPackage { add(BulwarkFcmModule(reactContext)) add(BulwarkUnifiedPushModule(reactContext)) add(BulwarkClientCertModule(reactContext)) + add(BulwarkDeviceSyncModule(reactContext)) // The bridge registers React Native's own module under this name. if (ReactNativeNewArchitectureFeatureFlags.enableBridgelessArchitecture()) { add(BulwarkHeadlessJsTaskSupportModule(reactContext)) diff --git a/android/app/src/main/java/com/anonymous/bulwarkmobile/sync/BulwarkAuthenticator.kt b/android/app/src/main/java/com/anonymous/bulwarkmobile/sync/BulwarkAuthenticator.kt new file mode 100644 index 00000000..e26aa44c --- /dev/null +++ b/android/app/src/main/java/com/anonymous/bulwarkmobile/sync/BulwarkAuthenticator.kt @@ -0,0 +1,79 @@ +package com.anonymous.bulwarkmobile.sync + +import android.accounts.AbstractAccountAuthenticator +import android.accounts.Account +import android.accounts.AccountAuthenticatorResponse +import android.accounts.AccountManager +import android.app.Service +import android.content.Context +import android.content.Intent +import android.net.Uri +import android.os.Bundle +import android.os.IBinder +import com.anonymous.bulwarkmobile.MainActivity + +/** + * The authenticator of the "Bulwark" account type. The app holds the + * credentials, not the account: accounts are only ever created by the app, + * from Settings → Contacts or Calendar, for an account that is signed in. + */ +class BulwarkAuthenticator(private val context: Context) : AbstractAccountAuthenticator(context) { + + /** Settings → Accounts → Add account opens the app's device sync settings instead. */ + override fun addAccount( + response: AccountAuthenticatorResponse?, + accountType: String?, + authTokenType: String?, + requiredFeatures: Array?, + options: Bundle?, + ): Bundle { + val intent = Intent(context, MainActivity::class.java) + .setAction(Intent.ACTION_VIEW) + .setData(Uri.parse(SETTINGS_URI)) + return Bundle().apply { putParcelable(AccountManager.KEY_INTENT, intent) } + } + + /** Removal from Android Settings is allowed; the app notices and turns device sync off. */ + override fun getAccountRemovalAllowed(response: AccountAuthenticatorResponse?, account: Account?): Bundle = + Bundle().apply { putBoolean(AccountManager.KEY_BOOLEAN_RESULT, true) } + + override fun getAuthToken( + response: AccountAuthenticatorResponse?, + account: Account?, + authTokenType: String?, + options: Bundle?, + ): Bundle = unsupported() + + override fun confirmCredentials(response: AccountAuthenticatorResponse?, account: Account?, options: Bundle?): Bundle = + unsupported() + + override fun updateCredentials( + response: AccountAuthenticatorResponse?, + account: Account?, + authTokenType: String?, + options: Bundle?, + ): Bundle = unsupported() + + override fun editProperties(response: AccountAuthenticatorResponse?, accountType: String?): Bundle = unsupported() + + override fun getAuthTokenLabel(authTokenType: String?): String? = null + + override fun hasFeatures(response: AccountAuthenticatorResponse?, account: Account?, features: Array?): Bundle = + Bundle().apply { putBoolean(AccountManager.KEY_BOOLEAN_RESULT, false) } + + private fun unsupported() = Bundle().apply { + putInt(AccountManager.KEY_ERROR_CODE, AccountManager.ERROR_CODE_UNSUPPORTED_OPERATION) + putString(AccountManager.KEY_ERROR_MESSAGE, "Bulwark signs in from the app") + } + + companion object { + /** Handled by the app's deep links: Settings → Contacts, with its "Sync to this device" section. */ + const val SETTINGS_URI = "bulwarkmobile://settings/contacts" + } +} + +class BulwarkAuthenticatorService : Service() { + private val authenticator by lazy { BulwarkAuthenticator(this) } + + override fun onBind(intent: Intent): IBinder? = authenticator.iBinder +} diff --git a/android/app/src/main/java/com/anonymous/bulwarkmobile/sync/BulwarkDeviceSyncModule.kt b/android/app/src/main/java/com/anonymous/bulwarkmobile/sync/BulwarkDeviceSyncModule.kt new file mode 100644 index 00000000..e60da9f2 --- /dev/null +++ b/android/app/src/main/java/com/anonymous/bulwarkmobile/sync/BulwarkDeviceSyncModule.kt @@ -0,0 +1,222 @@ +package com.anonymous.bulwarkmobile.sync + +import android.accounts.Account +import android.accounts.AccountManager +import android.accounts.OnAccountsUpdateListener +import android.content.ActivityNotFoundException +import android.content.ContentResolver +import android.content.Intent +import android.os.Build +import android.os.Handler +import android.os.Looper +import android.provider.Settings +import android.util.Log +import com.facebook.react.bridge.Arguments +import com.facebook.react.bridge.Promise +import com.facebook.react.bridge.ReactApplicationContext +import com.facebook.react.bridge.ReactContextBaseJavaModule +import com.facebook.react.bridge.ReactMethod +import com.facebook.react.bridge.WritableMap +import com.facebook.react.modules.core.DeviceEventManagerModule.RCTDeviceEventEmitter +import org.json.JSONObject + +/** + * `NativeModules.BulwarkDeviceSync`: Android accounts, sync settings and the + * run handshake of device sync (#34). The contract is `DeviceSyncNativeModule` + * in src/device-sync/types.ts; docs/device-sync.md explains it. + */ +class BulwarkDeviceSyncModule(reactContext: ReactApplicationContext) : + ReactContextBaseJavaModule(reactContext) { + + private val accountsListener = OnAccountsUpdateListener { + emit(ACCOUNTS_CHANGED_EVENT, null) + } + + init { + synchronized(BulwarkDeviceSyncModule::class.java) { currentInstance = this } + } + + override fun getName(): String = NAME + + override fun initialize() { + super.initialize() + val manager = AccountManager.get(reactApplicationContext) + val handler = Handler(Looper.getMainLooper()) + try { + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O) { + manager.addOnAccountsUpdatedListener(accountsListener, handler, false, arrayOf(accountType())) + } else { + manager.addOnAccountsUpdatedListener(accountsListener, handler, false) + } + } catch (e: RuntimeException) { + Log.w(TAG, "Account change listener not registered", e) + } + } + + override fun invalidate() { + try { + AccountManager.get(reactApplicationContext).removeOnAccountsUpdatedListener(accountsListener) + } catch (e: RuntimeException) { + // Not registered. + } + super.invalidate() + } + + private fun accountType() = DeviceSyncAccounts.accountType(reactApplicationContext) + + // ── accounts ──────────────────────────────────────────────── + + @ReactMethod + fun getInfo(promise: Promise) { + promise.resolve(Arguments.createMap().apply { + putString("accountType", accountType()) + putInt("sdkInt", Build.VERSION.SDK_INT) + }) + } + + @ReactMethod + fun listAccounts(promise: Promise) = settle(promise) { + Arguments.createArray().apply { + for ((account, registryId) in DeviceSyncAccounts.list(reactApplicationContext)) { + pushMap(Arguments.createMap().apply { + putString("name", account.name) + putString("registryId", registryId) + }) + } + } + } + + @ReactMethod + fun ensureAccount(name: String, registryId: String, promise: Promise) = settle(promise) { + DeviceSyncAccounts.ensure(reactApplicationContext, name, registryId) + } + + @ReactMethod + fun removeAccount(name: String, promise: Promise) = settle(promise) { + DeviceSyncAccounts.remove(reactApplicationContext, name) + } + + // ── sync settings ─────────────────────────────────────────── + + @ReactMethod + fun getSyncSettings(name: String, promise: Promise) = settle(promise) { + val account = DeviceSyncAccounts.account(reactApplicationContext, name) + Arguments.createMap().apply { + putBoolean("masterAutomatic", ContentResolver.getMasterSyncAutomatically()) + putMap("authorities", Arguments.createMap().apply { + for (authority in DeviceSyncAccounts.AUTHORITIES) putMap(authority, authoritySettings(account, authority)) + }) + } + } + + private fun authoritySettings(account: Account, authority: String): WritableMap = Arguments.createMap().apply { + putInt("syncable", ContentResolver.getIsSyncable(account, authority)) + putBoolean("automatic", ContentResolver.getSyncAutomatically(account, authority)) + val periodic = ContentResolver.getPeriodicSyncs(account, authority).minOfOrNull { it.period } ?: 0L + putDouble("periodicSeconds", periodic.toDouble()) + putBoolean("active", ContentResolver.isSyncActive(account, authority)) + putBoolean("pending", ContentResolver.isSyncPending(account, authority)) + } + + @ReactMethod + fun setSyncEnabled(name: String, authority: String, enabled: Boolean, promise: Promise) = settle(promise) { + DeviceSyncAccounts.setSyncEnabled(reactApplicationContext, name, authority, enabled) + null + } + + @ReactMethod + fun setPeriodicSync(name: String, authority: String, seconds: Double, promise: Promise) = settle(promise) { + DeviceSyncAccounts.setPeriodicSync(reactApplicationContext, name, authority, seconds.toLong()) + null + } + + @ReactMethod + fun requestSync(name: String, authority: String, optionsJson: String, promise: Promise) = settle(promise) { + val options = JSONObject(optionsJson) + val flags = SyncRequestFlags( + manual = options.optBoolean("manual"), + expedited = options.optBoolean("expedited"), + upload = options.optBoolean("upload"), + overrideTooManyDeletions = options.optBoolean("overrideTooManyDeletions"), + discardLocalDeletions = options.optBoolean("discardLocalDeletions"), + ) + DeviceSyncAccounts.requestSync(reactApplicationContext, name, authority, flags) + null + } + + /** The account's own sync screen where the platform has one, else the list of synced accounts. */ + @ReactMethod + fun openAccountSettings(name: String, promise: Promise) = settle(promise) { + val account = DeviceSyncAccounts.account(reactApplicationContext, name) + val perAccount = Intent(ACTION_ACCOUNT_SYNC_SETTINGS).putExtra(EXTRA_ACCOUNT, account) + val overview = Intent(Settings.ACTION_SYNC_SETTINGS) + .putExtra(Settings.EXTRA_ACCOUNT_TYPES, arrayOf(account.type)) + val activity = reactApplicationContext.currentActivity + for (intent in listOf(perAccount, overview)) { + try { + if (activity != null) { + activity.startActivity(intent) + } else { + reactApplicationContext.startActivity(intent.addFlags(Intent.FLAG_ACTIVITY_NEW_TASK)) + } + return@settle null + } catch (e: ActivityNotFoundException) { + // Try the next one. + } + } + throw IllegalStateException("No settings screen for sync accounts") + } + + // ── run handshake ─────────────────────────────────────────── + + @ReactMethod + fun finishRun(runId: String, reportJson: String, promise: Promise) { + val accepted = DeviceSyncRuns.finish(runId, reportJson) + if (!accepted) Log.i(TAG, "Report for run $runId ignored: nothing waits for it") + promise.resolve(accepted) + } + + @ReactMethod + fun isRunCancelled(runId: String, promise: Promise) { + promise.resolve(DeviceSyncRuns.isCancelled(runId)) + } + + // NativeEventEmitter bookkeeping. + @ReactMethod + fun addListener(eventName: String) {} + + @ReactMethod + fun removeListeners(count: Int) {} + + private inline fun settle(promise: Promise, block: () -> Any?) { + try { + promise.resolve(block()) + } catch (e: SecurityException) { + promise.reject("permission", e.message, e) + } catch (e: IllegalArgumentException) { + promise.reject("bad_args", e.message, e) + } catch (e: Exception) { + promise.reject("failed", e.message, e) + } + } + + companion object { + const val NAME = "BulwarkDeviceSync" + private const val TAG = "BulwarkDeviceSync" + const val ACCOUNTS_CHANGED_EVENT = "BulwarkDeviceSync:accountsChanged" + + /** Settings' per-account sync screen; public on AOSP-based settings apps, but not in the SDK. */ + private const val ACTION_ACCOUNT_SYNC_SETTINGS = "android.settings.ACCOUNT_SYNC_SETTINGS" + private const val EXTRA_ACCOUNT = "account" + + @Volatile private var currentInstance: BulwarkDeviceSyncModule? = null + + /** Dropped when no React instance or no JS listener is there, like BulwarkFcmModule.emit. */ + fun emit(eventName: String, params: WritableMap?) { + val module = currentInstance ?: return + val ctx = module.reactApplicationContext + if (!ctx.hasActiveReactInstance()) return + ctx.getJSModule(RCTDeviceEventEmitter::class.java).emit(eventName, params) + } + } +} diff --git a/android/app/src/main/java/com/anonymous/bulwarkmobile/sync/DeviceSyncAccounts.kt b/android/app/src/main/java/com/anonymous/bulwarkmobile/sync/DeviceSyncAccounts.kt new file mode 100644 index 00000000..2bbbcf86 --- /dev/null +++ b/android/app/src/main/java/com/anonymous/bulwarkmobile/sync/DeviceSyncAccounts.kt @@ -0,0 +1,111 @@ +package com.anonymous.bulwarkmobile.sync + +import android.accounts.Account +import android.accounts.AccountManager +import android.content.ContentResolver +import android.content.Context +import android.os.Bundle +import android.provider.CalendarContract +import android.provider.ContactsContract +import com.anonymous.bulwarkmobile.R + +/** + * The Android accounts of device sync (#34) and their sync settings. One + * account per app account that has device sync on; see docs/device-sync.md. + * + * Android's own sync settings are the source of truth for on/off: the + * Contacts and Calendar toggles in Settings → Accounts are + * `getSyncAutomatically`, and the app reads them back instead of keeping a + * copy. + */ +object DeviceSyncAccounts { + const val CONTACTS_AUTHORITY: String = ContactsContract.AUTHORITY + const val CALENDAR_AUTHORITY: String = CalendarContract.AUTHORITY + val AUTHORITIES = listOf(CONTACTS_AUTHORITY, CALENDAR_AUTHORITY) + + /** userData key holding the app account (`AccountEntry.id`) an Android account belongs to. */ + const val USER_DATA_REGISTRY_ID = "registryId" + + /** `.account`, set in build.gradle so the package name is never hard-coded. */ + fun accountType(context: Context): String = context.getString(R.string.device_sync_account_type) + + fun account(context: Context, name: String) = Account(name, accountType(context)) + + fun list(context: Context): List> { + val manager = AccountManager.get(context) + return manager.getAccountsByType(accountType(context)).map { it to manager.getUserData(it, USER_DATA_REGISTRY_ID) } + } + + /** + * Adds the account when it is missing and (re)writes its registry id. + * Both authorities start syncable with automatic sync off; the app turns + * on the ones the user picked. Returns true when the account was created. + */ + fun ensure(context: Context, name: String, registryId: String): Boolean { + val manager = AccountManager.get(context) + val account = account(context, name) + val exists = manager.getAccountsByType(account.type).any { it.name == name } + if (exists) { + manager.setUserData(account, USER_DATA_REGISTRY_ID, registryId) + return false + } + val userData = Bundle().apply { putString(USER_DATA_REGISTRY_ID, registryId) } + check(manager.addAccountExplicitly(account, null, userData)) { "Android refused to add the account" } + for (authority in AUTHORITIES) { + ContentResolver.setIsSyncable(account, authority, 1) + ContentResolver.setSyncAutomatically(account, authority, false) + } + return true + } + + /** Removes the account; the Contacts and Calendar providers drop all of its rows. */ + fun remove(context: Context, name: String): Boolean { + val manager = AccountManager.get(context) + val account = account(context, name) + if (manager.getAccountsByType(account.type).none { it.name == name }) return false + return manager.removeAccountExplicitly(account) + } + + fun setSyncEnabled(context: Context, name: String, authority: String, enabled: Boolean) { + requireAuthority(authority) + val account = account(context, name) + ContentResolver.setIsSyncable(account, authority, 1) + ContentResolver.setSyncAutomatically(account, authority, enabled) + } + + /** Replaces our periodic sync for the authority; 0 removes it. */ + fun setPeriodicSync(context: Context, name: String, authority: String, seconds: Long) { + requireAuthority(authority) + val account = account(context, name) + for (periodic in ContentResolver.getPeriodicSyncs(account, authority)) { + ContentResolver.removePeriodicSync(account, authority, periodic.extras) + } + if (seconds > 0) ContentResolver.addPeriodicSync(account, authority, Bundle.EMPTY, seconds) + } + + fun requestSync(context: Context, name: String, authority: String, flags: SyncRequestFlags) { + requireAuthority(authority) + ContentResolver.requestSync(account(context, name), authority, flags.toExtras()) + } + + fun requireAuthority(authority: String) { + require(authority in AUTHORITIES) { "Unknown authority $authority" } + } +} + +/** The SYNC_EXTRAS_* flags a request carries. Constant extras let repeated requests coalesce. */ +data class SyncRequestFlags( + val manual: Boolean = false, + val expedited: Boolean = false, + val upload: Boolean = false, + val overrideTooManyDeletions: Boolean = false, + val discardLocalDeletions: Boolean = false, +) { + fun toExtras(): Bundle = Bundle().apply { + if (manual) putBoolean(ContentResolver.SYNC_EXTRAS_MANUAL, true) + if (expedited) putBoolean(ContentResolver.SYNC_EXTRAS_EXPEDITED, true) + if (upload) putBoolean(ContentResolver.SYNC_EXTRAS_UPLOAD, true) + if (overrideTooManyDeletions) putBoolean(ContentResolver.SYNC_EXTRAS_OVERRIDE_TOO_MANY_DELETIONS, true) + if (discardLocalDeletions) putBoolean(ContentResolver.SYNC_EXTRAS_DISCARD_LOCAL_DELETIONS, true) + } +} diff --git a/android/app/src/main/java/com/anonymous/bulwarkmobile/sync/DeviceSyncAdapter.kt b/android/app/src/main/java/com/anonymous/bulwarkmobile/sync/DeviceSyncAdapter.kt new file mode 100644 index 00000000..6bf83444 --- /dev/null +++ b/android/app/src/main/java/com/anonymous/bulwarkmobile/sync/DeviceSyncAdapter.kt @@ -0,0 +1,188 @@ +package com.anonymous.bulwarkmobile.sync + +import android.accounts.Account +import android.accounts.AccountManager +import android.app.Application +import android.app.Service +import android.content.AbstractThreadedSyncAdapter +import android.content.ContentProviderClient +import android.content.ContentResolver +import android.content.Context +import android.content.Intent +import android.content.SyncResult +import android.os.Bundle +import android.os.Handler +import android.os.IBinder +import android.os.Looper +import android.os.SystemClock +import android.util.Log +import androidx.annotation.MainThread +import com.anonymous.bulwarkmobile.HeadlessJs +import com.facebook.react.bridge.Arguments +import com.facebook.react.bridge.WritableMap +import com.facebook.react.jstasks.HeadlessJsTaskConfig +import com.facebook.react.jstasks.HeadlessJsTaskContext +import com.facebook.react.jstasks.HeadlessJsTaskEventListener +import java.util.concurrent.TimeUnit + +/** + * The sync adapter behind both authorities (#34). It does no syncing itself: + * it starts the BulwarkDeviceSync headless JS task, where the engine lives + * (src/device-sync), waits for its report and hands that to SyncManager. See + * docs/device-sync.md. + * + * onPerformSync runs on its own SyncAdapterThread and may block; React Native + * is started from the main thread (see HeadlessJs), which stays free. + * SyncManager holds a wake lock for as long as onPerformSync runs. + */ +class DeviceSyncAdapter(context: Context) : + AbstractThreadedSyncAdapter(context, /* autoInitialize = */ true, /* allowParallelSyncs = */ false) { + + private val main = Handler(Looper.getMainLooper()) + + override fun onPerformSync( + account: Account, + extras: Bundle, + authority: String, + provider: ContentProviderClient, + syncResult: SyncResult, + ) { + val registryId = AccountManager.get(context).getUserData(account, DeviceSyncAccounts.USER_DATA_REGISTRY_ID) + if (registryId.isNullOrEmpty()) { + Log.w(TAG, "Sync of $authority skipped: the account has no app account") + syncResult.databaseError = true + return + } + val startedAt = SystemClock.elapsedRealtime() + // A sync scheduled as an expedited job (Android 12+) gets 3 minutes + // from JobScheduler instead of 10; stay inside either. + val expedited = extras.getBoolean(SYNC_EXTRAS_SCHEDULE_AS_EXPEDITED_JOB, false) + val deadline = startedAt + if (expedited) EXPEDITED_BUDGET_MS else BUDGET_MS + val run = DeviceSyncRuns.open(account.name, authority) + main.post { startRun(run, account, registryId, authority, extras, deadline) } + try { + while (!run.done.await(POLL_MS, TimeUnit.MILLISECONDS)) { + val boot = run.boot + if (boot != null && boot.isFaulted()) { + run.end("React Native did not start: ${boot.getError()?.message}") + } else if (SystemClock.elapsedRealtime() >= deadline) { + run.end("no report before the deadline") + } + } + val outcome = SyncReports.outcome(run.report.get(), run.endReason.get()) + SyncReports.apply(outcome, syncResult) + Log.i(TAG, "Sync ${run.runId} of $authority ended (${run.endReason.get()}) after " + + "${SystemClock.elapsedRealtime() - startedAt} ms: $outcome") + } catch (e: InterruptedException) { + // onSyncCanceled interrupts this thread. JS sees the run as + // cancelled at its next checkpoint (isRunCancelled). + run.end("cancelled") + Log.i(TAG, "Sync ${run.runId} of $authority cancelled") + Thread.currentThread().interrupt() + } finally { + DeviceSyncRuns.close(run) + // After startRun, which was posted first, so a listener it added is removed. + main.post { run.detach() } + } + } + + @MainThread + private fun startRun( + run: DeviceSyncRuns.Run, + account: Account, + registryId: String, + authority: String, + extras: Bundle, + deadline: Long, + ) { + if (run.cancelled) return + try { + run.boot = HeadlessJs.withReadyReactContext(context.applicationContext as Application) { reactContext -> + if (run.cancelled) return@withReadyReactContext + if (!reactContext.hasActiveReactInstance()) { + run.end("the React Native instance is gone") + return@withReadyReactContext + } + val tasks = HeadlessJsTaskContext.getInstance(reactContext) + run.attach(tasks, object : HeadlessJsTaskEventListener { + override fun onHeadlessJsTaskStart(taskId: Int) = Unit + + // JS settled without a report, timed out, or the context was reloaded. + override fun onHeadlessJsTaskFinish(taskId: Int) { + if (taskId == run.taskId) run.end("task finished") + } + }) + val taskTimeout = (deadline - SystemClock.elapsedRealtime() - LATCH_SLACK_MS).coerceAtLeast(MIN_TASK_MS) + val data = Arguments.createMap().apply { + putString("runId", run.runId) + putString("accountName", account.name) + putString("registryId", registryId) + putString("authority", authority) + putMap("extras", runExtras(extras)) + putDouble("deadline", (System.currentTimeMillis() + taskTimeout - JS_SLACK_MS).toDouble()) + } + HeadlessJs.startTask( + tasks, + HeadlessJsTaskConfig(TASK_KEY, data, taskTimeout, /* isAllowedInForeground = */ true), + onStarted = { run.taskId = it }, + onRefused = { run.end("refused: ${it.message}") }, + ) + } + } catch (e: RuntimeException) { + run.end("could not start: ${e.message}") + } + } + + private fun runExtras(extras: Bundle): WritableMap = Arguments.createMap().apply { + fun flag(key: String, name: String) { + if (extras.getBoolean(key, false)) putBoolean(name, true) + } + flag(ContentResolver.SYNC_EXTRAS_MANUAL, "manual") + flag(ContentResolver.SYNC_EXTRAS_UPLOAD, "upload") + flag(ContentResolver.SYNC_EXTRAS_EXPEDITED, "expedited") + flag(ContentResolver.SYNC_EXTRAS_IGNORE_BACKOFF, "ignoreBackoff") + flag(ContentResolver.SYNC_EXTRAS_OVERRIDE_TOO_MANY_DELETIONS, "overrideTooManyDeletions") + flag(ContentResolver.SYNC_EXTRAS_DISCARD_LOCAL_DELETIONS, "discardLocalDeletions") + } + + companion object { + private const val TAG = "BulwarkDeviceSync" + const val TASK_KEY = "BulwarkDeviceSync" + + /** ContentResolver.SYNC_EXTRAS_SCHEDULE_AS_EXPEDITED_JOB (API 31). */ + private const val SYNC_EXTRAS_SCHEDULE_AS_EXPEDITED_JOB = "schedule_as_expedited_job" + + private const val BUDGET_MS = 570_000L // 9.5 min of JobScheduler's 10 + private const val EXPEDITED_BUDGET_MS = 150_000L // 2.5 min of 3 + private const val LATCH_SLACK_MS = 20_000L // the task times out this long before our deadline + private const val JS_SLACK_MS = 30_000L // JS aims to be done this long before the task timeout + private const val MIN_TASK_MS = 30_000L + private const val POLL_MS = 500L + } +} + +/** Binds the contacts authority's adapter (xml/sync_contacts.xml). */ +class ContactsSyncService : Service() { + override fun onBind(intent: Intent): IBinder = adapter(this).syncAdapterBinder + + companion object { + @Volatile private var adapter: DeviceSyncAdapter? = null + + private fun adapter(context: Context): DeviceSyncAdapter = adapter ?: synchronized(this) { + adapter ?: DeviceSyncAdapter(context.applicationContext).also { adapter = it } + } + } +} + +/** Binds the calendar authority's adapter (xml/sync_calendar.xml). */ +class CalendarSyncService : Service() { + override fun onBind(intent: Intent): IBinder = adapter(this).syncAdapterBinder + + companion object { + @Volatile private var adapter: DeviceSyncAdapter? = null + + private fun adapter(context: Context): DeviceSyncAdapter = adapter ?: synchronized(this) { + adapter ?: DeviceSyncAdapter(context.applicationContext).also { adapter = it } + } + } +} diff --git a/android/app/src/main/java/com/anonymous/bulwarkmobile/sync/DeviceSyncRuns.kt b/android/app/src/main/java/com/anonymous/bulwarkmobile/sync/DeviceSyncRuns.kt new file mode 100644 index 00000000..cbe67561 --- /dev/null +++ b/android/app/src/main/java/com/anonymous/bulwarkmobile/sync/DeviceSyncRuns.kt @@ -0,0 +1,82 @@ +package com.anonymous.bulwarkmobile.sync + +import com.facebook.react.interfaces.TaskInterface +import com.facebook.react.jstasks.HeadlessJsTaskContext +import com.facebook.react.jstasks.HeadlessJsTaskEventListener +import java.util.UUID +import java.util.concurrent.ConcurrentHashMap +import java.util.concurrent.CountDownLatch +import java.util.concurrent.atomic.AtomicReference + +/** + * The device sync runs an adapter is waiting for, keyed by a random run id. + * + * A run ends on the first of: JS calling finishRun, the headless task ending + * (JS settled, timed out or was reloaded away), React refusing or failing to + * start, the adapter's deadline, or a cancel. Headless task ids restart at 1 + * with every React context, so they never identify a run; the run id does, + * and a report for a run nobody waits for any more is ignored. + */ +object DeviceSyncRuns { + const val NO_TASK = -1 + + class Run(val runId: String, val accountName: String, val authority: String) { + val done = CountDownLatch(1) + val report = AtomicReference(null) + val endReason = AtomicReference(null) + @Volatile var cancelled = false + @Volatile var boot: TaskInterface? = null + /** Written and compared on the UI thread only. */ + @Volatile var taskId = NO_TASK + @Volatile private var tasks: HeadlessJsTaskContext? = null + @Volatile private var listener: HeadlessJsTaskEventListener? = null + + /** Releases the waiting adapter. Idempotent: the first reason wins. */ + fun end(reason: String) { + endReason.compareAndSet(null, reason) + done.countDown() + } + + fun attach(tasks: HeadlessJsTaskContext, listener: HeadlessJsTaskEventListener) { + this.tasks = tasks + this.listener = listener + tasks.addTaskEventListener(listener) + } + + fun detach() { + val tasks = tasks ?: return + listener?.let { tasks.removeTaskEventListener(it) } + this.tasks = null + this.listener = null + } + } + + private val runs = ConcurrentHashMap() + + fun open(accountName: String, authority: String): Run = + Run(UUID.randomUUID().toString(), accountName, authority).also { runs[it.runId] = it } + + /** Forgets the run: a late finishRun is then ignored and isCancelled answers true. */ + fun close(run: Run) { + run.cancelled = true + runs.remove(run.runId) + } + + /** From JS. Returns false when nothing waits for this run (stale, duplicate or finished). */ + fun finish(runId: String, reportJson: String): Boolean { + val run = runs[runId] ?: return false + if (!run.report.compareAndSet(null, reportJson)) return false + run.end("reported") + return true + } + + /** Unknown runs count as cancelled: the adapter has stopped waiting for them. */ + fun isCancelled(runId: String): Boolean = runs[runId]?.cancelled ?: true + + fun cancel(runId: String) { + runs[runId]?.cancelled = true + } + + /** Runs currently waited for, newest last; for status and the push router's "sync again" flag. */ + fun active(): List = runs.values.toList() +} diff --git a/android/app/src/main/java/com/anonymous/bulwarkmobile/sync/SyncReports.kt b/android/app/src/main/java/com/anonymous/bulwarkmobile/sync/SyncReports.kt new file mode 100644 index 00000000..280523ef --- /dev/null +++ b/android/app/src/main/java/com/anonymous/bulwarkmobile/sync/SyncReports.kt @@ -0,0 +1,90 @@ +package com.anonymous.bulwarkmobile.sync + +import android.content.SyncResult +import org.json.JSONException +import org.json.JSONObject + +/** + * What a run's report (src/device-sync/types.ts `RunReport`) asks of the + * framework's SyncResult. Kept free of Android classes so JVM tests cover it; + * [SyncReports.apply] copies it onto the real SyncResult. + * + * SyncManager treats io errors as soft (retry with back-off), auth errors, + * database errors and too many deletions as hard (no retry until something + * changes), and `fullSyncRequested` as "schedule another sync now". + */ +data class SyncOutcome( + val ioErrors: Long = 0, + val authErrors: Long = 0, + val databaseError: Boolean = false, + val tooManyDeletions: Boolean = false, + val inserts: Long = 0, + val updates: Long = 0, + val deletes: Long = 0, + val entries: Long = 0, + val skipped: Long = 0, + /** Epoch seconds; 0 when unset. */ + val delayUntil: Long = 0, + val fullSyncRequested: Boolean = false, +) + +object SyncReports { + /** + * Translates a report. [reportJson] is null when the run ended without + * one ([endReason] says why: JS never answered, React failed to start, …), + * which is a soft error so the framework retries. + */ + fun outcome(reportJson: String?, endReason: String?): SyncOutcome { + if (reportJson == null) return SyncOutcome(ioErrors = 1) + val report = try { + JSONObject(reportJson) + } catch (e: JSONException) { + return SyncOutcome(ioErrors = 1) + } + val stats = report.optJSONObject("stats") + val down = stats?.optJSONObject("downloaded") + val up = stats?.optJSONObject("uploaded") + fun count(side: JSONObject?, key: String) = side?.optLong(key, 0) ?: 0 + val base = SyncOutcome( + inserts = count(down, "created") + count(up, "created"), + updates = count(down, "updated") + count(up, "updated"), + deletes = count(down, "deleted") + count(up, "deleted"), + entries = stats?.optLong("entries", 0) ?: 0, + skipped = stats?.optLong("skipped", 0) ?: 0, + delayUntil = report.optLong("delayUntil", 0), + fullSyncRequested = report.optBoolean("moreRecordsToGet", false), + ) + // SyncManager retries a failed sync at once, without back-off, when it + // made progress (inserts, updates, or deletes without + // tooManyDeletions). A hard error must therefore report none, or an + // auth failure after some work would loop. + val hard = base.copy(inserts = 0, updates = 0, deletes = 0) + return when (report.optString("outcome")) { + "ok", "cancelled", "disabled" -> base + "auth" -> hard.copy(authErrors = 1) + "permission", "unsupported", "safetyAbort" -> hard.copy(databaseError = true) + "tooManyDeletions" -> hard.copy( + tooManyDeletions = true, + // Shown in the system's "too many deletions" notification. + deletes = report.optJSONObject("tooManyDeletions")?.optLong("count", 0) ?: 0, + ) + // "io", "internal" and anything a newer JS might send: retry, at + // once if some work got done, else with back-off. + else -> base.copy(ioErrors = 1) + } + } + + fun apply(outcome: SyncOutcome, result: SyncResult) { + result.stats.numIoExceptions += outcome.ioErrors + result.stats.numAuthExceptions += outcome.authErrors + result.stats.numInserts += outcome.inserts + result.stats.numUpdates += outcome.updates + result.stats.numDeletes += outcome.deletes + result.stats.numEntries += outcome.entries + result.stats.numSkippedEntries += outcome.skipped + if (outcome.databaseError) result.databaseError = true + if (outcome.tooManyDeletions) result.tooManyDeletions = true + if (outcome.delayUntil > 0) result.delayUntil = outcome.delayUntil + if (outcome.fullSyncRequested) result.fullSyncRequested = true + } +} diff --git a/android/app/src/main/res/values/strings.xml b/android/app/src/main/res/values/strings.xml index f7c8e036..3e363054 100644 --- a/android/app/src/main/res/values/strings.xml +++ b/android/app/src/main/res/values/strings.xml @@ -2,4 +2,6 @@ Bulwark Mobile contain false + + Bulwark \ No newline at end of file diff --git a/android/app/src/main/res/xml/authenticator.xml b/android/app/src/main/res/xml/authenticator.xml new file mode 100644 index 00000000..ee861b32 --- /dev/null +++ b/android/app/src/main/res/xml/authenticator.xml @@ -0,0 +1,7 @@ + + + diff --git a/android/app/src/main/res/xml/contacts.xml b/android/app/src/main/res/xml/contacts.xml new file mode 100644 index 00000000..d6247191 --- /dev/null +++ b/android/app/src/main/res/xml/contacts.xml @@ -0,0 +1,29 @@ + + + + + + + + + + + + + + + + + + diff --git a/android/app/src/main/res/xml/sync_calendar.xml b/android/app/src/main/res/xml/sync_calendar.xml new file mode 100644 index 00000000..9c5c8d15 --- /dev/null +++ b/android/app/src/main/res/xml/sync_calendar.xml @@ -0,0 +1,9 @@ + + + diff --git a/android/app/src/main/res/xml/sync_contacts.xml b/android/app/src/main/res/xml/sync_contacts.xml new file mode 100644 index 00000000..25e25fbc --- /dev/null +++ b/android/app/src/main/res/xml/sync_contacts.xml @@ -0,0 +1,9 @@ + + + diff --git a/android/app/src/test/java/com/anonymous/bulwarkmobile/sync/SyncReportsTest.kt b/android/app/src/test/java/com/anonymous/bulwarkmobile/sync/SyncReportsTest.kt new file mode 100644 index 00000000..273e53b8 --- /dev/null +++ b/android/app/src/test/java/com/anonymous/bulwarkmobile/sync/SyncReportsTest.kt @@ -0,0 +1,84 @@ +package com.anonymous.bulwarkmobile.sync + +import org.junit.Assert.assertEquals +import org.junit.Test + +class SyncReportsTest { + private fun report(outcome: String, extra: String = "") = """ + {"v":1,"runId":"r","authority":"com.android.contacts","outcome":"$outcome", + "startedAt":0,"durationMs":5,"conflicts":0,"itemErrors":[], + "stats":{"downloaded":{"created":3,"updated":2,"deleted":1}, + "uploaded":{"created":1,"updated":4,"deleted":2},"entries":13,"skipped":1}$extra} + """.trimIndent() + + @Test + fun `a run without a report is a soft error`() { + assertEquals(SyncOutcome(ioErrors = 1), SyncReports.outcome(null, "task finished")) + } + + @Test + fun `an unreadable report is a soft error`() { + assertEquals(SyncOutcome(ioErrors = 1), SyncReports.outcome("{not json", "reported")) + } + + @Test + fun `ok sums both directions into the framework's stats`() { + assertEquals( + SyncOutcome(inserts = 4, updates = 6, deletes = 3, entries = 13, skipped = 1), + SyncReports.outcome(report("ok"), "reported"), + ) + } + + @Test + fun `io keeps the progress so the framework retries at once`() { + val outcome = SyncReports.outcome(report("io"), "reported") + assertEquals(1L, outcome.ioErrors) + assertEquals(4L, outcome.inserts) + } + + @Test + fun `auth is hard and reports no progress, or the framework would retry it at once`() { + assertEquals( + SyncOutcome(authErrors = 1, entries = 13, skipped = 1), + SyncReports.outcome(report("auth"), "reported"), + ) + } + + @Test + fun `permission, unsupported and safety aborts are database errors without progress`() { + for (outcome in listOf("permission", "unsupported", "safetyAbort")) { + assertEquals( + outcome, + SyncOutcome(databaseError = true, entries = 13, skipped = 1), + SyncReports.outcome(report(outcome), "reported"), + ) + } + } + + @Test + fun `too many deletions reports the pending deletions for the system notification`() { + assertEquals( + SyncOutcome(tooManyDeletions = true, deletes = 42, entries = 13, skipped = 1), + SyncReports.outcome(report("tooManyDeletions", ""","tooManyDeletions":{"count":42,"threshold":20}"""), "reported"), + ) + } + + @Test + fun `delayUntil and moreRecordsToGet reach the framework`() { + val outcome = SyncReports.outcome(report("ok", ""","delayUntil":1790000000,"moreRecordsToGet":true"""), "reported") + assertEquals(1790000000L, outcome.delayUntil) + assertEquals(true, outcome.fullSyncRequested) + } + + @Test + fun `cancelled and disabled runs are not errors`() { + for (outcome in listOf("cancelled", "disabled")) { + assertEquals(0L, SyncReports.outcome(report(outcome), "reported").ioErrors) + } + } + + @Test + fun `an outcome this build does not know is retried`() { + assertEquals(1L, SyncReports.outcome(report("somethingNew"), "reported").ioErrors) + } +} diff --git a/index.ts b/index.ts index e1ae8026..5dfad1b2 100644 --- a/index.ts +++ b/index.ts @@ -15,6 +15,14 @@ AppRegistry.registerHeadlessTask('BulwarkPushTask', () => async (data: Parameter await Promise.allSettled([pushBackgroundTask(data), refreshWidgetsInBackground()]); }); +// Device sync (Android, #34): the contacts and calendar sync adapters run the +// sync engine through this task, headless or next to the UI. The engine is +// required on first use so an app start does not load it. +AppRegistry.registerHeadlessTask('BulwarkDeviceSync', () => async (data: import('./src/device-sync/types').RunPayload) => { + const { runDeviceSyncTask } = require('./src/device-sync/task') as typeof import('./src/device-sync/task'); + await runDeviceSyncTask(data); +}); + // Home-screen widgets (Android): every widget event runs this headless task. registerWidgetTaskHandler(widgetTaskHandler); diff --git a/src/device-sync/task.ts b/src/device-sync/task.ts new file mode 100644 index 00000000..95e473d7 --- /dev/null +++ b/src/device-sync/task.ts @@ -0,0 +1,41 @@ +/** + * Entry of the BulwarkDeviceSync headless task, which the contacts and + * calendar sync adapters start for every sync (index.ts registers it). See + * docs/device-sync.md. + * + * The run must always end with `finishRun`, called before the task's promise + * settles: JS timers stop once the task is over, and a sync adapter waiting + * for a report that never comes holds its sync until the deadline. + * + * The sync engine is not wired in yet, so a run only reports that device + * sync is off; nothing in the app can turn it on at this point. + */ +import { NativeModules } from 'react-native'; +import type { DeviceSyncNativeModule, RunPayload, RunReport } from './types'; + +export async function runDeviceSyncTask(payload: RunPayload): Promise { + const native = NativeModules.BulwarkDeviceSync as DeviceSyncNativeModule | undefined; + if (!native || !payload?.runId) return; + const report: RunReport = { + v: 1, + runId: payload.runId, + authority: payload.authority, + outcome: 'disabled', + message: 'Device sync is not available in this build', + startedAt: Date.now(), + durationMs: 0, + stats: { + downloaded: { created: 0, updated: 0, deleted: 0 }, + uploaded: { created: 0, updated: 0, deleted: 0 }, + entries: 0, + skipped: 0, + }, + conflicts: 0, + itemErrors: [], + }; + try { + await native.finishRun(payload.runId, JSON.stringify(report)); + } catch { + // The adapter ends the run on its own when the task finishes. + } +} diff --git a/src/device-sync/types.ts b/src/device-sync/types.ts new file mode 100644 index 00000000..8bf1b899 --- /dev/null +++ b/src/device-sync/types.ts @@ -0,0 +1,369 @@ +/** + * Contracts of Android device sync (#34): the native bridge, the provider row + * and operation model, the headless payload and the run report. The design is + * in docs/device-sync.md; this file is its machine-checked half. + * + * Pure types and constants. Nothing here, nor in the mappers and merge rules + * built on it, may import react-native: vitest runs them in node. + */ + +// ─── Authorities and the headless task ───────────────── + +export const CONTACTS_AUTHORITY = 'com.android.contacts'; +export const CALENDAR_AUTHORITY = 'com.android.calendar'; +export type Authority = typeof CONTACTS_AUTHORITY | typeof CALENDAR_AUTHORITY; +export const AUTHORITIES: readonly Authority[] = [CONTACTS_AUTHORITY, CALENDAR_AUTHORITY]; + +/** The headless JS task the sync adapters start (registered in index.ts). */ +export const DEVICE_SYNC_TASK = 'BulwarkDeviceSync'; + +/** Event the native module emits when accounts of our type change. */ +export const ACCOUNTS_CHANGED_EVENT = 'BulwarkDeviceSync:accountsChanged'; + +// ─── Provider rows ───────────────────────────────────── + +/** Tables the native side lets the engine touch, per authority. */ +export type ContactsTable = 'raw_contacts' | 'data' | 'groups' | 'settings'; +export type CalendarTable = + | 'calendars' + | 'events' + | 'attendees' + | 'reminders' + | 'extended_properties' + | 'colors'; +export type ProviderTable = ContactsTable | CalendarTable; + +export const CONTACTS_TABLES: readonly ContactsTable[] = ['raw_contacts', 'data', 'groups', 'settings']; +export const CALENDAR_TABLES: readonly CalendarTable[] = [ + 'calendars', + 'events', + 'attendees', + 'reminders', + 'extended_properties', + 'colors', +]; + +/** + * A cell as it crosses the bridge: text, a number (booleans are 0/1, times are + * epoch milliseconds, ids fit a double) or null. Blob columns are written as + * `{ b64 }` (the photo bytes) and never read back through `query`. + */ +export type Cell = string | number | null; +export type BlobCell = { b64: string }; +export type WriteCell = Cell | BlobCell; +export type Row = Record; +export type WriteRow = Record; + +export interface ProviderQuery { + table: ProviderTable; + columns: string[]; + /** + * SQL WHERE clause with `?` placeholders. The native side ANDs the account + * scope onto it, so it never needs (and must not rely on) account columns. + */ + where?: string; + args?: Array; + orderBy?: string; +} + +/** Query results, column-major names and row-major cells. */ +export interface ProviderRows { + columns: string[]; + rows: Cell[][]; +} + +/** + * One provider operation. `id` addresses one row of our account by `_id`; + * `where`/`args` address a set (always ANDed with the account scope). + * + * - `insert.refs` maps a column to the index of an earlier `insert` in the + * same batch whose new row id fills it (ContentProviderOperation + * back-references), so one batch can create a raw contact and its data rows, + * or an event and its attendees. + * - `assert` fails the whole batch unless the addressed rows hold `values` + * (compared as text, as ContentProviderOperation does) and, when given, + * their number is `expectCount`. + * - `syncState` replaces this account's SyncState blob. + * - `yieldAllowed` marks the first op of an item group: the provider may + * commit everything before it and let other writers in. Batches must mark + * one at least every 400 ops (ContactsProvider refuses 500 without one). + */ +export type ProviderOp = + | { op: 'insert'; table: ProviderTable; values: WriteRow; refs?: Record; yieldAllowed?: boolean } + | { + op: 'update'; + table: ProviderTable; + id?: number; + where?: string; + args?: Array; + values: WriteRow; + expectCount?: number; + yieldAllowed?: boolean; + } + | { + op: 'delete'; + table: ProviderTable; + id?: number; + where?: string; + args?: Array; + expectCount?: number; + yieldAllowed?: boolean; + } + | { + op: 'assert'; + table: ProviderTable; + id?: number; + where?: string; + args?: Array; + values?: Row; + expectCount?: number; + yieldAllowed?: boolean; + } + | { op: 'syncState'; value: string; yieldAllowed?: boolean }; + +export interface OpResult { + /** New row id for an insert. */ + id?: number; + /** Rows touched by an update or delete. */ + count?: number; +} + +/** + * Why a batch failed. Nothing of a failed batch is applied except item groups + * the provider committed at yield points it actually took, which it does only + * under contention; callers must therefore re-read before retrying. + * - `assert`: an assert op did not hold (a concurrent edit), or an + * `expectCount` did not match. + * - `tooLarge`: the batch exceeded the Binder transaction limit; split it. + * - `scope`: the native side refused an op outside our account (an engine bug). + * - `permission`: the runtime permission for the authority is gone. + * - `provider`: anything else the provider threw. + */ +export type BatchFailure = 'assert' | 'tooLarge' | 'scope' | 'permission' | 'provider'; + +export type BatchResult = + | { ok: true; results: OpResult[] } + | { ok: false; reason: BatchFailure; message: string }; + +export interface PhotoData { + /** JPEG, downscaled so its longer side is at most the requested size. */ + jpegBase64: string; + /** `Photo.PHOTO_FILE_ID` of the display photo it was read from, if any. */ + fileId: number | null; +} + +/** + * The provider, bound to one Android account and one authority. Implemented + * by src/device-sync/native.ts on the device and by the in-memory fake in + * tests. All calls are scoped: rows of other accounts are invisible. + */ +export interface ProviderPort { + readonly accountName: string; + readonly authority: Authority; + query(q: ProviderQuery): Promise; + applyBatch(ops: ProviderOp[]): Promise; + readSyncState(): Promise; + /** Contacts only: the raw contact's display photo (or thumbnail), or null. */ + readPhoto(rawContactId: number, maxPx: number): Promise; +} + +// ─── Accounts and sync settings ──────────────────────── + +export interface AndroidAccount { + /** Account name as Android shows it (the login, or the registry id on a clash). */ + name: string; + /** `AccountEntry.id` of the app account it belongs to, from the account's userData. */ + registryId: string | null; +} + +export interface AuthoritySyncSettings { + /** `ContentResolver.getIsSyncable`: 1 yes, 0 no, -1 unknown. */ + syncable: number; + /** `getSyncAutomatically`: the toggle in Settings → Accounts. */ + automatic: boolean; + /** Period of our periodic sync in seconds, or 0 when there is none. */ + periodicSeconds: number; + active: boolean; + pending: boolean; +} + +export interface AccountSyncSettings { + /** `getMasterSyncAutomatically`: "Auto-sync data" for the whole device. */ + masterAutomatic: boolean; + authorities: Record; +} + +export interface RequestSyncOptions { + manual?: boolean; + expedited?: boolean; + upload?: boolean; + overrideTooManyDeletions?: boolean; + discardLocalDeletions?: boolean; +} + +// ─── The headless run ────────────────────────────────── + +/** SYNC_EXTRAS_* flags of the request that started a run. */ +export interface RunExtras { + manual?: boolean; + upload?: boolean; + expedited?: boolean; + ignoreBackoff?: boolean; + overrideTooManyDeletions?: boolean; + discardLocalDeletions?: boolean; +} + +/** Data of the BulwarkDeviceSync headless task, as Kotlin sends it. */ +export interface RunPayload { + runId: string; + accountName: string; + registryId: string; + authority: Authority; + extras: RunExtras; + /** Epoch ms after which Kotlin stops waiting; the run must have finished. */ + deadline: number; +} + +export type RunOutcome = + /** Everything was synced; per-item errors may still be listed. */ + | 'ok' + /** Network or server trouble: the framework retries with back-off. */ + | 'io' + /** Credentials rejected even after rebuilding the client: sign in again. */ + | 'auth' + /** The runtime permission for the authority is missing. */ + | 'permission' + /** The server does not offer the capability for this account. */ + | 'unsupported' + /** Local deletions above the threshold; waiting for the user's decision. */ + | 'tooManyDeletions' + /** A full reconcile found an empty server but a full device, and stopped. */ + | 'safetyAbort' + /** Cancelled by the framework, or out of time with work left. */ + | 'cancelled' + /** Sync is off for this account/authority in the app or in Android. */ + | 'disabled' + /** An exception the engine did not expect. */ + | 'internal'; + +export interface SideCounts { + created: number; + updated: number; + deleted: number; +} + +export interface RunStats { + /** Server → device: objects applied to rows. */ + downloaded: SideCounts; + /** Device → server: objects created, patched, destroyed. */ + uploaded: SideCounts; + /** Objects looked at. */ + entries: number; + /** Objects left for later (poisoned, backed off, read-only reverts). */ + skipped: number; +} + +export interface ItemError { + /** `SOURCE_ID`/`_SYNC_ID` when known, else `row:<_id>`. */ + ref: string; + side: 'upload' | 'download'; + /** SetError type (`invalidProperties`, …) or an engine error code. */ + type: string; + description?: string; + /** Epoch ms before which the item is not retried unless it changes. */ + retryAt?: number; +} + +export interface RunReport { + v: 1; + runId: string; + authority: Authority; + outcome: RunOutcome; + message?: string; + startedAt: number; + durationMs: number; + stats: RunStats; + /** Units where both sides changed differently; the server won each. */ + conflicts: number; + itemErrors: ItemError[]; + /** Set with outcome `tooManyDeletions`. */ + tooManyDeletions?: { count: number; threshold: number }; + /** Epoch seconds: do not sync before (a 429's Retry-After). */ + delayUntil?: number; + /** The run stopped at its time budget with work left; run again soon. */ + moreRecordsToGet?: boolean; +} + +// ─── Native module (the raw bridge) ──────────────────── + +/** + * `NativeModules.BulwarkDeviceSync`. Large payloads cross as JSON strings, + * which the bridge moves far faster than nested maps. Every provider call is + * scoped to `accountName` of our account type by the native side. + */ +export interface DeviceSyncNativeModule { + getInfo(): Promise<{ accountType: string; sdkInt: number }>; + + listAccounts(): Promise; + /** Adds the account if missing and (re)writes its registryId. Resolves true when it was created. */ + ensureAccount(name: string, registryId: string): Promise; + /** Removes the account; the providers drop its rows. */ + removeAccount(name: string): Promise; + + getSyncSettings(name: string): Promise; + /** Syncable = 1 plus `setSyncAutomatically(enabled)` for the authority. */ + setSyncEnabled(name: string, authority: Authority, enabled: boolean): Promise; + /** Replaces our periodic sync; 0 removes it. */ + setPeriodicSync(name: string, authority: Authority, seconds: number): Promise; + requestSync(name: string, authority: Authority, optionsJson: string): Promise; + /** Opens the account's sync screen in Android Settings. */ + openAccountSettings(name: string): Promise; + + /** Hands the report to the waiting adapter. Resolves false for an unknown or finished run. */ + finishRun(runId: string, reportJson: string): Promise; + isRunCancelled(runId: string): Promise; + + /** `ProviderQuery` JSON in, `ProviderRows` JSON out. */ + query(name: string, authority: Authority, queryJson: string): Promise; + /** `ProviderOp[]` JSON in, `BatchResult` JSON out (failures resolve, they do not reject). */ + applyBatch(name: string, authority: Authority, opsJson: string): Promise; + readSyncState(name: string, authority: Authority): Promise; + /** `PhotoData` JSON, or null when the raw contact has no photo. */ + readPhoto(name: string, rawContactId: number, maxPx: number): Promise; + + // NativeEventEmitter bookkeeping. + addListener(eventName: string): void; + removeListeners(count: number): void; +} + +// ─── App-side preferences the engine reads ───────────── + +/** Collections are keyed `/` everywhere. */ +export type CollectionKey = string; + +export type ReminderOwner = 'device' | 'bulwark'; + +/** + * What the engine needs from the app's device-sync store for one registry + * account. Selections list explicit choices only; a collection the user never + * touched follows the default rule: collections of the personal account on, + * shared ones off. + */ +export interface AccountSyncPrefs { + contactsSelection: Record; + calendarSelection: Record; + /** Where contacts created on the device go; unset means the server's default book. */ + newContactsAddressBook?: CollectionKey; + reminderOwner: ReminderOwner; +} + +/** Last run per authority, as the settings UI shows it. */ +export interface RunStatus { + at: number; + outcome: RunOutcome; + message?: string; + durationMs: number; + conflicts: number; + itemErrors: number; + stats: RunStats; +} From f66669964ac373589be8c6a9084a04517a9bab2a Mon Sep 17 00:00:00 2001 From: Linus Rath <139418639+rathlinus@users.noreply.github.com> Date: Sun, 27 Sep 2026 02:49:46 +0200 Subject: [PATCH 03/87] docs: describe how contacts and calendars will sync with the device #34 Issue #34 asks for contacts and calendars that show up in Android's own apps and whose edits there reach the server. docs/device-sync.md is the design the implementation follows: Android's sync framework drives the runs, a thin Kotlin shell starts a headless JS task, and the engine is TypeScript on a detached JMAP client, so token refresh, mTLS and session discovery keep a single owner. A pure-Kotlin engine was rejected: it would have had to redeem rotating refresh tokens outside that owner and re-implement the client. It pins down what every provider column holds, the JSContact and JSCalendar mapping tables, how device edits are told apart from provider normalisation (a baseline per row), how rows are matched when an editor re-inserts them, the merge and conflict rules, uploads as patches with `ifInState`, the native API, the failure matrix and the limitations. The platform and server facts it relies on come from AOSP's providers and calendar/contacts apps, Fossify, Etar and Stalwart v0.16.23's source. These were checked on the local test server: - map keys written through JMAP survive patches; `uid` duplicates are refused; unknown and plural recurrence properties are rejected - a start or `until` in a DST gap or overlap is silently dropped (a daily rule came back unbounded); `Etc/GMT-2` keeps both - `ifInState` yields `stateMismatch`; `relatedTo/: null` does not delete; result references to `/set` are refused An adversarial review of the first draft found 20 paths that could lose data, duplicate items or loop (own creates duplicated after a lost response, unsynced edits dropped on deselection or eviction, Fossify rewrites uploaded as edits, Etar moves turned into destroy + create, and more); the design was changed for each before any engine code. --- docs/device-sync.md | 1287 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 1287 insertions(+) create mode 100644 docs/device-sync.md diff --git a/docs/device-sync.md b/docs/device-sync.md new file mode 100644 index 00000000..75b790d7 --- /dev/null +++ b/docs/device-sync.md @@ -0,0 +1,1287 @@ +# Device sync: contacts and calendars in Android's providers + +Issue [#34](https://github.com/bulwarkmail/native/issues/34). Each signed-in +account can opt in to device sync. Its address books then appear in Android's +Contacts provider and its calendars in the Calendar provider, under an Android +account of type "Bulwark". Google Contacts and Calendar, AOSP/Fossify Contacts, +Etar, the dialer and Settings → Accounts all see them. Server changes reach the +device, and creates, edits and deletes made in any Android app reach the +server. + +This document is for maintainers. It describes the design: the architecture, +what every provider column holds, the mapping tables, the merge rules, the +native API, the failure matrix and the known limitations. Behaviour it relies +on was checked against Stalwart v0.16.23 (the server Bulwark targets) and +AOSP's ContactsProvider, CalendarProvider and the calendar/contacts apps; the +notable findings are quoted where they shape a rule. + +## Contents + +1. [Architecture](#architecture) +2. [The Android shell](#the-android-shell) +3. [Device data model](#device-data-model) +4. [A sync run](#a-sync-run) +5. [Change detection and merge rules](#change-detection-and-merge-rules) +6. [Uploads](#uploads) +7. [Contacts mapping](#contacts-mapping) +8. [Calendar mapping](#calendar-mapping) +9. [Headless payload, report and SyncResult](#headless-payload-report-and-syncresult) +10. [Native module API](#native-module-api) +11. [App integration](#app-integration) +12. [Failure matrix](#failure-matrix) +13. [Limits and performance](#limits-and-performance) +14. [Testing](#testing) +15. [Decisions and limitations](#decisions-and-limitations) +16. [Out of scope](#out-of-scope) + +## Architecture + +The Android sync framework drives everything. Kotlin is a thin shell; the sync +engine is TypeScript that runs as a headless JS task and reaches the providers +through a native module. + +``` +SyncManager ──onPerformSync──▶ DeviceSyncAdapter (Kotlin, SyncAdapterThread) + │ posts to the main thread: start React Native if needed, + │ start headless task BulwarkDeviceSync {runId, …} + │ waits on a latch (≤ 9.5 min, 2.5 min for expedited jobs) + ▼ + runDeviceSync (TypeScript, JS thread) + │ JMAP: a detached JMAPClient per registry account + │ provider: NativeModules.BulwarkDeviceSync (query / applyBatch) + ▼ + finishRun(runId, report) ──▶ latch released ──▶ SyncResult +``` + +Why this split: + +- Auth, token refresh, mTLS, 429 back-off and session discovery stay in the + existing JS client (`src/api/jmap-client.ts`). Refresh tokens rotate; a + second owner outside JS's de-duplication could evict the account. +- Mappers and merge rules are pure TypeScript with no React Native import, so + vitest covers them with golden fixtures and property tests. A test walks the + import graph of `src/device-sync/{contacts,calendar,engine}` and fails on + `react-native`, expo, stores or `jmap-client`: the global mocks in + `src/test-setup.ts` would otherwise hide such an import. +- Kotlin only translates: URIs, `ContentProviderOperation`s, `SyncResult`. Its + logic has JVM unit tests (`android/app/src/test`). + +A pure-Kotlin engine was rejected: it would have to redeem rotating refresh +tokens outside JS's de-duplication, re-implement session discovery with its +origin rewrite, mTLS, 429 handling and the first-touch gate, read +expo-secure-store's internal format, and start from zero test infrastructure. + +There is one ReactHost and one Hermes runtime per process. A sync that runs +while the UI is open runs in the UI's runtime (the task is started with +`allowedInForeground = true`), and a sync that cold-started the process leaves +its runtime for the UI to attach to. So the engine never touches the UI's +`jmapClient` singleton or zustand stores it does not own, and a module-level +mutex coordinates runs with app-side operations. + +### Code map + +| Path | What it holds | +|---|---| +| `android/app/src/main/java/com/anonymous/bulwarkmobile/HeadlessJs.kt` | starting React Native and a headless task; shared with the push job | +| `android/app/src/main/java/com/anonymous/bulwarkmobile/sync/` | authenticator, sync adapters, run registry, report mapping, native module, provider op translation, push router | +| `android/app/src/main/res/xml/{authenticator,sync_contacts,sync_calendar,contacts}.xml` | framework metadata | +| `android/app/src/debug/java/…/sync/DeviceSyncDebugReceiver.kt` | adb hooks for device tests (debug builds only) | +| `src/device-sync/types.ts`, `android-columns.ts` | contracts: bridge API, row model, ops, payload, report; provider column names | +| `src/device-sync/native.ts` | typed wrapper around `NativeModules.BulwarkDeviceSync` | +| `src/device-sync/common/` | pure helpers: canonical JSON, hashing, JSON Pointer patches, zoned time, durations | +| `src/device-sync/contacts/` | JSContact ↔ contacts rows, row matching, contact merge rules (pure) | +| `src/device-sync/calendar/` | JSCalendar ↔ calendar rows, RRULE, exceptions, merge rules (pure) | +| `src/device-sync/jmap/` | the JMAP side on a detached client: `/changes`, full reconcile, `/set`, uid lookup | +| `src/device-sync/engine/` | orchestration: run entry, mutex, checkpoints, batches, SyncState, report | +| `src/device-sync/task.ts` | the headless task entry (registered in `index.ts`) | +| `src/stores/device-sync-store.ts` | per-account preferences and the last run status | +| `src/components/settings/ContactsSettings.tsx`, `CalendarSettings.tsx` | the "Sync to this device" sections | + +## The Android shell + +### Account type and accounts + +The account type is `${applicationId}.account`, defined once in +`android/app/build.gradle` with `resValue` and referenced as +`@string/device_sync_account_type` from the XML and through +`R.string.device_sync_account_type` from Kotlin. The package name may change +before a store listing; nothing hard-codes it. The account label is "Bulwark" +and its icon is the app icon. + +One Android account exists per registry account (`AccountEntry.id`, +`username@hostname`) that has device sync on for at least one authority. Its +name is the account's login; when two registry accounts share a login on +different servers, the second one is named after its registry id. The registry +id is stored in the account's `userData` under `registryId` and is the only +link back to the app: the name is for people. `GET_ACCOUNTS` is not needed: +an authenticator always sees its own accounts, and `addAccountExplicitly` +needs only a UID match on API 23+. + +Other apps see the account (API 26+) when they hold `READ_CONTACTS` and we hold +a granted `WRITE_CONTACTS`; Contacts apps that list accounts rely on that. + +### Authenticator + +`BulwarkAuthenticator` and `BulwarkAuthenticatorService`: + +- `addAccount` (Settings → Accounts → Add account) never creates an account + without credentials. It returns an explicit Intent to MainActivity with + `bulwarkmobile://settings/contacts`, where the user turns sync on. (The + AccountManager only accepts an Intent that resolves inside our package.) +- `getAccountRemovalAllowed` allows removal. The providers then delete every + row of the account and its SyncState; the app notices and turns device sync + off for that account. It never recreates the account on its own. +- Everything else (`getAuthToken`, `confirmCredentials`, …) returns + `ERROR_CODE_UNSUPPORTED_OPERATION`. + +### Sync adapters + +Two `AbstractThreadedSyncAdapter` services (`ContactsSyncService`, +`CalendarSyncService`), declared with `supportsUploading="true"`, +`userVisible="true"`, `allowParallelSyncs="false"`, and in the main process so +they share the ReactHost with the UI. The contacts service also declares +`android.provider.CONTACTS_STRUCTURE` → `xml/contacts.xml`, whose `EditSchema` +lists exactly the data kinds in the [contacts mapping](#contacts-mapping). +AOSP Contacts rejects the whole account type when that XML breaks its rules +(every typed kind needs `` children, `name` needs all six `supports*` +attributes, `photo` must be present, single-value kinds need `maxOccurs="1"`), +and without an EditSchema it treats the account as read-only. + +Exported services are safe: `startSync`/`cancelSync` ignore callers other than +the system UID. + +`onPerformSync` (see `DeviceSyncAdapter.kt`): + +1. Reads `registryId` from the account's userData; a missing one is a hard + error (`databaseError`). +2. Opens a run in `DeviceSyncRuns`: a random `runId`, a latch, a cancel flag. + Headless task ids restart at 1 with every React context, so they never + identify a run. +3. Posts to the main thread: `HeadlessJs.withReadyReactContext` starts React + Native if needed (on the main thread only: the ReactHost is created lazily + without a lock), then starts the headless task `BulwarkDeviceSync` with the + [payload](#headless-payload-report-and-syncresult) and + `allowedInForeground = true`. The task timeout is the adapter's deadline + minus 20 s; the payload's `deadline` is 30 s earlier still. +4. Waits until the first of: `finishRun`, the headless task ending (JS settled + without a report, timed out, or the context was reloaded), React refusing or + failing to start (the boot task faults, e.g. a debug build without Metro), + or the deadline: 9.5 minutes, or 2.5 minutes when the sync runs as an + expedited job (`schedule_as_expedited_job`), inside JobScheduler's 10 and 3 + minute guarantees. +5. Maps the report onto `SyncResult`; no report is a soft error. + +`onSyncCanceled` interrupts the sync thread; the run is closed, and JS sees +`isRunCancelled(runId) == true` (also for any run nobody waits for) at its next +checkpoint. A run ends once: `finishRun` is accepted only while the run is +still waiting, so a report that arrives after the deadline, a boot fault or the +task's end is refused rather than half-applied. SyncManager holds a wake lock for as +long as `onPerformSync` runs, so the adapter takes none. + +SyncManager also cancels a sync that moves fewer than 10 bytes of network +traffic in any 60-second window. The engine sends a `Core/echo` when a local +phase has gone 40 s without a JMAP request. + +### Spike results + +Checked before building anything else on it, on the API 34 and API 32 +emulators with the debug build (commit `ed2525e`): + +| App state | Report received after | +|---|---| +| foreground (task runs next to the UI, `AppState` active) | 319-334 ms | +| background (process alive) | 320-327 ms | +| process killed (SyncManager starts it for the sync service; React starts headless, bundle from Metro) | 16.2 s (API 34), 8.5 s (API 32) | + +An io report was retried with back-off, an auth report was not retried, too +many deletions showed the system's "Attempted to delete too many Contacts." +notification, and with Metro unreachable the adapter gave up after 2 s. +Found on the way: SyncManager retries a failed sync at once when it made +progress (`madeSomeProgress`), so hard errors are reported without progress; +and without the runtime permission `AbstractThreadedSyncAdapter` fails the sync +itself (`databaseError`) before `onPerformSync` runs, so the app must detect a +revoked permission on its own. + +## Device data model + +### Identity + +| Row | Column | Value | +|---|---|---| +| raw contact | `RawContacts.SOURCE_ID` | `/`; null until created on the server | +| group | `Groups.SOURCE_ID` | `/` of a `kind: "group"` card | +| calendar | `Calendars._SYNC_ID` | `/` | +| event (master or single) | `Events._SYNC_ID` | `/`, or `~pending/` for a local event claimed for upload | +| event exception | `Events._SYNC_ID` + `ORIGINAL_SYNC_ID` | `/#` + the master's `_SYNC_ID` | + +JMAP ids use only `[A-Za-z0-9_-]` (RFC 8620 §1.2), so `/`, `#` and `~` never +occur in them: ids stay unique across the JMAP accounts, shared ones included, +that feed one Android account, and `~pending/…` can never be a server id. + +Identities are only unique for one server account's lifetime (Stalwart's ids +come from small per-account counters). So the SyncState records which registry +account and server origin the rows belong to, a run refuses to touch rows +written for another one, and a merge treats a server object whose `uid` +differs from its shadow's as a different object (delete and insert, never a +patch). + +Why a pending `_SYNC_ID`: CalendarProvider hard-deletes an app-deleted event +that has no `_SYNC_ID`, so a local event deleted while its create was in flight +would vanish without a trace and come back from the server. With a `_SYNC_ID` +the delete is a soft delete (`DELETED=1`) the engine can act on. It also lets +Etar create exceptions of a not-yet-uploaded series (it offers "this event" only +when the master has a `_SYNC_ID`); the provider rewrites their +`ORIGINAL_SYNC_ID` when the master's `_SYNC_ID` changes. Contacts need no such +marker: raw contacts of a sync account are always soft-deleted. + +A **new** local item is a raw contact without `SOURCE_ID`, or a master event +(`ORIGINAL_ID` and `ORIGINAL_SYNC_ID` both null) whose `_SYNC_ID` is null or +pending. DIRTY is not the test: a raw contact inserted without data rows is not +dirty. A row with `ORIGINAL_*` set is always an exception of its master and +never uploads as an event of its own. + +### Contacts columns + +| Column | Holds | +|---|---| +| `RawContacts.SOURCE_ID` | identity (above) | +| `RawContacts.SYNC1` | the card's collections: `/` joined by `,` | +| `RawContacts.SYNC2` | the shadow: the last server card, wire format, with each `data:` photo URI replaced by `sha256:` of its bytes | +| `RawContacts.SYNC3` | the pending create: `{"uid":…,"target":"/"}`, written before the `/set` and kept until the identity is | +| `RawContacts.SYNC4` | poison marker `{"fp":…,"type":…,"n":…,"until":…}` | +| `RawContacts.RAW_CONTACT_IS_READ_ONLY` | 1 when none of the card's address books grants `mayWrite` | +| `Data.DATA_SYNC1` | the entry key(s), e.g. `emails:work1`; see [units](#units) | +| `Data.DATA_SYNC2` | photo rows: the hash of the server photo applied to the row | +| `Data.DATA_SYNC3` | the baseline: the row's mapped columns as the provider stored them after our last write (JSON; text over 1 KB as its hash) | +| `Groups.SOURCE_ID`, `SYNC2`, `SYNC3`, `SYNC4` | as for raw contacts, for group cards (`SHOULD_SYNC` is never written: an update carrying it makes ContactsProvider request a sync) | +| `Settings` | one row per account: `UNGROUPED_VISIBLE=1`, `SHOULD_SYNC=1` (the default 0 hides every contact without a visible group) | + +Writing `DIRTY=0`, `SOURCE_ID` or `SYNC1-4` does not bump +`RawContacts.VERSION`; data-row inserts, updates and deletes do, whoever makes +them. So VERSION is used only as an optimistic-lock token in asserts, and DIRTY +for change detection. (A super-primary change in another account can bump our +VERSION without DIRTY; the assert then fails once and the row is re-read.) + +### Calendar columns + +| Column | Holds | +|---|---| +| `Calendars._SYNC_ID` | identity | +| `Calendars.CAL_SYNC1` | unused: CalendarProvider sends it as the `feed` extra of the sync it requests whenever a calendar with `SYNC_EVENTS=1` is inserted or changed, so anything there splits those requests apart | +| `Calendars.CAL_SYNC2` | the shadow: the last server `Calendar` object | +| `Calendars.CAL_SYNC3` | flags JSON: `readOnly` reason (`rights`, `subscription`), `taskOnly` | +| `Events._SYNC_ID` | identity | +| `Events.SYNC_DATA1` | masters and single events: the shadow, the full server event with its `recurrenceOverrides` | +| `Events.SYNC_DATA2` | exceptions: the recurrence id, exactly as the server's override key | +| `Events.SYNC_DATA3` | the pending create: `{"uid":…,"target":"/"}` | +| `Events.SYNC_DATA4` | the baseline: mapped event columns, attendees and reminders as stored after our last write (JSON; text over 1 KB as its hash) | +| `Events.SYNC_DATA5` | poison marker | +| `Events.UID_2445` | the event's `uid` | + +Attendees and reminders have no sync columns, which is why their baseline sits +on the event row. `Events.MUTATORS` (the packages that changed the event since +our last write) tells which app made a device edit. + +A description longer than 64 KB is written truncated, marked in the baseline, +and never uploaded: an event row carries the description, the shadow and the +baseline, and one Binder transaction holds 1 MB. + +### SyncState + +`ContactsContract.SyncState` and `CalendarContract.SyncState` hold one blob per +Android account (CalendarProvider requires sync-adapter URIs for it): UTF-8 +JSON, versioned. + +```json +{ + "v": 1, + "owner": { "registryId": "usera@example.org@mail.example.org", "origin": "https://mail.example.org" }, + "accounts": { + "c": { + "collectionsState": "s12eq", + "itemsState": "syufa", + "selected": ["c/b"], + "stale": [], + "reconcile": null, + "groups": ["c/e12"], + "taskOnly": [] + } + }, + "deviceZone": "Europe/Berlin", + "deviceZonePending": null +} +``` + +- `owner`: the registry account and the server origin the rows were written + for. A run for another owner stops with `internal` and the app offers to + start over (after uploading what it can). +- `itemsState` is the `ContactCard` or `CalendarEvent` state the rows describe. + It is written in the same `applyBatch` as the last item group of a `/changes` + page: the `syncState` op is the last op of that batch, is never + `yieldAllowed`, and the native side refuses a batch that breaks either rule, + so a provider yield can never commit the state ahead of its rows. +- No stored `itemsState` is newer than the shadow of an object changed before + it: an object whose rows could not be written (an item error) goes on + `stale` in the same batch, is fetched again at the start of every run, and + uploads nothing until it has been. +- `reconcile` marks a full reconcile in progress: `{ "from": , "phase": "ids" | "objects", "position": n }`, written with every + chunk. The next run continues it; its deletions are applied only when the + complete id list is known, and `itemsState` becomes `from` when it ends. +- `selected` is the selection the rows were written for; a collection is added + in the batch with its last loaded chunk, and removed with its last deleted + row. +- `groups` (contacts): the group cards present on the device, so a group an app + hard-deleted (Fossify does) is recognised by its absence. +- `taskOnly` lists calendars found to hold only tasks. +- `deviceZone` (calendar) is the zone floating events were written in; + `deviceZonePending` is set while a zone-change pass runs and cleared, with + `deviceZone` updated, in the batch of its last chunk. +- The providers delete the row with the account. + +## A sync run + +One run handles one Android account and one authority. A module-level mutex per +(registry account, authority) serialises it with other runs and with app-side +operations such as turning sync off. + +1. **Preflight.** + - Permissions for the authority (a revoked one → `permission`). + - The Android account and that the authority is still enabled + (`getSyncAutomatically`) → otherwise `disabled`. + - The SyncState's `owner` matches this registry account and server origin + (see [Identity](#identity)). + - `new JMAPClient().loadAccount(registryId)`: `false` or + `AuthenticationError` → rebuild once (another client may have rotated + the refresh token); still failing → `auth`, plus one notification that + deep-links to sign-in. Nothing local is touched. + - The capability on at least one JMAP account → else `unsupported`. + - An upload-only sync (`extras.upload`) with nothing dirty, deleted or new + for our account ends here with `ok`: ContactsProvider and CalendarProvider + schedule an upload sync for every account 30 s after any app write, other + accounts' included. +2. **Local checks.** Calendar: two masters sharing a `_SYNC_ID` (a + `CONTENT_EXCEPTION_URI` split, which also caps the old master's rule + without DIRTY) are resolved before any merge or baseline heal: the newer row + becomes a new event and the older master's rule is uploaded as it stands. + Objects on `stale` are fetched again. +3. **Collections.** `AddressBook/get` or `Calendar/get` (explicit properties) + for the primary JMAP account of the capability (`primaryAccounts[capability]`) + and every shared account that advertises it; calendar rows are inserted and + updated to match. Newly selected collections are loaded after step 4 has + brought the account up to date, restricted to them, so the stored state + never skips changes of the other collections. Deselected collections are + dropped after step 5, and only their clean rows go: dirty items there are + uploaded first (they are still valid objects on the server), and an item + whose upload failed keeps its rows and is reported. +4. **Download.** + - With a stored `itemsState`: `/changes` with `maxChanges: 256` and a + `hasMoreChanges` loop; each page's changed ids are fetched with explicit + `properties` in batches of `maxObjectsInGet`. + - Without a state, or on `cannotCalculateChanges`: a full reconcile, + checkpointed in `reconcile` (see [SyncState](#syncstate)). The state is + taken first (`/get` with `ids: []`), then every selected collection's ids + (`/query`, `inAddressBook`/`inCalendar`, `limit` 5000 with `position` + paging), then the objects in batches. Rows without an identity or with a + pending one (and exceptions of pending masters) are never reconcile + deletions. A reconcile that finds zero remote objects while the device + holds more than 10 rows for that JMAP account stops with `safetyAbort` + instead of wiping the device. + - An object without rows first looks for a local item it may already be: + a new row of the same JMAP account whose pending uid is the object's + `uid` (our own create whose identity never got written) is adopted: it + takes the identity and shadow and is merged as a dirty item. + - Each object is [merged](#merge-rules) into its rows. An object that left + every selected collection loses its rows only when they are clean; dirty + units are uploaded first. + - Rows are written in chunks; the new state goes into the last chunk of each + page, so every page is a checkpoint. +5. **Upload**, always after download so merges see the latest server truth; see + [Uploads](#uploads). +6. **Report** counts, conflicts, per-item errors and duration; persist the + status for the settings UI; call `finishRun` before the task's promise + settles (JS timers stop once the task is over). + +`/get` always lists `properties` explicitly: with `properties: null` Stalwart +leaves out JMAP-only fields such as `useDefaultAlerts`, and `ids: null` returns +at most 500 objects without saying so. + +Checkpoints sit between chunks and phases. At each one the run checks +`isRunCancelled`, its deadline and the time since the last JMAP request. When +the deadline is near it stops cleanly and reports `moreRecordsToGet`, which +the adapter turns into `fullSyncRequested` (SyncManager ignores +`moreRecordsToGet` itself). The step from a create's `/set` to its identity +write is not interruptible, and a create batch starts only while at least two +request timeouts (60 s) of budget remain. + +### Invariants + +Every reviewer checks against these. + +1. **No server data loss.** Uploads are JMAP patches (RFC 8620 §5.3, JSON + Pointer with `~0`/`~1` escaping) computed per mapped property against the + baseline. A property Android can't represent is never sent. A full-object + replace is never sent, and neither is a destroy + create for what was an + edit or a move on the device. Where Stalwart needs a whole map resent + (deleting a map entry, adding the first entry), the map is rebuilt from the + fresh shadow, so its other entries go back unchanged, and the request carries + `ifInState` so it cannot overwrite a server change the merge did not see. + What an editor did not model (a row it re-inserted without some column, an + entry it dropped) is never read as a deletion. +2. **No silent device-edit loss.** A dirty row is cleared only after the server + accepted the change, and only if it wasn't edited again during the upload: + the clearing batch asserts `RawContacts.VERSION` for contacts and the full + read projection (event row, attendees, reminders) for events, so check and + clear are one provider transaction. A row whose mapped projection equals its + baseline (only STARRED changed, say) is cleared without uploading. No dirty, + deleted or pending row is removed locally (deselection, move-away, disable, + logout, account reconciliation) before its upload succeeded or the user + confirmed losing it. +3. **Crash-safe and idempotent.** Every step can be re-run. The JMAP state is + written in the same `applyBatch` as the last chunk of rows it describes, as + that batch's last op. Creates carry a client-minted `uid`, written to the + row with the create's target before the `/set`; a download adopts a pending + row by uid, and an upload looks the uid up and adopts what it finds, so a + retry never duplicates on either side. +4. **No feedback loops.** Adapter writes use `CALLER_IS_SYNCADAPTER`, which + neither sets DIRTY nor schedules an upload sync, and never touch the + columns that make a provider request a sync (`CAL_SYNC1`, `Groups.SHOULD_SYNC`). + An echo of our own upload compares equal to the shadow and results in zero + provider writes. A run that failed reports progress only for rows it + actually wrote. +5. **Scoped.** Only rows of our own account type are read or written; the + native side enforces it for every table. +6. **Deletion safety.** Local→server deletions above the threshold set + `tooManyDeletions`; `SYNC_EXTRAS_OVERRIDE_TOO_MANY_DELETIONS` and + `SYNC_EXTRAS_DISCARD_LOCAL_DELETIONS` are honoured. Deletions inferred from + absence (groups) count toward the threshold. A full reconcile that returns + zero remote objects while the device holds more than 10 aborts. Deleting a + device item that belongs to several collections removes only the selected + memberships. +7. **Conflicts.** See [merge rules](#merge-rules). +8. **Limits respected:** `maxObjectsInGet/Set`, `maxCallsInRequest`, one + request in flight per run, `/changes` paging, provider batch sizes and + yield points, the Binder budget, the headless timeout and the JS thread + (yield between chunks). +9. **iOS untouched.** Everything is gated by `Platform.OS === 'android'` plus + the presence of the native module. +10. **No state ahead of a shadow.** No stored `itemsState` is newer than the + shadow of an object that changed before it (see `stale`). + +## Change detection and merge rules + +### Baselines + +Mappings are lossy in both directions, and the providers normalise what they +store: ContactsProvider splits a lone display name into components and fills a +missing formatted address, CalendarProvider fills the organizer, zeroes +all-day times and rewrites durations. Comparing rows against a fresh +`toDevice(shadow)` would see edits nobody made. + +So every row carries a **baseline**: its mapped columns exactly as the provider +stored them after our last write, read back after each chunk (text over 1 KB +as its hash). A column that differs from its baseline was written by someone +else. A clean row's baseline is by definition its current content, so whenever +the engine reads a clean row whose baseline differs, it rewrites the baseline; +that also heals the crash window between a chunk and its read-back. Two +exceptions: + +- the read-back also reads DIRTY (and VERSION): a row that turned dirty in the + meantime keeps its old baseline, so the user's edit still counts; +- `timing` and `rule` of a recurring master are never healed: a + `CONTENT_EXCEPTION_URI` split changes them without DIRTY (see the + [local checks](#a-sync-run)). + +### Units + +Changes are detected, merged and uploaded per **unit**: + +- contacts: the name (the StructuredName row), each entry of a map (one data + row: `emails:work1`, `phones:p-a`, `addresses:home`, …), the note, the + photo, and the card's group memberships; +- events: groups of related columns (`title`, `description`, `location`, + `timing`, `rule`, `status`, `availability`, `privacy`, `color`, `calendar`), + each attendee (by email), the reminder set, each EXDATE and each exception + row (recursively, the same units). + +A unit's upload is the set of patch paths for the sub-fields whose columns +changed against the baseline; sub-fields Android can't hold are never in it. + +### Row matching (contacts) + +Row keys are hints, not identity. Fossify Contacts deletes and re-inserts every +data row but the name on each save, dropping `DATA_SYNC*`, the columns it does +not model (DEPARTMENT, labels, a website's type), `IS_PRIMARY`, and every +nickname and organization but one; AOSP Contacts updates rows in place. So, +per data kind: + +1. Empty rows (an editor's blank nickname or note) are ignored. +2. Rows whose `DATA_SYNC1` key is in the shadow keep that key. +3. Rows without a usable key are matched to the shadow's unmatched entries by + the kind's primary value (email address case-insensitively, phone digits, + URL, date + type, …); then, only when the numbers of unmatched rows and + entries are equal, pairwise in order. +4. Rows still unmatched are new entries and get fresh keys: `b` + 8 random + base36 characters (Stalwart parses numeric keys as array indexes and + generates `k` itself, so neither is used). +5. Shadow entries still without a row count as deleted on the device **only + when the kind still has rows that carry keys** (an in-place editor removed + one). When every row of the kind is keyless, an editor rewrote them all, and + a missing entry may just be one it could not show: it stays on the server. + +A row matched by its key is compared column by column with its baseline; an +emptied column is a real deletion. A row matched by value has no baseline: its +primary value uploads when it differs from the shadow's projection, its TYPE +and LABEL only for phones, emails, addresses and events (editors that re-insert +rows write a constant type for websites), and never `IS_PRIMARY`; an empty +column never clears anything. + +The name: `""` and NULL are the same. `name/full` uploads when DISPLAY_NAME +changed against the baseline and is not simply the provider's join of the +current components (a typed display name); when only components changed, +`name/full` is re-derived with them only if the shadow's `full` was the derived +one (Stalwart never re-derives it), and left alone when it was custom +("Müller, Hans"). + +### Lossy calendar editors + +`Events.MUTATORS` names the app that changed the event. Fossify Calendar +re-inserts every attendee as a required attendee and rebuilds the RRULE from its +own model (FREQ, INTERVAL, COUNT or UNTIL, one BYDAY or BYMONTHDAY=-1, +BYMONTH). For an event it changed, attendee type and relationship changes are +ignored, and a rule change is applied part by part onto the shadow's rule: parts +Fossify models and changed are patched, parts it dropped stay. + +### Key reconciliation (downloads) + +Stalwart keeps the keys a JMAP client wrote. Cards and events written over +CardDAV/CalDAV, or received as invitations, have no stored keys, and Stalwart +generates them on every read: `k1…kN` by position for card entries and alerts, +`uuid5` of the address/name/URL for participants, locations and links. Such +keys shift when an entry is inserted before them. So before merging a new +server version, the engine maps the shadow's keys to the new keys by content +(equal entries first, then equal primary values) and renames the rows' keys; +only the remainder counts as added or removed. The first JMAP write to such an +object freezes its keys. + +### Merge rules + +For a dirty row whose object also changed on the server (base = shadow, +local = rows, remote = new server object), per unit: + +| Local changed? | Remote changed? | Result | +|---|---|---| +| no | no | nothing | +| no | yes | write remote, new baseline | +| yes | no | keep local, still dirty; uploads later against the new shadow | +| yes | yes, to the same value | converged; new baseline | +| yes | yes, differently | **server wins**: write remote, new baseline, `conflicts += 1` | + +"Remote changed" compares the mapped projection of base and remote, so a server +change to a sub-field Android doesn't show is not a conflict: the upload +patches only the paths the user changed and the server keeps its own. + +Whole-object rules: + +- Edit on the device + delete on the server → **the server delete wins**; the + rows are deleted and counted as a conflict. +- Delete on the device + edit on the server → **the device delete wins**; the + object is destroyed. A deletion is an explicit act; restoring the object + would resurrect something the user removed. +- Delete on both → purge, no conflict. +- A server object whose `uid` differs from the shadow's is a different object: + its rows are replaced, never patched. + +Every download write into an existing item is guarded: the group asserts the +DIRTY flag and VERSION (contacts) or projection (events) the plan was made from, +and ops that address one row by id expect exactly one row. Exception rows are +deleted by id only. A failed assert re-reads and re-plans the item (up to 3 +times); after that the item goes on `stale`. + +## Uploads + +Per JMAP account, in this order: pairs (below), creates, updates, deletions, +then group memberships; in `/set` calls of at most `maxObjectsInSet` objects +(creates: 50) and about 1 MB of JSON (Stalwart splits bigger calls into +several commits, which can apply partially), one request in flight, and one +`sendSchedulingMessages` value per call (it is a request-level flag). + +- **ifInState.** Every `/set` carries `ifInState` = the item state the merge + was based on (then the `newState` of our previous `/set`). A `stateMismatch` + means the server changed since the download: the run downloads again and + retries the upload, at most three times, then reports `io` without progress + (SyncManager would otherwise retry at once, without back-off). +- **Pairs.** Some apps turn an edit into a delete plus an insert: Etar moves an + event to another calendar that way, and turns a recurring series into a + single event (or edits "this and following" from its first instance) the + same way. A deleted row and a new row of the same run whose projections + match apart from the calendar (or the rule) are uploaded as a patch of the + existing object (`calendarIds` swapped; `recurrenceRule: null` with its + overrides removed); the new row takes the identity and exceptions of the old + one, which is purged. Across JMAP accounts a move is a create in the target + account first, identity written, then a destroy in the source (the engine + records its own destroys so the next download treats them as echoes). + Unpaired rows are ordinary creates and deletions. +- **New rows** (see [Identity](#identity) for what counts as new): + 1. Claim: write the pending create (`SYNC3` / `SYNC_DATA3`: the uid and the + target collection; for events also `UID_2445` and + `_SYNC_ID = ~pending/`). A row keeps a uid it already carries unless + another row, synced or pending, uses the same uid (a + `CONTENT_EXCEPTION_URI` split clones `UID_2445`); a server object with + that uid does not count, because it may be this row's own earlier create. + The target is fixed at the claim: contacts go to the address book chosen + in the settings if it is selected and writable, else the first selected + writable book of the personal account (none → the item is poisoned with + `noWritableAddressBook`); events go to the calendar of their row. + 2. Look the uid up: contacts in the target address book (card uids are + unique per book), events in the whole account (event uids are unique per + account). A hit is adopted: the row takes its identity and is merged as a + dirty item. + 3. Otherwise `create`. + 4. `invalidProperties` on `uid` means the object exists: Stalwart's + duplicate check is synchronous but the `uid` query reads an asynchronous + search index, so a just-created object can be invisible to the query. The + card error names the existing id; for events the lookup is retried with + back-off and, failing that, left to the next run's download, which adopts + the row by uid. + 5. From the `/set` to the identity write nothing is interruptible, and a + create batch starts only with at least two request timeouts (60 s) of + budget left. +- **Exception rows** (`ORIGINAL_*` set) upload only as + `recurrenceOverrides/` of their master, after the master's create when + the master is new. +- **Dirty rows**: the patch from the changed units. An empty patch clears + DIRTY without a request. +- **Deleted rows** (`DELETED=1`): an object that belongs to several + collections loses only the selected memberships (`addressBookIds/: null` + / `calendarIds/: null`) and is destroyed only when none would remain. + Then the rows are purged with sync-adapter deletes (for events also their + exceptions, by id: a sync-adapter delete removes only the row itself). + `notFound` counts as done. A deleted row without identity but with a pending + create is looked up first and destroyed if found. +- **Afterwards**: a `/get` of every touched id (a separate request for creates: + Stalwart does not resolve result references to `/set`). The result becomes + the new shadow, and the rows' baselines are refreshed from it, in one group + per item that first asserts the VERSION/projection read before the upload + and clears DIRTY. If the assert fails (edited again meanwhile), the shadow and + identity are still written, the baselines of the uploaded units are set to + the values that were uploaded, and DIRTY stays: the newer edit uploads next + time, and so does a revert to the old value. +- **SetErrors**: + - `notFound` on update → treated as a remote delete (the rows go); + - `forbidden` in a call that asked for scheduling messages → the item is + retried once without them and the report notes "invitations not sent" + (Stalwart refuses scheduling when it is disabled, when the account has no + calendar address, or without the scheduling permission); `forbidden` + otherwise → the rows are rewritten from the shadow (read-only on the server + after all); + - `stateMismatch` → as above; + - `invalidProperties` on `uid` → as above; + - `invalidProperties`, `invalidPatch`, `tooLarge`, `overQuota` and the rest → + the item is poisoned: its marker records a fingerprint of the patch and a + back-off (1 h, doubling to 1 day). It is not retried until the back-off + ends or the row changes, and the report lists it. + +### Deletion threshold + +More than 50 local deletions of an authority in one run, and more than 20 % of +that authority's synced objects, set `tooManyDeletions` unless the run carries +`overrideTooManyDeletions`. Deletions inferred from absence (groups) count. +No deletion of that run is uploaded; everything else proceeds. With +`discardLocalDeletions`, the deleted rows are purged and, in the same batch, the +affected JMAP accounts get a `reconcile` marker, so the next runs download the +objects again whatever their extras. + +## Contacts mapping + +JSContact (RFC 9553) as Stalwart stores it, to ContactsContract. Anything not +listed is preserved through patching. Data rows are written with both the +display name and the components (the provider recomputes whichever is missing), +and every entry row stores its key(s) in `DATA_SYNC1`. + +### Name (StructuredName, one row, key `name`) + +| JSContact | Android | +|---|---| +| `name.full` | `DISPLAY_NAME`; when absent, `deriveFullName(components)` is written and `full` is not uploaded unless the user changes it | +| component `title` (also `prefix`) | `PREFIX` | +| component `given` | `GIVEN_NAME` | +| component `given2` (also `middle`, `additional`) | `MIDDLE_NAME` | +| component `surname` (+ `surname2`, joined with a space) | `FAMILY_NAME` | +| component `generation`, `credential` (also `suffix`) | `SUFFIX` | +| component `phonetic` of given / given2 / surname | `PHONETIC_GIVEN_NAME` / `PHONETIC_MIDDLE_NAME` / `PHONETIC_FAMILY_NAME` | + +Upload: a changed column replaces the matching components (the first of their +kind; `surname2` is dropped only when `FAMILY_NAME` changed), keeps every other +component (separators, repeats, unknown kinds) in place, and sends the whole +`name/components` array (JSON Pointer can't address array elements). When +`name/full` goes along is in [row matching](#row-matching-contacts): a typed +display name uploads; a recomputed one re-derives `full` only when the server's +was derived too. + +### Entries + +| JSContact | Android kind | Columns | +|---|---|---| +| `nicknames/` | Nickname | `NAME`, `TYPE=DEFAULT` | +| `emails/` | Email | `ADDRESS`; `TYPE` from contexts: `private`→HOME, `work`→WORK, none→OTHER, `label`→CUSTOM+`LABEL`; `pref=1`→`IS_PRIMARY` | +| `phones/` | Phone | `NUMBER`; `TYPE` from features + contexts (table below); `label`→CUSTOM+`LABEL`; `pref=1`→`IS_PRIMARY` | +| `addresses/` | StructuredPostal | components → `STREET`, `POBOX`, `NEIGHBORHOOD`, `CITY`, `REGION`, `POSTCODE`, `COUNTRY` (table below), `full`→`FORMATTED_ADDRESS`; contexts → TYPE HOME/WORK/OTHER, `label`→CUSTOM | +| `organizations/` + `titles` pointing at it | Organization | `name`→`COMPANY`, `units` joined with `, `→`DEPARTMENT`, title with `kind: title`→`TITLE`, title with `kind: role`→`JOB_DESCRIPTION`; key `organizations:|titles:,`; titles without an organization get their own row | +| `links/` | Website | `URL` ← `uri`; contexts `private`→HOME, `work`→WORK, else OTHER; `label`→CUSTOM | +| `anniversaries/` | Event | `START_DATE`: `YYYY-MM-DD`, `--MM-DD` without year (`partialDateToString`); kind `birth`→BIRTHDAY, `wedding`→ANNIVERSARY, `other`→OTHER, anything else → CUSTOM + its kind as `LABEL` | +| `relatedTo/` | Relation | `NAME` ← the key (a `urn:uuid:` of a synced card shows that card's name), `TYPE` from the relation set (table below) | +| first entry of `notes` | Note | `NOTE` ← `note` | +| first `media` entry with `kind: "photo"` | Photo | full-size bytes into `PHOTO`; the provider scales them, stores the display photo and sets `PHOTO_FILE_ID` | +| group cards (`kind: "group"`) | Groups | `TITLE` ← `name.full`; `GROUP_VISIBLE=1`; `members` keys are uids of the member cards | +| membership | GroupMembership | one row per group card whose `members` holds this card's `uid` (`GROUP_SOURCE_ID`) | + +Phone types (first match wins; reverse: the table read right to left, and +`TYPE_OTHER` → no features): + +| Features / contexts | `Phone.TYPE` | +|---|---| +| mobile + work | WORK_MOBILE | +| mobile | MOBILE (read: also `cell`) | +| fax + work | FAX_WORK | +| fax + private | FAX_HOME | +| fax | OTHER_FAX | +| pager + work | WORK_PAGER | +| pager | PAGER | +| textphone | TTY_TDD | +| main-number | MAIN | +| work (voice) | WORK | +| private (voice) | HOME | +| anything else | OTHER | + +Postal components (reverse: an edited field replaces all components it was +built from with one component of the first kind, keeps the rest): + +| Android | JSContact component kinds, joined in order | +|---|---| +| `STREET` | `name` (street), `number`, `building`, `floor`, `apartment`, `room`, `block`, `direction`, `landmark` (with `separator`s) | +| `POBOX` | `postOfficeBox` | +| `NEIGHBORHOOD` | `district`, `subdistrict` | +| `CITY` | `locality` | +| `REGION` | `region` | +| `POSTCODE` | `postcode` | +| `COUNTRY` | `country` | + +Legacy flat address fields (`street`, `locality`, …) are read when there are no +components and never written. + +Relations: `friend`↔FRIEND, `spouse`↔SPOUSE, `child`↔CHILD, `parent`↔PARENT, +`sibling`→RELATIVE, `kin`→RELATIVE, `co-worker`/`colleague`→CUSTOM with the +type as label, `agent`→ASSISTANT, `emergency`→CUSTOM. Device types without an +RFC 9553 equivalent (BROTHER, SISTER, MOTHER, FATHER, MANAGER, ASSISTANT, +DOMESTIC_PARTNER, PARTNER, REFERRED_BY) upload as the nearest RFC type +(`sibling`, `parent`, `colleague`, `agent`, `spouse`, `spouse`, `contact`) and are +only sent when the user changed the type. + +Deleting an entry resends its whole map (Stalwart turns `map/key: null` into a +valueless vCard line or, for `relatedTo`, ignores it). Only `addresses`, +`members` and `keywords` take a `null` delete cleanly. + +Preserve-only (never written, never uploaded): `speakToAs`, +`preferredLanguages`, `cryptoKeys`, `directories`, `calendars`, +`schedulingAddresses`, `onlineServices`, `keywords`, `personalInfo`, +`localizations`, `language`, notes after the first, media other than the +first photo, and `kind` (an `org` card syncs like a person; `group` cards are +groups). Device-only columns (STARRED, custom ringtone, send-to-voicemail, +`IS_SUPER_PRIMARY`) never upload. + +Photos: a `data:` URI is decoded; a blob-backed `media` entry is downloaded +with the detached client. Upload: the display photo is read back, scaled to at +most 512 px, JPEG, and sent as a `data:image/jpeg;base64,…` URI in +`media//uri`, matching the app's picker. `DATA_SYNC2` holds the hash of +the server photo last applied, the baseline holds `PHOTO_FILE_ID`; a new file +id means the user changed the photo. Photos travel in batches of their own +(Binder budget). + +Groups: created before memberships (a membership naming an unknown +`GROUP_SOURCE_ID` makes the provider create an empty group). A group deleted on +the device is found by `DELETED`, or by its absence from the rows while +`SyncState.groups` still lists it (Fossify hard-deletes groups through a +sync-adapter URI); both count toward the deletion threshold. + +## Calendar mapping + +### Calendar rows + +| JMAP `Calendar` | Android `Calendars` | +|---|---| +| `id` | `_SYNC_ID` = `/` | +| `name` | `NAME`, `CALENDAR_DISPLAY_NAME` (shared accounts append the account's name) | +| `color` | `CALENDAR_COLOR` (CSS color → ARGB; absent → the app's default palette colour) | +| `myRights` | `CALENDAR_ACCESS_LEVEL`: `mayWriteAll` → OWNER (700); `mayWriteOwn` → CONTRIBUTOR (500); `mayRSVP` → RESPOND (300); else READ (200); subscribed iCal feed → READ | +| the user's calendar address | `OWNER_ACCOUNT` (never null: Etar crashes on it) | +| `timeZone` | `CALENDAR_TIME_ZONE` (device zone when null) | +| — | `SYNC_EVENTS=1` (the default 0 hides every instance), `VISIBLE=1` on insert only (local afterwards), `ALLOWED_REMINDERS="1,2"`, `ALLOWED_AVAILABILITY="0,1"`, `ALLOWED_ATTENDEE_TYPES="0,1,2,3"`, `MAX_REMINDERS=5` (0 when Bulwark owns reminders), `CAN_ORGANIZER_RESPOND=0`, `CAN_PARTIALLY_UPDATE=0` | + +Not synced: the virtual birthday calendar (client-side only), calendars +without `mayReadItems`, and task-only calendars: a calendar whose first 50 +objects are all `@type: "Task"` (`isTaskLikeObject`), re-checked on each full +reconcile. + +Deleting a calendar row deletes its events (provider trigger); the engine +first uploads the calendar's dirty events. + +### Timing + +| JSCalendar | Android | +|---|---| +| timed, `timeZone` set | `DTSTART` = instant of `start` in `timeZone`; `EVENT_TIMEZONE` = `timeZone` | +| floating (`timeZone` absent) | as timed, in the device zone; `SyncState.deviceZone` records it. When the device zone changes, clean floating events are rewritten so they keep their wall time | +| `showWithoutTime` | `ALL_DAY=1`, `DTSTART` = UTC midnight of the date, `EVENT_TIMEZONE="UTC"` | +| non-recurring | `DTEND` = start + `duration` (days nominal in the zone, the rest exact); `DURATION` null | +| recurring | `DURATION` = `PD` (all-day) or `PS` (timed), `DTEND` null | +| exception row | `DTEND` (the provider drops `DURATION` from exceptions) | + +Android's Duration parser takes `[+-]P` then `` (with `M` always +minutes, no fractions, no `Y`); all-day durations must be `PD` or the +provider throws. Server durations are parsed in full (weeks included, `P1W` +comes from CalDAV DTEND). A duration with a time part on an all-day event is +rounded up to whole days on the device and never uploaded unless changed. + +Upload of timing: only when a timing column differs from the baseline. The +local start is recomputed in the event's zone (floating events stay floating, +all-day events stay dates), and the duration from DTEND/DURATION with whole days +nominal. `EVENT_TIMEZONE` must be an IANA id: an unknown id silently becomes GMT +on Android, and makes the event floating on Stalwart; zones are validated with +`Intl` before either write. + +**DST edges.** Stalwart silently drops a `start`, an `until` or an override key +whose local time falls in a DST gap or overlap of the zone. Checked on the local +server: an event created at `2026-10-25T02:30` or `2026-03-29T02:30` +Europe/Berlin comes back without `start` and `timeZone`, and a daily rule with +`until: 2026-10-25T02:30` comes back without `until`, i.e. unbounded. So before +an upload: + +- a local time in a gap moves forward by the gap (RFC 5545 semantics); +- a non-recurring event whose start falls in an overlap is sent in the + fixed-offset zone of its instant (`Etc/GMT-2` for 02:30 CEST; `Etc/UTC` when + the offset is not a whole hour): same instant, same wall time, only the zone + name changes; +- a recurring event whose start or an override key falls in an overlap is not + uploaded: the item is poisoned with `dstAmbiguous`; +- an `until` in a gap or overlap moves one hour later. + +### Recurrence + +| JSCalendar | Android | +|---|---| +| `recurrenceRule` | `RRULE`: `FREQ`, `INTERVAL` (>1), `COUNT`, `UNTIL` (UTC `…Z` for timed events, `YYYYMMDD` for all-day), every `BY*` part, `BYSETPOS`, `WKST` (when not Monday) | +| `rscale` other than gregorian, `skip`, leap months (`byMonth` with `L`) | not representable: the rule is written without them, the event's timing and rule are never uploaded, and the item is marked `ruleNotRepresentable` | +| `excludedRecurrenceRule` | not supported by Stalwart; an `EXRULE` written by a device app is ignored | +| override `{excluded: true}` | `EXDATE`: comma list of UTC `YYYYMMDDTHHMMSSZ`, or `YYYYMMDD` for all-day (no prefix means UTC) | +| any other override, including `{}` | an exception row: master ⊕ override (applied as a PatchObject); `ORIGINAL_SYNC_ID`, `ORIGINAL_INSTANCE_TIME` = the key's instant in the master's zone, `ORIGINAL_ALL_DAY`; `SYNC_DATA2` = the key | +| — | `RDATE` is never written | + +Why no RDATE: an infinite rule plus RDATE cannot be inserted (the provider +tries to expand it to infinity), and RDATE-only events lose their DTSTART +instance. An exception row whose original instance the rule does not generate +simply shows as an extra event, which is what an extra JMAP instance is. + +AOSP expands BYSETPOS only for monthly rules with BYDAY, and walks at most 2000 +periods from DTSTART (a daily series older than about 5.5 years shows no +instances); both are listed under limitations. + +Device-side changes map back as follows: + +| On the device | Upload | +|---|---| +| exception row inserted (Etar "this event": `ORIGINAL_SYNC_ID` + `ORIGINAL_INSTANCE_TIME`, `DIRTY=1`) or edited | `recurrenceOverrides/`; the key is `ORIGINAL_INSTANCE_TIME` in the master's zone (a date at midnight for all-day). A new override carries the master's title, locations, participants and sequence (Stalwart stores a bare override as a separate event otherwise); an existing one is patched one level deep (`recurrenceOverrides//`), never deeper: Stalwart turns a deeper pointer into a partial override (other attendees vanish for that instance) | +| exception row with `STATUS_CANCELED` (Etar and Fossify "delete this event"), a deleted exception row, or an EXDATE added to the master | `recurrenceOverrides/: {excluded: true}` | +| EXDATE removed | `recurrenceOverrides/: null` | +| master `RRULE` capped with `UNTIL`/`COUNT` + a new event without identity (Etar "this and following") | `recurrenceRule` (whole object) with the new `until`/`count`, overrides after it removed (`recurrenceOverrides/: null`), and the new event created with a fresh uid | +| master start moved (Etar "all events") | `start`/`duration`/`timeZone`, and every override key moved by the same local offset (overrides whose `start` equalled their old key move too), sent as the whole `recurrenceOverrides` map | +| two masters with the same `_SYNC_ID` (a `CONTENT_EXCEPTION_URI` split clones `_SYNC_ID`, `SYNC_DATA*` and `UID_2445`, and caps the old master's rule without `DIRTY`) | found before any merge or baseline heal: the newer row becomes a new event (fresh uid), the older master's current RRULE is uploaded | +| event moved to another calendar, or a series turned into a single event (Etar deletes and re-inserts both) | paired with the deleted row and uploaded as a patch of the same object (see [Uploads](#uploads)) | +| an RSVP for one instance (Etar goes through `CONTENT_EXCEPTION_URI`) | the override's whole `participants` map with the user's new status | + +The first override of an event with none sends the whole `recurrenceOverrides` +map (a patch below a missing property fails). + +### Other properties + +| JSCalendar | Android | +|---|---| +| `title` | `TITLE` (exception rows always carry the master's title: Fossify Calendar treats an exception with an empty title as deleted) | +| `description` | `DESCRIPTION`; `descriptionContentType: text/html` is written as plain text and uploaded only when edited, then as `text/plain` | +| first entry of `locations` (server order) | `EVENT_LOCATION` ← `name`; edit patches `locations//name`, a new one adds `locations/`, clearing deletes the entry | +| `status` confirmed / tentative / cancelled | `STATUS` 1 / 0 / 2; absent → 1. Never NULL (the provider throws) | +| `freeBusyStatus` busy / free | `AVAILABILITY` 0 / 1; device TENTATIVE (2) uploads as `busy` | +| `privacy` absent / public / private / secret | `ACCESS_LEVEL` DEFAULT / PUBLIC / PRIVATE / CONFIDENTIAL | +| `color` | `EVENT_COLOR` (CSS name or `#rrggbb` → ARGB; upload `#rrggbb`); no Colors table in v1 | +| participants with a `mailto:` `calendarAddress` | Attendees rows: `ATTENDEE_EMAIL` (the user's own address normalised to exactly `OWNER_ACCOUNT`: Android matches "me" case-sensitively), `ATTENDEE_NAME`, `ATTENDEE_RELATIONSHIP` (owner or organizer address → ORGANIZER, else ATTENDEE), `ATTENDEE_TYPE` (`optional` role → OPTIONAL, kind `resource`/`location` → RESOURCE, else REQUIRED), `ATTENDEE_STATUS` (accepted 1, declined 2, needs-action 3, tentative 4, delegated/other 0); participant ids stay in the shadow | +| `organizerCalendarAddress` | `ORGANIZER` when the event has participants; otherwise unset (the provider uses the owner, which keeps the event editable) | +| — | `HAS_ATTENDEE_DATA=1`; `SELF_ATTENDEE_STATUS` is never written (the provider derives it and throws on update) | +| alerts with an offset before start (`relativeTo` start or absent, offset ≤ 0) | Reminders: `MINUTES` = −offset, `METHOD` = EMAIL for `action: email`, else ALERT | +| alerts relative to the end, absolute alerts, offsets after the start | preserved, not shown | +| `useDefaultAlerts: true` | the calendar's `defaultAlertsWithTime`/`WithoutTime` become the reminders (as the app's own scheduler does); a device edit uploads `useDefaultAlerts: false` plus explicit `alerts` | +| `uid` | `UID_2445` | +| `keywords`, `categories`, `priority`, `virtualLocations`, `links`, `relatedTo`, `sequence`, `locale`, `replyTo`, locations after the first, participants without a `mailto:` address | preserved only | + +Attendees upload: the user's own RSVP (their attendee row's status) patches +`participants//participationStatus`, with `sendSchedulingMessages: true` +like the app's `rsvpEvent`; on an exception row it goes into the override's +whole `participants` map. Added attendees get new participant keys with +`roles: {attendee: true}`, `participationStatus: needs-action` and +`expectReply: true`; removed ones are deleted (`participants/: null` is +clean on Stalwart). + +Reminders upload: the representable alerts are replaced by the device's +reminders, matched by (minutes, method) to keep existing alert keys; the others +stay. Only when the reminder owner is the device calendar app (below). + +### Scheduling messages + +Stalwart sends iTIP only when a `/set` asks for it +(`sendSchedulingMessages`, default false), whereas CalDAV clients get implicit +scheduling on every change. Device sync follows CalDAV's lead for changes a +calendar app would notify about: + +- the user is the organizer and the event has other participants: `true` when + title, description, timing, rule, overrides, location, status or attendees + changed, and on destroy (CANCEL); reminders, colour, availability and + privacy alone send nothing; +- the user is an attendee: `true` for their own RSVP change and on destroy + (Stalwart sends REPLY DECLINED); other edits are stored for the user only; +- events in the past: Stalwart sends nothing anyway. + +### Reminder owner + +When calendar sync is turned on while `calendarNotificationsEnabled` is on, the +app asks who should remind the user, per account: + +- **the calendar app** (default when Bulwark's reminders are off): Reminders + rows are written, and Bulwark's own scheduler skips events of the synced + calendars of that account (tasks keep their reminders); +- **Bulwark**: no Reminders rows; `MAX_REMINDERS=0` on the account's calendars + so apps offer no reminder editor; reminder rows an app adds anyway are neither + uploaded nor deleted. + +Changing the choice first uploads pending reminder edits (a normal run), then +rewrites the Reminders rows of that account's events together with their +baselines, one group per event that asserts the projection it read; no server +writes. + +## Headless payload, report and SyncResult + +Payload (`RunPayload` in `src/device-sync/types.ts`): +`{ runId, accountName, registryId, authority, extras: { manual, upload, +expedited, ignoreBackoff, overrideTooManyDeletions, discardLocalDeletions }, +deadline }`; only booleans cross from the framework's extras. + +Report (`RunReport`): `{ v: 1, runId, authority, outcome, message?, startedAt, +durationMs, stats: { downloaded: {created, updated, deleted}, uploaded: +{created, updated, deleted}, entries, skipped }, conflicts, itemErrors[], +tooManyDeletions?: {count, threshold}, delayUntil?, moreRecordsToGet? }`. + +| Outcome | Meaning | SyncResult (`SyncReports.kt`) | +|---|---|---| +| `ok` | synced; item errors may be listed | stats only | +| `io` | network or server trouble | `numIoExceptions=1` (soft: back-off; at once if progress was made) | +| `internal` | an unexpected exception | as `io` | +| `auth` | credentials rejected after a rebuild | `numAuthExceptions=1`, no progress (hard) | +| `permission` | runtime permission missing | `databaseError`, no progress (hard) | +| `unsupported` | no capability | `databaseError`, no progress | +| `safetyAbort` | empty server, full device | `databaseError`, no progress | +| `tooManyDeletions` | above the threshold | `tooManyDeletions=true`, `numDeletes` = pending deletions (the system's notification offers "delete" and "undo") | +| `cancelled` | framework cancel or out of time | stats only | +| `disabled` | sync is off | nothing | + +`delayUntil` (a 429's `Retry-After`, epoch seconds) and `moreRecordsToGet` +(→ `fullSyncRequested`) pass through. A run without a report is a soft error. +The stats count only rows actually written and objects actually sent, so a run +that repeated a page without writing anything is not "progress" to SyncManager. +A change pushed while a run is active (SyncManager drops a `requestSync` that +matches a running sync) flags the run; the adapter reads the flag after the run +is closed and then sets `fullSyncRequested`. + +## Native module API + +`NativeModules.BulwarkDeviceSync` (`DeviceSyncNativeModule` in `types.ts`). +Large payloads cross as JSON strings, which the bridge moves far faster than +nested maps. Every provider call is scoped to one account of our type. + +| Method | Does | +|---|---| +| `getInfo()` | `{ accountType, sdkInt }` | +| `listAccounts()` | our accounts with their `registryId` | +| `ensureAccount(name, registryId)` | adds the account (both authorities syncable, automatic off); true when created, false when it exists for the same registry id; rejects (`conflict`) when an account of that name belongs to another registry id | +| `removeAccount(name)` | removes it; the providers drop its rows | +| `getSyncSettings(name)` | master auto-sync, and per authority: syncable, automatic, periodic seconds, active, pending | +| `setSyncEnabled(name, authority, on)` | syncable = 1 and `setSyncAutomatically` | +| `setPeriodicSync(name, authority, seconds)` | replaces our periodic syncs (including the framework's default daily one); 0 removes them; the framework clamps to 15 min | +| `requestSync(name, authority, optionsJson)` | `{manual, expedited, upload, overrideTooManyDeletions, discardLocalDeletions}` as SYNC_EXTRAS_*; constant extras so repeated requests coalesce; flags an active run for `fullSyncRequested` | +| `openAccountSettings(name)` | the account's sync screen (`android.settings.ACCOUNT_SYNC_SETTINGS` with the Account), else `SYNC_SETTINGS` filtered to our type | +| `finishRun(runId, reportJson)` / `isRunCancelled(runId)` | the run handshake | +| `setPushRoutes(json)` | `{ jmapAccountId: [{ accountName, authorities }] }` for the push router, in SharedPreferences | +| `query(name, authority, queryJson)` | `ProviderQuery` → `ProviderRows` | +| `applyBatch(name, authority, opsJson)` | `ProviderOp[]` → `BatchResult`; failures resolve as `{ok: false, reason}` | +| `readSyncState(name, authority)` | the SyncState blob as text | +| `readPhoto(name, rawContactId, maxPx)` | the display photo (or thumbnail), scaled, as JPEG base64 + file id | + +Events: `BulwarkDeviceSync:accountsChanged` (an account of our type was added +or removed; JS re-lists). Events without a JS listener are dropped, so they are +hints; the UI re-reads state on mount and on foreground. + +Scoping, enforced in Kotlin for every op: + +- URIs always carry `caller_is_syncadapter=true`, `account_name`, + `account_type` (CalendarProvider accepts no other query parameters). +- Queries and updates/deletes by `where` get the account condition ANDed in: + `account_name/account_type` columns for raw_contacts, groups, settings, + calendars, events, colors; for data rows, `raw_contact_id IN` our raw + contacts; for attendees, reminders and extended properties, `event_id IN` + our events (resolved first; the providers may compile selections strictly, + so no subqueries). Ops by `id` verify the row belongs to the account first + and expect exactly one row. +- Inserts into data/attendees/reminders/extended_properties/events must name a + parent of the account (a back-reference to an insert of the same batch, or + an id that is verified). +- Tables are an enumerated set per authority; anything else is refused + (`scope`). +- Blobs (`{ b64 }`) are only accepted for `data.data15` (photos). +- A `syncState` op must be the last op of its batch and not `yieldAllowed`, + so a provider yield never commits the state ahead of the rows (refused as + `scope` otherwise). +- Batches: at most 499 ops between yield points (ContactsProvider throws at + 500), about 300 KB per call; `TransactionTooLargeException` resolves as + `tooLarge` so JS splits the batch. `OperationApplicationException` resolves + as `assert`, `SecurityException` as `permission`. + +## App integration + +### Store + +`src/stores/device-sync-store.ts`, keyed by `AccountEntry.id`, persisted under +`device-sync:v1` with `createPersistStorage({ writeDelayMs: 0 })` (not the +1-second delayed storage) and not part of the settings export. Per account: +`contactsSelection` / `calendarSelection` (explicit choices only; unlisted +collections follow the default: the personal account's on, shared accounts' +off), `newContactsAddressBook`, `reminderOwner`, `intervalSeconds` per +authority, the last `RunStatus` per authority, and `removedInAndroidSettings`. +Whether sync is on is not stored: Android's `getSyncAutomatically` is the +source of truth. Headless runs await hydration before any `set()` (zustand 5 +writes on every `set()`, even before hydration, and would overwrite the stored +preferences). + +### Settings sections + +Android only (`supportsDeviceSync` in `platform-capabilities.ts` and the +native module present). Settings → Contacts and Settings → Calendar each get a +"Sync to this device" section with a row per signed-in account: + +- an on/off switch (reflects `getSyncAutomatically`; "Paused in Android + settings" when the account exists but the authority is off there; a hint when + the device's master auto-sync is off); +- a chooser (sheet) for address books or calendars, with personal ones on and + shared ones off by default, read-only ones marked, and a warning when another + signed-in account already syncs the same shared collection; +- contacts: the address book for new contacts; +- calendar: who reminds you (calendar app or Bulwark); +- the interval: 15 min, 30 min, 1 h (default), 6 h, manual; +- "Sync now" (`requestSync` with manual + expedited); +- the last sync time and its result, or the error with an action ("Sign in + again", "Grant access", "Review deletions"); +- "Android account settings" (`openAccountSettings`). + +Turning sync on asks for `READ/WRITE_CONTACTS` or `READ/WRITE_CALENDAR` after a +rationale; denied → sync stays off and the section says why, with a link to the +app's system settings when Android no longer asks. + +### Lifecycle + +- **Enable:** permissions → `ensureAccount` → `setSyncEnabled(true)` → + `setPeriodicSync(interval)` → `requestSync(manual)`; push routes and push + types are refreshed. +- **Disable an authority:** the engine's teardown runs under the run mutex: + it uploads the authority's dirty, deleted and new items (bounded to 30 s; + it is not a sync run, so the "disabled" preflight does not stop it), then + deletes the authority's rows and SyncState. When items could not be uploaded + (offline, errors), nothing is deleted and the app asks: "N changes made on + this device haven't reached the server. Turn off anyway?" Only then + `setSyncEnabled(false)`; when both authorities are off the Android account + is removed. +- **Logout** (`auth-store.logout`, `logoutAll`, `removeAccount`): the same as + disabling both, before the credentials go, with the same question when + changes would be lost. +- **Registry evictions** (the app drops an account on its own, e.g. after an + `AuthenticationError`): reconciling Android accounts against the registry at + every engine start and on foreground finds an Android account whose registry + id is gone. It is not removed: automatic sync is turned off and one + notification says how many device changes are waiting ("Sign in again"). + Signing in again resumes it; removing it in the app or in Android Settings + drops it. An empty or unreadable registry is never acted on. +- **Account removed in Android Settings:** detected by the accounts listener + and on foreground; the store marks it, the switches show off, and the app + does not recreate the account. +- **Uninstall:** Android removes the accounts of an uninstalled authenticator, + and the providers delete their rows. +- **Permission revoked:** Android fails the sync before our code runs + (`databaseError`), so the section checks the permission itself and shows + "Grant access". + +### Triggers + +- **Periodic sync** per the user's interval (default 1 h: push covers + freshness). +- **Device edits:** the providers schedule upload syncs 30 s after any app + write (`supportsUploading`). +- **App running:** SSE `StateChange` for `AddressBook`/`ContactCard` or + `Calendar`/`CalendarEvent` on a synced account (`onStateChangeType` in + `state-change-bus.ts`; the stream serves the active account), the app's own + successful contact/calendar mutations (store actions), and returning to the + foreground → `requestSync` for that authority, debounced 5 s. A StateChange + whose state equals the one the engine last recorded is our own echo and is + skipped. +- **App closed:** while an account has device sync on, its push subscription + adds `ContactCard`, `AddressBook` (contacts) and `CalendarEvent`, `Calendar` + (calendar) to `EmailDelivery`, per account. The relay forwards the + StateChange's `changed` map verbatim (FCM data key `changed`; only its first + account key is repeated as `accountId`). A native router in the FCM and + UnifiedPush services walks every account key of `changed`, maps contact and + calendar types to `requestSync` through the routes JS stored, and starts the + mail push task only when some account's map has `EmailDelivery`. A guard at + the top of `pushBackgroundTask` returns for StateChanges without + `EmailDelivery`, so a contact change can never take the legacy "notify the + newest unread mail" path. + +## Failure matrix + +| Failure | What happens | Recovery | +|---|---|---| +| No network | `io` → back-off (30 s ×2, max 1 h) | automatic | +| Network lost mid-upload | the `/set` may have applied: a create's row is adopted by uid on the next download or lookup, patches re-apply idempotently | automatic | +| Process killed mid-sync | no report: soft error; the state was only advanced with complete chunks | next run repeats the page; echo writes are no-ops | +| OAuth token expired | the client refreshes; a rotated token held by another client → rebuild once | automatic, else `auth` + notification | +| Password revoked | `auth` (hard, no retry) + "Sign in again" | user signs in; the next trigger syncs | +| Permission revoked | Android fails the sync (`databaseError`) | "Grant access" in settings | +| `cannotCalculateChanges` | full reconcile of that JMAP account | automatic | +| Server `SetError` on an item | notFound → local delete; forbidden with scheduling → retried without it ("invitations not sent"); forbidden → revert; others → poisoned with back-off, listed in the report | automatic / user fixes the item | +| `stateMismatch` | re-download, retry up to 3 times, then `io` without progress | automatic (back-off) | +| 429 | `delayUntil` = Retry-After | automatic | +| Too many local deletions | `tooManyDeletions` + system notification | user picks delete or undo | +| Empty server on a full reconcile | `safetyAbort`, device untouched | user checks the server; turning sync off and on starts over | +| Edit during an upload | clearing assert fails → DIRTY stays, shadow updated | next run uploads the newer edit | +| JS crash or reload mid-run | task ends without report → soft error | automatic | +| React fails to start (debug without Metro) | boot fault → soft error in ~2 s | automatic | +| Account removed in Settings | providers drop rows; app turns sync off | user turns it on again | +| Logout / turning sync off | pending changes uploaded first; if that fails the user is asked before anything is deleted | — | +| Provider refuses a batch (`assert`/`tooLarge`/`provider`) | per-item retry / split; an item that still fails goes on `stale` and blocks its uploads | automatic, or listed in the report | +| App drops the account from its registry (e.g. after an auth error) | Android account kept, sync off, notification with the number of waiting changes | user signs in again, or removes it | +| Rows written for another server or registry account (SyncState `owner`) | the run stops with `internal` | the app offers to start over after uploading what it can | +| DST-ambiguous start on a recurring event, unrepresentable rule | item poisoned with a reason; device shows it, never uploads its timing | limitation | + +## Limits and performance + +- JMAP: `maxObjectsInGet/Set` (500 on Stalwart), `maxCallsInRequest` (16), + `maxConcurrentRequests` (4, shared with the UI: the engine keeps one request + in flight), `maxSizeRequest` (10 MB; the engine targets 1 MB), queries of at + most 5000 ids per call, `/changes` pages of 256 (Stalwart caps at 5000), + cards and events of at most 512 KiB. +- Provider: ≤ 499 ops between yield points, ~300 KB per `applyBatch`, photos in + their own batch; 50 raw contacts or 50 events per chunk. +- JS thread: a `setTimeout(0)` yield between chunks, so the UI keeps rendering + when the app is open; JSON crosses the bridge as strings. +- Time: 9.5 min per run (2.5 min expedited), checkpoints between chunks, + continuation through `fullSyncRequested`. + +The 2,000 + 2,000 scale run is recorded in the parity notes when #34 closes. + +## Testing + +- **Pure units** (vitest): mappers with golden fixtures, one field per test, + and round-trip properties (server → rows → projection equals the server's + mapped subset; unmapped data survives edits); RRULE, EXDATE, durations, time + zones, floating, all-day, DST edges of rules, exceptions, splits, reminders, + attendees; merge rules; the import-graph purity test. +- **Engine** (vitest): `src/device-sync/__tests__/fakes/fake-provider.ts`, an + in-memory ProviderPort with DIRTY/DELETED/VERSION, soft/hard deletes, + asserts, back-references, yield points and user-edit helpers (AOSP-style + in-place edits and Fossify-style delete+reinsert); and a fake JMAP server + for ContactCard/CalendarEvent/AddressBook/Calendar with `/changes`, + `cannotCalculateChanges`, `ifInState`, SetError injection, the async uid + index and Stalwart's key behaviour. Crash injection at every checkpoint; + poison items; the deletion guards; echo suppression; an edit during upload. +- **Kotlin** (JUnit, `gradlew :app:testDebugUnitTest`): report mapping, op and + URI translation, scoping. +- **Device** (API 34 and 32 emulators, local Stalwart only): the scenario matrix + in the #34 parity note, verified on three sides: the provider (`adb shell + content query …`), the server (JMAP calls) and the apps' UI. Debug builds + create accounts without UI: + + ``` + adb shell am broadcast -n com.anonymous.bulwarkmobile/.sync.DeviceSyncDebugReceiver \ + --es cmd ensure --es name --es registry --es enable contacts,calendar + adb shell requestsync -n -t com.anonymous.bulwarkmobile.account -a com.android.contacts --manual + ``` + +## Decisions and limitations + +Decisions made where the brief left a choice (safe defaults): + +- Default reminder owner when Bulwark's reminders are off: the calendar app. +- Delete on the device + edit on the server: the delete wins. +- Deletion threshold: > 50 and > 20 % of the authority's objects. +- Full-reconcile safety: more than 10 device rows against an empty server. +- Turning a Contacts/Calendar toggle off in Android Settings pauses sync and + keeps the data, like Google accounts; turning it off in the app also removes + the data. +- Events that are only in unselected calendars, and cards only in unselected + address books, are not on the device; an event in several calendars lives in + the first selected one (by id), a move on the device patches only that + membership, and a deletion removes only the selected memberships. +- A shared collection that another signed-in account already syncs gets a + warning in the chooser, not a refusal. +- An app account dropped from the registry keeps its Android account (sync + off) until the user signs in again or removes it: its device changes would + otherwise be lost. +- The user's own address for "me" and `OWNER_ACCOUNT`: the session login when + it is an email address, else the first identity's address; aliases from + identities also count as "me". + +Known limitations: + +- Google Contacts may not show or edit third-party accounts (closed source; + DAVx5 reports the same); AOSP-based and Fossify Contacts do. +- Fossify Contacts re-inserts rows on save: a field or entry it does not model + (a second nickname or organization, a department, a label) is kept on the + server, but an entry deleted or a field cleared in Fossify is not deleted or + cleared on the server. Fossify Calendar's rule edits upload only in the parts + its editor models. +- Android shows extra instances for BYSETPOS rules other than monthly-BYDAY, + and none for a daily series older than about 5.5 years (AOSP's 2000-period + expansion cap). +- Rules with `rscale`/`skip`/leap months: shown approximately, timing never + uploaded. `EXRULE` is not supported by Stalwart. +- Recurring events whose start or override falls in a DST overlap cannot be + uploaded (Stalwart drops ambiguous local times). +- A single occurrence that Stalwart holds only as an instance (no master) is + shown but read-only. +- Moving an event to a calendar of another JMAP account uploads as a create in + the target account and a destroy in the source (a new uid; participants are + notified accordingly). +- Descriptions over 64 KB are shown truncated and are not uploaded when edited. +- A shared address book or calendar that two signed-in accounts can both see + appears once per account when both sync it (each Android account holds its + own copy). +- Floating events follow the device zone; alerts relative to the end, absolute + alerts and alerts after the start are kept on the server but not shown. +- Photos travel as 512 px JPEG `data:` URIs; a larger original is replaced by + its scaled copy only when the user changes the photo on the device. + +## Out of scope + +Possible follow-ups: + +- **iOS.** iOS has no third-party sync-provider API for Contacts or Calendar. + The realistic route is a CalDAV/CardDAV configuration profile, which Stalwart + already serves. +- **Tasks** through the OpenTasks provider. +- **A contacts Directory provider** for server-side search. +- **Importing device-local contacts** into Bulwark. +- An account-scoped Colors table so calendar apps can offer event colours. +- Re-basing old daily/hourly series past AOSP's 2000-period expansion cap. From 59fcf386b97189851a9f7d37726eaf710a7baf49 Mon Sep 17 00:00:00 2001 From: Linus Rath <139418639+rathlinus@users.noreply.github.com> Date: Sun, 27 Sep 2026 02:53:22 +0200 Subject: [PATCH 04/87] test: add a fake device provider and jmap server for the sync tests #34 The device sync engine reaches Android's providers through a native bridge and the server through a detached JMAP client, and neither runs in vitest. This adds the stand-ins the engine and the mappers are tested against, and the contracts they implement: - `src/device-sync/types.ts`, `planner.ts`, `wire.ts`, `android-columns.ts`: the bridge, the provider op model, the JMAP port, and the interface between the engine and the contacts and calendar planners - a fake ContactsProvider and CalendarProvider with the behaviour the design relies on: the DIRTY and VERSION rules, soft and hard deletes, exception linking, normalisation, asserts, the yield-point limit, and a "user" face that edits the way AOSP Contacts, Fossify and Etar do - a fake Stalwart with `/changes` paging and compaction, `cannotCalculateChanges`, `ifInState`, uid checks behind a lagging search index, positional re-keying and fault injection (lost responses, 429, auth, SetErrors) - shared pure helpers (canonical JSON, PatchObjects with Stalwart's missing-parent rule, key and uid minting) and a test that walks the import graph, so no mapper or engine module pulls in React Native - `src/stores/device-sync-store.ts`: the per-account preferences and last run status the engine and the settings will share Checked: `npx vitest run src/device-sync` (91 tests), the full suite (2209 tests) and `npm run typecheck`. --- src/device-sync/__tests__/common.test.ts | 87 + .../__tests__/fake-jmap-server.test.ts | 1155 ++++++++ .../__tests__/fake-provider.test.ts | 293 ++ .../__tests__/fakes/fake-jmap-server.ts | 2510 +++++++++++++++++ .../__tests__/fakes/fake-provider.ts | 1209 ++++++++ src/device-sync/__tests__/fakes/fake-sql.ts | 269 ++ src/device-sync/__tests__/purity.test.ts | 113 + src/device-sync/android-columns.ts | 344 +++ src/device-sync/common/ids.ts | 78 + src/device-sync/common/json.ts | 72 + src/device-sync/common/patch.ts | 88 + src/device-sync/planner.ts | 357 +++ src/device-sync/types.ts | 65 + src/device-sync/wire.ts | 83 + src/stores/device-sync-store.ts | 139 + 15 files changed, 6862 insertions(+) create mode 100644 src/device-sync/__tests__/common.test.ts create mode 100644 src/device-sync/__tests__/fake-jmap-server.test.ts create mode 100644 src/device-sync/__tests__/fake-provider.test.ts create mode 100644 src/device-sync/__tests__/fakes/fake-jmap-server.ts create mode 100644 src/device-sync/__tests__/fakes/fake-provider.ts create mode 100644 src/device-sync/__tests__/fakes/fake-sql.ts create mode 100644 src/device-sync/__tests__/purity.test.ts create mode 100644 src/device-sync/android-columns.ts create mode 100644 src/device-sync/common/ids.ts create mode 100644 src/device-sync/common/json.ts create mode 100644 src/device-sync/common/patch.ts create mode 100644 src/device-sync/planner.ts create mode 100644 src/device-sync/wire.ts create mode 100644 src/stores/device-sync-store.ts diff --git a/src/device-sync/__tests__/common.test.ts b/src/device-sync/__tests__/common.test.ts new file mode 100644 index 00000000..43d10b6b --- /dev/null +++ b/src/device-sync/__tests__/common.test.ts @@ -0,0 +1,87 @@ +import { describe, expect, it } from 'vitest'; +import { canonicalJson, deepEqual, jsonHash, parseJsonColumn } from '../common/json'; +import { applyPatch, assertPatchWellFormed, ptr, ptrSegments, setInPatch } from '../common/patch'; +import { + exceptionRef, + makeKeyMinter, + objectRef, + parseObjectRef, + pendingRef, + pendingUidOf, + uuidFrom, +} from '../common/ids'; + +function seeded(seed: number) { + return () => { + seed = (seed * 1103515245 + 12345) % 2147483648; + return seed / 2147483648; + }; +} + +describe('common/json', () => { + it('compares and hashes independently of key order and undefined members', () => { + const a = { b: 1, a: { d: [1, { y: 2, x: 1 }], c: undefined } }; + const b = { a: { d: [1, { x: 1, y: 2 }] }, b: 1 }; + expect(canonicalJson(a)).toBe(canonicalJson(b)); + expect(deepEqual(a, b)).toBe(true); + expect(jsonHash(a)).toBe(jsonHash(b)); + expect(deepEqual({ a: [1, 2] }, { a: [2, 1] })).toBe(false); + expect(deepEqual({ a: null }, {})).toBe(true); + }); + + it('reads sync columns defensively', () => { + expect(parseJsonColumn('{"v":1}')).toEqual({ v: 1 }); + expect(parseJsonColumn('not json')).toBeNull(); + expect(parseJsonColumn(null)).toBeNull(); + }); +}); + +describe('common/patch', () => { + it('escapes pointer segments', () => { + expect(ptr('emails', 'x/y', 'label')).toBe('emails/x~1y/label'); + expect(ptrSegments('emails/x~1y/a~0b')).toEqual(['emails', 'x/y', 'a~b']); + }); + + it('applies patches like Stalwart, refusing a missing parent', () => { + const card = { emails: { e1: { address: 'a@x' } }, name: { full: 'A' } }; + expect(applyPatch(card, { 'emails/e1/address': 'b@x', 'emails/e2': { address: 'c@x' }, 'name/full': null })).toEqual({ + emails: { e1: { address: 'b@x' }, e2: { address: 'c@x' } }, + name: {}, + }); + expect(applyPatch(card, { 'phones/p1': { number: '1' } })).toBeNull(); + expect(card.emails.e1.address).toBe('a@x'); + }); + + it('keeps patches free of overlapping pointers', () => { + expect(() => assertPatchWellFormed({ emails: {}, 'emails/e1': null })).toThrow(/prefix/); + const patch: Record = { 'emails/e1/label': 'Office' }; + setInPatch(patch, 'emails', { e1: { address: 'a@x', label: 'Office' } }); + expect(patch).toEqual({ emails: { e1: { address: 'a@x', label: 'Office' } } }); + setInPatch(patch, 'emails/e1/address', 'b@x'); + expect(patch).toEqual({ emails: { e1: { address: 'b@x', label: 'Office' } } }); + assertPatchWellFormed(patch); + }); +}); + +describe('common/ids', () => { + it('mints keys Stalwart treats as plain keys', () => { + const mint = makeKeyMinter(seeded(7)); + const keys = Array.from({ length: 50 }, () => mint()); + for (const key of keys) { + expect(key).toMatch(/^b[0-9a-z]{8}$/); + expect(key).not.toMatch(/^k\d+$/); + } + expect(new Set(keys).size).toBe(keys.length); + const again = makeKeyMinter(() => 0); + expect(again(['b00000000'])).not.toBe('b00000000'); + }); + + it('builds v4 uuids and object refs', () => { + expect(uuidFrom(seeded(1))).toMatch(/^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/); + expect(objectRef('c', 'e9')).toBe('c/e9'); + expect(parseObjectRef('c/e9')).toEqual({ accountId: 'c', id: 'e9' }); + expect(parseObjectRef(pendingRef('u1'))).toBeNull(); + expect(pendingUidOf(pendingRef('urn:uuid:1'))).toBe('urn:uuid:1'); + expect(exceptionRef('c/e9', '2026-10-07T09:00:00')).toBe('c/e9#2026-10-07T09:00:00'); + }); +}); diff --git a/src/device-sync/__tests__/fake-jmap-server.test.ts b/src/device-sync/__tests__/fake-jmap-server.test.ts new file mode 100644 index 00000000..ceae4551 --- /dev/null +++ b/src/device-sync/__tests__/fake-jmap-server.test.ts @@ -0,0 +1,1155 @@ +// Tests for the fake JMAP server the device-sync engine tests run against: +// it has to answer like Stalwart v0.16.23 wherever the engine depends on it, +// so its own behaviour is pinned here. + +import { describe, expect, it } from 'vitest'; +import { JMAP_CALENDARS, JMAP_CONTACTS, JMAP_CORE, type JmapInvocation, type JmapPort } from '../types'; +import { FakeJmapServer, uuid5 } from './fakes/fake-jmap-server'; + +const USING = [JMAP_CORE, JMAP_CONTACTS, JMAP_CALENDARS]; + +type Args = Record; + +/** One method call in its own request: the response name and arguments. */ +async function call(port: JmapPort, name: string, args: Args): Promise<[string, Args]> { + const { methodResponses } = await port.request([[name, args, '0']], USING); + return [methodResponses[0][0], methodResponses[0][1] as Args]; +} + +function setup() { + const server = new FakeJmapServer(); + server.addAccount('a', { name: 'alice@example.com' }); + const book = server.addAddressBook('a', { name: 'Personal' }); + const cal = server.addCalendar('a', { name: 'Calendar' }); + return { server, port: server.port(), book, cal }; +} + +describe('FakeJmapServer', () => { + describe('session and ids', () => { + it('lists the accounts with their capabilities and the primary accounts', () => { + const server = new FakeJmapServer(); + server.addAccount('a', { name: 'alice@example.com' }); + server.addAccount('team', { name: 'Team', isPersonal: false, capabilities: ['contacts'] }); + + const session = server.port().session(); + + expect(session.username).toBe('alice@example.com'); + expect(session.primaryAccounts).toEqual({ [JMAP_CONTACTS]: 'a', [JMAP_CALENDARS]: 'a' }); + expect(session.accounts.team).toMatchObject({ name: 'Team', isPersonal: false, isReadOnly: false }); + expect(Object.keys(session.accounts.team.accountCapabilities)).toEqual([JMAP_CONTACTS]); + expect(Object.keys(session.accounts.a.accountCapabilities)).toEqual([JMAP_CONTACTS, JMAP_CALENDARS]); + expect(session.capabilities[JMAP_CORE]).toMatchObject({ + maxObjectsInGet: 500, + maxObjectsInSet: 500, + maxCallsInRequest: 16, + }); + expect(server.port('alice').session().username).toBe('alice'); + }); + + it('mints short ids per account and type and never reuses one', async () => { + const server = new FakeJmapServer(); + server.addAccount('a', { name: 'alice@example.com' }); + server.addAccount('b', { name: 'bob@example.com' }); + + expect(server.addAddressBook('a', { name: 'A' })).toBe('ab1'); + expect(server.addAddressBook('b', { name: 'B' })).toBe('ab1'); + expect(server.addCard('a', { uid: 'u1' })).toBe('c1'); + expect(server.addCard('b', { uid: 'u1' })).toBe('c1'); + const [, set] = await call(server.port(), 'ContactCard/set', { + accountId: 'a', + create: { n: { addressBookIds: { ab1: true } } }, + }); + expect(set.created).toEqual({ n: { id: 'c2' } }); + + server.serverDestroy('ContactCard', 'a', 'c2'); + expect(() => server.addCard('a', { id: 'c2' })).toThrow(/never reused/); + }); + + it('stamps with an injected clock', () => { + const t = Date.UTC(2026, 8, 27, 12, 0, 0); + const server = new FakeJmapServer({ now: () => t }); + server.addAccount('a', { name: 'alice@example.com' }); + server.addCalendar('a', { name: 'C' }); + + const id = server.addEvent('a', { title: 'x' }); + + expect(server.get('CalendarEvent', 'a', id)?.updated).toBe('2026-09-27T12:00:00Z'); + expect(() => server.setTime(0)).toThrow(); + }); + }); + + describe('AddressBook and Calendar', () => { + it('returns containers with Stalwart defaults and a computed isDefault', async () => { + const { server, port, book, cal } = setup(); + const shared = server.addAddressBook('a', { + name: 'Shared', + myRights: { mayRead: true, mayWrite: false, mayShare: false, mayDelete: false }, + }); + + const [, books] = await call(port, 'AddressBook/get', { accountId: 'a', ids: null }); + const [, cals] = await call(port, 'Calendar/get', { + accountId: 'a', + ids: [cal], + properties: ['name', 'defaultAlertsWithTime', 'nope'], + }); + + expect(books.list).toEqual([ + { + id: book, + name: 'Personal', + description: null, + sortOrder: 0, + isDefault: true, + isSubscribed: true, + shareWith: null, + myRights: { mayRead: true, mayWrite: true, mayShare: true, mayDelete: true }, + }, + expect.objectContaining({ id: shared, isDefault: false }), + ]); + expect(cals.list).toEqual([{ id: cal, name: 'Calendar', defaultAlertsWithTime: {}, nope: null }]); + }); + + it('forbids writes into an address book without mayWrite', async () => { + const { server, port } = setup(); + const shared = server.addAddressBook('a', { name: 'Shared', myRights: { mayRead: true, mayWrite: false } }); + const inShared = server.addCard('a', { uid: 'u', addressBookIds: { [shared]: true } }); + + const [, set] = await call(port, 'ContactCard/set', { + accountId: 'a', + create: { n: { addressBookIds: { [shared]: true } } }, + update: { [inShared]: { kind: 'org' } }, + }); + + expect(set.notCreated.n).toEqual({ + type: 'forbidden', + description: `You are not allowed to add contacts to address book ${shared}.`, + }); + expect(set.notUpdated[inShared]).toEqual({ + type: 'forbidden', + description: `You are not allowed to modify address book ${shared}.`, + }); + }); + }); + + describe('ContactCard', () => { + it('answers a create with only the id and returns the card normalised like Stalwart', async () => { + const { port, book } = setup(); + + const [, set] = await call(port, 'ContactCard/set', { + accountId: 'a', + create: { + new1: { + addressBookIds: { [book]: true }, + uid: 'urn:uuid:1', + name: { components: [{ kind: 'given', value: 'Jane' }, { kind: 'surname', value: 'Doe' }] }, + addresses: { home: { components: [{ kind: 'locality', value: 'Berlin' }], contexts: { private: true } } }, + titles: { t1: { name: 'CEO' } }, + anniversaries: { b: { kind: 'birth', date: { year: 1990, month: 5, day: 1 } } }, + media: { p: { kind: 'photo', uri: 'data:image/jpeg;base64,AAAA' } }, + }, + }, + }); + const [, get] = await call(port, 'ContactCard/get', { accountId: 'a', ids: ['c1'] }); + const [, update] = await call(port, 'ContactCard/set', { + accountId: 'a', + update: { c1: { 'name/full': 'J. Doe' } }, + }); + + expect(set.created).toEqual({ new1: { id: 'c1' } }); + expect(get.list).toEqual([ + { + id: 'c1', + addressBookIds: { [book]: true }, + '@type': 'Card', + version: '1.0', + uid: 'urn:uuid:1', + name: { + components: [{ kind: 'given', value: 'Jane' }, { kind: 'surname', value: 'Doe' }], + isOrdered: true, + full: 'Jane Doe', + }, + addresses: { + home: { components: [{ kind: 'locality', value: 'Berlin' }], contexts: { private: true }, isOrdered: true }, + }, + titles: { t1: { name: 'CEO', kind: 'title' } }, + anniversaries: { b: { kind: 'birth', date: { '@type': 'PartialDate', year: 1990, month: 5, day: 1 } } }, + media: { p: { kind: 'photo', uri: 'data:image/jpeg;base64,AAAA' } }, + }, + ]); + expect(update).toMatchObject({ updated: { c1: null } }); + expect(update).not.toHaveProperty('created'); + }); + + it('returns only the listed properties, plus the id', async () => { + const { server, port, book } = setup(); + const id = server.addCard('a', { uid: 'u1', kind: 'individual' }); + + const [, get] = await call(port, 'ContactCard/get', { + accountId: 'a', + ids: [id, 'missing'], + properties: ['uid', 'addressBookIds'], + }); + + expect(get.list).toEqual([{ id, uid: 'u1', addressBookIds: { [book]: true } }]); + expect(get.notFound).toEqual(['missing']); + expect(get.state).toBe(server.state('ContactCard', 'a')); + }); + + it('keeps map keys as written, slashes included, and resolves ~1 and ~0 in patch paths', async () => { + const { server, port } = setup(); + const id = server.addCard('a', { + emails: { + 'x/y': { address: 'a@example.com' }, + 'm~n': { address: 'b@example.com' }, + e3: { address: 'c@example.com' }, + }, + }); + + const [, set] = await call(port, 'ContactCard/set', { + accountId: 'a', + update: { [id]: { 'emails/x~1y/label': 'slash', 'emails/m~0n/label': 'tilde' } }, + }); + + expect(set.updated).toEqual({ [id]: null }); + expect(server.get('ContactCard', 'a', id)?.emails).toEqual({ + 'x/y': { address: 'a@example.com', label: 'slash' }, + 'm~n': { address: 'b@example.com', label: 'tilde' }, + e3: { address: 'c@example.com' }, + }); + }); + + it('fails a patch below a missing parent, also once the last entry of a map is gone', async () => { + const { server, port } = setup(); + const id = server.addCard('a', { uid: 'u1' }); + const update = async (patch: Args) => + (await call(port, 'ContactCard/set', { accountId: 'a', update: { [id]: patch } }))[1]; + + const first = await update({ 'emails/e1': { address: 'a@example.com' } }); + await update({ emails: { e1: { address: 'a@example.com' } } }); + await update({ emails: {} }); + const cardWithoutEmails = server.get('ContactCard', 'a', id); + const again = await update({ 'emails/e2/address': 'b@example.com' }); + const wildcard = await update({ 'emails/*/label': 'x' }); + + expect(first.notUpdated).toEqual({ + [id]: { type: 'invalidProperties', properties: ['emails/e1'], description: 'Patch operation failed.' }, + }); + expect(cardWithoutEmails).not.toHaveProperty('emails'); + expect(again.notUpdated[id]).toMatchObject({ properties: ['emails/e2/address'] }); + expect(wildcard.notUpdated[id]).toMatchObject({ description: 'Patch operation failed.' }); + }); + + it('reads an all-digit segment as an index that can replace a member but never add one', async () => { + const { server, port } = setup(); + const id = server.addCard('a', { + name: { components: [{ kind: 'given', value: 'Jane' }], full: 'Jane' }, + emails: { '7': { address: 'seven@example.com' } }, + }); + const update = async (patch: Args) => + (await call(port, 'ContactCard/set', { accountId: 'a', update: { [id]: patch } }))[1]; + + const replaced = await update({ 'name/components/0/value': 'Janet', 'emails/7/label': 'lucky' }); + const added = await update({ 'emails/8': { address: 'eight@example.com' } }); + const escaped = await update({ 'emails/a\\/b': { address: 'ab@example.com' } }); + + expect(replaced.updated).toEqual({ [id]: null }); + expect(added.notUpdated[id]).toMatchObject({ properties: ['emails/8'], description: 'Patch operation failed.' }); + expect(escaped.updated).toEqual({ [id]: null }); + expect(server.get('ContactCard', 'a', id)).toMatchObject({ + name: { components: [{ kind: 'given', value: 'Janet' }] }, + emails: { '7': { address: 'seven@example.com', label: 'lucky' }, 'a/b': { address: 'ab@example.com' } }, + }); + }); + + it('deletes a map entry or a property set to null', async () => { + const { server, port } = setup(); + const id = server.addCard('a', { + emails: { e1: { address: 'a@example.com' }, e2: { address: 'b@example.com' } }, + notes: { n1: { note: 'hi' } }, + }); + + await call(port, 'ContactCard/set', { accountId: 'a', update: { [id]: { 'emails/e1': null, notes: null } } }); + + const card = server.get('ContactCard', 'a', id); + expect(card?.emails).toEqual({ e2: { address: 'b@example.com' } }); + expect(card).not.toHaveProperty('notes'); + }); + + it('derives name.full once and keeps it through component patches until full is nulled', async () => { + const { server, port } = setup(); + const id = server.addCard('a', { + name: { components: [{ kind: 'given', value: 'Jane' }, { kind: 'surname', value: 'Doe' }] }, + }); + const separated = server.addCard('a', { + name: { + components: [ + { kind: 'surname', value: 'Doe' }, + { kind: 'separator', value: ', ' }, + { kind: 'given', value: 'Jane' }, + ], + }, + }); + const renamed = [{ kind: 'given', value: 'Janet' }, { kind: 'surname', value: 'Doe' }]; + const name = () => server.get('ContactCard', 'a', id)?.name; + + const derived = name(); + await call(port, 'ContactCard/set', { accountId: 'a', update: { [id]: { 'name/components': renamed } } }); + const stale = name(); + await call(port, 'ContactCard/set', { accountId: 'a', update: { [id]: { 'name/full': null } } }); + + expect(derived).toMatchObject({ full: 'Jane Doe', isOrdered: true }); + expect(stale).toMatchObject({ full: 'Jane Doe', components: renamed }); + expect(name()).toMatchObject({ full: 'Janet Doe' }); + expect(server.get('ContactCard', 'a', separated)?.name).toMatchObject({ full: 'Doe, Jane' }); + }); + + it('rejects an unknown top-level property and names it', async () => { + const { port, book } = setup(); + + const [, set] = await call(port, 'ContactCard/set', { + accountId: 'a', + create: { n: { addressBookIds: { [book]: true }, 'example.com:foo': 1 } }, + }); + + expect(set.notCreated.n).toEqual({ + type: 'invalidProperties', + properties: ['example.com:foo'], + description: 'Invalid property.', + }); + }); + + it('refuses a duplicate uid in the same address book and names the existing card', async () => { + const { server, port, book } = setup(); + const work = server.addAddressBook('a', { name: 'Work' }); + const existing = server.addCard('a', { uid: 'u1' }); + + const [, set] = await call(port, 'ContactCard/set', { + accountId: 'a', + create: { + same: { addressBookIds: { [book]: true }, uid: 'u1' }, + elsewhere: { addressBookIds: { [work]: true }, uid: 'u1' }, + }, + }); + + expect(set.notCreated).toEqual({ + same: { + type: 'invalidProperties', + properties: ['uid'], + description: `Contact with UID u1 already exists with id ${existing}.`, + }, + }); + expect(Object.keys(set.created)).toEqual(['elsewhere']); + }); + + it('checks uids against what was committed before the call, as Stalwart does', async () => { + const { port, book } = setup(); + + const [, set] = await call(port, 'ContactCard/set', { + accountId: 'a', + create: { + one: { addressBookIds: { [book]: true }, uid: 'u1' }, + two: { addressBookIds: { [book]: true }, uid: 'u1' }, + }, + }); + + expect(Object.keys(set.created)).toEqual(['one', 'two']); + }); + + it('lets a uid be stripped but not changed or added', async () => { + const { server, port } = setup(); + const withUid = server.addCard('a', { uid: 'u1' }); + const without = server.addCard('a', {}); + + const [, refused] = await call(port, 'ContactCard/set', { + accountId: 'a', + update: { [withUid]: { uid: 'u2' }, [without]: { uid: 'u3' } }, + }); + const [, stripped] = await call(port, 'ContactCard/set', { accountId: 'a', update: { [withUid]: { uid: null } } }); + + expect(refused.notUpdated[withUid]).toEqual({ + type: 'invalidProperties', + properties: ['uid'], + description: 'You cannot change the UID of a contact.', + }); + expect(refused.notUpdated[without]).toMatchObject({ properties: ['uid'] }); + expect(stripped.updated).toEqual({ [withUid]: null }); + expect(server.get('ContactCard', 'a', withUid)).not.toHaveProperty('uid'); + }); + }); + + describe('CalendarEvent', () => { + it('stamps updated on every write, inside overrides too', async () => { + const { server, port, cal } = setup(); + server.setTime(Date.UTC(2026, 8, 27, 10, 0, 0)); + + const [, created] = await call(port, 'CalendarEvent/set', { + accountId: 'a', + create: { + e: { + calendarIds: { [cal]: true }, + uid: 'ev-1', + title: 'Standup', + start: '2026-09-28T09:00:00', + timeZone: 'Europe/Berlin', + duration: 'PT15M', + recurrenceRule: { frequency: 'daily' }, + updated: '2000-01-01T00:00:00Z', + }, + }, + }); + const id = created.created.e.id; + const afterCreate = server.get('CalendarEvent', 'a', id); + server.advanceTime(60_000); + await call(port, 'CalendarEvent/set', { + accountId: 'a', + update: { + [id]: { + recurrenceOverrides: { + '2026-09-29T09:00:00': { title: 'Moved' }, + '2026-09-30T09:00:00': { excluded: true, title: 'ignored' }, + }, + }, + }, + }); + const afterOverride = server.get('CalendarEvent', 'a', id); + server.advanceTime(60_000); + await call(port, 'CalendarEvent/set', { accountId: 'a', update: { [id]: {} } }); + + expect(created.created.e).toEqual({ id }); + expect(afterCreate).toMatchObject({ + '@type': 'Event', + recurrenceRule: { frequency: 'daily' }, + updated: '2026-09-27T10:00:00Z', + }); + expect(afterOverride?.updated).toBe('2026-09-27T10:01:00Z'); + expect(afterOverride?.recurrenceOverrides).toEqual({ + '2026-09-29T09:00:00': { title: 'Moved', updated: '2026-09-27T10:01:00Z' }, + '2026-09-30T09:00:00': { excluded: true }, + }); + expect(server.get('CalendarEvent', 'a', id)?.updated).toBe('2026-09-27T10:02:00Z'); + }); + + it('leaves useDefaultAlerts out of properties: null and returns it when listed', async () => { + const { server, port, cal } = setup(); + const id = server.addEvent('a', { title: 'x', useDefaultAlerts: true, calendarIds: { [cal]: true } }); + + const [, all] = await call(port, 'CalendarEvent/get', { accountId: 'a', ids: [id], properties: null }); + const [, listed] = await call(port, 'CalendarEvent/get', { + accountId: 'a', + ids: [id], + properties: ['title', 'useDefaultAlerts'], + }); + + expect(all.list[0]).not.toHaveProperty('useDefaultAlerts'); + expect(all.list[0]).toMatchObject({ id, calendarIds: { [cal]: true }, isDraft: false, isOrigin: true, title: 'x' }); + expect(listed.list).toEqual([{ id, title: 'x', useDefaultAlerts: true }]); + }); + + it.each([ + ['recurrenceRules', [{ frequency: 'daily' }]], + ['excludedRecurrenceRule', { frequency: 'daily' }], + ['excludedRecurrenceRules', [{ frequency: 'daily' }]], + ['timeZones', {}], + ['progressUpdated', '2026-01-01T00:00:00Z'], + ['isOrigin', true], + ])('rejects %s on create and update', async (property, value) => { + const { server, port, cal } = setup(); + const id = server.addEvent('a', { title: 'x' }); + + const [, set] = await call(port, 'CalendarEvent/set', { + accountId: 'a', + create: { n: { calendarIds: { [cal]: true }, title: 'y', [property]: value } }, + update: { [id]: { [property]: value } }, + }); + + expect(set.notCreated.n).toMatchObject({ type: 'invalidProperties', properties: [property] }); + expect(set.notUpdated[id]).toMatchObject({ type: 'invalidProperties', properties: [property] }); + }); + + it('fails a recurrenceOverrides/ patch until the event has overrides', async () => { + const { server, port } = setup(); + const id = server.addEvent('a', { + title: 'x', + start: '2026-10-01T09:00:00', + recurrenceRule: { frequency: 'weekly' }, + }); + const pointer = 'recurrenceOverrides/2026-10-08T09:00:00'; + const update = async (patch: Args) => + (await call(port, 'CalendarEvent/set', { accountId: 'a', update: { [id]: patch } }))[1]; + + const refused = await update({ [pointer]: { excluded: true } }); + await update({ recurrenceOverrides: { '2026-10-08T09:00:00': { excluded: true } } }); + const accepted = await update({ 'recurrenceOverrides/2026-10-15T09:00:00': { excluded: true } }); + + expect(refused.notUpdated[id]).toEqual({ + type: 'invalidProperties', + properties: [pointer], + description: 'Patch operation failed.', + }); + expect(accepted.updated).toEqual({ [id]: null }); + expect(Object.keys(server.get('CalendarEvent', 'a', id)?.recurrenceOverrides as Args)).toEqual([ + '2026-10-08T09:00:00', + '2026-10-15T09:00:00', + ]); + }); + + it('refuses a duplicate uid anywhere in the account without naming the event', async () => { + const { server, port } = setup(); + const other = server.addCalendar('a', { name: 'Other' }); + server.addEvent('a', { uid: 'ev-1', title: 'x' }); + + const [, set] = await call(port, 'CalendarEvent/set', { + accountId: 'a', + create: { n: { calendarIds: { [other]: true }, uid: 'ev-1', title: 'y' } }, + }); + + expect(set.notCreated.n).toEqual({ + type: 'invalidProperties', + properties: ['uid'], + description: 'An event with UID ev-1 already exists.', + }); + }); + + it('generates a uid when a create has none', async () => { + const { server, port, cal } = setup(); + + const [, set] = await call(port, 'CalendarEvent/set', { + accountId: 'a', + create: { n: { calendarIds: { [cal]: true }, title: 'x' } }, + }); + + expect(server.get('CalendarEvent', 'a', set.created.n.id)?.uid).toMatch(/^[0-9a-f]{8}-[0-9a-f]{4}-4/); + }); + + it('returns tasks with @type Task', async () => { + const { server, port } = setup(); + const id = server.addEvent('a', { '@type': 'Task', title: 'Pay rent', due: '2026-10-01T00:00:00' }); + + const [, get] = await call(port, 'CalendarEvent/get', { accountId: 'a', ids: [id], properties: ['@type', 'title'] }); + + expect(get.list).toEqual([{ id, '@type': 'Task', title: 'Pay rent' }]); + }); + + it('records sendSchedulingMessages without sending anything', async () => { + const { server, port, cal } = setup(); + const id = server.addEvent('a', { title: 'x' }); + + await call(port, 'CalendarEvent/set', { + accountId: 'a', + sendSchedulingMessages: true, + create: { n: { calendarIds: { [cal]: true }, title: 'y' } }, + update: { [id]: { title: 'z' } }, + }); + await call(port, 'CalendarEvent/set', { accountId: 'a', destroy: [id] }); + + expect(server.scheduling.map(({ op }) => op)).toEqual(['create', 'update']); + }); + + it('makes the account the organizer of an event created with attendees', async () => { + const { server, port, cal } = setup(); + + const [, set] = await call(port, 'CalendarEvent/set', { + accountId: 'a', + create: { + n: { + calendarIds: { [cal]: true }, + title: 'Meet', + participants: { p1: { calendarAddress: 'mailto:bob@example.com', roles: { attendee: true } } }, + }, + }, + }); + + const event = server.get('CalendarEvent', 'a', set.created.n.id); + expect(event?.organizerCalendarAddress).toBe('mailto:alice@example.com'); + expect(event?.participants).toEqual({ + p1: { '@type': 'Participant', calendarAddress: 'mailto:bob@example.com', roles: { attendee: true } }, + [uuid5('mailto:alice@example.com')]: { + '@type': 'Participant', + calendarAddress: 'mailto:alice@example.com', + roles: { owner: true }, + }, + }); + }); + + it('copies the default alerts of the calendar into an event created with useDefaultAlerts', async () => { + const { server, port } = setup(); + const cal = server.addCalendar('a', { + name: 'With alerts', + defaultAlertsWithTime: { d1: { '@type': 'Alert', trigger: { '@type': 'OffsetTrigger', offset: '-PT15M' } } }, + }); + + const [, set] = await call(port, 'CalendarEvent/set', { + accountId: 'a', + create: { + n: { calendarIds: { [cal]: true }, title: 'x', useDefaultAlerts: true, alerts: { mine: { trigger: { offset: '-PT5M' } } } }, + }, + }); + + expect(server.get('CalendarEvent', 'a', set.created.n.id)?.alerts).toEqual({ + mine: { '@type': 'Alert', trigger: { '@type': 'OffsetTrigger', offset: '-PT5M' } }, + k2: { '@type': 'Alert', trigger: { '@type': 'OffsetTrigger', offset: '-PT15M' } }, + }); + }); + + it('returns at most maxObjectsInGet objects for ids: null', async () => { + const { server, port } = setup(); + for (let i = 0; i < 501; i++) server.addEvent('a', { title: `e${i}` }); + + const [, get] = await call(port, 'CalendarEvent/get', { accountId: 'a', ids: null, properties: ['title'] }); + + expect(get.list).toHaveLength(500); + expect(get.list[0]).toEqual({ id: 'ev1', title: 'e0' }); + }); + }); + + describe('/changes', () => { + it('compacts a window like Stalwart and reports a no-op patch as an update', async () => { + const { server, port } = setup(); + const kept = server.addCard('a', { uid: 'kept' }); + const doomed = server.addCard('a', { uid: 'doomed' }); + const since = server.state('ContactCard', 'a'); + + const created = server.addCard('a', { uid: 'created' }); + server.serverUpdate('ContactCard', 'a', created, { kind: 'individual' }); + const flash = server.addCard('a', { uid: 'flash' }); + await call(port, 'ContactCard/set', { accountId: 'a', destroy: [flash] }); + server.serverUpdate('ContactCard', 'a', doomed, { kind: 'individual' }); + await call(port, 'ContactCard/set', { accountId: 'a', destroy: [doomed] }); + await call(port, 'ContactCard/set', { accountId: 'a', update: { [kept]: {} } }); + const [, changes] = await call(port, 'ContactCard/changes', { accountId: 'a', sinceState: since }); + + expect(changes).toEqual({ + accountId: 'a', + oldState: since, + newState: server.state('ContactCard', 'a'), + hasMoreChanges: false, + created: [created], + updated: [kept], + destroyed: [doomed], + }); + }); + + it('reports a membership-only change as an update', async () => { + const { server, port, book } = setup(); + const work = server.addAddressBook('a', { name: 'Work' }); + const id = server.addCard('a', { uid: 'u' }); + const since = server.state('ContactCard', 'a'); + + await call(port, 'ContactCard/set', { accountId: 'a', update: { [id]: { [`addressBookIds/${work}`]: true } } }); + const [, changes] = await call(port, 'ContactCard/changes', { accountId: 'a', sinceState: since }); + + expect(changes.updated).toEqual([id]); + expect(server.get('ContactCard', 'a', id)?.addressBookIds).toEqual({ [book]: true, [work]: true }); + }); + + it('keeps address books and cards in one log but reports each type on its own', async () => { + const { server, port } = setup(); + const cardsSince = server.state('ContactCard', 'a'); + const booksSince = server.state('AddressBook', 'a'); + + const book2 = server.addAddressBook('a', { name: 'Second' }); + const card = server.addCard('a', { uid: 'u' }); + const [, books] = await call(port, 'AddressBook/changes', { accountId: 'a', sinceState: booksSince }); + const [, cards] = await call(port, 'ContactCard/changes', { accountId: 'a', sinceState: cardsSince }); + + expect(cardsSince).toBe('n'); + expect(booksSince).toMatch(/^s/); + expect(books).toMatchObject({ created: [book2], updated: [], destroyed: [], newState: server.state('AddressBook', 'a') }); + expect(cards).toMatchObject({ created: [card], updated: [], destroyed: [], newState: server.state('ContactCard', 'a') }); + expect(books.newState).not.toBe(cards.newState); + expect(server.state('CalendarEvent', 'a')).toBe('n'); + }); + + it('pages with maxChanges through intermediate states', async () => { + const { server, port } = setup(); + const ids = [1, 2, 3, 4, 5].map((n) => server.addCard('a', { uid: `u${n}` })); + + const pages: Args[] = []; + let state = 'n'; + let late = ''; + for (;;) { + const [, page] = await call(port, 'ContactCard/changes', { accountId: 'a', sinceState: state, maxChanges: 2 }); + pages.push(page); + state = page.newState; + if (!late) late = server.addCard('a', { uid: 'late' }); + if (!page.hasMoreChanges) break; + } + const [, rest] = await call(port, 'ContactCard/changes', { accountId: 'a', sinceState: state, maxChanges: 2 }); + + expect(pages.map((p) => p.created)).toEqual([ids.slice(0, 2), ids.slice(2, 4), ids.slice(4)]); + expect(pages.map((p) => p.hasMoreChanges)).toEqual([true, true, false]); + expect(pages[0].newState).toMatch(/^r/); + expect(pages[1].newState).toMatch(/^r/); + expect(pages[2].newState).toMatch(/^s/); + expect(rest).toMatchObject({ created: [late], hasMoreChanges: false, newState: server.state('ContactCard', 'a') }); + }); + + it('answers cannotCalculateChanges from a state older than the truncated log', async () => { + const { server, port } = setup(); + server.addCard('a', { uid: 'u1' }); + const old = server.state('ContactCard', 'a'); + server.addCard('a', { uid: 'u2' }); + const recent = server.state('ContactCard', 'a'); + const newest = server.addCard('a', { uid: 'u3' }); + const changesSince = async (sinceState: string) => + call(port, 'ContactCard/changes', { accountId: 'a', sinceState }); + + server.truncateChangeLog('a', 'contacts', 1); + const fromOld = await changesSince(old); + const fromRecent = await changesSince(recent); + server.truncateChangeLog('a', 'ContactCard'); + const fromRecentAgain = await changesSince(recent); + const current = server.state('ContactCard', 'a'); + const fromCurrent = await changesSince(current); + + expect(fromOld).toEqual(['error', { type: 'cannotCalculateChanges', description: 'Change log is truncated' }]); + expect(fromRecent[1].created).toEqual([newest]); + expect(fromRecentAgain[1].type).toBe('cannotCalculateChanges'); + expect(fromCurrent[1]).toMatchObject({ created: [], updated: [], destroyed: [], newState: current }); + }); + + it('answers cannotCalculateChanges for a state the log never reached', async () => { + const busy = new FakeJmapServer(); + busy.addAccount('a', { name: 'alice@example.com' }); + busy.addAddressBook('a', { name: 'Personal' }); + for (let i = 0; i < 10; i++) busy.addCard('a', { uid: `u${i}` }); + const { server, port } = setup(); + server.addCard('a', { uid: 'u' }); + + const [, error] = await call(port, 'ContactCard/changes', { + accountId: 'a', + sinceState: busy.state('ContactCard', 'a'), + }); + + expect(error).toEqual({ type: 'cannotCalculateChanges', description: 'Since state is invalid' }); + }); + }); + + describe('ifInState', () => { + it('refuses a /set unless ifInState is the current state', async () => { + const { server, port } = setup(); + const id = server.addCard('a', { uid: 'u' }); + const stale = server.state('ContactCard', 'a'); + server.serverUpdate('ContactCard', 'a', id, { kind: 'individual' }); + const current = server.state('ContactCard', 'a'); + + const refused = await call(port, 'ContactCard/set', { accountId: 'a', ifInState: stale, update: { [id]: { kind: 'org' } } }); + const unchanged = server.get('ContactCard', 'a', id)?.kind; + const [, accepted] = await call(port, 'ContactCard/set', { + accountId: 'a', + ifInState: current, + update: { [id]: { kind: 'org' } }, + }); + + expect(refused).toEqual(['error', { type: 'stateMismatch' }]); + expect(unchanged).toBe('individual'); + expect(accepted).toMatchObject({ oldState: current, updated: { [id]: null } }); + expect(accepted.newState).not.toBe(current); + expect(accepted.newState).toBe(server.state('ContactCard', 'a')); + }); + + it('sees a concurrent change made while the request is in flight', async () => { + const { server, port } = setup(); + const id = server.addCard('a', { uid: 'u' }); + const state = server.state('ContactCard', 'a'); + const stop = server.onBeforeRequest((request) => { + if (request.methods.includes('ContactCard/set')) server.serverUpdate('ContactCard', 'a', id, { kind: 'org' }); + }); + + const [, error] = await call(port, 'ContactCard/set', { + accountId: 'a', + ifInState: state, + update: { [id]: { kind: 'individual' } }, + }); + stop(); + + expect(error).toEqual({ type: 'stateMismatch' }); + }); + }); + + describe('uid index', () => { + it('finds a new object by uid only after the configured lag', async () => { + const { server, port, book } = setup(); + server.setUidIndexLag(1); + const [, set] = await call(port, 'ContactCard/set', { + accountId: 'a', + create: { n: { addressBookIds: { [book]: true }, uid: 'u1' } }, + }); + const id = set.created.n.id; + const byUid = { accountId: 'a', filter: { uid: 'u1', inAddressBook: book } }; + + const { methodResponses: next } = await port.request( + [ + ['ContactCard/query', byUid, 'uid'], + ['ContactCard/query', { accountId: 'a', filter: { inAddressBook: book } }, 'book'], + ], + USING, + ); + const [, later] = await call(port, 'ContactCard/query', byUid); + + expect(next[0][1].ids).toEqual([]); + expect(next[1][1].ids).toEqual([id]); + expect(later.ids).toEqual([id]); + }); + + it('finds objects by uid at once without a lag', async () => { + const { server, port, cal } = setup(); + const id = server.addEvent('a', { uid: 'ev-1', title: 'x' }); + + const [, query] = await call(port, 'CalendarEvent/query', { accountId: 'a', filter: { uid: 'ev-1', inCalendar: cal } }); + + expect(query.ids).toEqual([id]); + }); + }); + + describe('fault injection', () => { + it.each([ + ['network', 'NetworkError'], + ['timeout', 'RequestTimeoutError'], + ['auth', 'AuthenticationError'], + ['rateLimit', 'RateLimitError'], + ] as const)('fails the next request with %s before anything runs', async (kind, name) => { + const { server, port, book } = setup(); + server.failNextRequest(kind, { retryAfterMs: 7000 }); + const create: JmapInvocation[] = [ + ['ContactCard/set', { accountId: 'a', create: { n: { addressBookIds: { [book]: true } } } }, '0'], + ]; + + const error = await port.request(create, USING).catch((e: Error & { retryAfterMs?: number }) => e); + const cardsAfterFailure = server.all('ContactCard', 'a'); + const record = server.requests.at(-1); + await port.request(create, USING); + + expect(error).toBeInstanceOf(Error); + expect(error).toMatchObject({ name }); + if (kind === 'rateLimit') expect(error).toMatchObject({ retryAfterMs: 7000 }); + expect(cardsAfterFailure).toEqual([]); + expect(record).toMatchObject({ outcome: 'failed', fault: kind, responses: null }); + expect(server.all('ContactCard', 'a')).toHaveLength(1); + }); + + it('applies a request and then loses the response', async () => { + const { server, port, book } = setup(); + server.setUidIndexLag(2); + server.applyThenLoseResponse(); + const create: JmapInvocation[] = [ + ['ContactCard/set', { accountId: 'a', create: { n: { addressBookIds: { [book]: true }, uid: 'u1' } } }, '0'], + ]; + + await expect(port.request(create, USING)).rejects.toMatchObject({ name: 'NetworkError' }); + const [card] = server.all('ContactCard', 'a'); + const record = server.requests.at(-1); + // The retry protocol: the uid query can miss the card, the duplicate check cannot. + const [, query] = await call(port, 'ContactCard/query', { accountId: 'a', filter: { uid: 'u1', inAddressBook: book } }); + const { methodResponses } = await port.request(create, USING); + + expect(card).toMatchObject({ uid: 'u1' }); + expect(record).toMatchObject({ outcome: 'lost', fault: 'network' }); + expect(record?.responses?.[0][1]).toMatchObject({ created: { n: { id: card.id } } }); + expect(query.ids).toEqual([]); + expect((methodResponses[0][1] as Args).notCreated.n.description).toBe( + `Contact with UID u1 already exists with id ${card.id}.`, + ); + }); + + it('fails only a request that matches', async () => { + const { server, port, book } = setup(); + server.failNextRequest('network', { match: 'ContactCard/set' }); + + const get = await call(port, 'ContactCard/get', { accountId: 'a', ids: [] }); + const set = call(port, 'ContactCard/set', { accountId: 'a', create: { n: { addressBookIds: { [book]: true } } } }); + + expect(get[0]).toBe('ContactCard/get'); + await expect(set).rejects.toMatchObject({ name: 'NetworkError' }); + }); + + it('answers the next /set touching an object with an injected SetError, once', async () => { + const { server, port, book } = setup(); + const id = server.addCard('a', { uid: 'u1' }); + server.setErrorFor('ContactCard', 'a', id, { type: 'forbidden', description: 'read-only' }); + server.setErrorFor('ContactCard', 'a', 'u2', { type: 'overQuota' }); + const args = { + accountId: 'a', + update: { [id]: { kind: 'org' } }, + create: { n: { addressBookIds: { [book]: true }, uid: 'u2' } }, + }; + + const [, first] = await call(port, 'ContactCard/set', args); + const untouched = server.get('ContactCard', 'a', id); + const [, second] = await call(port, 'ContactCard/set', args); + + expect(first.notUpdated).toEqual({ [id]: { type: 'forbidden', description: 'read-only' } }); + expect(first.notCreated).toEqual({ n: { type: 'overQuota' } }); + expect(first.newState).toBe(first.oldState); + expect(untouched).not.toHaveProperty('kind'); + expect(second.updated).toEqual({ [id]: null }); + expect(Object.keys(second.created)).toEqual(['n']); + }); + + it('answers the next call of a method with an injected method error', async () => { + const { server, port } = setup(); + server.failNextMethod('ContactCard/set', { type: 'serverUnavailable' }); + + const first = await call(port, 'ContactCard/set', { accountId: 'a', update: {} }); + const second = await call(port, 'ContactCard/set', { accountId: 'a', update: {} }); + + expect(first).toEqual(['error', { type: 'serverUnavailable' }]); + expect(second[0]).toBe('ContactCard/set'); + }); + + it('serves blobs and can fail a download', async () => { + const { server, port } = setup(); + const blobId = server.addBlob('a', 'photo-bytes'); + server.failNextDownload('timeout'); + + await expect(port.downloadBlob('a', blobId)).rejects.toMatchObject({ name: 'RequestTimeoutError' }); + expect(new TextDecoder().decode(await port.downloadBlob('a', blobId))).toBe('photo-bytes'); + await expect(port.downloadBlob('a', 'missing')).rejects.toThrow('404'); + }); + }); + + describe('positional re-keying', () => { + it('gives card entries positional keys, as Stalwart does for a card written over CardDAV', async () => { + const { server, port } = setup(); + const id = server.addCard('a', { + emails: { work: { address: 'w@example.com' }, home: { address: 'h@example.com' } }, + organizations: { acme: { name: 'Acme' } }, + titles: { boss: { name: 'CEO', organizationId: 'acme' } }, + relatedTo: { 'urn:uuid:x': { relation: { friend: true } } }, + }); + const since = server.state('ContactCard', 'a'); + + server.rekeyPositionally('a', id); + const [, changes] = await call(port, 'ContactCard/changes', { accountId: 'a', sinceState: since }); + + expect(server.get('ContactCard', 'a', id)).toMatchObject({ + emails: { k1: { address: 'w@example.com' }, k2: { address: 'h@example.com' } }, + organizations: { k1: { name: 'Acme' } }, + titles: { k1: { name: 'CEO', organizationId: 'k1', kind: 'title' } }, + relatedTo: { 'urn:uuid:x': { relation: { friend: true } } }, + }); + expect(Object.keys(server.get('ContactCard', 'a', id)?.emails as Args)).toEqual(['k1', 'k2']); + expect(changes.updated).toEqual([id]); + }); + + it('keys event alerts by position and participants, locations and links by uuid5', () => { + const { server } = setup(); + const id = server.addEvent('a', { + title: 'x', + alerts: { a1: { trigger: { offset: '-PT5M' } }, a2: { trigger: { offset: '-PT1H' } } }, + participants: { p1: { calendarAddress: 'mailto:foo@example.com', locationId: 'l1' } }, + locations: { l1: { name: 'Room 1' } }, + mainLocationId: 'l1', + links: { x: { href: 'https://example.com/foo.pdf' } }, + }); + + server.rekeyPositionally('a', id); + + const event = server.get('CalendarEvent', 'a', id) as Args; + const room = uuid5('Room 1'); + expect(Object.keys(event.alerts)).toEqual(['k1', 'k2']); + expect(event.participants).toEqual({ + '59eb121c-e8f2-558a-9049-ef750a5976bd': { + '@type': 'Participant', + calendarAddress: 'mailto:foo@example.com', + locationId: room, + }, + }); + expect(Object.keys(event.locations)).toEqual([room]); + expect(event.mainLocationId).toBe(room); + expect(Object.keys(event.links)).toEqual(['245708bf-8e07-5d3b-a5da-2974a63c3b91']); + }); + + it('computes uuid5 keys as calcard does', () => { + expect(uuid5('mailto:hcabot@example.com')).toBe('0b235cc4-f04d-5fc4-98a3-c066650b3fbf'); + expect(uuid5('mailto:organizer@example.com')).toBe('251d3e9f-d83f-534c-8c45-c2896c75670c'); + expect(uuid5('mailto:bar@example.com')).toBe('5b6f4fa0-3695-53a7-904b-d0a6f8bc326f'); + }); + }); + + describe('result references', () => { + it('resolves #ids from /query and /changes results', async () => { + const { server, port, book } = setup(); + const c0 = server.addCard('a', { uid: 'u0' }); + const since = server.state('ContactCard', 'a'); + server.serverUpdate('ContactCard', 'a', c0, { kind: 'org' }); + const c1 = server.addCard('a', { uid: 'u1' }); + + const { methodResponses } = await port.request( + [ + ['ContactCard/query', { accountId: 'a', filter: { inAddressBook: book } }, 'q'], + ['ContactCard/get', { accountId: 'a', '#ids': { resultOf: 'q', name: 'ContactCard/query', path: '/ids' }, properties: ['uid'] }, 'g'], + ['ContactCard/changes', { accountId: 'a', sinceState: since }, 'c'], + ['ContactCard/get', { accountId: 'a', '#ids': { resultOf: 'c', name: 'ContactCard/changes', path: '/created' }, properties: ['uid'] }, 'gc'], + ['ContactCard/get', { accountId: 'a', '#ids': { resultOf: 'c', name: 'ContactCard/changes', path: '/updated' }, properties: ['uid'] }, 'gu'], + ], + USING, + ); + + expect(methodResponses[1][1].list).toEqual([{ id: c0, uid: 'u0' }, { id: c1, uid: 'u1' }]); + expect(methodResponses[3][1].list).toEqual([{ id: c1, uid: 'u1' }]); + expect(methodResponses[4][1].list).toEqual([{ id: c0, uid: 'u0' }]); + }); + + it('refuses a reference to a /set result or to a call that is not there', async () => { + const { port, book } = setup(); + + const { methodResponses } = await port.request( + [ + ['ContactCard/set', { accountId: 'a', create: { n: { addressBookIds: { [book]: true } } } }, 's'], + ['ContactCard/get', { accountId: 'a', '#ids': { resultOf: 's', name: 'ContactCard/set', path: '/created/n/id' } }, 'g'], + ['ContactCard/get', { accountId: 'a', '#ids': { resultOf: 'missing', name: 'ContactCard/query', path: '/ids' } }, 'm'], + ], + USING, + ); + + expect(methodResponses[0][0]).toBe('ContactCard/set'); + expect(methodResponses[1]).toEqual([ + 'error', + { type: 'invalidResultReference', description: 'Result reference to s#ContactCard/set not found.' }, + 'g', + ]); + expect(methodResponses[2][1]).toMatchObject({ type: 'invalidResultReference' }); + }); + }); + + describe('limits and method errors', () => { + it('rejects a request with too many calls before running any of them', async () => { + const { server, port, book } = setup(); + server.setLimits({ maxCallsInRequest: 2 }); + const calls: JmapInvocation[] = [ + ['ContactCard/set', { accountId: 'a', create: { n: { addressBookIds: { [book]: true } } } }, '0'], + ['Core/echo', {}, '1'], + ['Core/echo', {}, '2'], + ]; + + await expect(port.request(calls, USING)).rejects.toThrow(/maxCallsInRequest/); + + expect(port.session().capabilities[JMAP_CORE]).toMatchObject({ maxCallsInRequest: 2 }); + expect(server.all('ContactCard', 'a')).toEqual([]); + expect(server.requests.at(-1)?.outcome).toBe('rejected'); + }); + + it('counts how many requests were in flight at once', async () => { + const { server, port } = setup(); + + await call(port, 'Core/echo', {}); + await call(port, 'Core/echo', {}); + const sequential = server.maxInFlight; + await Promise.all([call(port, 'Core/echo', {}), call(port, 'Core/echo', {})]); + + expect(sequential).toBe(1); + expect(server.maxInFlight).toBe(2); + }); + + it('answers requestTooLarge for too many ids in /get or objects in /set', async () => { + const { server, port, book } = setup(); + server.setLimits({ maxObjectsInGet: 2, maxObjectsInSet: 1 }); + for (let i = 0; i < 3; i++) server.addCard('a', { uid: `u${i}` }); + + const get = await call(port, 'ContactCard/get', { accountId: 'a', ids: ['x', 'y', 'z'] }); + const set = await call(port, 'ContactCard/set', { + accountId: 'a', + create: { one: { addressBookIds: { [book]: true } }, two: { addressBookIds: { [book]: true } } }, + }); + const [, all] = await call(port, 'ContactCard/get', { accountId: 'a', ids: null }); + + expect(get).toEqual(['error', { type: 'requestTooLarge' }]); + expect(set).toEqual(['error', { type: 'requestTooLarge' }]); + expect(all.list).toHaveLength(2); + }); + + it('answers unknownMethod for unknown methods and for capabilities missing from using', async () => { + const { port } = setup(); + + const { methodResponses } = await port.request( + [ + ['Foo/get', {}, 'f'], + ['ContactCard/get', { accountId: 'a', ids: [] }, 'c'], + ['Core/echo', { hello: 1 }, 'e'], + ['CalendarEvent/get', { accountId: 'nobody', ids: [] }, 'x'], + ], + [JMAP_CORE, JMAP_CALENDARS], + ); + + expect(methodResponses[0]).toEqual(['error', expect.objectContaining({ type: 'unknownMethod' }), 'f']); + expect(methodResponses[1][1]).toMatchObject({ type: 'unknownMethod' }); + expect(methodResponses[2]).toEqual(['Core/echo', { hello: 1 }, 'e']); + expect(methodResponses[3]).toEqual(['error', { type: 'accountNotFound' }, 'x']); + }); + }); + + describe('/query', () => { + it('filters by address book and pages by position and limit, in id order', async () => { + const { server, port, book } = setup(); + const other = server.addAddressBook('a', { name: 'Other' }); + const ids = Array.from({ length: 7 }, (_, i) => server.addCard('a', { uid: `u${i}` })); + server.addCard('a', { uid: 'elsewhere', addressBookIds: { [other]: true } }); + const page = async (position: number) => + ( + await call(port, 'ContactCard/query', { + accountId: 'a', + filter: { inAddressBook: book }, + position, + limit: 3, + calculateTotal: true, + }) + )[1]; + + const first = await page(0); + const second = await page(3); + const third = await page(6); + server.setLimits({ maxQueryResults: 4 }); + const [, capped] = await call(port, 'ContactCard/query', { accountId: 'a', limit: 10 }); + + expect(first).toMatchObject({ ids: ids.slice(0, 3), position: 0, total: 7, limit: 3 }); + expect(first.queryState).toBe(server.state('ContactCard', 'a')); + expect(second.ids).toEqual(ids.slice(3, 6)); + expect(third.ids).toEqual(ids.slice(6)); + expect(capped).toMatchObject({ ids: ids.slice(0, 4), limit: 4 }); + }); + + it('refuses a filter it does not emulate', async () => { + const { port } = setup(); + + const [, error] = await call(port, 'ContactCard/query', { accountId: 'a', filter: { text: 'x' } }); + + expect(error.type).toBe('unsupportedFilter'); + }); + }); + + describe('edits by another client', () => { + it('destroys an address book with the cards only it holds', async () => { + const { server, port, book } = setup(); + const other = server.addAddressBook('a', { name: 'Other' }); + const only = server.addCard('a', { uid: 'only' }); + const both = server.addCard('a', { uid: 'both', addressBookIds: { [book]: true, [other]: true } }); + const cardsSince = server.state('ContactCard', 'a'); + const booksSince = server.state('AddressBook', 'a'); + + server.serverDestroy('AddressBook', 'a', book); + const [, cards] = await call(port, 'ContactCard/changes', { accountId: 'a', sinceState: cardsSince }); + const [, books] = await call(port, 'AddressBook/changes', { accountId: 'a', sinceState: booksSince }); + + expect(cards).toMatchObject({ created: [], updated: [both], destroyed: [only] }); + expect(books).toMatchObject({ created: [], updated: [], destroyed: [book] }); + expect(server.get('ContactCard', 'a', both)?.addressBookIds).toEqual({ [other]: true }); + expect(server.get('AddressBook', 'a', other)?.isDefault).toBe(true); + }); + + it('patches with JSON Pointers and stamps events unless the patch sets updated', () => { + const { server } = setup(); + server.setTime(Date.UTC(2026, 0, 2)); + const id = server.addEvent('a', { title: 'x', locations: { l1: { name: 'Old' } } }); + + server.advanceTime(1000); + server.serverUpdate('CalendarEvent', 'a', id, { 'locations/l1/name': 'New' }); + const stamped = server.get('CalendarEvent', 'a', id); + server.serverUpdate('CalendarEvent', 'a', id, { title: 'y', updated: '2020-01-01T00:00:00Z' }); + + expect(stamped).toMatchObject({ locations: { l1: { name: 'New' } }, updated: '2026-01-02T00:00:01Z' }); + expect(server.get('CalendarEvent', 'a', id)?.updated).toBe('2020-01-01T00:00:00Z'); + expect(() => server.serverUpdate('CalendarEvent', 'a', id, { 'alerts/k1': {} })).toThrow(/Patch operation failed/); + }); + }); +}); diff --git a/src/device-sync/__tests__/fake-provider.test.ts b/src/device-sync/__tests__/fake-provider.test.ts new file mode 100644 index 00000000..0a83ab5b --- /dev/null +++ b/src/device-sync/__tests__/fake-provider.test.ts @@ -0,0 +1,293 @@ +import { describe, expect, it } from 'vitest'; +import { + Attendees, + Calendars, + Data, + Events, + MimeType, + RawContacts, + StructuredName, + StructuredPostal, +} from '../android-columns'; +import { CALENDAR_AUTHORITY, CONTACTS_AUTHORITY, type ProviderOp } from '../types'; +import { FakeDeviceProviders, aospDurationValid, aospRruleValid } from './fakes/fake-provider'; +import { compileWhere } from './fakes/fake-sql'; + +const ME = 'usera@example.org'; +const OTHER = 'someone@example.org'; + +async function contactWithEmail(fake: FakeDeviceProviders, account = ME) { + const port = fake.port(account, CONTACTS_AUTHORITY); + const res = await port.applyBatch([ + { op: 'insert', table: 'raw_contacts', values: { [RawContacts.SOURCE_ID]: 'c/e1' } }, + { + op: 'insert', + table: 'data', + values: { [Data.MIMETYPE]: MimeType.EMAIL, [Data.DATA1]: 'a@example.org', [Data.DATA_SYNC1]: 'emails:e1' }, + refs: { [Data.RAW_CONTACT_ID]: 0 }, + }, + ]); + if (!res.ok) throw new Error(res.message); + return { port, rawId: res.results[0].id!, dataId: res.results[1].id! }; +} + +describe('fake-sql', () => { + const row = (r: Record) => (c: string) => r[c] ?? null; + + it('evaluates the documented subset with SQL precedence and NULL rules', () => { + const where = compileWhere('dirty = 1 OR deleted = 1 AND sourceid IS NOT NULL', []); + expect(where(row({ dirty: 0, deleted: 1, sourceid: null }))).toBe(false); + expect(where(row({ dirty: 1, deleted: 0, sourceid: null }))).toBe(true); + expect(compileWhere('x IN (?, ?, 3)', ['1', 2])(row({ x: 3 }))).toBe(true); + expect(compileWhere('x NOT IN (?)', ['1'])(row({ x: null }))).toBe(false); + expect(compileWhere("mimetype LIKE '%/email%'")(row({ mimetype: 'vnd.android.cursor.item/email_v2' }))).toBe(true); + expect(compileWhere('version = ?', ['2'])(row({ version: 2 }))).toBe(true); + expect(compileWhere('NOT (a = 1)')(row({ a: null }))).toBe(true); + expect(() => compileWhere('a = ?', [])).toThrow(/placeholders/); + }); +}); + +describe('FakeDeviceProviders: contacts', () => { + it('scopes every read and write to the port account', async () => { + const fake = new FakeDeviceProviders(); + const mine = await contactWithEmail(fake, ME); + const theirs = await contactWithEmail(fake, OTHER); + const rows = await mine.port.query({ table: 'raw_contacts', columns: [RawContacts._ID] }); + expect(rows.rows).toEqual([[mine.rawId]]); + const data = await mine.port.query({ table: 'data', columns: [Data._ID] }); + expect(data.rows).toEqual([[mine.dataId]]); + const denied = await mine.port.applyBatch([{ op: 'update', table: 'raw_contacts', id: theirs.rawId, values: { [RawContacts.SYNC1]: 'x' } }]); + expect(denied).toMatchObject({ ok: false, reason: 'scope' }); + const orphan = await mine.port.applyBatch([ + { op: 'insert', table: 'data', values: { [Data.RAW_CONTACT_ID]: theirs.rawId, [Data.MIMETYPE]: MimeType.NOTE } }, + ]); + expect(orphan).toMatchObject({ ok: false, reason: 'scope' }); + await expect(mine.port.query({ table: 'events', columns: ['_id'] })).rejects.toThrow(/scope/); + }); + + it('never sets DIRTY for the sync adapter but does for app edits, and only apps soft-delete', async () => { + const fake = new FakeDeviceProviders(); + const { rawId, dataId } = await contactWithEmail(fake); + expect(fake.row('raw_contacts', rawId)![RawContacts.DIRTY]).toBe(0); + fake.user.updateData(dataId, { [Data.DATA1]: 'b@example.org' }); + expect(fake.row('raw_contacts', rawId)![RawContacts.DIRTY]).toBe(1); + expect(fake.row('data', dataId)![Data.DATA_SYNC1]).toBe('emails:e1'); + fake.user.deleteContact(rawId); + expect(fake.row('raw_contacts', rawId)).toMatchObject({ [RawContacts.DELETED]: 1, [RawContacts.DIRTY]: 1 }); + const port = fake.port(ME, CONTACTS_AUTHORITY); + await port.applyBatch([{ op: 'delete', table: 'raw_contacts', id: rawId }]); + expect(fake.row('raw_contacts', rawId)).toBeUndefined(); + expect(fake.row('data', dataId)).toBeUndefined(); + }); + + it('bumps VERSION for data writes by anyone but not for DIRTY, SOURCE_ID or SYNC columns', async () => { + const fake = new FakeDeviceProviders(); + const { port, rawId, dataId } = await contactWithEmail(fake); + const version = () => fake.row('raw_contacts', rawId)![RawContacts.VERSION]; + expect(version()).toBe(2); // one bump for the batch's data inserts + await port.applyBatch([ + { op: 'update', table: 'raw_contacts', id: rawId, values: { [RawContacts.DIRTY]: 0, [RawContacts.SYNC2]: '{}', [RawContacts.SOURCE_ID]: 'c/e2' } }, + ]); + expect(version()).toBe(2); + await port.applyBatch([{ op: 'update', table: 'data', id: dataId, values: { [Data.DATA_SYNC3]: '{}' } }]); + expect(version()).toBe(3); + fake.user.star(rawId); + expect(version()).toBe(3); + expect(fake.row('raw_contacts', rawId)![RawContacts.DIRTY]).toBe(1); + }); + + it('applies a batch atomically and fails it on a stale VERSION assert', async () => { + const fake = new FakeDeviceProviders(); + const { port, rawId, dataId } = await contactWithEmail(fake); + fake.user.updateData(dataId, { [Data.DATA1]: 'edited@example.org' }); + const clear: ProviderOp[] = [ + { op: 'assert', table: 'raw_contacts', id: rawId, values: { [RawContacts.VERSION]: 2 } }, + { op: 'update', table: 'raw_contacts', id: rawId, values: { [RawContacts.DIRTY]: 0 } }, + ]; + expect(await port.applyBatch(clear)).toMatchObject({ ok: false, reason: 'assert' }); + expect(fake.row('raw_contacts', rawId)![RawContacts.DIRTY]).toBe(1); + const current = fake.row('raw_contacts', rawId)![RawContacts.VERSION] as number; + const ok = await port.applyBatch([ + { op: 'assert', table: 'raw_contacts', id: rawId, values: { [RawContacts.VERSION]: current } }, + { op: 'update', table: 'raw_contacts', id: rawId, values: { [RawContacts.DIRTY]: 0 } }, + { op: 'syncState', value: '{"v":1}' }, + ]); + expect(ok.ok).toBe(true); + expect(fake.readSyncState(ME, CONTACTS_AUTHORITY)).toBe('{"v":1}'); + }); + + it('runs a hook right before the next batch, for concurrent-edit tests', async () => { + const fake = new FakeDeviceProviders(); + const { port, rawId, dataId } = await contactWithEmail(fake); + const v = fake.row('raw_contacts', rawId)![RawContacts.VERSION] as number; + fake.beforeNextBatch(() => fake.user.updateData(dataId, { [Data.DATA1]: 'late@example.org' })); + const res = await port.applyBatch([ + { op: 'assert', table: 'raw_contacts', id: rawId, values: { [RawContacts.VERSION]: v } }, + { op: 'update', table: 'raw_contacts', id: rawId, values: { [RawContacts.DIRTY]: 0 } }, + ]); + expect(res.ok).toBe(false); + }); + + it('refuses 500 contacts ops without a yield point and oversized batches', async () => { + const fake = new FakeDeviceProviders({ maxBatchBytes: 50_000 }); + const port = fake.port(ME, CONTACTS_AUTHORITY); + const ops: ProviderOp[] = Array.from({ length: 500 }, () => ({ op: 'insert', table: 'raw_contacts', values: {} }) as ProviderOp); + expect(await port.applyBatch(ops)).toMatchObject({ ok: false, reason: 'provider' }); + ops[250] = { ...ops[250], yieldAllowed: true }; + expect((await port.applyBatch(ops)).ok).toBe(true); + const big = await port.applyBatch([{ op: 'insert', table: 'raw_contacts', values: { [RawContacts.SYNC2]: 'x'.repeat(60_000) } }]); + expect(big).toMatchObject({ ok: false, reason: 'tooLarge' }); + }); + + it('normalises names and addresses like ContactsProvider', async () => { + const fake = new FakeDeviceProviders(); + const { port, rawId } = await contactWithEmail(fake); + const res = await port.applyBatch([ + { op: 'insert', table: 'data', values: { [Data.RAW_CONTACT_ID]: rawId, [Data.MIMETYPE]: MimeType.STRUCTURED_NAME, [StructuredName.DISPLAY_NAME]: 'Ada Lovelace' } }, + { op: 'insert', table: 'data', values: { [Data.RAW_CONTACT_ID]: rawId, [Data.MIMETYPE]: MimeType.STRUCTURED_POSTAL, [StructuredPostal.CITY]: 'London', [StructuredPostal.COUNTRY]: 'UK' } }, + ]); + if (!res.ok) throw new Error(res.message); + expect(fake.row('data', res.results[0].id!)).toMatchObject({ [StructuredName.GIVEN_NAME]: 'Ada', [StructuredName.FAMILY_NAME]: 'Lovelace' }); + expect(fake.row('data', res.results[1].id!)![StructuredPostal.FORMATTED_ADDRESS]).toBe('London, UK'); + }); + + it('stores a photo written as a blob and serves it to readPhoto', async () => { + const fake = new FakeDeviceProviders(); + const { port, rawId } = await contactWithEmail(fake); + const res = await port.applyBatch([ + { op: 'insert', table: 'data', values: { [Data.RAW_CONTACT_ID]: rawId, [Data.MIMETYPE]: MimeType.PHOTO, [Data.DATA15]: { b64: 'AAAA' } } }, + ]); + expect(res.ok).toBe(true); + const photo = await port.readPhoto(rawId, 512); + expect(photo?.jpegBase64).toBe('AAAA'); + expect(photo?.fileId).toEqual(expect.any(Number)); + const rows = await port.query({ table: 'data', columns: [Data.DATA14, Data.DATA15], where: `${Data.MIMETYPE} = ?`, args: [MimeType.PHOTO] }); + expect(rows.rows[0]).toEqual([photo!.fileId, null]); + }); + + it('models Fossify saves: rows re-inserted without DATA_SYNC', async () => { + const fake = new FakeDeviceProviders(); + const { rawId, dataId } = await contactWithEmail(fake); + fake.user.fossifySave(rawId, null, [{ [Data.MIMETYPE]: MimeType.EMAIL, [Data.DATA1]: 'a@example.org' }]); + expect(fake.row('data', dataId)).toBeUndefined(); + const emails = fake.rows('data', `${Data.RAW_CONTACT_ID} = ?`, [rawId]); + expect(emails).toHaveLength(1); + expect(emails[0][Data.DATA_SYNC1]).toBeNull(); + expect(fake.row('raw_contacts', rawId)![RawContacts.DIRTY]).toBe(1); + }); +}); + +describe('FakeDeviceProviders: calendar', () => { + async function calendar(fake: FakeDeviceProviders) { + const port = fake.port(ME, CALENDAR_AUTHORITY); + const res = await port.applyBatch([ + { + op: 'insert', + table: 'calendars', + values: { + [Calendars._SYNC_ID]: 'c/b', + [Calendars.NAME]: 'Personal', + [Calendars.CALENDAR_DISPLAY_NAME]: 'Personal', + [Calendars.CALENDAR_COLOR]: -16776961, + [Calendars.CALENDAR_ACCESS_LEVEL]: 700, + [Calendars.OWNER_ACCOUNT]: ME, + [Calendars.SYNC_EVENTS]: 1, + }, + }, + ]); + if (!res.ok) throw new Error(res.message); + return { port, calendarId: res.results[0].id! }; + } + + it('requires the calendar columns the provider requires', async () => { + const fake = new FakeDeviceProviders(); + const port = fake.port(ME, CALENDAR_AUTHORITY); + const res = await port.applyBatch([{ op: 'insert', table: 'calendars', values: { [Calendars.NAME]: 'x' } }]); + expect(res).toMatchObject({ ok: false, reason: 'provider' }); + }); + + it('enforces recurring/exception column rules, all-day fixes and the NULL-status crash', async () => { + const fake = new FakeDeviceProviders(); + const { port, calendarId } = await calendar(fake); + const base = { [Events.CALENDAR_ID]: calendarId, [Events.EVENT_TIMEZONE]: 'UTC', [Events.STATUS]: 1 }; + const master = await port.applyBatch([ + { op: 'insert', table: 'events', values: { ...base, [Events._SYNC_ID]: 'c/e1', [Events.DTSTART]: 1_000, [Events.DTEND]: 5_000, [Events.RRULE]: 'FREQ=DAILY', [Events.DURATION]: 'P3600S' } }, + ]); + if (!master.ok) throw new Error(master.message); + const m = fake.row('events', master.results[0].id!)!; + expect(m[Events.DTEND]).toBeNull(); + expect(m[Events.ORGANIZER]).toBe(ME); + expect(await port.applyBatch([{ op: 'insert', table: 'events', values: { ...base, [Events.DTSTART]: 0, [Events.RRULE]: 'FREQ=DAILY' } }])).toMatchObject({ ok: false }); + expect(await port.applyBatch([{ op: 'insert', table: 'events', values: { ...base, [Events.DTSTART]: 0, [Events.RRULE]: 'FREQ=YEARLY;RSCALE=CHINESE', [Events.DURATION]: 'P1D' } }])).toMatchObject({ ok: false }); + expect(await port.applyBatch([{ op: 'insert', table: 'events', values: { ...base, [Events.ALL_DAY]: 1, [Events.DTSTART]: 0, [Events.RRULE]: 'FREQ=DAILY', [Events.DURATION]: 'PT0S' } }])).toMatchObject({ ok: false }); + const allDay = await port.applyBatch([ + { op: 'insert', table: 'events', values: { ...base, [Events.ALL_DAY]: 1, [Events.DTSTART]: 86_400_000 + 3_600_000, [Events.RRULE]: 'FREQ=DAILY', [Events.DURATION]: 'P86400S' } }, + ]); + if (!allDay.ok) throw new Error(allDay.message); + expect(fake.row('events', allDay.results[0].id!)).toMatchObject({ [Events.DTSTART]: 86_400_000, [Events.DURATION]: 'P1D' }); + expect(await port.applyBatch([{ op: 'update', table: 'events', id: master.results[0].id!, values: { [Events.STATUS]: null } }])).toMatchObject({ ok: false }); + expect(await port.applyBatch([{ op: 'update', table: 'events', id: master.results[0].id!, values: { [Events.SELF_ATTENDEE_STATUS]: 1 } }])).toMatchObject({ ok: false }); + }); + + it('links exceptions to their master and keeps them when a sync adapter deletes the master', async () => { + const fake = new FakeDeviceProviders(); + const { port, calendarId } = await calendar(fake); + const res = await port.applyBatch([ + { op: 'insert', table: 'events', values: { [Events.CALENDAR_ID]: calendarId, [Events._SYNC_ID]: 'c/e1', [Events.DTSTART]: 0, [Events.RRULE]: 'FREQ=DAILY', [Events.DURATION]: 'P3600S', [Events.EVENT_TIMEZONE]: 'UTC', [Events.STATUS]: 1 } }, + { op: 'insert', table: 'events', values: { [Events.CALENDAR_ID]: calendarId, [Events.ORIGINAL_SYNC_ID]: 'c/e1', [Events.ORIGINAL_INSTANCE_TIME]: 86_400_000, [Events.DTSTART]: 86_400_000, [Events.DTEND]: 90_000_000, [Events.EVENT_TIMEZONE]: 'UTC', [Events.STATUS]: 1 } }, + ]); + if (!res.ok) throw new Error(res.message); + const [masterId, exceptionId] = res.results.map((r) => r.id!); + expect(fake.row('events', exceptionId)![Events.ORIGINAL_ID]).toBe(masterId); + await port.applyBatch([{ op: 'update', table: 'events', id: masterId, values: { [Events._SYNC_ID]: 'c/e9' } }]); + expect(fake.row('events', exceptionId)![Events.ORIGINAL_SYNC_ID]).toBe('c/e9'); + await port.applyBatch([{ op: 'delete', table: 'events', id: masterId }]); + expect(fake.row('events', exceptionId)).toBeDefined(); + }); + + it('hard-deletes app-deleted events without _SYNC_ID and soft-deletes synced ones', async () => { + const fake = new FakeDeviceProviders(); + const { calendarId } = await calendar(fake); + const local = fake.user.insertEvent(calendarId, { [Events.DTSTART]: 0, [Events.DTEND]: 1000, [Events.EVENT_TIMEZONE]: 'UTC', [Events.TITLE]: 'x', [Events.STATUS]: 1 }); + expect(fake.row('events', local)).toMatchObject({ [Events.DIRTY]: 1, [Events.MUTATORS]: expect.any(String) }); + fake.user.deleteEvent(local); + expect(fake.row('events', local)).toBeUndefined(); + const synced = fake.user.insertEvent(calendarId, { [Events.DTSTART]: 0, [Events.DTEND]: 1000, [Events.EVENT_TIMEZONE]: 'UTC', [Events._SYNC_ID]: '~pending/u1', [Events.STATUS]: 1 }, { reminders: [{ minutes: 10, method: 1 }] }); + fake.user.deleteEvent(synced); + expect(fake.row('events', synced)).toMatchObject({ [Events.DELETED]: 1, [Events.DIRTY]: 1 }); + expect(fake.rows('reminders', 'event_id = ?', [synced])).toHaveLength(0); + }); + + it('models CONTENT_EXCEPTION_URI: only the exception is dirty; a split clones _SYNC_ID', async () => { + const fake = new FakeDeviceProviders(); + const { port, calendarId } = await calendar(fake); + const res = await port.applyBatch([ + { op: 'insert', table: 'events', values: { [Events.CALENDAR_ID]: calendarId, [Events._SYNC_ID]: 'c/e1', [Events.UID_2445]: 'u1', [Events.DTSTART]: 0, [Events.RRULE]: 'FREQ=DAILY', [Events.DURATION]: 'P3600S', [Events.EVENT_TIMEZONE]: 'UTC', [Events.STATUS]: 1 } }, + { op: 'insert', table: 'attendees', values: { [Attendees.ATTENDEE_EMAIL]: ME, [Attendees.ATTENDEE_STATUS]: 3 }, refs: { event_id: 0 } }, + ]); + if (!res.ok) throw new Error(res.message); + const masterId = res.results[0].id!; + const ex = fake.user.exceptionViaUri(masterId, 86_400_000, { [Events.SELF_ATTENDEE_STATUS]: 1 }); + expect(fake.row('events', masterId)![Events.DIRTY]).toBe(0); + expect(fake.row('events', ex)).toMatchObject({ [Events.DIRTY]: 1, [Events._SYNC_ID]: null, [Events.UID_2445]: 'u1', [Events.ORIGINAL_SYNC_ID]: 'c/e1' }); + expect(fake.rows('attendees', 'event_id = ?', [ex])[0][Attendees.ATTENDEE_STATUS]).toBe(1); + const split = fake.user.splitViaUri(masterId, 5 * 86_400_000, '20260101T000000Z', {}); + expect(fake.row('events', split)![Events._SYNC_ID]).toBe('c/e1'); + expect(fake.row('events', masterId)![Events.RRULE]).toContain('UNTIL=20260101T000000Z'); + expect(fake.row('events', masterId)![Events.DIRTY]).toBe(0); + }); +}); + +describe('AOSP format checks', () => { + it('accepts the Duration forms calendarcommon2 parses', () => { + for (const ok of ['P1D', 'PT1H30M', 'P1DT2H', 'P2W', 'P1W2D', '-PT15M', 'P3600S', 'P0DT1H30M0S']) expect(aospDurationValid(ok)).toBe(true); + for (const bad of ['1D', 'P1Y', 'PT1.5S', 'P', '']) expect(aospDurationValid(bad)).toBe(false); + }); + + it('rejects RRULE parts EventRecurrence does not know', () => { + expect(aospRruleValid('FREQ=MONTHLY;BYDAY=MO;BYSETPOS=-1;WKST=SU;X-FOO=1')).toBe(true); + expect(aospRruleValid('RRULE:FREQ=DAILY')).toBe(false); + expect(aospRruleValid('FREQ=YEARLY;RSCALE=GREGORIAN;SKIP=OMIT')).toBe(false); + expect(aospRruleValid('COUNT=2')).toBe(false); + }); +}); diff --git a/src/device-sync/__tests__/fakes/fake-jmap-server.ts b/src/device-sync/__tests__/fakes/fake-jmap-server.ts new file mode 100644 index 00000000..015a161d --- /dev/null +++ b/src/device-sync/__tests__/fakes/fake-jmap-server.ts @@ -0,0 +1,2510 @@ +/** + * An in-memory JMAP server for the device-sync engine tests (#34), standing + * in for Stalwart v0.16.23 behind a `JmapPort`. It holds several accounts (a + * personal one plus shared ones), each with address books, contact cards, + * calendars and calendar events, and answers the method calls the engine + * makes the way Stalwart does: per-type states over one change log per + * collection, `/changes` compaction and paging with intermediate states, JSON + * Pointer patches, the uid rules, `ifInState`, result references, limits and + * the normalisation Stalwart applies on write. docs/device-sync.md describes + * the engine; each Stalwart rule below names where it comes from ("verified": + * checked against the live server; otherwise read from the Stalwart and + * calcard sources). + * + * Tests also drive it from the outside: seed helpers, edits by another + * client, positional re-keying (an object written over DAV without PROP-IDs), + * and fault injection (transport errors, a lost response, a truncated change + * log, a lagging uid index, per-object SetErrors, limits). + * + * Not emulated: ghost entries left by `map/key: null` (deleted cleanly + * here), partial overrides from deep pointers, per-instance (synthetic) ids, + * `utcStart`/`utcEnd`, `expandRecurrences`, query anchors and text filters, + * DST gaps and unknown time zones, duration rewriting, object sizes and + * quotas (inject them with `setErrorFor`), and the state jump after a cache + * rebuild. Method names Stalwart knows but the fake does not implement + * answer `unknownMethod` with a description saying so. + * + * Pure TypeScript: no React Native, expo or node imports. + */ + +import type { + JmapAccountView, + JmapInvocation, + JmapPort, + JmapResponse, + JmapSessionView, +} from '../../types'; +import { JMAP_CALENDARS, JMAP_CONTACTS, JMAP_CORE } from '../../types'; + +// ─── Public types ────────────────────────────────────── + +export type DataType = 'AddressBook' | 'ContactCard' | 'Calendar' | 'CalendarEvent'; +export type ItemType = 'ContactCard' | 'CalendarEvent'; +export type ContainerType = 'AddressBook' | 'Calendar'; + +/** + * A change log. Stalwart keeps one per collection: AddressBook and + * ContactCard share one, Calendar and CalendarEvent the other, and each + * `/changes` call filters it to its own type. + */ +export type TypeGroup = 'contacts' | 'calendars'; + +export type FaultKind = 'network' | 'timeout' | 'auth' | 'rateLimit'; + +export interface SetErrorShape { + type: string; + description?: string; + properties?: string[]; + [extra: string]: unknown; +} + +export interface FakeLimits { + maxObjectsInGet: number; + maxObjectsInSet: number; + maxCallsInRequest: number; + /** Bytes of the serialised request. */ + maxSizeRequest: number; + /** Server cap on `/changes` `maxChanges` (Stalwart: 5000). */ + maxChanges: number; + /** Server cap on `/query` `limit` (Stalwart: 5000). */ + maxQueryResults: number; +} + +export interface AccountOptions { + name: string; + /** Default true. The first personal account with a capability is its primary account. */ + isPersonal?: boolean; + isReadOnly?: boolean; + /** Capabilities the account advertises; both by default. */ + capabilities?: Array<'contacts' | 'calendars'>; + /** + * The owner's calendar addresses, without `mailto:`: they decide + * `isOrigin` and the organizer Stalwart assigns. Default: the name, when it + * is an email address. + */ + addresses?: string[]; +} + +export interface RequestRecord { + seq: number; + using: string[]; + calls: JmapInvocation[]; + methods: string[]; + /** What the server answered; null when the request never ran. */ + responses: JmapInvocation[] | null; + /** + * `ok`: answered. `failed`: a transport fault before anything ran. `lost`: + * ran (writes applied), then the response was lost. `rejected`: refused as + * a whole (a request-level limit). + */ + outcome: 'ok' | 'failed' | 'lost' | 'rejected'; + fault?: FaultKind; +} + +export interface SchedulingRecord { + accountId: string; + op: 'create' | 'update' | 'destroy'; + id: string; + requestSeq: number; +} + +export interface FaultOptions { + /** Run the request (writes are applied), then fail it: a lost response. */ + applied?: boolean; + /** For `rateLimit`. Default 1000. */ + retryAfterMs?: number; + /** How many matching requests fail. Default 1. */ + times?: number; + /** Only requests with a method call of this name, or passing this test. */ + match?: string | ((calls: JmapInvocation[]) => boolean); +} + +/** One object a `/set` call touches, as `setErrorFor` matchers see it. */ +export interface SetTarget { + op: 'create' | 'update' | 'destroy'; + /** The creation id for creates, else the object id. */ + id: string; + /** The create object or the PatchObject. */ + object?: Record; +} + +export interface FakeJmapServerOptions { + /** Clock for `updated` stamps. Default: a manual clock (`setTime`, `advanceTime`). */ + now?: () => number; + /** Start of the manual clock. Default 2026-01-01T00:00:00Z. */ + startTime?: number; + limits?: Partial; +} + +type RequestListener = (request: RequestRecord) => void | Promise; + +// ─── Constants ───────────────────────────────────────── + +type Obj = Record; +type Side = 'container' | 'item'; +type ChangeOp = 'insert' | 'update' | 'delete'; + +const DEFAULT_LIMITS: FakeLimits = { + maxObjectsInGet: 500, + maxObjectsInSet: 500, + maxCallsInRequest: 16, + maxSizeRequest: 10_000_000, + maxChanges: 5000, + maxQueryResults: 5000, +}; + +const ID_PATTERN = /^[A-Za-z0-9_-]{1,255}$/; + +/** Prefix of the ids the fake mints, per type (`c1`, `c2`, … `ca`, …). */ +const ID_PREFIX: Record = { + AddressBook: 'ab', + ContactCard: 'c', + Calendar: 'cal', + CalendarEvent: 'ev', +}; + +const CAPABILITY_OF: Record = { + Core: JMAP_CORE, + AddressBook: JMAP_CONTACTS, + ContactCard: JMAP_CONTACTS, + Calendar: JMAP_CALENDARS, + CalendarEvent: JMAP_CALENDARS, +}; + +const SUPPORTED_METHODS = new Set([ + 'Core/echo', + 'AddressBook/get', + 'AddressBook/changes', + 'ContactCard/get', + 'ContactCard/set', + 'ContactCard/query', + 'ContactCard/changes', + 'Calendar/get', + 'Calendar/changes', + 'CalendarEvent/get', + 'CalendarEvent/set', + 'CalendarEvent/query', + 'CalendarEvent/changes', +]); + +/** Responses a result reference may point at (Stalwart: /get, /changes, /query, /queryChanges). */ +const REFERENCEABLE = /\/(get|changes|query|queryChanges)$/; + +/** Top-level ContactCard properties (RFC 9553, RFC 9610); anything else is "Invalid property." (verified). */ +const CARD_PROPERTIES = new Set([ + '@type', 'version', 'created', 'kind', 'language', 'members', 'prodId', 'uid', 'updated', 'name', + 'nicknames', 'organizations', 'speakToAs', 'titles', 'emails', 'onlineServices', 'phones', + 'preferredLanguages', 'calendars', 'schedulingAddresses', 'addresses', 'cryptoKeys', 'directories', + 'links', 'media', 'localizations', 'anniversaries', 'keywords', 'notes', 'personalInfo', 'relatedTo', + 'vCard', +]); + +/** + * Top-level CalendarEvent properties calcard maps: JSCalendar-bis names only. + * `recurrenceRules`, `excludedRecurrenceRule(s)`, `timeZones`, + * `progressUpdated` and other JSCalendar 1.0 names are "Invalid property." + * (verified). + */ +const EVENT_PROPERTIES = new Set([ + '@type', 'uid', 'relatedTo', 'prodId', 'created', 'updated', 'sequence', 'title', 'description', + 'descriptionContentType', 'showWithoutTime', 'locations', 'mainLocationId', 'virtualLocations', 'links', + 'locale', 'keywords', 'categories', 'color', 'recurrenceId', 'recurrenceIdTimeZone', 'recurrenceRule', + 'recurrenceOverrides', 'excluded', 'priority', 'freeBusyStatus', 'privacy', 'replyTo', 'sentBy', + 'participants', 'requestStatus', 'alerts', 'timeZone', 'start', 'duration', 'endTimeZone', 'status', + 'organizerCalendarAddress', 'iCalendar', 'due', 'estimatedDuration', 'percentComplete', 'progress', +]); + +/** JMAP-only event flags: booleans stored beside the iCalendar data. */ +const EVENT_FLAGS = ['isDraft', 'useDefaultAlerts', 'mayInviteSelf', 'mayInviteOthers', 'hideAttendees']; + +/** Computed on read; any value sent is "This property is immutable." (verified for isOrigin). */ +const IMMUTABLE_EVENT_PROPERTIES = new Set(['isOrigin', 'baseEventId', 'method']); + +/** Properties calcard drops from an override on read (`remove_forbidden_override_patches`). */ +const FORBIDDEN_OVERRIDE_PROPERTIES = [ + '@type', 'method', 'organizerCalendarAddress', 'privacy', 'prodId', 'recurrenceId', + 'recurrenceIdTimeZone', 'sentBy', 'uid', 'recurrenceRule', 'recurrenceOverrides', +]; + +/** Card maps that vanish when empty (no vCard line is written). `keywords` and `members` keep `{}`. */ +const CARD_MAPS = [ + 'nicknames', 'organizations', 'titles', 'emails', 'onlineServices', 'phones', 'preferredLanguages', + 'calendars', 'schedulingAddresses', 'addresses', 'cryptoKeys', 'directories', 'links', 'media', + 'localizations', 'anniversaries', 'notes', 'personalInfo', 'relatedTo', +]; + +/** Card maps whose keys are PROP-IDs, regenerated as `k1…kN` when those are lost. */ +const CARD_KEYED_MAPS = CARD_MAPS.filter((m) => m !== 'relatedTo' && m !== 'localizations'); + +/** Event maps that vanish when empty (no iCalendar line or component is written). */ +const EVENT_MAPS = [ + 'alerts', 'participants', 'locations', 'virtualLocations', 'links', 'recurrenceOverrides', 'keywords', + 'categories', 'relatedTo', 'replyTo', +]; + +/** Name component kinds that have a vCard N position and so make up a derived `full`. */ +const NAME_KINDS = new Set(['surname', 'given', 'given2', 'title', 'credential', 'surname2', 'generation']); + +const PRIVACY_RANK = ['public', 'private', 'secret']; + +const ADDRESS_BOOK_DEFAULTS: Obj = { + name: '', + description: null, + sortOrder: 0, + isSubscribed: true, + shareWith: null, + myRights: { mayRead: true, mayWrite: true, mayShare: true, mayDelete: true }, +}; + +const CALENDAR_DEFAULTS: Obj = { + name: '', + description: null, + color: null, + sortOrder: 0, + isSubscribed: true, + isVisible: true, + includeInAvailability: 'all', + defaultAlertsWithTime: {}, + defaultAlertsWithoutTime: {}, + timeZone: null, + shareWith: null, + myRights: { + mayReadFreeBusy: true, + mayReadItems: true, + mayWriteAll: true, + mayWriteOwn: true, + mayUpdatePrivate: true, + mayRSVP: true, + mayShare: true, + mayDelete: true, + }, +}; + +// ─── JSON helpers ────────────────────────────────────── + +function isObj(value: unknown): value is Obj { + return typeof value === 'object' && value !== null && !Array.isArray(value); +} + +function clone(value: T): T { + return value === undefined ? value : (JSON.parse(JSON.stringify(value)) as T); +} + +function hasOwn(obj: Obj, key: string): boolean { + return Object.prototype.hasOwnProperty.call(obj, key); +} + +/** Assign an own property, also for the key `__proto__`. */ +function setKey(obj: Obj, key: string, value: unknown): void { + if (key === '__proto__') { + Object.defineProperty(obj, key, { value, enumerable: true, writable: true, configurable: true }); + } else { + obj[key] = value; + } +} + +/** Drop null members of objects, recursively: the converters write nothing for null. */ +function stripNulls(value: unknown): void { + if (Array.isArray(value)) { + for (const item of value) stripNulls(item); + return; + } + if (!isObj(value)) return; + for (const key of Object.keys(value)) { + if (value[key] === null) delete value[key]; + else stripNulls(value[key]); + } +} + +function booleanSet(ids: string[]): Obj { + const out: Obj = {}; + for (const id of ids) setKey(out, id, true); + return out; +} + +function invalidProperties(properties: string[], description: string): SetErrorShape { + return { type: 'invalidProperties', properties, description }; +} + +class MethodError extends Error { + constructor(readonly type: string, readonly detail?: string) { + super(detail ?? type); + } +} + +// ─── State strings ───────────────────────────────────── + +// Stalwart's encoding (jmap-proto/src/types/state.rs): `n` for "no changes +// yet", `s` + base32(leb128(changeId)) for an exact state, and `r` + +// base32(leb128(from), leb128(to - from), leb128(itemsSent)) for the +// intermediate state of a paged `/changes`. Base32 uses Stalwart's alphabet. + +type ParsedState = + | { kind: 'initial' } + | { kind: 'exact'; id: number } + | { kind: 'intermediate'; from: number; to: number; sent: number }; + +const BASE32 = 'abcdefghijklmnopqrstuvwxyz792013'; + +function leb128(value: number, out: number[]): void { + let n = value; + do { + let byte = n % 128; + n = Math.floor(n / 128); + if (n > 0) byte |= 0x80; + out.push(byte); + } while (n > 0); +} + +function toBase32(bytes: number[]): string { + let out = ''; + let buffer = 0; + let bits = 0; + for (const byte of bytes) { + buffer = (buffer << 8) | byte; + bits += 8; + while (bits >= 5) { + out += BASE32[(buffer >> (bits - 5)) & 31]; + bits -= 5; + } + buffer &= (1 << bits) - 1; + } + if (bits > 0) out += BASE32[(buffer << (5 - bits)) & 31]; + return out; +} + +function fromBase32(text: string): number[] | null { + const out: number[] = []; + let buffer = 0; + let bits = 0; + for (const ch of text) { + const value = BASE32.indexOf(ch); + if (value < 0) return null; + buffer = (buffer << 5) | value; + bits += 5; + if (bits >= 8) { + out.push((buffer >> (bits - 8)) & 0xff); + bits -= 8; + buffer &= (1 << bits) - 1; + } + } + return out; +} + +function readLeb128(bytes: number[], pos: number): [number, number] | null { + let value = 0; + let scale = 1; + for (let i = pos; i < bytes.length; i++) { + value += (bytes[i] & 0x7f) * scale; + if ((bytes[i] & 0x80) === 0) return [value, i + 1]; + scale *= 128; + } + return null; +} + +function encodeState(state: ParsedState): string { + const bytes: number[] = []; + switch (state.kind) { + case 'initial': + return 'n'; + case 'exact': + leb128(state.id, bytes); + return `s${toBase32(bytes)}`; + case 'intermediate': + leb128(state.from, bytes); + leb128(state.to - state.from, bytes); + leb128(state.sent, bytes); + return `r${toBase32(bytes)}`; + } +} + +function parseState(value: string): ParsedState | null { + const bytes = fromBase32(value.slice(1)); + if (value[0] === 'n') return { kind: 'initial' }; + if (!bytes) return null; + if (value[0] === 's') { + const id = readLeb128(bytes, 0); + if (!id) return null; + return id[0] === 0 ? { kind: 'initial' } : { kind: 'exact', id: id[0] }; + } + if (value[0] === 'r') { + const from = readLeb128(bytes, 0); + const delta = from && readLeb128(bytes, from[1]); + const sent = delta && readLeb128(bytes, delta[1]); + if (!from || !delta || !sent || sent[0] === 0) return null; + return { kind: 'intermediate', from: from[0], to: from[0] + delta[0], sent: sent[0] }; + } + return null; +} + +function stateOfChangeId(changeId: number): ParsedState { + return changeId === 0 ? { kind: 'initial' } : { kind: 'exact', id: changeId }; +} + +// ─── JSON Pointer (jmap-tools semantics) ─────────────── + +// A PatchObject key containing `/` is a JSON Pointer. jmap-tools parses it +// with RFC 6901's `~0`/`~1` escapes, plus `\` escaping the next character; an +// all-digit segment (other than a leading-zero one like `01`) is an array +// index that can replace an existing member but never add one; `*` is a +// wildcard that makes the patch fail (jmap-tools pointer/parser.rs, eval.rs). + +type Segment = + | { kind: 'key'; key: string } + | { kind: 'index'; index: number; raw: string } + | { kind: 'wildcard' }; + +function parsePointer(value: string): Segment[] { + const segments: Segment[] = []; + let token: 'unknown' | 'number' | 'string' | 'wildcard' | 'escaped' = 'unknown'; + let buf = ''; + let num = 0; + let start = 0; + const flush = (end: number) => { + const raw = value.slice(start, end); + if (token === 'string') { + segments.push({ kind: 'key', key: buf }); + buf = ''; + } else if (token === 'number') { + segments.push(raw.length > 1 && raw[0] === '0' ? { kind: 'key', key: raw } : { kind: 'index', index: num, raw }); + num = 0; + } else if (token === 'wildcard') { + segments.push({ kind: 'wildcard' }); + } else if (token === 'unknown' && start > 0) { + segments.push({ kind: 'key', key: '' }); + } + }; + for (let i = 0; i < value.length; i++) { + const ch = value[i]; + if (ch >= '0' && ch <= '9' && (token === 'unknown' || token === 'number')) { + num = num * 10 + Number(ch); + token = 'number'; + } else if (ch === '*' && token === 'unknown') { + token = 'wildcard'; + } else if ((ch === '0' || ch === '1') && token === 'escaped') { + buf += ch === '0' ? '~' : '/'; + token = 'string'; + } else if (ch === '/') { + flush(i); + token = 'unknown'; + start = i + 1; + } else { + if ((token === 'number' || token === 'wildcard') && i > start) buf += value.slice(start, i); + if (ch === '~' && token !== 'escaped') { + token = 'escaped'; + } else if (ch === '\\') { + i++; + buf += i < value.length ? value[i] : '\\'; + token = 'string'; + } else { + buf += ch; + token = 'string'; + } + } + } + flush(value.length); + return segments; +} + +/** + * Apply one pointer patch the way `patch_jptr` does: every parent must exist, + * the last segment replaces or adds a member. One deliberate difference: + * Stalwart keeps `key: null` in the map, and for most contact maps its + * converter then leaves a valueless vCard line (or a JSPROP null) behind + * (research E2). The engine never relies on that and resends the whole map + * instead, so here a null simply deletes the member. + */ +function applyPointer(node: unknown, segments: Segment[], value: unknown, i = 0): boolean { + const segment = segments[i]; + if (!segment) return false; + const last = i === segments.length - 1; + if (segment.kind === 'key') { + if (!isObj(node)) return false; + if (hasOwn(node, segment.key)) { + if (!last) return applyPointer(node[segment.key], segments, value, i + 1); + assignOrDelete(node, segment.key, value); + return true; + } + if (!last) return false; + assignOrDelete(node, segment.key, value); + return true; + } + if (segment.kind === 'index') { + if (Array.isArray(node)) { + if (segment.index >= node.length) return false; + if (!last) return applyPointer(node[segment.index], segments, value, i + 1); + node[segment.index] = value; + return true; + } + const key = String(segment.index); + if (isObj(node) && hasOwn(node, key)) { + if (!last) return applyPointer(node[key], segments, value, i + 1); + assignOrDelete(node, key, value); + return true; + } + } + return false; +} + +function assignOrDelete(obj: Obj, key: string, value: unknown): void { + if (value === null) delete obj[key]; + else setKey(obj, key, value); +} + +/** A result reference path (RFC 8620 §3.7): a JSON Pointer where `*` maps over an array. */ +function resolveReferencePath(root: unknown, path: string): { value: unknown } | null { + if (path === '') return { value: root }; + if (!path.startsWith('/')) return null; + const tokens = path.slice(1).split('/').map((t) => t.replace(/~1/g, '/').replace(/~0/g, '~')); + const walk = (node: unknown, i: number): { value: unknown } | null => { + if (i === tokens.length) return { value: node }; + const token = tokens[i]; + if (Array.isArray(node)) { + if (token === '*') { + const out: unknown[] = []; + for (const item of node) { + const result = walk(item, i + 1); + if (!result) return null; + if (Array.isArray(result.value)) out.push(...result.value); + else out.push(result.value); + } + return { value: out }; + } + if (!/^\d+$/.test(token) || Number(token) >= node.length) return null; + return walk(node[Number(token)], i + 1); + } + if (isObj(node) && hasOwn(node, token)) return walk(node[token], i + 1); + return null; + }; + return walk(root, 0); +} + +// ─── uuid5 ───────────────────────────────────────────── + +// calcard keys participants, locations, links and virtual locations that have +// no JSID by uuid5 of their address, name, href or uri in its own namespace +// (7f1e1965-ae73-4454-b088-232c90730ce2, calcard jscalendar/mod.rs). + +const JSCAL_NAMESPACE = [0x7f, 0x1e, 0x19, 0x65, 0xae, 0x73, 0x44, 0x54, 0xb0, 0x88, 0x23, 0x2c, 0x90, 0x73, 0x0c, 0xe2]; + +function sha1(message: Uint8Array): Uint8Array { + const padded = new Uint8Array((((message.length + 8) >> 6) + 1) << 6); + padded.set(message); + padded[message.length] = 0x80; + const view = new DataView(padded.buffer); + const bits = message.length * 8; + view.setUint32(padded.length - 8, Math.floor(bits / 0x100000000)); + view.setUint32(padded.length - 4, bits >>> 0); + const h = [0x67452301, 0xefcdab89, 0x98badcfe, 0x10325476, 0xc3d2e1f0]; + const w = new Uint32Array(80); + for (let offset = 0; offset < padded.length; offset += 64) { + for (let i = 0; i < 16; i++) w[i] = view.getUint32(offset + i * 4); + for (let i = 16; i < 80; i++) { + const x = w[i - 3] ^ w[i - 8] ^ w[i - 14] ^ w[i - 16]; + w[i] = (x << 1) | (x >>> 31); + } + let [a, b, c, d, e] = h; + for (let i = 0; i < 80; i++) { + const f = i < 20 ? (b & c) | (~b & d) : i < 40 ? b ^ c ^ d : i < 60 ? (b & c) | (b & d) | (c & d) : b ^ c ^ d; + const k = i < 20 ? 0x5a827999 : i < 40 ? 0x6ed9eba1 : i < 60 ? 0x8f1bbcdc : 0xca62c1d6; + const t = (((a << 5) | (a >>> 27)) + f + e + k + w[i]) >>> 0; + e = d; + d = c; + c = ((b << 30) | (b >>> 2)) >>> 0; + b = a; + a = t; + } + h[0] = (h[0] + a) >>> 0; + h[1] = (h[1] + b) >>> 0; + h[2] = (h[2] + c) >>> 0; + h[3] = (h[3] + d) >>> 0; + h[4] = (h[4] + e) >>> 0; + } + const out = new Uint8Array(20); + const outView = new DataView(out.buffer); + h.forEach((value, i) => outView.setUint32(i * 4, value)); + return out; +} + +/** The key calcard gives an entry without a stored JSID, e.g. `uuid5('mailto:a@b.c')`. */ +export function uuid5(text: string): string { + const name = new TextEncoder().encode(text); + const input = new Uint8Array(JSCAL_NAMESPACE.length + name.length); + input.set(JSCAL_NAMESPACE); + input.set(name, JSCAL_NAMESPACE.length); + const hash = sha1(input); + hash[6] = (hash[6] & 0x0f) | 0x50; + hash[8] = (hash[8] & 0x3f) | 0x80; + const hex = Array.from(hash.slice(0, 16), (byte) => byte.toString(16).padStart(2, '0')).join(''); + return `${hex.slice(0, 8)}-${hex.slice(8, 12)}-${hex.slice(12, 16)}-${hex.slice(16, 20)}-${hex.slice(20)}`; +} + +// ─── Change log ──────────────────────────────────────── + +interface Change { + side: Side; + op: ChangeOp; + id: string; +} + +interface LogEntry { + changeId: number; + /** The truncation marker Stalwart writes where it trimmed the log. */ + marker: boolean; + changes: Change[]; +} + +interface ChangeWindow { + /** Compacted changes by `side:id`, in log order. */ + changes: Map; + fromChangeId: number; + toChangeId: number; + containerChangeId: number | null; + itemChangeId: number | null; + isTruncated: boolean; +} + +type LogQuery = { all: true } | { since: number } | { from: number; to: number }; + +/** + * Compaction inside a window (store/src/query/log.rs): an update after an + * insert is dropped, an insert then delete vanishes, an update then delete + * leaves only the delete, and a repeated update moves to the end. + */ +function compact(changes: Map, change: Change): void { + const key = `${change.side}:${change.id}`; + const current = changes.get(key); + if (change.op === 'insert') { + changes.set(key, change); + return; + } + if (current?.op === 'insert') { + if (change.op === 'delete') changes.delete(key); + return; + } + changes.delete(key); + changes.set(key, change); +} + +class ChangeLog { + entries: LogEntry[] = []; + /** The last change id with container (or item) changes: the type's state. */ + containerChangeId = 0; + itemChangeId = 0; + + append(changeId: number, changes: Change[]): void { + this.entries.push({ changeId, marker: false, changes }); + if (changes.some((c) => c.side === 'container')) this.containerChangeId = changeId; + if (changes.some((c) => c.side === 'item')) this.itemChangeId = changeId; + } + + sideChangeId(side: Side): number { + return side === 'container' ? this.containerChangeId : this.itemChangeId; + } + + /** `Store::changes`: the entries of a range, compacted. */ + query(q: LogQuery): ChangeWindow { + const [inclusive, from, to]: [boolean, number, number] = + 'all' in q ? [true, 0, Infinity] : 'since' in q ? [false, q.since, Infinity] : [true, q.from, q.to]; + const out: ChangeWindow = { + changes: new Map(), + fromChangeId: 0, + toChangeId: 0, + containerChangeId: null, + itemChangeId: null, + isTruncated: false, + }; + for (const entry of this.entries) { + const id = entry.changeId; + if (id < from || id > to) continue; + if (!inclusive && id === from) { + out.fromChangeId = id; + out.toChangeId = id; + continue; + } + if (entry.marker) { + out.isTruncated = true; + continue; + } + if (out.changes.size === 0) out.fromChangeId = id; + out.toChangeId = id; + let container = false; + let item = false; + for (const change of entry.changes) { + compact(out.changes, change); + if (change.side === 'container') container = true; + else item = true; + } + if (container) out.containerChangeId = id; + if (item) out.itemChangeId = id; + } + return out; + } + + /** + * `delete_changes`: keep the newest `keep` entries; the one before them + * becomes the truncation marker and everything older goes. + */ + truncate(keep: number): void { + if (this.entries.length <= keep) return; + const markerIndex = this.entries.length - keep - 1; + const marker: LogEntry = { changeId: this.entries[markerIndex].changeId, marker: true, changes: [] }; + this.entries = [marker, ...this.entries.slice(markerIndex + 1)]; + } +} + +/** The changes of one commit, as Stalwart's ChangeLogBuilder collects them. */ +class PendingChanges { + private readonly sets: Record>> = { + container: { insert: new Set(), update: new Set(), delete: new Set() }, + item: { insert: new Set(), update: new Set(), delete: new Set() }, + }; + + insert(side: Side, id: string): void { + const sets = this.sets[side]; + if (sets.delete.delete(id)) sets.update.add(id); + else sets.insert.add(id); + } + + update(side: Side, id: string): void { + this.sets[side].update.add(id); + } + + delete(side: Side, id: string): void { + const sets = this.sets[side]; + sets.update.delete(id); + sets.delete.add(id); + } + + toChanges(): Change[] { + const out: Change[] = []; + for (const side of ['container', 'item'] as const) { + for (const op of ['insert', 'update', 'delete'] as const) { + for (const id of this.sets[side][op]) out.push({ side, op, id }); + } + } + return out; + } +} + +// ─── Normalisation ───────────────────────────────────── + +/** RFC 3339 with offset → UTC at second precision with `Z`; null when unparsable. */ +function utcSeconds(value: string): string | null { + if (!/^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(\.\d+)?(Z|[+-]\d{2}:\d{2})$/i.test(value)) return null; + const time = Date.parse(value); + return Number.isNaN(time) ? null : new Date(time).toISOString().replace(/\.\d{3}Z$/, 'Z'); +} + +/** A LocalDateTime at second precision; a fraction and any `Z` or offset are ignored. */ +function localDateTime(value: string): string | null { + const match = /^(\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2})(?:\.\d+)?(?:Z|z|[+-]\d{2}:?\d{2})?$/.exec(value); + return match ? match[1] : null; +} + +/** + * `full` as calcard derives it when a name has components but no `full` + * (written as `FN;DERIVED=TRUE` and read back as `name.full`): the + * components in order joined by the default separator (a space unless + * `defaultSeparator` says otherwise), `separator` components verbatim. + */ +function deriveFullName(name: Obj): string { + if (!Array.isArray(name.components)) return ''; + const separator = + typeof name.defaultSeparator === 'string' && name.defaultSeparator !== '' ? name.defaultSeparator : ' '; + let full = ''; + let afterValue = false; + for (const component of name.components) { + if (!isObj(component) || typeof component.value !== 'string') continue; + if (component.kind === 'separator') { + full += component.value; + afterValue = false; + } else if (typeof component.kind === 'string' && NAME_KINDS.has(component.kind)) { + if (afterValue) full += separator; + full += component.value; + afterValue = true; + } + } + return full; +} + +/** Components make a name or address ordered (JSCOMPS is always written); without them the flag goes. */ +function normalizeOrdered(value: Obj): void { + if (Array.isArray(value.components) && value.components.length > 0) { + value.isOrdered = true; + } else { + delete value.isOrdered; + if (Array.isArray(value.components)) delete value.components; + } +} + +function dropEmptyMaps(obj: Obj, maps: string[]): void { + for (const key of maps) { + const value = obj[key]; + if (isObj(value) && Object.keys(value).length === 0) delete obj[key]; + } +} + +/** + * A card as `ContactCard/get` returns it after Stalwart stored it as vCard: + * `@type: "Card"` and `version: "1.0"` always (verified); `isOrdered: true` + * on a name and on addresses with components (verified); `kind: "title"` on + * titles without a kind (verified); a derived `name.full` when absent, which + * then sticks, since Stalwart never re-derives it after a component patch; + * `@type` on anniversary dates; `created`/`updated` in UTC at second + * precision; empty maps gone. `data:` URIs in media stay as sent (Stalwart + * re-encodes them as `data:;base64,…`, which the tests don't need). + */ +function normalizeCard(input: Obj): Obj { + stripNulls(input); + const card: Obj = { '@type': 'Card', version: '1.0' }; + for (const [key, value] of Object.entries(input)) { + if (key !== '@type' && key !== 'version') setKey(card, key, value); + } + for (const key of ['created', 'updated']) { + const value = card[key]; + if (typeof value !== 'string') continue; + const utc = utcSeconds(value); + if (utc) card[key] = utc; + else delete card[key]; + } + if (isObj(card.name)) { + const name = card.name; + normalizeOrdered(name); + if (typeof name.full !== 'string') { + const full = deriveFullName(name); + if (full) name.full = full; + } + } + if (isObj(card.addresses)) { + for (const address of Object.values(card.addresses)) if (isObj(address)) normalizeOrdered(address); + } + if (isObj(card.titles)) { + for (const title of Object.values(card.titles)) { + if (isObj(title) && title.kind === undefined) title.kind = 'title'; + } + } + if (isObj(card.anniversaries)) { + for (const anniversary of Object.values(card.anniversaries)) { + if (!isObj(anniversary) || !isObj(anniversary.date) || anniversary.date['@type'] !== undefined) continue; + const date = anniversary.date; + if (date.utc !== undefined) date['@type'] = 'Timestamp'; + else if (date.year !== undefined || date.month !== undefined || date.day !== undefined) date['@type'] = 'PartialDate'; + } + } + dropEmptyMaps(card, CARD_MAPS); + return card; +} + +/** `@type` calcard puts on nested event objects. */ +function addNestedTypes(component: Obj): void { + const typed: Array<[string, string]> = [ + ['participants', 'Participant'], + ['locations', 'Location'], + ['virtualLocations', 'VirtualLocation'], + ['links', 'Link'], + ['alerts', 'Alert'], + ]; + for (const [property, type] of typed) { + const map = component[property]; + if (!isObj(map)) continue; + for (const entry of Object.values(map)) { + if (!isObj(entry)) continue; + if (entry['@type'] === undefined) entry['@type'] = type; + const trigger = entry.trigger; + if (property === 'alerts' && isObj(trigger) && trigger['@type'] === undefined) { + if (trigger.offset !== undefined) trigger['@type'] = 'OffsetTrigger'; + else if (trigger.when !== undefined) trigger['@type'] = 'AbsoluteTrigger'; + } + } + } +} + +function normalizeEventTimes(component: Obj): void { + if (typeof component.start === 'string') { + const start = localDateTime(component.start); + if (start) component.start = start; + } + if (typeof component.created === 'string') { + const created = localDateTime(component.created); + if (created) component.created = `${created}Z`; + } +} + +/** + * An event as `CalendarEvent/get` returns it after Stalwart stored it as + * iCalendar: + * - `updated` is the server's time on every write (verified), and every + * override stored as its own VEVENT gets one too (verified for new and + * changed overrides; `stamp_updated` stamps all of them). In `server` mode + * (seeds, edits by another client) an `updated` that is already there is + * kept, so tests can pin it. + * - `@type` defaults to `Event` (`Task` stays); participants, locations, + * virtual locations, links and alerts get their `@type`. + * - `start` and override keys are LocalDateTimes (a fraction, `Z` or offset + * is ignored); an override key that is not one is dropped, `{excluded: + * true}` loses everything else, forbidden override properties go, and a + * stricter override `privacy` raises the event's. + * - Empty maps are gone. + */ +function normalizeEvent(input: Obj, now: string, mode: 'jmap' | 'server'): Obj { + stripNulls(input); + const event: Obj = { '@type': typeof input['@type'] === 'string' ? input['@type'] : 'Event' }; + for (const [key, value] of Object.entries(input)) if (key !== '@type') setKey(event, key, value); + normalizeEventTimes(event); + addNestedTypes(event); + if (isObj(event.recurrenceOverrides)) { + const overrides: Obj = {}; + let privacy = Math.max(0, PRIVACY_RANK.indexOf(String(event.privacy))); + for (const [key, value] of Object.entries(event.recurrenceOverrides)) { + const rid = localDateTime(key); + if (!rid || !isObj(value)) continue; + if (value.excluded === true) { + setKey(overrides, rid, { excluded: true }); + continue; + } + privacy = Math.max(privacy, PRIVACY_RANK.indexOf(String(value.privacy))); + for (const forbidden of FORBIDDEN_OVERRIDE_PROPERTIES) delete value[forbidden]; + if (Object.keys(value).length > 0) { + normalizeEventTimes(value); + addNestedTypes(value); + if (mode === 'jmap' || typeof value.updated !== 'string') value.updated = now; + } + setKey(overrides, rid, value); + } + event.recurrenceOverrides = overrides; + if (privacy > 0 && privacy > PRIVACY_RANK.indexOf(String(event.privacy))) event.privacy = PRIVACY_RANK[privacy]; + } + if (mode === 'jmap' || typeof event.updated !== 'string') event.updated = now; + dropEmptyMaps(event, EVENT_MAPS); + return event; +} + +// ─── Positional re-keying ────────────────────────────── + +interface Rekeyed { + map: Obj; + renames: Record; +} + +function rekeyMap(map: Obj, keyFor: (entry: unknown, index: number, oldKey: string) => string): Rekeyed { + const out: Obj = {}; + const renames: Record = {}; + Object.entries(map).forEach(([oldKey, entry], index) => { + let key = keyFor(entry, index, oldKey); + // jmap-tools renames a colliding key to `-` (json/object_vec.rs). + for (let n = 2; hasOwn(out, key); n++) key = `${keyFor(entry, index, oldKey)}-${n}`; + setKey(out, key, entry); + renames[oldKey] = key; + }); + return { map: out, renames }; +} + +const positional = (_entry: unknown, index: number) => `k${index + 1}`; + +function byContent(property: string): (entry: unknown, index: number, oldKey: string) => string { + return (entry, _index, oldKey) => { + const content = isObj(entry) ? entry[property] : undefined; + return typeof content === 'string' ? uuid5(content) : oldKey; + }; +} + +/** Card maps keyed by PROP-ID get `k1…kN` by position; titles follow their organization. */ +function rekeyCard(card: Obj): void { + let organizations: Record = {}; + for (const property of CARD_KEYED_MAPS) { + const map = card[property]; + if (!isObj(map)) continue; + const rekeyed = rekeyMap(map, positional); + card[property] = rekeyed.map; + if (property === 'organizations') organizations = rekeyed.renames; + } + if (isObj(card.speakToAs) && isObj(card.speakToAs.pronouns)) { + card.speakToAs.pronouns = rekeyMap(card.speakToAs.pronouns, positional).map; + } + if (isObj(card.titles)) { + for (const title of Object.values(card.titles)) { + if (isObj(title) && typeof title.organizationId === 'string' && organizations[title.organizationId]) { + title.organizationId = organizations[title.organizationId]; + } + } + } +} + +/** + * Alerts get `k1…kN` by position; participants, locations, virtual locations + * and links get uuid5 of their calendar address, name, uri and href. + */ +function rekeyEventComponent(component: Obj): void { + if (isObj(component.alerts)) component.alerts = rekeyMap(component.alerts, positional).map; + if (isObj(component.participants)) component.participants = rekeyMap(component.participants, byContent('calendarAddress')).map; + if (isObj(component.virtualLocations)) component.virtualLocations = rekeyMap(component.virtualLocations, byContent('uri')).map; + if (isObj(component.links)) component.links = rekeyMap(component.links, byContent('href')).map; + if (!isObj(component.locations)) return; + const locations = rekeyMap(component.locations, (entry, index, oldKey) => { + const name = isObj(entry) ? entry.name ?? entry.coordinates : undefined; + return typeof name === 'string' ? uuid5(name) : oldKey; + }); + component.locations = locations.map; + if (typeof component.mainLocationId === 'string' && locations.renames[component.mainLocationId]) { + component.mainLocationId = locations.renames[component.mainLocationId]; + } + if (isObj(component.participants)) { + for (const participant of Object.values(component.participants)) { + if (isObj(participant) && typeof participant.locationId === 'string' && locations.renames[participant.locationId]) { + participant.locationId = locations.renames[participant.locationId]; + } + } + } +} + +function rekeyEvent(event: Obj): void { + rekeyEventComponent(event); + if (isObj(event.recurrenceOverrides)) { + for (const override of Object.values(event.recurrenceOverrides)) if (isObj(override)) rekeyEventComponent(override); + } +} + +// ─── Server model ────────────────────────────────────── + +interface Stored { + id: string; + /** Document id: creation order, the order `/query` and `ids: null` return. */ + docId: number; + /** The JSContact/JSCalendar (or container) properties, without id, parents and JMAP-only flags. */ + data: Obj; + /** addressBookIds / calendarIds, in order. */ + parents: string[]; + /** Events: isDraft, useDefaultAlerts, mayInviteSelf, mayInviteOthers, hideAttendees. */ + flags: Obj; + /** First request whose `uid` query finds the object (the index lags). */ + uidVisibleFrom: number; + modSeq: number; +} + +interface Account { + id: string; + name: string; + isPersonal: boolean; + isReadOnly: boolean; + contacts: boolean; + calendars: boolean; + addresses: string[]; + /** Change ids are global to the account. */ + lastChangeId: number; + logs: Record; + objects: Record>; + usedIds: Record>; + idCounter: Record; + docCounter: Record; + defaultContainer: Partial>; + blobs: Map; +} + +interface Draft { + data: Obj; + parents: string[]; + flags: Obj; +} + +interface Fault { + kind: FaultKind; + target: 'request' | 'download'; + applied: boolean; + retryAfterMs: number; + remaining: number; + match?: (calls: JmapInvocation[]) => boolean; +} + +interface InjectedSetError { + type: ItemType; + accountId: string; + match: (target: SetTarget) => boolean; + error: SetErrorShape; + remaining: number; +} + +interface InjectedMethodError { + method: string; + accountId?: string; + error: { type: string; description?: string }; + remaining: number; +} + +interface CallContext { + seq: number; + /** Arguments that came from result references. */ + references: Set; +} + +function groupOf(type: DataType): TypeGroup { + return type === 'AddressBook' || type === 'ContactCard' ? 'contacts' : 'calendars'; +} + +function isContainer(type: DataType): type is ContainerType { + return type === 'AddressBook' || type === 'Calendar'; +} + +function containerOf(type: ItemType): ContainerType { + return type === 'ContactCard' ? 'AddressBook' : 'Calendar'; +} + +function parentProperty(type: ItemType): string { + return type === 'ContactCard' ? 'addressBookIds' : 'calendarIds'; +} + +function sideOf(type: DataType): Side { + return isContainer(type) ? 'container' : 'item'; +} + +function transportError(fault: Fault): Error & { retryAfterMs?: number } { + const [name, message] = { + network: ['NetworkError', 'Network request failed'], + timeout: ['RequestTimeoutError', 'Request timed out after 30s'], + auth: ['AuthenticationError', 'Session expired'], + rateLimit: ['RateLimitError', 'Rate limited by server'], + }[fault.kind]; + const error: Error & { retryAfterMs?: number } = new Error(message); + error.name = name; + if (fault.kind === 'rateLimit') error.retryAfterMs = fault.retryAfterMs; + return error; +} + +/** What jmap-client.ts throws for Stalwart's HTTP 400 request-level limit error. */ +function limitError(limit: string, detail: string): Error { + const body = JSON.stringify({ type: 'urn:ietf:params:jmap:error:limit', status: 400, limit, detail }); + return new Error(`JMAP request failed: 400 - ${body}`); +} + +// ─── The server ──────────────────────────────────────── + +export class FakeJmapServer { + /** Every request, in order, with its calls and what the server answered. */ + readonly requests: RequestRecord[] = []; + /** `/set` changes made with `sendSchedulingMessages: true` (no message is sent). */ + readonly scheduling: SchedulingRecord[] = []; + /** Blob downloads, in order. */ + readonly downloads: Array<{ accountId: string; blobId: string }> = []; + /** The most requests that were in flight at once (the engine keeps one per run). */ + maxInFlight = 0; + private inFlight = 0; + + private readonly accounts = new Map(); + private limits: FakeLimits; + private readonly customClock: (() => number) | null; + private manualTime: number; + private requestSeq = 0; + private uidLag = 0; + private uidCounter = 0; + private blobCounter = 0; + private modSeq = 0; + private sessionVersion = 0; + private faults: Fault[] = []; + private setErrors: InjectedSetError[] = []; + private methodErrors: InjectedMethodError[] = []; + private readonly beforeListeners = new Set(); + private readonly afterListeners = new Set(); + + constructor(options: FakeJmapServerOptions = {}) { + this.customClock = options.now ?? null; + this.manualTime = options.startTime ?? Date.UTC(2026, 0, 1); + this.limits = { ...DEFAULT_LIMITS, ...options.limits }; + } + + // ── Clock ── + + now(): number { + return this.customClock ? this.customClock() : this.manualTime; + } + + setTime(ms: number): void { + if (this.customClock) throw new Error('fake server: the clock was injected; move it instead'); + this.manualTime = ms; + } + + advanceTime(ms: number): void { + this.setTime(this.manualTime + ms); + } + + private nowUtc(): string { + return new Date(this.now()).toISOString().replace(/\.\d{3}Z$/, 'Z'); + } + + // ── Accounts and the port ── + + addAccount(id: string, options: AccountOptions): void { + if (!ID_PATTERN.test(id)) throw new Error(`fake server: invalid account id ${id}`); + if (this.accounts.has(id)) throw new Error(`fake server: account ${id} exists`); + const capabilities = options.capabilities ?? ['contacts', 'calendars']; + const perType = (make: () => T): Record => ({ + AddressBook: make(), + ContactCard: make(), + Calendar: make(), + CalendarEvent: make(), + }); + this.accounts.set(id, { + id, + name: options.name, + isPersonal: options.isPersonal ?? true, + isReadOnly: options.isReadOnly ?? false, + contacts: capabilities.includes('contacts'), + calendars: capabilities.includes('calendars'), + addresses: options.addresses ?? (options.name.includes('@') ? [options.name] : []), + lastChangeId: 0, + logs: { contacts: new ChangeLog(), calendars: new ChangeLog() }, + objects: perType(() => new Map()), + usedIds: perType(() => new Set()), + idCounter: perType(() => 0), + docCounter: perType(() => 0), + defaultContainer: {}, + blobs: new Map(), + }); + this.sessionVersion++; + } + + /** + * The engine's view of the server. `username` is the session's login + * (default: the first personal account's name); the primary account of a + * capability is the personal account named like it, else the first + * personal one that has the capability. + */ + port(username?: string): JmapPort { + return { + session: () => this.sessionView(username), + limits: () => ({ + maxObjectsInGet: this.limits.maxObjectsInGet, + maxObjectsInSet: this.limits.maxObjectsInSet, + maxCallsInRequest: this.limits.maxCallsInRequest, + maxConcurrentRequests: 4, + maxSizeRequest: this.limits.maxSizeRequest, + }), + request: (calls, using) => this.handleRequest(calls, using), + downloadBlob: (accountId, blobId) => this.handleDownload(accountId, blobId), + }; + } + + private sessionView(username?: string): JmapSessionView { + const accounts: Record = {}; + const all = [...this.accounts.values()]; + const personal = all.filter((a) => a.isPersonal); + const login = username ?? personal[0]?.name ?? ''; + const preferred = [...personal.filter((a) => a.name === login), ...personal.filter((a) => a.name !== login)]; + const primaryAccounts: Record = {}; + const contactsPrimary = preferred.find((a) => a.contacts); + const calendarsPrimary = preferred.find((a) => a.calendars); + if (contactsPrimary) primaryAccounts[JMAP_CONTACTS] = contactsPrimary.id; + if (calendarsPrimary) primaryAccounts[JMAP_CALENDARS] = calendarsPrimary.id; + for (const account of all) { + const accountCapabilities: Record = {}; + if (account.contacts) { + accountCapabilities[JMAP_CONTACTS] = { maxAddressBooksPerCard: null, mayCreateAddressBook: !account.isReadOnly }; + } + if (account.calendars) { + accountCapabilities[JMAP_CALENDARS] = { + maxCalendarsPerEvent: null, + minDateTime: '0001-01-01T00:00:00Z', + maxDateTime: '9999-12-31T23:59:59Z', + maxExpandedQueryDuration: 'P52W1D', + maxParticipantsPerEvent: 20, + mayCreateCalendar: !account.isReadOnly, + }; + } + accounts[account.id] = { + name: account.name, + isPersonal: account.isPersonal, + isReadOnly: account.isReadOnly, + accountCapabilities, + }; + } + const capabilities: Record = { + [JMAP_CORE]: { + maxSizeUpload: 50_000_000, + maxConcurrentUpload: 4, + maxSizeRequest: this.limits.maxSizeRequest, + maxConcurrentRequests: 4, + maxCallsInRequest: this.limits.maxCallsInRequest, + maxObjectsInGet: this.limits.maxObjectsInGet, + maxObjectsInSet: this.limits.maxObjectsInSet, + collationAlgorithms: [], + }, + }; + if (all.some((a) => a.contacts)) capabilities[JMAP_CONTACTS] = {}; + if (all.some((a) => a.calendars)) capabilities[JMAP_CALENDARS] = {}; + return { username: login, accounts, primaryAccounts, capabilities }; + } + + // ── Seeds and edits by another client ── + // Each call is one commit, so it shows up in `/changes` like any server-side + // change. Like a DAV client they skip the JMAP checks (uid uniqueness, + // rights) and get neither the organizer nor the default alerts a JMAP + // create adds; otherwise objects are normalised as Stalwart returns them. + + addAddressBook(accountId: string, book: Obj & { name: string }): string { + return this.insertContainer('AddressBook', accountId, book); + } + + addCalendar(accountId: string, calendar: Obj & { name: string }): string { + return this.insertContainer('Calendar', accountId, calendar); + } + + /** `addressBookIds` defaults to the default address book. */ + addCard(accountId: string, card: Obj = {}): string { + return this.insertItem('ContactCard', accountId, card); + } + + /** `calendarIds` defaults to the default calendar; `updated` is stamped unless given. */ + addEvent(accountId: string, event: Obj = {}): string { + return this.insertItem('CalendarEvent', accountId, event); + } + + serverCreate(type: DataType, accountId: string, object: Obj): string { + return isContainer(type) + ? this.insertContainer(type, accountId, object) + : this.insertItem(type, accountId, object); + } + + /** + * Another client patches an object (PatchObject keys, JSON Pointers + * included). Events get `updated` stamped unless the patch sets it. + */ + serverUpdate(type: DataType, accountId: string, id: string, patch: Obj): void { + const account = this.requireAccount(accountId); + const stored = this.requireObject(account, type, id); + const pending = new PendingChanges(); + if (isContainer(type)) { + const data = clone(stored.data); + for (const [key, value] of Object.entries(clone(patch))) { + if (key === 'id') continue; + if (key === 'isDefault') { + if (value === true) account.defaultContainer[type] = id; + continue; + } + if (key.includes('/')) { + if (!applyPointer(data, parsePointer(key), value)) throw new Error(`fake server: patch ${key} failed`); + } else { + assignOrDelete(data, key, value); + } + } + stored.data = data; + pending.update('container', id); + } else { + const draft: Draft = { data: clone(stored.data), parents: [...stored.parents], flags: { ...stored.flags } }; + const error = this.applyItemProperties(type, draft, clone(patch), id); + if (error) throw new Error(`fake server: ${error.type} ${error.properties?.join(',') ?? ''} ${error.description ?? ''}`); + for (const parent of draft.parents) this.requireObject(account, containerOf(type), parent); + if (type === 'ContactCard') { + draft.data = normalizeCard(draft.data); + } else { + const keepUpdated = hasOwn(patch, 'updated'); + if (!keepUpdated) delete draft.data.updated; + draft.data = normalizeEvent(draft.data, this.nowUtc(), 'server'); + } + Object.assign(stored, draft); + pending.update('item', id); + } + stored.modSeq = ++this.modSeq; + this.commit(account, groupOf(type), pending); + } + + /** + * Another client deletes an object. A container takes its items along, as + * `onDestroyRemoveContents` does: items only in it are destroyed, items in + * other containers too just lose the membership. + */ + serverDestroy(type: DataType, accountId: string, id: string): void { + const account = this.requireAccount(accountId); + this.requireObject(account, type, id); + const pending = new PendingChanges(); + account.objects[type].delete(id); + pending.delete(sideOf(type), id); + if (isContainer(type)) { + if (account.defaultContainer[type] === id) delete account.defaultContainer[type]; + const itemType: ItemType = type === 'AddressBook' ? 'ContactCard' : 'CalendarEvent'; + for (const item of [...account.objects[itemType].values()]) { + if (!item.parents.includes(id)) continue; + item.parents = item.parents.filter((p) => p !== id); + if (item.parents.length === 0) { + account.objects[itemType].delete(item.id); + pending.delete('item', item.id); + } else { + item.modSeq = ++this.modSeq; + pending.update('item', item.id); + } + } + } + this.commit(account, groupOf(type), pending); + } + + /** + * Rewrite a card's or event's map keys as Stalwart generates them for an + * object written over CardDAV/CalDAV without PROP-IDs/JSIDs: `k1…kN` by + * position for card entries and alerts, uuid5 of the address, name, uri or + * href for participants, locations, virtual locations and links. Logged + * as an update; nothing else changes (no `updated` stamp). + */ + rekeyPositionally(accountId: string, id: string, type?: ItemType): void { + const account = this.requireAccount(accountId); + const inCards = account.objects.ContactCard.has(id); + const inEvents = account.objects.CalendarEvent.has(id); + if (!type && inCards && inEvents) throw new Error(`fake server: ${id} is both a card and an event; pass the type`); + const resolved: ItemType | undefined = type ?? (inCards ? 'ContactCard' : inEvents ? 'CalendarEvent' : undefined); + if (!resolved) throw new Error(`fake server: no card or event ${id} in ${accountId}`); + const stored = this.requireObject(account, resolved, id); + if (resolved === 'ContactCard') rekeyCard(stored.data); + else rekeyEvent(stored.data); + stored.modSeq = ++this.modSeq; + const pending = new PendingChanges(); + pending.update('item', id); + this.commit(account, groupOf(resolved), pending); + } + + /** Store a blob for `downloadBlob` (e.g. a blob-backed `media` entry). */ + addBlob(accountId: string, data: Uint8Array | string, type = 'application/octet-stream'): string { + const account = this.requireAccount(accountId); + const blobId = `blob${++this.blobCounter}`; + const bytes = typeof data === 'string' ? new TextEncoder().encode(data) : data.slice(); + account.blobs.set(blobId, { bytes, type }); + return blobId; + } + + // ── Fault injection ── + + /** + * The next request (matching `opts.match`) throws the transport error + * jmap-client.ts would: NetworkError, RequestTimeoutError, + * AuthenticationError or RateLimitError with `retryAfterMs`. By default + * nothing of it runs; with `applied` it runs first (a lost response). + */ + failNextRequest(kind: FaultKind, opts: FaultOptions = {}): void { + this.faults.push(this.makeFault(kind, 'request', opts)); + } + + /** The next request runs (its writes are applied), then the response is lost: NetworkError. */ + applyThenLoseResponse(opts: Omit = {}): void { + this.failNextRequest('network', { ...opts, applied: true }); + } + + /** The next blob download throws the transport error for `kind`. */ + failNextDownload(kind: FaultKind, opts: Omit = {}): void { + this.faults.push(this.makeFault(kind, 'download', opts)); + } + + /** The next call of `method` (for `accountId`, if given) answers with this method-level error. */ + failNextMethod( + method: string, + error: { type: string; description?: string }, + opts: { accountId?: string; times?: number } = {}, + ): void { + this.methodErrors.push({ method, accountId: opts.accountId, error, remaining: opts.times ?? 1 }); + } + + /** + * The next `/set` touching `id` answers with `error` for that object (and + * applies nothing of it). `id` matches an update or destroy target, a + * creation id, or the `uid` of a create; or pass a predicate. + */ + setErrorFor( + type: ItemType, + accountId: string, + id: string | ((target: SetTarget) => boolean), + error: SetErrorShape, + times = 1, + ): void { + const match = + typeof id === 'function' + ? id + : (target: SetTarget) => + target.id === id || (target.op === 'create' && target.object?.uid === id); + this.setErrors.push({ type, accountId, match, error, remaining: times }); + } + + /** + * Trim the change log as Stalwart's daily purge does: keep the newest + * `keep` commits, mark the one before them as truncated, drop the rest. + * `/changes` from an older state then fails with `cannotCalculateChanges`. + */ + truncateChangeLog(accountId: string, group: TypeGroup | DataType, keep = 0): void { + const account = this.requireAccount(accountId); + const key: TypeGroup = group === 'contacts' || group === 'calendars' ? group : groupOf(group); + account.logs[key].truncate(keep); + } + + /** + * The `uid` query filter reads an index built after commit. With a lag of + * `n`, an object created during request R (or between R and R+1) is found + * by `uid` from request R+n+1 on. Applies to objects created afterwards. + */ + setUidIndexLag(n: number): void { + this.uidLag = Math.max(0, Math.floor(n)); + } + + setLimits(limits: Partial): void { + this.limits = { ...this.limits, ...limits }; + } + + /** Runs when a request arrives, before anything of it runs (e.g. a concurrent server edit). */ + onBeforeRequest(listener: RequestListener): () => void { + this.beforeListeners.add(listener); + return () => this.beforeListeners.delete(listener); + } + + /** Runs after a request ran, before its response reaches the caller (e.g. a device edit mid-upload). */ + onAfterRequest(listener: RequestListener): () => void { + this.afterListeners.add(listener); + return () => this.afterListeners.delete(listener); + } + + // ── Introspection ── + + /** Everything the server holds for an object: properties, parents and flags. */ + get(type: DataType, accountId: string, id: string): Obj | undefined { + const account = this.accounts.get(accountId); + const stored = account?.objects[type].get(id); + return account && stored ? this.snapshot(type, account, stored) : undefined; + } + + all(type: DataType, accountId: string): Obj[] { + const account = this.requireAccount(accountId); + return [...account.objects[type].values()].map((stored) => this.snapshot(type, account, stored)); + } + + state(type: DataType, accountId: string): string { + return this.stateOf(this.requireAccount(accountId), type); + } + + /** Method calls sent so far, in order; only those named `name` when given. */ + calls(name?: string): JmapInvocation[] { + const all = this.requests.flatMap((r) => r.calls); + return name ? all.filter(([method]) => method === name) : all; + } + + // ── Requests ── + + private async handleRequest(calls: JmapInvocation[], using: string[]): Promise { + this.inFlight++; + this.maxInFlight = Math.max(this.maxInFlight, this.inFlight); + try { + // Yield like a round trip, so requests a caller does not await overlap. + await Promise.resolve(); + return await this.runRequest(calls, using); + } finally { + this.inFlight--; + } + } + + private async runRequest(calls: JmapInvocation[], using: string[]): Promise { + const seq = ++this.requestSeq; + const record: RequestRecord = { + seq, + using: [...using], + calls: clone(calls), + methods: calls.map(([name]) => name), + responses: null, + outcome: 'ok', + }; + this.requests.push(record); + for (const listener of this.beforeListeners) await listener(record); + const fault = this.takeFault('request', calls); + if (fault && !fault.applied) { + record.outcome = 'failed'; + record.fault = fault.kind; + throw transportError(fault); + } + if (calls.length > this.limits.maxCallsInRequest) { + record.outcome = 'rejected'; + throw limitError('maxCallsInRequest', 'The request exceeds the maximum number of calls in a single request.'); + } + const size = new TextEncoder().encode(JSON.stringify({ using, methodCalls: calls })).length; + if (size > this.limits.maxSizeRequest) { + record.outcome = 'rejected'; + throw limitError('maxSizeRequest', 'The request is larger than the server is willing to process.'); + } + const responses: JmapInvocation[] = []; + for (const call of clone(calls)) responses.push(this.executeCall(call, using, responses, seq)); + record.responses = clone(responses); + for (const listener of this.afterListeners) await listener(record); + if (fault) { + record.outcome = 'lost'; + record.fault = fault.kind; + throw transportError(fault); + } + return { methodResponses: responses, sessionState: `fs${this.sessionVersion}` }; + } + + private async handleDownload(accountId: string, blobId: string): Promise { + this.downloads.push({ accountId, blobId }); + const fault = this.takeFault('download', null); + if (fault) throw transportError(fault); + const blob = this.accounts.get(accountId)?.blobs.get(blobId); + if (!blob) throw new Error('Failed to fetch blob: 404'); + return blob.bytes.slice(); + } + + private makeFault(kind: FaultKind, target: Fault['target'], opts: FaultOptions): Fault { + const match = opts.match; + return { + kind, + target, + applied: opts.applied ?? false, + retryAfterMs: opts.retryAfterMs ?? 1000, + remaining: opts.times ?? 1, + match: + typeof match === 'string' + ? (calls: JmapInvocation[]) => calls.some(([name]) => name === match) + : match, + }; + } + + private takeFault(target: Fault['target'], calls: JmapInvocation[] | null): Fault | null { + const index = this.faults.findIndex( + (f) => f.target === target && (!f.match || (calls !== null && f.match(calls))), + ); + if (index < 0) return null; + const fault = this.faults[index]; + if (--fault.remaining <= 0) this.faults.splice(index, 1); + return fault; + } + + private executeCall( + call: JmapInvocation, + using: string[], + prior: JmapInvocation[], + seq: number, + ): JmapInvocation { + const [name, rawArgs, callId] = call; + try { + if (!SUPPORTED_METHODS.has(name)) { + throw new MethodError('unknownMethod', `fake server: ${name} is not emulated`); + } + if (!isObj(rawArgs)) throw new MethodError('invalidArguments'); + const { args, references } = this.resolveReferences(rawArgs, prior); + const capability = CAPABILITY_OF[name.split('/')[0]]; + if (!using.includes(capability)) { + throw new MethodError( + 'unknownMethod', + `Method ${name} requires capability ${capability} which is not present in the "using" property.`, + ); + } + const injected = this.takeMethodError(name, args.accountId); + if (injected) throw new MethodError(injected.type, injected.description); + return [name, this.dispatch(name, args, { seq, references }), callId]; + } catch (err) { + if (!(err instanceof MethodError)) throw err; + return ['error', err.detail ? { type: err.type, description: err.detail } : { type: err.type }, callId]; + } + } + + private resolveReferences(raw: Obj, prior: JmapInvocation[]): { args: Obj; references: Set } { + const args: Obj = {}; + const references = new Set(); + for (const [key, value] of Object.entries(raw)) if (!key.startsWith('#')) setKey(args, key, value); + for (const [key, ref] of Object.entries(raw)) { + if (!key.startsWith('#')) continue; + const target = key.slice(1); + if (hasOwn(args, target)) throw new MethodError('invalidArguments', `Both ${target} and ${key} are given.`); + if (!isObj(ref) || typeof ref.resultOf !== 'string' || typeof ref.name !== 'string' || typeof ref.path !== 'string') { + throw new MethodError('invalidArguments', `Invalid result reference in ${key}.`); + } + // Stalwart resolves references to /get, /changes, /query and + // /queryChanges responses only; one to a /set is not found (verified). + const notFound = new MethodError('invalidResultReference', `Result reference to ${ref.resultOf}#${ref.name} not found.`); + const source = prior.find(([name, , id]) => id === ref.resultOf && name === ref.name); + if (!source || !REFERENCEABLE.test(source[0])) throw notFound; + const resolved = resolveReferencePath(source[1], ref.path); + if (!resolved) throw notFound; + setKey(args, target, clone(resolved.value)); + references.add(target); + } + return { args, references }; + } + + private takeMethodError(method: string, accountId: unknown): { type: string; description?: string } | null { + const index = this.methodErrors.findIndex( + (e) => e.method === method && (e.accountId === undefined || e.accountId === accountId), + ); + if (index < 0) return null; + const injected = this.methodErrors[index]; + if (--injected.remaining <= 0) this.methodErrors.splice(index, 1); + return injected.error; + } + + private dispatch(name: string, args: Obj, ctx: CallContext): Obj { + if (name === 'Core/echo') return args; + const [object, verb] = name.split('/') as [DataType, string]; + const account = this.methodAccount(args, object); + switch (verb) { + case 'get': + return this.methodGet(object, account, args); + case 'changes': + return this.methodChanges(object, account, args); + case 'query': + return this.methodQuery(object as ItemType, account, args, ctx); + case 'set': + return this.methodSet(object as ItemType, account, args, ctx); + default: + throw new MethodError('unknownMethod'); + } + } + + private methodAccount(args: Obj, type: DataType): Account { + if (typeof args.accountId !== 'string') throw new MethodError('invalidArguments', 'accountId is required.'); + const account = this.accounts.get(args.accountId); + if (!account) throw new MethodError('accountNotFound'); + if (!(groupOf(type) === 'contacts' ? account.contacts : account.calendars)) { + throw new MethodError('accountNotSupportedByMethod'); + } + return account; + } + + // ── /get ── + + private methodGet(type: DataType, account: Account, args: Obj): Obj { + const { ids, properties } = args; + if (ids != null && !Array.isArray(ids)) throw new MethodError('invalidArguments', 'ids must be an array or null.'); + if (properties != null && !Array.isArray(properties)) { + throw new MethodError('invalidArguments', 'properties must be an array or null.'); + } + if (Array.isArray(ids) && ids.length > this.limits.maxObjectsInGet) throw new MethodError('requestTooLarge'); + const store = account.objects[type]; + // `ids: null` silently returns only the first maxObjectsInGet objects (verified). + const targets: unknown[] = Array.isArray(ids) + ? ids + : [...store.keys()].slice(0, this.limits.maxObjectsInGet); + const list: Obj[] = []; + const notFound: unknown[] = []; + for (const id of targets) { + const stored = typeof id === 'string' ? store.get(id) : undefined; + if (stored) list.push(this.render(type, account, stored, properties as unknown[] | null | undefined)); + else notFound.push(id); + } + return { accountId: account.id, state: this.stateOf(account, type), list, notFound }; + } + + private render(type: DataType, account: Account, stored: Stored, properties: unknown[] | null | undefined): Obj { + const listed = properties ? properties.filter((p): p is string => typeof p === 'string') : null; + if (isContainer(type)) { + const full: Obj = { id: stored.id, ...clone(stored.data), isDefault: this.defaultContainerId(account, type) === stored.id }; + if (!listed) return full; + const out: Obj = { id: stored.id }; + for (const p of listed) setKey(out, p, hasOwn(full, p) ? full[p] : null); + return out; + } + const data = clone(stored.data); + if (type === 'ContactCard') { + if (!listed) return { id: stored.id, addressBookIds: booleanSet(stored.parents), ...data }; + const out: Obj = { id: stored.id }; + for (const p of listed) { + if (p === 'addressBookIds') out.addressBookIds = booleanSet(stored.parents); + else if (hasOwn(data, p)) setKey(out, p, data[p]); + } + return out; + } + // CalendarEvent: with `properties: null` Stalwart returns the JSCalendar + // properties plus only id, calendarIds, isDraft and isOrigin; the other + // JMAP-only fields (useDefaultAlerts, …) need an explicit list (verified). + const jmap: Obj = { + calendarIds: booleanSet(stored.parents), + isDraft: stored.flags.isDraft === true, + isOrigin: this.isOrigin(account, stored.data), + useDefaultAlerts: stored.flags.useDefaultAlerts === true, + mayInviteSelf: stored.flags.mayInviteSelf === true, + mayInviteOthers: stored.flags.mayInviteOthers === true, + hideAttendees: stored.flags.hideAttendees === true, + baseEventId: null, + }; + if (!listed) { + return { ...data, id: stored.id, calendarIds: jmap.calendarIds, isDraft: jmap.isDraft, isOrigin: jmap.isOrigin }; + } + const out: Obj = { id: stored.id }; + for (const p of listed) { + if (hasOwn(jmap, p)) setKey(out, p, jmap[p]); + else if (hasOwn(data, p)) setKey(out, p, data[p]); + } + return out; + } + + private snapshot(type: DataType, account: Account, stored: Stored): Obj { + if (isContainer(type)) { + return { id: stored.id, ...clone(stored.data), isDefault: this.defaultContainerId(account, type) === stored.id }; + } + return { + id: stored.id, + [parentProperty(type)]: booleanSet(stored.parents), + ...clone(stored.data), + ...clone(stored.flags), + }; + } + + private isOrigin(account: Account, data: Obj): boolean { + const organizer = data.organizerCalendarAddress; + if (typeof organizer !== 'string') return true; + const address = organizer.replace(/^mailto:/i, '').toLowerCase(); + return account.addresses.some((a) => a.toLowerCase() === address); + } + + // ── /changes ── + + /** jmap/src/changes/get.rs, step by step. */ + private methodChanges(type: DataType, account: Account, args: Obj): Obj { + const since = args.sinceState; + if (typeof since !== 'string') throw new MethodError('invalidArguments', 'sinceState is required.'); + const parsed = parseState(since); + if (!parsed) throw new MethodError('invalidArguments', `Invalid state ${since}.`); + const maxArg = args.maxChanges; + if (maxArg != null && (typeof maxArg !== 'number' || !Number.isInteger(maxArg) || maxArg < 0)) { + throw new MethodError('invalidArguments', 'maxChanges must be a positive integer.'); + } + const max = Math.min(typeof maxArg === 'number' && maxArg > 0 ? maxArg : Infinity, this.limits.maxChanges); + const side = sideOf(type); + const log = account.logs[groupOf(type)]; + const response = { + accountId: account.id, + oldState: since, + newState: 'n', + hasMoreChanges: false, + created: [] as string[], + updated: [] as string[], + destroyed: [] as string[], + }; + let newState: ParsedState = { kind: 'initial' }; + let itemsSent = 0; + let window: ChangeWindow; + if (parsed.kind === 'initial') { + window = log.query({ all: true }); + if (window.changes.size === 0 && window.fromChangeId === 0) return response; + } else if (parsed.kind === 'exact') { + const last = log.sideChangeId(side); + newState = stateOfChangeId(last); + if (last === parsed.id) return { ...response, newState: encodeState(newState) }; + window = log.query({ since: parsed.id }); + } else { + window = log.query({ from: parsed.from, to: parsed.to }); + const total = [...window.changes.values()].filter((c) => c.side === side).length; + if (parsed.sent >= total) window = log.query({ since: parsed.to }); + else itemsSent = parsed.sent; + } + if ((window.isTruncated || window.fromChangeId === 0) && parsed.kind !== 'initial') { + throw new MethodError( + 'cannotCalculateChanges', + window.isTruncated ? 'Change log is truncated' : 'Since state is invalid', + ); + } + const changes = [...window.changes.values()].filter((c) => c.side === side); + for (const change of changes.slice(itemsSent, itemsSent + max)) { + if (change.op === 'insert') response.created.push(change.id); + else if (change.op === 'update') response.updated.push(change.id); + else response.destroyed.push(change.id); + } + const changeId = (side === 'container' ? window.containerChangeId : window.itemChangeId) ?? window.toChangeId; + response.hasMoreChanges = changes.length > itemsSent + max; + if (response.hasMoreChanges) { + newState = { kind: 'intermediate', from: window.fromChangeId, to: changeId, sent: itemsSent + max }; + } else if (newState.kind === 'initial') { + newState = { kind: 'exact', id: changeId }; + } + response.newState = encodeState(newState); + return response; + } + + // ── /query ── + + private methodQuery(type: ItemType, account: Account, args: Obj, ctx: CallContext): Obj { + if (args.anchor != null) throw new MethodError('invalidArguments', 'fake server: anchor is not emulated'); + if (args.expandRecurrences === true) { + throw new MethodError('invalidArguments', 'fake server: expandRecurrences is not emulated'); + } + const inParent = type === 'ContactCard' ? 'inAddressBook' : 'inCalendar'; + const validate = (filter: unknown): void => { + if (filter == null) return; + if (!isObj(filter)) throw new MethodError('invalidArguments', 'filter must be an object.'); + if (filter.operator !== undefined) { + if (!['AND', 'OR', 'NOT'].includes(String(filter.operator)) || !Array.isArray(filter.conditions)) { + throw new MethodError('unsupportedFilter', `Invalid filter operator ${String(filter.operator)}.`); + } + filter.conditions.forEach(validate); + return; + } + for (const key of Object.keys(filter)) { + if (key !== inParent && key !== 'uid') { + throw new MethodError('unsupportedFilter', `fake server: filter ${key} is not emulated`); + } + } + }; + // `inAddressBook`/`inCalendar` read the synchronous cache; `uid` reads + // the search index, which lags behind writes (verified). + const matches = (stored: Stored, filter: unknown): boolean => { + if (!isObj(filter)) return true; + if (filter.operator !== undefined) { + const conditions = filter.conditions as unknown[]; + if (filter.operator === 'AND') return conditions.every((c) => matches(stored, c)); + if (filter.operator === 'OR') return conditions.some((c) => matches(stored, c)); + return !conditions.some((c) => matches(stored, c)); + } + if (filter[inParent] !== undefined && !stored.parents.includes(filter[inParent] as string)) return false; + if (filter.uid !== undefined && (stored.data.uid !== filter.uid || stored.uidVisibleFrom > ctx.seq)) return false; + return true; + }; + validate(args.filter); + let results = [...account.objects[type].values()].filter((stored) => matches(stored, args.filter)); + results = this.sortResults(type, results, args.sort); + + const total = results.length; + const cap = this.limits.maxQueryResults; + const requested = args.limit; + if (requested != null && (typeof requested !== 'number' || requested < 0)) { + throw new MethodError('invalidArguments', 'limit must be a positive integer.'); + } + const limit = typeof requested === 'number' ? Math.min(requested, cap) : cap; + const take = Math.min(limit, total); + const position = typeof args.position === 'number' ? Math.trunc(args.position) : 0; + let ids: string[]; + let responsePosition: number; + if (position >= 0) { + ids = results.slice(position, position + take).map((s) => s.id); + responsePosition = position <= total ? position : 0; + } else { + const start = Math.max(0, total + position); + ids = results.slice(start, start + take).map((s) => s.id); + responsePosition = start; + } + const response: Obj = { + accountId: account.id, + queryState: this.stateOf(account, type), + canCalculateChanges: true, + position: responsePosition, + ids, + }; + if (args.calculateTotal === true) response.total = total; + if (total > limit) response.limit = limit; + return response; + } + + private sortResults(type: ItemType, results: Stored[], sort: unknown): Stored[] { + if (sort == null) return results; + if (!Array.isArray(sort)) throw new MethodError('invalidArguments', 'sort must be an array.'); + const comparators = sort.map((comparator) => { + const property = isObj(comparator) ? comparator.property : undefined; + const ascending = !isObj(comparator) || comparator.isAscending !== false; + const allowed = type === 'ContactCard' ? ['created', 'updated'] : ['start', 'recurrenceId', 'uid']; + if (typeof property !== 'string' || !allowed.includes(property)) { + throw new MethodError('unsupportedSort', String(property)); + } + return (a: Stored, b: Stored): number => { + let diff: number; + if (property === 'created') diff = a.docId - b.docId; + else if (property === 'updated') diff = a.modSeq - b.modSeq; + else { + const field = property === 'uid' ? 'uid' : 'start'; + diff = String(a.data[field] ?? '').localeCompare(String(b.data[field] ?? '')); + } + return ascending ? diff : -diff; + }; + }); + return [...results].sort((a, b) => { + for (const compare of comparators) { + const diff = compare(a, b); + if (diff !== 0) return diff; + } + return a.docId - b.docId; + }); + } + + // ── /set ── + + private methodSet(type: ItemType, account: Account, args: Obj, ctx: CallContext): Obj { + const create = args.create ?? null; + const update = args.update ?? null; + const destroy = args.destroy ?? null; + if ((create !== null && !isObj(create)) || (update !== null && !isObj(update)) || (destroy !== null && !Array.isArray(destroy))) { + throw new MethodError('invalidArguments'); + } + const creates = Object.entries(create ?? {}); + const updates = Object.entries(update ?? {}); + const destroys = (destroy ?? []) as unknown[]; + // A destroy given as a result reference does not count (jmap-proto method/set.rs). + const counted = creates.length + updates.length + (ctx.references.has('destroy') ? 0 : destroys.length); + if (counted > this.limits.maxObjectsInSet) throw new MethodError('requestTooLarge'); + const oldState = this.stateOf(account, type); + if (args.ifInState != null) { + if (typeof args.ifInState !== 'string' || !parseState(args.ifInState)) { + throw new MethodError('invalidArguments', 'Invalid ifInState.'); + } + if (args.ifInState !== oldState) throw new MethodError('stateMismatch'); + } + if (account.isReadOnly) throw new MethodError('accountReadOnly'); + + const store = account.objects[type]; + const scheduling = type === 'CalendarEvent' && args.sendSchedulingMessages === true; + const record = (op: SchedulingRecord['op'], id: string) => { + if (scheduling) this.scheduling.push({ accountId: account.id, op, id, requestSeq: ctx.seq }); + }; + const pending = new PendingChanges(); + const createdInCall = new Set(); + const willDestroy = new Set(destroys.filter((id): id is string => typeof id === 'string')); + const created: Obj = {}; + const notCreated: Obj = {}; + const updated: Obj = {}; + const notUpdated: Obj = {}; + const destroyed: string[] = []; + const notDestroyed: Obj = {}; + + for (const [creationId, object] of creates) { + const injected = this.takeSetError(type, account.id, { + op: 'create', + id: creationId, + object: isObj(object) ? object : undefined, + }); + if (injected) { + notCreated[creationId] = injected; + continue; + } + if (!isObj(object)) { + notCreated[creationId] = { type: 'invalidProperties', description: 'Invalid object.' }; + continue; + } + const draft = this.prepareCreate(type, account, object, createdInCall); + if ('error' in draft) { + notCreated[creationId] = draft.error; + continue; + } + const id = this.mintId(account, type); + this.storeNew(account, type, id, draft); + createdInCall.add(id); + pending.insert('item', id); + created[creationId] = { id }; + record('create', id); + } + + for (const [id, patch] of updates) { + if (willDestroy.has(id)) { + notUpdated[id] = { type: 'willDestroy' }; + continue; + } + const injected = this.takeSetError(type, account.id, { op: 'update', id, object: isObj(patch) ? patch : undefined }); + if (injected) { + notUpdated[id] = injected; + continue; + } + const stored = store.get(id); + if (!stored || createdInCall.has(id)) { + notUpdated[id] = { type: 'notFound' }; + continue; + } + if (!isObj(patch)) { + notUpdated[id] = { type: 'invalidPatch' }; + continue; + } + const draft = this.prepareUpdate(type, account, stored, patch); + if ('error' in draft) { + notUpdated[id] = draft.error; + continue; + } + Object.assign(stored, draft); + stored.modSeq = ++this.modSeq; + pending.update('item', id); + updated[id] = null; + record('update', id); + } + + for (const id of destroys) { + const key = String(id); + const injected = this.takeSetError(type, account.id, { op: 'destroy', id: key }); + if (injected) { + notDestroyed[key] = injected; + continue; + } + const stored = typeof id === 'string' ? store.get(id) : undefined; + if (!stored || createdInCall.has(key)) { + notDestroyed[key] = { type: 'notFound' }; + continue; + } + const denied = stored.parents.find((p) => !this.canWrite(account, containerOf(type), p)); + if (denied) { + notDestroyed[key] = { + type: 'forbidden', + description: + type === 'ContactCard' + ? `You are not allowed to remove contacts from address book ${denied}.` + : `You are not allowed to remove calendar events from calendar ${denied}.`, + }; + continue; + } + store.delete(key); + pending.delete('item', key); + destroyed.push(key); + record('destroy', key); + } + + const changeId = this.commit(account, groupOf(type), pending); + // Stalwart leaves empty maps and lists out of a /set response; creates + // report only `{id}` and updates `null` (verified). + const response: Obj = { + accountId: account.id, + oldState, + newState: changeId === null ? oldState : encodeState({ kind: 'exact', id: changeId }), + }; + if (Object.keys(created).length) response.created = created; + if (Object.keys(updated).length) response.updated = updated; + if (destroyed.length) response.destroyed = destroyed; + if (Object.keys(notCreated).length) response.notCreated = notCreated; + if (Object.keys(notUpdated).length) response.notUpdated = notUpdated; + if (Object.keys(notDestroyed).length) response.notDestroyed = notDestroyed; + return response; + } + + private takeSetError(type: ItemType, accountId: string, target: SetTarget): SetErrorShape | null { + const index = this.setErrors.findIndex((e) => e.type === type && e.accountId === accountId && e.match(target)); + if (index < 0) return null; + const injected = this.setErrors[index]; + if (--injected.remaining <= 0) this.setErrors.splice(index, 1); + return clone(injected.error); + } + + /** + * Apply a create object or PatchObject the way `update_contact_card` and + * `update_calendar_event` do: keys with `/` are JSON Pointers (a missing + * parent fails the patch, verified), parent ids take a whole map or + * `/: true|false|null`, unknown top-level names are invalid. + */ + private applyItemProperties(type: ItemType, draft: Draft, props: Obj, expectedId: string | null): SetErrorShape | null { + const parentProp = parentProperty(type); + for (const [key, value] of Object.entries(props)) { + if (key.includes('/')) { + const segments = parsePointer(key); + const first = segments[0]; + if (first?.kind === 'key' && first.key === parentProp) { + const second = segments[1]; + const parentId = second?.kind === 'key' ? second.key : undefined; + if (parentId === undefined || (value !== true && value !== false && value !== null)) { + return invalidProperties([parentProp], `Invalid patch operation for ${parentProp}.`); + } + draft.parents = draft.parents.filter((p) => p !== parentId); + if (value === true) draft.parents.push(parentId); + continue; + } + if (!applyPointer(draft.data, segments, value)) return invalidProperties([key], 'Patch operation failed.'); + continue; + } + if (key === parentProp) { + if (!isObj(value)) return invalidProperties([parentProp], `Invalid patch operation for ${parentProp}.`); + const next = Object.keys(value).filter((id) => value[id] === true); + draft.parents = [...draft.parents.filter((p) => next.includes(p)), ...next.filter((p) => !draft.parents.includes(p))]; + continue; + } + if (key === 'id') { + if (expectedId === null || value !== expectedId) { + return invalidProperties(['id'], type === 'ContactCard' ? 'The id property is immutable.' : 'This property is immutable.'); + } + continue; + } + if (type === 'CalendarEvent') { + if (IMMUTABLE_EVENT_PROPERTIES.has(key)) return invalidProperties([key], 'This property is immutable.'); + if (EVENT_FLAGS.includes(key)) { + if (typeof value !== 'boolean') return invalidProperties([key], 'Invalid value.'); + draft.flags[key] = value; + continue; + } + if (key === 'utcStart' || key === 'utcEnd') { + return invalidProperties([key], 'fake server: utcStart/utcEnd writes are not emulated'); + } + if ((key === 'participants' || key === 'locations') && isObj(value) && Object.values(value).some(hasBlobLink)) { + return invalidProperties([key], 'blobIds in links is not supported.'); + } + } else if (key === 'media' && isObj(value) && Object.values(value).some((m) => isObj(m) && hasOwn(m, 'blobId'))) { + return invalidProperties(['media'], 'blobIds in media is not supported.'); + } + if (!(type === 'ContactCard' ? CARD_PROPERTIES : EVENT_PROPERTIES).has(key)) { + return invalidProperties([key], 'Invalid property.'); + } + assignOrDelete(draft.data, key, value); + } + if (draft.parents.length === 0) { + return invalidProperties( + [parentProp], + type === 'ContactCard' + ? 'Contact has to belong to at least one address book.' + : 'Event has to belong to at least one calendar.', + ); + } + return null; + } + + private prepareCreate(type: ItemType, account: Account, object: Obj, createdInCall: Set): Draft | { error: SetErrorShape } { + const draft: Draft = { data: {}, parents: [], flags: {} }; + const error = this.applyItemProperties(type, draft, object, null); + if (error) return { error }; + const containerType = containerOf(type); + for (const parent of draft.parents) { + if (!account.objects[containerType].has(parent)) { + return { + error: invalidProperties( + [parentProperty(type)], + type === 'ContactCard' ? `addressBookId ${parent} does not exist.` : `calendarId ${parent} does not exist.`, + ), + }; + } + if (!this.canWrite(account, containerType, parent)) { + return { + error: { + type: 'forbidden', + description: + type === 'ContactCard' + ? `You are not allowed to add contacts to address book ${parent}.` + : `You are not allowed to add calendar events to calendar ${parent}.`, + }, + }; + } + } + // Both uid checks read what was committed before this call, so two + // creates with one uid in the same call both pass (as on Stalwart). + const earlier = [...account.objects[type].values()].filter((s) => !createdInCall.has(s.id)); + if (type === 'ContactCard') { + draft.data = normalizeCard(draft.data); + const uid = draft.data.uid; + // Contact uids are unique per address book; the error names the card (verified). + const clash = typeof uid === 'string' + ? earlier.find((s) => s.data.uid === uid && s.parents.some((p) => draft.parents.includes(p))) + : undefined; + if (clash) { + return { error: invalidProperties(['uid'], `Contact with UID ${String(uid)} already exists with id ${clash.id}.`) }; + } + return draft; + } + // Stalwart generates a UUID when the create has no uid. + if (typeof draft.data.uid !== 'string') draft.data.uid = this.mintUid(); + if (draft.flags.useDefaultAlerts === true) this.addDefaultAlerts(account, draft); + draft.data = normalizeEvent(draft.data, this.nowUtc(), 'jmap'); + this.assignOrganizer(account, draft.data); + // Event uids are unique per account; the error names no event (verified). + const uid = draft.data.uid; + if (earlier.some((s) => s.data.uid === uid)) { + return { error: invalidProperties(['uid'], `An event with UID ${String(uid)} already exists.`) }; + } + return draft; + } + + private prepareUpdate(type: ItemType, account: Account, stored: Stored, patch: Obj): Draft | { error: SetErrorShape } { + const draft: Draft = { data: clone(stored.data), parents: [...stored.parents], flags: { ...stored.flags } }; + const error = this.applyItemProperties(type, draft, patch, stored.id); + if (error) return { error }; + if (type === 'ContactCard') { + draft.data = normalizeCard(draft.data); + } else { + draft.data = normalizeEvent(draft.data, this.nowUtc(), 'jmap'); + this.assignOrganizer(account, draft.data); + } + // A uid can be stripped (`uid: null`) but not changed or added. + if (draft.data.uid !== undefined && draft.data.uid !== stored.data.uid) { + return { + error: invalidProperties( + ['uid'], + type === 'ContactCard' ? 'You cannot change the UID of a contact.' : 'You cannot change the UID of a calendar event.', + ), + }; + } + const containerType = containerOf(type); + const noun = type === 'ContactCard' ? 'contacts' : 'calendar events'; + const place = type === 'ContactCard' ? 'address book' : 'calendar'; + for (const parent of draft.parents) { + if (stored.parents.includes(parent)) continue; + if (!account.objects[containerType].has(parent)) { + return { + error: invalidProperties( + [parentProperty(type)], + type === 'ContactCard' ? `addressBookId ${parent} does not exist.` : `calendarId ${parent} does not exist.`, + ), + }; + } + if (!this.canWrite(account, containerType, parent)) { + return { error: { type: 'forbidden', description: `You are not allowed to add ${noun} to ${place} ${parent}.` } }; + } + } + for (const parent of stored.parents) { + if (draft.parents.includes(parent) || this.canWrite(account, containerType, parent)) continue; + return { error: { type: 'forbidden', description: `You are not allowed to remove ${noun} from ${place} ${parent}.` } }; + } + for (const parent of draft.parents) { + if (!stored.parents.includes(parent) || this.canWrite(account, containerType, parent)) continue; + return { error: { type: 'forbidden', description: `You are not allowed to modify ${place} ${parent}.` } }; + } + return draft; + } + + /** + * `useDefaultAlerts: true` on create copies the calendars' default alerts + * into the event as alarms without a JSID, so they read back with the + * positional keys `k` after the event's own alerts. (Stalwart can then + * emit a duplicate key; here a taken key moves to the next free one.) + */ + private addDefaultAlerts(account: Account, draft: Draft): void { + const withoutTime = draft.data.showWithoutTime === true; + const alerts: Obj = isObj(draft.data.alerts) ? draft.data.alerts : {}; + let position = Object.keys(alerts).length; + for (const calendarId of draft.parents) { + const calendar = account.objects.Calendar.get(calendarId); + const defaults = calendar?.data[withoutTime ? 'defaultAlertsWithoutTime' : 'defaultAlertsWithTime']; + if (!isObj(defaults)) continue; + for (const alert of Object.values(defaults)) { + let key = `k${++position}`; + while (hasOwn(alerts, key)) key = `k${++position}`; + setKey(alerts, key, clone(alert)); + } + } + if (Object.keys(alerts).length > 0) draft.data.alerts = alerts; + } + + /** + * `itip_assign_organizer`: an event whose participants include attendees + * but that has no organizer gets the account's first address as organizer, + * which reads back as `organizerCalendarAddress` plus an owner participant + * keyed uuid5 of the address (merged into a participant with that address, + * if any). Approximated: only the master, and "attendee" means a + * participant with a calendar address that is not only an owner. + */ + private assignOrganizer(account: Account, event: Obj): void { + const address = account.addresses[0]; + const participants = event.participants; + if (!address || typeof event.organizerCalendarAddress === 'string' || !isObj(participants)) return; + const hasAttendee = Object.values(participants).some((p) => { + if (!isObj(p) || typeof p.calendarAddress !== 'string') return false; + const roles = isObj(p.roles) ? Object.keys(p.roles).filter((role) => (p.roles as Obj)[role] === true) : []; + return !(roles.length === 1 && roles[0] === 'owner'); + }); + if (!hasAttendee) return; + const uri = `mailto:${address}`; + event.organizerCalendarAddress = uri; + const existing = Object.values(participants).find((p) => isObj(p) && p.calendarAddress === uri); + if (isObj(existing)) { + existing.roles = { ...(isObj(existing.roles) ? existing.roles : {}), owner: true }; + } else { + setKey(participants, uuid5(uri), { '@type': 'Participant', calendarAddress: uri, roles: { owner: true } }); + } + } + + // ── Storage ── + + private insertContainer(type: ContainerType, accountId: string, input: Obj): string { + const account = this.requireAccount(accountId); + const { id: explicitId, isDefault, ...rest } = clone(input); + const id = this.mintId(account, type, explicitId); + const data = { ...clone(type === 'AddressBook' ? ADDRESS_BOOK_DEFAULTS : CALENDAR_DEFAULTS), ...rest }; + if (isDefault === true) account.defaultContainer[type] = id; + this.storeNew(account, type, id, { data, parents: [], flags: {} }); + const pending = new PendingChanges(); + pending.insert('container', id); + this.commit(account, groupOf(type), pending); + return id; + } + + private insertItem(type: ItemType, accountId: string, input: Obj): string { + const account = this.requireAccount(accountId); + const containerType = containerOf(type); + const object = clone(input); + const explicitId = object.id; + const parentInput = object[parentProperty(type)]; + delete object.id; + delete object[parentProperty(type)]; + let parents: string[]; + if (parentInput === undefined) { + const fallback = this.defaultContainerId(account, containerType); + if (!fallback) throw new Error(`fake server: ${accountId} has no ${containerType} for the new ${type}`); + parents = [fallback]; + } else if (isObj(parentInput)) { + parents = Object.keys(parentInput).filter((p) => parentInput[p] === true); + } else { + throw new Error(`fake server: ${parentProperty(type)} must be a map`); + } + for (const parent of parents) this.requireObject(account, containerType, parent); + const flags: Obj = {}; + if (type === 'CalendarEvent') { + for (const flag of EVENT_FLAGS) { + if (!hasOwn(object, flag)) continue; + flags[flag] = object[flag]; + delete object[flag]; + } + } + const data = type === 'ContactCard' ? normalizeCard(object) : normalizeEvent(object, this.nowUtc(), 'server'); + const id = this.mintId(account, type, explicitId); + this.storeNew(account, type, id, { data, parents, flags }); + const pending = new PendingChanges(); + pending.insert('item', id); + this.commit(account, groupOf(type), pending); + return id; + } + + private storeNew(account: Account, type: DataType, id: string, draft: Draft): void { + account.objects[type].set(id, { + id, + docId: ++account.docCounter[type], + data: draft.data, + parents: draft.parents, + flags: draft.flags, + uidVisibleFrom: this.uidLag === 0 ? 0 : this.requestSeq + this.uidLag + 1, + modSeq: ++this.modSeq, + }); + } + + /** ``, per account and type; ids are never reused. */ + private mintId(account: Account, type: DataType, explicit?: unknown): string { + const used = account.usedIds[type]; + if (explicit !== undefined) { + if (typeof explicit !== 'string' || !ID_PATTERN.test(explicit)) throw new Error(`fake server: invalid id ${String(explicit)}`); + if (used.has(explicit)) throw new Error(`fake server: ${type} id ${explicit} was used before (ids are never reused)`); + used.add(explicit); + return explicit; + } + let id: string; + do { + id = `${ID_PREFIX[type]}${(++account.idCounter[type]).toString(36)}`; + } while (used.has(id)); + used.add(id); + return id; + } + + private mintUid(): string { + return `00000000-0000-4000-8000-${(++this.uidCounter).toString(16).padStart(12, '0')}`; + } + + /** One commit: a new account-wide change id in the collection's log. */ + private commit(account: Account, group: TypeGroup, pending: PendingChanges): number | null { + const changes = pending.toChanges(); + if (changes.length === 0) return null; + const changeId = ++account.lastChangeId; + account.logs[group].append(changeId, changes); + return changeId; + } + + private stateOf(account: Account, type: DataType): string { + return encodeState(stateOfChangeId(account.logs[groupOf(type)].sideChangeId(sideOf(type)))); + } + + /** The container marked default, else the oldest one (Stalwart: the lowest id). */ + private defaultContainerId(account: Account, type: ContainerType): string | undefined { + const marked = account.defaultContainer[type]; + if (marked && account.objects[type].has(marked)) return marked; + return account.objects[type].keys().next().value; + } + + private canWrite(account: Account, type: ContainerType, id: string): boolean { + const rights = account.objects[type].get(id)?.data.myRights; + if (!isObj(rights)) return true; + return type === 'AddressBook' ? rights.mayWrite !== false : rights.mayWriteAll !== false; + } + + private requireAccount(accountId: string): Account { + const account = this.accounts.get(accountId); + if (!account) throw new Error(`fake server: no account ${accountId}`); + return account; + } + + private requireObject(account: Account, type: DataType, id: string): Stored { + const stored = account.objects[type].get(id); + if (!stored) throw new Error(`fake server: no ${type} ${id} in ${account.id}`); + return stored; + } +} + +function hasBlobLink(entry: unknown): boolean { + if (!isObj(entry) || !isObj(entry.links)) return false; + return Object.values(entry.links).some((link) => isObj(link) && hasOwn(link, 'blobId')); +} diff --git a/src/device-sync/__tests__/fakes/fake-provider.ts b/src/device-sync/__tests__/fakes/fake-provider.ts new file mode 100644 index 00000000..c642f0c6 --- /dev/null +++ b/src/device-sync/__tests__/fakes/fake-provider.ts @@ -0,0 +1,1209 @@ +/** + * In-memory stand-in for Android's Contacts and Calendar providers, as the + * device sync engine reaches them through the native bridge (`ProviderPort`), + * plus a "user" face that edits rows the way Contacts and Calendar apps do. + * + * It models the provider behaviour the engine relies on (docs/device-sync.md, + * "Device data model", and the AOSP facts quoted there): + * + * - Sync-adapter writes never set DIRTY; app writes do (data rows, events and + * their attendees/reminders). The provider never clears DIRTY. + * - RawContacts.VERSION goes up on every data-row insert, update or delete, + * whoever makes it, and on a DELETED change; never for DIRTY, SOURCE_ID or + * SYNC1-4. Groups bump VERSION on every update. + * - App deletes are soft for raw contacts (always) and for events with a + * `_sync_id`; an app-deleted event without one is gone at once. Sync-adapter + * deletes are hard, and an event's exceptions survive the master. + * - Batches are atomic here (a real provider may commit at yield points under + * contention; the engine must not rely on either), at most 499 contacts ops + * between yield points, asserts compare as text. + * - Normalisation: a lone display name is split, missing display name and + * formatted address are derived, the organizer defaults to the calendar + * owner, all-day times are zeroed in UTC and `PS` all-day durations become + * `PD`, while `PT0S`-style all-day durations, NULL STATUS updates, + * SELF_ATTENDEE_STATUS updates and RRULEs with unknown parts fail the batch. + * - Scoping: rows of other accounts are invisible and untouchable. + * + * Pure TypeScript, for vitest only. + */ +import { + Attendees, + Calendars, + Colors, + ContactsSettings, + Data, + Events, + ExtendedProperties, + GroupMembership, + Groups, + MimeType, + RawContacts, + Reminders, + StructuredName, + StructuredPostal, +} from '../../android-columns'; +import { + CALENDAR_AUTHORITY, + CALENDAR_TABLES, + CONTACTS_AUTHORITY, + CONTACTS_TABLES, + type Authority, + type BatchFailure, + type BatchResult, + type Cell, + type OpResult, + type PhotoData, + type ProviderOp, + type ProviderPort, + type ProviderQuery, + type ProviderRows, + type ProviderTable, + type Row, + type WriteCell, + type WriteRow, +} from '../../types'; +import { compileOrderBy, compileWhere, whereColumns } from './fake-sql'; + +export const FAKE_ACCOUNT_TYPE = 'com.anonymous.bulwarkmobile.account'; +const APP_PACKAGE = 'com.example.calendarapp'; + +type StoredRow = Record; + +/** Rows keyed by `_id`, which every stored row also carries as a column. */ +class RowMap extends Map { + override set(id: number, row: StoredRow): this { + row._id = id; + return super.set(id, row); + } +} +type Tables = Record; + +const numbered = (prefix: string, from: number, to: number) => + Array.from({ length: to - from + 1 }, (_, i) => `${prefix}${from + i}`); + +const COLUMNS: Record> = { + raw_contacts: new Set([...Object.values(RawContacts), 'custom_ringtone', 'send_to_voicemail', 'backup_id']), + data: new Set([...Object.values(Data), GroupMembership.GROUP_SOURCE_ID]), + groups: new Set([...Object.values(Groups), 'auto_add', 'favorites']), + settings: new Set(Object.values(ContactsSettings)), + calendars: new Set([...Object.values(Calendars), ...numbered('cal_sync', 4, 10), 'canPartiallyUpdate']), + events: new Set([...Object.values(Events), ...numbered('sync_data', 7, 10), 'lastSynced']), + attendees: new Set(Object.values(Attendees)), + reminders: new Set(Object.values(Reminders)), + extended_properties: new Set(Object.values(ExtendedProperties)), + colors: new Set(Object.values(Colors)), +}; + +/** Columns only the provider computes; writes to them are refused like SQLite would for a view. */ +const READ_ONLY_COLUMNS: Partial>> = { + data: new Set([GroupMembership.GROUP_SOURCE_ID]), + events: new Set([Events.ACCOUNT_NAME, Events.ACCOUNT_TYPE]), +}; + +class FakeProviderError extends Error { + constructor(readonly reason: BatchFailure, message: string) { + super(message); + } +} + +const isEmpty = (v: Cell | undefined) => v === null || v === undefined || v === ''; + +/** AOSP calendarcommon2 Duration.parse: `[+-]P` then ``, `T` ignored. */ +export function aospDurationValid(value: string): boolean { + return /^[+-]?P(T?\d+[WDHMS])*T?$/.test(value) && /\d/.test(value); +} + +/** AOSP EventRecurrence.parse: known parts only (X-* ignored), FREQ required, no RRULE: prefix. */ +export function aospRruleValid(value: string): boolean { + const known = new Set([ + 'FREQ', 'UNTIL', 'COUNT', 'INTERVAL', 'BYSECOND', 'BYMINUTE', 'BYHOUR', 'BYDAY', 'BYMONTHDAY', + 'BYYEARDAY', 'BYWEEKNO', 'BYMONTH', 'BYSETPOS', 'WKST', + ]); + return value.split('\n').every((rule) => { + const seen = new Set(); + let freq = false; + for (const part of rule.toUpperCase().split(';')) { + if (!part) continue; + const [name] = part.split('='); + if (name.startsWith('X-')) continue; + if (!known.has(name) || seen.has(name)) return false; + seen.add(name); + if (name === 'FREQ') freq = true; + } + return freq; + }); +} + +export interface FakeProviderOptions { + accountType?: string; + /** Derive/split names and addresses like ContactsProvider (default true). */ + normalize?: boolean; + /** Byte budget of one applyBatch before it fails with `tooLarge` (default 1 MB). */ + maxBatchBytes?: number; +} + +/** + * Both providers of one device. `port(account, authority)` is what the engine + * gets; `user` edits like apps do; the rest is for test setup and assertions. + */ +export class FakeDeviceProviders { + readonly accountType: string; + private readonly normalize: boolean; + private readonly maxBatchBytes: number; + private tables: Tables = FakeDeviceProviders.emptyTables(); + private syncState = new Map(); + private photos = new Map(); + private nextId = 1; + private nextPhotoFile = 1000; + private pendingFailure: { reason: BatchFailure; message: string } | null = null; + private beforeBatch: Array<() => void> = []; + /** Every batch applied or refused, for assertions on write counts. */ + readonly batches: Array<{ account: string; authority: Authority; ops: ProviderOp[]; ok: boolean }> = []; + + readonly user = new FakeUser(this); + + constructor(options: FakeProviderOptions = {}) { + this.accountType = options.accountType ?? FAKE_ACCOUNT_TYPE; + this.normalize = options.normalize ?? true; + this.maxBatchBytes = options.maxBatchBytes ?? 1_000_000; + } + + private static emptyTables(): Tables { + return { + raw_contacts: new RowMap(), + data: new RowMap(), + groups: new RowMap(), + settings: new RowMap(), + calendars: new RowMap(), + events: new RowMap(), + attendees: new RowMap(), + reminders: new RowMap(), + extended_properties: new RowMap(), + colors: new RowMap(), + }; + } + + port(accountName: string, authority: Authority): ProviderPort { + return new FakePort(this, accountName, authority); + } + + // ── test hooks ───────────────────────────────────────── + + /** The next applyBatch fails with this reason, applying nothing. */ + failNextBatch(reason: BatchFailure, message = `injected ${reason}`): void { + this.pendingFailure = { reason, message }; + } + + /** Runs `fn` right before the next applyBatch is applied (a concurrent user edit). */ + beforeNextBatch(fn: () => void): void { + this.beforeBatch.push(fn); + } + + /** Removes an account the way AccountManager does: the providers drop all its rows and SyncState. */ + removeAccount(accountName: string): void { + for (const table of ['raw_contacts', 'groups', 'settings', 'calendars', 'colors'] as const) { + for (const [id, row] of this.tables[table]) { + if (row.account_name === accountName && row.account_type === this.accountType) this.deleteRow(table, id); + } + } + for (const authority of [CONTACTS_AUTHORITY, CALENDAR_AUTHORITY]) this.syncState.delete(`${authority}|${accountName}`); + } + + // ── inspection ───────────────────────────────────────── + + rows(table: ProviderTable, where?: string, args: Array = []): Row[] { + const pred = compileWhere(where, args); + return [...this.tables[table].values()] + .filter((row) => pred((c) => this.readCell(table, row, c))) + .map((row) => ({ ...this.view(table, row) })); + } + + row(table: ProviderTable, id: number): Row | undefined { + const row = this.tables[table].get(id); + return row ? { ...this.view(table, row) } : undefined; + } + + /** Stored photo bytes (base64) by PHOTO_FILE_ID. */ + photo(fileId: number): string | undefined { + return this.photos.get(fileId); + } + + readSyncState(accountName: string, authority: Authority): string | null { + return this.syncState.get(`${authority}|${accountName}`) ?? null; + } + + // ── internals shared by the port and the user face ───── + + /** @internal */ allocId(): number { + return this.nextId++; + } + + /** @internal */ table(table: ProviderTable): RowMap { + return this.tables[table]; + } + + /** @internal */ readCell(table: ProviderTable, row: StoredRow, column: string): Cell { + if (table === 'data' && column === GroupMembership.GROUP_SOURCE_ID) { + if (row[Data.MIMETYPE] !== MimeType.GROUP_MEMBERSHIP || row[Data.DATA1] === null) return null; + return this.tables.groups.get(Number(row[Data.DATA1]))?.[Groups.SOURCE_ID] ?? null; + } + if (table === 'events' && (column === Events.ACCOUNT_NAME || column === Events.ACCOUNT_TYPE)) { + return this.tables.calendars.get(Number(row[Events.CALENDAR_ID]))?.[column] ?? null; + } + if (table === 'data' && column === Data.DATA15 && row[Data.MIMETYPE] === MimeType.PHOTO) return null; + const v = row[column]; + return v === undefined ? null : v; + } + + private view(table: ProviderTable, row: StoredRow): StoredRow { + const out: StoredRow = {}; + for (const column of COLUMNS[table]) out[column] = this.readCell(table, row, column); + return out; + } + + /** @internal */ owned(table: ProviderTable, row: StoredRow, accountName: string): boolean { + const mine = (r: StoredRow | undefined) => + !!r && r[RawContacts.ACCOUNT_NAME] === accountName && r[RawContacts.ACCOUNT_TYPE] === this.accountType; + switch (table) { + case 'raw_contacts': + case 'groups': + case 'settings': + case 'calendars': + case 'colors': + return mine(row); + case 'data': + return mine(this.tables.raw_contacts.get(Number(row[Data.RAW_CONTACT_ID]))); + case 'events': + return mine(this.tables.calendars.get(Number(row[Events.CALENDAR_ID]))); + case 'attendees': + case 'reminders': + case 'extended_properties': { + const event = this.tables.events.get(Number(row.event_id)); + return !!event && this.owned('events', event, accountName); + } + } + } + + /** @internal */ deleteRow(table: ProviderTable, id: number): void { + const row = this.tables[table].get(id); + if (!row) return; + this.tables[table].delete(id); + if (table === 'raw_contacts') { + for (const [dataId, d] of this.tables.data) if (d[Data.RAW_CONTACT_ID] === id) this.tables.data.delete(dataId); + } else if (table === 'groups') { + for (const [dataId, d] of this.tables.data) { + if (d[Data.MIMETYPE] === MimeType.GROUP_MEMBERSHIP && Number(d[Data.DATA1]) === id) { + this.tables.data.delete(dataId); + this.bumpVersion(Number(d[Data.RAW_CONTACT_ID])); + } + } + } else if (table === 'calendars') { + for (const [eventId, e] of this.tables.events) if (e[Events.CALENDAR_ID] === id) this.deleteRow('events', eventId); + } else if (table === 'events') { + for (const child of ['attendees', 'reminders', 'extended_properties'] as const) { + for (const [childId, c] of this.tables[child]) if (c.event_id === id) this.tables[child].delete(childId); + } + } + } + + /** @internal */ bumpVersion(rawContactId: number): void { + const rc = this.tables.raw_contacts.get(rawContactId); + if (rc) rc[RawContacts.VERSION] = Number(rc[RawContacts.VERSION] ?? 1) + 1; + } + + /** @internal */ markEventDirty(eventId: number): void { + const e = this.tables.events.get(eventId); + if (!e) return; + e[Events.DIRTY] = 1; + const mutators = String(e[Events.MUTATORS] ?? ''); + if (!mutators.split(',').includes(APP_PACKAGE)) e[Events.MUTATORS] = mutators ? `${mutators},${APP_PACKAGE}` : APP_PACKAGE; + } + + /** @internal */ storePhoto(b64: string): number { + const fileId = this.nextPhotoFile++; + this.photos.set(fileId, b64); + return fileId; + } + + /** @internal Provider-side fixes of a data row after a write (ContactsProvider's row handlers). */ + normalizeData(row: StoredRow): void { + if (!this.normalize) return; + if (row[Data.MIMETYPE] === MimeType.STRUCTURED_NAME) { + const parts = [ + StructuredName.PREFIX, + StructuredName.GIVEN_NAME, + StructuredName.MIDDLE_NAME, + StructuredName.FAMILY_NAME, + StructuredName.SUFFIX, + ]; + const anyPart = parts.some((c) => !isEmpty(row[c])); + if (isEmpty(row[StructuredName.DISPLAY_NAME]) && anyPart) { + row[StructuredName.DISPLAY_NAME] = parts.map((c) => row[c]).filter((v) => !isEmpty(v)).join(' '); + } else if (!isEmpty(row[StructuredName.DISPLAY_NAME]) && !anyPart) { + const words = String(row[StructuredName.DISPLAY_NAME]).trim().split(/\s+/); + row[StructuredName.GIVEN_NAME] = words[0] ?? null; + row[StructuredName.FAMILY_NAME] = words.length > 1 ? words.slice(1).join(' ') : null; + } + } else if (row[Data.MIMETYPE] === MimeType.STRUCTURED_POSTAL) { + const parts = [ + StructuredPostal.STREET, + StructuredPostal.POBOX, + StructuredPostal.NEIGHBORHOOD, + StructuredPostal.CITY, + StructuredPostal.REGION, + StructuredPostal.POSTCODE, + StructuredPostal.COUNTRY, + ]; + const anyPart = parts.some((c) => !isEmpty(row[c])); + if (isEmpty(row[StructuredPostal.FORMATTED_ADDRESS]) && anyPart) { + row[StructuredPostal.FORMATTED_ADDRESS] = parts.map((c) => row[c]).filter((v) => !isEmpty(v)).join(', '); + } else if (!isEmpty(row[StructuredPostal.FORMATTED_ADDRESS]) && !anyPart) { + row[StructuredPostal.STREET] = row[StructuredPostal.FORMATTED_ADDRESS]; + } + } + } + + /** @internal CalendarProvider's checks and fixes of an event row (sync adapter and app alike). */ + fixEvent(row: StoredRow, isInsert: boolean): void { + const recurring = !isEmpty(row[Events.RRULE]) || !isEmpty(row[Events.RDATE]); + const exception = !isEmpty(row[Events.ORIGINAL_SYNC_ID]) || !isEmpty(row[Events.ORIGINAL_ID]); + if (!isEmpty(row[Events.RRULE]) && !aospRruleValid(String(row[Events.RRULE]))) { + throw new FakeProviderError('provider', `Invalid recurrence rule: ${row[Events.RRULE]}`); + } + if (recurring) { + row[Events.DTEND] = null; + if (isEmpty(row[Events.DURATION])) throw new FakeProviderError('provider', 'Recurring event without DURATION'); + } else if (exception || !isEmpty(row[Events.DTEND])) { + row[Events.DURATION] = null; + } + if (!isEmpty(row[Events.DURATION]) && !aospDurationValid(String(row[Events.DURATION]))) { + throw new FakeProviderError('provider', `Invalid DURATION ${row[Events.DURATION]}`); + } + if (isInsert && isEmpty(row[Events.DTSTART])) throw new FakeProviderError('provider', 'DTSTART field missing'); + if (Number(row[Events.ALL_DAY]) === 1) { + for (const c of [Events.DTSTART, Events.DTEND]) { + if (!isEmpty(row[c])) row[c] = Math.floor(Number(row[c]) / 86_400_000) * 86_400_000; + } + const d = row[Events.DURATION]; + if (!isEmpty(d)) { + const s = String(d); + if (s.endsWith('S')) { + const n = Number(s.substring(1, s.length - 1)); // Integer.parseInt(duration.substring(1, len-1)) + if (!Number.isInteger(n)) throw new FakeProviderError('provider', `NumberFormatException: ${s}`); + row[Events.DURATION] = `P${Math.ceil(n / 86_400)}D`; + } + } + } + } + + /** @internal */ applyBatch(accountName: string, authority: Authority, ops: ProviderOp[]): BatchResult { + const record = (ok: boolean) => this.batches.push({ account: accountName, authority, ops, ok }); + if (this.pendingFailure) { + const { reason, message } = this.pendingFailure; + this.pendingFailure = null; + record(false); + return { ok: false, reason, message }; + } + if (JSON.stringify(ops).length > this.maxBatchBytes) { + record(false); + return { ok: false, reason: 'tooLarge', message: 'TransactionTooLargeException' }; + } + if (authority === CONTACTS_AUTHORITY) { + let sinceYield = 0; + for (let i = 0; i < ops.length; i++) { + if (i > 0 && ops[i].yieldAllowed) sinceYield = 0; + if (++sinceYield >= 500) { + record(false); + return { ok: false, reason: 'provider', message: 'Too many content provider operations between yield points' }; + } + } + } + for (const hook of this.beforeBatch.splice(0)) hook(); + + const snapshot = { + tables: cloneTables(this.tables), + syncState: new Map(this.syncState), + photos: new Map(this.photos), + nextId: this.nextId, + nextPhotoFile: this.nextPhotoFile, + }; + try { + const results = new BatchApplier(this, accountName, authority).apply(ops); + record(true); + return { ok: true, results }; + } catch (e) { + this.tables = snapshot.tables; + this.syncState = snapshot.syncState; + this.photos = snapshot.photos; + this.nextId = snapshot.nextId; + this.nextPhotoFile = snapshot.nextPhotoFile; + record(false); + if (e instanceof FakeProviderError) return { ok: false, reason: e.reason, message: e.message }; + return { ok: false, reason: 'provider', message: (e as Error).message }; + } + } + + /** @internal */ setSyncState(accountName: string, authority: Authority, value: string): void { + this.syncState.set(`${authority}|${accountName}`, value); + } +} + +function cloneTables(tables: Tables): Tables { + const out = {} as Tables; + for (const [name, rows] of Object.entries(tables) as Array<[ProviderTable, RowMap]>) { + const copy = new RowMap(); + for (const [id, row] of rows) copy.set(id, { ...row }); + out[name] = copy; + } + return out; +} + +function tablesFor(authority: Authority): readonly ProviderTable[] { + return authority === CONTACTS_AUTHORITY ? CONTACTS_TABLES : CALENDAR_TABLES; +} + +function checkColumns(table: ProviderTable, columns: Iterable, write: boolean): void { + for (const c of columns) { + if (!COLUMNS[table].has(c)) throw new FakeProviderError('provider', `no such column: ${c} in ${table}`); + if (write && READ_ONLY_COLUMNS[table]?.has(c)) throw new FakeProviderError('provider', `${c} is read-only in ${table}`); + } +} + +class FakePort implements ProviderPort { + constructor( + private readonly fake: FakeDeviceProviders, + readonly accountName: string, + readonly authority: Authority, + ) {} + + async query(q: ProviderQuery): Promise { + if (!tablesFor(this.authority).includes(q.table)) { + throw new Error(`scope: table ${q.table} is not part of ${this.authority}`); + } + try { + checkColumns(q.table, [...q.columns, ...whereColumns(q.where)], false); + } catch (e) { + throw new Error((e as Error).message); + } + const pred = compileWhere(q.where, q.args); + const order = compileOrderBy(q.orderBy); + const rows = [...this.fake.table(q.table).values()] + .filter((row) => this.fake.owned(q.table, row, this.accountName)) + .filter((row) => pred((c) => this.fake.readCell(q.table, row, c))) + .sort((a, b) => order((c) => this.fake.readCell(q.table, a, c), (c) => this.fake.readCell(q.table, b, c))); + return { + columns: q.columns, + rows: rows.map((row) => q.columns.map((c) => this.fake.readCell(q.table, row, c))), + }; + } + + async applyBatch(ops: ProviderOp[]): Promise { + return this.fake.applyBatch(this.accountName, this.authority, structuredClone(ops)); + } + + async readSyncState(): Promise { + return this.fake.readSyncState(this.accountName, this.authority); + } + + async readPhoto(rawContactId: number, _maxPx: number): Promise { + const rc = this.fake.table('raw_contacts').get(rawContactId); + if (!rc || !this.fake.owned('raw_contacts', rc, this.accountName)) throw new Error('scope: raw contact of another account'); + for (const row of this.fake.table('data').values()) { + if (row[Data.RAW_CONTACT_ID] === rawContactId && row[Data.MIMETYPE] === MimeType.PHOTO) { + const fileId = row[Data.DATA14] === null ? null : Number(row[Data.DATA14]); + const b64 = fileId === null ? undefined : this.fake.photo(fileId); + return b64 === undefined ? null : { jpegBase64: b64, fileId }; + } + } + return null; + } +} + +/** Applies one batch as a sync adapter; throws FakeProviderError to roll it back. */ +class BatchApplier { + private readonly results: OpResult[] = []; + private readonly insertedDataFor = new Set(); + + constructor( + private readonly fake: FakeDeviceProviders, + private readonly account: string, + private readonly authority: Authority, + ) {} + + apply(ops: ProviderOp[]): OpResult[] { + ops.forEach((op, index) => this.results.push(this.applyOne(op, index))); + // Data-row inserts bump each raw contact's VERSION once per transaction. + for (const id of this.insertedDataFor) this.fake.bumpVersion(id); + return this.results; + } + + private scopeError(msg: string): never { + throw new FakeProviderError('scope', msg); + } + + private checkTable(table: ProviderTable): void { + if (!tablesFor(this.authority).includes(table)) this.scopeError(`table ${table} is not part of ${this.authority}`); + } + + private target(table: ProviderTable, id: number | undefined, where: string | undefined, args: Array | undefined): number[] { + const rows = this.fake.table(table); + if (id !== undefined) { + const row = rows.get(id); + if (!row) return []; + if (!this.fake.owned(table, row, this.account)) this.scopeError(`${table} ${id} belongs to another account`); + return [id]; + } + checkColumns(table, whereColumns(where), false); + const pred = compileWhere(where, args); + return [...rows] + .filter(([, row]) => this.fake.owned(table, row, this.account)) + .filter(([, row]) => pred((c) => this.fake.readCell(table, row, c))) + .map(([rowId]) => rowId); + } + + private values(table: ProviderTable, values: WriteRow): StoredRow { + checkColumns(table, Object.keys(values), true); + const out: StoredRow = {}; + for (const [column, cell] of Object.entries(values) as Array<[string, WriteCell]>) { + if (cell !== null && typeof cell === 'object') { + if (!(table === 'data' && column === Data.DATA15)) this.scopeError(`blob not allowed in ${table}.${column}`); + out[column] = `blob:${cell.b64}`; + } else { + out[column] = cell; + } + } + return out; + } + + /** A photo row's full-size bytes become a stored display photo plus a file id, like PhotoProcessor. */ + private processPhoto(row: StoredRow): void { + const v = row[Data.DATA15]; + if (typeof v === 'string' && v.startsWith('blob:')) { + row[Data.DATA14] = this.fake.storePhoto(v.slice(5)); + row[Data.DATA15] = 'thumbnail'; + } + } + + private applyOne(op: ProviderOp, index: number): OpResult { + if (op.op === 'syncState') { + this.fake.setSyncState(this.account, this.authority, op.value); + return {}; + } + this.checkTable(op.table); + switch (op.op) { + case 'insert': + return this.insert(op.table, op.values, op.refs, index); + case 'update': + return this.update(op.table, op.id, op.where, op.args, op.values, op.expectCount); + case 'delete': + return this.delete(op.table, op.id, op.where, op.args, op.expectCount); + case 'assert': + return this.assert(op.table, op.id, op.where, op.args, op.values, op.expectCount); + } + } + + private insert(table: ProviderTable, values: WriteRow, refs: Record | undefined, index: number): OpResult { + const row = this.values(table, values); + for (const [column, ref] of Object.entries(refs ?? {})) { + if (ref >= index || this.results[ref]?.id === undefined) throw new FakeProviderError('provider', `bad back-reference ${ref}`); + row[column] = this.results[ref].id!; + } + const id = this.fake.allocId(); + const account = { account_name: this.account, account_type: this.fake.accountType }; + const parentOwned = (parentTable: ProviderTable, parentId: Cell) => { + const parent = this.fake.table(parentTable).get(Number(parentId)); + if (!parent || !this.fake.owned(parentTable, parent, this.account)) { + this.scopeError(`${table} insert names ${parentTable} ${String(parentId)} outside the account`); + } + return parent; + }; + const accountMismatch = (r: StoredRow) => + (r.account_name !== undefined && r.account_name !== this.account) || + (r.account_type !== undefined && r.account_type !== this.fake.accountType); + switch (table) { + case 'raw_contacts': + if (accountMismatch(row)) this.scopeError('raw contact of another account'); + Object.assign(row, { ...defaults(RAW_CONTACT_DEFAULTS), ...row, ...account }); + break; + case 'groups': + if (accountMismatch(row)) this.scopeError('group of another account'); + Object.assign(row, { ...defaults(GROUP_DEFAULTS), ...row, ...account }); + break; + case 'settings': { + for (const [sid, s] of this.fake.table('settings')) { + if (s.account_name === this.account && s.account_type === this.fake.accountType) this.fake.table('settings').delete(sid); + } + Object.assign(row, { [ContactsSettings.UNGROUPED_VISIBLE]: 0, [ContactsSettings.SHOULD_SYNC]: 1, ...row, ...account }); + break; + } + case 'data': { + parentOwned('raw_contacts', row[Data.RAW_CONTACT_ID]); + row[Data.RAW_CONTACT_ID] = Number(row[Data.RAW_CONTACT_ID]); + if (isEmpty(row[Data.MIMETYPE])) throw new FakeProviderError('provider', 'data row without mimetype'); + row[Data.DATA_VERSION] = 0; + if (row[Data.MIMETYPE] === MimeType.GROUP_MEMBERSHIP) this.resolveMembership(row); + this.processPhoto(row); + this.fake.normalizeData(row); + this.insertedDataFor.add(Number(row[Data.RAW_CONTACT_ID])); + break; + } + case 'calendars': { + if (accountMismatch(row)) this.scopeError('calendar of another account'); + for (const c of [Calendars.NAME, Calendars.CALENDAR_DISPLAY_NAME, Calendars.CALENDAR_COLOR, Calendars.CALENDAR_ACCESS_LEVEL, Calendars.OWNER_ACCOUNT]) { + if (isEmpty(row[c])) throw new FakeProviderError('provider', `calendar insert without ${c}`); + } + Object.assign(row, { ...defaults(CALENDAR_DEFAULTS), ...row, ...account }); + break; + } + case 'events': { + const calendar = parentOwned('calendars', row[Events.CALENDAR_ID])!; + row[Events.CALENDAR_ID] = Number(row[Events.CALENDAR_ID]); + Object.assign(row, { ...defaults(EVENT_DEFAULTS), ...row }); + this.fake.fixEvent(row, true); + if (isEmpty(row[Events.ORGANIZER])) row[Events.ORGANIZER] = calendar[Calendars.OWNER_ACCOUNT]; + this.linkException(row); + const self = row[Events.SELF_ATTENDEE_STATUS]; + this.fake.table('events').set(id, row); + if (!isEmpty(self)) { + this.fake.table('attendees').set(this.fake.allocId(), { + [Attendees.EVENT_ID]: id, + [Attendees.ATTENDEE_EMAIL]: calendar[Calendars.OWNER_ACCOUNT], + [Attendees.ATTENDEE_STATUS]: self, + }); + } + return { id }; + } + case 'attendees': + case 'reminders': + case 'extended_properties': + parentOwned('events', row.event_id); + row.event_id = Number(row.event_id); + break; + case 'colors': + if (accountMismatch(row)) this.scopeError('color of another account'); + Object.assign(row, { ...row, ...account }); + break; + } + this.fake.table(table).set(id, row); + return { id }; + } + + private resolveMembership(row: StoredRow): void { + const sourceId = row[GroupMembership.GROUP_SOURCE_ID]; + delete row[GroupMembership.GROUP_SOURCE_ID]; + if (row[Data.DATA1] !== null && row[Data.DATA1] !== undefined) return; + if (isEmpty(sourceId)) throw new FakeProviderError('provider', 'membership needs GROUP_ROW_ID or GROUP_SOURCE_ID'); + for (const [gid, g] of this.fake.table('groups')) { + if (g[Groups.SOURCE_ID] === sourceId && this.fake.owned('groups', g, this.account)) { + row[Data.DATA1] = gid; + return; + } + } + // Like the provider: an unknown source id creates an empty, invisible group. + const gid = this.fake.allocId(); + this.fake.table('groups').set(gid, { + ...defaults(GROUP_DEFAULTS), + [Groups.SOURCE_ID]: sourceId, + account_name: this.account, + account_type: this.fake.accountType, + }); + row[Data.DATA1] = gid; + } + + /** ORIGINAL_ID and ORIGINAL_SYNC_ID fill each other in from the master, as CalendarProvider does. */ + private linkException(row: StoredRow): void { + const events = this.fake.table('events'); + if (!isEmpty(row[Events.ORIGINAL_SYNC_ID]) && isEmpty(row[Events.ORIGINAL_ID])) { + for (const [mid, m] of events) { + if (m[Events._SYNC_ID] === row[Events.ORIGINAL_SYNC_ID] && m[Events.CALENDAR_ID] === row[Events.CALENDAR_ID]) { + row[Events.ORIGINAL_ID] = mid; + break; + } + } + } else if (!isEmpty(row[Events.ORIGINAL_ID]) && isEmpty(row[Events.ORIGINAL_SYNC_ID])) { + row[Events.ORIGINAL_SYNC_ID] = events.get(Number(row[Events.ORIGINAL_ID]))?.[Events._SYNC_ID] ?? null; + } + } + + private update( + table: ProviderTable, + id: number | undefined, + where: string | undefined, + args: Array | undefined, + values: WriteRow, + expectCount: number | undefined, + ): OpResult { + const ids = this.target(table, id, where, args); + if (expectCount !== undefined && ids.length !== expectCount) { + throw new FakeProviderError('assert', `wrong number of rows: ${ids.length}`); + } + const patch = this.values(table, values); + for (const rowId of ids) { + const row = this.fake.table(table).get(rowId)!; + const before = { ...row }; + if (table === 'events') { + if (Events.SELF_ATTENDEE_STATUS in patch) throw new FakeProviderError('provider', 'IllegalArgumentException: selfAttendeeStatus'); + if (Events.STATUS in patch && patch[Events.STATUS] === null) throw new FakeProviderError('provider', 'NullPointerException: eventStatus'); + } + Object.assign(row, patch); + switch (table) { + case 'raw_contacts': + if (before[RawContacts.DELETED] !== row[RawContacts.DELETED]) this.fake.bumpVersion(rowId); + break; + case 'groups': + row[Groups.VERSION] = Number(row[Groups.VERSION] ?? 1) + 1; + break; + case 'data': + row[Data.DATA_VERSION] = Number(row[Data.DATA_VERSION] ?? 0) + 1; + this.processPhoto(row); + this.fake.normalizeData(row); + this.fake.bumpVersion(Number(row[Data.RAW_CONTACT_ID])); + break; + case 'events': + this.fake.fixEvent(row, false); + if (Events.DIRTY in patch && Number(patch[Events.DIRTY]) === 0) row[Events.MUTATORS] = null; + if (before[Events._SYNC_ID] !== row[Events._SYNC_ID] && !isEmpty(before[Events._SYNC_ID])) { + for (const e of this.fake.table('events').values()) { + if (e[Events.ORIGINAL_SYNC_ID] === before[Events._SYNC_ID]) e[Events.ORIGINAL_SYNC_ID] = row[Events._SYNC_ID]; + } + } + break; + } + } + return { count: ids.length }; + } + + private delete( + table: ProviderTable, + id: number | undefined, + where: string | undefined, + args: Array | undefined, + expectCount: number | undefined, + ): OpResult { + const ids = this.target(table, id, where, args); + if (expectCount !== undefined && ids.length !== expectCount) { + throw new FakeProviderError('assert', `wrong number of rows: ${ids.length}`); + } + for (const rowId of ids) { + const row = this.fake.table(table).get(rowId); + if (!row) continue; + if (table === 'data') this.fake.bumpVersion(Number(row[Data.RAW_CONTACT_ID])); + this.fake.deleteRow(table, rowId); + } + return { count: ids.length }; + } + + private assert( + table: ProviderTable, + id: number | undefined, + where: string | undefined, + args: Array | undefined, + values: Row | undefined, + expectCount: number | undefined, + ): OpResult { + const ids = this.target(table, id, where, args); + if (expectCount !== undefined && ids.length !== expectCount) { + throw new FakeProviderError('assert', `wrong number of rows: ${ids.length}`); + } + if (values) { + checkColumns(table, Object.keys(values), false); + for (const rowId of ids) { + const row = this.fake.table(table).get(rowId)!; + for (const [column, expected] of Object.entries(values)) { + const actual = this.fake.readCell(table, row, column); + const same = actual === null || expected === null ? actual === expected : String(actual) === String(expected); + if (!same) { + throw new FakeProviderError('assert', `Found value ${String(actual)} when expected ${String(expected)} for column ${column}`); + } + } + } + } + return { count: ids.length }; + } +} + +const defaults = (d: StoredRow): StoredRow => ({ ...d }); + +const RAW_CONTACT_DEFAULTS: StoredRow = { + [RawContacts.SOURCE_ID]: null, + [RawContacts.VERSION]: 1, + [RawContacts.DIRTY]: 0, + [RawContacts.DELETED]: 0, + [RawContacts.SYNC1]: null, + [RawContacts.SYNC2]: null, + [RawContacts.SYNC3]: null, + [RawContacts.SYNC4]: null, + [RawContacts.RAW_CONTACT_IS_READ_ONLY]: 0, + [RawContacts.AGGREGATION_MODE]: 0, + [RawContacts.STARRED]: 0, +}; + +const GROUP_DEFAULTS: StoredRow = { + [Groups.SOURCE_ID]: null, + [Groups.VERSION]: 1, + [Groups.DIRTY]: 0, + [Groups.DELETED]: 0, + [Groups.GROUP_VISIBLE]: 0, + [Groups.SHOULD_SYNC]: 1, + [Groups.GROUP_IS_READ_ONLY]: 0, + auto_add: 0, + favorites: 0, +}; + +const CALENDAR_DEFAULTS: StoredRow = { + [Calendars.VISIBLE]: 1, + [Calendars.SYNC_EVENTS]: 0, + [Calendars.ALLOWED_REMINDERS]: '0,1', + [Calendars.ALLOWED_AVAILABILITY]: '0,1', + [Calendars.ALLOWED_ATTENDEE_TYPES]: '0,1,2', + [Calendars.MAX_REMINDERS]: 5, + [Calendars.CAN_ORGANIZER_RESPOND]: 1, + [Calendars.CAN_MODIFY_TIME_ZONE]: 1, + [Calendars.DIRTY]: 0, +}; + +const EVENT_DEFAULTS: StoredRow = { + [Events.DIRTY]: 0, + [Events.DELETED]: 0, + [Events.AVAILABILITY]: 0, + [Events.ACCESS_LEVEL]: 0, + [Events.HAS_ATTENDEE_DATA]: 0, + [Events.GUESTS_CAN_MODIFY]: 0, + [Events.GUESTS_CAN_INVITE_OTHERS]: 1, + [Events.GUESTS_CAN_SEE_GUESTS]: 1, + [Events.ALL_DAY]: 0, + [Events.STATUS]: 0, +}; + +/** + * Edits as Contacts and Calendar apps make them (not a sync adapter): they + * set DIRTY, soft-delete, and some re-insert rows. Named after the app whose + * behaviour they copy where it matters. + */ +export class FakeUser { + constructor(private readonly fake: FakeDeviceProviders) {} + + private rc(id: number): StoredRow { + const rc = this.fake.table('raw_contacts').get(id); + if (!rc) throw new Error(`no raw contact ${id}`); + return rc; + } + + private dirtyContact(id: number): void { + this.rc(id)[RawContacts.DIRTY] = 1; + } + + /** A new contact from a Contacts app: no SOURCE_ID, DIRTY once data rows exist. */ + insertContact(accountName: string, rows: Row[]): number { + const id = this.fake.allocId(); + this.fake.table('raw_contacts').set(id, { + ...defaults(RAW_CONTACT_DEFAULTS), + account_name: accountName, + account_type: this.fake.accountType, + }); + for (const row of rows) this.insertData(id, row); + return id; + } + + insertData(rawContactId: number, values: Row): number { + const id = this.fake.allocId(); + const row: StoredRow = { ...values, [Data.RAW_CONTACT_ID]: rawContactId, [Data.DATA_VERSION]: 0 }; + this.fake.normalizeData(row); + this.fake.table('data').set(id, row); + this.fake.bumpVersion(rawContactId); + this.dirtyContact(rawContactId); + return id; + } + + /** AOSP Contacts: in-place update of the changed columns only (DATA_SYNC kept). */ + updateData(dataId: number, values: Row): void { + const row = this.fake.table('data').get(dataId); + if (!row) throw new Error(`no data row ${dataId}`); + Object.assign(row, values); + row[Data.DATA_VERSION] = Number(row[Data.DATA_VERSION] ?? 0) + 1; + this.fake.normalizeData(row); + const rcId = Number(row[Data.RAW_CONTACT_ID]); + this.fake.bumpVersion(rcId); + this.dirtyContact(rcId); + } + + deleteData(dataId: number): void { + const row = this.fake.table('data').get(dataId); + if (!row) return; + const rcId = Number(row[Data.RAW_CONTACT_ID]); + this.fake.table('data').delete(dataId); + this.fake.bumpVersion(rcId); + this.dirtyContact(rcId); + } + + /** + * Fossify Contacts' save: the name row is updated in place, the rows of + * every other kind it models are deleted and `rows` inserted without + * DATA_SYNC; relations and foreign kinds are left alone. Also stars through + * the aggregate, which dirties the raw contact. + */ + fossifySave(rawContactId: number, name: Row | null, rows: Row[]): void { + const modeled = new Set([ + MimeType.NICKNAME, MimeType.PHONE, MimeType.EMAIL, MimeType.STRUCTURED_POSTAL, MimeType.EVENT, + MimeType.NOTE, MimeType.ORGANIZATION, MimeType.WEBSITE, MimeType.GROUP_MEMBERSHIP, + 'vnd.android.cursor.item/im', + ]); + for (const [id, row] of [...this.fake.table('data')]) { + if (row[Data.RAW_CONTACT_ID] !== rawContactId) continue; + if (name && row[Data.MIMETYPE] === MimeType.STRUCTURED_NAME) this.updateData(id, name); + else if (modeled.has(String(row[Data.MIMETYPE]))) this.deleteData(id); + } + for (const row of rows) this.insertData(rawContactId, row); + this.dirtyContact(rawContactId); + } + + /** Soft delete: DELETED=1, DIRTY=1, rows kept until the sync adapter purges them. */ + deleteContact(rawContactId: number): void { + const rc = this.rc(rawContactId); + if (Number(rc[RawContacts.DELETED]) !== 1) this.fake.bumpVersion(rawContactId); + rc[RawContacts.DELETED] = 1; + rc[RawContacts.DIRTY] = 1; + } + + /** Starring through the aggregate marks the raw contact dirty (any app, even a sync adapter). */ + star(rawContactId: number, starred = true): void { + const rc = this.rc(rawContactId); + rc[RawContacts.STARRED] = starred ? 1 : 0; + rc[RawContacts.DIRTY] = 1; + } + + /** A new photo from an editor: a new display photo file id on the photo row. */ + setPhoto(rawContactId: number, jpegBase64: string): void { + const fileId = this.fake.storePhoto(jpegBase64); + for (const [id, row] of this.fake.table('data')) { + if (row[Data.RAW_CONTACT_ID] === rawContactId && row[Data.MIMETYPE] === MimeType.PHOTO) { + this.updateData(id, { [Data.DATA14]: fileId, [Data.DATA15]: 'thumbnail' }); + return; + } + } + this.insertData(rawContactId, { [Data.MIMETYPE]: MimeType.PHOTO, [Data.DATA14]: fileId, [Data.DATA15]: 'thumbnail' }); + } + + insertGroup(accountName: string, title: string): number { + const id = this.fake.allocId(); + this.fake.table('groups').set(id, { + ...defaults(GROUP_DEFAULTS), + [Groups.TITLE]: title, + [Groups.DIRTY]: 1, + account_name: accountName, + account_type: this.fake.accountType, + }); + return id; + } + + updateGroup(groupId: number, values: Row): void { + const g = this.fake.table('groups').get(groupId); + if (!g) throw new Error(`no group ${groupId}`); + Object.assign(g, values, { [Groups.DIRTY]: 1 }); + g[Groups.VERSION] = Number(g[Groups.VERSION] ?? 1) + 1; + } + + /** App delete: memberships go at once, the group stays DELETED=1 until purged. */ + deleteGroup(groupId: number): void { + for (const [id, row] of [...this.fake.table('data')]) { + if (row[Data.MIMETYPE] === MimeType.GROUP_MEMBERSHIP && Number(row[Data.DATA1]) === groupId) { + this.fake.table('data').delete(id); + this.fake.bumpVersion(Number(row[Data.RAW_CONTACT_ID])); + } + } + const g = this.fake.table('groups').get(groupId); + if (g) Object.assign(g, { [Groups.DELETED]: 1, [Groups.DIRTY]: 1 }); + } + + /** Fossify deletes groups through a sync-adapter URI: gone at once, no DELETED marker. */ + fossifyDeleteGroup(groupId: number): void { + this.fake.deleteRow('groups', groupId); + } + + addToGroup(rawContactId: number, groupId: number): number { + return this.insertData(rawContactId, { [Data.MIMETYPE]: MimeType.GROUP_MEMBERSHIP, [Data.DATA1]: groupId }); + } + + // ── calendar ─────────────────────────────────────────── + + private event(id: number): StoredRow { + const e = this.fake.table('events').get(id); + if (!e) throw new Error(`no event ${id}`); + return e; + } + + /** A new event from a calendar app: no _SYNC_ID, DIRTY=1, organizer defaults to the owner. */ + insertEvent(calendarId: number, values: Row, children: { attendees?: Row[]; reminders?: Row[] } = {}): number { + const calendar = this.fake.table('calendars').get(calendarId); + if (!calendar) throw new Error(`no calendar ${calendarId}`); + const id = this.fake.allocId(); + const row: StoredRow = { ...defaults(EVENT_DEFAULTS), ...values, [Events.CALENDAR_ID]: calendarId }; + this.fake.fixEvent(row, true); + if (isEmpty(row[Events.ORGANIZER])) row[Events.ORGANIZER] = calendar[Calendars.OWNER_ACCOUNT]; + this.fake.table('events').set(id, row); + if (!isEmpty(row[Events.ORIGINAL_SYNC_ID]) && isEmpty(row[Events.ORIGINAL_ID])) { + for (const [mid, m] of this.fake.table('events')) { + if (mid !== id && m[Events._SYNC_ID] === row[Events.ORIGINAL_SYNC_ID]) row[Events.ORIGINAL_ID] = mid; + } + } + for (const a of children.attendees ?? []) this.fake.table('attendees').set(this.fake.allocId(), { ...a, event_id: id }); + for (const r of children.reminders ?? []) this.fake.table('reminders').set(this.fake.allocId(), { ...r, event_id: id }); + this.fake.markEventDirty(id); + return id; + } + + updateEvent(eventId: number, values: Row): void { + const e = this.event(eventId); + if (Events.STATUS in values && values[Events.STATUS] === null) throw new Error('NullPointerException: eventStatus'); + Object.assign(e, values); + this.fake.fixEvent(e, false); + this.fake.markEventDirty(eventId); + } + + /** + * App delete: without _SYNC_ID the row (and its un-synced exceptions) is + * gone at once; with one it becomes DELETED=1, DIRTY=1, its reminders and + * extended properties go at once, un-synced exceptions are hard-deleted and + * synced exceptions stay. + */ + deleteEvent(eventId: number): void { + const e = this.event(eventId); + const syncId = e[Events._SYNC_ID]; + const exceptionsOf = () => + [...this.fake.table('events')].filter(([, x]) => Number(x[Events.ORIGINAL_ID]) === eventId); + if (isEmpty(syncId)) { + for (const [xid] of exceptionsOf()) this.fake.deleteRow('events', xid); + this.fake.deleteRow('events', eventId); + return; + } + for (const [xid, x] of exceptionsOf()) if (isEmpty(x[Events._SYNC_ID])) this.fake.deleteRow('events', xid); + for (const child of ['reminders', 'extended_properties'] as const) { + for (const [cid, c] of [...this.fake.table(child)]) if (c.event_id === eventId) this.fake.table(child).delete(cid); + } + e[Events.DELETED] = 1; + this.fake.markEventDirty(eventId); + } + + /** Etar "this event" edit: a new row with ORIGINAL_SYNC_ID and ORIGINAL_INSTANCE_TIME (no UID_2445). */ + insertException(masterId: number, originalInstanceTime: number, values: Row): number { + const m = this.event(masterId); + return this.insertEvent(Number(m[Events.CALENDAR_ID]), { + [Events.TITLE]: m[Events.TITLE], + [Events.EVENT_TIMEZONE]: m[Events.EVENT_TIMEZONE], + [Events.ALL_DAY]: m[Events.ALL_DAY], + [Events.STATUS]: m[Events.STATUS], + ...values, + [Events.ORIGINAL_SYNC_ID]: m[Events._SYNC_ID], + [Events.ORIGINAL_ID]: masterId, + [Events.ORIGINAL_INSTANCE_TIME]: originalInstanceTime, + [Events.ORIGINAL_ALL_DAY]: m[Events.ALL_DAY], + }); + } + + /** Etar/Fossify "delete this event": a cancelled exception row, no EXDATE. */ + cancelInstance(masterId: number, originalInstanceTime: number, instanceEnd: number): number { + return this.insertException(masterId, originalInstanceTime, { + [Events.STATUS]: 2, + [Events.DTSTART]: originalInstanceTime, + [Events.DTEND]: instanceEnd, + }); + } + + /** + * CONTENT_EXCEPTION_URI without RRULE: clones the master except _SYNC_ID and + * SYNC_DATA*, copies attendees and reminders, marks only the exception dirty. + */ + exceptionViaUri(masterId: number, originalInstanceTime: number, values: Row): number { + const m = this.event(masterId); + const clone: StoredRow = { ...m }; + for (const c of [Events._SYNC_ID, Events.SYNC_DATA1, Events.SYNC_DATA2, Events.SYNC_DATA3, Events.SYNC_DATA4, Events.SYNC_DATA5, Events.SYNC_DATA6]) clone[c] = null; + for (const c of [Events.RRULE, Events.RDATE, Events.EXRULE, Events.EXDATE]) clone[c] = null; + const duration = String(m[Events.DURATION] ?? 'P0S'); + const secs = Number(/(\d+)S$/.exec(duration)?.[1] ?? 0) + Number(/(\d+)D$/.exec(duration)?.[1] ?? 0) * 86_400; + Object.assign(clone, { + [Events.DTSTART]: originalInstanceTime, + [Events.DTEND]: originalInstanceTime + secs * 1000, + [Events.DURATION]: null, + ...values, + [Events.ORIGINAL_ID]: masterId, + [Events.ORIGINAL_SYNC_ID]: m[Events._SYNC_ID], + [Events.ORIGINAL_INSTANCE_TIME]: originalInstanceTime, + [Events.ORIGINAL_ALL_DAY]: m[Events.ALL_DAY], + [Events.DIRTY]: 1, + [Events.DELETED]: 0, + }); + delete clone[Events.SELF_ATTENDEE_STATUS]; + const id = this.fake.allocId(); + this.fake.table('events').set(id, clone); + for (const child of ['attendees', 'reminders'] as const) { + for (const c of [...this.fake.table(child).values()]) { + if (c.event_id === masterId) this.fake.table(child).set(this.fake.allocId(), { ...c, event_id: id }); + } + } + const self = values[Events.SELF_ATTENDEE_STATUS]; + if (!isEmpty(self)) { + const owner = this.fake.table('calendars').get(Number(m[Events.CALENDAR_ID]))?.[Calendars.OWNER_ACCOUNT]; + const mine = [...this.fake.table('attendees').values()].filter((a) => a.event_id === id && a[Attendees.ATTENDEE_EMAIL] === owner); + if (mine.length !== 1) throw new Error('Status update WTF'); + mine[0][Attendees.ATTENDEE_STATUS] = self; + } + this.fake.markEventDirty(id); + return id; + } + + /** + * CONTENT_EXCEPTION_URI with an RRULE ("this and following" split by a third + * party): the master's rule gets UNTIL without DIRTY, and a clone INCLUDING + * _SYNC_ID, SYNC_DATA* and UID_2445 is inserted. + */ + splitViaUri(masterId: number, splitAt: number, untilUtc: string, values: Row): number { + const m = this.event(masterId); + m[Events.RRULE] = `${String(m[Events.RRULE]).replace(/;?(UNTIL|COUNT)=[^;]*/g, '')};UNTIL=${untilUtc}`; + const id = this.fake.allocId(); + this.fake.table('events').set(id, { ...m, [Events.DTSTART]: splitAt, ...values, [Events.DIRTY]: 1 }); + this.fake.markEventDirty(id); + return id; + } + + /** Etar RSVP "all events": the attendee row's status, event DIRTY. */ + setAttendeeStatus(eventId: number, email: string, status: number): void { + for (const a of this.fake.table('attendees').values()) { + if (a.event_id === eventId && a[Attendees.ATTENDEE_EMAIL] === email) a[Attendees.ATTENDEE_STATUS] = status; + } + this.fake.markEventDirty(eventId); + } + + addAttendee(eventId: number, values: Row): number { + const id = this.fake.allocId(); + this.fake.table('attendees').set(id, { ...values, event_id: eventId }); + this.fake.markEventDirty(eventId); + return id; + } + + removeAttendee(attendeeId: number): void { + const a = this.fake.table('attendees').get(attendeeId); + if (!a) return; + this.fake.table('attendees').delete(attendeeId); + this.fake.markEventDirty(Number(a.event_id)); + } + + setReminders(eventId: number, reminders: Array<{ minutes: number; method?: number }>): void { + for (const [id, r] of [...this.fake.table('reminders')]) if (r.event_id === eventId) this.fake.table('reminders').delete(id); + for (const r of reminders) { + this.fake.table('reminders').set(this.fake.allocId(), { + event_id: eventId, + [Reminders.MINUTES]: r.minutes, + [Reminders.METHOD]: r.method ?? 1, + }); + } + this.fake.markEventDirty(eventId); + } + + /** Fossify Calendar's save: the event row updated, attendees and reminders deleted and re-inserted. */ + fossifySaveEvent(eventId: number, values: Row, attendees: Row[], reminders: Array<{ minutes: number; method?: number }>): void { + this.updateEvent(eventId, values); + for (const [id, a] of [...this.fake.table('attendees')]) if (a.event_id === eventId) this.fake.table('attendees').delete(id); + for (const a of attendees) this.fake.table('attendees').set(this.fake.allocId(), { ...a, event_id: eventId }); + this.setReminders(eventId, reminders); + } +} diff --git a/src/device-sync/__tests__/fakes/fake-sql.ts b/src/device-sync/__tests__/fakes/fake-sql.ts new file mode 100644 index 00000000..8d93e98f --- /dev/null +++ b/src/device-sync/__tests__/fakes/fake-sql.ts @@ -0,0 +1,269 @@ +/** + * The small SQL subset the device sync engine may use in `ProviderQuery.where` + * and op selections, evaluated over in-memory rows for the fake providers. + * Keeping the engine to this subset keeps it portable to the real providers + * (SQLite) and testable here: + * + * expr := or + * or := and ('OR' and)* + * and := not ('AND' not)* + * not := 'NOT' not | primary + * primary := '(' expr ')' + * | column ('=' | '!=' | '<>' | '<' | '<=' | '>' | '>=') value + * | column 'IS' ['NOT'] 'NULL' + * | column ['NOT'] 'IN' '(' value (',' value)* ')' + * | column ['NOT'] 'LIKE' value + * value := '?' | 'string' | number + * + * Comparisons follow SQLite closely enough: NULL compares false, numbers and + * numeric strings compare as numbers (column affinity), LIKE is + * case-insensitive with `%` and `_`. + */ +import type { Cell } from '../../types'; + +type Value = { kind: 'arg'; index: number } | { kind: 'lit'; value: string | number }; + +type Node = + | { t: 'or'; parts: Node[] } + | { t: 'and'; parts: Node[] } + | { t: 'not'; inner: Node } + | { t: 'cmp'; column: string; op: string; value: Value } + | { t: 'isnull'; column: string; negate: boolean } + | { t: 'in'; column: string; values: Value[]; negate: boolean } + | { t: 'like'; column: string; value: Value; negate: boolean }; + +type Token = { k: 'id' | 'str' | 'num' | 'op' | 'arg' | 'lp' | 'rp' | 'comma'; v: string }; + +function tokenize(src: string): Token[] { + const out: Token[] = []; + let i = 0; + while (i < src.length) { + const c = src[i]; + if (/\s/.test(c)) { i++; continue; } + if (c === '(') { out.push({ k: 'lp', v: c }); i++; continue; } + if (c === ')') { out.push({ k: 'rp', v: c }); i++; continue; } + if (c === ',') { out.push({ k: 'comma', v: c }); i++; continue; } + if (c === '?') { out.push({ k: 'arg', v: c }); i++; continue; } + if (c === "'") { + let j = i + 1; + let s = ''; + while (j < src.length) { + if (src[j] === "'" && src[j + 1] === "'") { s += "'"; j += 2; continue; } + if (src[j] === "'") break; + s += src[j++]; + } + if (j >= src.length) throw new Error(`fake-sql: unterminated string in ${src}`); + out.push({ k: 'str', v: s }); + i = j + 1; + continue; + } + const two = src.slice(i, i + 2); + if (['!=', '<>', '<=', '>='].includes(two)) { out.push({ k: 'op', v: two }); i += 2; continue; } + if (['=', '<', '>'].includes(c)) { out.push({ k: 'op', v: c }); i++; continue; } + const num = /^-?\d+(\.\d+)?/.exec(src.slice(i)); + if (num) { out.push({ k: 'num', v: num[0] }); i += num[0].length; continue; } + const id = /^[A-Za-z_][A-Za-z0-9_]*/.exec(src.slice(i)); + if (id) { out.push({ k: 'id', v: id[0] }); i += id[0].length; continue; } + throw new Error(`fake-sql: unexpected '${c}' in ${src}`); + } + return out; +} + +class Parser { + private pos = 0; + private argIndex = 0; + constructor(private readonly tokens: Token[], private readonly src: string) {} + + parse(): Node { + const node = this.or(); + if (this.pos !== this.tokens.length) this.fail('trailing input'); + return node; + } + + private peek(): Token | undefined { return this.tokens[this.pos]; } + private kw(word: string): boolean { + const t = this.peek(); + if (t?.k === 'id' && t.v.toUpperCase() === word) { this.pos++; return true; } + return false; + } + private fail(msg: string): never { throw new Error(`fake-sql: ${msg} in "${this.src}"`); } + + private or(): Node { + const parts = [this.and()]; + while (this.kw('OR')) parts.push(this.and()); + return parts.length === 1 ? parts[0] : { t: 'or', parts }; + } + private and(): Node { + const parts = [this.not()]; + while (this.kw('AND')) parts.push(this.not()); + return parts.length === 1 ? parts[0] : { t: 'and', parts }; + } + private not(): Node { + if (this.kw('NOT')) return { t: 'not', inner: this.not() }; + return this.primary(); + } + private value(): Value { + const t = this.tokens[this.pos++]; + if (!t) this.fail('missing value'); + if (t.k === 'arg') return { kind: 'arg', index: this.argIndex++ }; + if (t.k === 'str') return { kind: 'lit', value: t.v }; + if (t.k === 'num') return { kind: 'lit', value: Number(t.v) }; + return this.fail(`expected a value, got ${t.v}`); + } + private primary(): Node { + const t = this.peek(); + if (!t) this.fail('unexpected end'); + if (t.k === 'lp') { + this.pos++; + const inner = this.or(); + if (this.peek()?.k !== 'rp') this.fail('missing )'); + this.pos++; + return inner; + } + if (t.k !== 'id') this.fail(`expected a column, got ${t.v}`); + this.pos++; + const column = t.v; + if (this.kw('IS')) { + const negate = this.kw('NOT'); + if (!this.kw('NULL')) this.fail('expected NULL'); + return { t: 'isnull', column, negate }; + } + const negate = this.kw('NOT'); + if (this.kw('IN')) { + if (this.peek()?.k !== 'lp') this.fail('expected ('); + this.pos++; + const values = [this.value()]; + while (this.peek()?.k === 'comma') { this.pos++; values.push(this.value()); } + if (this.peek()?.k !== 'rp') this.fail('missing )'); + this.pos++; + return { t: 'in', column, values, negate }; + } + if (this.kw('LIKE')) return { t: 'like', column, value: this.value(), negate }; + if (negate) this.fail('NOT must precede IN or LIKE here'); + const op = this.tokens[this.pos++]; + if (op?.k !== 'op') this.fail(`expected an operator after ${column}`); + return { t: 'cmp', column, op: op.v, value: this.value() }; + } +} + +const cache = new Map(); + +function compile(where: string): { node: Node; argCount: number } { + let hit = cache.get(where); + if (!hit) { + const tokens = tokenize(where); + const node = new Parser(tokens, where).parse(); + hit = { node, argCount: tokens.filter((t) => t.k === 'arg').length }; + cache.set(where, hit); + } + return hit; +} + +function asNumber(v: unknown): number | null { + if (typeof v === 'number') return v; + if (typeof v === 'string' && /^-?\d+(\.\d+)?$/.test(v.trim())) return Number(v); + return null; +} + +/** SQLite-like comparison: -1/0/1, or null when either side is NULL. */ +export function compareCells(a: Cell, b: Cell): number | null { + if (a === null || a === undefined || b === null || b === undefined) return null; + const na = asNumber(a); + const nb = asNumber(b); + if (na !== null && nb !== null) return na === nb ? 0 : na < nb ? -1 : 1; + const sa = String(a); + const sb = String(b); + return sa === sb ? 0 : sa < sb ? -1 : 1; +} + +function likeToRegex(pattern: string): RegExp { + const esc = pattern.replace(/[.*+^${}()|[\]\\]/g, '\\$&').replace(/%/g, '.*').replace(/_/g, '.'); + return new RegExp(`^${esc}$`, 'is'); +} + +export type ColumnReader = (column: string) => Cell; + +/** A predicate for rows, from a WHERE clause and its arguments. */ +export function compileWhere(where: string | undefined, args: Array = []): (read: ColumnReader) => boolean { + if (!where || !where.trim()) return () => true; + const { node, argCount } = compile(where); + if (argCount !== args.length) { + throw new Error(`fake-sql: "${where}" has ${argCount} placeholders but ${args.length} arguments`); + } + const val = (v: Value): Cell => (v.kind === 'arg' ? args[v.index] : v.value); + const evalNode = (n: Node, read: ColumnReader): boolean => { + switch (n.t) { + case 'or': return n.parts.some((p) => evalNode(p, read)); + case 'and': return n.parts.every((p) => evalNode(p, read)); + case 'not': return !evalNode(n.inner, read); + case 'isnull': { + const isNull = read(n.column) === null || read(n.column) === undefined; + return n.negate ? !isNull : isNull; + } + case 'in': { + const cell = read(n.column); + if (cell === null || cell === undefined) return false; + const hit = n.values.some((v) => compareCells(cell, val(v)) === 0); + return n.negate ? !hit : hit; + } + case 'like': { + const cell = read(n.column); + if (cell === null || cell === undefined) return false; + const hit = likeToRegex(String(val(n.value))).test(String(cell)); + return n.negate ? !hit : hit; + } + case 'cmp': { + const c = compareCells(read(n.column), val(n.value)); + if (c === null) return false; + switch (n.op) { + case '=': return c === 0; + case '!=': + case '<>': return c !== 0; + case '<': return c < 0; + case '<=': return c <= 0; + case '>': return c > 0; + case '>=': return c >= 0; + } + return false; + } + } + }; + return (read) => evalNode(node, read); +} + +/** Columns a WHERE clause mentions, so the fake can reject unknown ones like SQLite does. */ +export function whereColumns(where: string | undefined): string[] { + if (!where || !where.trim()) return []; + const { node } = compile(where); + const out: string[] = []; + const walk = (n: Node): void => { + switch (n.t) { + case 'or': + case 'and': n.parts.forEach(walk); break; + case 'not': walk(n.inner); break; + default: out.push(n.column); + } + }; + walk(node); + return out; +} + +/** A comparator for `orderBy` ("col [ASC|DESC], …"); `_id` ascending when absent. */ +export function compileOrderBy(orderBy: string | undefined): (a: ColumnReader, b: ColumnReader) => number { + const terms = (orderBy?.trim() ? orderBy.split(',') : ['_id']) + .map((term) => term.trim().split(/\s+/)) + .map(([column, dir]) => ({ column, desc: dir?.toUpperCase() === 'DESC' })); + return (a, b) => { + for (const { column, desc } of terms) { + const va = a(column); + const vb = b(column); + let c: number; + if (va === null && vb === null) c = 0; + else if (va === null) c = -1; // SQLite sorts NULL first + else if (vb === null) c = 1; + else c = compareCells(va, vb) ?? 0; + if (c !== 0) return desc ? -c : c; + } + return 0; + }; +} diff --git a/src/device-sync/__tests__/purity.test.ts b/src/device-sync/__tests__/purity.test.ts new file mode 100644 index 00000000..4f27efaf --- /dev/null +++ b/src/device-sync/__tests__/purity.test.ts @@ -0,0 +1,113 @@ +/** + * The device sync mappers, merge rules and engine must stay pure TypeScript: + * no react-native, expo, zustand stores or the jmapClient singleton anywhere + * in their import graph (docs/device-sync.md, "Architecture"). src/test-setup.ts + * mocks those modules globally, so an impure import would not fail any other + * test; this one walks the graph instead. + */ +import { readFileSync, readdirSync, statSync, existsSync } from 'node:fs'; +import path from 'node:path'; +import { describe, expect, it } from 'vitest'; + +const ROOT = path.resolve(__dirname, '..'); +/** + * Modules allowed to touch React Native: the bridge wrapper, the task entry, + * the detached client, and the app-side glue under `app/` (lifecycle, + * triggers), which the engine never imports. + */ +const RN_BOUNDARY = new Set(['native.ts', 'task.ts', path.join('jmap', 'client-port.ts')].map((p) => path.join(ROOT, p))); +const RN_BOUNDARY_DIRS = [path.join(ROOT, 'app') + path.sep]; +const isBoundary = (file: string) => RN_BOUNDARY.has(file) || RN_BOUNDARY_DIRS.some((dir) => file.startsWith(dir)); + +const FORBIDDEN = [ + /^react-native($|\/)/, + /^react($|\/)/, + /^expo/, + /^@expo\//, + /^zustand/, + /^@react-native/, + /^@react-navigation/, +]; +const FORBIDDEN_FILES = [/[\\/]src[\\/]stores[\\/]/, /[\\/]api[\\/]jmap-client\.ts$/, /[\\/]lib[\\/](random|uuid)\.ts$/]; + +function sourceFiles(dir: string): string[] { + if (!existsSync(dir)) return []; + return readdirSync(dir).flatMap((name) => { + const full = path.join(dir, name); + if (statSync(full).isDirectory()) return name === '__tests__' ? [] : sourceFiles(full); + return /\.tsx?$/.test(name) ? [full] : []; + }); +} + +function specifiers(file: string): string[] { + const src = readFileSync(file, 'utf8') + .replace(/\/\*[\s\S]*?\*\//g, '') + .replace(/^\s*\/\/.*$/gm, ''); + const out: string[] = []; + const patterns = [ + /^\s*import\s+(?!type\s)(?:[^'"]*?\s+from\s+)?['"]([^'"]+)['"]/gm, + /^\s*export\s+(?!type\s)[^'"]*?\s+from\s+['"]([^'"]+)['"]/gm, + /\brequire\(\s*['"]([^'"]+)['"]\s*\)/g, + /\bimport\(\s*['"]([^'"]+)['"]\s*\)/g, + ]; + for (const re of patterns) { + for (const m of src.matchAll(re)) { + // `import { type A, type B } from` is erased too. + const clause = m[0]; + if (/import\s*\{\s*(type\s+[\w$]+\s*,?\s*)+\}\s*from/.test(clause)) continue; + out.push(m[1]); + } + } + return out; +} + +function resolve(from: string, spec: string): string | null { + if (!spec.startsWith('.')) return null; + const base = path.resolve(path.dirname(from), spec); + for (const candidate of [base, `${base}.ts`, `${base}.tsx`, path.join(base, 'index.ts')]) { + if (existsSync(candidate) && statSync(candidate).isFile()) return candidate; + } + return null; +} + +function impureChains(entry: string): string[] { + const problems: string[] = []; + const seen = new Set(); + const walk = (file: string, chain: string[]) => { + if (seen.has(file)) return; + seen.add(file); + for (const spec of specifiers(file)) { + const resolved = resolve(file, spec); + const at = [...chain, path.relative(ROOT, file)]; + if (!resolved) { + if (FORBIDDEN.some((re) => re.test(spec))) problems.push(`${at.join(' → ')} → ${spec}`); + continue; + } + if (FORBIDDEN_FILES.some((re) => re.test(resolved))) { + problems.push(`${at.join(' → ')} → ${path.relative(ROOT, resolved)}`); + continue; + } + walk(resolved, at); + } + }; + walk(entry, []); + return problems; +} + +describe('device sync purity', () => { + const entries = sourceFiles(ROOT).filter((f) => !isBoundary(f)); + + it('scans the pure modules', () => { + expect(entries.map((f) => path.relative(ROOT, f))).toEqual(expect.arrayContaining(['types.ts', path.join('common', 'json.ts')])); + }); + + it('keeps react-native, expo, stores and the jmapClient singleton out of the mappers, merge rules and engine', () => { + const problems = entries.flatMap(impureChains); + expect(problems).toEqual([]); + }); + + it('would catch an impure import', () => { + const probe = path.join(ROOT, '..', 'lib', 'calendar-timezone.ts'); + expect(impureChains(probe).length).toBeGreaterThan(0); + }); +}); diff --git a/src/device-sync/android-columns.ts b/src/device-sync/android-columns.ts new file mode 100644 index 00000000..dc689821 --- /dev/null +++ b/src/device-sync/android-columns.ts @@ -0,0 +1,344 @@ +/** + * Column names and constant values of Android's ContactsContract and + * CalendarContract, as the providers spell them. The engine reads and writes + * rows by these names through the native bridge (src/device-sync/native.ts), + * so a typo here is a column the provider silently ignores or rejects. + * + * Only what device sync uses is listed. Pure constants: no React Native import. + */ + +// ─── ContactsContract ────────────────────────────────── + +export const RawContacts = { + _ID: '_id', + ACCOUNT_NAME: 'account_name', + ACCOUNT_TYPE: 'account_type', + SOURCE_ID: 'sourceid', + VERSION: 'version', + DIRTY: 'dirty', + DELETED: 'deleted', + SYNC1: 'sync1', + SYNC2: 'sync2', + SYNC3: 'sync3', + SYNC4: 'sync4', + RAW_CONTACT_IS_READ_ONLY: 'raw_contact_is_read_only', + AGGREGATION_MODE: 'aggregation_mode', + STARRED: 'starred', + DISPLAY_NAME_PRIMARY: 'display_name', +} as const; + +export const Data = { + _ID: '_id', + RAW_CONTACT_ID: 'raw_contact_id', + MIMETYPE: 'mimetype', + IS_PRIMARY: 'is_primary', + IS_SUPER_PRIMARY: 'is_super_primary', + IS_READ_ONLY: 'is_read_only', + DATA_VERSION: 'data_version', + DATA1: 'data1', + DATA2: 'data2', + DATA3: 'data3', + DATA4: 'data4', + DATA5: 'data5', + DATA6: 'data6', + DATA7: 'data7', + DATA8: 'data8', + DATA9: 'data9', + DATA10: 'data10', + DATA11: 'data11', + DATA12: 'data12', + DATA13: 'data13', + DATA14: 'data14', + DATA15: 'data15', + DATA_SYNC1: 'data_sync1', + DATA_SYNC2: 'data_sync2', + DATA_SYNC3: 'data_sync3', + DATA_SYNC4: 'data_sync4', +} as const; + +/** `Data.MIMETYPE` values of the kinds device sync maps. */ +export const MimeType = { + STRUCTURED_NAME: 'vnd.android.cursor.item/name', + NICKNAME: 'vnd.android.cursor.item/nickname', + EMAIL: 'vnd.android.cursor.item/email_v2', + PHONE: 'vnd.android.cursor.item/phone_v2', + STRUCTURED_POSTAL: 'vnd.android.cursor.item/postal-address_v2', + ORGANIZATION: 'vnd.android.cursor.item/organization', + WEBSITE: 'vnd.android.cursor.item/website', + EVENT: 'vnd.android.cursor.item/contact_event', + RELATION: 'vnd.android.cursor.item/relation', + NOTE: 'vnd.android.cursor.item/note', + PHOTO: 'vnd.android.cursor.item/photo', + GROUP_MEMBERSHIP: 'vnd.android.cursor.item/group_membership', +} as const; + +export const StructuredName = { + DISPLAY_NAME: 'data1', + GIVEN_NAME: 'data2', + FAMILY_NAME: 'data3', + PREFIX: 'data4', + MIDDLE_NAME: 'data5', + SUFFIX: 'data6', + PHONETIC_GIVEN_NAME: 'data7', + PHONETIC_MIDDLE_NAME: 'data8', + PHONETIC_FAMILY_NAME: 'data9', + FULL_NAME_STYLE: 'data10', + PHONETIC_NAME_STYLE: 'data11', +} as const; + +/** Columns shared by the typed kinds (Email, Phone, Postal, …). */ +export const CommonKind = { + DATA: 'data1', + TYPE: 'data2', + LABEL: 'data3', +} as const; + +export const Nickname = { NAME: 'data1', TYPE: 'data2', LABEL: 'data3' } as const; +export const NicknameType = { CUSTOM: 0, DEFAULT: 1, OTHER_NAME: 2, MAIDEN_NAME: 3, SHORT_NAME: 4, INITIALS: 5 } as const; + +export const Email = { ADDRESS: 'data1', TYPE: 'data2', LABEL: 'data3', DISPLAY_NAME: 'data4' } as const; +export const EmailType = { CUSTOM: 0, HOME: 1, WORK: 2, OTHER: 3, MOBILE: 4 } as const; + +export const Phone = { NUMBER: 'data1', TYPE: 'data2', LABEL: 'data3', NORMALIZED_NUMBER: 'data4' } as const; +export const PhoneType = { + CUSTOM: 0, + HOME: 1, + MOBILE: 2, + WORK: 3, + FAX_WORK: 4, + FAX_HOME: 5, + PAGER: 6, + OTHER: 7, + CALLBACK: 8, + CAR: 9, + COMPANY_MAIN: 10, + ISDN: 11, + MAIN: 12, + OTHER_FAX: 13, + RADIO: 14, + TELEX: 15, + TTY_TDD: 16, + WORK_MOBILE: 17, + WORK_PAGER: 18, + ASSISTANT: 19, + MMS: 20, +} as const; + +export const StructuredPostal = { + FORMATTED_ADDRESS: 'data1', + TYPE: 'data2', + LABEL: 'data3', + STREET: 'data4', + POBOX: 'data5', + NEIGHBORHOOD: 'data6', + CITY: 'data7', + REGION: 'data8', + POSTCODE: 'data9', + COUNTRY: 'data10', +} as const; +export const PostalType = { CUSTOM: 0, HOME: 1, WORK: 2, OTHER: 3 } as const; + +export const Organization = { + COMPANY: 'data1', + TYPE: 'data2', + LABEL: 'data3', + TITLE: 'data4', + DEPARTMENT: 'data5', + JOB_DESCRIPTION: 'data6', + SYMBOL: 'data7', + PHONETIC_NAME: 'data8', + OFFICE_LOCATION: 'data9', +} as const; +export const OrganizationType = { CUSTOM: 0, WORK: 1, OTHER: 2 } as const; + +export const Website = { URL: 'data1', TYPE: 'data2', LABEL: 'data3' } as const; +export const WebsiteType = { CUSTOM: 0, HOMEPAGE: 1, BLOG: 2, PROFILE: 3, HOME: 4, WORK: 5, FTP: 6, OTHER: 7 } as const; + +export const ContactEvent = { START_DATE: 'data1', TYPE: 'data2', LABEL: 'data3' } as const; +export const ContactEventType = { CUSTOM: 0, ANNIVERSARY: 1, OTHER: 2, BIRTHDAY: 3 } as const; + +export const Relation = { NAME: 'data1', TYPE: 'data2', LABEL: 'data3' } as const; +export const RelationType = { + CUSTOM: 0, + ASSISTANT: 1, + BROTHER: 2, + CHILD: 3, + DOMESTIC_PARTNER: 4, + FATHER: 5, + FRIEND: 6, + MANAGER: 7, + MOTHER: 8, + PARENT: 9, + PARTNER: 10, + REFERRED_BY: 11, + RELATIVE: 12, + SISTER: 13, + SPOUSE: 14, +} as const; + +export const Note = { NOTE: 'data1' } as const; + +export const Photo = { PHOTO_FILE_ID: 'data14', PHOTO: 'data15' } as const; + +export const GroupMembership = { GROUP_ROW_ID: 'data1', GROUP_SOURCE_ID: 'group_sourceid' } as const; + +export const Groups = { + _ID: '_id', + ACCOUNT_NAME: 'account_name', + ACCOUNT_TYPE: 'account_type', + SOURCE_ID: 'sourceid', + VERSION: 'version', + DIRTY: 'dirty', + DELETED: 'deleted', + TITLE: 'title', + NOTES: 'notes', + GROUP_VISIBLE: 'group_visible', + SHOULD_SYNC: 'should_sync', + GROUP_IS_READ_ONLY: 'group_is_read_only', + SYNC1: 'sync1', + SYNC2: 'sync2', + SYNC3: 'sync3', + SYNC4: 'sync4', +} as const; + +export const ContactsSettings = { + ACCOUNT_NAME: 'account_name', + ACCOUNT_TYPE: 'account_type', + UNGROUPED_VISIBLE: 'ungrouped_visible', + SHOULD_SYNC: 'should_sync', +} as const; + +// ─── CalendarContract ────────────────────────────────── + +export const Calendars = { + _ID: '_id', + ACCOUNT_NAME: 'account_name', + ACCOUNT_TYPE: 'account_type', + _SYNC_ID: '_sync_id', + DIRTY: 'dirty', + NAME: 'name', + CALENDAR_DISPLAY_NAME: 'calendar_displayName', + CALENDAR_COLOR: 'calendar_color', + CALENDAR_COLOR_KEY: 'calendar_color_index', + CALENDAR_ACCESS_LEVEL: 'calendar_access_level', + OWNER_ACCOUNT: 'ownerAccount', + VISIBLE: 'visible', + SYNC_EVENTS: 'sync_events', + CALENDAR_TIME_ZONE: 'calendar_timezone', + ALLOWED_REMINDERS: 'allowedReminders', + ALLOWED_AVAILABILITY: 'allowedAvailability', + ALLOWED_ATTENDEE_TYPES: 'allowedAttendeeTypes', + MAX_REMINDERS: 'maxReminders', + CAN_ORGANIZER_RESPOND: 'canOrganizerRespond', + CAN_MODIFY_TIME_ZONE: 'canModifyTimeZone', + IS_PRIMARY: 'isPrimary', + CAL_SYNC1: 'cal_sync1', + CAL_SYNC2: 'cal_sync2', + CAL_SYNC3: 'cal_sync3', +} as const; + +/** `Calendars.CALENDAR_ACCESS_LEVEL` values. */ +export const CalendarAccess = { + NONE: 0, + FREEBUSY: 100, + READ: 200, + RESPOND: 300, + OVERRIDE: 400, + CONTRIBUTOR: 500, + EDITOR: 600, + OWNER: 700, + ROOT: 800, +} as const; + +export const Events = { + _ID: '_id', + CALENDAR_ID: 'calendar_id', + ACCOUNT_NAME: 'account_name', + ACCOUNT_TYPE: 'account_type', + _SYNC_ID: '_sync_id', + DIRTY: 'dirty', + DELETED: 'deleted', + MUTATORS: 'mutators', + SYNC_DATA1: 'sync_data1', + SYNC_DATA2: 'sync_data2', + SYNC_DATA3: 'sync_data3', + SYNC_DATA4: 'sync_data4', + SYNC_DATA5: 'sync_data5', + SYNC_DATA6: 'sync_data6', + UID_2445: 'uid2445', + TITLE: 'title', + DESCRIPTION: 'description', + EVENT_LOCATION: 'eventLocation', + STATUS: 'eventStatus', + AVAILABILITY: 'availability', + ACCESS_LEVEL: 'accessLevel', + EVENT_COLOR: 'eventColor', + EVENT_COLOR_KEY: 'eventColor_index', + ORGANIZER: 'organizer', + IS_ORGANIZER: 'isOrganizer', + SELF_ATTENDEE_STATUS: 'selfAttendeeStatus', + HAS_ALARM: 'hasAlarm', + HAS_ATTENDEE_DATA: 'hasAttendeeData', + GUESTS_CAN_MODIFY: 'guestsCanModify', + GUESTS_CAN_INVITE_OTHERS: 'guestsCanInviteOthers', + GUESTS_CAN_SEE_GUESTS: 'guestsCanSeeGuests', + DTSTART: 'dtstart', + DTEND: 'dtend', + DURATION: 'duration', + EVENT_TIMEZONE: 'eventTimezone', + EVENT_END_TIMEZONE: 'eventEndTimezone', + ALL_DAY: 'allDay', + RRULE: 'rrule', + RDATE: 'rdate', + EXRULE: 'exrule', + EXDATE: 'exdate', + LAST_DATE: 'lastDate', + ORIGINAL_ID: 'original_id', + ORIGINAL_SYNC_ID: 'original_sync_id', + ORIGINAL_INSTANCE_TIME: 'originalInstanceTime', + ORIGINAL_ALL_DAY: 'originalAllDay', + CUSTOM_APP_PACKAGE: 'customAppPackage', + CUSTOM_APP_URI: 'customAppUri', +} as const; + +export const EventStatus = { TENTATIVE: 0, CONFIRMED: 1, CANCELED: 2 } as const; +export const EventAvailability = { BUSY: 0, FREE: 1, TENTATIVE: 2 } as const; +export const EventAccess = { DEFAULT: 0, CONFIDENTIAL: 1, PRIVATE: 2, PUBLIC: 3 } as const; + +export const Attendees = { + _ID: '_id', + EVENT_ID: 'event_id', + ATTENDEE_NAME: 'attendeeName', + ATTENDEE_EMAIL: 'attendeeEmail', + ATTENDEE_RELATIONSHIP: 'attendeeRelationship', + ATTENDEE_TYPE: 'attendeeType', + ATTENDEE_STATUS: 'attendeeStatus', +} as const; +export const AttendeeRelationship = { NONE: 0, ATTENDEE: 1, ORGANIZER: 2, PERFORMER: 3, SPEAKER: 4 } as const; +export const AttendeeType = { NONE: 0, REQUIRED: 1, OPTIONAL: 2, RESOURCE: 3 } as const; +export const AttendeeStatus = { NONE: 0, ACCEPTED: 1, DECLINED: 2, INVITED: 3, TENTATIVE: 4 } as const; + +export const Reminders = { + _ID: '_id', + EVENT_ID: 'event_id', + MINUTES: 'minutes', + METHOD: 'method', +} as const; +export const ReminderMethod = { DEFAULT: 0, ALERT: 1, EMAIL: 2, SMS: 3, ALARM: 4 } as const; + +export const ExtendedProperties = { + _ID: '_id', + EVENT_ID: 'event_id', + NAME: 'name', + VALUE: 'value', +} as const; + +export const Colors = { + _ID: '_id', + ACCOUNT_NAME: 'account_name', + ACCOUNT_TYPE: 'account_type', + COLOR_TYPE: 'color_type', + COLOR_KEY: 'color_index', + COLOR: 'color', +} as const; +export const ColorType = { CALENDAR: 0, EVENT: 1 } as const; diff --git a/src/device-sync/common/ids.ts b/src/device-sync/common/ids.ts new file mode 100644 index 00000000..4bcaee10 --- /dev/null +++ b/src/device-sync/common/ids.ts @@ -0,0 +1,78 @@ +/** + * Identifiers device sync mints and parses: map keys for new JSContact / + * JSCalendar entries, uids for new objects, and the `/` + * identities kept in SOURCE_ID / _SYNC_ID (docs/device-sync.md, "Identity"). + */ + +/** A random source in [0, 1); injectable so tests are deterministic. */ +export type RandomSource = () => number; + +const BASE36 = '0123456789abcdefghijklmnopqrstuvwxyz'; + +function randomChars(random: RandomSource, n: number): string { + let s = ''; + for (let i = 0; i < n; i++) s += BASE36[Math.floor(random() * 36) % 36]; + return s; +} + +/** + * Keys for new map entries: `b` + 8 base36 characters. Stalwart parses a + * numeric key as an array index and generates `k` keys itself, so neither + * shape is ever minted. + */ +export function makeKeyMinter(random: RandomSource = Math.random): (taken?: Iterable) => string { + let counter = 0; + return (taken) => { + const used = new Set(taken ?? []); + for (let attempt = 0; attempt < 16; attempt++) { + const key = `b${randomChars(random, 8)}`; + if (!used.has(key)) return key; + } + // A degenerate random source: count instead of looping forever. + for (;;) { + const key = `b${(counter++).toString(36).padStart(8, '0')}`; + if (!used.has(key)) return key; + } + }; +} + +/** An RFC 4122 v4 UUID from the random source. */ +export function uuidFrom(random: RandomSource): string { + const hex = Array.from({ length: 32 }, () => Math.floor(random() * 16).toString(16)); + hex[12] = '4'; + hex[16] = ((parseInt(hex[16], 16) & 0x3) | 0x8).toString(16); + const h = hex.join(''); + return `${h.slice(0, 8)}-${h.slice(8, 12)}-${h.slice(12, 16)}-${h.slice(16, 20)}-${h.slice(20)}`; +} + +/** SOURCE_ID / _SYNC_ID of a server object. JMAP ids never contain `/`. */ +export function objectRef(jmapAccountId: string, id: string): string { + return `${jmapAccountId}/${id}`; +} + +/** The parts of an object ref, or null for anything else (a pending marker, another app's value). */ +export function parseObjectRef(ref: unknown): { accountId: string; id: string } | null { + if (typeof ref !== 'string') return null; + const m = /^([A-Za-z0-9_-]+)\/([A-Za-z0-9_-]+)$/.exec(ref); + return m ? { accountId: m[1], id: m[2] } : null; +} + +/** `_SYNC_ID` of a local event claimed for upload; `~` never occurs in JMAP ids. */ +export const PENDING_PREFIX = '~pending/'; + +export function pendingRef(uid: string): string { + return `${PENDING_PREFIX}${uid}`; +} + +export function pendingUidOf(ref: unknown): string | null { + return typeof ref === 'string' && ref.startsWith(PENDING_PREFIX) ? ref.slice(PENDING_PREFIX.length) : null; +} + +/** `_SYNC_ID` of an exception row: the master's ref plus `#` and the recurrence id. */ +export function exceptionRef(masterRef: string, recurrenceId: string): string { + return `${masterRef}#${recurrenceId}`; +} + +/** Collection keys (`/`), as in SYNC1 and the selections. */ +export const collectionKey = objectRef; +export const parseCollectionKey = parseObjectRef; diff --git a/src/device-sync/common/json.ts b/src/device-sync/common/json.ts new file mode 100644 index 00000000..99afd240 --- /dev/null +++ b/src/device-sync/common/json.ts @@ -0,0 +1,72 @@ +/** + * JSON helpers for comparing server objects, shadows and baselines by value. + * Pure: device sync's mappers, merge rules and engine build on them. + */ +import { sha256Hex } from '../../lib/sha256'; + +type Json = null | boolean | number | string | Json[] | { [key: string]: Json }; + +/** JSON with object keys sorted and `undefined` members dropped, so equal values give equal text. */ +export function canonicalJson(value: unknown): string { + return JSON.stringify(canonicalize(value)); +} + +function canonicalize(value: unknown): Json { + if (value === null || value === undefined) return null; + if (Array.isArray(value)) return value.map((v) => (v === undefined ? null : canonicalize(v))); + if (typeof value === 'object') { + const out: { [key: string]: Json } = {}; + for (const key of Object.keys(value as object).sort()) { + const v = (value as Record)[key]; + if (v !== undefined) out[key] = canonicalize(v); + } + return out; + } + if (typeof value === 'number' && !Number.isFinite(value)) return null; + return value as Json; +} + +/** Structural equality of JSON values; key order and `undefined` members don't matter. */ +export function deepEqual(a: unknown, b: unknown): boolean { + if (a === b) return true; + if (a === undefined || b === undefined || a === null || b === null) { + return (a ?? null) === (b ?? null); + } + if (typeof a !== typeof b) return false; + if (Array.isArray(a)) { + if (!Array.isArray(b) || a.length !== b.length) return false; + return a.every((v, i) => deepEqual(v, b[i])); + } + if (typeof a === 'object') { + if (Array.isArray(b)) return false; + const ao = a as Record; + const bo = b as Record; + const keys = new Set([...Object.keys(ao), ...Object.keys(bo)]); + for (const key of keys) if (!deepEqual(ao[key], bo[key])) return false; + return true; + } + return false; +} + +export function utf8(text: string): Uint8Array { + return new TextEncoder().encode(text); +} + +/** SHA-256 (hex) of the canonical JSON of a value. */ +export function jsonHash(value: unknown): string { + return sha256Hex(utf8(canonicalJson(value))); +} + +/** Parses JSON a sync column holds; null for empty or unreadable text (a column another app wrote). */ +export function parseJsonColumn(text: unknown): T | null { + if (typeof text !== 'string' || !text) return null; + try { + return JSON.parse(text) as T; + } catch { + return null; + } +} + +export function clone(value: T): T { + return value === undefined ? value : (JSON.parse(JSON.stringify(value)) as T); +} diff --git a/src/device-sync/common/patch.ts b/src/device-sync/common/patch.ts new file mode 100644 index 00000000..a1d999c8 --- /dev/null +++ b/src/device-sync/common/patch.ts @@ -0,0 +1,88 @@ +/** + * JMAP PatchObjects (RFC 8620 §5.3): keys are JSON Pointers (RFC 6901) + * relative to the object, values replace what they point at, `null` removes. + * + * Stalwart specifics the engine relies on (docs/device-sync.md, "Uploads"): + * a pointer whose parent does not exist fails the whole update ("Patch + * operation failed"), so the first entry of a map is sent as the whole map; + * and no pointer may be a prefix of another in one patch. + */ +import { pointerToken, pointerTokenValue } from '../../api/patch-pointer'; +import { clone } from './json'; + +export type PatchObject = Record; + +/** Joins raw member names into a pointer, escaping `~` and `/` in each. */ +export function ptr(...segments: string[]): string { + return segments.map(pointerToken).join('/'); +} + +/** The raw member names of a pointer. */ +export function ptrSegments(pointer: string): string[] { + return pointer.split('/').map(pointerTokenValue); +} + +/** + * Throws when one pointer of the patch is a prefix of another (RFC 8620 + * §5.3 makes that an invalidPatch), or when a pointer is empty. + */ +export function assertPatchWellFormed(patch: PatchObject): void { + const keys = Object.keys(patch).sort(); + for (let i = 0; i < keys.length; i++) { + if (!keys[i]) throw new Error('empty pointer in patch'); + if (i > 0 && keys[i].startsWith(`${keys[i - 1]}/`)) { + throw new Error(`patch pointer ${keys[i - 1]} is a prefix of ${keys[i]}`); + } + } +} + +/** + * Applies a PatchObject to a copy of `target` the way Stalwart does. Returns + * null when a pointer's parent does not exist (Stalwart rejects the update). + * Array elements cannot be addressed; the engine never tries. + */ +export function applyPatch(target: T, patch: PatchObject): T | null { + const out = clone(target) as Record; + for (const [pointer, value] of Object.entries(patch)) { + const segments = ptrSegments(pointer); + let parent: Record = out; + for (let i = 0; i < segments.length - 1; i++) { + const next = parent[segments[i]]; + if (next === null || typeof next !== 'object' || Array.isArray(next)) return null; + parent = next as Record; + } + const leaf = segments[segments.length - 1]; + if (value === null) delete parent[leaf]; + else parent[leaf] = clone(value); + } + return out as T; +} + +/** + * Adds `value` at `pointer` to a patch, unless a shorter pointer of the patch + * already covers it (then the covering value is updated instead). + */ +export function setInPatch(patch: PatchObject, pointer: string, value: unknown): void { + for (const existing of Object.keys(patch)) { + if (pointer === existing) { + patch[existing] = value; + return; + } + if (pointer.startsWith(`${existing}/`)) { + const covered = patch[existing]; + if (covered === null || typeof covered !== 'object') { + throw new Error(`cannot patch ${pointer} below ${existing}, which is set to a scalar`); + } + const rest = pointer.slice(existing.length + 1); + const applied = applyPatch(covered as object, { [rest]: value }); + if (!applied) throw new Error(`cannot patch ${pointer}: missing parent below ${existing}`); + patch[existing] = applied; + return; + } + } + // A new covering pointer replaces the longer pointers it covers. + for (const existing of Object.keys(patch)) { + if (existing.startsWith(`${pointer}/`)) delete patch[existing]; + } + patch[pointer] = value; +} diff --git a/src/device-sync/planner.ts b/src/device-sync/planner.ts new file mode 100644 index 00000000..ebf16a05 --- /dev/null +++ b/src/device-sync/planner.ts @@ -0,0 +1,357 @@ +/** + * The contract between the sync engine (src/device-sync/engine, which owns all + * I/O: provider queries and batches, JMAP requests, retries, state) and the + * two mappers (src/device-sync/contacts and src/device-sync/calendar, which + * are pure planners). docs/device-sync.md describes the rules both follow. + * + * A planner never performs I/O. The engine queries rows with the planner's + * column lists, hands them to `decode*`, asks for a plan, and applies the + * plan's op groups. Every op group is applied atomically; asserts come first + * in a group, so a group whose rows changed since they were read fails as a + * whole and the engine re-reads and re-plans that item. + * + * `refs` inside a group's ops are indexes into THAT group; the engine rebases + * them when it packs several groups into one batch, and marks the first op of + * each group `yieldAllowed`. + */ +import type { AddressBookLike, CalendarLike, ContactCardWire, CalendarEventWire } from './wire'; +import type { ProviderOp, ReminderOwner, Row } from './types'; +import type { PatchObject } from './common/patch'; + +// ─── Shared ──────────────────────────────────────────── + +/** Ops for one item, applied atomically, asserts first. */ +export interface OpGroup { + /** For logs and reports: SOURCE_ID/_SYNC_ID or `row:<_id>`. */ + ref: string; + ops: ProviderOp[]; +} + +/** Why an item is left alone, with the fingerprint that lifts the back-off when the item changes. */ +export interface PoisonMarker { + /** Fingerprint of what failed (the patch, or the row projection). */ + fp: string; + /** SetError type or engine code (`dstAmbiguous`, `ruleNotRepresentable`, …). */ + type: string; + description?: string; + /** Attempts so far. */ + n: number; + /** Epoch ms before which the item is not retried unless `fp` changes. */ + until: number; +} + +/** A create claimed on the device: the uid and the collection it goes to, fixed at the claim. */ +export interface PendingCreate { + uid: string; + /** `/`. */ + target: string; +} + +export interface MapContext { + /** JMAP account of the object (the personal one or a shared one). */ + jmapAccountId: string; + now: number; + /** New map keys: `b` + 8 base36 characters, never one of `taken`. */ + mintKey(taken?: Iterable): string; + /** A new uid for a device-created object. */ + mintUid(): string; +} + +/** + * One server write an upload plan asks for. The engine batches them into + * `/set` calls per JMAP account, adds `ifInState`, and reports each result + * back through `planAccepted` / the SetError handling in the engine. + */ +export type UploadAction = + /** Create `object` (which carries `uid`) in `collectionId`, after looking the uid up. */ + | { kind: 'create'; uid: string; collectionId: string; object: Partial; sendSchedulingMessages?: boolean } + /** Patch the object; never empty. */ + | { kind: 'update'; id: string; patch: PatchObject; sendSchedulingMessages?: boolean } + /** Destroy the object (`id`), or whatever carries `uid` when the create's outcome is unknown. */ + | { kind: 'destroy'; id: string | null; uid: string | null; sendSchedulingMessages?: boolean }; + +/** + * What to do with a dirty, new or deleted local item. + * - `clean`: nothing mapped changed (only STARRED, say): apply `ops` (they clear DIRTY behind an assert). + * - `claim`: a new item without a pending uid: apply `ops` (they write the uid), then re-read and plan again. + * - `upload`: send `actions`; `local` is what they were computed from. + * - `purge`: a deleted item that never reached the server: apply `ops`. + * - `revert`: the item may not be changed on the server (read-only): apply `ops` (rewrite from the shadow). + * - `skip`: poisoned and backed off, or not representable; `reason` goes to the report. + */ +export type UploadPlan = + | { kind: 'clean'; ops: OpGroup } + | { kind: 'claim'; ops: OpGroup } + | { kind: 'upload'; actions: UploadAction[] } + | { kind: 'purge'; ops: OpGroup } + | { kind: 'revert'; ops: OpGroup; refetch?: boolean } + | { kind: 'skip'; reason: string }; + +export interface DownloadPlan { + ops: OpGroup; + /** Units where both sides changed differently; the server won each. */ + conflicts: number; + /** Local changes that survived the merge and still need an upload. */ + stillDirty: boolean; + /** `none` when rows already match (an echo): the engine then writes nothing. */ + effect: 'insert' | 'update' | 'none'; + /** Rows this plan inserts, updates or deletes; reported as progress only when applied. */ + writes: number; +} + +/** + * After the server accepted an upload (the engine re-fetched `server`): + * - `ops` write identity, shadow and baselines and clear DIRTY, behind the + * same assert the upload was read under (VERSION / projection); + * - `keepDirtyOps` are applied instead when that assert fails (edited again + * meanwhile): identity, shadow, and the baselines of the uploaded units set + * to the values that were uploaded; rows and DIRTY untouched. + */ +export interface AcceptedPlan { + ops: OpGroup; + keepDirtyOps: OpGroup; +} + +// ─── Contacts ────────────────────────────────────────── + +export interface LocalDataRow { + id: number; + mimetype: string; + /** The row's mapped cells (data1..data14, is_primary, group_sourceid), as read. */ + cells: Row; + /** DATA_SYNC1: entry key(s), a hint only (Fossify drops it). */ + key: string | null; + /** DATA_SYNC2: photo hash. */ + photoHash: string | null; + /** DATA_SYNC3 decoded: the mapped cells as stored after our last write. */ + baseline: Row | null; +} + +export interface LocalContact { + rawContactId: number; + /** `/`, null until created on the server. */ + sourceId: string | null; + version: number; + dirty: boolean; + deleted: boolean; + readOnly: boolean; + /** SYNC1: collection keys. */ + collections: string[]; + /** SYNC2: the last server card, photos reduced to hashes. */ + shadow: ContactCardWire | null; + /** SYNC3. */ + pending: PendingCreate | null; + /** SYNC4. */ + poison: PoisonMarker | null; + rows: LocalDataRow[]; +} + +export interface LocalGroup { + groupId: number; + sourceId: string | null; + version: number; + dirty: boolean; + deleted: boolean; + title: string | null; + shadow: ContactCardWire | null; + pending: PendingCreate | null; + poison: PoisonMarker | null; +} + +export interface ContactsContext extends MapContext { + /** Selected collection keys this card is in (`/`). */ + selectedCollections(card: ContactCardWire): string[]; + /** True when none of the card's address books grants `mayWrite`. */ + isReadOnly(card: ContactCardWire): boolean; + /** Whether a collection key is selected for sync on this device. */ + isSelected(collectionKey: string): boolean; + /** + * Where a contact created on the device goes, fixed at its claim: the chosen + * book when it is selected and writable, else the first selected writable + * book of the personal account; null when there is none (the item is poisoned). + */ + createTarget(): string | null; + /** Base64 bytes of a photo `media` entry; blob-backed ones are fetched by the engine beforehand. */ + photoBytes(entry: { uri?: string; blobId?: string; mediaType?: string }): string | null; + /** + * The device's photo of a raw contact as a JPEG (longer side ≤ 512 px, base64), read by the engine + * beforehand for every dirty contact that has a photo row; null when there is none. + */ + devicePhoto(rawContactId: number): string | null; + /** Display name of a card with this uid among the synced ones (Relation names). */ + nameForUid(uid: string): string | null; + /** Group rows by SOURCE_ID, for memberships. */ + groupRowIdBySourceId(sourceId: string): number | null; +} + +export interface ContactsPlanner { + readonly rawContactColumns: readonly string[]; + readonly dataColumns: readonly string[]; + readonly groupColumns: readonly string[]; + decodeContact(rawContact: Row, data: Row[]): LocalContact; + decodeGroup(group: Row): LocalGroup; + + /** + * A server individual/org card into a new, clean or dirty local contact. A + * local item without identity whose pending uid is the card's uid (our own + * create whose identity was never written) is adopted: identity and shadow + * are written, DIRTY stays, local changes are merged. + */ + planDownload(card: ContactCardWire, local: LocalContact | null, ctx: ContactsContext): DownloadPlan; + /** The server destroyed the card, or it left every selected address book. */ + planLocalDelete(local: LocalContact): OpGroup; + /** A clean contact whose rows drifted from their baselines (provider normalisation): rewrite the baselines. */ + planBaselineHeal(local: LocalContact): OpGroup | null; + planUpload(local: LocalContact, ctx: ContactsContext): UploadPlan; + planAccepted(local: LocalContact, server: ContactCardWire, ctx: ContactsContext): AcceptedPlan; + + /** Group cards (`kind: "group"`) ↔ Groups rows. */ + planGroupDownload(card: ContactCardWire, local: LocalGroup | null, ctx: ContactsContext): DownloadPlan; + planGroupLocalDelete(local: LocalGroup): OpGroup; + planGroupUpload(local: LocalGroup, ctx: ContactsContext): UploadPlan; + planGroupAccepted(local: LocalGroup, server: ContactCardWire, ctx: ContactsContext): AcceptedPlan; + /** + * Membership edits made on the device (GroupMembership rows of dirty + * contacts against the group cards' `members`) as patches of group cards. + * Called after contacts were uploaded, so new contacts have their uid. + */ + planMembershipUploads(contacts: LocalContact[], groups: LocalGroup[], ctx: ContactsContext): UploadAction[]; +} + +// ─── Calendar ────────────────────────────────────────── + +export interface LocalAttendee { + id: number; + cells: Row; +} + +export interface LocalReminder { + id: number; + cells: Row; +} + +/** Baseline of an event row: its mapped columns, attendees and reminders as stored (SYNC_DATA4). */ +export interface EventBaseline { + cells: Row; + attendees: Row[]; + reminders: Row[]; +} + +export interface LocalEventRow { + eventId: number; + calendarRowId: number; + /** `_SYNC_ID`: `/`, `/#` (exception), `~pending/`, or null. */ + syncId: string | null; + dirty: boolean; + deleted: boolean; + /** The mapped event columns as read (title, dtstart, rrule, status, …). */ + cells: Row; + attendees: LocalAttendee[]; + reminders: LocalReminder[]; + baseline: EventBaseline | null; + /** SYNC_DATA3. */ + pending: PendingCreate | null; + /** SYNC_DATA5. */ + poison: PoisonMarker | null; + /** Events.MUTATORS: packages that changed the row since our last write (lossy editors, see the design). */ + mutators: string | null; +} + +export interface LocalException extends LocalEventRow { + /** SYNC_DATA2 (our rows), else derived from ORIGINAL_INSTANCE_TIME by the planner. */ + recurrenceId: string | null; + originalInstanceTime: number | null; +} + +export interface LocalEvent extends LocalEventRow { + /** SYNC_DATA1: the last server event, with its overrides. */ + shadow: CalendarEventWire | null; + /** Rows whose ORIGINAL_SYNC_ID / ORIGINAL_ID point at this master. */ + exceptions: LocalException[]; + /** + * Set by `decodeEvents` when another master shares this `_SYNC_ID` (a + * CONTENT_EXCEPTION_URI split): `clone` on the newer row, which uploads as a + * new event; `source` on the older one, whose rule uploads as it stands. + */ + split?: 'clone' | 'source'; +} + +export interface LocalCalendar { + calendarRowId: number; + /** `/`. */ + syncId: string | null; + cells: Row; + /** CAL_SYNC2 (CAL_SYNC1 stays unused: CalendarProvider sends it as a sync extra). */ + shadow: CalendarLike | null; + /** CAL_SYNC3. */ + flags: { readOnly?: 'rights' | 'subscription'; taskOnly?: boolean } | null; +} + +export interface CalendarContext extends MapContext { + /** The device zone floating events are written in. */ + deviceZone: string; + /** `OWNER_ACCOUNT` of our calendars: the user's main calendar address, lowercase, no `mailto:`. */ + ownerAccount: string; + /** Every address of the user ("me"), lowercase, no `mailto:`. */ + selfAddresses: string[]; + reminderOwner: ReminderOwner; + /** Server calendars by id (for default alerts and rights). */ + calendar(calendarId: string): CalendarLike | undefined; + /** Android calendar row of a server calendar of `jmapAccountId`, if synced. */ + calendarRowId(calendarId: string): number | null; + /** Server calendar id of an Android calendar row, if it is one of ours. */ + calendarIdOfRow(calendarRowId: number): { jmapAccountId: string; calendarId: string } | null; + /** Read-only on the device: no write rights, or a subscribed iCal feed. */ + isReadOnly(calendarId: string): boolean; + /** Whether a collection key is selected for sync on this device. */ + isSelected(collectionKey: string): boolean; +} + +export interface CalendarPlanner { + readonly calendarColumns: readonly string[]; + readonly eventColumns: readonly string[]; + readonly attendeeColumns: readonly string[]; + readonly reminderColumns: readonly string[]; + decodeCalendar(row: Row): LocalCalendar; + /** Groups exception rows under their masters; orphan exceptions come back as masters with `deleted` untouched. */ + decodeEvents(events: Row[], attendees: Row[], reminders: Row[]): LocalEvent[]; + + /** + * Calendar rows for the selected server calendars: inserts, updates (name, + * colour, access, reminders allowed), deletes for deselected ones. The + * engine uploads dirty events of calendars to be deleted first. + */ + planCalendars( + selected: Array<{ jmapAccountId: string; calendar: CalendarLike; readOnly: boolean; accountName: string }>, + local: LocalCalendar[], + ctx: Omit, + ): { groups: OpGroup[]; deleteCalendarRows: number[] }; + + /** A server event (never a Task) into a new, clean or dirty local event with its exceptions; adopts a pending local event by uid like the contacts planner. */ + planDownload(event: CalendarEventWire, local: LocalEvent | null, ctx: CalendarContext): DownloadPlan; + /** Deletes the master and its exception rows (a sync-adapter delete does not cascade). */ + planLocalDelete(local: LocalEvent): OpGroup; + planBaselineHeal(local: LocalEvent): OpGroup | null; + /** May return several actions: a "this and following" split is a rule patch plus a create. */ + planUpload(local: LocalEvent, ctx: CalendarContext): UploadPlan; + planAccepted(local: LocalEvent, server: CalendarEventWire, ctx: CalendarContext): AcceptedPlan; + /** + * Deleted and new masters of one run that are one edit (Etar moves an event + * to another calendar, or turns a series into a single event, by deleting + * and re-inserting it): each pair uploads as a patch of the existing object; + * `ops` move the identity and exceptions to the new row and purge the old one + * once the patch is accepted. + */ + planPairs( + deleted: LocalEvent[], + fresh: LocalEvent[], + ctx: CalendarContext, + ): Array<{ deleted: LocalEvent; fresh: LocalEvent; actions: UploadAction[]; ops: OpGroup }>; + /** Floating events of a clean master re-written after the device zone changed. */ + planZoneChange(local: LocalEvent, previousZone: string, ctx: CalendarContext): OpGroup | null; + /** Reminder rows re-written after the reminder owner changed (no server writes). */ + planReminderOwnerChange(local: LocalEvent, ctx: CalendarContext): OpGroup | null; +} + +/** Re-exported so engine code imports one module. */ +export type { AddressBookLike, CalendarLike, ContactCardWire, CalendarEventWire }; diff --git a/src/device-sync/types.ts b/src/device-sync/types.ts index 8bf1b899..ce1f8fb8 100644 --- a/src/device-sync/types.ts +++ b/src/device-sync/types.ts @@ -322,6 +322,12 @@ export interface DeviceSyncNativeModule { /** Hands the report to the waiting adapter. Resolves false for an unknown or finished run. */ finishRun(runId: string, reportJson: string): Promise; isRunCancelled(runId: string): Promise; + /** + * Routes for the native push router: `{ [jmapAccountId]: [{ accountName, + * authorities }] }`, JSON. A push whose `changed` map names one of these + * accounts with a contact or calendar type requests those syncs. + */ + setPushRoutes(routesJson: string): Promise; /** `ProviderQuery` JSON in, `ProviderRows` JSON out. */ query(name: string, authority: Authority, queryJson: string): Promise; @@ -336,6 +342,65 @@ export interface DeviceSyncNativeModule { removeListeners(count: number): void; } +// ─── The JMAP side ───────────────────────────────────── + +/** A method call or response as on the wire: `[name, arguments, callId]`. */ +export type JmapInvocation = [name: string, args: Record, callId: string]; + +export interface JmapResponse { + methodResponses: JmapInvocation[]; + sessionState?: string; +} + +export interface JmapAccountView { + name: string; + isPersonal: boolean; + isReadOnly: boolean; + accountCapabilities: Record; +} + +export interface JmapSessionView { + username: string; + accounts: Record; + primaryAccounts: Record; + capabilities: Record; +} + +/** The core capability's limits the engine sizes its requests by. */ +export interface JmapCoreLimits { + maxObjectsInGet: number; + maxObjectsInSet: number; + maxCallsInRequest: number; + maxConcurrentRequests: number; + maxSizeRequest: number; +} + +/** + * The engine's view of one registry account's JMAP server: a detached + * JMAPClient on the device (src/device-sync/jmap/client-port.ts), the fake + * server in tests. Transport failures throw an Error whose `name` is + * `AuthenticationError`, `NetworkError`, `RequestTimeoutError` or + * `RateLimitError` (with `retryAfterMs`), as jmap-client.ts does; after a + * `NetworkError` or `RequestTimeoutError` the server may have applied the + * request. Any other thrown Error (jmap-client.ts throws plain ones for a + * non-OK HTTP status, e.g. `JMAP request failed: 503 - …`, and for a body that + * is not JSON) means no usable response and counts as `io`; a 400 for a limit + * is an engine bug. Method-level errors come back as + * `['error', { type }, callId]` responses. + */ +export interface JmapPort { + session(): JmapSessionView; + /** The session's core limits, with the RFC 8620 fallbacks where the server gives none. */ + limits(): JmapCoreLimits; + request(calls: JmapInvocation[], using: string[]): Promise; + /** Bytes of a blob-backed `media` entry (other servers; Stalwart keeps photos as `data:` URIs). */ + downloadBlob(accountId: string, blobId: string, type?: string): Promise; +} + +export const JMAP_CORE = 'urn:ietf:params:jmap:core'; +export const JMAP_CONTACTS = 'urn:ietf:params:jmap:contacts'; +export const JMAP_CALENDARS = 'urn:ietf:params:jmap:calendars'; + // ─── App-side preferences the engine reads ───────────── /** Collections are keyed `/` everywhere. */ diff --git a/src/device-sync/wire.ts b/src/device-sync/wire.ts new file mode 100644 index 00000000..cbbfb1c5 --- /dev/null +++ b/src/device-sync/wire.ts @@ -0,0 +1,83 @@ +/** + * JMAP objects as the device sync engine sees them: exactly as Stalwart sends + * and accepts them, without the client-only fields the app's stores add. + * Stalwart speaks JSCalendar 2.0, so `recurrenceRule` is singular on the wire + * (plural `recurrenceRules` and `excludedRecurrenceRule(s)` are rejected), and + * override values are PatchObjects. + */ +import type { + AddressBook, + Alert, + Calendar, + CalendarEvent, + ContactCard, + RecurrenceRule, +} from '../api/types'; + +type ContactClientOnly = 'accountId' | 'accountName' | 'isShared' | 'originalId'; +type EventClientOnly = + | 'recurrenceRules' + | 'excludedRecurrenceRules' + | 'recurrenceOverrides' + | 'originalId' + | 'originalCalendarIds' + | 'localAccountId' + | 'accountId' + | 'isShared' + | 'baseEventId' + | 'utcStart' + | 'utcEnd'; +type CalendarClientOnly = 'accountId' | 'originalId' | 'isShared' | 'colorIsLocalOverride'; + +/** A ContactCard (RFC 9553) on the wire. Unknown properties are kept as they came. */ +export type ContactCardWire = Omit & { + '@type'?: string; + version?: string; + [property: string]: unknown; +}; + +/** A CalendarEvent (JSCalendar 2.0 as Stalwart speaks it) on the wire. */ +export type CalendarEventWire = Omit & { + recurrenceRule?: RecurrenceRule | null; + /** Keys are LocalDateTimes in the event's zone; values are PatchObjects against the master. */ + recurrenceOverrides?: Record> | null; + [property: string]: unknown; +}; + +export type AddressBookLike = Omit; + +export type CalendarLike = Omit & { + defaultAlertsWithTime?: Record | null; + defaultAlertsWithoutTime?: Record | null; + timeZone?: string | null; + includeInAvailability?: string; +}; + +/** Every ContactCard property the engine asks `/get` for (explicit, see docs/device-sync.md). */ +export const CONTACT_CARD_PROPERTIES = [ + 'id', 'uid', 'addressBookIds', 'kind', 'language', 'name', 'nicknames', 'emails', 'phones', + 'onlineServices', 'preferredLanguages', 'organizations', 'titles', 'addresses', 'anniversaries', + 'personalInfo', 'notes', 'media', 'relatedTo', 'links', 'cryptoKeys', 'directories', 'keywords', + 'members', 'speakToAs', 'calendars', 'schedulingAddresses', 'localizations', 'prodId', 'created', + 'updated', '@type', 'version', +] as const; + +/** Every CalendarEvent property the engine asks `/get` for; `properties: null` would drop `useDefaultAlerts`. */ +export const CALENDAR_EVENT_PROPERTIES = [ + 'id', 'uid', '@type', 'calendarIds', 'isDraft', 'title', 'description', 'descriptionContentType', + 'start', 'duration', 'timeZone', 'endTimeZone', 'showWithoutTime', 'status', 'freeBusyStatus', + 'privacy', 'color', 'keywords', 'categories', 'priority', 'locale', 'sequence', 'method', + 'locations', 'mainLocationId', 'virtualLocations', 'links', 'relatedTo', 'participants', + 'organizerCalendarAddress', 'replyTo', 'sentBy', 'recurrenceRule', 'recurrenceOverrides', + 'recurrenceId', 'recurrenceIdTimeZone', 'alerts', 'useDefaultAlerts', 'created', 'updated', + 'prodId', 'progress', 'due', 'estimatedDuration', 'percentComplete', +] as const; + +export const ADDRESS_BOOK_PROPERTIES = [ + 'id', 'name', 'description', 'sortOrder', 'isDefault', 'isSubscribed', 'myRights', +] as const; + +export const CALENDAR_PROPERTIES = [ + 'id', 'name', 'description', 'color', 'sortOrder', 'isSubscribed', 'isVisible', 'isDefault', + 'includeInAvailability', 'defaultAlertsWithTime', 'defaultAlertsWithoutTime', 'timeZone', 'myRights', +] as const; diff --git a/src/stores/device-sync-store.ts b/src/stores/device-sync-store.ts new file mode 100644 index 00000000..2350fb4d --- /dev/null +++ b/src/stores/device-sync-store.ts @@ -0,0 +1,139 @@ +/** + * Device sync preferences and status per app account (Android only, #34; + * docs/device-sync.md, "App integration"). Keyed by `AccountEntry.id`. + * + * Whether sync is on is not stored here: Android's sync settings + * (`getSyncAutomatically` per authority) are the source of truth, so the + * toggles in Settings → Accounts and the app always agree. + * + * Persisted under `device-sync:v1` without the write delay the caches use: a + * headless sync run may be the only thing alive, and its status must survive + * the process. Not part of the settings export. A headless run reads and + * writes it only after `waitForDeviceSyncHydration()`: zustand 5 persists on + * every `set()`, even before hydration, and would overwrite the stored state. + */ +import { create } from 'zustand'; +import { persist } from 'zustand/middleware'; +import { createPersistStorage } from './persist-storage'; +import type { + AccountSyncPrefs, + Authority, + CollectionKey, + ReminderOwner, + RunStatus, +} from '../device-sync/types'; + +/** Periodic sync interval when the user has not picked one: push covers freshness. */ +export const DEFAULT_INTERVAL_SECONDS = 3600; + +export interface AccountDeviceSync { + /** Explicit choices only; unlisted collections follow the default (personal on, shared off). */ + contactsSelection: Record; + calendarSelection: Record; + /** Where contacts created on the device go; unset = the server's default address book. */ + newContactsAddressBook?: CollectionKey; + /** Who reminds the user of synced events; null until asked. */ + reminderOwner: ReminderOwner | null; + /** Periodic sync per authority in seconds; 0 = manual only. */ + intervalSeconds: Partial>; + lastRun: Partial>; + /** The Android account name this app account uses. */ + androidAccountName?: string; + /** Removed in Android Settings: the app shows sync as off and never recreates the account on its own. */ + removedInAndroidSettings?: boolean; + /** + * Latest item states the engine synced, per JMAP account and type + * (`ContactCard`, `CalendarEvent`, …). A StateChange naming the same state is + * our own echo and triggers no sync. + */ + knownStates?: Record>; +} + +export function emptyAccountDeviceSync(): AccountDeviceSync { + return { + contactsSelection: {}, + calendarSelection: {}, + reminderOwner: null, + intervalSeconds: {}, + lastRun: {}, + }; +} + +interface DeviceSyncState { + accounts: Record; + update: (registryId: string, patch: Partial) => void; + recordRunStatus: (registryId: string, authority: Authority, status: RunStatus) => void; + recordKnownState: (registryId: string, jmapAccountId: string, type: string, state: string) => void; + forget: (registryId: string) => void; +} + +export const useDeviceSyncStore = create()( + persist( + (set) => ({ + accounts: {}, + update: (registryId, patch) => + set((s) => ({ + accounts: { + ...s.accounts, + [registryId]: { ...(s.accounts[registryId] ?? emptyAccountDeviceSync()), ...patch }, + }, + })), + recordRunStatus: (registryId, authority, status) => + set((s) => { + const current = s.accounts[registryId] ?? emptyAccountDeviceSync(); + return { + accounts: { + ...s.accounts, + [registryId]: { ...current, lastRun: { ...current.lastRun, [authority]: status } }, + }, + }; + }), + recordKnownState: (registryId, jmapAccountId, type, state) => + set((s) => { + const current = s.accounts[registryId] ?? emptyAccountDeviceSync(); + const known = current.knownStates ?? {}; + return { + accounts: { + ...s.accounts, + [registryId]: { + ...current, + knownStates: { ...known, [jmapAccountId]: { ...(known[jmapAccountId] ?? {}), [type]: state } }, + }, + }, + }; + }), + forget: (registryId) => + set((s) => { + const { [registryId]: _gone, ...rest } = s.accounts; + return { accounts: rest }; + }), + }), + { + name: 'device-sync:v1', + storage: createPersistStorage({ writeDelayMs: 0 }), + partialize: (state) => ({ accounts: state.accounts }), + }, + ), +); + +/** Resolves once the persisted state has been read (at once when it already was). */ +export function waitForDeviceSyncHydration(): Promise { + if (useDeviceSyncStore.persist.hasHydrated()) return Promise.resolve(); + return new Promise((resolve) => { + const unsubscribe = useDeviceSyncStore.persist.onFinishHydration(() => { + unsubscribe(); + resolve(); + }); + }); +} + +/** The engine's view of one account's preferences. */ +export function accountSyncPrefs(registryId: string): AccountSyncPrefs { + const a = useDeviceSyncStore.getState().accounts[registryId] ?? emptyAccountDeviceSync(); + return { + contactsSelection: a.contactsSelection, + calendarSelection: a.calendarSelection, + newContactsAddressBook: a.newContactsAddressBook, + reminderOwner: a.reminderOwner ?? 'device', + }; +} From 95c8324f129263680b77ab0f26cd02c6d3e359e6 Mon Sep 17 00:00:00 2001 From: Linus Rath <139418639+rathlinus@users.noreply.github.com> Date: Sun, 27 Sep 2026 04:03:16 +0200 Subject: [PATCH 05/87] fix: let contacts apps edit contacts of the bulwark account #34 AOSP Contacts, and the apps built on its account-type parser, check the EditSchema in `xml/contacts.xml` and drop the whole account type when it breaks one of their rules: every data kind with a type column needs `` children, the other kinds must have none, and name, photo, nickname, organization, note and group membership need maxOccurs="1". The schema from ed2525e listed phone, email, postal, event and relationship without types, so the editors would not have offered the Bulwark account at all. The kinds now list the types device sync maps to JSContact, and the rules are written down in the file so the next edit keeps them. Checked on the API 32 emulator (AOSP Contacts): with a Bulwark account present, "Create new contact" offers it and opens the editor saving to it. --- android/app/src/main/res/xml/contacts.xml | 64 +++++++++++++++++++++-- 1 file changed, 59 insertions(+), 5 deletions(-) diff --git a/android/app/src/main/res/xml/contacts.xml b/android/app/src/main/res/xml/contacts.xml index d6247191..1aff029f 100644 --- a/android/app/src/main/res/xml/contacts.xml +++ b/android/app/src/main/res/xml/contacts.xml @@ -3,6 +3,13 @@ How Contacts apps show and edit contacts of the Bulwark account (#34). The EditSchema lists exactly the data kinds device sync maps to JSContact (docs/device-sync.md, "Contacts mapping"); apps offer nothing else. + + AOSP Contacts drops the whole account type when this schema breaks its rules + (BaseAccountType): every kind with a type column needs children and + the others must have none, name needs maxOccurs="1" and every supports* + attribute true (supportsDisplayName for older releases), and photo, + nickname, organization, note and group_membership need maxOccurs="1". + A Type's maxOccurs limits that type only; birthday gets none unless given. --> @@ -15,14 +22,61 @@ supportsPhoneticMiddleName="true" supportsPhoneticGivenName="true" /> - - + + + + + + + + + + + + + + + + + + + + + + - + + + + + + + - - + + + + + + + + + + + + + + + + + + + + + + + From f743ab0f7d72984f3566707994c8235d49b89a18 Mon Sep 17 00:00:00 2001 From: Linus Rath <139418639+rathlinus@users.noreply.github.com> Date: Sun, 27 Sep 2026 04:04:00 +0200 Subject: [PATCH 06/87] feat: give device sync scoped access to the contacts and calendar providers #34 The engine plans its provider reads and writes in JS, but only native code can reach ContactsProvider and CalendarProvider. This adds that bridge to the BulwarkDeviceSync module, with the rest of what the engine and the app need from it: - `query`, `applyBatch`, `readSyncState` and `readPhoto` take the `ProviderQuery`/`ProviderOp[]` JSON of `src/device-sync/types.ts`, check it (`sync/ProviderOps.kt`) and run it for one Android account of our type (`sync/ProviderScope.kt`, `sync/ProviderIo.kt`). Every selection gets the account ANDed in, every row an op names by id or points to as a parent must be the account's, and every URI carries `caller_is_syncadapter`, so nothing the sync writes marks rows dirty or schedules an upload. The calls run one at a time on the module's own thread through an unstable provider client, so a long batch never stalls other module calls and a provider that dies fails the call instead of the app. - `showSyncProblem` and `clearSyncProblem` post and clear the one notification per account that needs the user, e.g. "Sign in again". - `setPushRoutes` stores which Android accounts each JMAP account feeds, for the push router in `sync/PushRouting.kt`; the push services start using it in a later commit. - A run now ends exactly once and refuses a report that comes after that. A sync requested while a run is open makes the run ask for another sync when it ends, because SyncManager drops such requests as duplicates. `onPerformSync` lets nothing escape any more: an exception there killed the whole process, UI included. - `ensureAccount` no longer hands an existing Android account to another app account (`conflict`): its rows and SyncState belong to the first. Payloads cross as JSON strings, which the bridge moves much faster than nested maps. `src/device-sync/native.ts` is the typed JS side. The fake provider follows the rules the emulator runs turned up: contacts `data1`..`data14` read back as text, events and calendars the sync adapter inserts have DIRTY NULL, attendee rows are addressed by id, an op by id expects one row, and a vanished parent fails with `assert` while another account's fails with `scope`. Checked: 54 JVM tests (`gradlew :app:testDebugUnitTest`), debug and release (R8) builds, `npm run typecheck` and `npm test` on this commit, and on the API 32 emulator through the debug receiver: inserts with back-references and group memberships, SyncState, photo writes and scaled reads, asserts and expected counts, both scope refusals, the 499-op yield boundary, `tooLarge` for a 1.3 MB batch, and the notification. --- .../sync/DeviceSyncDebugReceiver.kt | 104 +++- .../sync/BulwarkDeviceSyncModule.kt | 107 +++- .../bulwarkmobile/sync/DeviceSyncAccounts.kt | 45 +- .../bulwarkmobile/sync/DeviceSyncAdapter.kt | 29 +- .../bulwarkmobile/sync/DeviceSyncRuns.kt | 87 ++- .../bulwarkmobile/sync/ProviderIo.kt | 368 +++++++++++++ .../bulwarkmobile/sync/ProviderOps.kt | 410 ++++++++++++++ .../bulwarkmobile/sync/ProviderScope.kt | 489 +++++++++++++++++ .../bulwarkmobile/sync/PushRouting.kt | 162 ++++++ .../bulwarkmobile/sync/SyncProblemNotifier.kt | 75 +++ .../sync/DeviceSyncAccountsTest.kt | 18 + .../bulwarkmobile/sync/DeviceSyncRunsTest.kt | 62 +++ .../bulwarkmobile/sync/ProviderOpsTest.kt | 515 ++++++++++++++++++ .../bulwarkmobile/sync/PushRoutingTest.kt | 122 +++++ docs/device-sync.md | 61 ++- .../__tests__/fake-provider.test.ts | 185 ++++++- .../__tests__/fakes/fake-provider.ts | 237 ++++++-- src/device-sync/__tests__/native.test.ts | 342 ++++++++++++ src/device-sync/native.ts | 255 +++++++++ src/device-sync/types.ts | 46 +- 20 files changed, 3591 insertions(+), 128 deletions(-) create mode 100644 android/app/src/main/java/com/anonymous/bulwarkmobile/sync/ProviderIo.kt create mode 100644 android/app/src/main/java/com/anonymous/bulwarkmobile/sync/ProviderOps.kt create mode 100644 android/app/src/main/java/com/anonymous/bulwarkmobile/sync/ProviderScope.kt create mode 100644 android/app/src/main/java/com/anonymous/bulwarkmobile/sync/PushRouting.kt create mode 100644 android/app/src/main/java/com/anonymous/bulwarkmobile/sync/SyncProblemNotifier.kt create mode 100644 android/app/src/test/java/com/anonymous/bulwarkmobile/sync/DeviceSyncAccountsTest.kt create mode 100644 android/app/src/test/java/com/anonymous/bulwarkmobile/sync/DeviceSyncRunsTest.kt create mode 100644 android/app/src/test/java/com/anonymous/bulwarkmobile/sync/ProviderOpsTest.kt create mode 100644 android/app/src/test/java/com/anonymous/bulwarkmobile/sync/PushRoutingTest.kt create mode 100644 src/device-sync/__tests__/native.test.ts create mode 100644 src/device-sync/native.ts diff --git a/android/app/src/debug/java/com/anonymous/bulwarkmobile/sync/DeviceSyncDebugReceiver.kt b/android/app/src/debug/java/com/anonymous/bulwarkmobile/sync/DeviceSyncDebugReceiver.kt index b08f283b..b1bd2a4d 100644 --- a/android/app/src/debug/java/com/anonymous/bulwarkmobile/sync/DeviceSyncDebugReceiver.kt +++ b/android/app/src/debug/java/com/anonymous/bulwarkmobile/sync/DeviceSyncDebugReceiver.kt @@ -3,47 +3,97 @@ package com.anonymous.bulwarkmobile.sync import android.content.BroadcastReceiver import android.content.Context import android.content.Intent +import android.util.Base64 import android.util.Log +import java.io.File +import kotlin.concurrent.thread /** - * Debug builds only: lets adb set up device sync accounts without the settings - * UI, for the device tests in docs/device-sync.md ("Testing"). Protected by - * android.permission.DUMP, which only the shell and the system hold. Syncs - * themselves are triggered with `adb shell requestsync`. + * Debug builds only: lets adb set up device sync accounts and drive the + * provider bridge without the settings UI or JS, for the device tests in + * docs/device-sync.md ("Testing"). Protected by android.permission.DUMP, + * which only the shell and the system hold. Syncs themselves are triggered + * with `adb shell requestsync`. * * adb shell am broadcast -n com.anonymous.bulwarkmobile/.sync.DeviceSyncDebugReceiver \ * --es cmd ensure --es name usera@example.org --es registry usera@example.org@10.0.2.2 \ * --es enable contacts,calendar * * Commands: `ensure` (name, registry, optional enable), `enable` (name, - * authority, optional --ez on false), `remove` (name) and `list`. + * authority, optional --ez on false), `remove` (name), `list`, and the + * provider calls `query` and `batch` (name, authority, and the + * `ProviderQuery`/`ProviderOp[]` JSON base64-encoded in `b64`, which spares + * the shell's quoting, or pushed to the app's external files directory and + * named by `file` when it is too long for a command line), `syncstate` + * (name, authority) and `photo` (name, `--el id`, optional `--ei max`); and + * the sync-problem notification, `problem` (name, title, text, uri, optional + * channel) and `clearproblem` (name). Results go to logcat (tag + * BulwarkDeviceSync) and the broadcast's result data. */ class DeviceSyncDebugReceiver : BroadcastReceiver() { override fun onReceive(context: Context, intent: Intent) { - val result = try { - val name = intent.getStringExtra("name") - when (intent.getStringExtra("cmd")) { - "ensure" -> { - val created = DeviceSyncAccounts.ensure(context, name!!, intent.getStringExtra("registry")!!) - for (which in intent.getStringExtra("enable").orEmpty().split(',').filter { it.isNotBlank() }) { - DeviceSyncAccounts.setSyncEnabled(context, name, authority(which), true) - } - "created=$created" - } - "enable" -> { - val on = intent.getBooleanExtra("on", true) - DeviceSyncAccounts.setSyncEnabled(context, name!!, authority(intent.getStringExtra("authority")!!), on) - "enabled=$on" - } - "remove" -> "removed=${DeviceSyncAccounts.remove(context, name!!)}" - "list" -> DeviceSyncAccounts.list(context).joinToString(";") { (account, registryId) -> "${account.name}=$registryId" } - else -> "unknown cmd" + val pending = goAsync() + // Provider calls block; keep them off the main thread. + thread(name = "DeviceSyncDebugReceiver") { + val result = try { + run(context, intent) + } catch (e: Exception) { + "error: ${e.javaClass.simpleName}: ${e.message}" } - } catch (e: Exception) { - "error: ${e.javaClass.simpleName}: ${e.message}" + Log.i(TAG, "debug command: $result") + pending.resultData = result + pending.finish() } - Log.i(TAG, "debug command: $result") - resultData = result + } + + private fun run(context: Context, intent: Intent): String { + val name = intent.getStringExtra("name") + return when (intent.getStringExtra("cmd")) { + "ensure" -> { + val created = DeviceSyncAccounts.ensure(context, name!!, intent.getStringExtra("registry")!!) + for (which in intent.getStringExtra("enable").orEmpty().split(',').filter { it.isNotBlank() }) { + DeviceSyncAccounts.setSyncEnabled(context, name, authority(which), true) + } + "created=$created" + } + "enable" -> { + val on = intent.getBooleanExtra("on", true) + DeviceSyncAccounts.setSyncEnabled(context, name!!, authority(intent.getStringExtra("authority")!!), on) + "enabled=$on" + } + "remove" -> "removed=${DeviceSyncAccounts.remove(context, name!!)}" + "list" -> DeviceSyncAccounts.list(context).joinToString(";") { (account, registryId) -> "${account.name}=$registryId" } + "query" -> io(context, intent) { it.query(json(context, intent)) } + "batch" -> io(context, intent) { it.applyBatch(json(context, intent)) } + "syncstate" -> io(context, intent) { it.readSyncState().toString() } + "photo" -> ProviderIo(context, name!!, DeviceSyncAccounts.CONTACTS_AUTHORITY).use { + it.readPhoto(intent.getLongExtra("id", -1), intent.getIntExtra("max", 512)).toString() + } + "problem" -> { + SyncProblemNotifier.show( + context, + name!!, + intent.getStringExtra("title").orEmpty(), + intent.getStringExtra("text").orEmpty(), + intent.getStringExtra("uri").orEmpty(), + intent.getStringExtra("channel"), + ) + "shown" + } + "clearproblem" -> { + SyncProblemNotifier.clear(context, name!!) + "cleared" + } + else -> "unknown cmd" + } + } + + private fun io(context: Context, intent: Intent, call: (ProviderIo) -> String): String = + ProviderIo(context, intent.getStringExtra("name")!!, authority(intent.getStringExtra("authority")!!)).use(call) + + private fun json(context: Context, intent: Intent): String { + intent.getStringExtra("file")?.let { return File(context.getExternalFilesDir(null), it).readText() } + return String(Base64.decode(intent.getStringExtra("b64")!!, Base64.DEFAULT), Charsets.UTF_8) } private fun authority(which: String) = when (which) { diff --git a/android/app/src/main/java/com/anonymous/bulwarkmobile/sync/BulwarkDeviceSyncModule.kt b/android/app/src/main/java/com/anonymous/bulwarkmobile/sync/BulwarkDeviceSyncModule.kt index e60da9f2..34b95d3f 100644 --- a/android/app/src/main/java/com/anonymous/bulwarkmobile/sync/BulwarkDeviceSyncModule.kt +++ b/android/app/src/main/java/com/anonymous/bulwarkmobile/sync/BulwarkDeviceSyncModule.kt @@ -19,11 +19,15 @@ import com.facebook.react.bridge.ReactMethod import com.facebook.react.bridge.WritableMap import com.facebook.react.modules.core.DeviceEventManagerModule.RCTDeviceEventEmitter import org.json.JSONObject +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors +import java.util.concurrent.RejectedExecutionException /** - * `NativeModules.BulwarkDeviceSync`: Android accounts, sync settings and the - * run handshake of device sync (#34). The contract is `DeviceSyncNativeModule` - * in src/device-sync/types.ts; docs/device-sync.md explains it. + * `NativeModules.BulwarkDeviceSync`: Android accounts, sync settings, the run + * handshake, provider I/O, push routes and the sync-problem notification of + * device sync (#34). The contract is `DeviceSyncNativeModule` in + * src/device-sync/types.ts; docs/device-sync.md explains it. */ class BulwarkDeviceSyncModule(reactContext: ReactApplicationContext) : ReactContextBaseJavaModule(reactContext) { @@ -32,6 +36,15 @@ class BulwarkDeviceSyncModule(reactContext: ReactApplicationContext) : emit(ACCOUNTS_CHANGED_EVENT, null) } + /** + * Provider I/O runs here, one call at a time: a batch can hold the + * provider for seconds (it sleeps at a contended yield point), which would + * stall every other native module call on React's native-modules thread. + */ + private val io: ExecutorService = Executors.newSingleThreadExecutor { task -> + Thread(task, "BulwarkDeviceSync-io").apply { isDaemon = true } + } + init { synchronized(BulwarkDeviceSyncModule::class.java) { currentInstance = this } } @@ -59,6 +72,7 @@ class BulwarkDeviceSyncModule(reactContext: ReactApplicationContext) : } catch (e: RuntimeException) { // Not registered. } + io.shutdown() super.invalidate() } @@ -181,6 +195,91 @@ class BulwarkDeviceSyncModule(reactContext: ReactApplicationContext) : promise.resolve(DeviceSyncRuns.isCancelled(runId)) } + // ── provider I/O ──────────────────────────────────────────── + + /** `ProviderQuery` JSON in, `ProviderRows` JSON out; rejects with `scope`, `permission` or `provider`. */ + @ReactMethod + fun query(name: String, authority: String, queryJson: String, promise: Promise) = onIo(promise) { + ProviderIo(reactApplicationContext, name, authority).use { it.query(queryJson) } + } + + /** `ProviderOp[]` JSON in, `BatchResult` JSON out. Never rejects: a failed batch is a result. */ + @ReactMethod + fun applyBatch(name: String, authority: String, opsJson: String, promise: Promise) { + val failed = { e: Throwable -> ProviderResults.failed(ProviderIo.failureOf(e), e.message ?: e.javaClass.simpleName) } + try { + io.execute { + val result = try { + ProviderIo(reactApplicationContext, name, authority).use { it.applyBatch(opsJson) } + } catch (e: Throwable) { + failed(e) + } + promise.resolve(result) + } + } catch (e: RejectedExecutionException) { + promise.resolve(failed(e)) + } + } + + @ReactMethod + fun readSyncState(name: String, authority: String, promise: Promise) = onIo(promise) { + ProviderIo(reactApplicationContext, name, authority).use { it.readSyncState() } + } + + /** `PhotoData` JSON, or null when the raw contact has no photo; rejects with `scope` for another account's. */ + @ReactMethod + fun readPhoto(name: String, rawContactId: Double, maxPx: Double, promise: Promise) = onIo(promise) { + ProviderIo(reactApplicationContext, name, DeviceSyncAccounts.CONTACTS_AUTHORITY) + .use { it.readPhoto(rawContactId.toLong(), maxPx.toInt()) } + } + + /** Settles [promise] with [block]'s result on the I/O thread; rejects with a `BatchFailure` code. */ + private fun onIo(promise: Promise, block: () -> Any?) { + try { + io.execute { + try { + promise.resolve(block()) + } catch (e: Throwable) { + promise.reject(ProviderIo.failureOf(e).wire, e.message ?: e.javaClass.simpleName, e) + } + } + } catch (e: RejectedExecutionException) { + promise.reject("provider", "Device sync is shutting down", e) + } + } + + // ── push routes and notifications ─────────────────────────── + + /** `{ jmapAccountId: [{ accountName, authorities }] }` for the push router (DeviceSyncPushRouter). */ + @ReactMethod + fun setPushRoutes(routesJson: String, promise: Promise) = settle(promise) { + DeviceSyncPushRouter.saveRoutes(reactApplicationContext, routesJson) + null + } + + /** + * Posts (or replaces) the account's sync-problem notification. [channelName] + * names the notification channel in the app's language; null keeps English. + */ + @ReactMethod + fun showSyncProblem( + accountName: String, + title: String, + text: String, + uri: String, + channelName: String?, + promise: Promise, + ) = settle(promise) { + SyncProblemNotifier.show(reactApplicationContext, accountName, title, text, uri, channelName) + null + } + + @ReactMethod + fun clearSyncProblem(accountName: String, promise: Promise) = settle(promise) { + SyncProblemNotifier.clear(reactApplicationContext, accountName) + null + } + // NativeEventEmitter bookkeeping. @ReactMethod fun addListener(eventName: String) {} @@ -191,6 +290,8 @@ class BulwarkDeviceSyncModule(reactContext: ReactApplicationContext) : private inline fun settle(promise: Promise, block: () -> Any?) { try { promise.resolve(block()) + } catch (e: AccountConflictException) { + promise.reject("conflict", e.message, e) } catch (e: SecurityException) { promise.reject("permission", e.message, e) } catch (e: IllegalArgumentException) { diff --git a/android/app/src/main/java/com/anonymous/bulwarkmobile/sync/DeviceSyncAccounts.kt b/android/app/src/main/java/com/anonymous/bulwarkmobile/sync/DeviceSyncAccounts.kt index 2bbbcf86..29e2fa7c 100644 --- a/android/app/src/main/java/com/anonymous/bulwarkmobile/sync/DeviceSyncAccounts.kt +++ b/android/app/src/main/java/com/anonymous/bulwarkmobile/sync/DeviceSyncAccounts.kt @@ -36,18 +36,32 @@ object DeviceSyncAccounts { return manager.getAccountsByType(accountType(context)).map { it to manager.getUserData(it, USER_DATA_REGISTRY_ID) } } + fun exists(context: Context, name: String): Boolean = + AccountManager.get(context).getAccountsByType(accountType(context)).any { it.name == name } + /** - * Adds the account when it is missing and (re)writes its registry id. - * Both authorities start syncable with automatic sync off; the app turns - * on the ones the user picked. Returns true when the account was created. + * Adds the account when it is missing. Both authorities start syncable + * with automatic sync off; the app turns on the ones the user picked. + * Returns true when the account was created, false when it exists for + * [registryId] already. An account that belongs to another registry id + * keeps it ([AccountConflictException]): its rows and SyncState were + * written for that app account. */ fun ensure(context: Context, name: String, registryId: String): Boolean { val manager = AccountManager.get(context) val account = account(context, name) val exists = manager.getAccountsByType(account.type).any { it.name == name } - if (exists) { - manager.setUserData(account, USER_DATA_REGISTRY_ID, registryId) - return false + val current = if (exists) manager.getUserData(account, USER_DATA_REGISTRY_ID) else null + when (ensureAction(exists, current, registryId)) { + EnsureAction.KEEP -> return false + EnsureAction.ADOPT -> { + manager.setUserData(account, USER_DATA_REGISTRY_ID, registryId) + return false + } + EnsureAction.CONFLICT -> throw AccountConflictException( + "The Android account $name belongs to another app account ($current)", + ) + EnsureAction.CREATE -> Unit } val userData = Bundle().apply { putString(USER_DATA_REGISTRY_ID, registryId) } check(manager.addAccountExplicitly(account, null, userData)) { "Android refused to add the account" } @@ -58,6 +72,16 @@ object DeviceSyncAccounts { return true } + enum class EnsureAction { CREATE, KEEP, ADOPT, CONFLICT } + + /** An existing account without a registry id (never completed) is adopted; one with another id is not touched. */ + fun ensureAction(exists: Boolean, currentRegistryId: String?, registryId: String): EnsureAction = when { + !exists -> EnsureAction.CREATE + currentRegistryId == registryId -> EnsureAction.KEEP + currentRegistryId.isNullOrEmpty() -> EnsureAction.ADOPT + else -> EnsureAction.CONFLICT + } + /** Removes the account; the Contacts and Calendar providers drop all of its rows. */ fun remove(context: Context, name: String): Boolean { val manager = AccountManager.get(context) @@ -83,8 +107,14 @@ object DeviceSyncAccounts { if (seconds > 0) ContentResolver.addPeriodicSync(account, authority, Bundle.EMPTY, seconds) } + /** + * Requests a sync. SyncManager drops a request that matches a sync it is + * running, so a run of that account and authority is also flagged to + * ask for another sync when it ends (see [DeviceSyncRuns.flagSyncAgain]). + */ fun requestSync(context: Context, name: String, authority: String, flags: SyncRequestFlags) { requireAuthority(authority) + DeviceSyncRuns.flagSyncAgain(name, authority) ContentResolver.requestSync(account(context, name), authority, flags.toExtras()) } @@ -93,6 +123,9 @@ object DeviceSyncAccounts { } } +/** [DeviceSyncAccounts.ensure] refused to hand an account of one app account to another. */ +class AccountConflictException(message: String) : Exception(message) + /** The SYNC_EXTRAS_* flags a request carries. Constant extras let repeated requests coalesce. */ data class SyncRequestFlags( val manual: Boolean = false, diff --git a/android/app/src/main/java/com/anonymous/bulwarkmobile/sync/DeviceSyncAdapter.kt b/android/app/src/main/java/com/anonymous/bulwarkmobile/sync/DeviceSyncAdapter.kt index 6bf83444..aa4b851d 100644 --- a/android/app/src/main/java/com/anonymous/bulwarkmobile/sync/DeviceSyncAdapter.kt +++ b/android/app/src/main/java/com/anonymous/bulwarkmobile/sync/DeviceSyncAdapter.kt @@ -40,6 +40,13 @@ class DeviceSyncAdapter(context: Context) : private val main = Handler(Looper.getMainLooper()) + /** + * Anything thrown out of here is rethrown on the sync thread + * (AbstractThreadedSyncAdapter's SyncThread) and kills the process, the UI + * included. So every failure becomes a result: a soft error SyncManager + * retries with back-off, or a database error for a missing permission, + * as the framework would report it. + */ override fun onPerformSync( account: Account, extras: Bundle, @@ -47,6 +54,18 @@ class DeviceSyncAdapter(context: Context) : provider: ContentProviderClient, syncResult: SyncResult, ) { + try { + performSync(account, extras, authority, syncResult) + } catch (e: SecurityException) { + Log.w(TAG, "Sync of $authority refused by a permission check", e) + syncResult.databaseError = true + } catch (t: Throwable) { + Log.e(TAG, "Sync of $authority failed", t) + syncResult.stats.numIoExceptions++ + } + } + + private fun performSync(account: Account, extras: Bundle, authority: String, syncResult: SyncResult) { val registryId = AccountManager.get(context).getUserData(account, DeviceSyncAccounts.USER_DATA_REGISTRY_ID) if (registryId.isNullOrEmpty()) { Log.w(TAG, "Sync of $authority skipped: the account has no app account") @@ -69,9 +88,9 @@ class DeviceSyncAdapter(context: Context) : run.end("no report before the deadline") } } - val outcome = SyncReports.outcome(run.report.get(), run.endReason.get()) + val outcome = SyncReports.outcome(run.report, run.endReason) SyncReports.apply(outcome, syncResult) - Log.i(TAG, "Sync ${run.runId} of $authority ended (${run.endReason.get()}) after " + + Log.i(TAG, "Sync ${run.runId} of $authority ended (${run.endReason}) after " + "${SystemClock.elapsedRealtime() - startedAt} ms: $outcome") } catch (e: InterruptedException) { // onSyncCanceled interrupts this thread. JS sees the run as @@ -84,6 +103,12 @@ class DeviceSyncAdapter(context: Context) : // After startRun, which was posted first, so a listener it added is removed. main.post { run.detach() } } + // A change pushed while the run was open: SyncManager dropped its + // request as a duplicate of this sync, so ask for the next one here. + if (run.syncAgainRequested) { + Log.i(TAG, "Sync ${run.runId} of $authority: changes arrived meanwhile, syncing again") + syncResult.fullSyncRequested = true + } } @MainThread diff --git a/android/app/src/main/java/com/anonymous/bulwarkmobile/sync/DeviceSyncRuns.kt b/android/app/src/main/java/com/anonymous/bulwarkmobile/sync/DeviceSyncRuns.kt index cbe67561..9e46adeb 100644 --- a/android/app/src/main/java/com/anonymous/bulwarkmobile/sync/DeviceSyncRuns.kt +++ b/android/app/src/main/java/com/anonymous/bulwarkmobile/sync/DeviceSyncRuns.kt @@ -6,24 +6,34 @@ import com.facebook.react.jstasks.HeadlessJsTaskEventListener import java.util.UUID import java.util.concurrent.ConcurrentHashMap import java.util.concurrent.CountDownLatch +import java.util.concurrent.atomic.AtomicBoolean import java.util.concurrent.atomic.AtomicReference /** * The device sync runs an adapter is waiting for, keyed by a random run id. * - * A run ends on the first of: JS calling finishRun, the headless task ending - * (JS settled, timed out or was reloaded away), React refusing or failing to - * start, the adapter's deadline, or a cancel. Headless task ids restart at 1 - * with every React context, so they never identify a run; the run id does, - * and a report for a run nobody waits for any more is ignored. + * A run ends exactly once, on the first of: JS calling finishRun, the + * headless task ending (JS settled, timed out or was reloaded away), React + * refusing or failing to start, the adapter's deadline, or a cancel. A report + * that arrives after that is refused whole rather than half-applied. Headless + * task ids restart at 1 with every React context, so they never identify a + * run; the run id does. */ object DeviceSyncRuns { const val NO_TASK = -1 + enum class State { WAITING, ENDED } + class Run(val runId: String, val accountName: String, val authority: String) { + private val state = AtomicReference(State.WAITING) + private val syncAgain = AtomicBoolean(false) val done = CountDownLatch(1) - val report = AtomicReference(null) - val endReason = AtomicReference(null) + + /** Written before [done] opens, read after it: the latch orders them. */ + @Volatile var report: String? = null + private set + @Volatile var endReason: String? = null + private set @Volatile var cancelled = false @Volatile var boot: TaskInterface? = null /** Written and compared on the UI thread only. */ @@ -31,10 +41,24 @@ object DeviceSyncRuns { @Volatile private var tasks: HeadlessJsTaskContext? = null @Volatile private var listener: HeadlessJsTaskEventListener? = null - /** Releases the waiting adapter. Idempotent: the first reason wins. */ - fun end(reason: String) { - endReason.compareAndSet(null, reason) + val isWaiting: Boolean get() = state.get() == State.WAITING + + /** True once a sync was requested for this account and authority while the run was open. */ + val syncAgainRequested: Boolean get() = syncAgain.get() + + /** Releases the waiting adapter. Only the first end counts; returns whether this was it. */ + fun end(reason: String): Boolean = end(reason, null) + + internal fun end(reason: String, reportJson: String?): Boolean { + if (!state.compareAndSet(State.WAITING, State.ENDED)) return false + report = reportJson + endReason = reason done.countDown() + return true + } + + internal fun requestSyncAgain() { + syncAgain.set(true) } fun attach(tasks: HeadlessJsTaskContext, listener: HeadlessJsTaskEventListener) { @@ -56,27 +80,46 @@ object DeviceSyncRuns { fun open(accountName: String, authority: String): Run = Run(UUID.randomUUID().toString(), accountName, authority).also { runs[it.runId] = it } - /** Forgets the run: a late finishRun is then ignored and isCancelled answers true. */ + /** + * Forgets the run: a late finishRun is then refused and isCancelled + * answers true. Read [Run.syncAgainRequested] after this, so a request + * can no longer slip in between reading the flag and closing. + */ fun close(run: Run) { + run.end("closed") run.cancelled = true runs.remove(run.runId) } - /** From JS. Returns false when nothing waits for this run (stale, duplicate or finished). */ - fun finish(runId: String, reportJson: String): Boolean { - val run = runs[runId] ?: return false - if (!run.report.compareAndSet(null, reportJson)) return false - run.end("reported") - return true - } + /** From JS. Accepted only while the adapter still waits for this run: false when it ended, timed out or is unknown. */ + fun finish(runId: String, reportJson: String): Boolean = + runs[runId]?.end("reported", reportJson) ?: false - /** Unknown runs count as cancelled: the adapter has stopped waiting for them. */ - fun isCancelled(runId: String): Boolean = runs[runId]?.cancelled ?: true + /** Runs nobody waits for, or that already ended, count as cancelled: JS should stop. */ + fun isCancelled(runId: String): Boolean { + val run = runs[runId] ?: return true + return run.cancelled || !run.isWaiting + } fun cancel(runId: String) { runs[runId]?.cancelled = true } - /** Runs currently waited for, newest last; for status and the push router's "sync again" flag. */ - fun active(): List = runs.values.toList() + /** + * Flags the open runs of [accountName] and [authority] to request another + * sync when they end (`SyncResult.fullSyncRequested`): SyncManager drops a + * `requestSync` that matches a sync it is running, so a change pushed + * mid-run would otherwise wait for the next trigger. Returns whether a + * run was flagged. + */ + fun flagSyncAgain(accountName: String, authority: String): Boolean { + var flagged = false + for (run in runs.values) { + if (run.accountName == accountName && run.authority == authority) { + run.requestSyncAgain() + flagged = true + } + } + return flagged + } } diff --git a/android/app/src/main/java/com/anonymous/bulwarkmobile/sync/ProviderIo.kt b/android/app/src/main/java/com/anonymous/bulwarkmobile/sync/ProviderIo.kt new file mode 100644 index 00000000..cb0e250d --- /dev/null +++ b/android/app/src/main/java/com/anonymous/bulwarkmobile/sync/ProviderIo.kt @@ -0,0 +1,368 @@ +package com.anonymous.bulwarkmobile.sync + +import android.accounts.Account +import android.content.ContentProviderClient +import android.content.ContentProviderOperation +import android.content.ContentUris +import android.content.ContentValues +import android.content.Context +import android.content.OperationApplicationException +import android.database.Cursor +import android.graphics.Bitmap +import android.graphics.BitmapFactory +import android.net.Uri +import android.os.TransactionTooLargeException +import android.provider.CalendarContract +import android.provider.ContactsContract +import android.provider.SyncStateContract +import android.util.Base64 +import org.json.JSONObject +import java.io.ByteArrayOutputStream +import java.io.FileNotFoundException + +/** + * Runs device sync's provider calls for one Android account of our type and + * one authority: `query`, `applyBatch`, `readSyncState` and `readPhoto` of + * `NativeModules.BulwarkDeviceSync`. [ScopePlanner] decides what may run; this + * class only builds the calls and reads the answers. + * + * Every URI carries `caller_is_syncadapter=true` and the account + * (CalendarProvider accepts no other query parameter), so writes neither set + * DIRTY nor schedule upload syncs, and ContactsProvider filters data rows by + * the account. The scope lookups are the exception: they read ids and owners + * of rows of any account. + * + * The client is unstable: a provider process that dies fails the call instead + * of taking this process (and the UI) with it. Call from a background thread; + * close when done. + */ +class ProviderIo(context: Context, accountName: String, private val authority: String) : ScopeLookup, AutoCloseable { + private val appContext: Context = context.applicationContext + private val account: AccountRef + private val client: ContentProviderClient + + init { + if (authority !in DeviceSyncAccounts.AUTHORITIES) { + throw ProviderRefusal(BatchFailure.SCOPE, "$authority is not a device sync authority") + } + account = AccountRef(accountName, DeviceSyncAccounts.accountType(appContext)) + client = appContext.contentResolver.acquireUnstableContentProviderClient(authority) + ?: throw IllegalStateException("No content provider for $authority") + } + + override fun close() { + client.close() + } + + // ── the bridge calls ──────────────────────────────────────── + + /** `ProviderQuery` JSON in, `ProviderRows` JSON out. Throws [ProviderRefusal] or what the provider threw. */ + fun query(queryJson: String): String { + val plan = ScopePlanner(account, this).planQuery(ProviderOpParser.parseQuery(authority, queryJson)) + val writer = ProviderRowsWriter(plan.columns) + val projection = plan.columns.toTypedArray() + for (selection in plan.selections) { + query(uri(plan.table), projection, selection, plan.orderBy).use { cursor -> + while (cursor.moveToNext()) { + writer.beginRow() + for (i in projection.indices) { + when (cursor.getType(i)) { + Cursor.FIELD_TYPE_INTEGER -> writer.longCell(cursor.getLong(i)) + Cursor.FIELD_TYPE_FLOAT -> writer.doubleCell(cursor.getDouble(i)) + Cursor.FIELD_TYPE_STRING -> writer.textCell(cursor.getString(i)) + // Blobs (photo thumbnails) never cross the bridge; readPhoto reads them. + else -> writer.nullCell() + } + } + writer.endRow() + } + } + } + return writer.finish() + } + + /** `ProviderOp[]` JSON in, `BatchResult` JSON out. Never throws: failures are results. */ + fun applyBatch(opsJson: String): String = try { + val ops = ProviderOpParser.parseBatch(authority, opsJson) + // Rows written for an account that is gone would outlive it: the + // providers purge an account's rows only when accounts change. + if (!DeviceSyncAccounts.exists(appContext, account.name)) { + throw ProviderRefusal(BatchFailure.SCOPE, "The account ${account.name} no longer exists") + } + if (ops.isEmpty()) { + ProviderResults.ok(emptyList()) + } else { + val planned = ScopePlanner(account, this).planBatch(ops) + val results = client.applyBatch(ArrayList(planned.map(::build))) + ProviderResults.ok(planned.zip(results) { op, result -> outcome(op, result.uri, result.count) }) + } + } catch (e: Throwable) { + ProviderResults.failed(failureOf(e), e.message ?: e.javaClass.simpleName) + } + + /** The account's SyncState blob as text, or null when there is none. */ + fun readSyncState(): String? = + SyncStateContract.Helpers.get(client, syncStateUri(), Account(account.name, account.type)) + ?.toString(Charsets.UTF_8) + + /** + * The raw contact's display photo, or its thumbnail when it has none, + * scaled so the longer side is at most [maxPx], as `PhotoData` JSON; null + * when the raw contact has no photo. + */ + fun readPhoto(rawContactId: Long, maxPx: Int): String? { + require(authority == DeviceSyncAccounts.CONTACTS_AUTHORITY) { "Photos are contacts data" } + require(maxPx > 0) { "maxPx must be positive" } + if (rootAccounts(ProviderTable.RAW_CONTACTS, setOf(rawContactId))[rawContactId] != account) { + throw ProviderRefusal(BatchFailure.SCOPE, "Raw contact $rawContactId is not part of the account") + } + var fileId: Long? = null + var thumbnail: ByteArray? = null + val photoRow = Selection( + "raw_contact_id = ? AND mimetype = ?", + listOf(rawContactId.toString(), ContactsContract.CommonDataKinds.Photo.CONTENT_ITEM_TYPE), + ) + query( + uri(ProviderTable.DATA), + arrayOf(ContactsContract.CommonDataKinds.Photo.PHOTO_FILE_ID, ContactsContract.CommonDataKinds.Photo.PHOTO), + photoRow, + "${ContactsContract.Data.IS_PRIMARY} DESC", + ).use { cursor -> + if (cursor.moveToFirst()) { + if (!cursor.isNull(0)) fileId = cursor.getLong(0) + if (cursor.getType(1) == Cursor.FIELD_TYPE_BLOB) thumbnail = cursor.getBlob(1) + } + } + val bytes = readDisplayPhoto(rawContactId) ?: thumbnail ?: return null + val jpeg = scaledJpeg(bytes, maxPx) ?: return null + return JSONObject() + .put("jpegBase64", Base64.encodeToString(jpeg, Base64.NO_WRAP)) + .put("fileId", fileId ?: JSONObject.NULL) + .toString() + } + + private fun readDisplayPhoto(rawContactId: Long): ByteArray? { + val uri = ContentUris.withAppendedId(ContactsContract.RawContacts.CONTENT_URI, rawContactId).buildUpon() + .appendPath(ContactsContract.RawContacts.DisplayPhoto.CONTENT_DIRECTORY) + .scoped() + .build() + return try { + client.openAssetFile(uri, "r")?.use { fd -> fd.createInputStream().use { it.readBytes() } } + } catch (e: FileNotFoundException) { + // No display photo stored (small photos only have the thumbnail). + null + } + } + + private fun scaledJpeg(bytes: ByteArray, maxPx: Int): ByteArray? { + val bounds = BitmapFactory.Options().apply { inJustDecodeBounds = true } + BitmapFactory.decodeByteArray(bytes, 0, bytes.size, bounds) + if (bounds.outWidth <= 0 || bounds.outHeight <= 0) return null + val options = BitmapFactory.Options().apply { + inSampleSize = PhotoScaling.sampleSize(bounds.outWidth, bounds.outHeight, maxPx) + } + val decoded = BitmapFactory.decodeByteArray(bytes, 0, bytes.size, options) ?: return null + val (width, height) = PhotoScaling.targetSize(decoded.width, decoded.height, maxPx) + val scaled = if (width == decoded.width && height == decoded.height) { + decoded + } else { + Bitmap.createScaledBitmap(decoded, width, height, true) + } + return ByteArrayOutputStream().use { out -> + scaled.compress(Bitmap.CompressFormat.JPEG, JPEG_QUALITY, out) + out.toByteArray() + } + } + + // ── ScopeLookup ───────────────────────────────────────────── + + override fun ownCalendarIds(): Set { + val ids = HashSet() + val selection = Selection( + "${ScopePlanner.ACCOUNT_NAME} = ? AND ${ScopePlanner.ACCOUNT_TYPE} = ?", + listOf(account.name, account.type), + ) + query(uri(ProviderTable.CALENDARS), arrayOf(ID), selection, null).use { c -> + while (c.moveToNext()) ids += c.getLong(0) + } + return ids + } + + override fun rootAccounts(table: ProviderTable, ids: Set): Map { + val owners = HashMap() + forEachRow(table, arrayOf(ID, ScopePlanner.ACCOUNT_NAME, ScopePlanner.ACCOUNT_TYPE), ID, ids) { c -> + owners[c.getLong(0)] = AccountRef(c.getString(1).orEmpty(), c.getString(2).orEmpty()) + } + return owners + } + + override fun dataRawContacts(ids: Set): Map = + idPairs(ProviderTable.DATA, "raw_contact_id", ids) + + override fun eventCalendars(ids: Set): Map = + idPairs(ProviderTable.EVENTS, "calendar_id", ids) + + override fun childEvents(table: ProviderTable, ids: Set): Map = + idPairs(table, "event_id", ids) + + override fun childEventIdsMatching(table: ProviderTable, where: String?, args: List): Set { + val events = HashSet() + query(uri(table, scoped = false), arrayOf("event_id"), Selection(where, args), null).use { c -> + while (c.moveToNext()) if (!c.isNull(0)) events += c.getLong(0) + } + return events + } + + /** Row id → [column] for the rows of [ids] that exist, in any account. */ + private fun idPairs(table: ProviderTable, column: String, ids: Set): Map { + val pairs = HashMap() + forEachRow(table, arrayOf(ID, column), table.idColumn!!, ids) { c -> + if (!c.isNull(1)) pairs[c.getLong(0)] = c.getLong(1) + } + return pairs + } + + private fun forEachRow( + table: ProviderTable, + projection: Array, + idColumn: String, + ids: Set, + read: (Cursor) -> Unit, + ) { + for (chunk in ids.chunked(ScopePlanner.ID_LIST_CHUNK)) { + val selection = Selection(ScopePlanner.idList(idColumn, chunk), emptyList()) + query(uri(table, scoped = false), projection, selection, null).use { c -> + while (c.moveToNext()) read(c) + } + } + } + + private fun query(uri: Uri, projection: Array, selection: Selection, orderBy: String?): Cursor = + client.query(uri, projection, selection.sql, selection.args.toTypedArray(), orderBy) + ?: throw IllegalStateException("$authority returned no cursor for $uri") + + // ── operations ────────────────────────────────────────────── + + private fun build(op: PlannedOp): ContentProviderOperation = when (op) { + is PlannedOp.Insert -> ContentProviderOperation.newInsert(uri(op.table)) + .withValues(contentValues(op.values)) + .apply { for ((column, index) in op.backReferences) withValueBackReference(column, index) } + .withYieldAllowed(op.yieldAllowed) + .build() + is PlannedOp.Update -> ContentProviderOperation + .newUpdate(op.itemId?.let { uri(op.table, itemId = it) } ?: uri(op.table)) + .withValues(contentValues(op.values)) + .selectRows(op.selection) + .expectRows(op.expectedCount) + .withYieldAllowed(op.yieldAllowed) + .build() + is PlannedOp.Delete -> ContentProviderOperation.newDelete(uri(op.table)) + .selectRows(op.selection) + .expectRows(op.expectedCount) + .withYieldAllowed(op.yieldAllowed) + .build() + is PlannedOp.Assert -> ContentProviderOperation.newAssertQuery(uri(op.table)) + .selectRows(op.selection) + .apply { + op.values?.let { values -> + withValues(ContentValues().apply { for ((column, text) in values) put(column, text) }) + } + } + .expectRows(op.expectedCount) + .withYieldAllowed(op.yieldAllowed) + .build() + // An insert that replaces the account's row; never a yield point. + is PlannedOp.SetSyncState -> SyncStateContract.Helpers.newSetOperation( + syncStateUri(), + Account(account.name, account.type), + op.value.toByteArray(Charsets.UTF_8), + ) + } + + private fun ContentProviderOperation.Builder.selectRows(selection: Selection) = apply { + // An item URI takes no selection at all (CalendarProvider refuses one). + if (selection.sql != null) withSelection(selection.sql, selection.args.toTypedArray()) + } + + private fun ContentProviderOperation.Builder.expectRows(count: Int?) = apply { + if (count != null) withExpectedCount(count) + } + + private fun outcome(op: PlannedOp, uri: Uri?, count: Int?): OpOutcome = when (op) { + // Settings inserts answer with the account's URI, which has no id. + is PlannedOp.Insert -> OpOutcome(id = uri?.lastPathSegment?.toLongOrNull()) + is PlannedOp.SetSyncState -> OpOutcome() + else -> OpOutcome(count = count) + } + + private fun contentValues(values: Map) = ContentValues(values.size).apply { + for ((column, cell) in values) { + when (cell) { + is Cell.Text -> put(column, cell.value) + is Cell.Integer -> put(column, cell.value) + is Cell.Real -> put(column, cell.value) + is Cell.Blob -> put(column, decodeBlob(column, cell.base64)) + Cell.Null -> putNull(column) + } + } + } + + private fun decodeBlob(column: String, base64: String): ByteArray = try { + Base64.decode(base64, Base64.DEFAULT) + } catch (e: IllegalArgumentException) { + throw ProviderRefusal(BatchFailure.PROVIDER, "$column is not base64") + } + + // ── URIs ──────────────────────────────────────────────────── + + private fun uri(table: ProviderTable, itemId: Long? = null, scoped: Boolean = true): Uri { + val base = when (table) { + ProviderTable.RAW_CONTACTS -> ContactsContract.RawContacts.CONTENT_URI + ProviderTable.DATA -> ContactsContract.Data.CONTENT_URI + ProviderTable.GROUPS -> ContactsContract.Groups.CONTENT_URI + ProviderTable.SETTINGS -> ContactsContract.Settings.CONTENT_URI + ProviderTable.CALENDARS -> CalendarContract.Calendars.CONTENT_URI + ProviderTable.EVENTS -> CalendarContract.Events.CONTENT_URI + ProviderTable.ATTENDEES -> CalendarContract.Attendees.CONTENT_URI + ProviderTable.REMINDERS -> CalendarContract.Reminders.CONTENT_URI + ProviderTable.EXTENDED_PROPERTIES -> CalendarContract.ExtendedProperties.CONTENT_URI + ProviderTable.COLORS -> CalendarContract.Colors.CONTENT_URI + } + val builder = (itemId?.let { ContentUris.withAppendedId(base, it) } ?: base).buildUpon() + return if (scoped) builder.scoped().build() else builder.syncAdapter().build() + } + + private fun syncStateUri(): Uri = when (authority) { + DeviceSyncAccounts.CONTACTS_AUTHORITY -> ContactsContract.SyncState.CONTENT_URI + else -> CalendarContract.SyncState.CONTENT_URI + }.buildUpon().scoped().build() + + private fun Uri.Builder.syncAdapter() = appendQueryParameter(CALLER_IS_SYNCADAPTER, "true") + + private fun Uri.Builder.scoped() = syncAdapter() + .appendQueryParameter(ScopePlanner.ACCOUNT_NAME, account.name) + .appendQueryParameter(ScopePlanner.ACCOUNT_TYPE, account.type) + + companion object { + private const val ID = "_id" + private const val JPEG_QUALITY = 85 + + /** Same parameter name in ContactsContract and CalendarContract. */ + private const val CALLER_IS_SYNCADAPTER = ContactsContract.CALLER_IS_SYNCADAPTER + + /** The `BatchFailure` of an exception from planning or from the provider (possibly wrapped). */ + fun failureOf(e: Throwable): BatchFailure { + var t: Throwable? = e + while (t != null) { + when (t) { + is ProviderRefusal -> return t.reason + is TransactionTooLargeException -> return BatchFailure.TOO_LARGE + is SecurityException -> return BatchFailure.PERMISSION + is OperationApplicationException -> return ProviderResults.failureOfOperationApplication(t.message) + } + t = t.cause?.takeIf { it !== t } + } + return BatchFailure.PROVIDER + } + } +} diff --git a/android/app/src/main/java/com/anonymous/bulwarkmobile/sync/ProviderOps.kt b/android/app/src/main/java/com/anonymous/bulwarkmobile/sync/ProviderOps.kt new file mode 100644 index 00000000..167bbd4d --- /dev/null +++ b/android/app/src/main/java/com/anonymous/bulwarkmobile/sync/ProviderOps.kt @@ -0,0 +1,410 @@ +package com.anonymous.bulwarkmobile.sync + +import org.json.JSONArray +import org.json.JSONException +import org.json.JSONObject +import java.math.BigDecimal +import java.math.BigInteger +import kotlin.math.abs + +/** + * The provider operations of device sync (#34) as the engine sends them: the + * `ProviderQuery` and `ProviderOp[]` JSON of src/device-sync/types.ts, parsed + * into a validated model. [ScopePlanner] turns the model into provider calls + * scoped to one account and [ProviderIo] runs them. Nothing here touches + * Android types, so JVM tests cover all of it. + */ + +/** `BatchFailure` of types.ts: why a batch was refused or failed. */ +enum class BatchFailure(val wire: String) { + ASSERT("assert"), + TOO_LARGE("tooLarge"), + SCOPE("scope"), + PERMISSION("permission"), + PROVIDER("provider"), +} + +/** A query or batch refused before it reached the provider. */ +class ProviderRefusal(val reason: BatchFailure, message: String) : Exception(message) + +/** How the rows of a table are tied to an account; see [ScopePlanner]. */ +enum class TableScope { ACCOUNT_COLUMNS, URI_PARAMETERS, CALENDAR_IDS, EVENT_IDS } + +/** The tables the engine may touch, per authority. Any other table is refused. */ +enum class ProviderTable( + val wire: String, + val authority: String, + val scope: TableScope, + /** The table whose row each row here belongs to, named by [parentColumn]. */ + val parent: ProviderTable? = null, + val parentColumn: String? = null, + /** + * How a selection names the row id; null when rows have none (Settings + * rows are keyed by their account). CalendarProvider queries attendees + * joined with events and calendars, where a bare `_id` is ambiguous. + */ + val idColumn: String? = "_id", +) { + RAW_CONTACTS("raw_contacts", DeviceSyncAccounts.CONTACTS_AUTHORITY, TableScope.ACCOUNT_COLUMNS), + DATA("data", DeviceSyncAccounts.CONTACTS_AUTHORITY, TableScope.URI_PARAMETERS, RAW_CONTACTS, "raw_contact_id"), + GROUPS("groups", DeviceSyncAccounts.CONTACTS_AUTHORITY, TableScope.ACCOUNT_COLUMNS), + SETTINGS("settings", DeviceSyncAccounts.CONTACTS_AUTHORITY, TableScope.ACCOUNT_COLUMNS, idColumn = null), + CALENDARS("calendars", DeviceSyncAccounts.CALENDAR_AUTHORITY, TableScope.ACCOUNT_COLUMNS), + EVENTS("events", DeviceSyncAccounts.CALENDAR_AUTHORITY, TableScope.CALENDAR_IDS, CALENDARS, "calendar_id"), + ATTENDEES( + "attendees", DeviceSyncAccounts.CALENDAR_AUTHORITY, TableScope.EVENT_IDS, EVENTS, "event_id", "Attendees._id", + ), + REMINDERS( + "reminders", DeviceSyncAccounts.CALENDAR_AUTHORITY, TableScope.EVENT_IDS, EVENTS, "event_id", "Reminders._id", + ), + EXTENDED_PROPERTIES( + "extended_properties", + DeviceSyncAccounts.CALENDAR_AUTHORITY, + TableScope.EVENT_IDS, + EVENTS, + "event_id", + "ExtendedProperties._id", + ), + COLORS("colors", DeviceSyncAccounts.CALENDAR_AUTHORITY, TableScope.ACCOUNT_COLUMNS), + ; + + /** An insert returns the new row id, which later ops can back-reference. */ + val insertReturnsId: Boolean get() = idColumn != null + + companion object { + fun of(authority: String, wire: String): ProviderTable = + entries.firstOrNull { it.wire == wire && it.authority == authority } + ?: throw ProviderRefusal(BatchFailure.SCOPE, "Table $wire is not part of $authority") + } +} + +/** + * A cell as it crosses the bridge: text, a number, null, or (written to + * `data.data15` only) photo bytes as base64. Numbers that are integers become + * [Integer], so an assert compares `5` with the provider's `5`, not `5.0`. + */ +sealed interface Cell { + data class Text(val value: String) : Cell + data class Integer(val value: Long) : Cell + data class Real(val value: Double) : Cell + data class Blob(val base64: String) : Cell + data object Null : Cell + + /** The text ContentProviderOperation compares an assert with; blobs never reach an assert. */ + fun text(): String? = when (this) { + is Text -> value + is Integer -> value.toString() + is Real -> value.toString() + is Blob -> throw IllegalStateException("Blobs have no text form") + Null -> null + } + + /** The row id this cell names, if it names one (`12` or `"12"`). */ + fun rowId(): Long? = when (this) { + is Integer -> value + is Text -> value.toLongOrNull() + else -> null + } +} + +/** Which rows an update, delete or assert addresses: one row by id, a `where` set, or the id row if it matches. */ +data class RowTarget(val id: Long?, val where: String?, val args: List) + +data class ProviderQuery( + val table: ProviderTable, + val columns: List, + val where: String?, + val args: List, + val orderBy: String?, +) + +/** One `ProviderOp` of types.ts, validated. */ +sealed interface ProviderOp { + val yieldAllowed: Boolean + + data class Insert( + val table: ProviderTable, + val values: Map, + /** Column → index of an earlier insert of the batch whose new row id fills it. */ + val refs: Map, + override val yieldAllowed: Boolean, + ) : ProviderOp + + data class Update( + val table: ProviderTable, + val target: RowTarget, + val values: Map, + val expectCount: Int?, + override val yieldAllowed: Boolean, + ) : ProviderOp + + data class Delete( + val table: ProviderTable, + val target: RowTarget, + val expectCount: Int?, + override val yieldAllowed: Boolean, + ) : ProviderOp + + data class Assert( + val table: ProviderTable, + val target: RowTarget, + val values: Map?, + val expectCount: Int?, + override val yieldAllowed: Boolean, + ) : ProviderOp + + data class SetSyncState(val value: String, override val yieldAllowed: Boolean) : ProviderOp +} + +const val GROUP_MEMBERSHIP_MIMETYPE = "vnd.android.cursor.item/group_membership" + +/** + * The table a written column points into when the row it names must belong to + * the account as well: the parent of a row, the master of an exception event, + * the group of a membership row. + */ +internal fun referencedTable(table: ProviderTable, column: String, values: Map): ProviderTable? = when { + column == table.parentColumn -> table.parent + table == ProviderTable.EVENTS && column == "original_id" -> ProviderTable.EVENTS + table == ProviderTable.DATA && column == "data1" && + values["mimetype"] == Cell.Text(GROUP_MEMBERSHIP_MIMETYPE) -> ProviderTable.GROUPS + else -> null +} + +object ProviderOpParser { + /** + * ContactsProvider throws when its op counter reaches this. It counts an + * op before resetting at a yield point, so at most 499 ops may follow a + * yield point up to and including the next one. + */ + const val CONTACTS_OPS_PER_YIELD_POINT = 500 + + /** JS numbers beyond this are not exact integers any more. */ + private const val MAX_SAFE_INTEGER = 9_007_199_254_740_991.0 + + fun parseQuery(authority: String, json: String): ProviderQuery { + val o = parseObject(json, "query") + val table = ProviderTable.of(authority, o.requireString("table")) + val rawColumns = o.opt("columns") as? JSONArray ?: throw invalid("query needs a columns array") + val columns = (0 until rawColumns.length()).map { i -> + rawColumns.opt(i) as? String ?: throw invalid("columns[$i] must be a string") + } + if (columns.isEmpty()) throw invalid("query needs at least one column") + val (where, args) = o.whereArgs("query") + return ProviderQuery(table, columns, where, args, o.optionalString("orderBy")) + } + + fun parseBatch(authority: String, json: String): List { + val array = try { + JSONArray(json) + } catch (e: JSONException) { + throw invalid("ops are not a JSON array: ${e.message}") + } + val ops = (0 until array.length()).map { i -> + val o = array.opt(i) as? JSONObject ?: throw invalid("op $i is not an object") + parseOp(authority, i, o) + } + checkSyncState(ops) + checkBackReferences(ops) + if (authority == DeviceSyncAccounts.CONTACTS_AUTHORITY) checkYieldPoints(ops) + return ops + } + + private fun parseOp(authority: String, index: Int, o: JSONObject): ProviderOp { + val where = "op $index" + val kind = o.opt("op") as? String ?: throw invalid("$where has no op") + val yieldAllowed = o.optBoolean("yieldAllowed", false) + if (kind == "syncState") { + val value = o.opt("value") as? String ?: throw invalid("$where: syncState needs a string value") + return ProviderOp.SetSyncState(value, yieldAllowed) + } + val table = ProviderTable.of(authority, o.requireString("table")) + return when (kind) { + "insert" -> ProviderOp.Insert(table, o.writeValues(table, where), o.refs(where), yieldAllowed) + "update" -> { + val values = o.writeValues(table, where) + if (values.isEmpty()) throw invalid("$where: an update needs values") + ProviderOp.Update(table, o.target(table, where), values, o.expectCount(where), yieldAllowed) + } + "delete" -> ProviderOp.Delete(table, o.target(table, where), o.expectCount(where), yieldAllowed) + "assert" -> { + val target = o.target(table, where) + val values = o.assertValues(where) + val expectCount = o.expectCount(where) + if (values.isNullOrEmpty() && expectCount == null && target.id == null) { + throw invalid("$where: an assert needs values, an expected count or an id") + } + ProviderOp.Assert(table, target, values?.takeIf { it.isNotEmpty() }, expectCount, yieldAllowed) + } + else -> throw invalid("$where: unknown op $kind") + } + } + + /** + * The state describes the rows of its batch, so it must commit with them: + * as the last op, and not at a yield point, where the provider may commit + * everything before it on its own. + */ + private fun checkSyncState(ops: List) { + ops.forEachIndexed { i, op -> + if (op !is ProviderOp.SetSyncState) return@forEachIndexed + if (i != ops.lastIndex) { + throw ProviderRefusal(BatchFailure.SCOPE, "op $i: syncState must be the last op of its batch") + } + if (op.yieldAllowed) throw ProviderRefusal(BatchFailure.SCOPE, "op $i: syncState must not be a yield point") + } + } + + private fun checkBackReferences(ops: List) { + ops.forEachIndexed { i, op -> + if (op !is ProviderOp.Insert) return@forEachIndexed + for ((column, ref) in op.refs) { + val target = ops.getOrNull(ref) + if (ref >= i || target !is ProviderOp.Insert || !target.table.insertReturnsId) { + throw invalid("op $i: $column refers to op $ref, which is not an earlier insert with a row id") + } + val expected = referencedTable(op.table, column, op.values) + if (expected != null && target.table != expected) { + throw ProviderRefusal( + BatchFailure.SCOPE, + "op $i: $column must refer to a ${expected.wire} insert, not ${target.table.wire}", + ) + } + } + val parentColumn = op.table.parentColumn ?: return@forEachIndexed + val named = parentColumn in op.refs || (op.values[parentColumn] ?: Cell.Null) != Cell.Null + if (!named) { + throw ProviderRefusal(BatchFailure.SCOPE, "op $i: a ${op.table.wire} insert must name its $parentColumn") + } + } + } + + private fun checkYieldPoints(ops: List) { + var count = 0 + ops.forEachIndexed { i, op -> + if (++count >= CONTACTS_OPS_PER_YIELD_POINT) { + throw invalid( + "op $i: more than ${CONTACTS_OPS_PER_YIELD_POINT - 1} ops without a yield point " + + "(ContactsProvider refuses the batch)", + ) + } + if (i > 0 && op.yieldAllowed) count = 0 + } + } + + // ── JSON helpers ──────────────────────────────────────────── + + private fun invalid(message: String) = ProviderRefusal(BatchFailure.PROVIDER, message) + + private fun parseObject(json: String, what: String): JSONObject = try { + JSONObject(json) + } catch (e: JSONException) { + throw invalid("$what is not a JSON object: ${e.message}") + } + + private fun JSONObject.requireString(key: String): String = + opt(key) as? String ?: throw invalid("$key must be a string") + + private fun JSONObject.optionalString(key: String): String? = when (val v = opt(key)) { + null, JSONObject.NULL -> null + is String -> v.takeIf { it.isNotBlank() } + else -> throw invalid("$key must be a string") + } + + private fun JSONObject.target(table: ProviderTable, where: String): RowTarget { + val id = when (val raw = opt("id")) { + null, JSONObject.NULL -> null + else -> (scalar(raw, "$where: id") as? Cell.Integer)?.value?.takeIf { it >= 0 } + ?: throw invalid("$where: id must be a row id") + } + if (id != null && table.idColumn == null) throw invalid("$where: ${table.wire} rows have no id") + val (sql, args) = whereArgs(where) + return RowTarget(id, sql, args) + } + + private fun JSONObject.whereArgs(where: String): Pair> { + val sql = optionalString("where") + val args = when (val raw = opt("args")) { + null, JSONObject.NULL -> emptyList() + is JSONArray -> (0 until raw.length()).map { i -> + scalar(raw.opt(i), "$where: args[$i]").text() + ?: throw invalid("$where: args[$i] must be a string or a number") + } + else -> throw invalid("$where: args must be an array") + } + if (sql == null && args.isNotEmpty()) throw invalid("$where: args without a where") + return sql to args + } + + private fun JSONObject.expectCount(where: String): Int? = when (val raw = opt("expectCount")) { + null, JSONObject.NULL -> null + else -> (scalar(raw, "$where: expectCount") as? Cell.Integer)?.value + ?.takeIf { it in 0..Int.MAX_VALUE }?.toInt() + ?: throw invalid("$where: expectCount must be a count") + } + + private fun JSONObject.refs(where: String): Map { + val raw = opt("refs") ?: return emptyMap() + if (raw == JSONObject.NULL) return emptyMap() + if (raw !is JSONObject) throw invalid("$where: refs must be an object") + val out = LinkedHashMap() + for (column in raw.keys()) { + val index = (scalar(raw.opt(column), "$where: refs.$column") as? Cell.Integer)?.value + ?.takeIf { it in 0..Int.MAX_VALUE } + ?: throw invalid("$where: refs.$column must be an op index") + out[column] = index.toInt() + } + return out + } + + private fun JSONObject.writeValues(table: ProviderTable, where: String): Map { + val raw = opt("values") as? JSONObject ?: throw invalid("$where: values must be an object") + val out = LinkedHashMap() + for (column in raw.keys()) { + val v = raw.opt(column) + out[column] = if (v is JSONObject) blob(table, column, v, where) else scalar(v, "$where: $column") + } + return out + } + + private fun JSONObject.assertValues(where: String): Map? { + val raw = opt("values") ?: return null + if (raw == JSONObject.NULL) return null + if (raw !is JSONObject) throw invalid("$where: values must be an object") + val out = LinkedHashMap() + for (column in raw.keys()) out[column] = scalar(raw.opt(column), "$where: $column") + return out + } + + /** Only photo rows carry bytes (`data.data15`); the provider stores and scales them itself. */ + private fun blob(table: ProviderTable, column: String, v: JSONObject, where: String): Cell.Blob { + if (table != ProviderTable.DATA || column != "data15") { + throw ProviderRefusal(BatchFailure.SCOPE, "$where: bytes are only accepted for data.data15, not ${table.wire}.$column") + } + val base64 = v.opt("b64") as? String ?: throw invalid("$where: $column needs { b64 }") + return Cell.Blob(base64) + } + + private fun scalar(v: Any?, what: String): Cell = when (v) { + null, JSONObject.NULL -> Cell.Null + is String -> Cell.Text(v) + is Boolean -> Cell.Integer(if (v) 1 else 0) + is Number -> numberCell(v) + else -> throw invalid("$what must be text, a number or null") + } + + /** org.json yields Integer, Long, Double, BigInteger or BigDecimal depending on the platform and the value. */ + internal fun numberCell(n: Number): Cell = when (n) { + is Int, is Long, is Short, is Byte -> Cell.Integer(n.toLong()) + is BigInteger -> if (n.bitLength() < 64) Cell.Integer(n.toLong()) else Cell.Real(n.toDouble()) + is BigDecimal -> try { + Cell.Integer(n.longValueExact()) + } catch (e: ArithmeticException) { + Cell.Real(n.toDouble()) + } + else -> { + val d = n.toDouble() + if (d.isFinite() && d == Math.rint(d) && abs(d) <= MAX_SAFE_INTEGER) Cell.Integer(d.toLong()) else Cell.Real(d) + } + } +} diff --git a/android/app/src/main/java/com/anonymous/bulwarkmobile/sync/ProviderScope.kt b/android/app/src/main/java/com/anonymous/bulwarkmobile/sync/ProviderScope.kt new file mode 100644 index 00000000..01729241 --- /dev/null +++ b/android/app/src/main/java/com/anonymous/bulwarkmobile/sync/ProviderScope.kt @@ -0,0 +1,489 @@ +package com.anonymous.bulwarkmobile.sync + +import org.json.JSONArray +import org.json.JSONObject + +/** + * Plans provider calls for one account: every selection gets the account + * condition ANDed in, and every row an operation names by id, or points to + * from a written column, is checked to belong to the account first + * (docs/device-sync.md, "Native module API", Scoping). + * + * How a table is tied to the account ([TableScope]): + * - raw contacts, groups, settings, calendars and colors carry it in their + * `account_name`/`account_type` columns; + * - contacts data rows are filtered by ContactsProvider itself through the + * `account_name`/`account_type` URI parameters. Their account columns are + * never selected on: Android 17 drops them from the Data view for apps + * targeting more than API 36; + * - events get `calendar_id IN (…)` of the account's calendars, not the + * account columns: CalendarProvider updates events on the Events table, + * which has none; + * - attendees, reminders and extended properties get `event_id IN (…)` of the + * account's events among those the operation touches. CalendarProvider + * applies no account filter to them. + * + * Selections never use subqueries: the providers may compile them strictly. + */ +class ScopePlanner(private val account: AccountRef, private val lookup: ScopeLookup) { + private val calendarIds: Set by lazy { lookup.ownCalendarIds() } + + fun planQuery(query: ProviderQuery): PlannedQuery { + val target = RowTarget(null, query.where, query.args) + val selections = if (query.table.scope == TableScope.EVENT_IDS) { + // Mostly a handful of events (the engine reads children per chunk + // of events); a query over all of them is split so no selection + // outgrows a Binder transaction. Rows are then ordered per chunk. + ownEventsMatching(query.table, target) + .sorted() + .chunked(ID_LIST_CHUNK) + .map { chunk -> selection(query.table, target, chunk) } + } else { + listOf(selection(query.table, target, null)) + } + return PlannedQuery(query.table, query.columns, selections, query.orderBy) + } + + fun planBatch(ops: List): List { + val owners = Owners(ownershipRequests(ops)) + return ops.mapIndexed { i, op -> plan(i, op, owners) } + } + + private fun plan(i: Int, op: ProviderOp, owners: Owners): PlannedOp = when (op) { + is ProviderOp.SetSyncState -> PlannedOp.SetSyncState(op.value) + is ProviderOp.Insert -> { + checkAccountColumns(i, op.table, op.values) + checkReferences(i, op.table, op.values - op.refs.keys, owners) + val values = if (op.table.scope == TableScope.ACCOUNT_COLUMNS) op.values + accountCells() else op.values + PlannedOp.Insert(op.table, values, op.refs, op.yieldAllowed) + } + is ProviderOp.Update -> { + checkAccountColumns(i, op.table, op.values) + checkReferences(i, op.table, op.values, owners) + checkTarget(i, op.table, op.target, owners) + if (op.table == ProviderTable.EXTENDED_PROPERTIES) { + // CalendarProvider updates extended properties only through + // their item URI, which takes no selection: the id check above + // is the scope. + val id = op.target.id + if (id == null || op.target.where != null) { + throw ProviderRefusal(BatchFailure.PROVIDER, "op $i: extended properties can only be updated by id") + } + PlannedOp.Update(op.table, Selection.NONE, id, op.values, op.expectCount ?: 1, op.yieldAllowed) + } else { + val selection = selection(op.table, op.target, eventScope(op.table, op.target, owners)) + PlannedOp.Update(op.table, selection, null, op.values, expectedCount(op.target, op.expectCount), op.yieldAllowed) + } + } + is ProviderOp.Delete -> { + checkTarget(i, op.table, op.target, owners) + val selection = selection(op.table, op.target, eventScope(op.table, op.target, owners)) + PlannedOp.Delete(op.table, selection, expectedCount(op.target, op.expectCount), op.yieldAllowed) + } + is ProviderOp.Assert -> { + checkTarget(i, op.table, op.target, owners) + val selection = selection(op.table, op.target, eventScope(op.table, op.target, owners)) + val values = op.values?.mapValues { (_, cell) -> cell.text() } + PlannedOp.Assert(op.table, selection, values, expectedCount(op.target, op.expectCount), op.yieldAllowed) + } + } + + /** An op that names one row by id expects exactly that row: the engine relies on it to notice a vanished row. */ + private fun expectedCount(target: RowTarget, expectCount: Int?): Int? = + expectCount ?: if (target.id != null) 1 else null + + private fun accountCells() = mapOf( + ACCOUNT_NAME to Cell.Text(account.name), + ACCOUNT_TYPE to Cell.Text(account.type), + ) + + /** + * Account columns may only restate the account, on the tables that have + * them; a data row, event or attendee follows its parent's account. + */ + private fun checkAccountColumns(i: Int, table: ProviderTable, values: Map) { + for (column in listOf(ACCOUNT_NAME, ACCOUNT_TYPE, DATA_SET)) { + val cell = values[column] ?: continue + val restatesAccount = table.scope == TableScope.ACCOUNT_COLUMNS && when (column) { + ACCOUNT_NAME -> cell == Cell.Text(account.name) + ACCOUNT_TYPE -> cell == Cell.Text(account.type) + else -> cell == Cell.Null + } + if (!restatesAccount) { + throw ProviderRefusal(BatchFailure.SCOPE, "op $i: ${table.wire}.$column would leave the account") + } + } + } + + /** + * Parents, masters and groups a written column names must be rows of the + * account that still exist. (Back-references were checked by the parser: + * they name inserts of the same batch.) + */ + private fun checkReferences(i: Int, table: ProviderTable, values: Map, owners: Owners) { + for ((column, cell) in values) { + val referenced = referencedTable(table, column, values) ?: continue + if (cell == Cell.Null) { + if (column == table.parentColumn) { + throw ProviderRefusal(BatchFailure.SCOPE, "op $i: ${table.wire}.$column must name a ${referenced.wire} row") + } + continue + } + val id = cell.rowId() + ?: throw ProviderRefusal(BatchFailure.PROVIDER, "op $i: ${table.wire}.$column must be a row id") + when (owners.of(referenced, id)) { + Owner.OURS -> Unit + Owner.FOREIGN -> throw ProviderRefusal( + BatchFailure.SCOPE, + "op $i: ${table.wire}.$column names ${referenced.wire} $id of another account", + ) + // Deleted meanwhile, as far as the engine can tell: re-read and re-plan. + Owner.MISSING -> throw ProviderRefusal( + BatchFailure.ASSERT, + "op $i: ${table.wire}.$column names ${referenced.wire} $id, which no longer exists", + ) + } + } + } + + /** A row named by id must not belong to another account; a missing one fails on the expected count instead. */ + private fun checkTarget(i: Int, table: ProviderTable, target: RowTarget, owners: Owners) { + val id = target.id ?: return + if (owners.of(table, id) == Owner.FOREIGN) { + throw ProviderRefusal(BatchFailure.SCOPE, "op $i: ${table.wire} $id belongs to another account") + } + } + + /** For attendees, reminders and extended properties: the account's events the op may touch. */ + private fun eventScope(table: ProviderTable, target: RowTarget, owners: Owners): Collection? { + if (table.scope != TableScope.EVENT_IDS) return null + val id = target.id ?: return ownEventsMatching(table, target) + return listOfNotNull(owners.eventOf(table, id)) + } + + private fun ownEventsMatching(table: ProviderTable, target: RowTarget): Set { + val candidates = lookup.childEventIdsMatching(table, target.where, target.args) + if (candidates.isEmpty()) return emptySet() + return lookup.eventCalendars(candidates).filterValues { it in calendarIds }.keys + } + + private fun selection(table: ProviderTable, target: RowTarget, eventIds: Collection?): Selection { + val parts = ArrayList(3) + val args = ArrayList() + target.id?.let { + parts += "${table.idColumn} = ?" + args += it.toString() + } + target.where?.let { + parts += it + args += target.args + } + when (table.scope) { + TableScope.ACCOUNT_COLUMNS -> { + parts += "$ACCOUNT_NAME = ? AND $ACCOUNT_TYPE = ?" + args += account.name + args += account.type + } + TableScope.URI_PARAMETERS -> Unit + TableScope.CALENDAR_IDS -> parts += idList("calendar_id", calendarIds) + TableScope.EVENT_IDS -> parts += idList("event_id", eventIds.orEmpty()) + } + return Selection(parts.takeIf { it.isNotEmpty() }?.joinToString(" AND ") { "($it)" }, args) + } + + /** Every row id the batch names, by table: op targets and referenced rows. */ + private fun ownershipRequests(ops: List): Map> { + val requests = HashMap>() + fun request(table: ProviderTable, id: Long) { + requests.getOrPut(table) { HashSet() } += id + } + fun requestReferences(table: ProviderTable, values: Map) { + for ((column, cell) in values) { + val referenced = referencedTable(table, column, values) ?: continue + cell.rowId()?.let { request(referenced, it) } + } + } + for (op in ops) { + when (op) { + is ProviderOp.Insert -> requestReferences(op.table, op.values) + is ProviderOp.Update -> { + requestReferences(op.table, op.values) + op.target.id?.let { request(op.table, it) } + } + is ProviderOp.Delete -> op.target.id?.let { request(op.table, it) } + is ProviderOp.Assert -> op.target.id?.let { request(op.table, it) } + is ProviderOp.SetSyncState -> Unit + } + } + return requests + } + + private enum class Owner { OURS, FOREIGN, MISSING } + + /** Who owns the rows a batch names, resolved up front with as few lookups as the tables allow. */ + private inner class Owners(requests: Map>) { + private val owners = HashMap>() + private val childEvents = HashMap>() + + init { + // Children lead to events, data rows to raw contacts; both are + // added to their parent's lookup. + val eventIds = HashSet(requests[ProviderTable.EVENTS].orEmpty()) + for (table in CHILD_TABLES) { + val ids = requests[table] ?: continue + val events = lookup.childEvents(table, ids) + childEvents[table] = events + eventIds += events.values + } + val rawContactIds = HashSet(requests[ProviderTable.RAW_CONTACTS].orEmpty()) + val dataParents = requests[ProviderTable.DATA]?.let { lookup.dataRawContacts(it) }.orEmpty() + rawContactIds += dataParents.values + + if (eventIds.isNotEmpty()) { + owners[ProviderTable.EVENTS] = lookup.eventCalendars(eventIds) + .mapValues { (_, calendarId) -> if (calendarId in calendarIds) Owner.OURS else Owner.FOREIGN } + } + for ((table, events) in childEvents) { + // A child whose event is gone is an orphan, not ours. + owners[table] = events.mapValues { (_, eventId) -> ownerOrForeign(ProviderTable.EVENTS, eventId) } + } + val roots = HashMap>() + for (table in ROOT_TABLES) requests[table]?.let { roots[table] = it } + if (rawContactIds.isNotEmpty()) roots[ProviderTable.RAW_CONTACTS] = rawContactIds + for ((table, ids) in roots) { + owners[table] = lookup.rootAccounts(table, ids) + .mapValues { (_, owner) -> if (owner == account) Owner.OURS else Owner.FOREIGN } + } + if (dataParents.isNotEmpty()) { + owners[ProviderTable.DATA] = dataParents.mapValues { (_, rawContactId) -> + ownerOrForeign(ProviderTable.RAW_CONTACTS, rawContactId) + } + } + } + + private fun ownerOrForeign(table: ProviderTable, id: Long): Owner = + owners[table]?.get(id)?.takeIf { it != Owner.MISSING } ?: Owner.FOREIGN + + fun of(table: ProviderTable, id: Long): Owner = owners[table]?.get(id) ?: Owner.MISSING + + /** The event of a child row, when both exist and are the account's. */ + fun eventOf(table: ProviderTable, id: Long): Long? = + childEvents[table]?.get(id)?.takeIf { of(table, id) == Owner.OURS } + } + + companion object { + const val ACCOUNT_NAME = "account_name" + const val ACCOUNT_TYPE = "account_type" + const val DATA_SET = "data_set" + + /** Ids per `IN (…)` list: a few KB of selection, far from SQLite's and Binder's limits. */ + const val ID_LIST_CHUNK = 500 + + private val CHILD_TABLES = ProviderTable.entries.filter { it.scope == TableScope.EVENT_IDS } + private val ROOT_TABLES = ProviderTable.entries.filter { it.scope == TableScope.ACCOUNT_COLUMNS } + + /** `column IN (1,2,3)`, or a condition no row meets. The ids are ours to inline: numbers we read. */ + fun idList(column: String, ids: Collection): String = + if (ids.isEmpty()) "0" else ids.sorted().joinToString(",", "$column IN (", ")") + } +} + +/** An Android account of our type: the scope of every provider call. */ +data class AccountRef(val name: String, val type: String) + +/** + * What [ScopePlanner] needs to know about rows beyond the operations + * themselves. [ProviderIo] answers with provider queries: except for + * [ownCalendarIds], these read rows of any account, but only their ids and + * owners, to tell rows of another account from rows that are gone. + */ +interface ScopeLookup { + /** `_id`s of the account's calendars. */ + fun ownCalendarIds(): Set + + /** The account of each row of a root table that exists. */ + fun rootAccounts(table: ProviderTable, ids: Set): Map + + /** The raw contact of each data row that exists. */ + fun dataRawContacts(ids: Set): Map + + /** The calendar of each event that exists. */ + fun eventCalendars(ids: Set): Map + + /** The event of each attendee, reminder or extended property that exists. */ + fun childEvents(table: ProviderTable, ids: Set): Map + + /** The events of the rows of a child table that match [where]. */ + fun childEventIdsMatching(table: ProviderTable, where: String?, args: List): Set +} + +/** A selection with its `?` arguments; a null [sql] selects every row the URI allows. */ +data class Selection(val sql: String?, val args: List) { + companion object { + val NONE = Selection(null, emptyList()) + } +} + +/** A provider query, split into [selections] when its scope list is long; no selection means no rows. */ +data class PlannedQuery( + val table: ProviderTable, + val columns: List, + val selections: List, + val orderBy: String?, +) + +/** One ContentProviderOperation, ready to build on the account's sync-adapter URI. */ +sealed interface PlannedOp { + val yieldAllowed: Boolean + + data class Insert( + val table: ProviderTable, + val values: Map, + val backReferences: Map, + override val yieldAllowed: Boolean, + ) : PlannedOp + + /** [itemId] set: an update through the row's own URI, without a selection. */ + data class Update( + val table: ProviderTable, + val selection: Selection, + val itemId: Long?, + val values: Map, + val expectedCount: Int?, + override val yieldAllowed: Boolean, + ) : PlannedOp + + data class Delete( + val table: ProviderTable, + val selection: Selection, + val expectedCount: Int?, + override val yieldAllowed: Boolean, + ) : PlannedOp + + /** [values] compared as text, as ContentProviderOperation does. */ + data class Assert( + val table: ProviderTable, + val selection: Selection, + val values: Map?, + val expectedCount: Int?, + override val yieldAllowed: Boolean, + ) : PlannedOp + + data class SetSyncState(val value: String) : PlannedOp { + override val yieldAllowed: Boolean get() = false + } +} + +/** The result of one op, as `OpResult` of types.ts. */ +data class OpOutcome(val id: Long? = null, val count: Int? = null) + +object ProviderResults { + fun ok(results: List): String { + val array = JSONArray() + for (result in results) { + array.put(JSONObject().apply { + result.id?.let { put("id", it) } + result.count?.let { put("count", it) } + }) + } + return JSONObject().put("ok", true).put("results", array).toString() + } + + fun failed(reason: BatchFailure, message: String?): String = JSONObject() + .put("ok", false) + .put("reason", reason.wire) + .put("message", message ?: reason.wire) + .toString() + + /** + * An OperationApplicationException is an assert that did not hold or an + * expected count that did not match (`assert`), except when an insert got + * no row back or ContactsProvider refused the batch's yield points. + */ + fun failureOfOperationApplication(message: String?): BatchFailure = when { + message == null -> BatchFailure.ASSERT + message.startsWith("Insert into ") -> BatchFailure.PROVIDER + message.startsWith("Too many content provider operations") -> BatchFailure.PROVIDER + else -> BatchFailure.ASSERT + } +} + +/** + * Query results as `ProviderRows` JSON, built by hand: a chunk of contacts + * can carry megabytes of shadows, and this is the hot path of every run. + */ +class ProviderRowsWriter(columns: List) { + private val out = StringBuilder(4096) + private var rows = 0 + private var cells = 0 + + init { + out.append("{\"columns\":[") + columns.forEachIndexed { i, column -> + if (i > 0) out.append(',') + appendString(column) + } + out.append("],\"rows\":[") + } + + fun beginRow() { + if (rows++ > 0) out.append(',') + out.append('[') + cells = 0 + } + + fun endRow() { + out.append(']') + } + + fun nullCell() = cell { out.append("null") } + + fun longCell(value: Long) = cell { out.append(value) } + + fun doubleCell(value: Double) = cell { if (value.isFinite()) out.append(value) else out.append("null") } + + fun textCell(value: String) = cell { appendString(value) } + + fun finish(): String = out.append("]}").toString() + + private inline fun cell(write: () -> Unit) { + if (cells++ > 0) out.append(',') + write() + } + + private fun appendString(s: String) { + out.append('"') + for (c in s) { + when { + c == '"' -> out.append("\\\"") + c == '\\' -> out.append("\\\\") + c == '\n' -> out.append("\\n") + c == '\r' -> out.append("\\r") + c == '\t' -> out.append("\\t") + c < ' ' || c.code == 0x2028 || c.code == 0x2029 -> + out.append("\\u").append(Integer.toHexString(c.code).padStart(4, '0')) + else -> out.append(c) + } + } + out.append('"') + } +} + +/** Sizes for [ProviderIo.readPhoto]: decode close to the target, then scale the longer side down to it. */ +object PhotoScaling { + /** The largest power of two that keeps the decoded image at least [maxPx] on its longer side. */ + fun sampleSize(width: Int, height: Int, maxPx: Int): Int { + require(maxPx > 0) { "maxPx must be positive" } + var sample = 1 + val longer = maxOf(width, height) + while (longer / (sample * 2) >= maxPx) sample *= 2 + return sample + } + + fun targetSize(width: Int, height: Int, maxPx: Int): Pair { + val longer = maxOf(width, height) + if (longer <= maxPx) return width to height + val scale = maxPx.toDouble() / longer + return maxOf(1, Math.round(width * scale).toInt()) to maxOf(1, Math.round(height * scale).toInt()) + } +} diff --git a/android/app/src/main/java/com/anonymous/bulwarkmobile/sync/PushRouting.kt b/android/app/src/main/java/com/anonymous/bulwarkmobile/sync/PushRouting.kt new file mode 100644 index 00000000..3bdcca1b --- /dev/null +++ b/android/app/src/main/java/com/anonymous/bulwarkmobile/sync/PushRouting.kt @@ -0,0 +1,162 @@ +package com.anonymous.bulwarkmobile.sync + +import android.content.Context +import android.util.Log +import org.json.JSONArray +import org.json.JSONException +import org.json.JSONObject + +/** + * What a push asks for (#34): syncs of the Android accounts that contacts and + * calendar changes feed, and whether the mail push task should run at all. + * + * The relay forwards the JMAP StateChange's `changed` map verbatim (FCM data + * key `changed`: JMAP account id → type → state; only its first account is + * repeated as `accountId`), so every account key is looked at. The mail task + * runs only for mail: when some account's map has `EmailDelivery`, for an + * EmailPush, or when `changed` is missing or unreadable (older relays), so a + * contact change never ends in a mail notification. + */ +object PushRouting { + /** An Android account and the authorities it syncs, fed by one JMAP account. */ + data class Route(val accountName: String, val authorities: Set) + + data class Decision( + /** (Android account name, authority) pairs to request a sync for. */ + val syncs: Set>, + val startMailTask: Boolean, + ) + + private const val EMAIL_PUSH_KIND = "jmap-email-push" + private const val EMAIL_DELIVERY = "EmailDelivery" + private val CONTACT_TYPES = setOf("ContactCard", "AddressBook") + private val CALENDAR_TYPES = setOf("CalendarEvent", "Calendar") + + fun decide(data: Map, routes: Map>): Decision { + val changed = data["changed"]?.let(::parseChanged) + var mail = data["kind"] == EMAIL_PUSH_KIND || changed == null + val syncs = LinkedHashSet>() + for ((jmapAccountId, types) in changed.orEmpty()) { + if (types == null) { + // One account's entry is unreadable: it may be mail. + mail = true + continue + } + if (EMAIL_DELIVERY in types) mail = true + val contacts = types.any { it in CONTACT_TYPES } + val calendar = types.any { it in CALENDAR_TYPES } + for (route in routes[jmapAccountId].orEmpty()) { + if (contacts && DeviceSyncAccounts.CONTACTS_AUTHORITY in route.authorities) { + syncs += route.accountName to DeviceSyncAccounts.CONTACTS_AUTHORITY + } + if (calendar && DeviceSyncAccounts.CALENDAR_AUTHORITY in route.authorities) { + syncs += route.accountName to DeviceSyncAccounts.CALENDAR_AUTHORITY + } + } + } + return Decision(syncs, mail) + } + + /** + * JMAP account id → its changed types (null for an entry that is not an + * object). Null for a map that is unreadable or names no type at all, + * which the relay sends as `{}` when it had no map. + */ + private fun parseChanged(json: String): Map?>? { + val o = try { + JSONObject(json) + } catch (e: JSONException) { + return null + } + val out = LinkedHashMap?>() + for (jmapAccountId in o.keys()) { + val types = o.opt(jmapAccountId) as? JSONObject + out[jmapAccountId] = types?.keys()?.asSequence()?.toSet() + } + return out.takeIf { entries -> entries.values.any { it == null || it.isNotEmpty() } } + } + + /** + * `{ jmapAccountId: [{ accountName, authorities }] }` as JS stores it. + * [strict] refuses anything malformed (IllegalArgumentException); otherwise + * malformed entries are skipped, so a bad route never stops mail. + */ + fun parseRoutes(json: String?, strict: Boolean = false): Map> { + if (json.isNullOrEmpty()) return emptyMap() + fun bad(message: String): Nothing = throw IllegalArgumentException("Push routes: $message") + val o = try { + JSONObject(json) + } catch (e: JSONException) { + if (strict) bad("not a JSON object") else return emptyMap() + } + val out = LinkedHashMap>() + for (jmapAccountId in o.keys()) { + val entries = o.opt(jmapAccountId) as? JSONArray + ?: if (strict) bad("$jmapAccountId must map to an array") else continue + val routes = ArrayList() + for (i in 0 until entries.length()) { + val entry = entries.opt(i) as? JSONObject + val name = entry?.opt("accountName") as? String + val authorities = entry?.opt("authorities") as? JSONArray + if (name.isNullOrEmpty() || authorities == null) { + if (strict) bad("$jmapAccountId[$i] needs accountName and authorities") else continue + } + val known = (0 until authorities.length()).mapNotNull { j -> + val authority = authorities.opt(j) as? String + if (authority !in DeviceSyncAccounts.AUTHORITIES) { + if (strict) bad("$jmapAccountId[$i]: unknown authority $authority") else null + } else { + authority + } + } + routes += Route(name, known.toSet()) + } + out[jmapAccountId] = routes + } + return out + } +} + +/** + * Applies [PushRouting] to incoming pushes (BulwarkMessagingService, + * BulwarkUnifiedPushService) with the routes JS stored through + * `setPushRoutes`. + */ +object DeviceSyncPushRouter { + private const val TAG = "BulwarkDeviceSync" + private const val PREFS = "bulwark_device_sync" + private const val KEY_ROUTES = "pushRoutes" + + /** Validates and stores the routes; they must survive the process, pushes arrive without JS. */ + fun saveRoutes(context: Context, json: String) { + PushRouting.parseRoutes(json, strict = true) + val saved = context.getSharedPreferences(PREFS, Context.MODE_PRIVATE).edit() + .putString(KEY_ROUTES, json) + .commit() + check(saved) { "Push routes could not be saved" } + } + + /** + * Requests the syncs the push asks for, with constant (empty) extras so + * repeated pushes coalesce into one pending sync. Returns whether the mail + * push task should run; any failure here says yes, so mail never depends + * on device sync. + */ + fun route(context: Context, data: Map): Boolean = try { + val routes = PushRouting.parseRoutes( + context.getSharedPreferences(PREFS, Context.MODE_PRIVATE).getString(KEY_ROUTES, null), + ) + val decision = PushRouting.decide(data, routes) + for ((accountName, authority) in decision.syncs) { + try { + DeviceSyncAccounts.requestSync(context, accountName, authority, SyncRequestFlags()) + } catch (e: RuntimeException) { + Log.w(TAG, "Push: no sync requested for $authority", e) + } + } + decision.startMailTask + } catch (e: Exception) { + Log.w(TAG, "Push not routed", e) + true + } +} diff --git a/android/app/src/main/java/com/anonymous/bulwarkmobile/sync/SyncProblemNotifier.kt b/android/app/src/main/java/com/anonymous/bulwarkmobile/sync/SyncProblemNotifier.kt new file mode 100644 index 00000000..d76a88fc --- /dev/null +++ b/android/app/src/main/java/com/anonymous/bulwarkmobile/sync/SyncProblemNotifier.kt @@ -0,0 +1,75 @@ +package com.anonymous.bulwarkmobile.sync + +import android.app.NotificationChannel +import android.app.NotificationManager +import android.app.PendingIntent +import android.content.Context +import android.content.Intent +import android.net.Uri +import android.os.Build +import androidx.core.app.NotificationCompat +import com.anonymous.bulwarkmobile.MainActivity +import com.anonymous.bulwarkmobile.R + +/** + * The one notification per Android account that device sync needs the user + * for (#34), e.g. "Sign in again" after the server rejected the credentials. + * The engine posts it through `showSyncProblem`; the app clears it once the + * problem is solved. Tapping it opens a `bulwarkmobile://` deep link in the + * app. + */ +object SyncProblemNotifier { + const val CHANNEL_ID = "bulwark_sync" + private const val DEFAULT_CHANNEL_NAME = "Sync problems" + private const val TAG_PREFIX = "bulwark-sync:" + private const val NOTIFICATION_ID = 1 + private const val DEEP_LINK_SCHEME = "bulwarkmobile" + + /** + * Creates the channel, or renames it: JS passes the name in the app's + * language. Nothing urgent, so default importance and no vibration. + */ + fun ensureChannel(context: Context, name: String?) { + if (Build.VERSION.SDK_INT < Build.VERSION_CODES.O) return + val manager = context.getSystemService(NotificationManager::class.java) + val channel = NotificationChannel( + CHANNEL_ID, + name?.takeIf { it.isNotBlank() } ?: DEFAULT_CHANNEL_NAME, + NotificationManager.IMPORTANCE_DEFAULT, + ).apply { enableVibration(false) } + manager.createNotificationChannel(channel) + } + + fun show(context: Context, accountName: String, title: String, text: String, uri: String, channelName: String?) { + val link = Uri.parse(uri) + require(link.scheme == DEEP_LINK_SCHEME) { "Only $DEEP_LINK_SCHEME:// links open from a sync notification" } + ensureChannel(context, channelName) + val tag = TAG_PREFIX + accountName + // Explicit, so no other app can answer the tap. + val intent = Intent(Intent.ACTION_VIEW, link, context, MainActivity::class.java) + .addFlags(Intent.FLAG_ACTIVITY_NEW_TASK or Intent.FLAG_ACTIVITY_SINGLE_TOP or Intent.FLAG_ACTIVITY_CLEAR_TOP) + val pending = PendingIntent.getActivity( + context, + tag.hashCode(), + intent, + PendingIntent.FLAG_UPDATE_CURRENT or PendingIntent.FLAG_IMMUTABLE, + ) + val notification = NotificationCompat.Builder(context, CHANNEL_ID) + .setSmallIcon(R.drawable.ic_notification) + .setContentTitle(title) + .setContentText(text) + .setStyle(NotificationCompat.BigTextStyle().bigText(text)) + .setCategory(NotificationCompat.CATEGORY_ERROR) + .setPriority(NotificationCompat.PRIORITY_DEFAULT) + // Runs repeat the same problem until it is solved; say it once. + .setOnlyAlertOnce(true) + .setAutoCancel(true) + .setContentIntent(pending) + .build() + context.getSystemService(NotificationManager::class.java).notify(tag, NOTIFICATION_ID, notification) + } + + fun clear(context: Context, accountName: String) { + context.getSystemService(NotificationManager::class.java).cancel(TAG_PREFIX + accountName, NOTIFICATION_ID) + } +} diff --git a/android/app/src/test/java/com/anonymous/bulwarkmobile/sync/DeviceSyncAccountsTest.kt b/android/app/src/test/java/com/anonymous/bulwarkmobile/sync/DeviceSyncAccountsTest.kt new file mode 100644 index 00000000..abbaa92e --- /dev/null +++ b/android/app/src/test/java/com/anonymous/bulwarkmobile/sync/DeviceSyncAccountsTest.kt @@ -0,0 +1,18 @@ +package com.anonymous.bulwarkmobile.sync + +import com.anonymous.bulwarkmobile.sync.DeviceSyncAccounts.EnsureAction +import org.junit.Assert.assertEquals +import org.junit.Test + +class DeviceSyncAccountsTest { + @Test + fun `ensureAccount creates, keeps, adopts or refuses`() { + assertEquals(EnsureAction.CREATE, DeviceSyncAccounts.ensureAction(false, null, "alice@example.org@mail")) + assertEquals(EnsureAction.KEEP, DeviceSyncAccounts.ensureAction(true, "alice@example.org@mail", "alice@example.org@mail")) + // An account whose setup never wrote a registry id. + assertEquals(EnsureAction.ADOPT, DeviceSyncAccounts.ensureAction(true, null, "alice@example.org@mail")) + assertEquals(EnsureAction.ADOPT, DeviceSyncAccounts.ensureAction(true, "", "alice@example.org@mail")) + // Its rows and SyncState belong to another app account: never rewritten. + assertEquals(EnsureAction.CONFLICT, DeviceSyncAccounts.ensureAction(true, "alice@example.org@old", "alice@example.org@mail")) + } +} diff --git a/android/app/src/test/java/com/anonymous/bulwarkmobile/sync/DeviceSyncRunsTest.kt b/android/app/src/test/java/com/anonymous/bulwarkmobile/sync/DeviceSyncRunsTest.kt new file mode 100644 index 00000000..dbdfc040 --- /dev/null +++ b/android/app/src/test/java/com/anonymous/bulwarkmobile/sync/DeviceSyncRunsTest.kt @@ -0,0 +1,62 @@ +package com.anonymous.bulwarkmobile.sync + +import org.junit.Assert.assertEquals +import org.junit.Assert.assertFalse +import org.junit.Assert.assertNull +import org.junit.Assert.assertTrue +import org.junit.Test + +class DeviceSyncRunsTest { + private val contacts = "com.android.contacts" + private val calendar = "com.android.calendar" + + @Test + fun `a report is accepted once, while the adapter waits`() { + val run = DeviceSyncRuns.open("alice", contacts) + assertTrue(DeviceSyncRuns.finish(run.runId, "{\"outcome\":\"ok\"}")) + assertFalse(DeviceSyncRuns.finish(run.runId, "{\"outcome\":\"io\"}")) + assertFalse(run.end("no report before the deadline")) + assertEquals(0L, run.done.count) + assertEquals("{\"outcome\":\"ok\"}", run.report) + assertEquals("reported", run.endReason) + DeviceSyncRuns.close(run) + } + + @Test + fun `a report after the run ended is refused whole`() { + val run = DeviceSyncRuns.open("alice", contacts) + assertTrue(run.end("no report before the deadline")) + assertFalse(DeviceSyncRuns.finish(run.runId, "{\"outcome\":\"ok\"}")) + assertNull(run.report) + assertEquals("no report before the deadline", run.endReason) + DeviceSyncRuns.close(run) + assertFalse(DeviceSyncRuns.finish(run.runId, "{\"outcome\":\"ok\"}")) + } + + @Test + fun `runs nobody waits for, or that ended, count as cancelled`() { + assertTrue(DeviceSyncRuns.isCancelled("unknown")) + val run = DeviceSyncRuns.open("alice", calendar) + assertFalse(DeviceSyncRuns.isCancelled(run.runId)) + run.end("task finished") + assertTrue(DeviceSyncRuns.isCancelled(run.runId)) + DeviceSyncRuns.close(run) + assertTrue(DeviceSyncRuns.isCancelled(run.runId)) + } + + @Test + fun `a sync requested during a run flags that run only, and the flag outlives the close`() { + val mine = DeviceSyncRuns.open("alice", contacts) + val otherAuthority = DeviceSyncRuns.open("alice", calendar) + val otherAccount = DeviceSyncRuns.open("bob", contacts) + assertTrue(DeviceSyncRuns.flagSyncAgain("alice", contacts)) + DeviceSyncRuns.close(mine) + assertTrue(mine.syncAgainRequested) + assertFalse(otherAuthority.syncAgainRequested) + assertFalse(otherAccount.syncAgainRequested) + // Closed runs are not flagged any more. + assertFalse(DeviceSyncRuns.flagSyncAgain("alice", contacts)) + DeviceSyncRuns.close(otherAuthority) + DeviceSyncRuns.close(otherAccount) + } +} diff --git a/android/app/src/test/java/com/anonymous/bulwarkmobile/sync/ProviderOpsTest.kt b/android/app/src/test/java/com/anonymous/bulwarkmobile/sync/ProviderOpsTest.kt new file mode 100644 index 00000000..751f0ad5 --- /dev/null +++ b/android/app/src/test/java/com/anonymous/bulwarkmobile/sync/ProviderOpsTest.kt @@ -0,0 +1,515 @@ +package com.anonymous.bulwarkmobile.sync + +import org.json.JSONArray +import org.json.JSONObject +import org.junit.Assert.assertEquals +import org.junit.Assert.assertNull +import org.junit.Assert.assertTrue +import org.junit.Assert.fail +import org.junit.Test +import java.math.BigDecimal +import java.math.BigInteger + +private const val CONTACTS = "com.android.contacts" +private const val CALENDAR = "com.android.calendar" +private val ME = AccountRef("alice@example.org", "com.anonymous.bulwarkmobile.account") +private val OTHER = AccountRef("bob@gmail.com", "com.google") + +/** The provider as far as the planner asks: who owns which row; [me] is the account the lookup is bound to. */ +private class FakeLookup(private val me: AccountRef = ME) : ScopeLookup { + val rawContacts = HashMap() + val groups = HashMap() + val calendars = HashMap() + val colors = HashMap() + /** data id → raw contact */ + val data = HashMap() + /** event id → calendar */ + val events = HashMap() + /** child table → child id → event */ + val children = HashMap>() + /** (child table, where) → events of the matching rows; a null where matches every row. */ + val matching = HashMap, Set>() + + override fun ownCalendarIds() = calendars.filterValues { it == me }.keys + + override fun rootAccounts(table: ProviderTable, ids: Set): Map = when (table) { + ProviderTable.RAW_CONTACTS -> rawContacts + ProviderTable.GROUPS -> groups + ProviderTable.CALENDARS -> calendars + ProviderTable.COLORS -> colors + else -> throw AssertionError("no root lookup for $table") + }.filterKeys { it in ids } + + override fun dataRawContacts(ids: Set) = data.filterKeys { it in ids } + + override fun eventCalendars(ids: Set) = events.filterKeys { it in ids } + + override fun childEvents(table: ProviderTable, ids: Set): Map = + children[table].orEmpty().filterKeys { it in ids } + + override fun childEventIdsMatching(table: ProviderTable, where: String?, args: List): Set = + matching[table to where] ?: if (where == null) children[table].orEmpty().values.toSet() else emptySet() + + fun child(table: ProviderTable, id: Long, event: Long) { + children.getOrPut(table) { HashMap() }[id] = event + } +} + +class ProviderOpsTest { + private val lookup = FakeLookup().apply { + rawContacts[1] = ME + rawContacts[2] = OTHER + groups[10] = ME + groups[11] = OTHER + data[100] = 1 + data[101] = 2 + calendars[20] = ME + calendars[21] = ME + calendars[22] = OTHER + colors[40] = ME + events[30] = 20 + events[31] = 21 + events[32] = 22 + child(ProviderTable.ATTENDEES, 50, 30) + child(ProviderTable.ATTENDEES, 51, 32) + child(ProviderTable.REMINDERS, 60, 31) + child(ProviderTable.EXTENDED_PROPERTIES, 70, 30) + child(ProviderTable.EXTENDED_PROPERTIES, 71, 32) + } + private val planner = ScopePlanner(ME, lookup) + + private fun ops(vararg op: String) = "[${op.joinToString(",")}]" + + private fun plan(authority: String, vararg op: String) = + planner.planBatch(ProviderOpParser.parseBatch(authority, ops(*op))) + + private fun refused(reason: BatchFailure, block: () -> Unit): String { + try { + block() + } catch (e: ProviderRefusal) { + assertEquals(e.message, reason, e.reason) + return e.message.orEmpty() + } + fail("expected a $reason refusal") + throw AssertionError() + } + + // ── tables and parsing ────────────────────────────────────── + + @Test + fun `tables are enumerated per authority and anything else is out of scope`() { + assertEquals(ProviderTable.EXTENDED_PROPERTIES, ProviderTable.of(CALENDAR, "extended_properties")) + assertEquals(ProviderTable.SETTINGS, ProviderTable.of(CONTACTS, "settings")) + refused(BatchFailure.SCOPE) { ProviderTable.of(CONTACTS, "events") } + refused(BatchFailure.SCOPE) { ProviderTable.of(CALENDAR, "raw_contacts") } + refused(BatchFailure.SCOPE) { ProviderTable.of(CONTACTS, "contacts") } + refused(BatchFailure.SCOPE) { plan(CALENDAR, """{"op":"delete","table":"instances","where":"1"}""") } + } + + @Test + fun `numbers keep integers exact whatever org_json parsed them into`() { + assertEquals(Cell.Integer(5), ProviderOpParser.numberCell(5)) + assertEquals(Cell.Integer(5), ProviderOpParser.numberCell(5.0)) + assertEquals(Cell.Integer(5), ProviderOpParser.numberCell(BigDecimal("5.0"))) + assertEquals(Cell.Integer(1L shl 60), ProviderOpParser.numberCell(BigInteger.ONE.shiftLeft(60))) + assertEquals(Cell.Integer(1_700_000_000_000), ProviderOpParser.numberCell(1_700_000_000_000L)) + assertEquals(Cell.Real(1.5), ProviderOpParser.numberCell(BigDecimal("1.5"))) + assertEquals(Cell.Real(1.5), ProviderOpParser.numberCell(1.5)) + } + + @Test + fun `values keep text, numbers, booleans as 0 and 1, and nulls`() { + val insert = ProviderOpParser.parseBatch( + CONTACTS, + ops("""{"op":"insert","table":"raw_contacts","values":{"sync1":"a","dirty":0,"starred":true,"sync2":null}}"""), + ).single() as ProviderOp.Insert + assertEquals( + mapOf("sync1" to Cell.Text("a"), "dirty" to Cell.Integer(0), "starred" to Cell.Integer(1), "sync2" to Cell.Null), + insert.values, + ) + } + + @Test + fun `malformed ops are provider failures`() { + refused(BatchFailure.PROVIDER) { ProviderOpParser.parseBatch(CONTACTS, "{}") } + refused(BatchFailure.PROVIDER) { plan(CONTACTS, """{"op":"upsert","table":"groups","values":{}}""") } + refused(BatchFailure.PROVIDER) { plan(CONTACTS, """{"op":"update","table":"groups","id":10,"values":{}}""") } + refused(BatchFailure.PROVIDER) { plan(CONTACTS, """{"op":"assert","table":"groups","where":"dirty = 1"}""") } + refused(BatchFailure.PROVIDER) { plan(CONTACTS, """{"op":"delete","table":"settings","id":1}""") } + refused(BatchFailure.PROVIDER) { plan(CONTACTS, """{"op":"delete","table":"groups","args":["x"]}""") } + refused(BatchFailure.PROVIDER) { plan(CONTACTS, """{"op":"delete","table":"groups","id":1.5}""") } + refused(BatchFailure.PROVIDER) { plan(CONTACTS, """{"op":"delete","table":"groups","where":"a = ?","args":[null]}""") } + } + + @Test + fun `bytes are only accepted for data_data15`() { + val planned = plan( + CONTACTS, + """{"op":"insert","table":"data","values":{"raw_contact_id":1,"mimetype":"vnd.android.cursor.item/photo","data15":{"b64":"AAEC"}}}""", + ).single() as PlannedOp.Insert + assertEquals(Cell.Blob("AAEC"), planned.values["data15"]) + refused(BatchFailure.SCOPE) { + plan(CONTACTS, """{"op":"update","table":"raw_contacts","id":1,"values":{"sync2":{"b64":"AAEC"}}}""") + } + refused(BatchFailure.SCOPE) { + plan(CONTACTS, """{"op":"insert","table":"data","values":{"raw_contact_id":1,"data1":{"b64":"AA"}}}""") + } + refused(BatchFailure.PROVIDER) { + plan(CONTACTS, """{"op":"insert","table":"data","values":{"raw_contact_id":1,"data15":{"bytes":"AA"}}}""") + } + } + + // ── batch rules ───────────────────────────────────────────── + + @Test + fun `a syncState op must be the last op and not a yield point`() { + val planned = plan( + CONTACTS, + """{"op":"update","table":"groups","id":10,"values":{"sync2":"x"},"yieldAllowed":true}""", + """{"op":"syncState","value":"{\"v\":1}"}""", + ) + assertEquals(PlannedOp.SetSyncState("{\"v\":1}"), planned.last()) + refused(BatchFailure.SCOPE) { + plan(CONTACTS, """{"op":"syncState","value":"{}"}""", """{"op":"delete","table":"groups","id":10}""") + } + refused(BatchFailure.SCOPE) { + plan( + CONTACTS, + """{"op":"delete","table":"groups","id":10}""", + """{"op":"syncState","value":"{}","yieldAllowed":true}""", + ) + } + } + + private fun deletes(count: Int, yieldAt: Set = emptySet()) = (0 until count).joinToString(",", "[", "]") { i -> + """{"op":"delete","table":"groups","where":"sync1 = ?","args":["$i"]${if (i in yieldAt) ""","yieldAllowed":true""" else ""}}""" + } + + @Test + fun `contacts batches stop at 499 ops between yield points, counted as ContactsProvider does`() { + assertEquals(499, ProviderOpParser.parseBatch(CONTACTS, deletes(499)).size) + refused(BatchFailure.PROVIDER) { ProviderOpParser.parseBatch(CONTACTS, deletes(500)) } + // The yielding op still counts toward the ops before it. + assertEquals(998, ProviderOpParser.parseBatch(CONTACTS, deletes(998, setOf(498))).size) + refused(BatchFailure.PROVIDER) { ProviderOpParser.parseBatch(CONTACTS, deletes(999, setOf(498))) } + refused(BatchFailure.PROVIDER) { ProviderOpParser.parseBatch(CONTACTS, deletes(600, setOf(499))) } + // A yield on the first op is ignored, as the provider ignores it. + refused(BatchFailure.PROVIDER) { ProviderOpParser.parseBatch(CONTACTS, deletes(500, setOf(0))) } + } + + @Test + fun `calendar batches have no yield point limit`() { + val many = (0 until 600).joinToString(",", "[", "]") { """{"op":"delete","table":"colors","where":"color_index = '$it'"}""" } + assertEquals(600, ProviderOpParser.parseBatch(CALENDAR, many).size) + } + + @Test + fun `back-references name earlier inserts, and parents name their parent table`() { + val planned = plan( + CONTACTS, + """{"op":"insert","table":"raw_contacts","values":{"sourceid":"c/1"}}""", + """{"op":"insert","table":"data","values":{"mimetype":"vnd.android.cursor.item/name","data1":"A"},"refs":{"raw_contact_id":0}}""", + ) + assertEquals(mapOf("raw_contact_id" to 0), (planned[1] as PlannedOp.Insert).backReferences) + refused(BatchFailure.PROVIDER) { + plan(CONTACTS, """{"op":"insert","table":"data","values":{"mimetype":"x"},"refs":{"raw_contact_id":0}}""") + } + refused(BatchFailure.PROVIDER) { + plan( + CONTACTS, + """{"op":"delete","table":"groups","id":10}""", + """{"op":"insert","table":"data","values":{"mimetype":"x"},"refs":{"raw_contact_id":0}}""", + ) + } + // Settings rows have no id to refer to. + refused(BatchFailure.PROVIDER) { + plan( + CONTACTS, + """{"op":"insert","table":"settings","values":{"ungrouped_visible":1}}""", + """{"op":"insert","table":"data","values":{"mimetype":"x"},"refs":{"raw_contact_id":0}}""", + ) + } + refused(BatchFailure.SCOPE) { + plan( + CONTACTS, + """{"op":"insert","table":"groups","values":{"title":"g"}}""", + """{"op":"insert","table":"data","values":{"mimetype":"x"},"refs":{"raw_contact_id":0}}""", + ) + } + refused(BatchFailure.SCOPE) { plan(CONTACTS, """{"op":"insert","table":"data","values":{"mimetype":"x"}}""") } + refused(BatchFailure.SCOPE) { + plan(CALENDAR, """{"op":"insert","table":"attendees","values":{"event_id":null,"attendeeEmail":"a@b"}}""") + } + } + + // ── scope: inserts ────────────────────────────────────────── + + @Test + fun `root inserts are written into the account, whatever the op says`() { + val planned = plan( + CONTACTS, + """{"op":"insert","table":"raw_contacts","values":{"sourceid":"c/1","account_name":"alice@example.org"}}""", + ).single() as PlannedOp.Insert + assertEquals(Cell.Text(ME.name), planned.values["account_name"]) + assertEquals(Cell.Text(ME.type), planned.values["account_type"]) + val calendar = plan(CALENDAR, """{"op":"insert","table":"calendars","values":{"name":"c"}}""").single() + assertEquals(Cell.Text(ME.type), (calendar as PlannedOp.Insert).values["account_type"]) + } + + @Test + fun `account columns never move a row out of the account`() { + refused(BatchFailure.SCOPE) { + plan(CONTACTS, """{"op":"insert","table":"raw_contacts","values":{"account_name":"bob@gmail.com"}}""") + } + refused(BatchFailure.SCOPE) { + plan(CONTACTS, """{"op":"update","table":"groups","id":10,"values":{"account_type":"com.google"}}""") + } + refused(BatchFailure.SCOPE) { + plan(CONTACTS, """{"op":"insert","table":"groups","values":{"title":"g","data_set":"plugin"}}""") + } + refused(BatchFailure.SCOPE) { + plan(CONTACTS, """{"op":"insert","table":"data","values":{"raw_contact_id":1,"account_name":"alice@example.org"}}""") + } + refused(BatchFailure.SCOPE) { + plan(CALENDAR, """{"op":"update","table":"events","id":30,"values":{"account_name":"alice@example.org"}}""") + } + } + + @Test + fun `inserts must name a parent of the account that still exists`() { + plan(CONTACTS, """{"op":"insert","table":"data","values":{"raw_contact_id":1,"mimetype":"x"}}""") + plan(CONTACTS, """{"op":"insert","table":"data","values":{"raw_contact_id":"1","mimetype":"x"}}""") + refused(BatchFailure.SCOPE) { plan(CONTACTS, """{"op":"insert","table":"data","values":{"raw_contact_id":2,"mimetype":"x"}}""") } + refused(BatchFailure.ASSERT) { plan(CONTACTS, """{"op":"insert","table":"data","values":{"raw_contact_id":9,"mimetype":"x"}}""") } + refused(BatchFailure.PROVIDER) { + plan(CONTACTS, """{"op":"insert","table":"data","values":{"raw_contact_id":"me","mimetype":"x"}}""") + } + plan(CALENDAR, """{"op":"insert","table":"events","values":{"calendar_id":21,"dtstart":0}}""") + refused(BatchFailure.SCOPE) { plan(CALENDAR, """{"op":"insert","table":"events","values":{"calendar_id":22,"dtstart":0}}""") } + refused(BatchFailure.SCOPE) { + plan(CALENDAR, """{"op":"insert","table":"events","values":{"calendar_id":20,"original_id":32}}""") + } + plan(CALENDAR, """{"op":"insert","table":"reminders","values":{"event_id":31,"minutes":10}}""") + refused(BatchFailure.SCOPE) { plan(CALENDAR, """{"op":"insert","table":"reminders","values":{"event_id":32,"minutes":10}}""") } + refused(BatchFailure.ASSERT) { plan(CALENDAR, """{"op":"insert","table":"attendees","values":{"event_id":99}}""") } + } + + @Test + fun `a membership row may only name a group of the account`() { + val membership = """"mimetype":"vnd.android.cursor.item/group_membership"""" + plan(CONTACTS, """{"op":"insert","table":"data","values":{"raw_contact_id":1,$membership,"data1":10}}""") + plan(CONTACTS, """{"op":"insert","table":"data","values":{"raw_contact_id":1,$membership,"group_sourceid":"c/g"}}""") + refused(BatchFailure.SCOPE) { + plan(CONTACTS, """{"op":"insert","table":"data","values":{"raw_contact_id":1,$membership,"data1":11}}""") + } + // data1 of other kinds is plain data. + plan(CONTACTS, """{"op":"insert","table":"data","values":{"raw_contact_id":1,"mimetype":"x","data1":11}}""") + } + + @Test + fun `moving a row to another parent is checked like an insert`() { + refused(BatchFailure.SCOPE) { plan(CALENDAR, """{"op":"update","table":"events","id":30,"values":{"calendar_id":22}}""") } + refused(BatchFailure.SCOPE) { plan(CONTACTS, """{"op":"update","table":"data","id":100,"values":{"raw_contact_id":null}}""") } + plan(CALENDAR, """{"op":"update","table":"events","id":30,"values":{"calendar_id":21}}""") + } + + // ── scope: selections ─────────────────────────────────────── + + @Test + fun `root tables are selected by their account columns`() { + val op = plan( + CONTACTS, + """{"op":"update","table":"raw_contacts","where":"dirty = ? AND deleted = 0","args":[1],"values":{"dirty":0}}""", + ).single() as PlannedOp.Update + assertEquals( + Selection("(dirty = ? AND deleted = 0) AND (account_name = ? AND account_type = ?)", listOf("1", ME.name, ME.type)), + op.selection, + ) + assertNull(op.expectedCount) + } + + @Test + fun `an op by id selects that row of the account and expects exactly one row`() { + val op = plan(CONTACTS, """{"op":"delete","table":"groups","id":10}""").single() as PlannedOp.Delete + assertEquals(Selection("(_id = ?) AND (account_name = ? AND account_type = ?)", listOf("10", ME.name, ME.type)), op.selection) + assertEquals(1, op.expectedCount) + val counted = plan(CONTACTS, """{"op":"assert","table":"raw_contacts","id":1,"expectCount":0}""").single() + assertEquals(0, (counted as PlannedOp.Assert).expectedCount) + } + + @Test + fun `an id of another account is refused, a vanished one is left to the expected count`() { + refused(BatchFailure.SCOPE) { plan(CONTACTS, """{"op":"delete","table":"raw_contacts","id":2}""") } + refused(BatchFailure.SCOPE) { plan(CONTACTS, """{"op":"update","table":"data","id":101,"values":{"data1":"x"}}""") } + refused(BatchFailure.SCOPE) { plan(CALENDAR, """{"op":"assert","table":"events","id":32,"values":{"dirty":0}}""") } + refused(BatchFailure.SCOPE) { plan(CALENDAR, """{"op":"delete","table":"attendees","id":51}""") } + refused(BatchFailure.SCOPE) { plan(CALENDAR, """{"op":"update","table":"calendars","id":22,"values":{"name":"x"}}""") } + val gone = plan(CONTACTS, """{"op":"delete","table":"raw_contacts","id":9}""").single() as PlannedOp.Delete + assertEquals(1, gone.expectedCount) + } + + @Test + fun `data rows rely on the URI parameters and never select on the Data view's account columns`() { + val op = plan(CONTACTS, """{"op":"update","table":"data","id":100,"values":{"data1":"x"}}""").single() as PlannedOp.Update + assertEquals(Selection("(_id = ?)", listOf("100")), op.selection) + val query = planner.planQuery(ProviderOpParser.parseQuery(CONTACTS, """{"table":"data","columns":["_id"],"where":"raw_contact_id IN (1)"}""")) + assertEquals(listOf(Selection("(raw_contact_id IN (1))", emptyList())), query.selections) + val all = planner.planQuery(ProviderOpParser.parseQuery(CONTACTS, """{"table":"data","columns":["_id"]}""")) + assertEquals(listOf(Selection.NONE), all.selections) + } + + @Test + fun `events are selected by the account's calendars`() { + val query = planner.planQuery( + ProviderOpParser.parseQuery(CALENDAR, """{"table":"events","columns":["_id","title"],"where":"dirty = 1","orderBy":"_id"}"""), + ) + assertEquals(listOf(Selection("(dirty = 1) AND (calendar_id IN (20,21))", emptyList())), query.selections) + assertEquals("_id", query.orderBy) + val update = plan(CALENDAR, """{"op":"update","table":"events","id":30,"values":{"dirty":0}}""").single() as PlannedOp.Update + assertEquals(Selection("(_id = ?) AND (calendar_id IN (20,21))", listOf("30")), update.selection) + } + + @Test + fun `an account without calendars selects no events`() { + val carol = AccountRef("carol", ME.type) + val empty = ScopePlanner(carol, FakeLookup(carol).apply { calendars.putAll(lookup.calendars) }) + val query = empty.planQuery(ProviderOpParser.parseQuery(CALENDAR, """{"table":"events","columns":["_id"]}""")) + assertEquals(listOf(Selection("(0)", emptyList())), query.selections) + } + + @Test + fun `children of events are selected by the account's events among those they touch`() { + lookup.matching[ProviderTable.ATTENDEES to "event_id IN (30,32)"] = setOf(30, 32) + val query = planner.planQuery( + ProviderOpParser.parseQuery(CALENDAR, """{"table":"attendees","columns":["attendeeEmail"],"where":"event_id IN (30,32)"}"""), + ) + assertEquals(listOf(Selection("(event_id IN (30,32)) AND (event_id IN (30))", emptyList())), query.selections) + + val byId = plan(CALENDAR, """{"op":"delete","table":"attendees","id":50}""").single() as PlannedOp.Delete + assertEquals(Selection("(Attendees._id = ?) AND (event_id IN (30))", listOf("50")), byId.selection) + assertEquals(1, byId.expectedCount) + + val gone = plan(CALENDAR, """{"op":"delete","table":"reminders","id":69}""").single() as PlannedOp.Delete + assertEquals(Selection("(Reminders._id = ?) AND (0)", listOf("69")), gone.selection) + + lookup.matching[ProviderTable.REMINDERS to "event_id = ?"] = setOf(31) + val byWhere = plan(CALENDAR, """{"op":"delete","table":"reminders","where":"event_id = ?","args":[31]}""").single() + assertEquals(Selection("(event_id = ?) AND (event_id IN (31))", listOf("31")), (byWhere as PlannedOp.Delete).selection) + } + + @Test + fun `a child query that touches none of the account's events reads nothing`() { + lookup.matching[ProviderTable.ATTENDEES to "event_id = 32"] = setOf(32) + val query = planner.planQuery( + ProviderOpParser.parseQuery(CALENDAR, """{"table":"attendees","columns":["_id"],"where":"event_id = 32"}"""), + ) + assertTrue(query.selections.isEmpty()) + } + + @Test + fun `long event lists are split into chunks`() { + val many = (1000L until 1000L + 1200).toSet() + for (id in many) lookup.events[id] = 20 + lookup.matching[ProviderTable.REMINDERS to null] = many + val query = planner.planQuery(ProviderOpParser.parseQuery(CALENDAR, """{"table":"reminders","columns":["minutes"]}""")) + assertEquals(3, query.selections.size) + assertTrue(query.selections.all { it.sql!!.startsWith("(event_id IN (") }) + } + + @Test + fun `extended properties are updated through their item URI, so only by id`() { + val op = plan(CALENDAR, """{"op":"update","table":"extended_properties","id":70,"values":{"value":"v"}}""").single() + assertEquals(PlannedOp.Update(ProviderTable.EXTENDED_PROPERTIES, Selection.NONE, 70, mapOf("value" to Cell.Text("v")), 1, false), op) + refused(BatchFailure.SCOPE) { + plan(CALENDAR, """{"op":"update","table":"extended_properties","id":71,"values":{"value":"v"}}""") + } + refused(BatchFailure.PROVIDER) { + plan(CALENDAR, """{"op":"update","table":"extended_properties","where":"name = 'x'","values":{"value":"v"}}""") + } + } + + @Test + fun `asserts compare text and keep nulls`() { + val op = plan( + CONTACTS, + """{"op":"assert","table":"raw_contacts","id":1,"values":{"version":3,"dirty":0,"sync3":null,"sourceid":"c/1"}}""", + ).single() as PlannedOp.Assert + assertEquals(mapOf("version" to "3", "dirty" to "0", "sync3" to null, "sourceid" to "c/1"), op.values) + assertEquals(1, op.expectedCount) + } + + @Test + fun `settings are written by account`() { + val insert = plan(CONTACTS, """{"op":"insert","table":"settings","values":{"ungrouped_visible":1,"should_sync":1}}""").single() + assertEquals(Cell.Text(ME.name), (insert as PlannedOp.Insert).values["account_name"]) + val update = plan(CONTACTS, """{"op":"update","table":"settings","values":{"ungrouped_visible":1}}""").single() + assertEquals( + Selection("(account_name = ? AND account_type = ?)", listOf(ME.name, ME.type)), + (update as PlannedOp.Update).selection, + ) + } + + // ── results ───────────────────────────────────────────────── + + @Test + fun `batch results are BatchResult JSON`() { + val ok = JSONObject(ProviderResults.ok(listOf(OpOutcome(id = 7), OpOutcome(count = 1), OpOutcome()))) + assertEquals(true, ok.getBoolean("ok")) + val results = ok.getJSONArray("results") + assertEquals(7L, results.getJSONObject(0).getLong("id")) + assertEquals(1, results.getJSONObject(1).getInt("count")) + assertEquals(0, results.getJSONObject(2).length()) + val failed = JSONObject(ProviderResults.failed(BatchFailure.TOO_LARGE, "TransactionTooLargeException")) + assertEquals(false, failed.getBoolean("ok")) + assertEquals("tooLarge", failed.getString("reason")) + assertEquals("TransactionTooLargeException", failed.getString("message")) + } + + @Test + fun `an OperationApplicationException is an assert unless an insert got no row`() { + assertEquals(BatchFailure.ASSERT, ProviderResults.failureOfOperationApplication("Found value 4 when expected 3 for column version")) + assertEquals(BatchFailure.ASSERT, ProviderResults.failureOfOperationApplication("Expected 1 rows but actual 0")) + assertEquals(BatchFailure.ASSERT, ProviderResults.failureOfOperationApplication("wrong number of rows: 0")) + assertEquals( + BatchFailure.PROVIDER, + ProviderResults.failureOfOperationApplication("Insert into content://com.android.contacts/settings returned no result"), + ) + assertEquals( + BatchFailure.PROVIDER, + ProviderResults.failureOfOperationApplication("Too many content provider operations between yield points."), + ) + } + + @Test + fun `query rows are ProviderRows JSON with escaped text`() { + val writer = ProviderRowsWriter(listOf("_id", "title", "x")) + writer.beginRow() + writer.longCell(9_007_199_254_740_991) + writer.textCell("a \"quoted\"\\ line\nbreak\u0001 and \u2028") + writer.nullCell() + writer.endRow() + writer.beginRow() + writer.longCell(2) + writer.doubleCell(1.5) + writer.doubleCell(Double.NaN) + writer.endRow() + val parsed = JSONObject(writer.finish()) + assertEquals(JSONArray(listOf("_id", "title", "x")).toString(), parsed.getJSONArray("columns").toString()) + val rows = parsed.getJSONArray("rows") + assertEquals(9_007_199_254_740_991L, rows.getJSONArray(0).getLong(0)) + assertEquals("a \"quoted\"\\ line\nbreak\u0001 and \u2028", rows.getJSONArray(0).getString(1)) + assertTrue(rows.getJSONArray(0).isNull(2)) + assertEquals(1.5, rows.getJSONArray(1).getDouble(1), 0.0) + assertTrue(rows.getJSONArray(1).isNull(2)) + assertEquals("{\"columns\":[],\"rows\":[]}", ProviderRowsWriter(emptyList()).finish()) + } + + @Test + fun `photos decode close to the target size and scale their longer side down to it`() { + assertEquals(1, PhotoScaling.sampleSize(720, 720, 512)) + assertEquals(4, PhotoScaling.sampleSize(4000, 3000, 512)) + assertEquals(512 to 384, PhotoScaling.targetSize(1000, 750, 512)) + assertEquals(96 to 96, PhotoScaling.targetSize(96, 96, 512)) + assertEquals(1 to 512, PhotoScaling.targetSize(3, 4000, 512)) + } +} diff --git a/android/app/src/test/java/com/anonymous/bulwarkmobile/sync/PushRoutingTest.kt b/android/app/src/test/java/com/anonymous/bulwarkmobile/sync/PushRoutingTest.kt new file mode 100644 index 00000000..1a13656d --- /dev/null +++ b/android/app/src/test/java/com/anonymous/bulwarkmobile/sync/PushRoutingTest.kt @@ -0,0 +1,122 @@ +package com.anonymous.bulwarkmobile.sync + +import org.junit.Assert.assertEquals +import org.junit.Assert.assertFalse +import org.junit.Assert.assertTrue +import org.junit.Test + +class PushRoutingTest { + private val contacts = "com.android.contacts" + private val calendar = "com.android.calendar" + + private val routes = PushRouting.parseRoutes( + """ + { + "c": [{ "accountName": "alice@example.org", "authorities": ["$contacts", "$calendar"] }], + "team": [ + { "accountName": "alice@example.org", "authorities": ["$contacts"] }, + { "accountName": "alice-work", "authorities": ["$calendar"] } + ] + } + """.trimIndent(), + strict = true, + ) + + private fun stateChange(changed: String) = mapOf( + "kind" to "jmap-state-change", + "accountLabel" to "alice", + "accountId" to "c", + "emailIds" to "[]", + "changed" to changed, + ) + + @Test + fun `a contact change requests a contacts sync and no mail task`() { + val decision = PushRouting.decide(stateChange("""{"c":{"ContactCard":"s1"}}"""), routes) + assertEquals(setOf("alice@example.org" to contacts), decision.syncs) + assertFalse(decision.startMailTask) + } + + @Test + fun `calendar types request calendar syncs of the routed accounts only`() { + val decision = PushRouting.decide(stateChange("""{"team":{"Calendar":"s1","CalendarEvent":"s2"}}"""), routes) + assertEquals(setOf("alice-work" to calendar), decision.syncs) + assertFalse(decision.startMailTask) + } + + @Test + fun `every account key is routed, not only the one repeated as accountId`() { + val decision = PushRouting.decide( + stateChange("""{"c":{"AddressBook":"s1"},"team":{"ContactCard":"s2","CalendarEvent":"s3"}}"""), + routes, + ) + // alice@example.org's contacts sync is requested once. + assertEquals(setOf("alice@example.org" to contacts, "alice-work" to calendar), decision.syncs) + } + + @Test + fun `mail anywhere in the map starts the mail task, next to the syncs`() { + val decision = PushRouting.decide( + stateChange("""{"team":{"CalendarEvent":"s1"},"c":{"EmailDelivery":"s2","Email":"s3"}}"""), + routes, + ) + assertTrue(decision.startMailTask) + assertEquals(setOf("alice-work" to calendar), decision.syncs) + } + + @Test + fun `an email push always starts the mail task`() { + val data = mapOf("kind" to "jmap-email-push", "accountId" to "c", "changed" to """{"c":{"EmailDelivery":"s"}}""") + assertTrue(PushRouting.decide(data, routes).startMailTask) + assertTrue(PushRouting.decide(mapOf("kind" to "jmap-email-push"), routes).startMailTask) + } + + @Test + fun `payloads without a readable map are legacy mail pushes`() { + for (changed in listOf(null, "", "{}", "not json", "[1]", """{"c":{}}""")) { + val data = stateChange(changed ?: "").let { if (changed == null) it - "changed" else it } + val decision = PushRouting.decide(data, routes) + assertTrue("changed=$changed", decision.startMailTask) + assertTrue("changed=$changed", decision.syncs.isEmpty()) + } + } + + @Test + fun `an unreadable account entry may be mail`() { + val decision = PushRouting.decide(stateChange("""{"c":"garbled","team":{"ContactCard":"s"}}"""), routes) + assertTrue(decision.startMailTask) + assertEquals(setOf("alice@example.org" to contacts), decision.syncs) + } + + @Test + fun `accounts without routes are ignored`() { + val decision = PushRouting.decide(stateChange("""{"elsewhere":{"ContactCard":"s"}}"""), routes) + assertTrue(decision.syncs.isEmpty()) + assertFalse(decision.startMailTask) + assertTrue(PushRouting.decide(stateChange("""{"c":{"ContactCard":"s"}}"""), emptyMap()).syncs.isEmpty()) + } + + @Test + fun `stored routes are validated strictly and read leniently`() { + val bad = listOf( + "[]", + """{"c":{}}""", + """{"c":[{"authorities":["$contacts"]}]}""", + """{"c":[{"accountName":"a","authorities":["com.example.tasks"]}]}""", + ) + for (json in bad) { + try { + PushRouting.parseRoutes(json, strict = true) + throw AssertionError("accepted $json") + } catch (e: IllegalArgumentException) { + // expected + } + } + assertEquals(emptyMap>(), PushRouting.parseRoutes("[]")) + assertEquals( + mapOf("c" to listOf(PushRouting.Route("a", setOf(contacts)))), + PushRouting.parseRoutes("""{"c":[{"accountName":"a","authorities":["$contacts","com.example.tasks"]},{"x":1}]}"""), + ) + assertEquals(emptyMap>(), PushRouting.parseRoutes(null)) + } +} diff --git a/docs/device-sync.md b/docs/device-sync.md index 75b790d7..16a0e106 100644 --- a/docs/device-sync.md +++ b/docs/device-sync.md @@ -1018,39 +1018,76 @@ nested maps. Every provider call is scoped to one account of our type. | `openAccountSettings(name)` | the account's sync screen (`android.settings.ACCOUNT_SYNC_SETTINGS` with the Account), else `SYNC_SETTINGS` filtered to our type | | `finishRun(runId, reportJson)` / `isRunCancelled(runId)` | the run handshake | | `setPushRoutes(json)` | `{ jmapAccountId: [{ accountName, authorities }] }` for the push router, in SharedPreferences | +| `showSyncProblem(name, title, text, uri, channelName)` / `clearSyncProblem(name)` | the account's one sync-problem notification; a tap opens `uri` (`bulwarkmobile://` only) in the app; `channelName` names the channel in the app's language | | `query(name, authority, queryJson)` | `ProviderQuery` → `ProviderRows` | | `applyBatch(name, authority, opsJson)` | `ProviderOp[]` → `BatchResult`; failures resolve as `{ok: false, reason}` | | `readSyncState(name, authority)` | the SyncState blob as text | | `readPhoto(name, rawContactId, maxPx)` | the display photo (or thumbnail), scaled, as JPEG base64 + file id | +Provider calls run one at a time on a thread of the module's own (a batch +can hold the provider for seconds at a contended yield point), through an +unstable provider client, so a provider process that dies fails the call +instead of this process. `query`, `readSyncState` and `readPhoto` reject with +code `scope`, `permission` or `provider`. + Events: `BulwarkDeviceSync:accountsChanged` (an account of our type was added or removed; JS re-lists). Events without a JS listener are dropped, so they are hints; the UI re-reads state on mount and on foreground. -Scoping, enforced in Kotlin for every op: +Scoping, enforced in Kotlin for every op +(`android/app/src/main/java/com/anonymous/bulwarkmobile/sync/ProviderScope.kt`): - URIs always carry `caller_is_syncadapter=true`, `account_name`, `account_type` (CalendarProvider accepts no other query parameters). - Queries and updates/deletes by `where` get the account condition ANDed in: - `account_name/account_type` columns for raw_contacts, groups, settings, - calendars, events, colors; for data rows, `raw_contact_id IN` our raw - contacts; for attendees, reminders and extended properties, `event_id IN` - our events (resolved first; the providers may compile selections strictly, - so no subqueries). Ops by `id` verify the row belongs to the account first - and expect exactly one row. + - `account_name`/`account_type` columns for raw_contacts, groups, + settings, calendars and colors; + - data rows are filtered by ContactsProvider itself through the account + URI parameters (their account columns are never selected on: Android 17 + drops them from the Data view for apps targeting API 37); + - events get `calendar_id IN` the account's calendars (CalendarProvider + updates events on the Events table, which has no account columns); + - attendees, reminders and extended properties get `event_id IN` the + account's events among those the op touches (resolved first; the + providers may compile selections strictly, so no subqueries). +- Ops by `id` verify the row belongs to the account first (`scope` + otherwise) and expect exactly one row unless they set `expectCount`, so a + row that vanished fails with `assert`. `id` and `where` together address + the id row if it matches. On attendees, reminders and extended properties + the id column is table-qualified (`Attendees._id`), since CalendarProvider + reads attendees through a join where a bare `_id` is ambiguous; the engine + therefore never writes `_id` into a `where` there. Extended properties are + updated through their item URI, so by `id` only. - Inserts into data/attendees/reminders/extended_properties/events must name a - parent of the account (a back-reference to an insert of the same batch, or - an id that is verified). + parent (a back-reference to an insert of the same batch, or an id); an + insert or update naming a parent, exception master (`original_id`) or group + (`data1` of a membership row) checks it: gone fails with `assert` (re-read + and re-plan), another account's with `scope`. A back-reference must point + to an earlier insert of the right table; settings inserts return no id. +- `account_name`/`account_type`/`data_set` may only be written to restate the + account, on the tables that have them (`scope` otherwise). - Tables are an enumerated set per authority; anything else is refused (`scope`). - Blobs (`{ b64 }`) are only accepted for `data.data15` (photos). - A `syncState` op must be the last op of its batch and not `yieldAllowed`, so a provider yield never commits the state ahead of the rows (refused as `scope` otherwise). -- Batches: at most 499 ops between yield points (ContactsProvider throws at - 500), about 300 KB per call; `TransactionTooLargeException` resolves as +- Batches: at most 499 ops up to and including each yield point + (ContactsProvider counts an op before resetting at a yield point and throws + at 500), about 300 KB per call; `TransactionTooLargeException` resolves as `tooLarge` so JS splits the batch. `OperationApplicationException` resolves - as `assert`, `SecurityException` as `permission`. + as `assert` (except the provider's own "too many operations" and failed + inserts, `provider`), `SecurityException` as `permission`. The whole batch + is also refused (`scope`) once the Android account is gone, so no rows + outlive it. + +What the providers do with values, which the planners account for: + +- contacts `data1`..`data14` are TEXT columns: a number written there reads + back as a string, so mappers normalise types before comparing; +- `Events.dirty` and `Calendars.dirty` are NULL after a sync-adapter insert: + planners write `dirty = 0` on every insert and select dirty rows with + `dirty = 1`. ## App integration diff --git a/src/device-sync/__tests__/fake-provider.test.ts b/src/device-sync/__tests__/fake-provider.test.ts index 0a83ab5b..335c8047 100644 --- a/src/device-sync/__tests__/fake-provider.test.ts +++ b/src/device-sync/__tests__/fake-provider.test.ts @@ -62,7 +62,7 @@ describe('FakeDeviceProviders: contacts', () => { { op: 'insert', table: 'data', values: { [Data.RAW_CONTACT_ID]: theirs.rawId, [Data.MIMETYPE]: MimeType.NOTE } }, ]); expect(orphan).toMatchObject({ ok: false, reason: 'scope' }); - await expect(mine.port.query({ table: 'events', columns: ['_id'] })).rejects.toThrow(/scope/); + await expect(mine.port.query({ table: 'events', columns: ['_id'] })).rejects.toMatchObject({ code: 'scope' }); }); it('never sets DIRTY for the sync adapter but does for app edits, and only apps soft-delete', async () => { @@ -162,7 +162,8 @@ describe('FakeDeviceProviders: contacts', () => { expect(photo?.jpegBase64).toBe('AAAA'); expect(photo?.fileId).toEqual(expect.any(Number)); const rows = await port.query({ table: 'data', columns: [Data.DATA14, Data.DATA15], where: `${Data.MIMETYPE} = ?`, args: [MimeType.PHOTO] }); - expect(rows.rows[0]).toEqual([photo!.fileId, null]); + // data14 is a TEXT column: the file id reads back as text. + expect(rows.rows[0]).toEqual([String(photo!.fileId), null]); }); it('models Fossify saves: rows re-inserted without DATA_SYNC', async () => { @@ -268,13 +269,13 @@ describe('FakeDeviceProviders: calendar', () => { if (!res.ok) throw new Error(res.message); const masterId = res.results[0].id!; const ex = fake.user.exceptionViaUri(masterId, 86_400_000, { [Events.SELF_ATTENDEE_STATUS]: 1 }); - expect(fake.row('events', masterId)![Events.DIRTY]).toBe(0); + expect(fake.row('events', masterId)![Events.DIRTY]).toBeNull(); expect(fake.row('events', ex)).toMatchObject({ [Events.DIRTY]: 1, [Events._SYNC_ID]: null, [Events.UID_2445]: 'u1', [Events.ORIGINAL_SYNC_ID]: 'c/e1' }); expect(fake.rows('attendees', 'event_id = ?', [ex])[0][Attendees.ATTENDEE_STATUS]).toBe(1); const split = fake.user.splitViaUri(masterId, 5 * 86_400_000, '20260101T000000Z', {}); expect(fake.row('events', split)![Events._SYNC_ID]).toBe('c/e1'); expect(fake.row('events', masterId)![Events.RRULE]).toContain('UNTIL=20260101T000000Z'); - expect(fake.row('events', masterId)![Events.DIRTY]).toBe(0); + expect(fake.row('events', masterId)![Events.DIRTY]).toBeNull(); }); }); @@ -291,3 +292,179 @@ describe('AOSP format checks', () => { expect(aospRruleValid('COUNT=2')).toBe(false); }); }); + +describe('FakeDeviceProviders: the native bridge rules', () => { + async function calendarWithEvent(fake: FakeDeviceProviders, account = ME) { + const port = fake.port(account, CALENDAR_AUTHORITY); + const res = await port.applyBatch([ + { + op: 'insert', + table: 'calendars', + values: { + [Calendars.NAME]: 'Personal', + [Calendars.CALENDAR_DISPLAY_NAME]: 'Personal', + [Calendars.CALENDAR_COLOR]: -16776961, + [Calendars.CALENDAR_ACCESS_LEVEL]: 700, + [Calendars.OWNER_ACCOUNT]: account, + }, + }, + { + op: 'insert', + table: 'events', + values: { [Events.DTSTART]: 0, [Events.DTEND]: 3_600_000, [Events.EVENT_TIMEZONE]: 'UTC', [Events.STATUS]: 1 }, + refs: { [Events.CALENDAR_ID]: 0 }, + }, + { op: 'insert', table: 'attendees', values: { [Attendees.ATTENDEE_EMAIL]: OTHER }, refs: { event_id: 1 } }, + { op: 'insert', table: 'extended_properties', values: { name: 'x', value: '1' }, refs: { event_id: 1 } }, + ]); + if (!res.ok) throw new Error(res.message); + return { + port, + calendarId: res.results[0].id!, + eventId: res.results[1].id!, + attendeeId: res.results[2].id!, + propertyId: res.results[3].id!, + }; + } + + it('stores numbers written to data1..data14 as text', async () => { + const fake = new FakeDeviceProviders(); + const { port, rawId } = await contactWithEmail(fake); + const res = await port.applyBatch([ + { + op: 'insert', + table: 'data', + values: { [Data.RAW_CONTACT_ID]: rawId, [Data.MIMETYPE]: MimeType.PHONE, [Data.DATA1]: '+1 555', [Data.DATA2]: 2 }, + }, + ]); + if (!res.ok) throw new Error(res.message); + const rows = await port.query({ + table: 'data', + columns: [Data.DATA2, Data.RAW_CONTACT_ID], + where: `${Data.MIMETYPE} = ?`, + args: [MimeType.PHONE], + }); + expect(rows.rows).toEqual([['2', rawId]]); + // Asserts compare as text, so a number still matches. + const ok = await port.applyBatch([{ op: 'assert', table: 'data', id: res.results[0].id!, values: { [Data.DATA2]: 2 } }]); + expect(ok.ok).toBe(true); + }); + + it('leaves DIRTY NULL on events and calendars the sync adapter inserts', async () => { + const fake = new FakeDeviceProviders(); + const { port, calendarId, eventId } = await calendarWithEvent(fake); + expect(fake.row('calendars', calendarId)![Calendars.DIRTY]).toBeNull(); + expect(fake.row('events', eventId)![Events.DIRTY]).toBeNull(); + expect((await port.query({ table: 'events', columns: ['_id'], where: 'dirty = 0' })).rows).toEqual([]); + expect((await port.query({ table: 'events', columns: ['_id'], where: 'dirty = 1' })).rows).toEqual([]); + }); + + it('expects exactly the id row unless told otherwise, and applies where to it', async () => { + const fake = new FakeDeviceProviders(); + const { port, rawId } = await contactWithEmail(fake); + const gone = await port.applyBatch([{ op: 'delete', table: 'raw_contacts', id: rawId + 1000 }]); + expect(gone).toMatchObject({ ok: false, reason: 'assert' }); + const tolerated = await port.applyBatch([{ op: 'delete', table: 'raw_contacts', id: rawId + 1000, expectCount: 0 }]); + expect(tolerated).toEqual({ ok: true, results: [{ count: 0 }] }); + const mismatch = await port.applyBatch([ + { op: 'update', table: 'raw_contacts', id: rawId, where: `${RawContacts.DIRTY} = 1`, values: { [RawContacts.SYNC1]: 'x' } }, + ]); + expect(mismatch).toMatchObject({ ok: false, reason: 'assert' }); + const matched = await port.applyBatch([ + { op: 'update', table: 'raw_contacts', id: rawId, where: `${RawContacts.DIRTY} = 0`, values: { [RawContacts.SYNC1]: 'x' } }, + ]); + expect(matched).toEqual({ ok: true, results: [{ count: 1 }] }); + }); + + it('refuses _id in where on calendar child tables, and updates extended properties by id only', async () => { + const fake = new FakeDeviceProviders(); + const { port, eventId, attendeeId, propertyId } = await calendarWithEvent(fake); + await expect( + port.query({ table: 'attendees', columns: ['event_id'], where: '_id = ?', args: [attendeeId] }), + ).rejects.toMatchObject({ code: 'provider' }); + const byWhere = await port.applyBatch([{ op: 'delete', table: 'attendees', where: '_id = ?', args: [attendeeId] }]); + expect(byWhere).toMatchObject({ ok: false, reason: 'provider' }); + const byId = await port.applyBatch([{ op: 'delete', table: 'attendees', id: attendeeId }]); + expect(byId).toEqual({ ok: true, results: [{ count: 1 }] }); + const propertiesByWhere = await port.applyBatch([ + { op: 'update', table: 'extended_properties', where: 'event_id = ?', args: [eventId], values: { value: '2' } }, + ]); + expect(propertiesByWhere).toMatchObject({ ok: false, reason: 'provider' }); + const propertiesById = await port.applyBatch([ + { op: 'update', table: 'extended_properties', id: propertyId, values: { value: '2' } }, + ]); + expect(propertiesById.ok).toBe(true); + }); + + it('fails with assert for a parent that is gone and with scope for one of another account', async () => { + const fake = new FakeDeviceProviders(); + const mine = await calendarWithEvent(fake); + const theirs = await calendarWithEvent(fake, OTHER); + const gone = await mine.port.applyBatch([ + { op: 'insert', table: 'reminders', values: { event_id: theirs.eventId + 1000, minutes: 10 } }, + ]); + expect(gone).toMatchObject({ ok: false, reason: 'assert' }); + const foreign = await mine.port.applyBatch([ + { op: 'insert', table: 'reminders', values: { event_id: theirs.eventId, minutes: 10 } }, + ]); + expect(foreign).toMatchObject({ ok: false, reason: 'scope' }); + const unnamed = await mine.port.applyBatch([{ op: 'insert', table: 'reminders', values: { minutes: 10 } }]); + expect(unnamed).toMatchObject({ ok: false, reason: 'scope' }); + const moveAway = await mine.port.applyBatch([ + { op: 'update', table: 'events', id: mine.eventId, values: { [Events.CALENDAR_ID]: theirs.calendarId } }, + ]); + expect(moveAway).toMatchObject({ ok: false, reason: 'scope' }); + const wrongRef = await mine.port.applyBatch([ + { op: 'insert', table: 'colors', values: { color_type: 0, color_index: '1', color: 1 } }, + { op: 'insert', table: 'reminders', values: { minutes: 10 }, refs: { event_id: 0 } }, + ]); + expect(wrongRef).toMatchObject({ ok: false, reason: 'scope' }); + const leaves = await mine.port.applyBatch([ + { op: 'update', table: 'calendars', id: mine.calendarId, values: { [Calendars.ACCOUNT_NAME]: OTHER } }, + ]); + expect(leaves).toMatchObject({ ok: false, reason: 'scope' }); + }); + + it('wants syncState last and never at a yield point', async () => { + const fake = new FakeDeviceProviders(); + const { port, rawId } = await contactWithEmail(fake); + const early = await port.applyBatch([ + { op: 'syncState', value: '{}' }, + { op: 'update', table: 'raw_contacts', id: rawId, values: { [RawContacts.SYNC1]: 'x' } }, + ]); + expect(early).toMatchObject({ ok: false, reason: 'scope' }); + const yielding = await port.applyBatch([ + { op: 'update', table: 'raw_contacts', id: rawId, values: { [RawContacts.SYNC1]: 'x' } }, + { op: 'syncState', value: '{}', yieldAllowed: true }, + ]); + expect(yielding).toMatchObject({ ok: false, reason: 'scope' }); + expect(fake.readSyncState(ME, CONTACTS_AUTHORITY)).toBeNull(); + }); + + it('returns no id for a settings insert, which cannot be addressed by id either', async () => { + const fake = new FakeDeviceProviders(); + const port = fake.port(ME, CONTACTS_AUTHORITY); + const res = await port.applyBatch([{ op: 'insert', table: 'settings', values: { ungrouped_visible: 1 } }]); + expect(res).toEqual({ ok: true, results: [{}] }); + const byId = await port.applyBatch([{ op: 'delete', table: 'settings', id: 1 }]); + expect(byId).toMatchObject({ ok: false, reason: 'provider' }); + }); + + it('counts the op at a yield point before resetting, like ContactsProvider', async () => { + const fake = new FakeDeviceProviders(); + const { port, rawId } = await contactWithEmail(fake); + const update = (yieldAllowed = false): ProviderOp => ({ + op: 'update', + table: 'raw_contacts', + id: rawId, + values: { [RawContacts.SYNC2]: 'y' }, + yieldAllowed, + }); + // 499 ops, then a yield point: fine. + const fits = [...Array.from({ length: 498 }, () => update()), update(true), ...Array.from({ length: 10 }, () => update())]; + expect((await port.applyBatch(fits)).ok).toBe(true); + // The yield point is the 500th op since the start: refused. + const late = [...Array.from({ length: 499 }, () => update()), update(true)]; + expect(await port.applyBatch(late)).toMatchObject({ ok: false, reason: 'provider' }); + }); +}); diff --git a/src/device-sync/__tests__/fakes/fake-provider.ts b/src/device-sync/__tests__/fakes/fake-provider.ts index c642f0c6..3a611bb2 100644 --- a/src/device-sync/__tests__/fakes/fake-provider.ts +++ b/src/device-sync/__tests__/fakes/fake-provider.ts @@ -7,7 +7,10 @@ * "Device data model", and the AOSP facts quoted there): * * - Sync-adapter writes never set DIRTY; app writes do (data rows, events and - * their attendees/reminders). The provider never clears DIRTY. + * their attendees/reminders). The provider never clears DIRTY. Events and + * calendars the sync adapter inserts have DIRTY NULL unless it writes one. + * - Contacts `data1`..`data14` are TEXT columns: numbers are stored, and read + * back, as their text. * - RawContacts.VERSION goes up on every data-row insert, update or delete, * whoever makes it, and on a DELETED change; never for DIRTY, SOURCE_ID or * SYNC1-4. Groups bump VERSION on every update. @@ -23,6 +26,15 @@ * `PD`, while `PT0S`-style all-day durations, NULL STATUS updates, * SELF_ATTENDEE_STATUS updates and RRULEs with unknown parts fail the batch. * - Scoping: rows of other accounts are invisible and untouchable. + * - The native bridge's rules (android/.../sync/ProviderOps.kt and + * ProviderScope.kt): an op with `id` expects exactly one row unless it says + * otherwise, and applies `where` too; attendees, reminders and extended + * properties refuse `_id` in `where` (ambiguous in CalendarProvider's + * join), and extended properties are updated by `id` only; a named parent, + * exception master or group that is gone fails with `assert`, one of + * another account with `scope`; `syncState` must be the batch's last op + * and no yield point; settings inserts return no id. Query and photo + * errors carry the native rejection `code`. * * Pure TypeScript, for vitest only. */ @@ -106,8 +118,93 @@ class FakeProviderError extends Error { } } +/** A rejected query or photo read, with the `code` the native module rejects with. */ +function rejection(code: BatchFailure, message: string): Error { + return Object.assign(new Error(message), { code }); +} + const isEmpty = (v: Cell | undefined) => v === null || v === undefined || v === ''; +/** The parent row each row of a table belongs to. */ +const PARENTS: Partial> = { + data: { column: Data.RAW_CONTACT_ID, table: 'raw_contacts' }, + events: { column: Events.CALENDAR_ID, table: 'calendars' }, + attendees: { column: Attendees.EVENT_ID, table: 'events' }, + reminders: { column: Reminders.EVENT_ID, table: 'events' }, + extended_properties: { column: ExtendedProperties.EVENT_ID, table: 'events' }, +}; + +/** Tables read through CalendarProvider's join, where a bare `_id` is ambiguous. */ +const JOINED_CHILD_TABLES: ReadonlySet = new Set(['attendees', 'reminders', 'extended_properties']); + +/** Tables that carry the account in `account_name`/`account_type`. */ +const ACCOUNT_TABLES: ReadonlySet = new Set(['raw_contacts', 'groups', 'settings', 'calendars', 'colors']); + +/** + * The table a written column points into when the row it names must be the + * account's too (ProviderOps.kt `referencedTable`): a parent, an exception's + * master, a membership's group. + */ +function referencedTable(table: ProviderTable, column: string, values: WriteRow): ProviderTable | null { + if (PARENTS[table]?.column === column) return PARENTS[table].table; + if (table === 'events' && column === Events.ORIGINAL_ID) return 'events'; + if (table === 'data' && column === Data.DATA1 && values[Data.MIMETYPE] === MimeType.GROUP_MEMBERSHIP) return 'groups'; + return null; +} + +const TEXT_DATA_COLUMNS: ReadonlySet = new Set(numbered('data', 1, 14)); + +/** ContactsProvider's `data1`..`data14` have TEXT affinity: SQLite stores numbers as their text. */ +function textifyData(row: StoredRow): void { + for (const column of TEXT_DATA_COLUMNS) { + const v = row[column]; + if (typeof v === 'number') row[column] = String(v); + } +} + +/** + * What the native side refuses before touching the provider (ProviderOps.kt + * `parseBatch`): a misplaced syncState, bad back-references, inserts that + * name no parent, and contacts batches without enough yield points. + */ +function refuseBatch(authority: Authority, ops: ProviderOp[]): { reason: BatchFailure; message: string } | null { + for (let i = 0; i < ops.length; i++) { + const op = ops[i]; + if (op.op !== 'syncState') continue; + if (i !== ops.length - 1) return { reason: 'scope', message: `op ${i}: syncState must be the last op of its batch` }; + if (op.yieldAllowed) return { reason: 'scope', message: `op ${i}: syncState must not be a yield point` }; + } + for (let i = 0; i < ops.length; i++) { + const op = ops[i]; + if (op.op !== 'insert') continue; + for (const [column, ref] of Object.entries(op.refs ?? {})) { + const target = ops[ref]; + if (!Number.isInteger(ref) || ref < 0 || ref >= i || target?.op !== 'insert' || target.table === 'settings') { + return { reason: 'provider', message: `op ${i}: ${column} refers to op ${ref}, which is not an earlier insert with a row id` }; + } + const expected = referencedTable(op.table, column, op.values); + if (expected && target.table !== expected) { + return { reason: 'scope', message: `op ${i}: ${column} must refer to a ${expected} insert, not ${target.table}` }; + } + } + const parent = PARENTS[op.table]; + if (parent && !(parent.column in (op.refs ?? {})) && (op.values[parent.column] ?? null) === null) { + return { reason: 'scope', message: `op ${i}: a ${op.table} insert must name its ${parent.column}` }; + } + } + if (authority === CONTACTS_AUTHORITY) { + // ContactsProvider counts an op before resetting at a yield point. + let sinceYield = 0; + for (let i = 0; i < ops.length; i++) { + if (++sinceYield >= 500) { + return { reason: 'provider', message: 'Too many content provider operations between yield points' }; + } + if (i > 0 && ops[i].yieldAllowed) sinceYield = 0; + } + } + return null; +} + /** AOSP calendarcommon2 Duration.parse: `[+-]P` then ``, `T` ignored. */ export function aospDurationValid(value: string): boolean { return /^[+-]?P(T?\d+[WDHMS])*T?$/.test(value) && /\d/.test(value); @@ -408,15 +505,10 @@ export class FakeDeviceProviders { record(false); return { ok: false, reason: 'tooLarge', message: 'TransactionTooLargeException' }; } - if (authority === CONTACTS_AUTHORITY) { - let sinceYield = 0; - for (let i = 0; i < ops.length; i++) { - if (i > 0 && ops[i].yieldAllowed) sinceYield = 0; - if (++sinceYield >= 500) { - record(false); - return { ok: false, reason: 'provider', message: 'Too many content provider operations between yield points' }; - } - } + const refusal = refuseBatch(authority, ops); + if (refusal) { + record(false); + return { ok: false, ...refusal }; } for (const hook of this.beforeBatch.splice(0)) hook(); @@ -469,6 +561,13 @@ function checkColumns(table: ProviderTable, columns: Iterable, write: bo } } +/** SQLite on CalendarProvider's attendee join: "ambiguous column name: _id". Address those rows with `id`. */ +function checkNoBareId(table: ProviderTable, where: string | undefined): void { + if (JOINED_CHILD_TABLES.has(table) && whereColumns(where).includes('_id')) { + throw new FakeProviderError('provider', `ambiguous column name: _id (address ${table} rows with id)`); + } +} + class FakePort implements ProviderPort { constructor( private readonly fake: FakeDeviceProviders, @@ -478,12 +577,13 @@ class FakePort implements ProviderPort { async query(q: ProviderQuery): Promise { if (!tablesFor(this.authority).includes(q.table)) { - throw new Error(`scope: table ${q.table} is not part of ${this.authority}`); + throw rejection('scope', `table ${q.table} is not part of ${this.authority}`); } try { checkColumns(q.table, [...q.columns, ...whereColumns(q.where)], false); + checkNoBareId(q.table, q.where); } catch (e) { - throw new Error((e as Error).message); + throw rejection('provider', (e as Error).message); } const pred = compileWhere(q.where, q.args); const order = compileOrderBy(q.orderBy); @@ -507,7 +607,9 @@ class FakePort implements ProviderPort { async readPhoto(rawContactId: number, _maxPx: number): Promise { const rc = this.fake.table('raw_contacts').get(rawContactId); - if (!rc || !this.fake.owned('raw_contacts', rc, this.accountName)) throw new Error('scope: raw contact of another account'); + if (!rc || !this.fake.owned('raw_contacts', rc, this.accountName)) { + throw rejection('scope', `Raw contact ${rawContactId} is not part of the account`); + } for (const row of this.fake.table('data').values()) { if (row[Data.RAW_CONTACT_ID] === rawContactId && row[Data.MIMETYPE] === MimeType.PHOTO) { const fileId = row[Data.DATA14] === null ? null : Number(row[Data.DATA14]); @@ -545,22 +647,70 @@ class BatchApplier { if (!tablesFor(this.authority).includes(table)) this.scopeError(`table ${table} is not part of ${this.authority}`); } + /** The rows an op addresses: the `id` row if it also matches `where`, or every row of the account matching `where`. */ private target(table: ProviderTable, id: number | undefined, where: string | undefined, args: Array | undefined): number[] { + if (id !== undefined && table === 'settings') throw new FakeProviderError('provider', 'settings rows have no id'); + checkColumns(table, whereColumns(where), false); + checkNoBareId(table, where); + const pred = compileWhere(where, args); const rows = this.fake.table(table); if (id !== undefined) { const row = rows.get(id); if (!row) return []; if (!this.fake.owned(table, row, this.account)) this.scopeError(`${table} ${id} belongs to another account`); - return [id]; + return pred((c) => this.fake.readCell(table, row, c)) ? [id] : []; } - checkColumns(table, whereColumns(where), false); - const pred = compileWhere(where, args); return [...rows] .filter(([, row]) => this.fake.owned(table, row, this.account)) .filter(([, row]) => pred((c) => this.fake.readCell(table, row, c))) .map(([rowId]) => rowId); } + /** An op that names one row by id expects exactly that row unless it says otherwise. */ + private checkCount(ids: number[], id: number | undefined, expectCount: number | undefined): void { + const expected = expectCount ?? (id !== undefined ? 1 : undefined); + if (expected !== undefined && ids.length !== expected) { + throw new FakeProviderError('assert', `wrong number of rows: ${ids.length}, expected ${expected}`); + } + } + + /** Account columns may only restate the account, and only on the tables that have them. */ + private checkAccountColumns(table: ProviderTable, values: WriteRow): void { + for (const column of ['account_name', 'account_type', 'data_set']) { + if (!(column in values)) continue; + const cell = values[column]; + const restates = + ACCOUNT_TABLES.has(table) && + (column === 'account_name' ? cell === this.account : column === 'account_type' ? cell === this.fake.accountType : cell === null); + if (!restates) this.scopeError(`${table}.${column} would leave the account`); + } + } + + /** + * Parents, masters and groups a written column names must be rows of the + * account that still exist: a gone one fails with `assert` (the engine + * re-reads), one of another account with `scope`. `skip`: columns filled by + * back-references, checked before the batch ran. + */ + private checkReferences(table: ProviderTable, values: WriteRow, skip: ReadonlySet = new Set()): void { + for (const [column, cell] of Object.entries(values)) { + if (skip.has(column)) continue; + const referenced = referencedTable(table, column, values); + if (!referenced) continue; + if (cell === null) { + if (PARENTS[table]?.column === column) this.scopeError(`${table}.${column} must name a ${referenced} row`); + continue; + } + const id = typeof cell === 'number' ? cell : typeof cell === 'string' && /^\d+$/.test(cell) ? Number(cell) : NaN; + if (!Number.isInteger(id)) throw new FakeProviderError('provider', `${table}.${column} must be a row id`); + const row = this.fake.table(referenced).get(id); + if (!row) throw new FakeProviderError('assert', `${table}.${column} names ${referenced} ${id}, which no longer exists`); + if (!this.fake.owned(referenced, row, this.account)) { + this.scopeError(`${table}.${column} names ${referenced} ${id} of another account`); + } + } + } + private values(table: ProviderTable, values: WriteRow): StoredRow { checkColumns(table, Object.keys(values), true); const out: StoredRow = {}; @@ -603,30 +753,21 @@ class BatchApplier { } private insert(table: ProviderTable, values: WriteRow, refs: Record | undefined, index: number): OpResult { + this.checkAccountColumns(table, values); + this.checkReferences(table, values, new Set(Object.keys(refs ?? {}))); const row = this.values(table, values); + // Back-references win over values, as ContentProviderOperation applies them last. for (const [column, ref] of Object.entries(refs ?? {})) { if (ref >= index || this.results[ref]?.id === undefined) throw new FakeProviderError('provider', `bad back-reference ${ref}`); row[column] = this.results[ref].id!; } const id = this.fake.allocId(); const account = { account_name: this.account, account_type: this.fake.accountType }; - const parentOwned = (parentTable: ProviderTable, parentId: Cell) => { - const parent = this.fake.table(parentTable).get(Number(parentId)); - if (!parent || !this.fake.owned(parentTable, parent, this.account)) { - this.scopeError(`${table} insert names ${parentTable} ${String(parentId)} outside the account`); - } - return parent; - }; - const accountMismatch = (r: StoredRow) => - (r.account_name !== undefined && r.account_name !== this.account) || - (r.account_type !== undefined && r.account_type !== this.fake.accountType); switch (table) { case 'raw_contacts': - if (accountMismatch(row)) this.scopeError('raw contact of another account'); Object.assign(row, { ...defaults(RAW_CONTACT_DEFAULTS), ...row, ...account }); break; case 'groups': - if (accountMismatch(row)) this.scopeError('group of another account'); Object.assign(row, { ...defaults(GROUP_DEFAULTS), ...row, ...account }); break; case 'settings': { @@ -634,21 +775,22 @@ class BatchApplier { if (s.account_name === this.account && s.account_type === this.fake.accountType) this.fake.table('settings').delete(sid); } Object.assign(row, { [ContactsSettings.UNGROUPED_VISIBLE]: 0, [ContactsSettings.SHOULD_SYNC]: 1, ...row, ...account }); - break; + this.fake.table(table).set(id, row); + // Keyed by the account: the insert answers without a row id. + return {}; } case 'data': { - parentOwned('raw_contacts', row[Data.RAW_CONTACT_ID]); row[Data.RAW_CONTACT_ID] = Number(row[Data.RAW_CONTACT_ID]); if (isEmpty(row[Data.MIMETYPE])) throw new FakeProviderError('provider', 'data row without mimetype'); row[Data.DATA_VERSION] = 0; if (row[Data.MIMETYPE] === MimeType.GROUP_MEMBERSHIP) this.resolveMembership(row); this.processPhoto(row); this.fake.normalizeData(row); + textifyData(row); this.insertedDataFor.add(Number(row[Data.RAW_CONTACT_ID])); break; } case 'calendars': { - if (accountMismatch(row)) this.scopeError('calendar of another account'); for (const c of [Calendars.NAME, Calendars.CALENDAR_DISPLAY_NAME, Calendars.CALENDAR_COLOR, Calendars.CALENDAR_ACCESS_LEVEL, Calendars.OWNER_ACCOUNT]) { if (isEmpty(row[c])) throw new FakeProviderError('provider', `calendar insert without ${c}`); } @@ -656,8 +798,9 @@ class BatchApplier { break; } case 'events': { - const calendar = parentOwned('calendars', row[Events.CALENDAR_ID])!; - row[Events.CALENDAR_ID] = Number(row[Events.CALENDAR_ID]); + const calendarId = Number(row[Events.CALENDAR_ID]); + row[Events.CALENDAR_ID] = calendarId; + const calendar = this.fake.table('calendars').get(calendarId)!; Object.assign(row, { ...defaults(EVENT_DEFAULTS), ...row }); this.fake.fixEvent(row, true); if (isEmpty(row[Events.ORGANIZER])) row[Events.ORGANIZER] = calendar[Calendars.OWNER_ACCOUNT]; @@ -676,11 +819,9 @@ class BatchApplier { case 'attendees': case 'reminders': case 'extended_properties': - parentOwned('events', row.event_id); row.event_id = Number(row.event_id); break; case 'colors': - if (accountMismatch(row)) this.scopeError('color of another account'); Object.assign(row, { ...row, ...account }); break; } @@ -733,10 +874,14 @@ class BatchApplier { values: WriteRow, expectCount: number | undefined, ): OpResult { - const ids = this.target(table, id, where, args); - if (expectCount !== undefined && ids.length !== expectCount) { - throw new FakeProviderError('assert', `wrong number of rows: ${ids.length}`); + this.checkAccountColumns(table, values); + this.checkReferences(table, values); + if (table === 'extended_properties' && (id === undefined || where !== undefined)) { + // CalendarProvider updates them only through the item URI, which takes no selection. + throw new FakeProviderError('provider', 'extended properties can only be updated by id'); } + const ids = this.target(table, id, where, args); + this.checkCount(ids, id, expectCount); const patch = this.values(table, values); for (const rowId of ids) { const row = this.fake.table(table).get(rowId)!; @@ -757,6 +902,7 @@ class BatchApplier { row[Data.DATA_VERSION] = Number(row[Data.DATA_VERSION] ?? 0) + 1; this.processPhoto(row); this.fake.normalizeData(row); + textifyData(row); this.fake.bumpVersion(Number(row[Data.RAW_CONTACT_ID])); break; case 'events': @@ -781,9 +927,7 @@ class BatchApplier { expectCount: number | undefined, ): OpResult { const ids = this.target(table, id, where, args); - if (expectCount !== undefined && ids.length !== expectCount) { - throw new FakeProviderError('assert', `wrong number of rows: ${ids.length}`); - } + this.checkCount(ids, id, expectCount); for (const rowId of ids) { const row = this.fake.table(table).get(rowId); if (!row) continue; @@ -802,9 +946,7 @@ class BatchApplier { expectCount: number | undefined, ): OpResult { const ids = this.target(table, id, where, args); - if (expectCount !== undefined && ids.length !== expectCount) { - throw new FakeProviderError('assert', `wrong number of rows: ${ids.length}`); - } + this.checkCount(ids, id, expectCount); if (values) { checkColumns(table, Object.keys(values), false); for (const rowId of ids) { @@ -859,11 +1001,12 @@ const CALENDAR_DEFAULTS: StoredRow = { [Calendars.MAX_REMINDERS]: 5, [Calendars.CAN_ORGANIZER_RESPOND]: 1, [Calendars.CAN_MODIFY_TIME_ZONE]: 1, - [Calendars.DIRTY]: 0, + // No column default: NULL until someone writes it. + [Calendars.DIRTY]: null, }; const EVENT_DEFAULTS: StoredRow = { - [Events.DIRTY]: 0, + [Events.DIRTY]: null, [Events.DELETED]: 0, [Events.AVAILABILITY]: 0, [Events.ACCESS_LEVEL]: 0, @@ -909,6 +1052,7 @@ export class FakeUser { const id = this.fake.allocId(); const row: StoredRow = { ...values, [Data.RAW_CONTACT_ID]: rawContactId, [Data.DATA_VERSION]: 0 }; this.fake.normalizeData(row); + textifyData(row); this.fake.table('data').set(id, row); this.fake.bumpVersion(rawContactId); this.dirtyContact(rawContactId); @@ -922,6 +1066,7 @@ export class FakeUser { Object.assign(row, values); row[Data.DATA_VERSION] = Number(row[Data.DATA_VERSION] ?? 0) + 1; this.fake.normalizeData(row); + textifyData(row); const rcId = Number(row[Data.RAW_CONTACT_ID]); this.fake.bumpVersion(rcId); this.dirtyContact(rcId); diff --git a/src/device-sync/__tests__/native.test.ts b/src/device-sync/__tests__/native.test.ts new file mode 100644 index 00000000..37b83ffd --- /dev/null +++ b/src/device-sync/__tests__/native.test.ts @@ -0,0 +1,342 @@ +import { afterEach, describe, expect, it, vi } from 'vitest'; +import { NativeEventEmitter, NativeModules, Platform } from 'react-native'; +import { + clearSyncProblem, + createProviderPort, + ensureAndroidAccount, + finishRun, + getDeviceSyncModule, + getSyncSettings, + isDeviceSyncAvailable, + isRunCancelled, + listAndroidAccounts, + onAccountsChanged, + openAccountSettings, + removeAndroidAccount, + requestSync, + setPeriodicSync, + setPushRoutes, + setSyncEnabled, + showSyncProblem, +} from '../native'; +import { + ACCOUNTS_CHANGED_EVENT, + CALENDAR_AUTHORITY, + CONTACTS_AUTHORITY, + type AccountSyncSettings, + type Authority, + type ProviderOp, + type RunReport, +} from '../types'; + +const modules = NativeModules as Record; +const ACCOUNT = 'alice@example.org'; + +const SETTINGS: AccountSyncSettings = { + masterAutomatic: true, + authorities: { + [CONTACTS_AUTHORITY]: { syncable: 1, automatic: true, periodicSeconds: 3600, active: false, pending: false }, + [CALENDAR_AUTHORITY]: { syncable: 1, automatic: false, periodicSeconds: 0, active: false, pending: true }, + }, +}; + +/** Installs a BulwarkDeviceSync module whose methods resolve like the Kotlin one. */ +function installModule(overrides: Record = {}) { + const module = { + getInfo: vi.fn(async () => ({ accountType: 'com.anonymous.bulwarkmobile.account', sdkInt: 34 })), + listAccounts: vi.fn(async () => [{ name: ACCOUNT, registryId: 'alice@example.org@mail.example.org' }]), + ensureAccount: vi.fn(async () => true), + removeAccount: vi.fn(async () => true), + getSyncSettings: vi.fn(async () => SETTINGS), + setSyncEnabled: vi.fn(async () => undefined), + setPeriodicSync: vi.fn(async () => undefined), + requestSync: vi.fn(async () => undefined), + openAccountSettings: vi.fn(async () => undefined), + finishRun: vi.fn(async () => true), + isRunCancelled: vi.fn(async () => false), + setPushRoutes: vi.fn(async () => undefined), + query: vi.fn(async () => JSON.stringify({ columns: ['_id', 'sourceid'], rows: [[1, 'c/a'], [2, null]] })), + applyBatch: vi.fn(async () => JSON.stringify({ ok: true, results: [{ id: 5 }, { count: 1 }] })), + readSyncState: vi.fn(async (): Promise => null), + readPhoto: vi.fn(async (): Promise => null), + showSyncProblem: vi.fn(async () => undefined), + clearSyncProblem: vi.fn(async () => undefined), + addListener: vi.fn(), + removeListeners: vi.fn(), + ...overrides, + }; + modules.BulwarkDeviceSync = module; + return module; +} + +const TWO_OPS: ProviderOp[] = [ + { op: 'insert', table: 'raw_contacts', values: { sourceid: null } }, + { op: 'update', table: 'data', id: 7, values: { data1: 'x' } }, +]; + +afterEach(() => { + delete modules.BulwarkDeviceSync; + (Platform as { OS: string }).OS = 'android'; + vi.restoreAllMocks(); +}); + +describe('module lookup', () => { + it('reads NativeModules on every call instead of caching the module', () => { + expect(getDeviceSyncModule()).toBeNull(); + const first = installModule(); + expect(getDeviceSyncModule()).toBe(first); + const second = installModule(); + expect(getDeviceSyncModule()).toBe(second); + delete modules.BulwarkDeviceSync; + expect(getDeviceSyncModule()).toBeNull(); + }); + + it('is off on iOS even if a module of that name exists', () => { + installModule(); + (Platform as { OS: string }).OS = 'ios'; + expect(getDeviceSyncModule()).toBeNull(); + expect(isDeviceSyncAvailable()).toBe(false); + }); + + it('is available on Android with the module', () => { + expect(isDeviceSyncAvailable()).toBe(false); + installModule(); + expect(isDeviceSyncAvailable()).toBe(true); + }); + + it('rejects everything, and subscribes nothing, without the module', async () => { + const calls: Array<() => Promise> = [ + () => listAndroidAccounts(), + () => ensureAndroidAccount(ACCOUNT, 'r'), + () => removeAndroidAccount(ACCOUNT), + () => getSyncSettings(ACCOUNT), + () => setSyncEnabled(ACCOUNT, CONTACTS_AUTHORITY, true), + () => setPeriodicSync(ACCOUNT, CONTACTS_AUTHORITY, 3600), + () => requestSync(ACCOUNT, CONTACTS_AUTHORITY), + () => openAccountSettings(ACCOUNT), + () => finishRun('run', {} as RunReport), + () => isRunCancelled('run'), + () => setPushRoutes({}), + () => showSyncProblem(ACCOUNT, 't', 'x', 'bulwarkmobile://login'), + () => clearSyncProblem(ACCOUNT), + () => createProviderPort(ACCOUNT, CONTACTS_AUTHORITY).query({ table: 'raw_contacts', columns: ['_id'] }), + () => createProviderPort(ACCOUNT, CONTACTS_AUTHORITY).applyBatch([]), + () => createProviderPort(ACCOUNT, CONTACTS_AUTHORITY).readSyncState(), + () => createProviderPort(ACCOUNT, CONTACTS_AUTHORITY).readPhoto(1, 512), + ]; + for (const call of calls) { + await expect(call()).rejects.toMatchObject({ code: 'unavailable' }); + } + const unsubscribe = onAccountsChanged(() => undefined); + expect(() => unsubscribe()).not.toThrow(); + }); +}); + +describe('createProviderPort', () => { + it('is bound to its account and authority', () => { + const port = createProviderPort(ACCOUNT, CALENDAR_AUTHORITY); + expect(port.accountName).toBe(ACCOUNT); + expect(port.authority).toBe(CALENDAR_AUTHORITY); + }); + + it('sends a query as JSON and parses the rows', async () => { + const module = installModule(); + const port = createProviderPort(ACCOUNT, CONTACTS_AUTHORITY); + const rows = await port.query({ table: 'raw_contacts', columns: ['_id', 'sourceid'], where: 'dirty = ?', args: [1] }); + expect(rows).toEqual({ columns: ['_id', 'sourceid'], rows: [[1, 'c/a'], [2, null]] }); + expect(module.query).toHaveBeenCalledWith( + ACCOUNT, + CONTACTS_AUTHORITY, + JSON.stringify({ table: 'raw_contacts', columns: ['_id', 'sourceid'], where: 'dirty = ?', args: [1] }), + ); + }); + + it('throws on query replies that are not ProviderRows', async () => { + const port = createProviderPort(ACCOUNT, CONTACTS_AUTHORITY); + const q = { table: 'raw_contacts' as const, columns: ['_id'] }; + for (const reply of [ + 'not json', + JSON.stringify({ columns: ['_id'] }), + JSON.stringify({ columns: ['_id'], rows: [[1, 2]] }), + JSON.stringify({ columns: ['_id'], rows: [[{ b64: 'AA' }]] }), + JSON.stringify({ columns: [7], rows: [] }), + 42, + ]) { + installModule({ query: vi.fn(async () => reply) }); + await expect(port.query(q)).rejects.toThrow(/BulwarkDeviceSync\.query/); + } + }); + + it('passes native rejections through with their code', async () => { + const scope = Object.assign(new Error('Table instances is not part of com.android.calendar'), { code: 'scope' }); + installModule({ query: vi.fn(async () => Promise.reject(scope)) }); + await expect( + createProviderPort(ACCOUNT, CALENDAR_AUTHORITY).query({ table: 'events', columns: ['_id'] }), + ).rejects.toBe(scope); + }); + + it('sends ops as JSON and returns the batch result', async () => { + const module = installModule(); + const result = await createProviderPort(ACCOUNT, CONTACTS_AUTHORITY).applyBatch(TWO_OPS); + expect(result).toEqual({ ok: true, results: [{ id: 5 }, { count: 1 }] }); + expect(module.applyBatch).toHaveBeenCalledWith(ACCOUNT, CONTACTS_AUTHORITY, JSON.stringify(TWO_OPS)); + }); + + it('returns failed batches as results', async () => { + installModule({ + applyBatch: vi.fn(async () => JSON.stringify({ ok: false, reason: 'assert', message: 'Expected 1 rows but actual 0' })), + }); + await expect(createProviderPort(ACCOUNT, CONTACTS_AUTHORITY).applyBatch(TWO_OPS)).resolves.toEqual({ + ok: false, + reason: 'assert', + message: 'Expected 1 rows but actual 0', + }); + }); + + it('throws on batch replies that are not BatchResults', async () => { + const port = createProviderPort(ACCOUNT, CONTACTS_AUTHORITY); + for (const reply of [ + JSON.stringify({ ok: false, reason: 'weird', message: 'x' }), + JSON.stringify({ ok: true, results: [{ id: 'five' }, {}] }), + JSON.stringify({ ok: true, results: [{ id: 5 }] }), + JSON.stringify({ ok: true }), + '[]', + ]) { + installModule({ applyBatch: vi.fn(async () => reply) }); + await expect(port.applyBatch(TWO_OPS)).rejects.toThrow(/BulwarkDeviceSync\.applyBatch/); + } + }); + + it('reads the SyncState as text', async () => { + const module = installModule(); + const port = createProviderPort(ACCOUNT, CALENDAR_AUTHORITY); + await expect(port.readSyncState()).resolves.toBeNull(); + module.readSyncState.mockResolvedValueOnce('{"v":1}'); + await expect(port.readSyncState()).resolves.toBe('{"v":1}'); + expect(module.readSyncState).toHaveBeenCalledWith(ACCOUNT, CALENDAR_AUTHORITY); + }); + + it('reads photos of contacts only', async () => { + const module = installModule(); + const port = createProviderPort(ACCOUNT, CONTACTS_AUTHORITY); + await expect(port.readPhoto(3, 512)).resolves.toBeNull(); + module.readPhoto.mockResolvedValueOnce(JSON.stringify({ jpegBase64: '/9j/', fileId: 12 })); + await expect(port.readPhoto(3, 512)).resolves.toEqual({ jpegBase64: '/9j/', fileId: 12 }); + expect(module.readPhoto).toHaveBeenLastCalledWith(ACCOUNT, 3, 512); + module.readPhoto.mockResolvedValueOnce(JSON.stringify({ jpegBase64: '/9j/' })); + await expect(port.readPhoto(3, 512)).rejects.toThrow(/readPhoto/); + await expect(createProviderPort(ACCOUNT, CALENDAR_AUTHORITY).readPhoto(3, 512)).rejects.toThrow(/contacts/); + }); +}); + +describe('accounts and sync settings', () => { + it('passes account calls through', async () => { + const module = installModule(); + await expect(listAndroidAccounts()).resolves.toEqual([{ name: ACCOUNT, registryId: 'alice@example.org@mail.example.org' }]); + await expect(ensureAndroidAccount(ACCOUNT, 'reg')).resolves.toBe(true); + expect(module.ensureAccount).toHaveBeenCalledWith(ACCOUNT, 'reg'); + await expect(removeAndroidAccount(ACCOUNT)).resolves.toBe(true); + expect(module.removeAccount).toHaveBeenCalledWith(ACCOUNT); + await expect(getSyncSettings(ACCOUNT)).resolves.toEqual(SETTINGS); + await setSyncEnabled(ACCOUNT, CALENDAR_AUTHORITY, false); + expect(module.setSyncEnabled).toHaveBeenCalledWith(ACCOUNT, CALENDAR_AUTHORITY, false); + await openAccountSettings(ACCOUNT); + expect(module.openAccountSettings).toHaveBeenCalledWith(ACCOUNT); + }); + + it('lets a registry id conflict reject with its code', async () => { + const conflict = Object.assign(new Error('belongs to another app account'), { code: 'conflict' }); + installModule({ ensureAccount: vi.fn(async () => Promise.reject(conflict)) }); + await expect(ensureAndroidAccount(ACCOUNT, 'other')).rejects.toMatchObject({ code: 'conflict' }); + }); + + it('sends whole seconds for periodic syncs and refuses nonsense', async () => { + const module = installModule(); + await setPeriodicSync(ACCOUNT, CONTACTS_AUTHORITY, 899.6); + expect(module.setPeriodicSync).toHaveBeenCalledWith(ACCOUNT, CONTACTS_AUTHORITY, 900); + await setPeriodicSync(ACCOUNT, CONTACTS_AUTHORITY, 0); + expect(module.setPeriodicSync).toHaveBeenLastCalledWith(ACCOUNT, CONTACTS_AUTHORITY, 0); + await expect(setPeriodicSync(ACCOUNT, CONTACTS_AUTHORITY, -1)).rejects.toBeInstanceOf(RangeError); + await expect(setPeriodicSync(ACCOUNT, CONTACTS_AUTHORITY, Number.NaN)).rejects.toBeInstanceOf(RangeError); + }); + + it('sends sync request options as JSON', async () => { + const module = installModule(); + await requestSync(ACCOUNT, CONTACTS_AUTHORITY); + expect(module.requestSync).toHaveBeenLastCalledWith(ACCOUNT, CONTACTS_AUTHORITY, '{}'); + await requestSync(ACCOUNT, CALENDAR_AUTHORITY, { manual: true, expedited: true }); + expect(module.requestSync).toHaveBeenLastCalledWith( + ACCOUNT, + CALENDAR_AUTHORITY, + JSON.stringify({ manual: true, expedited: true }), + ); + }); +}); + +describe('run handshake', () => { + it('sends the report as JSON and returns whether it was accepted', async () => { + const module = installModule({ finishRun: vi.fn(async () => false) }); + const report = { v: 1, runId: 'r1', authority: CONTACTS_AUTHORITY, outcome: 'ok' } as RunReport; + await expect(finishRun('r1', report)).resolves.toBe(false); + expect(module.finishRun).toHaveBeenCalledWith('r1', JSON.stringify(report)); + }); + + it('asks whether a run was cancelled', async () => { + const module = installModule({ isRunCancelled: vi.fn(async () => true) }); + await expect(isRunCancelled('r1')).resolves.toBe(true); + expect(module.isRunCancelled).toHaveBeenCalledWith('r1'); + }); +}); + +describe('push routes and notifications', () => { + it('stores push routes as JSON', async () => { + const module = installModule(); + const routes: Record> = { + c: [{ accountName: ACCOUNT, authorities: [CONTACTS_AUTHORITY, CALENDAR_AUTHORITY] }], + }; + await setPushRoutes(routes); + expect(module.setPushRoutes).toHaveBeenCalledWith(JSON.stringify(routes)); + }); + + it('posts and clears the sync problem notification', async () => { + const module = installModule(); + await showSyncProblem(ACCOUNT, 'Sign in again', 'Contacts stopped syncing', 'bulwarkmobile://login'); + expect(module.showSyncProblem).toHaveBeenLastCalledWith( + ACCOUNT, + 'Sign in again', + 'Contacts stopped syncing', + 'bulwarkmobile://login', + null, + ); + await showSyncProblem(ACCOUNT, 'Erneut anmelden', 'Kontakte', 'bulwarkmobile://login', 'Synchronisierungsprobleme'); + expect(module.showSyncProblem).toHaveBeenLastCalledWith( + ACCOUNT, + 'Erneut anmelden', + 'Kontakte', + 'bulwarkmobile://login', + 'Synchronisierungsprobleme', + ); + await clearSyncProblem(ACCOUNT); + expect(module.clearSyncProblem).toHaveBeenCalledWith(ACCOUNT); + }); + + it('listens for account changes until unsubscribed', () => { + installModule(); + const remove = vi.fn(); + let handler: (() => void) | undefined; + const addListener = vi + .spyOn(NativeEventEmitter.prototype, 'addListener') + .mockImplementation(((event: string, h: () => void) => { + expect(event).toBe(ACCOUNTS_CHANGED_EVENT); + handler = h; + return { remove }; + }) as never); + const listener = vi.fn(); + const unsubscribe = onAccountsChanged(listener); + expect(addListener).toHaveBeenCalledTimes(1); + handler?.(); + expect(listener).toHaveBeenCalledTimes(1); + unsubscribe(); + expect(remove).toHaveBeenCalledTimes(1); + }); +}); diff --git a/src/device-sync/native.ts b/src/device-sync/native.ts new file mode 100644 index 00000000..528fd1e6 --- /dev/null +++ b/src/device-sync/native.ts @@ -0,0 +1,255 @@ +/** + * Typed wrapper around `NativeModules.BulwarkDeviceSync` (#34): the one module + * of device sync that talks to React Native's bridge. The raw contract is + * `DeviceSyncNativeModule` in ./types; docs/device-sync.md ("Native module + * API") explains it. + * + * The native module is looked up on every call and never cached, so a dev + * reload or an APK built before device sync never leaves a stale reference. + * Everything except `getDeviceSyncModule` and `isDeviceSyncAvailable` + * rejects when the module is missing (iOS, or an old APK), and + * `onAccountsChanged` then returns an unsubscribe that does nothing. + * + * Provider data crosses the bridge as JSON strings; replies are checked here, + * so a malformed one fails loudly instead of reaching the engine. + */ +import { NativeEventEmitter, NativeModules, Platform } from 'react-native'; +import { + ACCOUNTS_CHANGED_EVENT, + CONTACTS_AUTHORITY, + type AccountSyncSettings, + type AndroidAccount, + type Authority, + type BatchFailure, + type BatchResult, + type Cell, + type DeviceSyncNativeModule, + type OpResult, + type PhotoData, + type ProviderPort, + type ProviderRows, + type RequestSyncOptions, + type RunReport, +} from './types'; + +export function getDeviceSyncModule(): DeviceSyncNativeModule | null { + if (Platform.OS !== 'android') return null; + return ( + ((NativeModules as Record).BulwarkDeviceSync as DeviceSyncNativeModule | undefined) ?? null + ); +} + +/** Android with a build that carries the native module. */ +export function isDeviceSyncAvailable(): boolean { + return getDeviceSyncModule() !== null; +} + +function requireModule(): DeviceSyncNativeModule { + const module = getDeviceSyncModule(); + if (!module) { + throw Object.assign(new Error('Device sync is not available in this build'), { code: 'unavailable' }); + } + return module; +} + +// ─── Provider ────────────────────────────────────────── + +/** The provider bound to one Android account of our type and one authority. */ +export function createProviderPort(accountName: string, authority: Authority): ProviderPort { + return { + accountName, + authority, + async query(q) { + const json = await requireModule().query(accountName, authority, JSON.stringify(q)); + return toRows(parseReply(json, 'query')); + }, + async applyBatch(ops) { + const json = await requireModule().applyBatch(accountName, authority, JSON.stringify(ops)); + const result = toBatchResult(parseReply(json, 'applyBatch')); + if (result.ok && result.results.length !== ops.length) { + throw malformed('applyBatch', `${result.results.length} results for ${ops.length} ops`); + } + return result; + }, + async readSyncState() { + const state: unknown = await requireModule().readSyncState(accountName, authority); + if (state === null || state === undefined) return null; + if (typeof state !== 'string') throw malformed('readSyncState', 'not text'); + return state; + }, + async readPhoto(rawContactId, maxPx) { + if (authority !== CONTACTS_AUTHORITY) throw new Error('readPhoto reads contacts: this port is for calendars'); + const json: unknown = await requireModule().readPhoto(accountName, rawContactId, maxPx); + return json === null || json === undefined ? null : toPhoto(parseReply(json, 'readPhoto')); + }, + }; +} + +function malformed(method: string, detail: string): Error { + return new Error(`BulwarkDeviceSync.${method} replied with malformed data: ${detail}`); +} + +function parseReply(json: unknown, method: string): unknown { + if (typeof json !== 'string') throw malformed(method, `a ${typeof json}, not JSON text`); + try { + return JSON.parse(json); + } catch (e) { + throw malformed(method, (e as Error).message); + } +} + +const isObject = (v: unknown): v is Record => typeof v === 'object' && v !== null && !Array.isArray(v); +const isCell = (v: unknown): v is Cell => v === null || typeof v === 'string' || typeof v === 'number'; +const isOptionalNumber = (v: unknown) => v === undefined || typeof v === 'number'; + +function toRows(value: unknown): ProviderRows { + if (!isObject(value) || !Array.isArray(value.columns) || !Array.isArray(value.rows)) { + throw malformed('query', 'no columns and rows'); + } + const { columns, rows } = value; + if (!columns.every((c) => typeof c === 'string')) throw malformed('query', 'a column name is not text'); + for (const row of rows) { + if (!Array.isArray(row) || row.length !== columns.length || !row.every(isCell)) { + throw malformed('query', 'a row does not match its columns'); + } + } + return { columns: columns as string[], rows: rows as Cell[][] }; +} + +const BATCH_FAILURES: readonly BatchFailure[] = ['assert', 'tooLarge', 'scope', 'permission', 'provider']; + +function toBatchResult(value: unknown): BatchResult { + if (isObject(value) && value.ok === true && Array.isArray(value.results)) { + const results: OpResult[] = value.results.map((r) => { + if (!isObject(r) || !isOptionalNumber(r.id) || !isOptionalNumber(r.count)) { + throw malformed('applyBatch', 'an op result is not { id?, count? }'); + } + return r as OpResult; + }); + return { ok: true, results }; + } + if ( + isObject(value) && + value.ok === false && + BATCH_FAILURES.includes(value.reason as BatchFailure) && + typeof value.message === 'string' + ) { + return { ok: false, reason: value.reason as BatchFailure, message: value.message }; + } + throw malformed('applyBatch', 'not a BatchResult'); +} + +function toPhoto(value: unknown): PhotoData { + if ( + !isObject(value) || + typeof value.jpegBase64 !== 'string' || + !(value.fileId === null || typeof value.fileId === 'number') + ) { + throw malformed('readPhoto', 'not { jpegBase64, fileId }'); + } + return { jpegBase64: value.jpegBase64, fileId: value.fileId }; +} + +// ─── Accounts and sync settings ──────────────────────── + +export async function listAndroidAccounts(): Promise { + return requireModule().listAccounts(); +} + +/** + * Adds the Android account for an app account. Resolves true when it was + * created, false when it exists for this registry id already; rejects with + * code `conflict` when an account of that name belongs to another one. + */ +export async function ensureAndroidAccount(name: string, registryId: string): Promise { + return requireModule().ensureAccount(name, registryId); +} + +/** Removes the Android account; the providers drop all of its rows. */ +export async function removeAndroidAccount(name: string): Promise { + return requireModule().removeAccount(name); +} + +export async function getSyncSettings(name: string): Promise { + return requireModule().getSyncSettings(name); +} + +export async function setSyncEnabled(name: string, authority: Authority, enabled: boolean): Promise { + await requireModule().setSyncEnabled(name, authority, enabled); +} + +/** Replaces the periodic sync; 0 removes it. Android raises anything shorter than 15 minutes to that. */ +export async function setPeriodicSync(name: string, authority: Authority, seconds: number): Promise { + if (!Number.isFinite(seconds) || seconds < 0) throw new RangeError(`Invalid sync interval: ${seconds}`); + await requireModule().setPeriodicSync(name, authority, Math.round(seconds)); +} + +/** + * Asks SyncManager for a sync. Repeated requests with the same options + * coalesce; one that arrives while the same sync runs makes that run ask for + * another one when it ends. + */ +export async function requestSync(name: string, authority: Authority, options: RequestSyncOptions = {}): Promise { + await requireModule().requestSync(name, authority, JSON.stringify(options)); +} + +/** The account's sync screen in Android Settings (or the list of synced accounts). */ +export async function openAccountSettings(name: string): Promise { + await requireModule().openAccountSettings(name); +} + +// ─── The run handshake ───────────────────────────────── + +/** Hands the report to the waiting adapter; false when the run already ended (deadline, cancel) or is unknown. */ +export async function finishRun(runId: string, report: RunReport): Promise { + return requireModule().finishRun(runId, JSON.stringify(report)); +} + +/** True once the adapter stopped waiting for the run (cancelled, out of time, or unknown). */ +export async function isRunCancelled(runId: string): Promise { + return requireModule().isRunCancelled(runId); +} + +// ─── Push routes and notifications ───────────────────── + +/** + * Tells the native push router which Android accounts each JMAP account feeds, + * so a pushed contact or calendar change requests their syncs without JS. + * Replaces the previous routes. + */ +export async function setPushRoutes( + routes: Record>, +): Promise { + await requireModule().setPushRoutes(JSON.stringify(routes)); +} + +/** + * Posts (or replaces) the account's "sync problem" notification; tapping it + * opens `uri`, a `bulwarkmobile://` deep link. `channelName` names the + * notification channel in the app's language (English when omitted). + */ +export async function showSyncProblem( + accountName: string, + title: string, + text: string, + uri: string, + channelName?: string, +): Promise { + await requireModule().showSyncProblem(accountName, title, text, uri, channelName ?? null); +} + +export async function clearSyncProblem(accountName: string): Promise { + await requireModule().clearSyncProblem(accountName); +} + +/** + * Calls `listener` when an account of our type is added or removed (also in + * Android Settings). A hint only: events without a JS listener are dropped, + * so re-read the accounts on mount and on foreground too. + */ +export function onAccountsChanged(listener: () => void): () => void { + const module = getDeviceSyncModule(); + if (!module) return () => undefined; + const subscription = new NativeEventEmitter(module).addListener(ACCOUNTS_CHANGED_EVENT, () => listener()); + return () => subscription.remove(); +} diff --git a/src/device-sync/types.ts b/src/device-sync/types.ts index ce1f8fb8..d84bbf19 100644 --- a/src/device-sync/types.ts +++ b/src/device-sync/types.ts @@ -74,19 +74,38 @@ export interface ProviderRows { /** * One provider operation. `id` addresses one row of our account by `_id`; - * `where`/`args` address a set (always ANDed with the account scope). + * `where`/`args` address a set (always ANDed with the account scope). With + * both, the op addresses the id row if it matches `where`. An op with `id` + * expects exactly one row unless `expectCount` says otherwise, so a row that + * vanished fails the batch with `assert`. * * - `insert.refs` maps a column to the index of an earlier `insert` in the * same batch whose new row id fills it (ContentProviderOperation * back-references), so one batch can create a raw contact and its data rows, - * or an event and its attendees. + * or an event and its attendees. A column in both `refs` and `values` takes + * the back-reference. Settings rows return no id and cannot be referenced. * - `assert` fails the whole batch unless the addressed rows hold `values` * (compared as text, as ContentProviderOperation does) and, when given, * their number is `expectCount`. - * - `syncState` replaces this account's SyncState blob. + * - `syncState` replaces this account's SyncState blob. It must be the last + * op of its batch and never a yield point (refused with `scope`). * - `yieldAllowed` marks the first op of an item group: the provider may - * commit everything before it and let other writers in. Batches must mark - * one at least every 400 ops (ContactsProvider refuses 500 without one). + * commit everything before it and let other writers in. Contacts batches + * must mark one at least every 400 ops (ContactsProvider refuses a batch + * with 500 ops up to and including a yield point). + * + * What the real providers do with the cells (the fake provider copies it): + * - contacts `data1`..`data14` are TEXT columns, so a number written there + * reads back as a string (`2` → `"2"`); + * - `Events.dirty` and `Calendars.dirty` are NULL, not 0, after an insert by + * the sync adapter: write `dirty: 0` explicitly, and select dirty rows with + * `dirty = 1`; + * - attendees, reminders and extended properties are read through a join, + * where `_id` is ambiguous: address one of their rows with `id`, never + * with `_id` in `where`. Extended properties are updated by `id` only. + * - an insert (or an update) naming a parent row, exception master or group + * that no longer exists fails with `assert`, so the engine re-reads; one of + * another account fails with `scope`. */ export type ProviderOp = | { op: 'insert'; table: ProviderTable; values: WriteRow; refs?: Record; yieldAllowed?: boolean } @@ -305,7 +324,12 @@ export interface DeviceSyncNativeModule { getInfo(): Promise<{ accountType: string; sdkInt: number }>; listAccounts(): Promise; - /** Adds the account if missing and (re)writes its registryId. Resolves true when it was created. */ + /** + * Adds the account if missing. Resolves true when it was created, false + * when it exists for `registryId` already (one left without a registryId + * is adopted); rejects with code `conflict` when it belongs to another + * registry id, whose rows and SyncState it holds. + */ ensureAccount(name: string, registryId: string): Promise; /** Removes the account; the providers drop its rows. */ removeAccount(name: string): Promise; @@ -328,7 +352,17 @@ export interface DeviceSyncNativeModule { * accounts with a contact or calendar type requests those syncs. */ setPushRoutes(routesJson: string): Promise; + /** + * Posts (or replaces) the account's one sync-problem notification; a tap + * opens `uri`, which must be a `bulwarkmobile://` link. `channelName` names + * the notification channel in the app's language; null keeps English. + */ + showSyncProblem(accountName: string, title: string, text: string, uri: string, channelName: string | null): Promise; + clearSyncProblem(accountName: string): Promise; + // Provider I/O runs on one native thread, a call at a time. `query`, + // `readSyncState` and `readPhoto` reject with code `scope`, `permission` or + // `provider` (as `BatchFailure`). /** `ProviderQuery` JSON in, `ProviderRows` JSON out. */ query(name: string, authority: Authority, queryJson: string): Promise; /** `ProviderOp[]` JSON in, `BatchResult` JSON out (failures resolve, they do not reject). */ From 96a6b3832132e281fbd8b6ca1f310ba7edd0de5e Mon Sep 17 00:00:00 2001 From: Linus Rath <139418639+rathlinus@users.noreply.github.com> Date: Sun, 27 Sep 2026 04:19:21 +0200 Subject: [PATCH 07/87] feat: sync the device's contacts and calendars with the server in the background #34 The sync adapters from ed2525e start a headless task for every sync, but the task only reported "disabled". This adds the engine it runs (`src/device-sync/engine`, `src/device-sync/jmap`): for one Android account and authority it downloads the server's changes, uploads what the device changed, and hands a report back to SyncManager. - Downloads follow `/changes` from the state kept in the provider's SyncState. Rows and the state they reach are written in one batch, so a killed run resumes where it stopped. Without a state, after `cannotCalculateChanges` or after the user discarded deletions, a full reconcile compares ids instead. - Uploads go in three phases across accounts (creates, updates, deletions), so moving an item to another account never destroys it before the copy exists. A create carries a uid written to the device first, and the server is asked for that uid before creating, so a lost response or a killed run never makes a duplicate. Updates are patches guarded by `ifInState`; nothing is replaced whole. - More than 50 deletions that are also more than a fifth of the synced items hold every deletion back and report `tooManyDeletions`, so Android asks the user; "undo" restores the items from the server. - An item the server refuses is marked and not sent again until it changes or its back-off (1 hour, doubling to a day) ends. Such refusals and planner errors stay with their item; the rest of the run goes on. - Runs of one account and authority take a lock, stop at checkpoints when cancelled or close to the deadline, and never throw: every failure becomes an outcome, and a rejected sign-in posts one "Sign in again" notification. - The JMAP side uses a client of its own per app account, never the UI's, since a run can share the JS runtime with the app. Turning sync off or signing out runs the same upload first (`teardownAuthority`) and deletes the rows only once nothing is waiting, or when the user agreed to lose it. The planners that map items to rows land in the next commits; until then a run fails before it touches a row. Checked: 94 engine tests against the fake provider and JMAP server (sync flows, failures, crashes and retries, teardown), and `npm run typecheck`, `npm test` and `npm run i18n:check` on this commit. --- locales/rn/en.json | 5 + .../__tests__/engine/calendar-flow.test.ts | 259 ++++ .../__tests__/engine/calendar-passes.test.ts | 119 ++ .../__tests__/engine/contacts-flow.test.ts | 372 +++++ .../__tests__/engine/failures.test.ts | 610 ++++++++ src/device-sync/__tests__/engine/harness.ts | 196 +++ .../__tests__/engine/internals.test.ts | 328 +++++ src/device-sync/__tests__/engine/task.test.ts | 63 + .../__tests__/engine/toy-planners.ts | 657 +++++++++ src/device-sync/engine/batch.ts | 216 +++ src/device-sync/engine/calendar-sync.ts | 663 +++++++++ src/device-sync/engine/checkpoint.ts | 55 + src/device-sync/engine/contacts-sync.ts | 743 ++++++++++ src/device-sync/engine/context.ts | 39 + src/device-sync/engine/deps.ts | 113 ++ src/device-sync/engine/errors.ts | 88 ++ src/device-sync/engine/index.ts | 10 + src/device-sync/engine/item-sync.ts | 1303 +++++++++++++++++ src/device-sync/engine/kinds.ts | 95 ++ src/device-sync/engine/mutex.ts | 70 + src/device-sync/engine/poison.ts | 34 + src/device-sync/engine/provider.ts | 77 + src/device-sync/engine/report.ts | 95 ++ src/device-sync/engine/run.ts | 232 +++ src/device-sync/engine/selection.ts | 36 + src/device-sync/engine/sync-state.ts | 188 +++ src/device-sync/engine/teardown.ts | 127 ++ src/device-sync/jmap/caller.ts | 333 +++++ src/device-sync/jmap/client-port.ts | 87 ++ src/device-sync/jmap/connection.ts | 62 + src/device-sync/jmap/errors.ts | 62 + src/device-sync/jmap/session.ts | 87 ++ src/device-sync/planner.ts | 13 +- src/device-sync/task.ts | 195 ++- 34 files changed, 7608 insertions(+), 24 deletions(-) create mode 100644 src/device-sync/__tests__/engine/calendar-flow.test.ts create mode 100644 src/device-sync/__tests__/engine/calendar-passes.test.ts create mode 100644 src/device-sync/__tests__/engine/contacts-flow.test.ts create mode 100644 src/device-sync/__tests__/engine/failures.test.ts create mode 100644 src/device-sync/__tests__/engine/harness.ts create mode 100644 src/device-sync/__tests__/engine/internals.test.ts create mode 100644 src/device-sync/__tests__/engine/task.test.ts create mode 100644 src/device-sync/__tests__/engine/toy-planners.ts create mode 100644 src/device-sync/engine/batch.ts create mode 100644 src/device-sync/engine/calendar-sync.ts create mode 100644 src/device-sync/engine/checkpoint.ts create mode 100644 src/device-sync/engine/contacts-sync.ts create mode 100644 src/device-sync/engine/context.ts create mode 100644 src/device-sync/engine/deps.ts create mode 100644 src/device-sync/engine/errors.ts create mode 100644 src/device-sync/engine/index.ts create mode 100644 src/device-sync/engine/item-sync.ts create mode 100644 src/device-sync/engine/kinds.ts create mode 100644 src/device-sync/engine/mutex.ts create mode 100644 src/device-sync/engine/poison.ts create mode 100644 src/device-sync/engine/provider.ts create mode 100644 src/device-sync/engine/report.ts create mode 100644 src/device-sync/engine/run.ts create mode 100644 src/device-sync/engine/selection.ts create mode 100644 src/device-sync/engine/sync-state.ts create mode 100644 src/device-sync/engine/teardown.ts create mode 100644 src/device-sync/jmap/caller.ts create mode 100644 src/device-sync/jmap/client-port.ts create mode 100644 src/device-sync/jmap/connection.ts create mode 100644 src/device-sync/jmap/errors.ts create mode 100644 src/device-sync/jmap/session.ts diff --git a/locales/rn/en.json b/locales/rn/en.json index b6c558d8..b880e06a 100644 --- a/locales/rn/en.json +++ b/locales/rn/en.json @@ -252,6 +252,11 @@ "deep_link": { "open_failed": "Couldn't open this" }, + "device_sync": { + "auth_problem_text": "The server no longer accepts the sign-in of {account}, so its contacts and calendars on this device stopped syncing. Sign in again to resume. Changes made on this device are kept.", + "auth_problem_title": "Sign in again to keep syncing", + "problem_channel": "Sync problems" + }, "email_composer": { "attach_camera": "Camera", "attachment_reminder_body": "Your message mentions \"{keyword}\" but no file is attached. Send anyway?", diff --git a/src/device-sync/__tests__/engine/calendar-flow.test.ts b/src/device-sync/__tests__/engine/calendar-flow.test.ts new file mode 100644 index 00000000..9025737e --- /dev/null +++ b/src/device-sync/__tests__/engine/calendar-flow.test.ts @@ -0,0 +1,259 @@ +// Engine runs for calendars, and the teardown the app calls when sync is +// turned off (docs/device-sync.md, "Calendar mapping", "App integration"). + +import { describe, expect, it } from 'vitest'; +import { Events } from '../../android-columns'; +import { parseJsonColumn } from '../../common/json'; +import type { PoisonMarker } from '../../planner'; +import { + addDeviceContact, + addServerCards, + ANDROID_ACCOUNT, + CALENDAR_AUTHORITY, + CONTACTS_AUTHORITY, + createHarness, + renameDeviceContact, + type Harness, +} from './harness'; + +const START = '2026-09-28T09:00:00'; + +function addServerEvent(h: Harness, title: string, calendarId = h.calendar, extra: Record = {}): string { + return h.server.addEvent('a', { + uid: `uid-${title}`, + title, + start: START, + duration: 'PT1H', + timeZone: 'Etc/UTC', + calendarIds: { [calendarId]: true }, + ...extra, + }); +} + +function calendarRow(h: Harness, calendarId: string) { + return h.device.rows('calendars').find((c) => c._sync_id === `a/${calendarId}`); +} + +function titles(h: Harness): string[] { + return h.events().map((e) => String(e[Events.TITLE])).sort(); +} + +describe('device sync engine: calendar', () => { + it('writes calendar rows for the synced calendars and downloads their events, never tasks', async () => { + const h = createHarness(); + const standup = addServerEvent(h, 'Standup'); + h.server.addEvent('a', { uid: 't1', '@type': 'Task', title: 'Buy milk', calendarIds: { [h.calendar]: true } }); + + const report = await h.run(CALENDAR_AUTHORITY); + + expect(report).toMatchObject({ outcome: 'ok', stats: { downloaded: { created: 1 } } }); + expect(calendarRow(h, h.calendar)).toMatchObject({ ownerAccount: 'alice@example.com', sync_events: 1, name: 'Calendar' }); + expect(h.events()).toEqual([expect.objectContaining({ title: 'Standup', _sync_id: `a/${standup}`, uid2445: 'uid-Standup' })]); + expect(h.state(CALENDAR_AUTHORITY)).toMatchObject({ + accounts: { a: { selected: [`a/${h.calendar}`], reconcile: null } }, + deviceZone: 'Europe/Berlin', + reminderOwner: 'device', + }); + }); + + it('uploads a new device event, an edit and a deletion', async () => { + const h = createHarness(); + const standup = addServerEvent(h, 'Standup'); + const retro = addServerEvent(h, 'Retro'); + await h.run(CALENDAR_AUTHORITY); + const rowId = Number(calendarRow(h, h.calendar)!._id); + const byTitle = (t: string) => h.events().find((e) => e.title === t)!; + + h.device.user.insertEvent(rowId, { title: 'Lunch', dtstart: Date.UTC(2026, 8, 28, 12), dtend: Date.UTC(2026, 8, 28, 13), eventTimezone: 'UTC' }); + h.device.user.updateEvent(Number(byTitle('Standup')._id), { title: 'Daily standup' }); + h.device.user.deleteEvent(Number(byTitle('Retro')._id)); + const report = await h.run(CALENDAR_AUTHORITY); + + expect(report).toMatchObject({ outcome: 'ok', stats: { uploaded: { created: 1, updated: 1, deleted: 1 } } }); + expect(h.server.get('CalendarEvent', 'a', standup)).toMatchObject({ title: 'Daily standup' }); + expect(h.server.get('CalendarEvent', 'a', retro)).toBeUndefined(); + const lunch = h.server.all('CalendarEvent', 'a').find((e) => e.title === 'Lunch')!; + expect(lunch).toMatchObject({ start: '2026-09-28T12:00:00', calendarIds: { [h.calendar]: true } }); + expect(titles(h)).toEqual(['Daily standup', 'Lunch']); + expect(byTitle('Lunch')).toMatchObject({ _sync_id: `a/${lunch.id}`, dirty: 0 }); + }); + + it('gives a calendar that holds only tasks no row', async () => { + const h = createHarness(); + const todo = h.server.addCalendar('a', { name: 'To do' }); + for (let i = 0; i < 3; i++) h.server.addEvent('a', { uid: `t${i}`, '@type': 'Task', title: `Task ${i}`, calendarIds: { [todo]: true } }); + addServerEvent(h, 'Standup'); + + await h.run(CALENDAR_AUTHORITY); + + expect(calendarRow(h, todo)).toBeUndefined(); + expect(h.state(CALENDAR_AUTHORITY)?.accounts.a.taskOnly).toEqual([todo]); + expect(titles(h)).toEqual(['Standup']); + }); + + it('drops a deselected calendar once its events are uploaded, keeping the others', async () => { + const h = createHarness(); + const work = h.server.addCalendar('a', { name: 'Work' }); + addServerEvent(h, 'Standup'); + const review = addServerEvent(h, 'Review', work); + addServerEvent(h, 'Planning', work); + await h.run(CALENDAR_AUTHORITY); + expect(titles(h)).toEqual(['Planning', 'Review', 'Standup']); + + h.device.user.updateEvent(Number(h.events().find((e) => e.title === 'Review')!._id), { title: 'Code review' }); + h.prefs.calendarSelection[`a/${work}`] = false; + await h.run(CALENDAR_AUTHORITY); + + expect(h.server.get('CalendarEvent', 'a', review)).toMatchObject({ title: 'Code review' }); + expect(calendarRow(h, work)).toBeUndefined(); + expect(titles(h)).toEqual(['Standup']); + expect(h.state(CALENDAR_AUTHORITY)?.accounts.a.selected).toEqual([`a/${h.calendar}`]); + }); + + it('keeps a deselected calendar while one of its events could not be uploaded', async () => { + const h = createHarness(); + const work = h.server.addCalendar('a', { name: 'Work' }); + const review = addServerEvent(h, 'Review', work); + addServerEvent(h, 'Planning', work); + await h.run(CALENDAR_AUTHORITY); + h.device.user.updateEvent(Number(h.events().find((e) => e.title === 'Review')!._id), { title: 'Code review' }); + h.server.setErrorFor('CalendarEvent', 'a', review, { type: 'tooLarge' }); + h.prefs.calendarSelection[`a/${work}`] = false; + + const report = await h.run(CALENDAR_AUTHORITY); + + expect(report.itemErrors).toEqual([expect.objectContaining({ ref: `a/${review}`, type: 'tooLarge' })]); + expect(calendarRow(h, work)).toBeDefined(); + expect(titles(h)).toEqual(['Code review']); + expect(h.state(CALENDAR_AUTHORITY)?.accounts.a.selected).toEqual([`a/${h.calendar}`, `a/${work}`]); + }); + + it('marks a calendar that mirrors a feed subscription read-only', async () => { + const h = createHarness(); + const feed = h.server.addCalendar('a', { name: 'Holidays' }); + h.deps.subscriptionCalendars = async () => [{ jmapAccountId: null, calendarId: feed }]; + + await h.run(CALENDAR_AUTHORITY); + + expect(calendarRow(h, feed)).toMatchObject({ calendar_access_level: 200 }); + expect(calendarRow(h, h.calendar)).toMatchObject({ calendar_access_level: 700 }); + }); + + it('never gives a device event the identity of an unrelated event that has its uid (event uids are unique per account)', async () => { + const h = createHarness(); + const hidden = h.server.addCalendar('a', { name: 'Hidden' }); + h.prefs.calendarSelection[`a/${hidden}`] = false; + const other = addServerEvent(h, 'Other', hidden, { uid: 'dup' }); + await h.run(CALENDAR_AUTHORITY); + const rowId = Number(calendarRow(h, h.calendar)!._id); + const id = h.device.user.insertEvent(rowId, { title: 'Mine', dtstart: Date.UTC(2026, 8, 28, 12), dtend: Date.UTC(2026, 8, 28, 13), eventTimezone: 'UTC' }); + // An app wrote the uid, and the claim keeps it. + await h.device.port(ANDROID_ACCOUNT, CALENDAR_AUTHORITY).applyBatch([ + { + op: 'update', + table: 'events', + id, + values: { _sync_id: '~pending/dup', uid2445: 'dup', sync_data3: JSON.stringify({ uid: 'dup', target: `a/${h.calendar}` }) }, + }, + ]); + + const report = await h.run(CALENDAR_AUTHORITY); + + expect(report.itemErrors).toEqual([expect.objectContaining({ type: 'uidConflict' })]); + expect(h.server.get('CalendarEvent', 'a', other)).toMatchObject({ title: 'Other', calendarIds: { [hidden]: true } }); + const row = h.events().find((e) => Number(e._id) === id)!; + expect(row).toMatchObject({ title: 'Mine', _sync_id: '~pending/dup' }); + expect(parseJsonColumn(row.sync_data5)).toMatchObject({ type: 'uidConflict', n: 1 }); + }); + + it('adopts an event its lost create left on the server, with a lagging uid index', async () => { + const h = createHarness(); + addServerEvent(h, 'Standup'); + await h.run(CALENDAR_AUTHORITY); + h.server.setUidIndexLag(3); + const rowId = Number(calendarRow(h, h.calendar)!._id); + const id = h.device.user.insertEvent(rowId, { title: 'Lunch', dtstart: Date.UTC(2026, 8, 28, 12), dtend: Date.UTC(2026, 8, 28, 13), eventTimezone: 'UTC' }); + h.server.applyThenLoseResponse({ match: 'CalendarEvent/set' }); + + expect((await h.run(CALENDAR_AUTHORITY)).outcome).toBe('io'); + expect((await h.run(CALENDAR_AUTHORITY)).outcome).toBe('ok'); + + expect(h.server.all('CalendarEvent', 'a').map((e) => e.title).sort()).toEqual(['Lunch', 'Standup']); + expect(h.events()).toHaveLength(2); + expect(h.events().find((e) => Number(e._id) === id)).toMatchObject({ dirty: 0, _sync_id: expect.stringMatching(/^a\//) }); + }); +}); + +describe('device sync engine: teardown', () => { + it('uploads waiting changes, then removes every row and the SyncState of the authority', async () => { + const h = createHarness(); + const [ada] = addServerCards(h, ['Ada']); + await h.run(); + renameDeviceContact(h, h.contactNamed('Ada')!.id, 'Ada King'); + addDeviceContact(h, 'Hedy'); + + const result = await h.teardown(CONTACTS_AUTHORITY); + + expect(result).toEqual({ pending: 0 }); + expect(h.server.get('ContactCard', 'a', ada)).toMatchObject({ name: { full: 'Ada King' } }); + expect(h.serverNames()).toEqual(['Ada King', 'Hedy']); + expect(h.device.rows('raw_contacts')).toEqual([]); + expect(h.device.rows('groups')).toEqual([]); + expect(h.device.rows('settings')).toEqual([]); + expect(h.device.readSyncState(ANDROID_ACCOUNT, CONTACTS_AUTHORITY)).toBe(''); + }); + + it('deletes nothing while changes could not be uploaded, unless forced', async () => { + const h = createHarness(); + addServerCards(h, ['Ada']); + await h.run(); + addDeviceContact(h, 'Hedy'); + h.deps.jmap = async () => { + const error = new Error('Network request failed'); + error.name = 'NetworkError'; + throw error; + }; + + expect(await h.teardown(CONTACTS_AUTHORITY)).toEqual({ pending: 1 }); + expect(h.contacts()).toHaveLength(2); + + expect(await h.teardown(CONTACTS_AUTHORITY, { force: true })).toEqual({ pending: 0 }); + expect(h.contacts()).toHaveLength(0); + expect(h.serverNames()).toEqual(['Ada']); + }); + + it('holds mass deletions back like a sync run, and asks', async () => { + const h = createHarness(); + addServerCards(h, Array.from({ length: 80 }, (_, i) => `P${i}`)); + await h.run(); + for (const contact of h.contacts().slice(0, 60)) h.device.user.deleteContact(contact.id); + + expect(await h.teardown(CONTACTS_AUTHORITY)).toEqual({ pending: 60 }); + expect(h.serverNames()).toHaveLength(80); + }); + + it('removes the calendar rows, and their events with them', async () => { + const h = createHarness(); + addServerEvent(h, 'Standup'); + await h.run(CALENDAR_AUTHORITY); + + expect(await h.teardown(CALENDAR_AUTHORITY)).toEqual({ pending: 0 }); + expect(h.device.rows('calendars')).toEqual([]); + expect(h.device.rows('events')).toEqual([]); + }); + + it('stops a running sync of the account and waits for it', async () => { + const h = createHarness(); + addServerCards(h, Array.from({ length: 200 }, (_, i) => `P${i}`)); + let teardown: Promise<{ pending: number }> | null = null; + h.checkpoints.onCheckpoint = (n) => { + if (n === 3) teardown = h.teardown(CONTACTS_AUTHORITY); + }; + + const report = await h.run(); + + expect(report).toMatchObject({ outcome: 'cancelled' }); + expect(await teardown).toEqual({ pending: 0 }); + expect(h.contacts()).toHaveLength(0); + }); +}); diff --git a/src/device-sync/__tests__/engine/calendar-passes.test.ts b/src/device-sync/__tests__/engine/calendar-passes.test.ts new file mode 100644 index 00000000..f3cc50f2 --- /dev/null +++ b/src/device-sync/__tests__/engine/calendar-passes.test.ts @@ -0,0 +1,119 @@ +// The calendar passes that need no server writes (device zone change, +// reminder owner change) and the local checks before an upload (a series +// split through CONTENT_EXCEPTION_URI). + +import { describe, expect, it, vi } from 'vitest'; +import { Events } from '../../android-columns'; +import type { CalendarPlanner } from '../../planner'; +import { ANDROID_ACCOUNT, CALENDAR_AUTHORITY, createHarness, type Harness } from './harness'; +import { toyCalendarPlanner } from './toy-planners'; + +function withPlanner(h: Harness, overrides: Partial): CalendarPlanner { + const planner = { ...toyCalendarPlanner, ...overrides } as CalendarPlanner; + h.deps.planners = { ...h.deps.planners, calendar: planner }; + return planner; +} + +function addServerEvent(h: Harness, title: string): string { + return h.server.addEvent('a', { uid: `uid-${title}`, title, start: '2026-09-28T09:00:00', duration: 'PT1H', calendarIds: { [h.calendar]: true } }); +} + +describe('device sync engine: calendar passes', () => { + it('rewrites clean events for a new device zone, and stores the zone with the last chunk', async () => { + const h = createHarness({ tuning: { chunkSize: 2 } }); + for (const title of ['A', 'B', 'C']) addServerEvent(h, title); + await h.run(CALENDAR_AUTHORITY); + const planZoneChange = vi.fn((local, previousZone: string, ctx) => ({ + ref: String(local.syncId), + ops: [{ op: 'update' as const, table: 'events' as const, id: local.eventId, values: { eventTimezone: ctx.deviceZone } }], + previousZone, + })); + withPlanner(h, { planZoneChange }); + h.deps.deviceZone = () => 'America/New_York'; + + const report = await h.run(CALENDAR_AUTHORITY); + + expect(report.outcome).toBe('ok'); + expect(planZoneChange).toHaveBeenCalledTimes(3); + expect(planZoneChange.mock.calls.every(([, previous]) => previous === 'Europe/Berlin')).toBe(true); + expect(h.events().map((e) => e[Events.EVENT_TIMEZONE])).toEqual(['America/New_York', 'America/New_York', 'America/New_York']); + expect(h.state(CALENDAR_AUTHORITY)).toMatchObject({ deviceZone: 'America/New_York', deviceZonePending: null }); + }); + + it('finishes an interrupted zone pass in the next run', async () => { + const h = createHarness({ tuning: { chunkSize: 1 } }); + for (const title of ['A', 'B', 'C']) addServerEvent(h, title); + await h.run(CALENDAR_AUTHORITY); + const planZoneChange = vi.fn((local: { eventId: number; syncId: string | null }) => ({ + ref: String(local.syncId), + ops: [{ op: 'update' as const, table: 'events' as const, id: local.eventId, values: { eventTimezone: 'America/New_York' } }], + })); + withPlanner(h, { planZoneChange }); + h.deps.deviceZone = () => 'America/New_York'; + let zoneChunks = 0; + h.checkpoints.onCheckpoint = () => { + if (planZoneChange.mock.calls.length > 0 && ++zoneChunks === 2) h.checkpoints.crashAt = h.checkpoints.count; + }; + + await h.run(CALENDAR_AUTHORITY); + expect(h.state(CALENDAR_AUTHORITY)).toMatchObject({ deviceZone: 'Europe/Berlin', deviceZonePending: 'America/New_York' }); + + h.checkpoints.crashAt = null; + h.checkpoints.onCheckpoint = undefined; + await h.run(CALENDAR_AUTHORITY); + expect(h.state(CALENDAR_AUTHORITY)).toMatchObject({ deviceZone: 'America/New_York', deviceZonePending: null }); + expect(h.events().every((e) => e[Events.EVENT_TIMEZONE] === 'America/New_York')).toBe(true); + }); + + it('uploads with the old reminder owner, then rewrites the reminders and the calendar rows for the new one', async () => { + const h = createHarness(); + addServerEvent(h, 'Standup'); + await h.run(CALENDAR_AUTHORITY); + expect(h.device.rows('calendars')[0]).toMatchObject({ maxReminders: 5 }); + const owners: string[] = []; + const planReminderOwnerChange = vi.fn((local: { eventId: number; syncId: string | null }, ctx: { reminderOwner: string }) => { + owners.push(ctx.reminderOwner); + return { ref: String(local.syncId), ops: [{ op: 'delete' as const, table: 'reminders' as const, where: 'event_id = ?', args: [local.eventId] }] }; + }); + const planUpload = vi.fn(toyCalendarPlanner.planUpload); + withPlanner(h, { planReminderOwnerChange, planUpload }); + h.device.user.setReminders(Number(h.events()[0]._id), [{ minutes: 10 }]); + h.device.user.updateEvent(Number(h.events()[0]._id), { title: 'Daily standup' }); + h.prefs.reminderOwner = 'bulwark'; + + await h.run(CALENDAR_AUTHORITY); + + expect(planUpload.mock.calls.map(([, ctx]) => ctx.reminderOwner)).toEqual(['device']); + expect(owners).toEqual(['bulwark']); + expect(h.device.rows('reminders')).toEqual([]); + expect(h.device.rows('calendars')[0]).toMatchObject({ maxReminders: 0 }); + expect(h.state(CALENDAR_AUTHORITY)?.reminderOwner).toBe('bulwark'); + }); + + it('uploads the clone of a CONTENT_EXCEPTION_URI split as a new event after its source', async () => { + const h = createHarness(); + const series = addServerEvent(h, 'Series'); + await h.run(CALENDAR_AUTHORITY); + const master = h.events()[0]; + // A recurring master, as a calendar app sees it. + await h.device.port(ANDROID_ACCOUNT, CALENDAR_AUTHORITY).applyBatch([ + { op: 'update', table: 'events', id: Number(master._id), values: { rrule: 'FREQ=DAILY', duration: 'P3600S', dtend: null } }, + ]); + const planUpload = vi.fn(toyCalendarPlanner.planUpload); + withPlanner(h, { planUpload }); + const clone = h.device.user.splitViaUri(Number(master._id), Date.UTC(2026, 9, 1, 9), '20260930T235959Z', { title: 'Series (later)' }); + + const report = await h.run(CALENDAR_AUTHORITY); + + expect(report.outcome).toBe('ok'); + // The source was planned while the split was visible, before its clone was claimed. + const splits = planUpload.mock.calls.map(([local]) => local.split); + expect(splits.indexOf('source')).toBeLessThan(splits.indexOf('clone')); + const events = h.server.all('CalendarEvent', 'a'); + expect(events.map((e) => e.title).sort()).toEqual(['Series', 'Series (later)']); + const cloneRow = h.events().find((e) => Number(e._id) === clone)!; + expect(cloneRow._sync_id).not.toBe(`a/${series}`); + expect(cloneRow._sync_id).toMatch(/^a\//); + expect(new Set(h.events().map((e) => e._sync_id)).size).toBe(2); + }); +}); diff --git a/src/device-sync/__tests__/engine/contacts-flow.test.ts b/src/device-sync/__tests__/engine/contacts-flow.test.ts new file mode 100644 index 00000000..d786b20e --- /dev/null +++ b/src/device-sync/__tests__/engine/contacts-flow.test.ts @@ -0,0 +1,372 @@ +// Engine runs for contacts against the fake provider and the fake JMAP +// server: first sync, incremental /changes, uploads, echoes, selection +// changes and groups (docs/device-sync.md, "A sync run"). + +import { describe, expect, it } from 'vitest'; +import { Data, MimeType } from '../../android-columns'; +import { runDeviceSync } from '../../engine/run'; +import { + addDeviceContact, + addServerCards, + ANDROID_ACCOUNT, + CONTACTS_AUTHORITY, + createHarness, + REGISTRY_ID, + renameDeviceContact, + RUN_BUDGET, + rowWrites, + type Harness, +} from './harness'; + +function runAs(h: Harness, registryId: string) { + return runDeviceSync( + { runId: 'other', accountName: ANDROID_ACCOUNT, registryId, authority: CONTACTS_AUTHORITY, extras: {}, deadline: h.clock.now + RUN_BUDGET }, + h.deps, + ); +} + +describe('device sync engine: contacts', () => { + it('downloads every card of the synced address books on the first sync (a full reconcile)', async () => { + const h = createHarness(); + addServerCards(h, ['Ada', 'Grace', 'Linus']); + + const report = await h.run(); + + expect(report).toMatchObject({ outcome: 'ok', stats: { downloaded: { created: 3, updated: 0, deleted: 0 } } }); + expect(h.contacts().map((c) => c.name).sort()).toEqual(['Ada', 'Grace', 'Linus']); + expect(h.contacts().every((c) => c.sourceId?.startsWith('a/') && !c.dirty)).toBe(true); + const state = h.state(); + expect(state?.owner).toEqual({ registryId: REGISTRY_ID, origin: 'https://mail.example.com' }); + expect(state?.accounts.a).toMatchObject({ + itemsState: h.server.state('ContactCard', 'a'), + reconcile: null, + selected: [`a/${h.book}`], + stale: [], + }); + expect(h.device.rows('settings')).toEqual([expect.objectContaining({ ungrouped_visible: 1, should_sync: 1 })]); + expect(h.deps.recordStatus).toHaveBeenCalledWith(REGISTRY_ID, CONTACTS_AUTHORITY, expect.objectContaining({ outcome: 'ok' })); + }); + + it('applies server creates, updates and deletes from /changes', async () => { + const h = createHarness(); + const [ada, grace] = addServerCards(h, ['Ada', 'Grace']); + await h.run(); + + h.server.serverUpdate('ContactCard', 'a', ada, { 'name/full': 'Ada Lovelace' }); + h.server.serverDestroy('ContactCard', 'a', grace); + addServerCards(h, ['Katherine']); + const report = await h.run(); + + expect(report.stats.downloaded).toEqual({ created: 1, updated: 1, deleted: 1 }); + expect(h.contacts().map((c) => c.name).sort()).toEqual(['Ada Lovelace', 'Katherine']); + expect(h.server.calls('ContactCard/changes').length).toBeGreaterThan(0); + expect(h.state()?.accounts.a.itemsState).toBe(h.server.state('ContactCard', 'a')); + }); + + it('pages through /changes and stores the state with every page', async () => { + const h = createHarness(); + await h.run(); + addServerCards(h, Array.from({ length: 300 }, (_, i) => `Person ${i}`)); + const before = h.server.calls('ContactCard/changes').length; + + const report = await h.run(); + + const pages = h.server.calls('ContactCard/changes').slice(before); + expect(pages.map(([, args]) => args.maxChanges)).toEqual([256, 256]); + expect(report.stats.downloaded.created).toBe(300); + expect(h.contacts()).toHaveLength(300); + // One state op per page, each the last op of its batch. + const stateBatches = h.batches.log.filter((ops) => ops.some((op) => op.op === 'syncState')); + for (const ops of stateBatches) expect(ops[ops.length - 1].op).toBe('syncState'); + }); + + it('uploads a device edit as a patch and a new device contact as a create with a client uid', async () => { + const h = createHarness(); + const [ada] = addServerCards(h, ['Ada']); + await h.run(); + + renameDeviceContact(h, h.contactNamed('Ada')!.id, 'Ada King'); + const fresh = addDeviceContact(h, 'Hedy'); + const report = await h.run(); + + expect(report).toMatchObject({ outcome: 'ok', stats: { uploaded: { created: 1, updated: 1, deleted: 0 } } }); + expect(h.server.get('ContactCard', 'a', ada)).toMatchObject({ name: { full: 'Ada King' } }); + const created = h.server.all('ContactCard', 'a').find((c) => (c.name as { full: string }).full === 'Hedy')!; + expect(created.uid).toMatch(/^[0-9a-f-]{36}$/); + expect(h.contacts().find((c) => c.id === fresh)).toMatchObject({ sourceId: `a/${created.id}`, dirty: false }); + expect(h.contacts().every((c) => !c.dirty)).toBe(true); + const set = h.server.calls('ContactCard/set'); + expect(set.every(([, args]) => typeof args.ifInState === 'string')).toBe(true); + }); + + it('uploads to an account without any card yet, whose /changes state is spelled differently from its type state', async () => { + const h = createHarness(); + await h.run(); + addDeviceContact(h, 'Hedy'); + + const report = await h.run(); + + expect(report.outcome).toBe('ok'); + expect(h.serverNames()).toEqual(['Hedy']); + expect(h.contacts()).toEqual([expect.objectContaining({ name: 'Hedy', dirty: false, sourceId: expect.stringMatching(/^a\//) })]); + }); + + it('writes nothing but the state when its own upload comes back through /changes (echo)', async () => { + const h = createHarness(); + addServerCards(h, ['Ada']); + await h.run(); + renameDeviceContact(h, h.contactNamed('Ada')!.id, 'Ada King'); + addDeviceContact(h, 'Hedy'); + await h.run(); + const before = h.batches.log.length; + + const report = await h.run(); + + expect(report.stats.downloaded).toEqual({ created: 0, updated: 0, deleted: 0 }); + expect(rowWrites(h.batches.log.slice(before))).toEqual([]); + expect(h.contacts().map((c) => c.name).sort()).toEqual(['Ada King', 'Hedy']); + }); + + it('keeps a contact dirty when the user edits it again while its upload is on the way', async () => { + const h = createHarness(); + const [ada] = addServerCards(h, ['Ada']); + await h.run(); + const id = h.contactNamed('Ada')!.id; + renameDeviceContact(h, id, 'Ada King'); + const stop = h.server.onAfterRequest((request) => { + if (request.methods.includes('ContactCard/set')) renameDeviceContact(h, id, 'Ada, Countess'); + }); + + await h.run(); + stop(); + + expect(h.server.get('ContactCard', 'a', ada)).toMatchObject({ name: { full: 'Ada King' } }); + expect(h.contacts().find((c) => c.id === id)).toMatchObject({ name: 'Ada, Countess', dirty: true }); + + await h.run(); + + expect(h.server.get('ContactCard', 'a', ada)).toMatchObject({ name: { full: 'Ada, Countess' } }); + expect(h.contacts().find((c) => c.id === id)).toMatchObject({ dirty: false }); + }); + + it('lets the server win a conflicting edit of the same unit and counts it', async () => { + const h = createHarness(); + const [ada] = addServerCards(h, ['Ada']); + await h.run(); + renameDeviceContact(h, h.contactNamed('Ada')!.id, 'Ada (device)'); + h.server.serverUpdate('ContactCard', 'a', ada, { 'name/full': 'Ada (server)' }); + + const report = await h.run(); + + expect(report.conflicts).toBe(1); + expect(h.contacts().map((c) => c.name)).toEqual(['Ada (server)']); + expect(h.server.get('ContactCard', 'a', ada)).toMatchObject({ name: { full: 'Ada (server)' } }); + }); + + it('deletes on the server what the user deleted, and purges the rows', async () => { + const h = createHarness(); + const [ada] = addServerCards(h, ['Ada', 'Grace']); + await h.run(); + + h.device.user.deleteContact(h.contactNamed('Ada')!.id); + const report = await h.run(); + + expect(report.stats.uploaded.deleted).toBe(1); + expect(h.server.get('ContactCard', 'a', ada)).toBeUndefined(); + expect(h.contacts().map((c) => c.name)).toEqual(['Grace']); + }); + + it('drops only the clean rows of a deselected address book, after uploading its dirty ones', async () => { + const h = createHarness(); + const work = h.server.addAddressBook('a', { name: 'Work' }); + addServerCards(h, ['Ada']); + const [boss, colleague] = addServerCards(h, ['Boss', 'Colleague'], 'a', work); + await h.run(); + expect(h.contacts()).toHaveLength(3); + + renameDeviceContact(h, h.contactNamed('Boss')!.id, 'The Boss'); + h.prefs.contactsSelection[`a/${work}`] = false; + const report = await h.run(); + + expect(report.outcome).toBe('ok'); + expect(h.contacts().map((c) => c.name)).toEqual(['Ada']); + expect(h.server.get('ContactCard', 'a', boss)).toMatchObject({ name: { full: 'The Boss' } }); + expect(h.server.get('ContactCard', 'a', colleague)).toBeDefined(); + expect(h.state()?.accounts.a.selected).toEqual([`a/${h.book}`]); + }); + + it("leaves the webmail's Trusted Senders book off the device unless the user picked it", async () => { + const h = createHarness(); + const trusted = h.server.addAddressBook('a', { name: 'Trusted Senders' }); + addServerCards(h, ['Ada']); + addServerCards(h, ['newsletter@example.com'], 'a', trusted); + + await h.run(); + expect(h.contacts().map((c) => c.name)).toEqual(['Ada']); + + h.prefs.contactsSelection[`a/${trusted}`] = true; + await h.run(); + expect(h.contacts().map((c) => c.name).sort()).toEqual(['Ada', 'newsletter@example.com']); + }); + + it('loads a newly selected address book once the account is up to date', async () => { + const h = createHarness({ accounts: 'withShared' }); + addServerCards(h, ['Ada']); + const teamBook = h.server.all('AddressBook', 'team')[0].id as string; + h.server.addCard('team', { uid: 'u-team', name: { full: 'Teammate' }, addressBookIds: { [teamBook]: true } }); + await h.run(); + expect(h.contacts().map((c) => c.name)).toEqual(['Ada']); + + h.prefs.contactsSelection[`team/${teamBook}`] = true; + const report = await h.run(); + + expect(report.stats.downloaded.created).toBe(1); + expect(h.contacts().map((c) => c.name).sort()).toEqual(['Ada', 'Teammate']); + expect(h.contactNamed('Teammate')?.sourceId).toMatch(/^team\//); + expect(h.state()?.accounts.team.selected).toEqual([`team/${teamBook}`]); + }); + + it('writes group cards before the contacts that are their members', async () => { + const h = createHarness(); + const [ada] = addServerCards(h, ['Ada']); + const adaUid = h.server.get('ContactCard', 'a', ada)!.uid as string; + h.server.addCard('a', { uid: 'g1', kind: 'group', name: { full: 'Friends' }, members: { [adaUid]: true } }); + + await h.run(); + + const groups = h.device.rows('groups'); + expect(groups).toHaveLength(1); + expect(groups[0]).toMatchObject({ title: 'Friends', group_visible: 1 }); + const memberships = h.device.rows('data').filter((d) => d.mimetype === MimeType.GROUP_MEMBERSHIP); + expect(memberships).toHaveLength(1); + // data1 is a TEXT column: the group row id may read back as text. + expect(String(memberships[0][Data.DATA1])).toBe(String(groups[0]._id)); + expect(memberships[0].group_sourceid).toBe(groups[0].sourceid); + expect(h.state()?.accounts.a.groups).toEqual([groups[0].sourceid]); + }); + + it("updates a member's rows when only the group card's members changed on the server", async () => { + const h = createHarness(); + const [ada, grace] = addServerCards(h, ['Ada', 'Grace']); + const adaUid = h.server.get('ContactCard', 'a', ada)!.uid as string; + const graceUid = h.server.get('ContactCard', 'a', grace)!.uid as string; + const group = h.server.addCard('a', { uid: 'g1', kind: 'group', name: { full: 'Friends' }, members: { [adaUid]: true } }); + await h.run(); + const members = () => + h.device + .rows('data') + .filter((d) => d.mimetype === MimeType.GROUP_MEMBERSHIP) + .map((d) => h.contacts().find((c) => c.id === Number(d.raw_contact_id))?.name) + .sort(); + expect(members()).toEqual(['Ada']); + + // Another client moves the membership: only the group card changes. + h.server.serverUpdate('ContactCard', 'a', group, { members: { [graceUid]: true } }); + await h.run(); + + expect(members()).toEqual(['Grace']); + expect(h.contacts().every((c) => !c.dirty)).toBe(true); + }); + + it('uploads a membership added on the device as a patch of the group card', async () => { + const h = createHarness(); + const [ada] = addServerCards(h, ['Ada']); + const group = h.server.addCard('a', { uid: 'g1', kind: 'group', name: { full: 'Friends' }, members: {} }); + await h.run(); + + const groupRow = h.device.rows('groups')[0]; + h.device.user.addToGroup(h.contactNamed('Ada')!.id, Number(groupRow._id)); + await h.run(); + + const adaUid = h.server.get('ContactCard', 'a', ada)!.uid as string; + expect(h.server.get('ContactCard', 'a', group)?.members).toEqual({ [adaUid]: true }); + expect(h.contacts().every((c) => !c.dirty)).toBe(true); + }); + + it('skips the network for an upload-only sync with nothing to upload', async () => { + const h = createHarness(); + addServerCards(h, ['Ada']); + await h.run(); + const requests = h.server.requests.length; + + const report = await h.run(CONTACTS_AUTHORITY, { upload: true }); + + expect(report.outcome).toBe('ok'); + expect(h.server.requests.length).toBe(requests); + }); + + it('reports disabled, unsupported and rows of another owner without touching anything', async () => { + const off = createHarness(); + off.deps.isSyncEnabled = async () => false; + expect((await off.run()).outcome).toBe('disabled'); + expect(off.server.requests).toHaveLength(0); + + const bare = createHarness(); + const port = bare.server.port(); + bare.deps.jmap = async () => ({ + origin: 'https://mail.example.com', + port: { ...port, session: () => ({ ...port.session(), primaryAccounts: {}, accounts: {} }) }, + }); + expect((await bare.run()).outcome).toBe('unsupported'); + + const other = createHarness(); + addServerCards(other, ['Ada']); + await other.run(); + other.deps.jmap = async () => ({ port: other.server.port(), origin: 'https://other.example.com' }); + const moved = await other.run(); + expect(moved.outcome).toBe('internal'); + expect(moved.message).toMatch(/another server/); + expect(other.contacts()).toHaveLength(1); + const foreign = await runAs(other, 'bob@mail.example.com'); + expect(foreign.outcome).toBe('internal'); + expect(foreign.message).toMatch(/another account/); + }); + + it('keeps one request in flight and stays within the server limits', async () => { + const h = createHarness(); + h.server.setLimits({ maxObjectsInGet: 20, maxObjectsInSet: 10 }); + addServerCards(h, Array.from({ length: 60 }, (_, i) => `Server ${i}`)); + await h.run(); + for (let i = 0; i < 25; i++) addDeviceContact(h, `Device ${i}`); + + const report = await h.run(); + + expect(report.outcome).toBe('ok'); + expect(h.server.maxInFlight).toBe(1); + for (const [, args] of h.server.calls('ContactCard/get')) { + expect(Array.isArray(args.ids)).toBe(true); + expect((args.ids as string[]).length).toBeLessThanOrEqual(20); + } + for (const [, args] of h.server.calls('ContactCard/set')) { + const count = Object.keys(args.create ?? {}).length + Object.keys(args.update ?? {}).length + ((args.destroy as string[]) ?? []).length; + expect(count).toBeLessThanOrEqual(10); + } + expect(h.serverNames()).toHaveLength(85); + expect(h.contacts()).toHaveLength(85); + expect(ANDROID_ACCOUNT).toBe('alice@example.com'); + }); + + it('packs provider batches of at most 400 ops with a yield point at every group and the state op last', async () => { + const h = createHarness(); + addServerCards(h, Array.from({ length: 450 }, (_, i) => `Person ${i}`)); + + await h.run(); + + expect(h.contacts()).toHaveLength(450); + for (const ops of h.batches.log) { + expect(ops.length).toBeLessThanOrEqual(400); + const state = ops.findIndex((op) => op.op === 'syncState'); + if (state >= 0) { + expect(state).toBe(ops.length - 1); + expect((ops[state] as { yieldAllowed?: boolean }).yieldAllowed).toBeFalsy(); + } + // Every download insert starts a group with its "not there yet" guard: a yield point, and nothing inside a group is one. + ops.forEach((op, i) => { + if (op.op === 'insert' && op.table === 'raw_contacts') { + expect(ops[i - 1]).toMatchObject({ op: 'assert', table: 'raw_contacts', expectCount: 0, yieldAllowed: true }); + expect(op.yieldAllowed).toBeFalsy(); + } + }); + } + }); +}); diff --git a/src/device-sync/__tests__/engine/failures.test.ts b/src/device-sync/__tests__/engine/failures.test.ts new file mode 100644 index 00000000..e27ec3f4 --- /dev/null +++ b/src/device-sync/__tests__/engine/failures.test.ts @@ -0,0 +1,610 @@ +// How the engine survives crashes, lost responses and server trouble, and +// its guards (docs/device-sync.md, "Invariants", "Failure matrix", +// "Deletion threshold"). + +import { describe, expect, it } from 'vitest'; +import { parseJsonColumn } from '../../common/json'; +import type { PoisonMarker } from '../../planner'; +import type { JmapInvocation, JmapResponse } from '../../types'; +import { + addDeviceContact, + addServerCards, + CONTACTS_AUTHORITY, + createHarness, + CrashError, + renameDeviceContact, + REGISTRY_ID, + type Harness, +} from './harness'; + +const FINAL = ['Ada King', 'Grace Hopper', 'Hedy', 'Margaret']; + +/** Work on both sides: device edit, create and delete; server update, create and delete. */ +async function scenario(options: Parameters[0] = {}): Promise { + const h = createHarness(options); + const [, grace, , katherine] = addServerCards(h, ['Ada', 'Grace', 'Linus', 'Katherine']); + await h.run(); + renameDeviceContact(h, h.contactNamed('Ada')!.id, 'Ada King'); + addDeviceContact(h, 'Hedy'); + h.device.user.deleteContact(h.contactNamed('Linus')!.id); + h.server.serverUpdate('ContactCard', 'a', grace, { 'name/full': 'Grace Hopper' }); + addServerCards(h, ['Margaret']); + h.server.serverDestroy('ContactCard', 'a', katherine); + h.checkpoints.count = 0; + h.batches.applied = 0; + return h; +} + +/** Both sides hold exactly the expected contacts, once each, and nothing waits for an upload. */ +function expectConverged(h: Harness, names = FINAL): void { + const device = h.contacts(); + expect(device.map((c) => c.name).sort()).toEqual(names); + expect(device.every((c) => !c.dirty && !c.deleted && c.sourceId)).toBe(true); + expect(h.serverNames()).toEqual(names); + const refs = device.map((c) => c.sourceId); + expect(new Set(refs).size).toBe(refs.length); + for (const contact of device) { + const card = h.server.get('ContactCard', 'a', contact.sourceId!.split('/')[1]); + expect((card?.name as { full: string }).full).toBe(contact.name); + } + const uids = h.server.all('ContactCard', 'a').map((c) => c.uid); + expect(new Set(uids).size).toBe(uids.length); +} + +describe('device sync engine: crashes and lost responses', () => { + it('converges after a crash at any checkpoint, with no duplicates and no lost edits', async () => { + const probe = await scenario(); + await probe.run(); + expectConverged(probe); + const checkpoints = probe.checkpoints.count; + expect(checkpoints).toBeGreaterThan(5); + + for (let n = 1; n <= checkpoints; n++) { + const h = await scenario(); + h.checkpoints.crashAt = n; + const crashed = await h.run(); + expect(crashed.outcome, `crash at checkpoint ${n}`).toBe('internal'); + h.checkpoints.crashAt = null; + const report = await h.run(); + expect(report.outcome, `after a crash at checkpoint ${n}`).toBe('ok'); + expectConverged(h); + } + }); + + it('converges after a crash after any number of provider batches', async () => { + const probe = await scenario(); + await probe.run(); + const batches = probe.batches.applied; + expect(batches).toBeGreaterThan(3); + + for (let n = 0; n < batches; n++) { + const h = await scenario(); + h.batches.crashAfter = n; + await h.run(); + h.batches.crashAfter = null; + const report = await h.run(); + expect(report.outcome, `after a crash following batch ${n}`).toBe('ok'); + expectConverged(h); + } + }); + + it('converges when the server applied any request but its response was lost', async () => { + const probe = await scenario(); + const before = probe.server.requests.length; + await probe.run(); + const requests = probe.server.requests.length - before; + + for (let n = 1; n <= requests; n++) { + const h = await scenario(); + // The uid index lags behind, as on Stalwart: a lookup right after the lost create misses it. + h.server.setUidIndexLag(2); + let seen = 0; + const stop = h.server.onBeforeRequest(() => { + if (++seen === n) h.server.applyThenLoseResponse(); + }); + const lost = await h.run(); + stop(); + expect(lost.outcome, `lost response ${n}`).toBe('io'); + const report = await h.run(); + expect(report.outcome, `after losing response ${n}`).toBe('ok'); + expectConverged(h); + } + }); + + it('converges from a crash anywhere in a first sync (a checkpointed full reconcile)', async () => { + const names = Array.from({ length: 120 }, (_, i) => `P${String(i).padStart(3, '0')}`); + const fresh = () => { + const h = createHarness(); + addServerCards(h, names); + return h; + }; + const probe = fresh(); + await probe.run(); + const checkpoints = probe.checkpoints.count; + + for (let n = 1; n <= checkpoints; n += 2) { + const h = fresh(); + h.checkpoints.crashAt = n; + await h.run(); + h.checkpoints.crashAt = null; + expect((await h.run()).outcome).toBe('ok'); + expect(h.contacts().map((c) => c.name).sort()).toEqual(names); + expect(new Set(h.contacts().map((c) => c.sourceId)).size).toBe(120); + } + }); + + it('resumes a reconcile after its last stored chunk instead of starting over', async () => { + const names = Array.from({ length: 150 }, (_, i) => `P${String(i).padStart(3, '0')}`); + const h = createHarness(); + addServerCards(h, names); + // Batches: the Settings row, the reconcile marker, then one per chunk of 50. Stop after the second chunk. + h.batches.crashAfter = 4; + await h.run(); + const marker = h.state()?.accounts.a.reconcile; + expect(marker).toMatchObject({ phase: 'objects', position: 100 }); + h.batches.crashAfter = null; + const gets = h.server.calls('ContactCard/get').length; + + await h.run(); + + const fetched = h.server + .calls('ContactCard/get') + .slice(gets) + .filter(([, args]) => (args.properties as string[]).includes('name')) + .flatMap(([, args]) => args.ids as string[]); + expect(fetched).toHaveLength(50); + expect(h.contacts()).toHaveLength(150); + expect(h.state()?.accounts.a.reconcile).toBeNull(); + }); + + it('adopts its own create whose response was lost instead of creating it twice', async () => { + const h = createHarness(); + addServerCards(h, ['Ada']); + await h.run(); + h.server.setUidIndexLag(3); + const id = addDeviceContact(h, 'Hedy'); + h.server.applyThenLoseResponse({ match: 'ContactCard/set' }); + + expect((await h.run()).outcome).toBe('io'); + expect(h.serverNames()).toEqual(['Ada', 'Hedy']); + expect(h.contacts().find((c) => c.id === id)?.sourceId).toBeNull(); + + expect((await h.run()).outcome).toBe('ok'); + expect(h.serverNames()).toEqual(['Ada', 'Hedy']); + expect(h.contacts().find((c) => c.id === id)).toMatchObject({ name: 'Hedy', dirty: false, sourceId: expect.stringMatching(/^a\//) }); + expect(h.contacts()).toHaveLength(2); + }); + + it("adopts the card a duplicate-uid error names (the client's automatic retry of a create)", async () => { + const h = createHarness(); + await h.run(); + addDeviceContact(h, 'Hedy'); + // The first attempt was applied; the retry answers with Stalwart's duplicate-uid error. + const port = h.server.port(); + let retried = false; + h.deps.jmap = async () => ({ + origin: 'https://mail.example.com', + port: { + ...port, + request: async (calls: JmapInvocation[], using: string[]): Promise => { + const response = await port.request(calls, using); + const [name, args] = response.methodResponses[0]; + if (retried || name !== 'ContactCard/set' || !args.created) return response; + retried = true; + const [[key, created]] = Object.entries(args.created as Record); + const uid = (calls[0][1].create as Record)[key].uid; + const notCreated = { + [key]: { type: 'invalidProperties', properties: ['uid'], description: `Contact with UID ${uid} already exists with id ${created.id}.` }, + }; + return { ...response, methodResponses: [[name, { ...args, created: undefined, notCreated }, response.methodResponses[0][2]]] }; + }, + }, + }); + + const report = await h.run(); + + expect(report.outcome).toBe('ok'); + expect(retried).toBe(true); + expect(h.serverNames()).toEqual(['Hedy']); + expect(h.contacts()).toEqual([expect.objectContaining({ name: 'Hedy', dirty: false, sourceId: expect.stringMatching(/^a\//) })]); + }); +}); + +describe('device sync engine: server trouble', () => { + it('downloads again and retries the upload after a stateMismatch', async () => { + const h = createHarness(); + const [ada, grace] = addServerCards(h, ['Ada', 'Grace']); + await h.run(); + renameDeviceContact(h, h.contactNamed('Ada')!.id, 'Ada King'); + let sets = 0; + const stop = h.server.onBeforeRequest((request) => { + // Another client writes right before our first upload. + if (request.methods.includes('ContactCard/set') && ++sets === 1) { + h.server.serverUpdate('ContactCard', 'a', grace, { 'name/full': 'Grace Hopper' }); + } + }); + + const report = await h.run(); + stop(); + + expect(report.outcome).toBe('ok'); + const set = h.server.requests.filter((r) => r.methods.includes('ContactCard/set')); + expect(set[0].responses?.[0]).toEqual(['error', { type: 'stateMismatch' }, 's']); + expect(h.server.get('ContactCard', 'a', ada)).toMatchObject({ name: { full: 'Ada King' } }); + expect(h.contacts().map((c) => c.name).sort()).toEqual(['Ada King', 'Grace Hopper']); + }); + + it('gives up with io and no progress when the server keeps changing', async () => { + const h = createHarness(); + const [, grace] = addServerCards(h, ['Ada', 'Grace']); + await h.run(); + renameDeviceContact(h, h.contactNamed('Ada')!.id, 'Ada King'); + let n = 0; + const stop = h.server.onBeforeRequest((request) => { + if (request.methods.includes('ContactCard/set')) h.server.serverUpdate('ContactCard', 'a', grace, { 'name/full': `Grace ${++n}` }); + }); + + const report = await h.run(); + stop(); + + expect(report.outcome).toBe('io'); + expect(report.stats.downloaded).toEqual({ created: 0, updated: 0, deleted: 0 }); + expect(n).toBe(4); + expect(h.contactNamed('Ada King')?.dirty).toBe(true); + }); + + it('holds back the uploads of an account that refuses writes, and uploads the other accounts', async () => { + const h = createHarness({ accounts: 'withShared' }); + const teamBook = h.server.all('AddressBook', 'team')[0].id as string; + h.prefs.contactsSelection[`team/${teamBook}`] = true; + addServerCards(h, ['Ada']); + const [mate] = addServerCards(h, ['Teammate'], 'team', teamBook); + await h.run(); + renameDeviceContact(h, h.contactNamed('Ada')!.id, 'Ada King'); + renameDeviceContact(h, h.contactNamed('Teammate')!.id, 'Team mate'); + h.server.failNextMethod('ContactCard/set', { type: 'accountReadOnly' }, { accountId: 'team' }); + + const report = await h.run(); + + expect(report.outcome).toBe('ok'); + expect(report.itemErrors).toEqual([expect.objectContaining({ ref: 'team', type: 'accountReadOnly' })]); + expect(h.serverNames()).toEqual(['Ada King']); + expect(h.server.get('ContactCard', 'team', mate)).toMatchObject({ name: { full: 'Teammate' } }); + expect(h.contactNamed('Team mate')?.dirty).toBe(true); + + await h.run(); + expect(h.server.get('ContactCard', 'team', mate)).toMatchObject({ name: { full: 'Team mate' } }); + }); + + it('reports a 429 as io with delayUntil from Retry-After', async () => { + const h = createHarness(); + await h.run(); + h.server.failNextRequest('rateLimit', { retryAfterMs: 120_000 }); + + const report = await h.run(); + + expect(report.outcome).toBe('io'); + expect(report.delayUntil).toBe(Math.ceil((h.clock.now + 120_000) / 1000)); + }); + + it('rebuilds the client once when the server rejects the credentials, then reports auth', async () => { + const h = createHarness(); + addServerCards(h, ['Ada']); + h.server.failNextRequest('auth'); + expect((await h.run()).outcome).toBe('ok'); + expect(h.deps.notifyAuthProblem).not.toHaveBeenCalled(); + + h.server.failNextRequest('auth', { times: 2 }); + const report = await h.run(); + + expect(report.outcome).toBe('auth'); + expect(h.deps.notifyAuthProblem).toHaveBeenCalledWith(REGISTRY_ID, 'alice@example.com', CONTACTS_AUTHORITY); + expect(h.contacts()).toHaveLength(1); + }); + + it('reports auth without touching anything when the account has no stored credentials', async () => { + const h = createHarness(); + addServerCards(h, ['Ada']); + await h.run(); + const writes = h.batches.log.length; + h.deps.jmap = async () => { + const error = new Error('No usable credentials for this account'); + error.name = 'AuthenticationError'; + throw error; + }; + + const report = await h.run(); + + expect(report.outcome).toBe('auth'); + expect(h.batches.log.length).toBe(writes); + }); + + it('full-reconciles an account after cannotCalculateChanges', async () => { + const h = createHarness(); + const [ada, grace] = addServerCards(h, ['Ada', 'Grace', 'Linus']); + await h.run(); + h.server.serverUpdate('ContactCard', 'a', ada, { 'name/full': 'Ada King' }); + h.server.serverDestroy('ContactCard', 'a', grace); + h.server.truncateChangeLog('a', 'contacts'); + + const report = await h.run(); + + expect(report.outcome).toBe('ok'); + expect(h.server.requests.some((r) => r.responses?.some(([, args]) => (args as { type?: string }).type === 'cannotCalculateChanges'))).toBe(true); + expect(h.contacts().map((c) => c.name).sort()).toEqual(['Ada King', 'Linus']); + expect(h.state()?.accounts.a).toMatchObject({ reconcile: null, itemsState: h.server.state('ContactCard', 'a') }); + }); + + it('stops a reconcile that finds an empty server while the device is full (safetyAbort)', async () => { + const h = createHarness(); + const ids = addServerCards(h, Array.from({ length: 12 }, (_, i) => `P${i}`)); + await h.run(); + for (const id of ids) h.server.serverDestroy('ContactCard', 'a', id); + h.server.truncateChangeLog('a', 'contacts'); + + const report = await h.run(); + + expect(report.outcome).toBe('safetyAbort'); + expect(h.contacts()).toHaveLength(12); + }); +}); + +describe('device sync engine: time and cancellation', () => { + it('stops cleanly near the deadline and asks for another sync', async () => { + const h = createHarness(); + addServerCards(h, Array.from({ length: 150 }, (_, i) => `P${i}`)); + h.checkpoints.onCheckpoint = (n) => { + // Time runs out in the middle of the download. + if (n === 4) h.clock.now += 9 * 60_000; + }; + + const report = await h.run(); + + expect(report).toMatchObject({ outcome: 'cancelled', moreRecordsToGet: true }); + expect(h.contacts().length).toBeGreaterThan(0); + expect(h.contacts().length).toBeLessThan(150); + h.checkpoints.onCheckpoint = undefined; + + expect((await h.run()).outcome).toBe('ok'); + expect(h.contacts()).toHaveLength(150); + }); + + it('stops at the next checkpoint when the framework cancels the run', async () => { + const h = createHarness(); + addServerCards(h, Array.from({ length: 150 }, (_, i) => `P${i}`)); + h.deps.isCancelled = async () => true; + + const report = await h.run(); + + expect(report).toMatchObject({ outcome: 'cancelled', moreRecordsToGet: true }); + expect(h.contacts()).toHaveLength(0); + }); + + it('sends Core/echo when a local phase went 40 s without a JMAP request', async () => { + const h = createHarness(); + addServerCards(h, Array.from({ length: 120 }, (_, i) => `P${i}`)); + let n = 0; + const port = h.deps.provider; + h.deps.provider = (name, authority) => { + const inner = port(name, authority); + return { + ...inner, + applyBatch: async (ops) => { + h.clock.now += 45_000; + n++; + return inner.applyBatch(ops); + }, + }; + }; + + await h.run(); + + expect(n).toBeGreaterThan(2); + expect(h.server.calls('Core/echo').length).toBeGreaterThan(0); + }); + + it('runs one sync of an account and authority at a time', async () => { + const h = createHarness(); + addServerCards(h, Array.from({ length: 60 }, (_, i) => `P${i}`)); + + const [first, second] = await Promise.all([h.run(), h.run()]); + + expect([first.outcome, second.outcome]).toEqual(['ok', 'ok']); + expect(h.server.maxInFlight).toBe(1); + expect(h.contacts()).toHaveLength(60); + }); +}); + +describe('device sync engine: poisoned items', () => { + it('backs a refused item off, retries it after the back-off, and at once when it changes', async () => { + const h = createHarness(); + const [ada] = addServerCards(h, ['Ada', 'Grace']); + await h.run(); + const id = h.contactNamed('Ada')!.id; + renameDeviceContact(h, id, 'Ada King'); + h.server.setErrorFor('ContactCard', 'a', (t) => t.op === 'update' && t.id === ada, { + type: 'invalidProperties', + properties: ['name'], + description: 'Invalid property.', + }, 3); + + const first = await h.run(); + + expect(first.outcome).toBe('ok'); + expect(first.stats.skipped).toBe(1); + expect(first.itemErrors).toEqual([ + expect.objectContaining({ ref: `a/${ada}`, side: 'upload', type: 'invalidProperties', retryAt: h.clock.now + 3_600_000 }), + ]); + const marker = parseJsonColumn(h.contacts().find((c) => c.id === id)!.row.sync4); + expect(marker).toMatchObject({ type: 'invalidProperties', n: 1, until: h.clock.now + 3_600_000 }); + + // Not retried hot. + const sets = h.server.calls('ContactCard/set').length; + const again = await h.run(); + expect(h.server.calls('ContactCard/set').length).toBe(sets); + expect(again.itemErrors).toEqual([expect.objectContaining({ type: 'invalidProperties' })]); + + // Retried once the back-off is over; refused again, the back-off doubles. + h.clock.now += 3_600_001; + await h.run(); + expect(h.server.calls('ContactCard/set').length).toBe(sets + 1); + expect(parseJsonColumn(h.contacts().find((c) => c.id === id)!.row.sync4)).toMatchObject({ n: 2, until: h.clock.now + 7_200_000 }); + + // Changed by the user: retried at once (the server refuses once more), then accepted. + renameDeviceContact(h, id, 'Ada, Countess'); + await h.run(); + expect(h.server.calls('ContactCard/set').length).toBe(sets + 2); + renameDeviceContact(h, id, 'Ada Lovelace'); + const fixed = await h.run(); + expect(fixed.itemErrors).toEqual([]); + expect(h.server.get('ContactCard', 'a', ada)).toMatchObject({ name: { full: 'Ada Lovelace' } }); + expect(h.contacts().find((c) => c.id === id)!.row.sync4).toBeNull(); + }); + + it('never uploads an item whose rows could not be written until it was fetched again', async () => { + const h = createHarness(); + const [ada] = addServerCards(h, ['Ada']); + await h.run(); + h.server.serverUpdate('ContactCard', 'a', ada, { 'name/full': 'Ada King' }); + // The provider refuses the item's rows (in its batch and alone): it goes on `stale`, the state still moves on. + const provider = h.deps.provider; + let refusals = 2; + h.deps.provider = (name, authority) => { + const port = provider(name, authority); + return { + ...port, + applyBatch: async (ops) => + refusals > 0 && ops.some((op) => op.op === 'update' && op.table === 'data') && refusals-- + ? { ok: false, reason: 'provider', message: 'SQLiteException' } + : port.applyBatch(ops), + }; + }; + const failing = await h.run(); + expect(refusals).toBe(0); + expect(failing.itemErrors).toEqual([expect.objectContaining({ ref: `a/${ada}`, side: 'download', type: 'provider' })]); + expect(h.state()?.accounts.a.stale).toEqual([ada]); + expect(h.state()?.accounts.a.itemsState).toBe(h.server.state('ContactCard', 'a')); + + const next = await h.run(); + + expect(next.outcome).toBe('ok'); + expect(h.contacts().map((c) => c.name)).toEqual(['Ada King']); + expect(h.state()?.accounts.a.stale).toEqual([]); + }); +}); + +describe('device sync engine: planner failures', () => { + it('keeps syncing everything else when a planner throws on one item, and tries that one again next run', async () => { + const h = createHarness(); + const [, grace] = addServerCards(h, ['Ada', 'Grace', 'Linus']); + let broken = true; + const planner = h.deps.planners.contacts; + h.deps.planners = { + ...h.deps.planners, + contacts: { + ...planner, + planDownload: (card, local, ctx) => { + if (broken && card.id === grace) throw new TypeError('cannot read the card'); + return planner.planDownload(card, local, ctx); + }, + planUpload: (local, ctx) => { + if (broken && local.rows.some((r) => r.cells.data1 === 'Hedy')) throw new TypeError('cannot read the rows'); + return planner.planUpload(local, ctx); + }, + }, + }; + addDeviceContact(h, 'Hedy'); + addDeviceContact(h, 'Mary'); + + const report = await h.run(); + + expect(report.outcome).toBe('ok'); + expect(report.itemErrors).toEqual( + expect.arrayContaining([ + expect.objectContaining({ ref: `a/${grace}`, side: 'download', type: 'plannerError' }), + expect.objectContaining({ side: 'upload', type: 'plannerError' }), + ]), + ); + expect(h.contacts().map((c) => c.name).sort()).toEqual(['Ada', 'Hedy', 'Linus', 'Mary']); + expect(h.serverNames()).toEqual(['Ada', 'Grace', 'Linus', 'Mary']); + // The reconcile completed past the item that could not be planned. + expect(h.state()?.accounts.a).toMatchObject({ stale: [grace], reconcile: null, itemsState: expect.any(String) }); + + broken = false; + expect((await h.run()).itemErrors).toEqual([]); + expect(h.contacts().map((c) => c.name).sort()).toEqual(['Ada', 'Grace', 'Hedy', 'Linus', 'Mary']); + expect(h.serverNames()).toEqual(['Ada', 'Grace', 'Hedy', 'Linus', 'Mary']); + expect(h.state()?.accounts.a.stale).toEqual([]); + }); +}); + +describe('device sync engine: deletion guards', () => { + async function withDeletions(total: number, deleted: number): Promise { + const h = createHarness(); + addServerCards(h, Array.from({ length: total }, (_, i) => `P${String(i).padStart(3, '0')}`)); + await h.run(); + for (const contact of h.contacts().slice(0, deleted)) h.device.user.deleteContact(contact.id); + return h; + } + + it('holds every deletion back above 50 and 20 % (tooManyDeletions)', async () => { + const h = await withDeletions(100, 55); + + const report = await h.run(); + + expect(report).toMatchObject({ outcome: 'tooManyDeletions', tooManyDeletions: { count: 55, threshold: 50 } }); + expect(h.serverNames()).toHaveLength(100); + expect(h.contacts().filter((c) => c.deleted)).toHaveLength(55); + }); + + it('uploads the deletions when the run overrides the threshold', async () => { + const h = await withDeletions(100, 55); + + const report = await h.run(CONTACTS_AUTHORITY, { overrideTooManyDeletions: true }); + + expect(report.outcome).toBe('ok'); + expect(report.stats.uploaded.deleted).toBe(55); + expect(h.serverNames()).toHaveLength(45); + expect(h.contacts()).toHaveLength(45); + }); + + it('restores the deleted contacts from the server when the run discards the deletions', async () => { + const h = await withDeletions(100, 55); + + const discard = await h.run(CONTACTS_AUTHORITY, { discardLocalDeletions: true }); + expect(discard.outcome).toBe('ok'); + expect(h.serverNames()).toHaveLength(100); + + const next = await h.run(CONTACTS_AUTHORITY, { upload: true }); + expect(next.outcome).toBe('ok'); + expect(h.contacts()).toHaveLength(100); + expect(h.contacts().every((c) => !c.deleted && !c.dirty)).toBe(true); + }); + + it('uploads deletions below the threshold', async () => { + const h = await withDeletions(100, 20); + + const report = await h.run(); + + expect(report.outcome).toBe('ok'); + expect(h.serverNames()).toHaveLength(80); + }); + + it('counts groups an app hard-deleted, found by their absence', async () => { + const h = createHarness(); + const group = h.server.addCard('a', { uid: 'g1', kind: 'group', name: { full: 'Friends' } }); + addServerCards(h, ['Ada']); + await h.run(); + + h.device.user.fossifyDeleteGroup(Number(h.device.rows('groups')[0]._id)); + const report = await h.run(); + + expect(report.stats.uploaded.deleted).toBe(1); + expect(h.server.get('ContactCard', 'a', group)).toBeUndefined(); + expect(h.state()?.accounts.a.groups).toEqual([]); + }); + + it('fails a crash-free run with CrashError only when injected', () => { + expect(new CrashError('x').name).toBe('CrashError'); + }); +}); diff --git a/src/device-sync/__tests__/engine/harness.ts b/src/device-sync/__tests__/engine/harness.ts new file mode 100644 index 00000000..dc13849e --- /dev/null +++ b/src/device-sync/__tests__/engine/harness.ts @@ -0,0 +1,196 @@ +/** + * The engine under test, wired to the fake providers, the fake JMAP server + * and the toy planners, with helpers to read both sides and inject faults. + */ +import { vi } from 'vitest'; +import { Data, MimeType } from '../../android-columns'; +import { parseJsonColumn } from '../../common/json'; +import { runDeviceSync } from '../../engine/run'; +import { teardownAuthority, type TeardownOptions } from '../../engine/teardown'; +import type { EngineDeps, JmapConnection, Tuning } from '../../engine/deps'; +import { openWithAuthRebuild } from '../../jmap/connection'; +import type { SyncState } from '../../engine/sync-state'; +import { + CALENDAR_AUTHORITY, + CONTACTS_AUTHORITY, + type AccountSyncPrefs, + type Authority, + type ProviderOp, + type ProviderPort, + type RunExtras, + type RunReport, + type Row, +} from '../../types'; +import { FakeDeviceProviders } from '../fakes/fake-provider'; +import { FakeJmapServer } from '../fakes/fake-jmap-server'; +import { toyCalendarPlanner, toyContactsPlanner } from './toy-planners'; + +export const ANDROID_ACCOUNT = 'alice@example.com'; +export const REGISTRY_ID = 'alice@mail.example.com'; +export const ORIGIN = 'https://mail.example.com'; +export const START = Date.UTC(2026, 8, 27, 10, 0, 0); +export const RUN_BUDGET = 9 * 60_000; + +/** Deterministic randomness: uids and keys differ between items but repeat between test runs. */ +export function seededRandom(seed = 42): () => number { + let s = seed >>> 0; + return () => { + s = (s * 1664525 + 1013904223) >>> 0; + return s / 2 ** 32; + }; +} + +export class CrashError extends Error { + constructor(where: string) { + super(`crash injected ${where}`); + this.name = 'CrashError'; + } +} + +export interface Harness { + server: FakeJmapServer; + device: FakeDeviceProviders; + deps: EngineDeps; + prefs: AccountSyncPrefs; + book: string; + calendar: string; + clock: { now: number }; + /** Checkpoints seen by the last runs (each call of isCancelled). */ + checkpoints: { count: number; crashAt: number | null; onCheckpoint?: (n: number) => void }; + /** Batches applied through the engine's provider port; crash after `crashAfter` of them. */ + batches: { applied: number; crashAfter: number | null; log: ProviderOp[][] }; + run(authority?: Authority, extras?: RunExtras): Promise; + teardown(authority: Authority, options?: TeardownOptions): Promise<{ pending: number }>; + state(authority?: Authority): SyncState | null; + contacts(): Array<{ id: number; sourceId: string | null; name: string | null; dirty: boolean; deleted: boolean; row: Row }>; + contactNamed(name: string): { id: number; sourceId: string | null; dirty: boolean; deleted: boolean } | undefined; + nameDataRow(rawContactId: number): Row | undefined; + events(): Row[]; + serverNames(accountId?: string): string[]; +} + +export function createHarness(options: { tuning?: Partial; accounts?: 'personal' | 'withShared'; rebuild?: boolean } = {}): Harness { + const clock = { now: START }; + const server = new FakeJmapServer({ now: () => clock.now }); + server.addAccount('a', { name: ANDROID_ACCOUNT }); + const book = server.addAddressBook('a', { name: 'Personal', isDefault: true }); + const calendar = server.addCalendar('a', { name: 'Calendar', isDefault: true }); + if (options.accounts === 'withShared') { + server.addAccount('team', { name: 'Team', isPersonal: false }); + server.addAddressBook('team', { name: 'Team contacts' }); + server.addCalendar('team', { name: 'Team calendar' }); + } + const device = new FakeDeviceProviders(); + const prefs: AccountSyncPrefs = { contactsSelection: {}, calendarSelection: {}, reminderOwner: 'device' }; + const checkpoints: Harness['checkpoints'] = { count: 0, crashAt: null }; + const batches: Harness['batches'] = { applied: 0, crashAfter: null, log: [] }; + + const wrap = (port: ProviderPort): ProviderPort => ({ + accountName: port.accountName, + authority: port.authority, + query: (q) => port.query(q), + readSyncState: () => port.readSyncState(), + readPhoto: (id, px) => port.readPhoto(id, px), + applyBatch: async (ops) => { + if (batches.crashAfter !== null && batches.applied >= batches.crashAfter) throw new CrashError(`after ${batches.applied} batches`); + const result = await port.applyBatch(ops); + if (result.ok) { + batches.applied++; + batches.log.push(ops); + } + return result; + }, + }); + + const connection = (): JmapConnection => ({ port: server.port(), origin: ORIGIN }); + const deps: EngineDeps = { + provider: (name, authority) => wrap(device.port(name, authority)), + jmap: async () => (options.rebuild === false ? connection() : openWithAuthRebuild(async () => connection())), + prefs: async () => prefs, + planners: { contacts: toyContactsPlanner, calendar: toyCalendarPlanner }, + now: () => clock.now, + isCancelled: async () => { + checkpoints.count++; + checkpoints.onCheckpoint?.(checkpoints.count); + if (checkpoints.crashAt !== null && checkpoints.count >= checkpoints.crashAt) throw new CrashError(`at checkpoint ${checkpoints.count}`); + return false; + }, + random: seededRandom(), + subscriptionCalendars: async () => [], + isSyncEnabled: async () => true, + deviceZone: () => 'Europe/Berlin', + recordStatus: vi.fn(), + recordKnownState: vi.fn(), + notifyAuthProblem: vi.fn(), + yieldThread: async () => undefined, + sleep: async () => undefined, + tuning: options.tuning, + }; + + let runs = 0; + const harness: Harness = { + server, + device, + deps, + prefs, + book, + calendar, + clock, + checkpoints, + batches, + run: (authority = CONTACTS_AUTHORITY, extras = {}) => + runDeviceSync( + { runId: `run-${++runs}`, accountName: ANDROID_ACCOUNT, registryId: REGISTRY_ID, authority, extras, deadline: clock.now + RUN_BUDGET }, + deps, + ), + teardown: (authority, teardownOptions) => teardownAuthority(deps, REGISTRY_ID, ANDROID_ACCOUNT, authority, teardownOptions), + state: (authority = CONTACTS_AUTHORITY) => parseJsonColumn(device.readSyncState(ANDROID_ACCOUNT, authority)), + contacts: () => + device.rows('raw_contacts').map((row) => { + const id = Number(row._id); + const nameRow = harness.nameDataRow(id); + return { + id, + sourceId: (row.sourceid as string | null) ?? null, + name: (nameRow?.[Data.DATA1] as string | null) ?? null, + dirty: Number(row.dirty) === 1, + deleted: Number(row.deleted) === 1, + row, + }; + }), + contactNamed: (name) => harness.contacts().find((c) => c.name === name), + nameDataRow: (rawContactId) => + device.rows('data').find((d) => Number(d.raw_contact_id) === rawContactId && d.mimetype === MimeType.STRUCTURED_NAME), + events: () => device.rows('events'), + serverNames: (accountId = 'a') => + server + .all('ContactCard', accountId) + .filter((c) => c.kind !== 'group') + .map((c) => String((c.name as { full?: string } | undefined)?.full ?? '')) + .sort(), + }; + return harness; +} + +/** A contact added on the device the way a Contacts app does (DIRTY, no SOURCE_ID). */ +export function addDeviceContact(h: Harness, name: string): number { + return h.device.user.insertContact(ANDROID_ACCOUNT, [{ [Data.MIMETYPE]: MimeType.STRUCTURED_NAME, [Data.DATA1]: name }]); +} + +/** An in-place edit of a contact's name (AOSP Contacts). */ +export function renameDeviceContact(h: Harness, rawContactId: number, name: string): void { + const row = h.nameDataRow(rawContactId); + if (!row) throw new Error(`no name row for ${rawContactId}`); + h.device.user.updateData(Number(row._id), { [Data.DATA1]: name, [Data.DATA2]: null, [Data.DATA3]: null }); +} + +export function addServerCards(h: Harness, names: string[], accountId = 'a', book = h.book): string[] { + return names.map((name, i) => h.server.addCard(accountId, { uid: `uid-${name}-${i}`, name: { full: name }, addressBookIds: { [book]: true } })); +} + +/** Batches that wrote rows (anything but the SyncState op). */ +export function rowWrites(batches: ProviderOp[][]): ProviderOp[][] { + return batches.filter((ops) => ops.some((op) => op.op !== 'syncState' && op.op !== 'assert')); +} + +export { CALENDAR_AUTHORITY, CONTACTS_AUTHORITY }; diff --git a/src/device-sync/__tests__/engine/internals.test.ts b/src/device-sync/__tests__/engine/internals.test.ts new file mode 100644 index 00000000..8b58b263 --- /dev/null +++ b/src/device-sync/__tests__/engine/internals.test.ts @@ -0,0 +1,328 @@ +// The engine's building blocks on their own: provider batches, the +// SyncState blob, the run lock, SetError handling, query paging, the auth +// rebuild and how failures map to outcomes. + +import { describe, expect, it, vi } from 'vitest'; +import { BatchWriter, buildBatch, prependOps, type Work } from '../../engine/batch'; +import { classifyFailure, RunAbort, StateMismatch, StopRun } from '../../engine/errors'; +import { acquireLock } from '../../engine/mutex'; +import { isBackedOff, nextMarker } from '../../engine/poison'; +import { collectionDefaultOn, isCollectionSelected } from '../../engine/selection'; +import { emptySyncState, parseSyncState, serializeSyncState, StateStore } from '../../engine/sync-state'; +import { classifySetError, existingIdFromUidError, JmapCaller } from '../../jmap/caller'; +import { openWithAuthRebuild } from '../../jmap/connection'; +import { JMAPMethodError } from '../../jmap/errors'; +import { accountsWithCapability, calendarAddresses } from '../../jmap/session'; +import { JMAP_CALENDARS, JMAP_CONTACTS, type BatchResult, type ProviderOp, type ProviderPort } from '../../types'; +import { FakeJmapServer } from '../fakes/fake-jmap-server'; + +function port(apply: (ops: ProviderOp[]) => BatchResult | Promise): ProviderPort & { sent: ProviderOp[][] } { + const sent: ProviderOp[][] = []; + return { + accountName: 'x', + authority: 'com.android.contacts', + sent, + query: async () => ({ columns: [], rows: [] }), + readSyncState: async () => null, + readPhoto: async () => null, + applyBatch: async (ops) => { + sent.push(ops); + return apply(ops); + }, + }; +} + +const ok = (ops: ProviderOp[]): BatchResult => ({ ok: true, results: ops.map(() => ({ count: 1 })) }); +const insert = (table: 'raw_contacts' | 'data', values: Record = {}): ProviderOp => ({ op: 'insert', table, values }); +const writer = (p: ProviderPort, options: Partial<{ maxOps: number; maxBytes: number; maxReplans: number }> = {}) => + new BatchWriter(p, { maxOps: 400, maxBytes: 300_000, maxReplans: 3, ...options }); + +describe('provider batches', () => { + it('rebases refs, makes each group start at a yield point and puts the state op last', () => { + const group = { ref: 'g', ops: [insert('raw_contacts'), { op: 'insert', table: 'data', values: {}, refs: { raw_contact_id: 0 }, yieldAllowed: true } as ProviderOp] }; + + const ops = buildBatch([group, group], { op: 'syncState', value: '{}', yieldAllowed: true } as ProviderOp); + + expect(ops.map((op) => (op as { yieldAllowed?: boolean }).yieldAllowed ?? false)).toEqual([true, false, true, false, false]); + expect((ops[3] as { refs: Record }).refs).toEqual({ raw_contact_id: 2 }); + expect(ops[4]).toEqual({ op: 'syncState', value: '{}' }); + }); + + it('prepends ops to a group, shifting its refs', () => { + const group = prependOps({ ref: 'g', ops: [insert('raw_contacts'), { op: 'insert', table: 'data', values: {}, refs: { raw_contact_id: 0 } }] }, [ + { op: 'assert', table: 'raw_contacts', where: 'sourceid = ?', args: ['a/1'], expectCount: 0 }, + ]); + expect((group.ops[2] as { refs: Record }).refs).toEqual({ raw_contact_id: 1 }); + }); + + it('never splits a group, keeps batches within the op budget and sends photo groups alone', async () => { + const p = port(ok); + const works: Work[] = [ + ...Array.from({ length: 5 }, (_, i) => ({ group: { ref: `g${i}`, ops: [insert('raw_contacts'), insert('data'), insert('data')] } })), + { group: { ref: 'photo', ops: [{ op: 'insert', table: 'data', values: { data15: { b64: 'AAAA' } } }] } }, + { group: { ref: 'g5', ops: [insert('raw_contacts')] } }, + ]; + + await writer(p, { maxOps: 7 }).write(works, { op: () => ({ op: 'syncState', value: 's' }), applied: () => undefined }); + + expect(p.sent.map((b) => b.length)).toEqual([6, 6, 3, 1, 2]); + expect(p.sent[3]).toEqual([expect.objectContaining({ values: { data15: { b64: 'AAAA' } } })]); + expect(p.sent[4][1]).toEqual({ op: 'syncState', value: 's' }); + }); + + it('retries the groups of a failed batch one by one and re-plans a failed assert at most three times', async () => { + let failing = 5; + const p = port((ops) => + ops.some((op) => op.op === 'assert') && failing-- > 0 ? { ok: false, reason: 'assert', message: 'stale' } : ok(ops), + ); + const applied: string[] = []; + const failed = vi.fn(); + let replans = 0; + const stale: Work = { + group: { ref: 'stale', ops: [{ op: 'assert', table: 'raw_contacts', id: 1, values: { version: 1 } }, insert('data')] }, + applied: () => { + applied.push('stale'); + }, + failed, + }; + stale.replan = async () => { + replans++; + return stale; + }; + const fine: Work = { + group: { ref: 'fine', ops: [insert('raw_contacts')] }, + applied: () => { + applied.push('fine'); + }, + }; + const tail = { op: () => ({ op: 'syncState', value: 's' }) as ProviderOp, applied: vi.fn() }; + + await writer(p).write([fine, stale], tail); + + expect(applied).toEqual(['fine']); + expect(replans).toBe(3); + expect(failed).toHaveBeenCalledWith('assert', 'stale'); + expect(tail.applied).toHaveBeenCalledOnce(); + expect(p.sent.at(-1)).toEqual([{ op: 'syncState', value: 's' }]); + }); + + it('splits a batch the Binder refused (tooLarge) and gives up a single group that is too large', async () => { + const p = port((ops) => (ops.length > 2 ? { ok: false, reason: 'tooLarge', message: 'TransactionTooLargeException' } : ok(ops))); + const failed = vi.fn(); + const works: Work[] = [ + ...Array.from({ length: 4 }, (_, i) => ({ group: { ref: `g${i}`, ops: [insert('raw_contacts')] } })), + { group: { ref: 'big', ops: [insert('raw_contacts'), insert('data'), insert('data')] }, failed }, + ]; + + await writer(p).write(works); + + expect(failed).toHaveBeenCalledWith('tooLarge', 'TransactionTooLargeException'); + expect(p.sent.filter((b) => b.length <= 2).flat()).toHaveLength(4); + }); + + it('ends the run when the permission is gone and refuses to retry an op outside the account', async () => { + const denied = port(() => ({ ok: false, reason: 'permission', message: 'SecurityException' })); + await expect(writer(denied).write([{ group: { ref: 'g', ops: [insert('raw_contacts')] } }])).rejects.toMatchObject({ outcome: 'permission' }); + const scope = port(() => ({ ok: false, reason: 'scope', message: 'not our account' })); + await expect(writer(scope).write([{ group: { ref: 'g', ops: [insert('raw_contacts')] } }])).rejects.toThrow(/not our account/); + }); + + it('skips groups that only assert', async () => { + const p = port(ok); + await writer(p).write([{ group: { ref: 'g', ops: [{ op: 'assert', table: 'raw_contacts', id: 1, values: { dirty: 0 } }] } }]); + expect(p.sent).toEqual([]); + }); +}); + +describe('SyncState', () => { + it('round-trips and tolerates missing parts', () => { + const state = emptySyncState(); + state.owner = { registryId: 'r', origin: 'https://o' }; + state.accounts.a = { collectionsState: 's1', itemsState: 's2', selected: ['a/b'], stale: ['c1'], reconcile: null, groups: ['a/g'] }; + state.deviceZone = 'Europe/Berlin'; + + expect(parseSyncState(serializeSyncState(state))).toEqual({ state, readable: true }); + expect(parseSyncState(JSON.stringify({ v: 1, accounts: { a: { itemsState: 'x' } } })).state.accounts.a).toEqual({ + collectionsState: null, + itemsState: 'x', + selected: [], + stale: [], + reconcile: null, + }); + }); + + it('starts over when the blob is unreadable or of another version', () => { + expect(parseSyncState('{nope').readable).toBe(false); + expect(parseSyncState(JSON.stringify({ v: 2, accounts: {} })).readable).toBe(false); + expect(parseSyncState(null)).toEqual({ state: emptySyncState(), readable: true }); + expect(parseSyncState('')).toEqual({ state: emptySyncState(), readable: true }); + }); + + it('adopts a proposed state only after its batch applied', () => { + const store = new StateStore(emptySyncState(), { registryId: 'r', origin: 'o' }); + const tail = store.tail((next) => { + next.deviceZone = 'UTC'; + }); + const op = tail.op(); + expect(store.committed.deviceZone).toBeUndefined(); + expect(JSON.parse((op as { value: string }).value)).toMatchObject({ deviceZone: 'UTC', owner: { registryId: 'r' } }); + tail.applied(); + expect(store.committed.deviceZone).toBe('UTC'); + }); +}); + +describe('run lock', () => { + it('serialises holders and hands on the turn of a waiter that gave up', async () => { + const order: string[] = []; + const first = await acquireLock('k'); + const second = acquireLock('k', 10); + const third = acquireLock('k'); + expect(await second).toBeNull(); + first!(); + const release = await third; + order.push('third'); + release!(); + expect(order).toEqual(['third']); + const again = await acquireLock('k', 10); + expect(again).not.toBeNull(); + again!(); + }); +}); + +describe('poison markers', () => { + it('backs off an hour, doubling to a day, while the item stays the same', () => { + const first = nextMarker(null, 'fp', { type: 'invalidPatch' }, 0, { firstMs: 3_600_000, maxMs: 86_400_000 }); + const second = nextMarker(first, 'fp', { type: 'invalidPatch' }, 0, { firstMs: 3_600_000, maxMs: 86_400_000 }); + let marker = second; + for (let i = 0; i < 10; i++) marker = nextMarker(marker, 'fp', { type: 'invalidPatch' }, 0, { firstMs: 3_600_000, maxMs: 86_400_000 }); + expect([first.until, second.until, marker.until]).toEqual([3_600_000, 7_200_000, 86_400_000]); + expect(nextMarker(second, 'changed', { type: 'invalidPatch' }, 0, { firstMs: 1, maxMs: 10 }).n).toBe(1); + expect(isBackedOff(first, 'fp', 1)).toBe(true); + expect(isBackedOff(first, 'other', 1)).toBe(false); + expect(isBackedOff(first, 'fp', 3_600_001)).toBe(false); + }); +}); + +describe('JMAP side', () => { + it('classifies SetErrors', () => { + expect(classifySetError({ type: 'notFound' }, 'update')).toBe('notFound'); + expect(classifySetError({ type: 'forbidden' }, 'destroy')).toBe('forbidden'); + expect(classifySetError({ type: 'invalidProperties', properties: ['uid'] }, 'create')).toBe('uidExists'); + expect(classifySetError({ type: 'invalidProperties', properties: ['uid'] }, 'update')).toBe('poison'); + expect(classifySetError({ type: 'overQuota' }, 'create')).toBe('poison'); + expect(existingIdFromUidError({ type: 'invalidProperties', description: 'Contact with UID u1 already exists with id c5.' })).toBe('c5'); + expect(existingIdFromUidError({ type: 'invalidProperties', description: 'An event with UID u1 already exists.' })).toBeNull(); + }); + + it('pages /query by position until the total, with overlapping pages', async () => { + const server = new FakeJmapServer(); + server.addAccount('a', { name: 'alice@example.com' }); + const book = server.addAddressBook('a', { name: 'Personal' }); + for (let i = 0; i < 30; i++) server.addCard('a', { uid: `u${i}` }); + server.setLimits({ maxQueryResults: 8 }); + const caller = new JmapCaller(server.port(), () => 0); + + const ids = await caller.queryAll('ContactCard', 'a', { inAddressBook: book }); + + expect(ids).toHaveLength(30); + expect(server.calls('ContactCard/query').length).toBeGreaterThan(3); + }); + + it('looks uids up in one query and maps them back', async () => { + const server = new FakeJmapServer(); + server.addAccount('a', { name: 'alice@example.com' }); + const book = server.addAddressBook('a', { name: 'Personal' }); + const other = server.addAddressBook('a', { name: 'Other' }); + const c1 = server.addCard('a', { uid: 'u1', addressBookIds: { [book]: true } }); + server.addCard('a', { uid: 'u2', addressBookIds: { [other]: true } }); + const caller = new JmapCaller(server.port(), () => 0); + + const found = await caller.lookupUids('ContactCard', 'a', ['u1', 'u2', 'u3'], book); + + expect([...found]).toEqual([['u1', [c1]]]); + }); + + it('names the accounts with a capability, the primary first', () => { + const server = new FakeJmapServer(); + server.addAccount('team', { name: 'Team', isPersonal: false, capabilities: ['contacts'] }); + server.addAccount('a', { name: 'alice@example.com' }); + const session = server.port().session(); + + expect(accountsWithCapability(session, JMAP_CONTACTS).map((a) => [a.id, a.primary, a.personal])).toEqual([ + ['a', true, true], + ['team', false, false], + ]); + expect(accountsWithCapability(session, JMAP_CALENDARS).map((a) => a.id)).toEqual(['a']); + }); + + it('takes the login as the owner address and survives a server without identities', async () => { + const server = new FakeJmapServer(); + server.addAccount('a', { name: 'Alice@Example.com' }); + const addresses = await calendarAddresses(new JmapCaller(server.port(), () => 0), 'fallback'); + expect(addresses).toEqual({ ownerAccount: 'alice@example.com', selfAddresses: ['alice@example.com'] }); + }); + + it('rebuilds the connection once when the credentials are rejected', async () => { + const server = new FakeJmapServer(); + server.addAccount('a', { name: 'alice@example.com' }); + const load = vi.fn(async () => ({ port: server.port(), origin: 'o' })); + const connection = await openWithAuthRebuild(load); + server.failNextRequest('auth'); + + await connection.port.request([['Core/echo', {}, '0']], ['urn:ietf:params:jmap:core']); + expect(load).toHaveBeenCalledTimes(2); + + server.failNextRequest('auth'); + await expect(connection.port.request([['Core/echo', {}, '0']], ['urn:ietf:params:jmap:core'])).rejects.toMatchObject({ name: 'AuthenticationError' }); + expect(load).toHaveBeenCalledTimes(2); + }); + + it('opens with a second client when the first has no session, and gives up after that', async () => { + const connection = { port: new FakeJmapServer().port(), origin: 'o' }; + const loads = [null, connection]; + await expect(openWithAuthRebuild(async () => loads.shift() ?? null)).resolves.toMatchObject({ origin: 'o' }); + expect(loads).toEqual([]); + const rejected = Object.assign(new Error('401'), { name: 'AuthenticationError' }); + await expect(openWithAuthRebuild(async () => { throw rejected; })).rejects.toMatchObject({ name: 'AuthenticationError' }); + await expect(openWithAuthRebuild(async () => null)).rejects.toMatchObject({ name: 'AuthenticationError' }); + const offline = Object.assign(new Error('offline'), { name: 'NetworkError' }); + await expect(openWithAuthRebuild(async () => { throw offline; })).rejects.toBe(offline); + }); +}); + +describe('selection defaults', () => { + it("syncs the personal account's collections, not shared ones nor the Trusted Senders book, unless chosen", () => { + expect(collectionDefaultOn(true, 'com.android.contacts', 'Personal')).toBe(true); + expect(collectionDefaultOn(true, 'com.android.contacts', 'Trusted Senders')).toBe(false); + expect(collectionDefaultOn(true, 'com.android.calendar', 'Trusted Senders')).toBe(true); + expect(collectionDefaultOn(false, 'com.android.contacts', 'Team')).toBe(false); + expect(collectionDefaultOn(true, 'com.android.calendar', null)).toBe(true); + + const personal = { id: 'a', personal: true }; + const trusted = { id: 'ab9', name: 'Trusted Senders' }; + expect(isCollectionSelected({}, personal, 'com.android.contacts', trusted)).toBe(false); + expect(isCollectionSelected({ 'a/ab9': true }, personal, 'com.android.contacts', trusted)).toBe(true); + expect(isCollectionSelected({ 'a/ab1': false }, personal, 'com.android.contacts', { id: 'ab1', name: 'Personal' })).toBe(false); + expect(isCollectionSelected({ 't/c1': true }, { id: 't', personal: false }, 'com.android.calendar', { id: 'c1' })).toBe(true); + }); +}); + +describe('outcomes', () => { + it('maps failures to report outcomes', () => { + const named = (name: string, extra: object = {}) => Object.assign(new Error(name), { name, ...extra }); + expect(classifyFailure(new StopRun('deadline'), 0)).toMatchObject({ outcome: 'cancelled', moreRecordsToGet: true }); + expect(classifyFailure(new RunAbort('safetyAbort', 'x'), 0)).toMatchObject({ outcome: 'safetyAbort' }); + expect(classifyFailure(new StateMismatch('a'), 0)).toMatchObject({ outcome: 'io', noProgress: true }); + expect(classifyFailure(named('AuthenticationError'), 0)).toMatchObject({ outcome: 'auth', authProblem: true }); + expect(classifyFailure(named('NetworkError'), 0)).toMatchObject({ outcome: 'io' }); + expect(classifyFailure(named('RequestTimeoutError'), 0)).toMatchObject({ outcome: 'io' }); + expect(classifyFailure(named('RateLimitError', { retryAfterMs: 5000 }), 10_000)).toMatchObject({ outcome: 'io', delayUntil: 15 }); + expect(classifyFailure(new JMAPMethodError('serverFail'), 0)).toMatchObject({ outcome: 'io' }); + expect(classifyFailure(new Error('JMAP request failed: 503 - busy'), 0)).toMatchObject({ outcome: 'io' }); + expect( + classifyFailure(new Error('JMAP request failed: 400 - {"type":"urn:ietf:params:jmap:error:limit","limit":"maxCallsInRequest"}'), 0), + ).toMatchObject({ outcome: 'internal' }); + expect(classifyFailure(Object.assign(new Error('denied'), { code: 'permission' }), 0)).toMatchObject({ outcome: 'permission' }); + expect(classifyFailure(new TypeError('x is undefined'), 0)).toMatchObject({ outcome: 'internal' }); + }); +}); diff --git a/src/device-sync/__tests__/engine/task.test.ts b/src/device-sync/__tests__/engine/task.test.ts new file mode 100644 index 00000000..384fc21c --- /dev/null +++ b/src/device-sync/__tests__/engine/task.test.ts @@ -0,0 +1,63 @@ +// The headless task entry: every run ends with finishRun and nothing +// escapes it (docs/device-sync.md, "The Android shell"). + +import { beforeEach, describe, expect, it, vi } from 'vitest'; + +vi.mock('../../native', () => ({ + createProviderPort: vi.fn(), + finishRun: vi.fn(async () => true), + getSyncSettings: vi.fn(async () => ({ + masterAutomatic: true, + authorities: { 'com.android.contacts': { automatic: false, syncable: 1, periodicSeconds: 0, active: false, pending: false } }, + })), + isRunCancelled: vi.fn(async () => false), + showSyncProblem: vi.fn(async () => undefined), +})); + +import * as native from '../../native'; +import { runDeviceSyncTask, teardownAuthority } from '../../task'; +import { useDeviceSyncStore } from '../../../stores/device-sync-store'; +import type { RunPayload } from '../../types'; + +const payload: RunPayload = { + runId: 'r1', + accountName: 'alice@example.com', + registryId: 'alice@mail.example.com', + authority: 'com.android.contacts', + extras: {}, + deadline: Date.now() + 60_000, +}; + +describe('device sync task', () => { + beforeEach(() => { + vi.mocked(native.finishRun).mockClear(); + }); + + it('hands the report to the adapter before settling, and records the status', async () => { + await runDeviceSyncTask(payload); + + expect(native.finishRun).toHaveBeenCalledWith('r1', expect.objectContaining({ v: 1, runId: 'r1', outcome: 'disabled' })); + expect(useDeviceSyncStore.getState().accounts['alice@mail.example.com']?.lastRun?.['com.android.contacts']).toMatchObject({ + outcome: 'disabled', + }); + }); + + it('reports a failure instead of throwing, and survives a refused finishRun', async () => { + vi.mocked(native.getSyncSettings).mockRejectedValueOnce(new Error('module gone')); + vi.mocked(native.finishRun).mockRejectedValueOnce(new Error('bridge down')); + + await expect(runDeviceSyncTask({ ...payload, runId: 'r2' })).resolves.toBeUndefined(); + + expect(native.finishRun).toHaveBeenCalledWith('r2', expect.objectContaining({ outcome: 'internal', message: 'module gone' })); + }); + + it('ignores a payload without a run', async () => { + await runDeviceSyncTask({} as RunPayload); + expect(native.finishRun).not.toHaveBeenCalled(); + }); + + it('exports the teardown the app calls', () => { + expect(teardownAuthority).toBeTypeOf('function'); + expect(teardownAuthority.length).toBe(4); + }); +}); diff --git a/src/device-sync/__tests__/engine/toy-planners.ts b/src/device-sync/__tests__/engine/toy-planners.ts new file mode 100644 index 00000000..8bb7136a --- /dev/null +++ b/src/device-sync/__tests__/engine/toy-planners.ts @@ -0,0 +1,657 @@ +/** + * Small planners for the engine tests, faithful to the planner contract + * (src/device-sync/planner.ts) for a toy mapping: + * + * - contacts: `name.full` ↔ one StructuredName row (DISPLAY_NAME), group + * cards ↔ Groups rows (TITLE), memberships ↔ GroupMembership rows; + * - calendar: `title` ↔ one event row (plus the columns the provider + * requires), calendars ↔ Calendars rows. + * + * Like the real planners they assert what they read first (VERSION and + * DIRTY, or the event projection), keep baselines in DATA_SYNC3 / + * SYNC_DATA4, merge dirty items per unit (server wins a real conflict), + * adopt pending creates by uid, claim new items before creating them, and + * return `effect: 'none'` for echoes. + */ +import { Data, Events, GroupMembership, MimeType } from '../../android-columns'; +import { collectionKey, parseCollectionKey, parseObjectRef, pendingRef, pendingUidOf } from '../../common/ids'; +import { deepEqual, parseJsonColumn } from '../../common/json'; +import { ptr } from '../../common/patch'; +import type { + AcceptedPlan, + CalendarContext, + CalendarEventWire, + CalendarPlanner, + ContactCardWire, + ContactsContext, + ContactsPlanner, + DownloadPlan, + LocalCalendar, + LocalContact, + LocalEvent, + LocalGroup, + OpGroup, + UploadAction, + UploadPlan, +} from '../../planner'; +import type { ProviderOp, Row } from '../../types'; + +const num = (v: unknown) => Number(v ?? 0); +const flag = (v: unknown) => Number(v ?? 0) === 1; +const str = (v: unknown): string | null => (typeof v === 'string' && v ? v : null); +const onIds = (map: Record | undefined | null) => + Object.entries(map ?? {}).filter(([, on]) => on === true).map(([id]) => id); + +interface ToyContactsContext extends ContactsContext { + groupsOf?(uid: string): string[]; +} + +// ─── Contacts ─────────────────────────────────────────── + +function nameOf(card: ContactCardWire | null | undefined): string { + return card?.name?.full ?? ''; +} + +function nameRow(local: LocalContact) { + return local.rows.find((r) => r.mimetype === MimeType.STRUCTURED_NAME) ?? null; +} + +function membershipRows(local: LocalContact) { + return local.rows.filter((r) => r.mimetype === MimeType.GROUP_MEMBERSHIP); +} + +function cardCollections(acct: string, card: ContactCardWire): string { + return onIds(card.addressBookIds).map((id) => collectionKey(acct, id)).join(','); +} + +function shadowOf(card: ContactCardWire): string { + return JSON.stringify(card); +} + +/** Group SOURCE_IDs a card should be a member of, with their row ids. */ +function wantedGroups(card: ContactCardWire, ctx: ToyContactsContext): Array<{ sourceId: string; rowId: number }> { + if (typeof card.uid !== 'string' || !ctx.groupsOf) return []; + return ctx + .groupsOf(card.uid) + .map((sourceId) => ({ sourceId, rowId: ctx.groupRowIdBySourceId(sourceId) })) + .filter((g): g is { sourceId: string; rowId: number } => g.rowId !== null); +} + +function membershipOps(local: LocalContact | null, card: ContactCardWire, ctx: ToyContactsContext, parent: number | null): ProviderOp[] { + const wanted = wantedGroups(card, ctx); + const have = local ? membershipRows(local) : []; + const ops: ProviderOp[] = []; + for (const row of have) { + if (!wanted.some((g) => g.sourceId === row.cells.group_sourceid)) ops.push({ op: 'delete', table: 'data', id: row.id, expectCount: 1 }); + } + for (const group of wanted) { + if (have.some((r) => r.cells.group_sourceid === group.sourceId)) continue; + const values = { [Data.MIMETYPE]: MimeType.GROUP_MEMBERSHIP, [GroupMembership.GROUP_ROW_ID]: group.rowId }; + ops.push( + parent === null + ? { op: 'insert', table: 'data', values, refs: { [Data.RAW_CONTACT_ID]: 0 } } + : { op: 'insert', table: 'data', values: { ...values, [Data.RAW_CONTACT_ID]: parent } }, + ); + } + return ops; +} + +export const toyContactsPlanner: ContactsPlanner = { + rawContactColumns: ['_id', 'sourceid', 'version', 'dirty', 'deleted', 'sync1', 'sync2', 'sync3', 'sync4', 'raw_contact_is_read_only'], + dataColumns: ['_id', 'raw_contact_id', 'mimetype', 'data1', 'data_sync1', 'data_sync3', 'group_sourceid'], + groupColumns: ['_id', 'sourceid', 'version', 'dirty', 'deleted', 'title', 'sync2', 'sync3', 'sync4'], + + decodeContact(rc: Row, data: Row[]): LocalContact { + return { + rawContactId: num(rc._id), + sourceId: str(rc.sourceid), + version: num(rc.version), + dirty: flag(rc.dirty), + deleted: flag(rc.deleted), + readOnly: flag(rc.raw_contact_is_read_only), + collections: (str(rc.sync1) ?? '').split(',').filter(Boolean), + shadow: parseJsonColumn(rc.sync2), + pending: parseJsonColumn(rc.sync3), + poison: parseJsonColumn(rc.sync4), + rows: data.map((d) => ({ + id: num(d._id), + mimetype: String(d.mimetype), + cells: { data1: d.data1 ?? null, group_sourceid: d.group_sourceid ?? null }, + key: str(d.data_sync1), + photoHash: null, + baseline: parseJsonColumn(d.data_sync3), + })), + }; + }, + + decodeGroup(row: Row): LocalGroup { + return { + groupId: num(row._id), + sourceId: str(row.sourceid), + version: num(row.version), + dirty: flag(row.dirty), + deleted: flag(row.deleted), + title: str(row.title), + shadow: parseJsonColumn(row.sync2), + pending: parseJsonColumn(row.sync3), + poison: parseJsonColumn(row.sync4), + }; + }, + + planDownload(card, local, context): DownloadPlan { + const ctx = context as ToyContactsContext; + const acct = ctx.jmapAccountId; + const ref = `${acct}/${card.id}`; + const name = nameOf(card); + const raw = { sourceid: ref, sync1: cardCollections(acct, card), sync2: shadowOf(card), raw_contact_is_read_only: ctx.isReadOnly(card) ? 1 : 0 }; + if (!local) { + const ops: ProviderOp[] = [ + { op: 'insert', table: 'raw_contacts', values: raw }, + { + op: 'insert', + table: 'data', + values: { [Data.MIMETYPE]: MimeType.STRUCTURED_NAME, data1: name, data_sync1: 'name', data_sync3: JSON.stringify({ data1: name }) }, + refs: { [Data.RAW_CONTACT_ID]: 0 }, + }, + ...membershipOps(null, card, ctx, null), + ]; + return { ops: { ref, ops }, conflicts: 0, stillDirty: false, effect: 'insert', writes: ops.length }; + } + const row = nameRow(local); + const guard: ProviderOp = { + op: 'assert', + table: 'raw_contacts', + id: local.rawContactId, + values: { version: local.version, dirty: local.dirty ? 1 : 0 }, + expectCount: 1, + }; + const ops: ProviderOp[] = [guard]; + let conflicts = 0; + let stillDirty = false; + const writeName = (value: string) => { + if (row) { + ops.push({ op: 'update', table: 'data', id: row.id, values: { data1: value, data_sync3: JSON.stringify({ data1: value }) }, expectCount: 1 }); + } else { + ops.push({ + op: 'insert', + table: 'data', + values: { [Data.RAW_CONTACT_ID]: local.rawContactId, [Data.MIMETYPE]: MimeType.STRUCTURED_NAME, data1: value, data_sync1: 'name', data_sync3: JSON.stringify({ data1: value }) }, + }); + } + }; + const adopting = !local.sourceId && local.pending?.uid === card.uid; + if (local.deleted) { + // The device delete wins over a server edit; the destroy is on its way. + return { ops: { ref, ops: [guard] }, conflicts: 0, stillDirty: true, effect: 'none', writes: 0 }; + } + const localValue = String(row?.cells.data1 ?? ''); + if (adopting) { + // Our own create: identity and shadow; the rows stay the user's, measured against the server's value. + ops.push({ op: 'update', table: 'raw_contacts', id: local.rawContactId, values: { ...raw, sync3: null } }); + if (row) ops.push({ op: 'update', table: 'data', id: row.id, values: { data_sync3: JSON.stringify({ data1: name }) }, expectCount: 1 }); + return { ops: { ref, ops }, conflicts: 0, stillDirty: localValue !== name, effect: 'update', writes: ops.length - 1 }; + } + const shadowName = nameOf(local.shadow); + const baseValue = String(row?.baseline?.data1 ?? shadowName); + const localChanged = local.dirty && localValue !== baseValue; + const remoteChanged = name !== shadowName; + if (!localChanged) { + if (localValue !== name) writeName(name); + } else if (remoteChanged) { + if (localValue === name) writeName(name); + else { + writeName(name); + conflicts++; + } + } else { + stillDirty = true; + } + ops.push(...membershipOps(local, card, ctx, local.rawContactId)); + const shadowSame = deepEqual(local.shadow, card) && local.collections.join(',') === raw.sync1 && local.readOnly === !!raw.raw_contact_is_read_only; + if (!shadowSame) ops.push({ op: 'update', table: 'raw_contacts', id: local.rawContactId, values: raw }); + const writes = ops.length - 1; + return { ops: { ref, ops }, conflicts, stillDirty, effect: writes > 0 ? 'update' : 'none', writes }; + }, + + planLocalDelete(local): OpGroup { + return { ref: local.sourceId ?? `row:${local.rawContactId}`, ops: [{ op: 'delete', table: 'raw_contacts', id: local.rawContactId }] }; + }, + + planBaselineHeal(local): OpGroup | null { + const row = nameRow(local); + if (local.dirty || !row || deepEqual(row.baseline, { data1: row.cells.data1 })) return null; + return { + ref: local.sourceId ?? `row:${local.rawContactId}`, + ops: [ + { op: 'assert', table: 'raw_contacts', id: local.rawContactId, values: { version: local.version, dirty: 0 }, expectCount: 1 }, + { op: 'update', table: 'data', id: row.id, values: { data_sync3: JSON.stringify({ data1: row.cells.data1 }) }, expectCount: 1 }, + ], + }; + }, + + planUpload(local, ctx): UploadPlan { + const ref = local.sourceId ?? `row:${local.rawContactId}`; + const guard: ProviderOp = { op: 'assert', table: 'raw_contacts', id: local.rawContactId, values: { version: local.version }, expectCount: 1 }; + const parsed = parseObjectRef(local.sourceId); + if (local.deleted) { + if (parsed) { + const books = onIds(local.shadow?.addressBookIds); + const synced = books.filter((b) => ctx.isSelected(collectionKey(parsed.accountId, b))); + if (synced.length && synced.length < books.length) { + return { kind: 'upload', actions: [{ kind: 'update', id: parsed.id, patch: Object.fromEntries(synced.map((b) => [`addressBookIds/${b}`, null])) }] }; + } + return { kind: 'upload', actions: [{ kind: 'destroy', id: parsed.id, uid: null }] }; + } + if (local.pending) return { kind: 'upload', actions: [{ kind: 'destroy', id: null, uid: local.pending.uid }] }; + return { kind: 'purge', ops: { ref, ops: [{ op: 'delete', table: 'raw_contacts', id: local.rawContactId }] } }; + } + const row = nameRow(local); + const value = String(row?.cells.data1 ?? ''); + if (!parsed) { + if (!local.pending) { + const target = ctx.createTarget(); + if (!target) return { kind: 'skip', reason: 'noWritableAddressBook' }; + return { + kind: 'claim', + ops: { ref, ops: [guard, { op: 'update', table: 'raw_contacts', id: local.rawContactId, values: { sync3: JSON.stringify({ uid: ctx.mintUid(), target }) } }] }, + }; + } + const target = parseCollectionKey(local.pending.target); + if (!target) return { kind: 'skip', reason: 'badTarget' }; + return { + kind: 'upload', + actions: [{ kind: 'create', uid: local.pending.uid, collectionId: target.id, object: { uid: local.pending.uid, name: { full: value } } }], + }; + } + if (local.readOnly) { + const shadowName = nameOf(local.shadow); + return { + kind: 'revert', + ops: { + ref, + ops: [ + guard, + ...(row ? [{ op: 'update' as const, table: 'data' as const, id: row.id, values: { data1: shadowName, data_sync3: JSON.stringify({ data1: shadowName }) } }] : []), + { op: 'update', table: 'raw_contacts', id: local.rawContactId, values: { dirty: 0 } }, + ], + }, + }; + } + const base = String(row?.baseline?.data1 ?? nameOf(local.shadow)); + if (value === base) { + return { kind: 'clean', ops: { ref, ops: [{ ...guard, values: { version: local.version, dirty: 1 } }, { op: 'update', table: 'raw_contacts', id: local.rawContactId, values: { dirty: 0 } }] } }; + } + const patch = local.shadow?.name ? { 'name/full': value } : { name: { full: value } }; + return { kind: 'upload', actions: [{ kind: 'update', id: parsed.id, patch }] }; + }, + + planAccepted(local, server, ctx): AcceptedPlan { + const acct = ctx.jmapAccountId; + const ref = `${acct}/${server.id}`; + const row = nameRow(local); + const identity = { sourceid: ref, sync1: cardCollections(acct, server), sync2: shadowOf(server), sync3: null }; + const baseline: ProviderOp[] = row + ? [{ op: 'update', table: 'data', id: row.id, values: { data_sync3: JSON.stringify({ data1: row.cells.data1 }) }, expectCount: 1 }] + : []; + return { + ops: { + ref, + ops: [ + { op: 'assert', table: 'raw_contacts', id: local.rawContactId, values: { version: local.version }, expectCount: 1 }, + { op: 'update', table: 'raw_contacts', id: local.rawContactId, values: { ...identity, dirty: 0 } }, + ...baseline, + ], + }, + keepDirtyOps: { ref, ops: [{ op: 'update', table: 'raw_contacts', id: local.rawContactId, values: identity }, ...baseline] }, + }; + }, + + planGroupDownload(card, local, ctx): DownloadPlan { + const acct = ctx.jmapAccountId; + const ref = `${acct}/${card.id}`; + const values = { sourceid: ref, title: nameOf(card), group_visible: 1, sync2: shadowOf(card) }; + if (!local) { + return { ops: { ref, ops: [{ op: 'insert', table: 'groups', values }] }, conflicts: 0, stillDirty: false, effect: 'insert', writes: 1 }; + } + if (local.deleted) return { ops: { ref, ops: [] }, conflicts: 0, stillDirty: true, effect: 'none', writes: 0 }; + const adopting = !local.sourceId && local.pending?.uid === card.uid; + if (!adopting && local.title === values.title && deepEqual(local.shadow, card)) { + return { ops: { ref, ops: [] }, conflicts: 0, stillDirty: local.dirty, effect: 'none', writes: 0 }; + } + const localChanged = local.dirty && local.title !== nameOf(local.shadow); + const keepTitle = adopting || (localChanged && values.title === nameOf(local.shadow)); + return { + ops: { + ref, + ops: [ + { op: 'assert', table: 'groups', id: local.groupId, values: { version: local.version }, expectCount: 1 }, + { op: 'update', table: 'groups', id: local.groupId, values: { ...values, ...(keepTitle ? { title: local.title } : {}), sync3: null } }, + ], + }, + conflicts: localChanged && !keepTitle && values.title !== local.title ? 1 : 0, + stillDirty: keepTitle && local.dirty, + effect: 'update', + writes: 1, + }; + }, + + planGroupLocalDelete(local): OpGroup { + return { ref: local.sourceId ?? `group:${local.groupId}`, ops: [{ op: 'delete', table: 'groups', id: local.groupId }] }; + }, + + planGroupUpload(local, ctx): UploadPlan { + const ref = local.sourceId ?? `group:${local.groupId}`; + const parsed = parseObjectRef(local.sourceId); + if (local.deleted) { + if (parsed) return { kind: 'upload', actions: [{ kind: 'destroy', id: parsed.id, uid: null }] }; + return { kind: 'purge', ops: { ref, ops: [{ op: 'delete', table: 'groups', id: local.groupId }] } }; + } + if (!parsed) { + if (!local.pending) { + const target = ctx.createTarget(); + if (!target) return { kind: 'skip', reason: 'noWritableAddressBook' }; + return { + kind: 'claim', + ops: { + ref, + ops: [ + { op: 'assert', table: 'groups', id: local.groupId, values: { version: local.version }, expectCount: 1 }, + { op: 'update', table: 'groups', id: local.groupId, values: { sync3: JSON.stringify({ uid: ctx.mintUid(), target }) } }, + ], + }, + }; + } + const target = parseCollectionKey(local.pending.target); + if (!target) return { kind: 'skip', reason: 'badTarget' }; + return { + kind: 'upload', + actions: [{ kind: 'create', uid: local.pending.uid, collectionId: target.id, object: { uid: local.pending.uid, kind: 'group', name: { full: local.title ?? '' } } }], + }; + } + if ((local.title ?? '') === nameOf(local.shadow)) { + return { + kind: 'clean', + ops: { ref, ops: [{ op: 'assert', table: 'groups', id: local.groupId, values: { version: local.version }, expectCount: 1 }, { op: 'update', table: 'groups', id: local.groupId, values: { dirty: 0 } }] }, + }; + } + return { kind: 'upload', actions: [{ kind: 'update', id: parsed.id, patch: { 'name/full': local.title ?? '' } }] }; + }, + + planGroupAccepted(local, server, ctx): AcceptedPlan { + const ref = `${ctx.jmapAccountId}/${server.id}`; + const values = { sourceid: ref, sync2: shadowOf(server), sync3: null }; + return { + ops: { + ref, + ops: [ + { op: 'assert', table: 'groups', id: local.groupId, values: { version: local.version }, expectCount: 1 }, + { op: 'update', table: 'groups', id: local.groupId, values: { ...values, dirty: 0 } }, + ], + }, + keepDirtyOps: { ref, ops: [{ op: 'update', table: 'groups', id: local.groupId, values }] }, + }; + }, + + planMembershipUploads(contacts, groups, ctx): UploadAction[] { + const acct = ctx.jmapAccountId; + const patches = new Map>(); + for (const contact of contacts) { + const uid = contact.shadow?.uid ?? contact.pending?.uid; + if (!contact.sourceId || typeof uid !== 'string') continue; + const wanted = new Set(membershipRows(contact).map((r) => String(r.cells.group_sourceid))); + for (const group of groups) { + const parsed = parseObjectRef(group.sourceId); + if (!parsed || parsed.accountId !== acct) continue; + const member = group.shadow?.members?.[uid] === true; + const want = wanted.has(group.sourceId as string); + if (member === want) continue; + const patch = patches.get(parsed.id) ?? {}; + patch[ptr('members', uid)] = want ? true : null; + patches.set(parsed.id, patch); + } + } + return [...patches].map(([id, patch]) => ({ kind: 'update', id, patch })); + }, +}; + +// ─── Calendar ─────────────────────────────────────────── + +const EVENT_HOUR = 3_600_000; + +function startMs(event: CalendarEventWire): number { + return Date.parse(`${String(event.start ?? '2026-01-01T00:00:00').slice(0, 19)}Z`); +} + +function localStart(ms: number): string { + return new Date(ms).toISOString().slice(0, 19); +} + +function eventRef(ctx: CalendarContext, event: CalendarEventWire): string { + return `${ctx.jmapAccountId}/${event.id}`; +} + +export const toyCalendarPlanner: CalendarPlanner = { + calendarColumns: ['_id', '_sync_id', 'name', 'calendar_displayName', 'calendar_access_level', 'maxReminders', 'cal_sync2', 'cal_sync3'], + eventColumns: [ + '_id', 'calendar_id', '_sync_id', 'dirty', 'deleted', 'title', 'dtstart', 'dtend', 'eventTimezone', 'original_id', + 'original_sync_id', 'originalInstanceTime', 'sync_data1', 'sync_data2', 'sync_data3', 'sync_data4', 'sync_data5', 'uid2445', 'mutators', + ], + attendeeColumns: ['_id', 'event_id', 'attendeeEmail'], + reminderColumns: ['_id', 'event_id', 'minutes'], + + decodeCalendar(row: Row): LocalCalendar { + return { + calendarRowId: num(row._id), + syncId: str(row._sync_id), + cells: row, + shadow: parseJsonColumn(row.cal_sync2), + flags: parseJsonColumn(row.cal_sync3), + }; + }, + + decodeEvents(events: Row[], attendees: Row[], reminders: Row[]): LocalEvent[] { + const toRow = (row: Row) => ({ + eventId: num(row._id), + calendarRowId: num(row.calendar_id), + syncId: str(row._sync_id), + dirty: flag(row.dirty), + deleted: flag(row.deleted), + cells: { title: row.title ?? null, dtstart: row.dtstart ?? null, dtend: row.dtend ?? null }, + attendees: attendees.filter((a) => num(a.event_id) === num(row._id)).map((a) => ({ id: num(a._id), cells: { attendeeEmail: a.attendeeEmail ?? null } })), + reminders: reminders.filter((r) => num(r.event_id) === num(row._id)).map((r) => ({ id: num(r._id), cells: { minutes: r.minutes ?? null } })), + baseline: parseJsonColumn<{ cells: Row; attendees: Row[]; reminders: Row[] }>(row.sync_data4), + pending: parseJsonColumn(row.sync_data3), + poison: parseJsonColumn(row.sync_data5), + mutators: str(row.mutators), + }); + const isException = (row: Row) => row.original_id !== null && row.original_id !== undefined || !!str(row.original_sync_id); + const masters = events.filter((r) => !isException(r)).map((row) => ({ + ...toRow(row), + shadow: parseJsonColumn(row.sync_data1), + exceptions: events + .filter((x) => isException(x) && (num(x.original_id) === num(row._id) || (str(x.original_sync_id) !== null && str(x.original_sync_id) === str(row._sync_id)))) + .map((x) => ({ ...toRow(x), recurrenceId: str(x.sync_data2), originalInstanceTime: x.originalInstanceTime === null ? null : num(x.originalInstanceTime) })), + })) as LocalEvent[]; + const bySyncId = new Map(); + for (const m of masters) if (m.syncId) bySyncId.set(m.syncId, [...(bySyncId.get(m.syncId) ?? []), m]); + for (const pair of bySyncId.values()) { + if (pair.length < 2) continue; + pair.sort((a, b) => a.eventId - b.eventId); + pair[0].split = 'source'; + for (const clone of pair.slice(1)) clone.split = 'clone'; + } + return masters; + }, + + planCalendars(selected, local, ctx) { + const groups: OpGroup[] = []; + const keys = new Set(); + for (const { jmapAccountId, calendar, readOnly, accountName } of selected) { + const key = collectionKey(jmapAccountId, calendar.id); + keys.add(key); + const values = { + name: calendar.name, + calendar_displayName: accountName ? `${calendar.name} (${accountName})` : calendar.name, + calendar_access_level: readOnly ? 200 : 700, + maxReminders: ctx.reminderOwner === 'device' ? 5 : 0, + cal_sync2: JSON.stringify(calendar), + cal_sync3: JSON.stringify(readOnly ? { readOnly: 'rights' } : {}), + }; + const existing = local.find((l) => l.syncId === key); + if (!existing) { + groups.push({ + ref: key, + ops: [{ op: 'insert', table: 'calendars', values: { ...values, _sync_id: key, calendar_color: -16776961, ownerAccount: ctx.ownerAccount, sync_events: 1, visible: 1 } }], + }); + } else if (Object.entries(values).some(([k, v]) => String(existing.cells[k] ?? '') !== String(v))) { + groups.push({ ref: key, ops: [{ op: 'update', table: 'calendars', id: existing.calendarRowId, values }] }); + } + } + return { groups, deleteCalendarRows: local.filter((l) => !l.syncId || !keys.has(l.syncId)).map((l) => l.calendarRowId) }; + }, + + planDownload(event, local, ctx): DownloadPlan { + const ref = eventRef(ctx, event); + const calendarRowId = onIds(event.calendarIds).map((id) => ctx.calendarRowId(id)).find((id) => id !== null) ?? null; + const start = startMs(event); + const cells = { title: event.title ?? '', dtstart: start, dtend: start + EVENT_HOUR }; + const baseline = JSON.stringify({ cells: { title: cells.title }, attendees: [], reminders: [] }); + if (!local) { + if (calendarRowId === null) return { ops: { ref, ops: [] }, conflicts: 0, stillDirty: false, effect: 'none', writes: 0 }; + const ops: ProviderOp[] = [{ + op: 'insert', + table: 'events', + // CalendarProvider leaves DIRTY NULL on a sync-adapter insert unless it is written. + values: { calendar_id: calendarRowId, _sync_id: ref, ...cells, eventTimezone: 'UTC', eventStatus: 1, dirty: 0, uid2445: event.uid, sync_data1: JSON.stringify(event), sync_data4: baseline }, + }]; + return { ops: { ref, ops }, conflicts: 0, stillDirty: false, effect: 'insert', writes: 1 }; + } + const guard: ProviderOp = { op: 'assert', table: 'events', id: local.eventId, values: { dirty: local.dirty ? 1 : 0, title: local.cells.title }, expectCount: 1 }; + if (local.deleted) return { ops: { ref, ops: [guard] }, conflicts: 0, stillDirty: true, effect: 'none', writes: 0 }; + const adopting = !parseObjectRef(local.syncId) && (local.pending?.uid ?? pendingUidOf(local.syncId)) === event.uid; + if (adopting) { + return { + ops: { ref, ops: [guard, { op: 'update', table: 'events', id: local.eventId, values: { _sync_id: ref, sync_data1: JSON.stringify(event), sync_data3: null, sync_data4: baseline } }] }, + conflicts: 0, + stillDirty: local.cells.title !== event.title, + effect: 'update', + writes: 1, + }; + } + const shadowTitle = local.shadow?.title ?? ''; + const baseTitle = String(local.baseline?.cells.title ?? shadowTitle); + const localChanged = local.dirty && local.cells.title !== baseTitle; + const remoteChanged = (event.title ?? '') !== shadowTitle; + const values: Row = {}; + let conflicts = 0; + if (!localChanged || remoteChanged) { + if (local.cells.title !== cells.title || local.cells.dtstart !== cells.dtstart) Object.assign(values, cells, { sync_data4: baseline }); + if (localChanged && local.cells.title !== cells.title) conflicts = 1; + } + if (!deepEqual(local.shadow, event)) values.sync_data1 = JSON.stringify(event); + if (!Object.keys(values).length) return { ops: { ref, ops: [guard] }, conflicts: 0, stillDirty: localChanged && !remoteChanged, effect: 'none', writes: 0 }; + return { + ops: { ref, ops: [guard, { op: 'update', table: 'events', id: local.eventId, values, expectCount: 1 }] }, + conflicts, + stillDirty: localChanged && !remoteChanged, + effect: 'update', + writes: 1, + }; + }, + + planLocalDelete(local): OpGroup { + return { + ref: local.syncId ?? `row:${local.eventId}`, + ops: [ + ...local.exceptions.map((x) => ({ op: 'delete' as const, table: 'events' as const, id: x.eventId })), + { op: 'delete', table: 'events', id: local.eventId }, + ], + }; + }, + + planBaselineHeal() { + return null; + }, + + planUpload(local, ctx): UploadPlan { + const ref = local.syncId ?? `row:${local.eventId}`; + const identity = local.split === 'clone' ? null : parseObjectRef(local.syncId); + const pendingUid = local.pending?.uid ?? pendingUidOf(local.syncId); + if (local.deleted) { + if (identity) return { kind: 'upload', actions: [{ kind: 'destroy', id: identity.id, uid: null }] }; + if (pendingUid) return { kind: 'upload', actions: [{ kind: 'destroy', id: null, uid: pendingUid }] }; + return { kind: 'purge', ops: { ref, ops: [{ op: 'delete', table: 'events', id: local.eventId }] } }; + } + const guard: ProviderOp = { op: 'assert', table: 'events', id: local.eventId, values: { title: local.cells.title }, expectCount: 1 }; + if (!identity) { + const calendar = ctx.calendarIdOfRow(local.calendarRowId); + if (!calendar) return { kind: 'skip', reason: 'notOurCalendar' }; + if (!pendingUid || local.split === 'clone') { + const uid = ctx.mintUid(); + return { + kind: 'claim', + ops: { + ref, + ops: [guard, { + op: 'update', + table: 'events', + id: local.eventId, + values: { _sync_id: pendingRef(uid), uid2445: uid, sync_data3: JSON.stringify({ uid, target: collectionKey(calendar.jmapAccountId, calendar.calendarId) }) }, + }], + }, + }; + } + const start = Number(local.cells.dtstart ?? 0); + return { + kind: 'upload', + actions: [{ + kind: 'create', + uid: pendingUid, + collectionId: calendar.calendarId, + object: { uid: pendingUid, title: String(local.cells.title ?? ''), start: localStart(start), duration: 'PT1H', timeZone: 'Etc/UTC' }, + }], + }; + } + if (ctx.isReadOnly(onIds(local.shadow?.calendarIds)[0] ?? '')) { + return { kind: 'revert', ops: { ref, ops: [guard, { op: 'update', table: 'events', id: local.eventId, values: { title: local.shadow?.title ?? '', dirty: 0 } }] } }; + } + const base = String(local.baseline?.cells.title ?? local.shadow?.title ?? ''); + if (local.cells.title === base) { + return { kind: 'clean', ops: { ref, ops: [guard, { op: 'update', table: 'events', id: local.eventId, values: { dirty: 0 } }] } }; + } + return { kind: 'upload', actions: [{ kind: 'update', id: identity.id, patch: { title: String(local.cells.title ?? '') } }] }; + }, + + planAccepted(local, server, ctx): AcceptedPlan { + const ref = eventRef(ctx, server); + const values = { + _sync_id: ref, + sync_data1: JSON.stringify(server), + sync_data3: null, + sync_data4: JSON.stringify({ cells: { title: local.cells.title }, attendees: [], reminders: [] }), + }; + return { + ops: { + ref, + ops: [ + { op: 'assert', table: 'events', id: local.eventId, values: { title: local.cells.title }, expectCount: 1 }, + { op: 'update', table: 'events', id: local.eventId, values: { ...values, dirty: 0 } }, + ], + }, + keepDirtyOps: { ref, ops: [{ op: 'update', table: 'events', id: local.eventId, values }] }, + }; + }, + + planPairs() { + return []; + }, + planZoneChange() { + return null; + }, + planReminderOwnerChange() { + return null; + }, +}; + +/** An Events column list the fake accepts in `where`, for tests that read rows. */ +export const EVENT_TITLE = Events.TITLE; diff --git a/src/device-sync/engine/batch.ts b/src/device-sync/engine/batch.ts new file mode 100644 index 00000000..2b4095ee --- /dev/null +++ b/src/device-sync/engine/batch.ts @@ -0,0 +1,216 @@ +/** + * Packs item op groups into provider batches and applies them + * (docs/device-sync.md, "SyncState", "Native module API", "Failure matrix"): + * + * - a group is never split; `refs` are rebased to the batch and the group's + * first op is its yield point, so the provider may commit between groups + * but never inside one; + * - at most `maxOps` ops and about `maxBytes` of JSON per batch; groups that + * carry photo bytes travel alone (Binder budget); + * - the SyncState op, when there is one, is the last op of the last batch + * and never a yield point: the stored state can't get ahead of its rows. + * + * A failed batch may have been committed up to a yield point the provider + * took, so its groups are retried one by one (their asserts make a retry of + * a committed group fail harmlessly); a group that fails an assert is + * re-read and re-planned up to `maxReplans` times, then given up. `tooLarge` + * splits the batch; `permission` ends the run. + */ +import type { OpGroup } from '../planner'; +import type { BatchFailure, OpResult, ProviderOp, ProviderPort } from '../types'; +import { RunAbort } from './errors'; +import { hasWrites } from './provider'; +import type { Tail } from './sync-state'; + +export interface Work { + group: OpGroup; + /** The group applied; `results` are its ops' results (new row ids for inserts). */ + applied?(results: OpResult[]): void | Promise; + /** After an assert failure: read the item again and plan it again; null when nothing is left to write. */ + replan?(): Promise; + /** Given up: the item's rows stay as they are. */ + failed?(reason: BatchFailure, message: string): void | Promise; +} + +export interface BatchWriterOptions { + maxOps: number; + maxBytes: number; + maxReplans: number; +} + +function sizeOf(ops: readonly ProviderOp[]): number { + return JSON.stringify(ops).length; +} + +function carriesBlob(ops: readonly ProviderOp[]): boolean { + return ops.some( + (op) => + (op.op === 'insert' || op.op === 'update') && + Object.values(op.values).some((v) => v !== null && typeof v === 'object'), + ); +} + +/** A group's ops at `offset` in a batch: refs rebased, the first op a yield point, no other. */ +function placeGroup(ops: readonly ProviderOp[], offset: number): ProviderOp[] { + return ops.map((op, i) => { + const copy = { ...op } as ProviderOp & { yieldAllowed?: boolean }; + if (copy.op === 'insert' && copy.refs) { + const refs: Record = {}; + for (const [column, index] of Object.entries(copy.refs)) refs[column] = index + offset; + copy.refs = refs; + } + if (i === 0) copy.yieldAllowed = true; + else delete copy.yieldAllowed; + return copy; + }); +} + +export function buildBatch(groups: readonly OpGroup[], tailOp?: ProviderOp): ProviderOp[] { + const out: ProviderOp[] = []; + for (const group of groups) out.push(...placeGroup(group.ops, out.length)); + if (tailOp) { + const { yieldAllowed: _never, ...last } = tailOp as ProviderOp & { yieldAllowed?: boolean }; + out.push(last as ProviderOp); + } + return out; +} + +/** Prepends ops to a group (e.g. an engine-side assert), shifting the group's refs. */ +export function prependOps(group: OpGroup, ops: ProviderOp[]): OpGroup { + return { ref: group.ref, ops: [...ops, ...placeGroup(group.ops, ops.length).map(stripYield)] }; +} + +/** One group running `first` then `second` atomically. */ +export function concatGroups(ref: string, first: OpGroup, second: OpGroup): OpGroup { + return { + ref, + ops: [...first.ops.map(stripYield), ...placeGroup(second.ops, first.ops.length).map(stripYield)], + }; +} + +function stripYield(op: ProviderOp): ProviderOp { + const { yieldAllowed: _y, ...rest } = op as ProviderOp & { yieldAllowed?: boolean }; + return rest as ProviderOp; +} + +export class BatchWriter { + /** Batches sent, for tests and logs. */ + batchesSent = 0; + + constructor( + private readonly port: ProviderPort, + private readonly options: BatchWriterOptions, + ) {} + + /** + * Applies the works in order; `tail` (the SyncState op) goes last, after + * every group of `works` was applied or given up. + */ + async write(works: readonly Work[], tail?: Tail): Promise { + const todo = works.filter((w) => hasWrites(w.group.ops)); + const batches = this.pack(todo); + if (batches.length === 0) { + if (tail) await this.applyTail(tail); + return; + } + for (let i = 0; i < batches.length; i++) { + const last = i === batches.length - 1; + const batch = batches[i]; + // A photo batch is already at the Binder budget: its state op goes alone. + const tailHere = last && tail && !(batch.length === 1 && carriesBlob(batch[0].group.ops)) ? tail : undefined; + await this.applyBatch(batch, tailHere); + if (last && tail && !tailHere) await this.applyTail(tail); + } + } + + private pack(works: readonly Work[]): Work[][] { + const batches: Work[][] = []; + let current: Work[] = []; + let ops = 0; + let bytes = 0; + const flush = () => { + if (current.length) batches.push(current); + current = []; + ops = 0; + bytes = 0; + }; + for (const work of works) { + if (carriesBlob(work.group.ops)) { + flush(); + batches.push([work]); + continue; + } + const n = work.group.ops.length; + const b = sizeOf(work.group.ops); + if (current.length && (ops + n > this.options.maxOps || bytes + b > this.options.maxBytes)) flush(); + current.push(work); + ops += n; + bytes += b; + } + flush(); + return batches; + } + + private async send(ops: ProviderOp[]) { + this.batchesSent++; + const result = await this.port.applyBatch(ops); + if (!result.ok) { + if (result.reason === 'permission') throw new RunAbort('permission', result.message); + // The native side refused an op outside our account: an engine bug, never retried. + if (result.reason === 'scope') throw new Error(`Provider refused an op: ${result.message}`); + } + return result; + } + + private async applyBatch(works: Work[], tail?: Tail): Promise { + const tailOp = tail?.op(); + const result = await this.send(buildBatch(works.map((w) => w.group), tailOp)); + if (result.ok) { + let offset = 0; + for (const work of works) { + const n = work.group.ops.length; + await work.applied?.(result.results.slice(offset, offset + n)); + offset += n; + } + tail?.applied(); + return; + } + if (result.reason === 'tooLarge' && works.length > 1) { + const middle = Math.ceil(works.length / 2); + await this.applyBatch(works.slice(0, middle)); + await this.applyBatch(works.slice(middle), tail); + return; + } + if (works.length === 1 && !tail) { + await this.retryAlone(works[0], result.reason, result.message); + return; + } + for (const work of works) await this.applyAlone(work, 0); + if (tail) await this.applyTail(tail); + } + + private async applyAlone(work: Work, attempt: number): Promise { + if (!hasWrites(work.group.ops)) return; + const result = await this.send(buildBatch([work.group])); + if (result.ok) { + await work.applied?.(result.results); + return; + } + await this.retryAlone(work, result.reason, result.message, attempt); + } + + private async retryAlone(work: Work, reason: BatchFailure, message: string, attempt = 0): Promise { + if (reason === 'assert' && work.replan && attempt < this.options.maxReplans) { + const next = await work.replan(); + if (next) await this.applyAlone(next, attempt + 1); + return; + } + await work.failed?.(reason, message); + } + + private async applyTail(tail: Tail): Promise { + const result = await this.send([...buildBatch([], tail.op())]); + if (!result.ok) throw new Error(`Could not store the sync state: ${result.reason} ${result.message}`); + tail.applied(); + } +} diff --git a/src/device-sync/engine/calendar-sync.ts b/src/device-sync/engine/calendar-sync.ts new file mode 100644 index 00000000..38997b03 --- /dev/null +++ b/src/device-sync/engine/calendar-sync.ts @@ -0,0 +1,663 @@ +/** + * Calendars: server calendars ↔ Calendars rows, events ↔ Events rows with + * their exceptions, attendees and reminders (docs/device-sync.md, "Calendar + * mapping"). Tasks never sync; a calendar that holds only tasks gets no row. + */ +import { findTasksOnlyCalendarIds, isTaskLikeObject, SCAN_PROPERTIES, type ScannedCalendarObject } from '../../lib/calendar-component-detection'; +import { collectionKey, parseCollectionKey, parseObjectRef, pendingUidOf } from '../common/ids'; +import type { CalendarContext, CalendarEventWire, CalendarLike, CalendarPlanner, LocalCalendar, LocalEvent, OpGroup } from '../planner'; +import { CALENDAR_AUTHORITY, type ReminderOwner, type Row } from '../types'; +import { CALENDAR_EVENT_PROPERTIES, CALENDAR_PROPERTIES } from '../wire'; +import { calendarAddresses, type CalendarAddresses, type CapabilityAccount } from '../jmap/session'; +import type { Work } from './batch'; +import type { RunEnv } from './context'; +import type { SubscriptionCalendar } from './deps'; +import { ItemSync, type Pending } from './item-sync'; +import { accountOfRef, idInAccount, refOf, type Held, type Kind, type ServerObject } from './kinds'; +import { poisonFingerprint } from './poison'; +import { flag, num, str } from './provider'; +import { isCollectionSelected } from './selection'; +import { accountOf, type SyncState } from './sync-state'; + +const MASTER = 'original_id IS NULL AND original_sync_id IS NULL'; +const WITH_IDENTITY = "_sync_id IS NOT NULL AND _sync_id NOT LIKE '~pending/%'"; +/** CalendarProvider leaves DIRTY (and may leave DELETED) NULL on sync-adapter inserts: NULL is clean. */ +const CLEAN = '(dirty IS NULL OR dirty = 0) AND (deleted IS NULL OR deleted = 0)'; + +function onIds(map: Record | undefined | null): string[] { + return Object.entries(map ?? {}) + .filter(([, on]) => on === true) + .map(([id]) => id); +} + +function union(a: readonly string[], b: readonly string[]): string[] { + return [...new Set([...a, ...b])]; +} + +export class CalendarSync extends ItemSync { + protected readonly itemType = 'CalendarEvent' as const; + protected readonly itemProperties = CALENDAR_EVENT_PROPERTIES; + protected readonly parentFilter = 'inCalendar' as const; + protected readonly parentProperty = 'calendarIds' as const; + + readonly eventKind: Kind; + private readonly planner: CalendarPlanner; + private readonly calendarColumns: string[]; + private readonly eventColumns: string[]; + private readonly attendeeColumns: string[]; + private readonly reminderColumns: string[]; + + private readonly calendars = new Map(); + private readonly syncedKeys = new Set(); + private readonly readOnlyKeys = new Set(); + private readonly rowByKey = new Map(); + private readonly keyByRow = new Map(); + /** Calendar rows of deselected calendars: dropped after the upload. */ + private deleteRows: number[] = []; + /** Per account: calendars found to hold only tasks (true) or events again (false). */ + private readonly taskOnlyNotes = new Map>(); + private readonly collectionStates = new Map(); + private addresses: CalendarAddresses = { ownerAccount: '', selfAddresses: [] }; + private subscriptions: SubscriptionCalendar[] = []; + private zone = 'UTC'; + /** Who the Reminders rows of this run are written for (the stored owner until the owner-change pass). */ + private reminderOwner: ReminderOwner = 'device'; + + constructor(env: RunEnv) { + super(env); + this.planner = env.deps.planners.calendar; + this.calendarColumns = union(this.planner.calendarColumns, ['_id']); + this.eventColumns = union(this.planner.eventColumns, ['_id', 'calendar_id', '_sync_id', 'original_id', 'original_sync_id']); + this.attendeeColumns = union(this.planner.attendeeColumns, ['_id', 'event_id']); + this.reminderColumns = union(this.planner.reminderColumns, ['_id', 'event_id']); + this.eventKind = this.makeEventKind(); + } + + // ── Kind ── + + private makeEventKind(): Kind { + const planner = this.planner; + return { + name: 'event', + table: 'events', + identityColumn: '_sync_id', + poisonColumn: 'sync_data5', + meta: (e) => { + const identity = parseObjectRef(e.syncId) ? e.syncId : null; + const pendingUid = pendingUidOf(e.syncId); + const calendarKey = this.keyByRow.get(e.calendarRowId) ?? null; + const acct = accountOfRef(identity) ?? accountOfRef(calendarKey); + return { + rowId: e.eventId, + // A split clone shares its source's _SYNC_ID but uploads as a new event. + sourceId: e.split === 'clone' ? null : identity, + pending: e.pending ?? (pendingUid && calendarKey ? { uid: pendingUid, target: calendarKey } : null), + dirty: e.dirty || e.split !== undefined || e.exceptions.some((x) => x.dirty || x.deleted), + deleted: e.deleted, + isNew: !identity || e.split === 'clone', + poison: e.poison, + collections: e.shadow && acct + ? onIds(e.shadow.calendarIds as Record).map((id) => collectionKey(acct, id)) + : calendarKey ? [calendarKey] : [], + }; + }, + fingerprint: (e) => + poisonFingerprint({ + deleted: e.deleted, + shadow: e.shadow, + cells: e.cells, + attendees: e.attendees.map((a) => JSON.stringify(a.cells)).sort(), + reminders: e.reminders.map((r) => JSON.stringify(r.cells)).sort(), + exceptions: e.exceptions + .map((x) => JSON.stringify([x.deleted, x.cells, x.attendees.map((a) => a.cells), x.reminders.map((r) => r.cells)])) + .sort(), + }), + loadByRefs: async (refs) => { + const out = new Map(); + for (const e of await this.loadMastersIn('_sync_id', refs)) { + if (e.split !== 'clone' && e.syncId && parseObjectRef(e.syncId)) out.set(e.syncId, e); + } + return out; + }, + loadByRowIds: (ids) => this.loadMastersIn('_id', ids), + loadNew: () => this.loadMastersWhere("_sync_id IS NULL OR _sync_id LIKE '~pending/%'"), + planDownload: (event, local, acct) => planner.planDownload(event, local, this.ctx(acct)), + planLocalDelete: (e) => planner.planLocalDelete(e), + planBaselineHeal: (e) => planner.planBaselineHeal(e), + planUpload: (e, acct) => planner.planUpload(e, this.ctx(acct)), + planAccepted: (e, event, acct) => planner.planAccepted(e, event, this.ctx(acct)), + }; + } + + protected kinds(): Kind[] { + return [this.eventKind]; + } + + /** Tasks never sync (explicit `@type: "Task"`, or CalDAV VTODOs recognised by their task-only fields). */ + protected kindOf(object: ServerObject): Kind | null { + return isTaskLikeObject(object as ScannedCalendarObject) ? null : this.eventKind; + } + + // ── Loading ── + + private async decodeWithExceptions(masters: Row[]): Promise { + if (!masters.length) return []; + const byId = new Map(); + for (const row of masters) byId.set(num(row._id), row); + const ids = masters.map((r) => num(r._id)); + const syncIds = masters.map((r) => str(r._sync_id)).filter((s): s is string => !!s); + for (const row of await this.env.reader.rowsIn('events', this.eventColumns, 'original_id', ids)) byId.set(num(row._id), row); + if (syncIds.length) { + for (const row of await this.env.reader.rowsIn('events', this.eventColumns, 'original_sync_id', syncIds)) byId.set(num(row._id), row); + } + const all = [...byId.values()]; + const eventIds = all.map((r) => num(r._id)); + const attendees = await this.env.reader.rowsIn('attendees', this.attendeeColumns, 'event_id', eventIds); + const reminders = await this.env.reader.rowsIn('reminders', this.reminderColumns, 'event_id', eventIds); + return this.planner.decodeEvents(all, attendees, reminders); + } + + private async loadMastersIn(column: '_id' | '_sync_id', values: ReadonlyArray): Promise { + if (!values.length) return []; + return this.decodeWithExceptions(await this.env.reader.rowsIn('events', this.eventColumns, column, values, { where: MASTER })); + } + + private async loadMastersWhere(where: string, args: Array = []): Promise { + return this.decodeWithExceptions(await this.env.reader.rows('events', this.eventColumns, `(${where}) AND ${MASTER}`, args)); + } + + private async masterIds(where: string, args: Array = []): Promise { + return (await this.env.reader.rows('events', ['_id'], `(${where}) AND ${MASTER}`, args)).map((r) => num(r._id)); + } + + private async loadCalendars(): Promise { + return (await this.env.reader.rows('calendars', this.calendarColumns)).map((r) => this.planner.decodeCalendar(r)); + } + + private async refreshCalendarRows(): Promise { + this.rowByKey.clear(); + this.keyByRow.clear(); + for (const calendar of await this.loadCalendars()) { + if (!calendar.syncId || !parseCollectionKey(calendar.syncId)) continue; + this.rowByKey.set(calendar.syncId, calendar.calendarRowId); + this.keyByRow.set(calendar.calendarRowId, calendar.syncId); + } + } + + // ── Collections ── + + private isFeed(account: CapabilityAccount, calendarId: string): boolean { + return this.subscriptions.some( + (s) => s.calendarId === calendarId && (s.jmapAccountId === account.id || (s.jmapAccountId === null && account.primary)), + ); + } + + private noteTaskOnly(acct: string, calendarId: string, taskOnly: boolean): void { + let notes = this.taskOnlyNotes.get(acct); + if (!notes) this.taskOnlyNotes.set(acct, (notes = new Map())); + notes.set(calendarId, taskOnly); + } + + private isTaskOnly(acct: string, calendarId: string): boolean { + return this.taskOnlyNotes.get(acct)?.get(calendarId) ?? (this.env.store.account(acct).taskOnly ?? []).includes(calendarId); + } + + /** A calendar whose first objects are all tasks (Stalwart offers no per-calendar component set). */ + private async scanTaskOnly(acct: string, calendarId: string): Promise { + const ids = await this.env.jmap.queryFirst('CalendarEvent', acct, { inCalendar: calendarId }, this.env.tuning.taskScanLimit); + if (!ids.length) return false; + const { list } = await this.env.jmap.get('CalendarEvent', acct, ids, SCAN_PROPERTIES); + return findTasksOnlyCalendarIds(list.map((o) => ({ ...o, calendarIds: { [calendarId]: true } })), [calendarId]).has(calendarId); + } + + protected async collections(write: boolean): Promise { + this.zone = this.env.deps.deviceZone(); + this.reminderOwner = this.env.store.committed.reminderOwner ?? this.env.prefs.reminderOwner; + this.addresses = await calendarAddresses(this.env.jmap, this.env.accountName); + this.subscriptions = await this.env.deps.subscriptionCalendars(); + this.syncedKeys.clear(); + this.readOnlyKeys.clear(); + const selected: Array<{ jmapAccountId: string; calendar: CalendarLike; readOnly: boolean; accountName: string }> = []; + for (const account of this.env.accounts) { + const { list, state } = await this.env.jmap.getContainers('Calendar', account.id, CALENDAR_PROPERTIES); + this.calendars.set(account.id, list); + if (state) { + this.collectionStates.set(account.id, state); + this.env.recordKnownState(account.id, 'Calendar', state); + } + const stored = this.env.store.account(account.id); + const reconciling = !stored.itemsState || !!stored.reconcile; + for (const calendar of list) { + if (calendar.myRights?.mayReadItems === false) continue; + if (!isCollectionSelected(this.env.prefs.calendarSelection, account, CALENDAR_AUTHORITY, calendar)) continue; + const key = collectionKey(account.id, calendar.id); + // Checked when a calendar comes to the device, and again on every full reconcile. + if (write && (reconciling || (!stored.selected.includes(key) && !this.isTaskOnly(account.id, calendar.id)))) { + this.noteTaskOnly(account.id, calendar.id, await this.scanTaskOnly(account.id, calendar.id)); + } + if (this.isTaskOnly(account.id, calendar.id)) continue; + const rights = calendar.myRights; + const readOnly = this.isFeed(account, calendar.id) || (!!rights && !rights.mayWriteAll && !rights.mayWriteOwn); + this.syncedKeys.add(key); + if (readOnly) this.readOnlyKeys.add(key); + selected.push({ jmapAccountId: account.id, calendar, readOnly, accountName: account.personal ? '' : account.name }); + } + } + await this.refreshCalendarRows(); + this.deleteRows = []; + if (!write) return; + const plan = this.planner.planCalendars(selected, await this.loadCalendars(), this.calendarCtx()); + await this.env.writer.write(plan.groups.map((group) => ({ group }))); + this.deleteRows = plan.deleteCalendarRows; + await this.refreshCalendarRows(); + } + + protected syncedCollections(acct: string): string[] { + return (this.calendars.get(acct) ?? []) + .filter((c) => this.syncedKeys.has(collectionKey(acct, c.id))) + .map((c) => c.id); + } + + protected inSelection(acct: string, object: ServerObject): boolean { + return onIds(object.calendarIds as Record).some((id) => this.syncedKeys.has(collectionKey(acct, id))); + } + + private findCalendar(calendarId: string): CalendarLike | undefined { + for (const account of this.env.accounts) { + const found = this.calendars.get(account.id)?.find((c) => c.id === calendarId); + if (found) return found; + } + return undefined; + } + + private calendarCtx(): Omit { + const env = this.env; + const primary = this.env.accounts.find((a) => a.primary)?.id ?? this.env.accounts[0]?.id ?? ''; + return { + now: env.now(), + mintKey: (taken) => env.mintKey(taken), + mintUid: () => env.mintUid(), + deviceZone: this.zone, + ownerAccount: this.addresses.ownerAccount, + selfAddresses: this.addresses.selfAddresses, + reminderOwner: this.reminderOwner, + calendar: (calendarId) => this.findCalendar(calendarId), + calendarRowId: (calendarId) => this.rowByKey.get(collectionKey(primary, calendarId)) ?? null, + calendarIdOfRow: (rowId) => { + const parsed = parseCollectionKey(this.keyByRow.get(rowId)); + return parsed ? { jmapAccountId: parsed.accountId, calendarId: parsed.id } : null; + }, + isReadOnly: (calendarId) => [...this.readOnlyKeys].some((k) => parseCollectionKey(k)?.id === calendarId), + isSelected: (key) => this.syncedKeys.has(key), + }; + } + + ctx(acct: string): CalendarContext { + return { + ...this.calendarCtx(), + jmapAccountId: acct, + calendar: (calendarId) => this.calendars.get(acct)?.find((c) => c.id === calendarId), + calendarRowId: (calendarId) => this.rowByKey.get(collectionKey(acct, calendarId)) ?? null, + isReadOnly: (calendarId) => this.readOnlyKeys.has(collectionKey(acct, calendarId)), + }; + } + + protected decorateState(acct: string, next: SyncState): () => void { + const account = accountOf(next, acct); + const notes = this.taskOnlyNotes.get(acct); + const consumed = new Map(notes ?? []); + if (consumed.size) { + const taskOnly = new Set(account.taskOnly ?? []); + for (const [id, only] of consumed) { + if (only) taskOnly.add(id); + else taskOnly.delete(id); + } + account.taskOnly = [...taskOnly]; + } + const collections = this.collectionStates.get(acct); + if (collections) account.collectionsState = collections; + return () => { + for (const [id, only] of consumed) if (notes?.get(id) === only) notes.delete(id); + }; + } + + protected async localIdentities(acct: string): Promise> { + const out = new Map(); + const rows = await this.env.reader.rows('events', ['_id', '_sync_id', 'dirty', 'deleted'], `_sync_id LIKE ? AND ${MASTER}`, [`${acct}/%`]); + for (const row of rows) { + const id = idInAccount(str(row._sync_id), acct); + if (id) out.set(id, { dirty: flag(row.dirty), deleted: flag(row.deleted) }); + } + return out; + } + + // ── Uploads ── + + /** + * Masters with local changes: dirty, deleted or new ones, masters of dirty + * or deleted exception rows, and both masters of a CONTENT_EXCEPTION_URI + * split (two masters sharing a _SYNC_ID; the older one's rule changed + * without DIRTY), loaded together so the planner sees the split. + */ + protected async loadUploadItems(): Promise { + const rows = await this.env.reader.rows( + 'events', + ['_id', '_sync_id', 'original_id', 'original_sync_id', 'uid2445'], + "dirty = 1 OR deleted = 1 OR _sync_id IS NULL OR _sync_id LIKE '~pending/%'", + ); + const ids = new Set(); + const refs = new Set(); + const newUids = new Set(); + for (const row of rows) { + if (row.original_id !== null && row.original_id !== undefined) ids.add(num(row.original_id)); + else if (str(row.original_sync_id)) refs.add(str(row.original_sync_id) as string); + else { + ids.add(num(row._id)); + const uid = str(row.uid2445); + if (uid && !parseObjectRef(str(row._sync_id))) newUids.add(uid); + } + } + const bySyncId = new Map(); + for (const row of await this.env.reader.rows('events', ['_id', '_sync_id'], `_sync_id IS NOT NULL AND ${MASTER}`)) { + const syncId = str(row._sync_id) as string; + bySyncId.set(syncId, [...(bySyncId.get(syncId) ?? []), num(row._id)]); + } + for (const [syncId, masters] of bySyncId) { + if (masters.length > 1 || refs.has(syncId)) for (const id of masters) ids.add(id); + } + // A new master may carry a uid another row has (an app copied an event): those rows come along, so + // the planner sees the collision and gives the new one a fresh uid instead of adopting the other's object. + if (newUids.size) { + for (const row of await this.env.reader.rowsIn('events', ['_id'], 'uid2445', [...newUids], { where: MASTER })) ids.add(num(row._id)); + } + const events = await this.loadMastersIn('_id', [...ids]); + return events.map((local) => ({ kind: this.eventKind as Kind, local })); + } + + /** Event uids are unique per account: only an event in the create's calendar can be the row's own create. */ + protected adoptionTargetMatches(acct: string, object: ServerObject, target: string): boolean { + const parsed = parseCollectionKey(target); + return !!parsed && parsed.accountId === acct && (object.calendarIds as Record | undefined)?.[parsed.id] === true; + } + + protected accountOfItem(held: Held): string | null { + return super.accountOfItem(held) ?? accountOfRef(this.keyByRow.get((held.local as LocalEvent).calendarRowId) ?? null); + } + + protected claimAccount(held: Held): string | null { + return accountOfRef(this.keyByRow.get((held.local as LocalEvent).calendarRowId) ?? null); + } + + /** A split clone is claimed after its source's rule was planned (see prePhase). */ + protected claimable(held: Held): boolean { + return (held.local as LocalEvent).split !== 'clone'; + } + + protected async countDeletions(items: Held[]): Promise<{ count: number; synced: number; accounts: Set }> { + let count = 0; + const accounts = new Set(); + for (const held of items) { + const meta = held.kind.meta(held.local); + if (!meta.deleted || !meta.sourceId) continue; + count++; + const acct = accountOfRef(meta.sourceId); + if (acct) accounts.add(acct); + } + const synced = (await this.masterIds(WITH_IDENTITY)).length; + return { count, synced: Math.max(synced, count), accounts }; + } + + /** + * Before the creates: a split's source uploads its rule while the split is + * still visible, then its clone is claimed as a new event; and deleted + + * new masters of the run that are one edit (a move, a series turned + * single) upload as patches of the existing object. + */ + protected async prePhase(acct: string): Promise { + let items = (await this.loadUploadItems()).filter((h) => this.accountOfItem(h) === acct); + const sources = items.filter((h) => (h.local as LocalEvent).split === 'source'); + const clones = items.filter((h) => (h.local as LocalEvent).split === 'clone'); + if (sources.length || clones.length) { + const works: Work[] = []; + const ready: Pending[] = []; + for (const held of sources) { + const pending = await this.planItem(acct, held, works); + if (pending) ready.push(pending); + } + // A clone (of a split, or a new master whose uid another row carries) needs no source: + // it claims a fresh uid here and is created with the other new events. + for (const held of clones) { + const pending = await this.planItem(acct, held, works); + if (pending) ready.push(pending); + } + await this.env.writer.write(works); + if (ready.length) await this.send(acct, ready); + items = (await this.loadUploadItems()).filter((h) => this.accountOfItem(h) === acct); + } + const deleted = items.map((h) => h.local as LocalEvent).filter((e) => { + const m = this.eventKind.meta(e); + return m.deleted && !!m.sourceId; + }); + const fresh = items.map((h) => h.local as LocalEvent).filter((e) => { + const m = this.eventKind.meta(e); + return m.isNew && !m.deleted && !!m.pending; + }); + if (!deleted.length || !fresh.length) return; + let pairs: ReturnType; + try { + pairs = this.planner.planPairs(deleted, fresh, this.ctx(acct)); + } catch (error) { + // Unpaired, they upload as ordinary creates and deletions. + this.plannerFailed(acct, `pairs:${acct}`, 'upload', error); + return; + } + if (!pairs.length) return; + await this.env.checkpoints.check('upload:pairs'); + const pending: Pending[] = pairs.map((pair) => { + const meta = this.eventKind.meta(pair.deleted); + const ownId = idInAccount(meta.sourceId, acct); + return { + held: { kind: this.eventKind, local: pair.deleted }, + meta, + fingerprint: this.eventKind.fingerprint(pair.deleted), + actions: pair.actions as Pending['actions'], + accept: async (objects) => { + // The new row takes the identity and exceptions of the old one, which is purged. + await this.env.writer.write([{ group: pair.ops }]); + const [moved] = await this.eventKind.loadByRowIds([pair.fresh.eventId]); + const server = ownId ? objects.get(ownId) : undefined; + if (!moved || !server) { + if (ownId) this.markStale(acct, ownId); + return []; + } + const held: Held = { kind: this.eventKind, local: moved }; + try { + return [this.acceptWork(acct, { held, meta: this.eventKind.meta(moved), fingerprint: '', actions: [] }, server, false)]; + } catch (error) { + this.plannerFailed(acct, refOf(acct, server.id), 'upload', error, server.id); + return []; + } + }, + }; + }); + await this.send(acct, pending); + } + + // ── After the upload ── + + /** A master with nothing waiting for an upload. */ + private isClean(e: LocalEvent): boolean { + const m = this.eventKind.meta(e); + return !m.dirty && !m.deleted && !m.isNew; + } + + /** + * Deselected calendars lose their rows (the provider deletes their events) + * once nothing in them waits for an upload; until then only their clean + * events go. Events that left every synced calendar go when clean. + */ + protected async dropOutsideSelection(acct: string): Promise { + const synced = new Set(this.syncedCollections(acct).map((id) => collectionKey(acct, id))); + const deselected = this.env.store.account(acct).selected.filter((key) => !synced.has(key)); + const rows = this.deleteRows.filter((row) => accountOfRef(this.keyByRow.get(row) ?? null) === acct); + const outside = [...(this.outside.get(acct) ?? [])]; + if (!deselected.length && !rows.length && !outside.length) return; + const kept = new Set(); + for (const rowId of rows) { + await this.env.checkpoints.check('deselect'); + const key = this.keyByRow.get(rowId) as string; + if (await this.dropCalendarRow(rowId)) continue; + kept.add(key); + } + const works: Work[] = []; + for (const local of (await this.eventKind.loadByRefs(outside)).values()) { + const meta = this.eventKind.meta(local); + const id = idInAccount(meta.sourceId, acct); + if (!id || !meta.collections.length || meta.collections.some((k) => synced.has(k))) continue; + const work = this.removeWork(acct, id, { kind: this.eventKind, local }, 'outside'); + if (work) works.push(work); + } + const selected = [...synced, ...kept]; + await this.env.writer.write(works, this.tail([acct], (next) => { + accountOf(next, acct).selected = selected; + })); + this.outside.delete(acct); + } + + /** Deletes a calendar row when none of its events waits for an upload; else only its clean events. True when the row went. */ + private async dropCalendarRow(rowId: number): Promise { + const waiting = await this.env.reader.rows( + 'events', + ['_id'], + "calendar_id = ? AND (dirty = 1 OR deleted = 1 OR _sync_id IS NULL OR _sync_id LIKE '~pending/%')", + [rowId], + ); + if (!waiting.length) { + await this.env.writer.write([{ group: { ref: `calendar:${rowId}`, ops: [{ op: 'delete', table: 'calendars', id: rowId }] } }]); + return true; + } + const clean = await this.masterIds(`calendar_id = ? AND ${CLEAN} AND ${WITH_IDENTITY}`, [rowId]); + for (let i = 0; i < clean.length; i += this.env.tuning.chunkSize) { + await this.env.checkpoints.check('deselect'); + const works: Work[] = []; + for (const local of await this.eventKind.loadByRowIds(clean.slice(i, i + this.env.tuning.chunkSize))) { + const meta = this.eventKind.meta(local); + const acct = accountOfRef(meta.sourceId); + const id = acct ? idInAccount(meta.sourceId, acct) : null; + if (!acct || !id || !this.isClean(local)) continue; + const work = this.removeWork(acct, id, { kind: this.eventKind, local }, 'outside'); + if (work) works.push(work); + } + await this.env.writer.write(works); + } + return false; + } + + protected async afterSync(): Promise { + // Rows of calendars that belong to no account we know (garbage, or a vanished account). + for (const rowId of this.deleteRows) { + if (this.keyByRow.has(rowId) && accountOfRef(this.keyByRow.get(rowId) ?? null)) continue; + await this.dropCalendarRow(rowId); + } + await this.zonePass(); + await this.reminderOwnerPass(); + } + + /** Every clean master, in chunks, through `plan`; `finish` rides with the last chunk's batch. */ + private async passOverCleanMasters( + label: string, + plan: (e: LocalEvent, acct: string) => OpGroup | null, + finish: (next: SyncState) => void, + ): Promise { + const ids = await this.masterIds(`${CLEAN} AND ${WITH_IDENTITY}`); + const size = this.env.tuning.chunkSize; + if (!ids.length) { + await this.env.writer.write([], this.env.store.tail(finish)); + return; + } + for (let i = 0; i < ids.length; i += size) { + await this.env.checkpoints.check(label); + const works: Work[] = []; + for (const local of await this.eventKind.loadByRowIds(ids.slice(i, i + size))) { + const sourceId = this.eventKind.meta(local).sourceId; + const acct = accountOfRef(sourceId); + if (!acct || !this.isClean(local)) continue; + try { + const group = plan(local, acct); + // A group that fails its assert was edited meanwhile: the upload takes it from here. + if (group) works.push({ group }); + } catch (error) { + this.plannerFailed(acct, sourceId as string, 'download', error); + } + } + await this.env.writer.write(works, i + size >= ids.length ? this.env.store.tail(finish) : undefined); + } + } + + /** + * The device zone changed: clean floating events are rewritten so they + * keep their wall time. `deviceZonePending` marks the pass until its last + * chunk stores the new zone. + */ + private async zonePass(): Promise { + const committed = this.env.store.committed; + const stored = committed.deviceZone ?? null; + if (!stored) { + await this.env.writer.write([], this.env.store.tail((next) => { + next.deviceZone = this.zone; + next.deviceZonePending = null; + })); + return; + } + if (stored === this.zone && !committed.deviceZonePending) return; + if (committed.deviceZonePending !== this.zone) { + await this.env.writer.write([], this.env.store.tail((next) => { + next.deviceZonePending = this.zone; + })); + } + await this.passOverCleanMasters( + 'zone', + (e, acct) => this.planner.planZoneChange(e, stored, this.ctx(acct)), + (next) => { + next.deviceZone = this.zone; + next.deviceZonePending = null; + }, + ); + } + + /** + * The reminder owner changed (the run's uploads used the old one, so + * pending reminder edits went up first): Reminders rows and baselines are + * rewritten for the new owner, and the calendars' MAX_REMINDERS follows. + */ + private async reminderOwnerPass(): Promise { + const stored = this.env.store.committed.reminderOwner ?? null; + const wanted = this.env.prefs.reminderOwner; + if (stored === wanted) return; + this.reminderOwner = wanted; + if (!stored) { + // The first run wrote everything for the wanted owner already. + await this.env.writer.write([], this.env.store.tail((next) => { + next.reminderOwner = wanted; + })); + return; + } + await this.passOverCleanMasters( + 'reminders', + (e, acct) => this.planner.planReminderOwnerChange(e, this.ctx(acct)), + (next) => { + next.reminderOwner = wanted; + }, + ); + const selected: Array<{ jmapAccountId: string; calendar: CalendarLike; readOnly: boolean; accountName: string }> = []; + for (const account of this.env.accounts) { + for (const calendar of this.calendars.get(account.id) ?? []) { + const key = collectionKey(account.id, calendar.id); + if (!this.syncedKeys.has(key)) continue; + selected.push({ jmapAccountId: account.id, calendar, readOnly: this.readOnlyKeys.has(key), accountName: account.personal ? '' : account.name }); + } + } + const plan = this.planner.planCalendars(selected, await this.loadCalendars(), this.calendarCtx()); + await this.env.writer.write(plan.groups.map((group) => ({ group }))); + } +} diff --git a/src/device-sync/engine/checkpoint.ts b/src/device-sync/engine/checkpoint.ts new file mode 100644 index 00000000..2cd8603c --- /dev/null +++ b/src/device-sync/engine/checkpoint.ts @@ -0,0 +1,55 @@ +/** + * Checkpoints sit between chunks and phases (docs/device-sync.md, "A sync + * run"): the run stops cleanly when the framework cancelled it or its time + * is nearly up, keeps SyncManager's traffic monitor happy during long local + * phases (it cancels a sync with no network traffic for 60 s), and lets the + * JS thread breathe so the UI keeps rendering when it shares the runtime. + */ +import type { JmapCaller } from '../jmap/caller'; +import { StopRun } from './errors'; + +export interface CheckpointOptions { + now(): number; + /** Epoch ms by which the run must have finished. */ + deadline: number; + /** Stop starting work this long before the deadline. */ + marginMs: number; + echoAfterMs: number; + isCancelled(): Promise; + yieldThread(): Promise; +} + +export class Checkpoints { + private jmap: JmapCaller | null = null; + /** Test hook: runs at every checkpoint (crash injection). */ + onCheckpoint?: (label: string) => void | Promise; + + constructor(private readonly options: CheckpointOptions) {} + + attach(jmap: JmapCaller): void { + this.jmap = jmap; + } + + budgetMs(): number { + return this.options.deadline - this.options.now(); + } + + /** Throws StopRun when the run must end here. */ + async check(label = ''): Promise { + await this.onCheckpoint?.(label); + if (this.budgetMs() < this.options.marginMs) throw new StopRun('deadline'); + if (await this.options.isCancelled()) throw new StopRun('cancelled'); + if (this.jmap && this.jmap.idleMs() >= this.options.echoAfterMs) await this.jmap.echo(); + await this.options.yieldThread(); + } + + /** Throws StopRun('deadline') unless at least `ms` of budget remain (a create batch needs two request timeouts). */ + requireBudget(ms: number): void { + if (this.budgetMs() < ms) throw new StopRun('deadline'); + } + + /** Keeps traffic flowing during a long local phase without the other checks. */ + async keepAlive(): Promise { + if (this.jmap && this.jmap.idleMs() >= this.options.echoAfterMs) await this.jmap.echo(); + } +} diff --git a/src/device-sync/engine/contacts-sync.ts b/src/device-sync/engine/contacts-sync.ts new file mode 100644 index 00000000..2064a4b4 --- /dev/null +++ b/src/device-sync/engine/contacts-sync.ts @@ -0,0 +1,743 @@ +/** + * Contacts: address books ↔ raw contacts and groups of one Android account + * (docs/device-sync.md, "Contacts mapping"). Group cards (`kind: "group"`) + * are Groups rows and are always written before the contacts of the same + * download, so memberships never name a group that does not exist yet. + */ +import { MimeType } from '../android-columns'; +import { collectionKey, parseCollectionKey } from '../common/ids'; +import { parseJsonColumn } from '../common/json'; +import type { + AddressBookLike, + ContactCardWire, + ContactsContext, + ContactsPlanner, + LocalContact, + LocalGroup, +} from '../planner'; +import { CONTACTS_AUTHORITY, type Row } from '../types'; +import { ADDRESS_BOOK_PROPERTIES, CONTACT_CARD_PROPERTIES } from '../wire'; +import type { Work } from './batch'; +import type { RunEnv } from './context'; +import { ItemSync, type Pending } from './item-sync'; +import { accountOfRef, idInAccount, refOf, type Held, type Kind, type ServerObject } from './kinds'; +import { poisonFingerprint } from './poison'; +import { flag, num, str } from './provider'; +import { isCollectionSelected } from './selection'; +import { accountOf, type SyncState } from './sync-state'; + +/** + * The context the engine hands the contacts planner. `groupsOf` is + * not part of `ContactsContext` yet: memberships are stored on the group + * cards (`members` holds member uids), so a contact's GroupMembership rows + * need the reverse lookup. + */ +export interface ContactsRunContext extends ContactsContext { + /** SOURCE_IDs of the group cards whose `members` include this uid. */ + groupsOf(uid: string): string[]; +} + +const BASE64 = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/'; + +export function toBase64(bytes: Uint8Array): string { + let out = ''; + for (let i = 0; i < bytes.length; i += 3) { + const a = bytes[i]; + const b = i + 1 < bytes.length ? bytes[i + 1] : 0; + const c = i + 2 < bytes.length ? bytes[i + 2] : 0; + const n = (a << 16) | (b << 8) | c; + out += BASE64[(n >> 18) & 63] + BASE64[(n >> 12) & 63]; + out += i + 1 < bytes.length ? BASE64[(n >> 6) & 63] : '='; + out += i + 2 < bytes.length ? BASE64[n & 63] : '='; + } + return out; +} + +/** Base64 of a `data:` URI's bytes, or null. */ +export function dataUriBase64(uri: unknown): string | null { + if (typeof uri !== 'string' || !/^data:/i.test(uri)) return null; + const comma = uri.indexOf(','); + if (comma < 0) return null; + const header = uri.slice(5, comma); + const payload = uri.slice(comma + 1); + if (/;base64$/i.test(header)) return payload.replace(/\s+/g, ''); + const bytes: number[] = []; + for (let i = 0; i < payload.length; i++) { + if (payload[i] === '%' && /^[0-9a-f]{2}$/i.test(payload.slice(i + 1, i + 3))) { + bytes.push(parseInt(payload.slice(i + 1, i + 3), 16)); + i += 2; + } else { + bytes.push(payload.charCodeAt(i) & 0xff); + } + } + return toBase64(Uint8Array.from(bytes)); +} + +/** The name a Relation row shows for a card. */ +export function displayNameOf(card: Pick | null | undefined): string | null { + const full = card?.name?.full; + if (typeof full === 'string' && full.trim()) return full.trim(); + const parts = (card?.name?.components ?? []) + .filter((c) => c && c.kind !== 'separator' && typeof c.value === 'string' && c.value.trim()) + .map((c) => (c.value as string).trim()); + return parts.length ? parts.join(' ') : null; +} + +function onIds(map: Record | undefined | null): string[] { + return Object.entries(map ?? {}) + .filter(([, on]) => on === true) + .map(([id]) => id); +} + +function union(a: readonly string[], b: readonly string[]): string[] { + return [...new Set([...a, ...b])]; +} + +export class ContactsSync extends ItemSync { + protected readonly itemType = 'ContactCard' as const; + protected readonly itemProperties = CONTACT_CARD_PROPERTIES; + protected readonly parentFilter = 'inAddressBook' as const; + protected readonly parentProperty = 'addressBookIds' as const; + + readonly contactKind: Kind; + readonly groupKind: Kind; + private readonly planner: ContactsPlanner; + private readonly rawColumns: string[]; + private readonly dataColumns: string[]; + private readonly groupColumns: string[]; + + private readonly books = new Map(); + private readonly syncedKeys = new Set(); + private readonly collectionStates = new Map(); + /** Group SOURCE_ID → Groups row id, and member uid → group SOURCE_IDs; rebuilt after group writes. */ + private readonly groupRows = new Map(); + private readonly membership = new Map>(); + private groupsStale = true; + /** Per account: group refs that appeared (true) or went (false), for `SyncState.groups`. */ + private readonly groupChanges = new Map>(); + /** Per account: group refs listed in `groups` whose rows an app hard-deleted. */ + private absent = new Map(); + private readonly blobPhotos = new Map(); + private readonly devicePhotos = new Map(); + private uidNames: Map | null = null; + private uidRefs: Map | null = null; + /** Per account: raw contacts the upload phases looked at, for their memberships. */ + private readonly uploadRows = new Map>(); + + constructor(env: RunEnv) { + super(env); + this.planner = env.deps.planners.contacts; + this.rawColumns = union(this.planner.rawContactColumns, ['_id']); + this.dataColumns = union(this.planner.dataColumns, ['_id', 'raw_contact_id', 'mimetype']); + this.groupColumns = union(this.planner.groupColumns, ['_id']); + this.contactKind = this.makeContactKind(); + this.groupKind = this.makeGroupKind(); + } + + // ── Kinds ── + + private makeContactKind(): Kind { + const planner = this.planner; + return { + name: 'contact', + table: 'raw_contacts', + identityColumn: 'sourceid', + poisonColumn: 'sync4', + meta: (c) => ({ + rowId: c.rawContactId, + sourceId: c.sourceId, + pending: c.pending, + dirty: c.dirty, + deleted: c.deleted, + isNew: !c.sourceId, + poison: c.poison, + collections: c.collections, + }), + fingerprint: (c) => + poisonFingerprint({ + deleted: c.deleted, + shadow: c.shadow, + rows: c.rows.map((r) => JSON.stringify([r.mimetype, r.cells])).sort(), + }), + loadByRefs: async (refs) => { + const out = new Map(); + for (const c of await this.loadContactsIn('sourceid', refs)) if (c.sourceId) out.set(c.sourceId, c); + return out; + }, + loadByRowIds: (ids) => this.loadContactsIn('_id', ids), + loadNew: () => this.loadContactsWhere('sourceid IS NULL'), + planDownload: (card, local, acct) => planner.planDownload(card, local, this.ctx(acct)), + planLocalDelete: (c) => planner.planLocalDelete(c), + planBaselineHeal: (c) => planner.planBaselineHeal(c), + planUpload: (c, acct) => planner.planUpload(c, this.ctx(acct)), + planAccepted: (c, card, acct) => planner.planAccepted(c, card, this.ctx(acct)), + }; + } + + private makeGroupKind(): Kind { + const planner = this.planner; + return { + name: 'group', + table: 'groups', + identityColumn: 'sourceid', + poisonColumn: 'sync4', + meta: (g) => ({ + rowId: g.groupId, + sourceId: g.sourceId, + pending: g.pending, + dirty: g.dirty, + deleted: g.deleted, + isNew: !g.sourceId, + poison: g.poison, + collections: this.cardKeys(accountOfRef(g.sourceId), g.shadow), + }), + fingerprint: (g) => poisonFingerprint({ deleted: g.deleted, title: g.title, shadow: g.shadow }), + loadByRefs: async (refs) => { + const out = new Map(); + for (const g of await this.loadGroupsIn('sourceid', refs)) if (g.sourceId) out.set(g.sourceId, g); + return out; + }, + loadByRowIds: (ids) => this.loadGroupsIn('_id', ids), + loadNew: () => this.loadGroupsWhere('sourceid IS NULL'), + planDownload: (card, local, acct) => planner.planGroupDownload(card, local, this.ctx(acct)), + planLocalDelete: (g) => planner.planGroupLocalDelete(g), + planBaselineHeal: () => null, + planUpload: (g, acct) => planner.planGroupUpload(g, this.ctx(acct)), + planAccepted: (g, card, acct) => planner.planGroupAccepted(g, card, this.ctx(acct)), + }; + } + + protected kinds(): Kind[] { + return [this.groupKind, this.contactKind]; + } + + protected kindOf(object: ServerObject): Kind { + return object.kind === 'group' ? this.groupKind : this.contactKind; + } + + // ── Loading ── + + private async decodeContacts(rawRows: Row[]): Promise { + if (!rawRows.length) return []; + const data = await this.env.reader.rowsIn('data', this.dataColumns, 'raw_contact_id', rawRows.map((r) => num(r._id))); + const byContact = new Map(); + for (const row of data) { + const id = num(row.raw_contact_id); + let list = byContact.get(id); + if (!list) byContact.set(id, (list = [])); + list.push(row); + } + return rawRows.map((rc) => this.planner.decodeContact(rc, byContact.get(num(rc._id)) ?? [])); + } + + private async loadContactsIn(column: '_id' | 'sourceid', values: ReadonlyArray): Promise { + if (!values.length) return []; + return this.decodeContacts(await this.env.reader.rowsIn('raw_contacts', this.rawColumns, column, values)); + } + + private async loadContactsWhere(where?: string, args?: Array): Promise { + return this.decodeContacts(await this.env.reader.rows('raw_contacts', this.rawColumns, where, args)); + } + + private async loadGroupsIn(column: '_id' | 'sourceid', values: ReadonlyArray): Promise { + if (!values.length) return []; + const rows = await this.env.reader.rowsIn('groups', this.groupColumns, column, values); + return rows.map((r) => this.planner.decodeGroup(r)); + } + + private async loadGroupsWhere(where?: string, args?: Array): Promise { + const rows = await this.env.reader.rows('groups', this.groupColumns, where, args); + return rows.map((r) => this.planner.decodeGroup(r)); + } + + /** Group rows by SOURCE_ID and who is a member of which group, from the group cards' shadows. */ + private async refreshGroups(): Promise { + this.groupRows.clear(); + this.membership.clear(); + for (const group of await this.loadGroupsWhere()) { + if (!group.sourceId || group.deleted) continue; + this.groupRows.set(group.sourceId, group.groupId); + for (const uid of onIds(group.shadow?.members as Record | undefined)) { + let set = this.membership.get(uid); + if (!set) this.membership.set(uid, (set = new Set())); + set.add(group.sourceId); + } + } + this.groupsStale = false; + } + + // ── Collections ── + + private cardKeys(acct: string | null, card: Pick | null | undefined): string[] { + if (!acct || !card) return []; + return onIds(card.addressBookIds).map((id) => collectionKey(acct, id)); + } + + private book(key: string): AddressBookLike | undefined { + const parsed = parseCollectionKey(key); + return parsed ? this.books.get(parsed.accountId)?.find((b) => b.id === parsed.id) : undefined; + } + + private writable(key: string): boolean { + const book = this.book(key); + return !!book && book.myRights?.mayWrite !== false; + } + + protected async collections(write: boolean): Promise { + this.syncedKeys.clear(); + for (const account of this.env.accounts) { + const { list, state } = await this.env.jmap.getContainers('AddressBook', account.id, ADDRESS_BOOK_PROPERTIES); + this.books.set(account.id, list); + for (const book of list) { + if (isCollectionSelected(this.env.prefs.contactsSelection, account, CONTACTS_AUTHORITY, book)) { + this.syncedKeys.add(collectionKey(account.id, book.id)); + } + } + if (state) { + this.collectionStates.set(account.id, state); + this.env.recordKnownState(account.id, 'AddressBook', state); + } + } + if (write) await this.ensureSettingsRow(); + } + + /** One Settings row per account with UNGROUPED_VISIBLE=1: the default 0 hides every contact without a visible group. */ + private async ensureSettingsRow(): Promise { + const rows = await this.env.reader.rows('settings', ['ungrouped_visible', 'should_sync']); + const values = { ungrouped_visible: 1, should_sync: 1 }; + if (!rows.length) { + await this.env.writer.write([{ group: { ref: 'settings', ops: [{ op: 'insert', table: 'settings', values }] } }]); + } else if (rows.some((r) => num(r.ungrouped_visible) !== 1 || num(r.should_sync) !== 1)) { + await this.env.writer.write([{ group: { ref: 'settings', ops: [{ op: 'update', table: 'settings', values }] } }]); + } + } + + protected syncedCollections(acct: string): string[] { + return (this.books.get(acct) ?? []) + .filter((b) => this.syncedKeys.has(collectionKey(acct, b.id))) + .map((b) => b.id); + } + + protected inSelection(acct: string, object: ServerObject): boolean { + return this.cardKeys(acct, object as ContactCardWire).some((k) => this.syncedKeys.has(k)); + } + + /** + * Where a contact created on the device goes: the chosen book when it + * syncs and is writable, else the personal account's default (or first) + * synced writable book; null when there is none. + */ + createTarget(): string | null { + const chosen = this.env.prefs.newContactsAddressBook; + if (chosen && this.syncedKeys.has(chosen) && this.writable(chosen)) return chosen; + const primary = this.env.accounts.find((a) => a.primary) ?? this.env.accounts[0]; + if (!primary) return null; + const books = [...(this.books.get(primary.id) ?? [])].sort( + (a, b) => Number(!!b.isDefault) - Number(!!a.isDefault) || (a.sortOrder ?? 0) - (b.sortOrder ?? 0) || a.id.localeCompare(b.id), + ); + const book = books.find((b) => { + const key = collectionKey(primary.id, b.id); + return this.syncedKeys.has(key) && this.writable(key); + }); + return book ? collectionKey(primary.id, book.id) : null; + } + + private photoBytes(entry: { uri?: string; blobId?: string }): string | null { + if (entry.blobId) return this.blobPhotos.get(entry.blobId) ?? null; + return dataUriBase64(entry.uri); + } + + ctx(acct: string): ContactsRunContext { + const env = this.env; + return { + jmapAccountId: acct, + now: env.now(), + mintKey: (taken) => env.mintKey(taken), + mintUid: () => env.mintUid(), + selectedCollections: (card) => this.cardKeys(acct, card).filter((k) => this.syncedKeys.has(k)), + isReadOnly: (card) => { + const keys = this.cardKeys(acct, card); + return keys.length > 0 && keys.every((k) => this.book(k)?.myRights?.mayWrite === false); + }, + isSelected: (key) => this.syncedKeys.has(key), + createTarget: () => this.createTarget(), + photoBytes: (entry) => this.photoBytes(entry), + devicePhoto: (rawContactId) => this.devicePhotos.get(rawContactId) ?? null, + nameForUid: (uid) => this.uidNames?.get(uid) ?? null, + groupRowIdBySourceId: (sourceId) => this.groupRows.get(sourceId) ?? null, + groupsOf: (uid) => [...(this.membership.get(uid) ?? [])], + }; + } + + // ── Downloads ── + + /** + * Group cards first: a membership naming an unknown group makes the + * provider create an empty one. Memberships live on the group cards, so + * when a group's `members` changed, its old and new members come along in + * the same download (before its state op): their GroupMembership rows + * change although their own cards did not. + */ + protected async order(acct: string, ids: string[]): Promise<{ first: string[]; rest: string[]; missing: string[] }> { + if (!ids.length) return { first: [], rest: [], missing: [] }; + const { list, notFound } = await this.env.jmap.get<{ id: string; kind?: string; members?: Record }>( + 'ContactCard', + acct, + ids, + ['id', 'kind', 'members'], + ); + const groups = list.filter((c) => c.kind === 'group'); + const groupIds = new Set(groups.map((c) => c.id)); + const found = new Set(list.map((c) => c.id)); + const rest = ids.filter((id) => found.has(id) && !groupIds.has(id)); + const members = (await this.membersToRefresh(acct, groups)).filter((id) => !groupIds.has(id)); + return { + first: ids.filter((id) => groupIds.has(id)), + rest: members.length ? [...new Set([...rest, ...members])].sort() : rest, + missing: notFound, + }; + } + + /** Contacts of `acct` whose membership in one of these group cards flipped since the group's shadow. */ + private async membersToRefresh(acct: string, groups: Array<{ id: string; members?: Record }>): Promise { + if (!groups.length) return []; + const locals = await this.groupKind.loadByRefs(groups.map((g) => refOf(acct, g.id))); + const flipped = new Set(); + for (const group of groups) { + const before = new Set(onIds(locals.get(refOf(acct, group.id))?.shadow?.members as Record | undefined)); + const after = new Set(onIds(group.members)); + for (const uid of before) if (!after.has(uid)) flipped.add(uid); + for (const uid of after) if (!before.has(uid)) flipped.add(uid); + } + if (!flipped.size) return []; + await this.ensureShadowIndex(); + const out: string[] = []; + for (const uid of flipped) { + for (const ref of this.uidRefs?.get(uid) ?? []) { + const id = idInAccount(ref, acct); + if (id) out.push(id); + } + } + return out; + } + + protected async beforePlanning(acct: string, objects: ServerObject[]): Promise { + const cards = objects as ContactCardWire[]; + if (this.groupsStale && cards.some((c) => c.kind !== 'group')) await this.refreshGroups(); + for (const card of cards) { + const photo = Object.values(card.media ?? {}).find((m) => m?.kind === 'photo') as + | { blobId?: string; mediaType?: string } + | undefined; + if (!photo?.blobId || this.blobPhotos.has(photo.blobId)) continue; + try { + this.blobPhotos.set(photo.blobId, toBase64(await this.env.jmap.downloadBlob(acct, photo.blobId, photo.mediaType))); + } catch (error) { + this.env.log('photo download failed', error); + this.blobPhotos.set(photo.blobId, null); + } + } + if (cards.some((c) => Object.keys(c.relatedTo ?? {}).length > 0)) await this.ensureShadowIndex(); + for (const card of cards) this.indexCard(acct, card); + } + + private indexCard(acct: string, card: ContactCardWire & { id?: string }): void { + if (!this.uidNames || !this.uidRefs || typeof card.uid !== 'string' || card.kind === 'group') return; + const name = displayNameOf(card); + if (name) this.uidNames.set(card.uid, name); + if (card.id) { + const ref = refOf(acct, card.id); + const refs = this.uidRefs.get(card.uid) ?? []; + if (!refs.includes(ref)) this.uidRefs.set(card.uid, [...refs, ref]); + } + } + + /** + * The synced cards by uid (their names for Relation rows, their rows for + * membership changes), read from the shadows on first need and kept up to + * date with every card downloaded afterwards. + */ + private async ensureShadowIndex(): Promise { + if (this.uidNames && this.uidRefs) return; + const names = new Map(); + const refs = new Map(); + for (const row of await this.env.reader.rows('raw_contacts', ['sourceid', 'sync2'], 'sourceid IS NOT NULL')) { + const shadow = parseJsonColumn(row.sync2); + const ref = str(row.sourceid); + if (!shadow || typeof shadow.uid !== 'string' || !ref) continue; + const name = displayNameOf(shadow); + if (name) names.set(shadow.uid, name); + refs.set(shadow.uid, [...(refs.get(shadow.uid) ?? []), ref]); + } + this.uidNames = names; + this.uidRefs = refs; + } + + protected adoptionTargetMatches(acct: string, object: ServerObject, target: string): boolean { + return this.cardKeys(acct, object as ContactCardWire).includes(target); + } + + protected lookupScope(_acct: string, collectionId: string): string | null { + // Card uids are unique per address book. + return collectionId; + } + + private groupChange(acct: string, ref: string, present: boolean): void { + let changes = this.groupChanges.get(acct); + if (!changes) this.groupChanges.set(acct, (changes = new Map())); + changes.set(ref, present); + this.groupsStale = true; + } + + protected onDownloaded(kind: Kind, acct: string, object: ServerObject): void { + if (kind === this.groupKind) this.groupChange(acct, refOf(acct, object.id), true); + else this.indexCard(acct, object as ContactCardWire); + } + + protected onRemoved(kind: Kind, acct: string, id: string): void { + if (kind === this.groupKind) this.groupChange(acct, refOf(acct, id), false); + } + + protected onAccepted(kind: Kind, acct: string, server: ServerObject): void { + if (kind === this.groupKind) this.groupChange(acct, refOf(acct, server.id), true); + else this.indexCard(acct, server as ContactCardWire); + } + + protected decorateState(acct: string, next: SyncState): () => void { + const account = accountOf(next, acct); + const changes = this.groupChanges.get(acct); + const consumed = new Map(changes ?? []); + if (consumed.size || account.groups === undefined) { + const groups = new Set(account.groups ?? []); + for (const [ref, present] of consumed) { + if (present) groups.add(ref); + else groups.delete(ref); + } + account.groups = [...groups]; + } + const collections = this.collectionStates.get(acct); + if (collections) account.collectionsState = collections; + return () => { + for (const [ref, present] of consumed) if (changes?.get(ref) === present) changes.delete(ref); + }; + } + + protected async localIdentities(acct: string): Promise> { + const out = new Map(); + for (const table of ['groups', 'raw_contacts'] as const) { + for (const row of await this.env.reader.rows(table, ['_id', 'sourceid', 'dirty', 'deleted'], 'sourceid LIKE ?', [`${acct}/%`])) { + const id = idInAccount(str(row.sourceid), acct); + if (id) out.set(id, { dirty: flag(row.dirty), deleted: flag(row.deleted) }); + } + } + return out; + } + + // ── Uploads ── + + protected async loadUploadItems(): Promise { + const groups = await this.loadGroupsWhere('dirty = 1 OR deleted = 1 OR sourceid IS NULL'); + const contacts = await this.loadContactsWhere('dirty = 1 OR deleted = 1 OR sourceid IS NULL'); + return [ + ...groups.map((local) => ({ kind: this.groupKind as Kind, local })), + ...contacts.map((local) => ({ kind: this.contactKind as Kind, local })), + ]; + } + + protected claimAccount(): string | null { + const target = this.createTarget(); + return accountOfRef(target) ?? this.env.accounts.find((a) => a.primary)?.id ?? this.env.accounts[0]?.id ?? null; + } + + protected noteUploadItems(acct: string, items: Held[]): void { + let rows = this.uploadRows.get(acct); + if (!rows) this.uploadRows.set(acct, (rows = new Set())); + for (const held of items) if (held.kind === this.contactKind) rows.add(held.kind.meta(held.local).rowId); + } + + /** The device photo of every contact about to be planned that has a photo row (JPEG, ≤ 512 px). */ + protected async beforeUploadPlanning(items: Held[]): Promise { + for (const held of items) { + if (held.kind !== this.contactKind) continue; + const contact = held.local as LocalContact; + if (this.devicePhotos.has(contact.rawContactId) || !contact.rows.some((r) => r.mimetype === MimeType.PHOTO)) continue; + try { + const photo = await this.env.reader.port.readPhoto(contact.rawContactId, 512); + this.devicePhotos.set(contact.rawContactId, photo?.jpegBase64 ?? null); + } catch (error) { + this.env.log('photo read failed', error); + this.devicePhotos.set(contact.rawContactId, null); + } + } + } + + /** Group refs the SyncState lists whose rows are gone: an app hard-deleted them (Fossify does). */ + private async absentGroups(): Promise> { + const present = new Set(); + for (const row of await this.env.reader.rows('groups', ['sourceid'], 'sourceid IS NOT NULL')) { + const ref = str(row.sourceid); + if (ref) present.add(ref); + } + const out = new Map(); + for (const [acct, account] of Object.entries(this.env.store.committed.accounts)) { + const changes = this.groupChanges.get(acct); + const refs = (account.groups ?? []).filter((ref) => !present.has(ref) && changes?.get(ref) !== false); + if (refs.length) out.set(acct, refs); + } + return out; + } + + protected async countDeletions(items: Held[]): Promise<{ count: number; synced: number; accounts: Set }> { + let count = 0; + const accounts = new Set(); + for (const held of items) { + const meta = held.kind.meta(held.local); + if (!meta.deleted || !meta.sourceId) continue; + count++; + const acct = accountOfRef(meta.sourceId); + if (acct) accounts.add(acct); + } + this.absent = await this.absentGroups(); + for (const [acct, refs] of this.absent) { + count += refs.length; + accounts.add(acct); + } + const synced = + (await this.env.reader.rows('raw_contacts', ['_id'], 'sourceid IS NOT NULL')).length + + (await this.env.reader.rows('groups', ['_id'], 'sourceid IS NOT NULL')).length; + return { count, synced: Math.max(synced, count), accounts }; + } + + protected onDiscard(next: SyncState, accounts: Set): void { + for (const acct of accounts) { + const gone = new Set(this.absent.get(acct) ?? []); + if (!gone.size) continue; + const account = accountOf(next, acct); + account.groups = (account.groups ?? []).filter((ref) => !gone.has(ref)); + } + this.absent = new Map(); + } + + protected async extraPhase(acct: string, deletionsAllowed: boolean): Promise { + if (deletionsAllowed) await this.deleteAbsentGroups(acct); + await this.uploadMemberships(acct); + } + + /** + * A group hard-deleted on the device: destroyed on the server, or, when + * it is also in address books that do not sync, removed from the synced + * ones only. + */ + private async deleteAbsentGroups(acct: string): Promise { + const refs = (this.absent.get(acct) ?? []).filter((ref) => this.groupChanges.get(acct)?.get(ref) !== false); + if (!refs.length) return; + await this.env.checkpoints.check('upload:groups'); + const ids = refs.map((ref) => idInAccount(ref, acct)).filter((id): id is string => !!id); + const { list, notFound } = await this.env.jmap.get('ContactCard', acct, ids, ['id', 'addressBookIds']); + for (const id of notFound) this.groupChange(acct, refOf(acct, id), false); + const update: Record> = {}; + const destroy: string[] = []; + for (const card of list) { + const books = onIds(card.addressBookIds); + const synced = books.filter((id) => this.syncedKeys.has(collectionKey(acct, id))); + if (synced.length && synced.length < books.length) { + update[card.id] = Object.fromEntries(synced.map((id) => [`addressBookIds/${id}`, null])); + } else { + destroy.push(card.id); + } + } + if (!destroy.length && !Object.keys(update).length) return; + const { response } = await this.setChecked(acct, { update, destroy }); + const done = [...(response.destroyed ?? []), ...Object.keys(response.updated ?? {})]; + for (const id of done) { + this.groupChange(acct, refOf(acct, id), false); + this.env.report.stats.uploaded.deleted++; + } + for (const [id, error] of [...Object.entries(response.notDestroyed ?? {}), ...Object.entries(response.notUpdated ?? {})]) { + if (error?.type === 'notFound') this.groupChange(acct, refOf(acct, id), false); + else this.env.report.itemError({ ref: refOf(acct, id), side: 'upload', type: error?.type ?? 'serverFail', description: error?.description }); + } + } + + /** Memberships edited on the device become patches of the group cards' `members`. */ + private async uploadMemberships(acct: string): Promise { + const rows = [...(this.uploadRows.get(acct) ?? [])]; + if (!rows.length) return; + await this.env.checkpoints.check('upload:memberships'); + const contacts = (await this.contactKind.loadByRowIds(rows)).filter((c) => !c.deleted); + const groups = (await this.loadGroupsWhere('sourceid LIKE ?', [`${acct}/%`])).filter( + (g) => accountOfRef(g.sourceId) === acct && !g.deleted, + ); + const actions = this.planner.planMembershipUploads(contacts, groups, this.ctx(acct)); + if (!actions.length) return; + const byGroup = new Map(); + for (const action of actions) { + const id = action.kind === 'create' ? null : action.id; + const group = id ? groups.find((g) => g.sourceId === refOf(acct, id)) : undefined; + if (!group) { + this.env.report.itemError({ ref: id ? refOf(acct, id) : 'membership', side: 'upload', type: 'unknownGroup' }); + continue; + } + let pending = byGroup.get(group.sourceId as string); + if (!pending) { + const held: Held = { kind: this.groupKind, local: group }; + pending = { held, meta: this.groupKind.meta(group), fingerprint: this.groupKind.fingerprint(group), actions: [] }; + byGroup.set(group.sourceId as string, pending); + } + pending.actions.push(action as Pending['actions'][number]); + } + if (byGroup.size) await this.send(acct, [...byGroup.values()]); + } + + // ── Deselection ── + + /** + * After the upload: rows of contacts in no synced address book go, when + * clean (a deselected book, or a card moved away on the server). Dirty + * ones stay until their upload succeeds, and keep their book selected. + */ + protected async dropOutsideSelection(acct: string): Promise { + const synced = new Set(this.syncedCollections(acct).map((id) => collectionKey(acct, id))); + const committed = this.env.store.account(acct); + const deselected = committed.selected.filter((key) => !synced.has(key)); + const outside = [...(this.outside.get(acct) ?? [])]; + if (!deselected.length && !outside.length) return; + + const kept = new Set(); + const consider = (held: Held, works: Work[]) => { + const meta = held.kind.meta(held.local); + const id = idInAccount(meta.sourceId, acct); + // Rows whose collections are unknown are never dropped. + if (!id || !meta.collections.length || meta.collections.some((k) => synced.has(k))) return; + const work = this.removeWork(acct, id, held, 'outside'); + if (work) works.push(work); + else for (const key of meta.collections) if (deselected.includes(key)) kept.add(key); + }; + if (deselected.length) { + const groupWorks: Work[] = []; + for (const group of await this.loadGroupsWhere('sourceid LIKE ?', [`${acct}/%`])) consider({ kind: this.groupKind, local: group }, groupWorks); + await this.env.writer.write(groupWorks); + const rows = await this.env.reader.rows('raw_contacts', ['_id', 'sourceid'], 'sourceid LIKE ?', [`${acct}/%`]); + const ids = rows.filter((r) => accountOfRef(str(r.sourceid)) === acct).map((r) => num(r._id)); + for (let i = 0; i < ids.length; i += this.env.tuning.chunkSize) { + await this.env.checkpoints.check('deselect'); + const works: Work[] = []; + for (const contact of await this.contactKind.loadByRowIds(ids.slice(i, i + this.env.tuning.chunkSize))) { + consider({ kind: this.contactKind, local: contact }, works); + } + await this.env.writer.write(works); + } + const selected = [...synced, ...kept]; + // The book leaves `selected` with its last dropped row; one still holding a dirty item stays. + await this.env.writer.write([], this.tail([acct], (next) => { + accountOf(next, acct).selected = selected; + })); + } else { + const works: Work[] = []; + for (const kind of this.kinds()) { + for (const local of (await kind.loadByRefs(outside)).values()) consider({ kind, local }, works); + } + await this.env.writer.write(works); + } + this.outside.delete(acct); + } +} diff --git a/src/device-sync/engine/context.ts b/src/device-sync/engine/context.ts new file mode 100644 index 00000000..9162a6fe --- /dev/null +++ b/src/device-sync/engine/context.ts @@ -0,0 +1,39 @@ +/** + * What one run (or teardown) of one Android account and authority carries + * through its phases. + */ +import type { AccountSyncPrefs, Authority, RunExtras } from '../types'; +import type { JmapCaller } from '../jmap/caller'; +import type { CapabilityAccount } from '../jmap/session'; +import type { BatchWriter } from './batch'; +import type { Checkpoints } from './checkpoint'; +import type { EngineDeps, Tuning } from './deps'; +import type { ProviderReader } from './provider'; +import type { ReportBuilder } from './report'; +import type { StateStore } from './sync-state'; + +export interface RunEnv { + readonly deps: EngineDeps; + readonly tuning: Tuning; + /** A teardown only uploads (and never writes calendar rows). */ + readonly mode: 'sync' | 'teardown'; + readonly registryId: string; + readonly accountName: string; + readonly authority: Authority; + readonly extras: RunExtras; + readonly reader: ProviderReader; + readonly writer: BatchWriter; + readonly report: ReportBuilder; + readonly checkpoints: Checkpoints; + readonly store: StateStore; + readonly jmap: JmapCaller; + /** Accounts with the authority's capability, the primary first. */ + readonly accounts: CapabilityAccount[]; + readonly prefs: AccountSyncPrefs; + now(): number; + mintKey(taken?: Iterable): string; + mintUid(): string; + sleep(ms: number): Promise; + log(message: string, detail?: unknown): void; + recordKnownState(jmapAccountId: string, type: string, state: string): void; +} diff --git a/src/device-sync/engine/deps.ts b/src/device-sync/engine/deps.ts new file mode 100644 index 00000000..f7bbf4a2 --- /dev/null +++ b/src/device-sync/engine/deps.ts @@ -0,0 +1,113 @@ +/** + * Everything the sync engine reaches outside itself. The engine is pure + * TypeScript: src/device-sync/task.ts wires these to the native module, a + * detached JMAPClient and the app's stores; the tests wire them to the fake + * provider, the fake JMAP server and test planners. + */ +import type { CalendarPlanner, ContactsPlanner } from '../planner'; +import type { RandomSource } from '../common/ids'; +import type { AccountSyncPrefs, Authority, ProviderPort, RunStatus } from '../types'; +import type { JmapConnection } from '../jmap/connection'; + +export type { JmapConnection }; + +/** A calendar that mirrors an iCal feed (read-only on the device). */ +export interface SubscriptionCalendar { + /** JMAP account of the mirror calendar; null for subscriptions saved before accounts were recorded (the primary). */ + jmapAccountId: string | null; + calendarId: string; +} + +export interface EngineDeps { + /** The provider of one Android account and authority. */ + provider(accountName: string, authority: Authority): ProviderPort; + /** A JMAP connection for the registry account (rebuilt once on an auth failure); throws the client's errors. */ + jmap(registryId: string): Promise; + /** The account's device-sync preferences (the store is hydrated first). */ + prefs(registryId: string): Promise; + planners: { contacts: ContactsPlanner; calendar: CalendarPlanner }; + /** Epoch milliseconds. */ + now(): number; + /** Whether the framework cancelled the run, or nobody waits for it any more. */ + isCancelled(runId: string): Promise; + random: RandomSource; + /** Feed-subscription calendars (read-only on the device). */ + subscriptionCalendars(): Promise; + /** Android's `getSyncAutomatically` for the account and authority. */ + isSyncEnabled(accountName: string, authority: Authority): Promise; + /** The device's IANA time zone (floating events are written in it). */ + deviceZone(): string; + /** The last run's status for the settings UI. */ + recordStatus(registryId: string, authority: Authority, status: RunStatus): void | Promise; + /** The latest state the engine synced for a JMAP account and type, so the app skips the echo of our own writes. */ + recordKnownState(registryId: string, jmapAccountId: string, type: string, state: string): void | Promise; + /** One notification that deep-links to sign-in, after an auth failure the rebuild did not fix. */ + notifyAuthProblem(registryId: string, accountName: string, authority: Authority): void | Promise; + /** Lets the JS thread render between chunks (default: `setTimeout(0)`). */ + yieldThread?(): Promise; + /** Waits before a retry, e.g. the uid lookup's back-off (default: `setTimeout`). */ + sleep?(ms: number): Promise; + log?(message: string, detail?: unknown): void; + /** Overrides for tests. */ + tuning?: Partial; +} + +/** Sizes and limits from docs/device-sync.md ("Limits and performance", "Deletion threshold"). */ +export interface Tuning { + /** Raw contacts or events planned per chunk. */ + chunkSize: number; + /** Provider ops per applyBatch, with a yield point at every item group. */ + maxBatchOps: number; + /** Serialized bytes per applyBatch (the Binder budget is 1 MB for the whole transaction). */ + maxBatchBytes: number; + /** Re-reads and re-plans of an item whose group failed an assert. */ + maxReplans: number; + /** Creates per `/set` (a create batch is not interruptible). */ + maxCreatesPerSet: number; + /** Serialized bytes per `/set` call (Stalwart splits bigger calls into several commits). */ + maxSetBytes: number; + /** `stateMismatch` retries of an account's upload. */ + maxStateMismatchRetries: number; + /** Stop starting work this long before the deadline. */ + deadlineMarginMs: number; + /** A create batch starts only with this much budget left (two request timeouts). */ + createBudgetMs: number; + /** A `Core/echo` when a local phase has gone this long without a JMAP request. */ + echoAfterMs: number; + /** Deletions above this count … */ + deletionThresholdCount: number; + /** … and above this share of the authority's synced objects set `tooManyDeletions`. */ + deletionThresholdRatio: number; + /** A full reconcile that finds nothing while the device holds more than this aborts. */ + safetyAbortRows: number; + /** Back-off of a poisoned item: first delay, doubling up to the cap. */ + poisonBackoffMs: number; + poisonBackoffMaxMs: number; + /** Retries of a uid lookup after a duplicate-uid error, with these delays. */ + uidLookupDelaysMs: number[]; + /** Objects scanned to tell a task-only calendar. */ + taskScanLimit: number; + /** Item errors kept in a report. */ + maxItemErrors: number; +} + +export const DEFAULT_TUNING: Tuning = { + chunkSize: 50, + maxBatchOps: 400, + maxBatchBytes: 300_000, + maxReplans: 3, + maxCreatesPerSet: 50, + maxSetBytes: 1_000_000, + maxStateMismatchRetries: 3, + deadlineMarginMs: 30_000, + createBudgetMs: 60_000, + echoAfterMs: 40_000, + deletionThresholdCount: 50, + deletionThresholdRatio: 0.2, + safetyAbortRows: 10, + poisonBackoffMs: 60 * 60_000, + poisonBackoffMaxMs: 24 * 60 * 60_000, + uidLookupDelaysMs: [500, 1500, 4000], + taskScanLimit: 50, + maxItemErrors: 50, +}; diff --git a/src/device-sync/engine/errors.ts b/src/device-sync/engine/errors.ts new file mode 100644 index 00000000..d2fe7372 --- /dev/null +++ b/src/device-sync/engine/errors.ts @@ -0,0 +1,88 @@ +/** + * How a run stops early, and what each failure means for the report + * (docs/device-sync.md, "Headless payload, report and SyncResult" and + * "Failure matrix"). + */ +import type { RunOutcome } from '../types'; +import { isMethodError, isRequestLimitError, retryAfterMs, transportFailure } from '../jmap/errors'; + +/** Out of time, or cancelled: stop cleanly between chunks, ask for another sync. */ +export class StopRun extends Error { + constructor(readonly reason: 'cancelled' | 'deadline') { + super(reason === 'deadline' ? 'Out of time; the rest follows in the next sync' : 'Cancelled'); + this.name = 'StopRun'; + } +} + +/** A run ends with this outcome. `noProgress` zeroes the counts so SyncManager backs off instead of retrying at once. */ +export class RunAbort extends Error { + constructor( + readonly outcome: RunOutcome, + message: string, + readonly options: { noProgress?: boolean; delayUntil?: number } = {}, + ) { + super(message); + this.name = 'RunAbort'; + } +} + +/** The server changed since the download an upload was planned from. */ +export class StateMismatch extends Error { + constructor(readonly jmapAccountId: string) { + super(`stateMismatch in ${jmapAccountId}`); + this.name = 'StateMismatch'; + } +} + +export interface ClassifiedFailure { + outcome: RunOutcome; + message: string; + delayUntil?: number; + moreRecordsToGet?: boolean; + noProgress?: boolean; + /** Tell the user to sign in again. */ + authProblem?: boolean; +} + +function messageOf(error: unknown): string { + const message = error instanceof Error ? error.message : String(error); + return message.length > 300 ? `${message.slice(0, 300)}…` : message; +} + +/** A provider refusal for a missing runtime permission (the native module rejects with it). */ +export function isPermissionFailure(error: unknown): boolean { + const e = error as { code?: unknown; message?: unknown } | null; + return e?.code === 'permission' || (typeof e?.message === 'string' && /\bpermission\b|SecurityException/i.test(e.message)); +} + +export function classifyFailure(error: unknown, now: number): ClassifiedFailure { + if (error instanceof StopRun) { + return { outcome: 'cancelled', message: error.message, moreRecordsToGet: true }; + } + if (error instanceof RunAbort) { + return { outcome: error.outcome, message: error.message, noProgress: error.options.noProgress, delayUntil: error.options.delayUntil }; + } + if (error instanceof StateMismatch) { + return { outcome: 'io', message: 'The server kept changing during the upload', noProgress: true }; + } + switch (transportFailure(error)) { + case 'auth': + return { outcome: 'auth', message: messageOf(error), authProblem: true }; + case 'network': + case 'timeout': + return { outcome: 'io', message: messageOf(error) }; + case 'rateLimit': { + const wait = retryAfterMs(error) ?? 60_000; + return { outcome: 'io', message: messageOf(error), delayUntil: Math.ceil((now + wait) / 1000) }; + } + default: + break; + } + if (isPermissionFailure(error)) return { outcome: 'permission', message: messageOf(error) }; + if (isRequestLimitError(error)) return { outcome: 'internal', message: messageOf(error) }; + if (isMethodError(error)) return { outcome: 'io', message: `${error.type}: ${messageOf(error)}` }; + if (error instanceof Error && /^(JMAP request failed|Invalid JSON response)/.test(error.message)) { + return { outcome: 'io', message: messageOf(error) }; + } + return { outcome: 'internal', message: messageOf(error) }; +} diff --git a/src/device-sync/engine/index.ts b/src/device-sync/engine/index.ts new file mode 100644 index 00000000..5434709f --- /dev/null +++ b/src/device-sync/engine/index.ts @@ -0,0 +1,10 @@ +/** + * The device sync engine (docs/device-sync.md). Pure TypeScript: every + * dependency comes in through `EngineDeps`; src/device-sync/task.ts wires + * the real ones. + */ +export { runDeviceSync, hasLocalWork } from './run'; +export { teardownAuthority, countPending, type TeardownOptions } from './teardown'; +export { DEFAULT_TUNING, type EngineDeps, type JmapConnection, type SubscriptionCalendar, type Tuning } from './deps'; +export { parseSyncState, serializeSyncState, type SyncState, type AccountState } from './sync-state'; +export { lockKey, isLocked } from './mutex'; diff --git a/src/device-sync/engine/item-sync.ts b/src/device-sync/engine/item-sync.ts new file mode 100644 index 00000000..7f0f731c --- /dev/null +++ b/src/device-sync/engine/item-sync.ts @@ -0,0 +1,1303 @@ +/** + * The download and upload machinery shared by the contacts and calendar sync + * (docs/device-sync.md, "A sync run", "Change detection and merge rules", + * "Uploads"). The authorities plug in their kinds of local items, their + * collections and their extra phases; everything that talks to the server, + * the batching, the retries and the guards is here. + */ +import type { OpGroup, UploadAction } from '../planner'; +import { collectionKey } from '../common/ids'; +import { + classifySetError, + existingIdFromUidError, + type ItemType, + type SetArgs, + type SetErrorShape, + type SetResponse, +} from '../jmap/caller'; +import { isMethodError } from '../jmap/errors'; +import { concatGroups, prependOps, type Work } from './batch'; +import type { RunEnv } from './context'; +import { RunAbort, StateMismatch } from './errors'; +import { + accountOfRef, + idInAccount, + insertGuard, + poisonOp, + refOf, + type Held, + type ItemMeta, + type Kind, + type ServerObject, +} from './kinds'; +import { isBackedOff, nextMarker } from './poison'; +import { hasWrites } from './provider'; +import { accountOf, type SyncState, type Tail } from './sync-state'; + +type Action = UploadAction; + +/** An item's upload, on its way to the server. */ +export interface Pending { + held: Held; + meta: ItemMeta; + fingerprint: string; + actions: Action[]; + /** Resent without scheduling messages after a `forbidden`. */ + noScheduling?: boolean; + /** Custom handling once every action was accepted (pairs, memberships); replaces `planAccepted`. */ + accept?(objects: Map): Promise; +} + +type Outcome = + | { action: Action; ok: true; id: string } + | { action: Action; ok: false; error: SetErrorShape; scheduling: boolean }; + +interface SetCallPlan { + args: { create: Record>; update: Record>; destroy: string[]; sendSchedulingMessages: boolean }; + entries: Array<{ pending: Pending; action: Action; key: string }>; + touched: Set; + updateIds: string[]; + creates: number; + bytes: number; +} + +type Removal = 'destroyed' | 'outside'; + +/** Method-level refusals of a whole `/set` that concern the account, not the objects. */ +const ACCOUNT_REFUSALS = new Set(['accountReadOnly', 'forbidden', 'accountNotFound', 'accountNotSupportedByMethod']); + +export abstract class ItemSync { + /** Per account: object ids put on `stale` (true) or fetched fine again (false), for the next state op. */ + private readonly staleMarks = new Map>(); + /** Per account: refs of dirty items whose object left every synced collection; dropped once uploaded. */ + protected readonly outside = new Map>(); + /** Per account: objects our uploads created for other rows (a series split), adopted by the download. */ + private readonly extraCreated = new Map>(); + /** Per account: `ifInState` of the next `/set` (the download's state, then our last `newState`). */ + private readonly setStates = new Map(); + + constructor(protected readonly env: RunEnv) {} + + // ── What the authority provides ── + + protected abstract readonly itemType: ItemType; + protected abstract readonly itemProperties: readonly string[]; + protected abstract readonly parentFilter: 'inAddressBook' | 'inCalendar'; + protected abstract readonly parentProperty: 'addressBookIds' | 'calendarIds'; + + protected abstract kinds(): Kind[]; + /** The kind whose rows hold the object; null when it does not sync (a task). */ + protected abstract kindOf(object: ServerObject): Kind | null; + /** Whether the object is in a collection that syncs. */ + protected abstract inSelection(jmapAccountId: string, object: ServerObject): boolean; + /** Ids of the collections of the account that sync now. */ + protected abstract syncedCollections(jmapAccountId: string): string[]; + /** Reads the collections, and (`write`) brings the collection rows up to date. */ + protected abstract collections(write: boolean): Promise; + /** Local identities of the account: object id → local flags, for reconcile deletions. */ + protected abstract localIdentities(jmapAccountId: string): Promise>; + /** Every item with local changes: dirty, deleted, or new. */ + protected abstract loadUploadItems(): Promise; + /** The JMAP account a new item without a pending create is claimed in. */ + protected abstract claimAccount(held: Held): string | null; + /** Local deletions waiting for an upload, synced objects, and the accounts concerned. */ + protected abstract countDeletions(items: Held[]): Promise<{ count: number; synced: number; accounts: Set }>; + /** Drops the clean rows of deselected collections and of objects that left the selection. */ + protected abstract dropOutsideSelection(jmapAccountId: string): Promise; + + // ── Hooks ── + + /** Processing order of a download: `first` completely before `rest` (group cards before contacts); `missing` are gone. */ + protected async order(_acct: string, ids: string[]): Promise<{ first: string[]; rest: string[]; missing: string[] }> { + return { first: [], rest: ids, missing: [] }; + } + protected async beforePlanning(_acct: string, _objects: ServerObject[]): Promise {} + protected async beforeUploadPlanning(_items: Held[]): Promise {} + protected async beforeReconcile(_acct: string): Promise {} + /** Authority changes folded into a state op of the account; the callback runs once that op is stored. */ + protected decorateState(_acct: string, _next: SyncState): (() => void) | void {} + protected onDownloaded(_kind: Kind, _acct: string, _object: ServerObject): void {} + protected onRemoved(_kind: Kind, _acct: string, _id: string): void {} + protected onAccepted(_kind: Kind, _acct: string, _server: ServerObject): void {} + protected onDiscard(_next: SyncState, _accounts: Set): void {} + /** Whether a pending create of `target` may adopt this object (contacts: the target book holds it). */ + protected adoptionTargetMatches(acct: string, _object: ServerObject, target: string): boolean { + return accountOfRef(target) === acct; + } + /** Where a uid is looked up for a create in `collectionId`: that collection, or null for the whole account. */ + protected lookupScope(_acct: string, _collectionId: string): string | null { + return null; + } + /** Whether a new item is claimed with the others (calendar: a split clone waits for its source). */ + protected claimable(_held: Held): boolean { + return true; + } + /** The JMAP account an item uploads to. */ + protected accountOfItem(held: Held): string | null { + const meta = held.kind.meta(held.local); + return accountOfRef(meta.sourceId) ?? accountOfRef(meta.pending?.target ?? null); + } + /** Before the creates (calendar: split series and pairs). */ + protected async prePhase(_acct: string): Promise {} + /** After the deletions (contacts: groups deleted by absence, memberships). */ + protected async extraPhase(_acct: string, _deletionsAllowed: boolean): Promise {} + /** After uploads and deselection (calendar: zone and reminder-owner passes). */ + protected async afterSync(): Promise {} + + // ── Runs ── + + /** A sync run's phases after the preflight. */ + async sync(): Promise { + await this.collections(true); + for (const acct of this.accountIds()) await this.refetchStale(acct); + for (const acct of this.accountIds()) { + await this.changesLoop(acct); + await this.loadNewlySelected(acct); + } + await this.uploadAll(); + for (const acct of this.knownAccountIds()) { + await this.env.checkpoints.check('deselect'); + await this.dropOutsideSelection(acct); + } + await this.afterSync(); + await this.flushState(); + } + + /** A teardown's upload: everything waiting on the device, nothing else. */ + async uploadOnly(): Promise { + await this.collections(false); + await this.uploadAll(); + await this.flushState(); + } + + protected accountIds(): string[] { + return this.env.accounts.map((a) => a.id); + } + + /** The session's accounts plus accounts the rows were written for that are gone from it. */ + protected knownAccountIds(): string[] { + return [...new Set([...this.accountIds(), ...Object.keys(this.env.store.committed.accounts)])]; + } + + // ── State ── + + private marks(acct: string): Map { + let marks = this.staleMarks.get(acct); + if (!marks) this.staleMarks.set(acct, (marks = new Map())); + return marks; + } + + protected markStale(acct: string, id: string): void { + this.marks(acct).set(id, true); + } + + protected markResolved(acct: string, id: string): void { + if (this.marks(acct).has(id) || this.env.store.account(acct).stale.includes(id)) this.marks(acct).set(id, false); + } + + protected isStale(acct: string, sourceId: string | null): boolean { + const id = idInAccount(sourceId, acct); + if (!id) return false; + return this.marks(acct).get(id) ?? this.env.store.account(acct).stale.includes(id); + } + + /** + * A planner threw on one item (a bug, or data it cannot handle): that + * item is reported and left alone, the rest of the run goes on. A + * download's object goes on `stale`, so the state still moves and the + * object is tried again next run. + */ + protected plannerFailed(acct: string, ref: string, side: 'download' | 'upload', error: unknown, staleId?: string): void { + if (staleId) this.markStale(acct, staleId); + if (side === 'upload') this.env.report.stats.skipped++; + const description = error instanceof Error ? error.message : String(error); + this.env.report.itemError({ ref, side, type: 'plannerError', description: description.slice(0, 300) }); + this.env.log(`planner failed on ${ref}`, error); + } + + protected outsideOf(acct: string): Set { + let set = this.outside.get(acct); + if (!set) this.outside.set(acct, (set = new Set())); + return set; + } + + /** A state op for `accts`, carrying their stale marks and the authority's changes along with `mutate`. */ + protected tail(accts: readonly string[], mutate?: (next: SyncState) => void): Tail { + const consumed: Array<[string, string, boolean]> = []; + const commits: Array<() => void> = []; + const base = this.env.store.tail((next) => { + consumed.length = 0; + commits.length = 0; + mutate?.(next); + for (const acct of accts) { + const account = accountOf(next, acct); + const marks = this.staleMarks.get(acct); + if (marks?.size) { + const stale = new Set(account.stale); + for (const [id, isStale] of marks) { + consumed.push([acct, id, isStale]); + if (isStale) stale.add(id); + else stale.delete(id); + } + account.stale = [...stale]; + } + const commit = this.decorateState(acct, next); + if (commit) commits.push(commit); + } + }); + return { + op: base.op, + applied: () => { + base.applied(); + for (const [acct, id, isStale] of consumed) { + const marks = this.staleMarks.get(acct); + if (marks?.get(id) === isStale) marks.delete(id); + } + for (const commit of commits) commit(); + }, + }; + } + + /** Stores what is still pending (stale marks, collection states, …) at the end of a run. */ + protected async flushState(): Promise { + const accts = this.knownAccountIds(); + const probe = this.tail(accts); + const before = JSON.stringify(this.env.store.committed); + const op = probe.op(); + if (op.op === 'syncState' && op.value === before) return; + await this.env.writer.write([], { op: () => op, applied: probe.applied }); + } + + // ── Downloads ── + + /** Objects put on `stale` by an earlier run: fetched again first (invariant 10). */ + protected async refetchStale(acct: string): Promise { + const ids = [...this.env.store.account(acct).stale]; + if (ids.length) await this.processIds(acct, ids, [], () => undefined); + } + + /** `/changes` pages from the stored state; a full reconcile without one or after `cannotCalculateChanges`. */ + protected async changesLoop(acct: string): Promise { + let reconciled = false; + for (let round = 0; round < 10_000; round++) { + const account = this.env.store.account(acct); + if (!account.itemsState || account.reconcile) { + if (reconciled) return; + await this.fullReconcile(acct); + reconciled = true; + continue; + } + const since = account.itemsState; + await this.env.checkpoints.check('changes'); + let page; + try { + page = await this.env.jmap.changes(this.itemType, acct, since); + } catch (error) { + if (isMethodError(error, 'cannotCalculateChanges') && !reconciled) { + this.env.log(`cannotCalculateChanges for ${acct}: full reconcile`); + await this.requestReconcile(acct); + continue; + } + throw error; + } + const ids = [...new Set([...page.created, ...page.updated])]; + const newState = page.newState; + if (!ids.length && !page.destroyed.length && newState === since) return; + await this.processIds(acct, ids, page.destroyed, (next) => { + accountOf(next, acct).itemsState = newState; + }); + if (!page.hasMoreChanges || newState === since) return; + } + } + + protected async requestReconcile(acct: string): Promise { + await this.env.writer.write( + [], + this.tail([acct], (next) => { + accountOf(next, acct).reconcile = { from: null, phase: 'ids', position: 0, after: null }; + }), + ); + } + + /** + * A full reconcile, checkpointed in `reconcile`: the state first, then the + * ids of every synced collection, then the objects in chunks; deletions + * only once the complete id list is known. Zero objects on the server + * against a full device stops the run (`safetyAbort`). + */ + protected async fullReconcile(acct: string): Promise { + let marker = this.env.store.account(acct).reconcile; + if (!marker?.from) { + await this.env.checkpoints.check('reconcile'); + const from = await this.env.jmap.takeState(this.itemType, acct); + const started = { from, phase: 'ids' as const, position: 0, after: null }; + await this.env.writer.write([], this.tail([acct], (next) => { + accountOf(next, acct).reconcile = { ...started }; + })); + marker = started; + } + const from = marker.from as string; + await this.beforeReconcile(acct); + const collections = this.syncedCollections(acct); + const listed = new Set(); + for (const collection of collections) { + await this.env.checkpoints.check('reconcile:ids'); + for (const id of await this.env.jmap.queryAll(this.itemType, acct, { [this.parentFilter]: collection })) listed.add(id); + } + const local = await this.localIdentities(acct); + if (listed.size === 0 && collections.length > 0 && local.size > this.env.tuning.safetyAbortRows) { + throw new RunAbort( + 'safetyAbort', + `The server lists no objects in ${acct} while this device holds ${local.size}; nothing was changed`, + ); + } + const { first, rest, missing } = await this.order(acct, [...listed].sort()); + const size = this.env.tuning.chunkSize; + for (let i = 0; i < first.length; i += size) { + await this.env.checkpoints.check('reconcile:objects'); + await this.processChunk(acct, first.slice(i, i + size), [], undefined); + } + const resumeAfter = marker.after; + const todo = resumeAfter ? rest.filter((id) => id > resumeAfter) : rest; + let position = rest.length - todo.length; + for (let i = 0; i < todo.length; i += size) { + await this.env.checkpoints.check('reconcile:objects'); + const part = todo.slice(i, i + size); + position += part.length; + const mark = { from, phase: 'objects' as const, position, after: part[part.length - 1] }; + await this.processChunk(acct, part, [], (next) => { + accountOf(next, acct).reconcile = { ...mark }; + }); + } + const gone = new Set(missing); + const verify: string[] = []; + for (const [id, flags] of local) { + if (listed.has(id)) continue; + if (flags.dirty || flags.deleted) verify.push(id); + else gone.add(id); + } + if (verify.length) { + // Destroyed, or moved out of the synced collections? Only a destroy takes dirty rows. + const { list, notFound } = await this.env.jmap.get(this.itemType, acct, verify, ['id']); + for (const id of notFound) gone.add(id); + for (const object of list) this.outsideOf(acct).add(refOf(acct, object.id)); + } + const selected = collections.map((c) => collectionKey(acct, c)); + await this.processIds(acct, [], [...gone], (next) => { + const account = accountOf(next, acct); + account.itemsState = from; + account.reconcile = null; + account.selected = selected; + }); + } + + /** Collections selected since the rows were written: loaded once the account is up to date. */ + protected async loadNewlySelected(acct: string): Promise { + const account = this.env.store.account(acct); + if (!account.itemsState || account.reconcile) return; + for (const collection of this.syncedCollections(acct)) { + const key = collectionKey(acct, collection); + if (this.env.store.account(acct).selected.includes(key)) continue; + await this.env.checkpoints.check('select'); + const ids = await this.env.jmap.queryAll(this.itemType, acct, { [this.parentFilter]: collection }); + await this.processIds(acct, ids, [], (next) => { + const a = accountOf(next, acct); + if (!a.selected.includes(key)) a.selected.push(key); + }); + } + } + + /** Downloads `ids` and removes the rows of `destroyed`, in chunks; `mutate` rides with the last one. */ + protected async processIds( + acct: string, + ids: readonly string[], + destroyed: readonly string[], + mutate?: (next: SyncState) => void, + ): Promise { + const { first, rest, missing } = ids.length ? await this.order(acct, [...ids]) : { first: [], rest: [], missing: [] }; + const size = this.env.tuning.chunkSize; + const chunks: Array<{ ids: string[]; gone: string[] }> = []; + for (const part of [first, rest]) { + for (let i = 0; i < part.length; i += size) chunks.push({ ids: part.slice(i, i + size), gone: [] }); + } + const gone = [...new Set([...destroyed, ...missing])]; + for (let i = 0; i < gone.length; i += size) chunks.push({ ids: [], gone: gone.slice(i, i + size) }); + if (chunks.length === 0) { + if (mutate) await this.env.writer.write([], this.tail([acct], mutate)); + return; + } + for (let i = 0; i < chunks.length; i++) { + await this.env.checkpoints.check('download'); + await this.processChunk(acct, chunks[i].ids, chunks[i].gone, i === chunks.length - 1 ? mutate : undefined); + } + } + + /** One chunk: fetch, find the local items (adopting pending creates by uid), plan, write, read back. */ + protected async processChunk( + acct: string, + ids: readonly string[], + gone: readonly string[], + mutate?: (next: SyncState) => void, + ): Promise { + let objects: ServerObject[] = []; + let notFound: string[] = []; + if (ids.length) { + ({ list: objects, notFound } = await this.env.jmap.get(this.itemType, acct, ids, this.itemProperties)); + } + await this.beforePlanning(acct, objects); + const allGone = [...new Set([...gone, ...notFound])]; + const held = await this.heldByRefs([...objects.map((o) => refOf(acct, o.id)), ...allGone.map((id) => refOf(acct, id))]); + let candidates: Held[] | null = null; + const works: Work[] = []; + const written = new Map>(); + for (const object of objects) { + this.env.report.stats.entries++; + const kind = this.kindOf(object); + let found = held.get(refOf(acct, object.id)) ?? null; + if (!found && kind && this.inSelection(acct, object)) { + candidates ??= await this.adoptionCandidates(acct); + found = this.takeAdoptable(candidates, kind, acct, object); + } + let work: Work | null; + try { + work = this.downloadDecision(acct, object, kind, found, written); + } catch (error) { + this.plannerFailed(acct, refOf(acct, object.id), 'download', error, object.id); + continue; + } + if (work) works.push(work); + else this.markResolved(acct, object.id); + } + for (const id of allGone) { + const found = held.get(refOf(acct, id)); + let work: Work | null; + try { + work = found ? this.removeWork(acct, id, found, 'destroyed') : null; + } catch (error) { + this.plannerFailed(acct, refOf(acct, id), 'download', error, id); + continue; + } + if (work) works.push(work); + else this.markResolved(acct, id); + } + await this.env.writer.write(works, mutate ? this.tail([acct], mutate) : undefined); + await this.readBack(written); + } + + private async heldByRefs(refs: readonly string[]): Promise> { + const out = new Map(); + if (!refs.length) return out; + for (const kind of this.kinds()) { + for (const [ref, local] of await kind.loadByRefs(refs)) if (!out.has(ref)) out.set(ref, { kind, local }); + } + return out; + } + + private async adoptionCandidates(acct: string): Promise { + const out: Held[] = []; + for (const kind of this.kinds()) { + for (const local of await kind.loadNew()) { + const meta = kind.meta(local); + if (meta.pending && accountOfRef(meta.pending.target) === acct) out.push({ kind, local }); + } + } + return out; + } + + /** A new local item whose pending create carries the object's uid: our own create whose identity was never written. */ + private takeAdoptable(candidates: Held[], kind: Kind, acct: string, object: ServerObject): Held | null { + if (typeof object.uid !== 'string') return null; + const index = candidates.findIndex((c) => { + if (c.kind !== kind) return false; + const pending = kind.meta(c.local).pending; + return !!pending && pending.uid === object.uid && this.adoptionTargetMatches(acct, object, pending.target); + }); + return index < 0 ? null : candidates.splice(index, 1)[0]; + } + + private downloadDecision( + acct: string, + object: ServerObject, + kind: Kind | null, + found: Held | null, + written: Map> | null, + ): Work | null { + if (!kind || !this.inSelection(acct, object)) { + return found ? this.removeWork(acct, object.id, found, 'outside') : null; + } + if (found && found.kind !== kind) { + const meta = found.kind.meta(found.local); + if (meta.dirty || meta.deleted) { + this.env.report.itemError({ ref: refOf(acct, object.id), side: 'download', type: 'kindChanged' }); + return null; + } + return this.replaceWork(acct, object, kind, found); + } + return this.downloadWork(kind, acct, object, found ? found.local : null, written); + } + + /** The object merged into its rows (inserted, updated, or adopted); an echo writes nothing but heals the baselines. */ + protected downloadWork( + kind: Kind, + acct: string, + object: ServerObject, + local: unknown, + written: Map> | null, + ): Work | null { + const ref = refOf(acct, object.id); + const plan = kind.planDownload(object, local ?? null, acct); + if (plan.effect === 'none' || !hasWrites(plan.ops.ops)) { + if (!local || kind.meta(local).dirty) return null; + const heal = kind.planBaselineHeal(local); + return heal && hasWrites(heal.ops) ? { group: heal } : null; + } + const group = local ? plan.ops : prependOps(plan.ops, [insertGuard(kind, ref)]); + return { + group, + applied: () => { + if (plan.effect === 'insert') this.env.report.stats.downloaded.created++; + else this.env.report.stats.downloaded.updated++; + this.env.report.conflicts += plan.conflicts; + this.markResolved(acct, object.id); + this.onDownloaded(kind, acct, object); + if (written) { + let refs = written.get(kind); + if (!refs) written.set(kind, (refs = new Set())); + refs.add(ref); + } + }, + replan: async () => { + const current = local ? await this.reload(kind, local) : ((await kind.loadByRefs([ref])).get(ref) ?? null); + try { + return this.downloadWork(kind, acct, object, current, written); + } catch (error) { + this.plannerFailed(acct, ref, 'download', error, object.id); + return null; + } + }, + failed: (reason, message) => { + this.markStale(acct, object.id); + this.env.report.itemError({ ref, side: 'download', type: reason, description: message }); + }, + }; + } + + /** An object that changed kind (a card that became a group): its old rows go, new ones come, atomically. */ + private replaceWork(acct: string, object: ServerObject, kind: Kind, found: Held): Work { + const ref = refOf(acct, object.id); + const plan = kind.planDownload(object, null, acct); + return { + group: concatGroups(ref, found.kind.planLocalDelete(found.local), plan.ops), + applied: () => { + this.env.report.stats.downloaded.updated++; + this.markResolved(acct, object.id); + this.onRemoved(found.kind, acct, object.id); + this.onDownloaded(kind, acct, object); + }, + failed: (reason, message) => { + this.markStale(acct, object.id); + this.env.report.itemError({ ref, side: 'download', type: reason, description: message }); + }, + }; + } + + /** + * Rows of an object the server destroyed (the server delete wins over a + * local edit: a conflict), or that left every synced collection (only + * clean rows go; dirty ones upload first and are dropped afterwards). + */ + protected removeWork(acct: string, id: string, found: Held, reason: Removal): Work | null { + const ref = refOf(acct, id); + const meta = found.kind.meta(found.local); + if (reason === 'outside' && (meta.dirty || meta.deleted || meta.isNew)) { + this.outsideOf(acct).add(ref); + return null; + } + const conflict = reason === 'destroyed' && meta.dirty && !meta.deleted; + return { + group: found.kind.planLocalDelete(found.local), + applied: () => { + this.env.report.stats.downloaded.deleted++; + if (conflict) this.env.report.conflicts++; + this.markResolved(acct, id); + this.onRemoved(found.kind, acct, id); + }, + replan: async () => { + const again = (await found.kind.loadByRefs([ref])).get(ref); + try { + return again === undefined ? null : this.removeWork(acct, id, { kind: found.kind, local: again }, reason); + } catch (error) { + this.plannerFailed(acct, ref, 'download', error, id); + return null; + } + }, + failed: (failure, message) => { + this.markStale(acct, id); + this.env.report.itemError({ ref, side: 'download', type: failure, description: message }); + }, + }; + } + + protected async reload(kind: Kind, local: unknown): Promise { + const meta = kind.meta(local); + if (meta.sourceId) return (await kind.loadByRefs([meta.sourceId])).get(meta.sourceId) ?? null; + const [again] = await kind.loadByRowIds([meta.rowId]); + return again ?? null; + } + + /** Rows as the provider stored them: clean items whose baselines drifted (normalisation) get them rewritten. */ + private async readBack(written: Map>): Promise { + const works: Work[] = []; + for (const [kind, refs] of written) { + for (const local of (await kind.loadByRefs([...refs])).values()) { + if (kind.meta(local).dirty) continue; + const heal = kind.planBaselineHeal(local); + if (heal && hasWrites(heal.ops)) works.push({ group: heal }); + } + } + if (works.length) await this.env.writer.write(works); + } + + // ── Uploads ── + + /** + * Local changes to the server, per phase across the accounts: pairs and + * splits, creates, updates, deletions, then memberships. Creates of every + * account come before deletions of any, so a move between accounts never + * destroys first. A `stateMismatch` downloads the account again and + * repeats the phase, at most three times. + */ + protected async uploadAll(): Promise { + await this.claimNewItems(); + const deletionsAllowed = await this.deletionPolicy(await this.loadUploadItems()); + const accounts = this.accountIds(); + for (const acct of accounts) await this.withRetries(acct, () => this.prePhase(acct)); + for (const phase of ['create', 'update', 'delete'] as const) { + for (const acct of accounts) await this.withRetries(acct, () => this.phase(acct, phase, deletionsAllowed)); + } + for (const acct of accounts) await this.withRetries(acct, () => this.extraPhase(acct, deletionsAllowed)); + for (const acct of accounts) await this.adoptExtraCreated(acct); + } + + private async withRetries(acct: string, run: () => Promise): Promise { + for (let attempt = 0; ; attempt++) { + try { + await run(); + return; + } catch (error) { + if (isMethodError(error) && ACCOUNT_REFUSALS.has(error.type)) { + // The account takes no writes (a shared account turned read-only, or gone): its items wait, the others go on. + this.env.report.itemError({ ref: acct, side: 'upload', type: error.type, description: error.message }); + return; + } + if (!(error instanceof StateMismatch) || error.jmapAccountId !== acct) throw error; + if (attempt >= this.env.tuning.maxStateMismatchRetries) { + throw new RunAbort('io', 'The server kept changing while this device uploaded', { noProgress: true }); + } + await this.catchUp(acct); + } + } + } + + /** + * After a `stateMismatch`: download what changed, then guard the next + * `/set` with the type's current state when nothing changed after the + * stored one. Stalwart can name one point of the history two ways (a + * `/changes` state against the type's state, e.g. `s…` against `n` before + * the first item change), and `ifInState` compares them as text. + */ + private async catchUp(acct: string): Promise { + this.setStates.delete(acct); + await this.changesLoop(acct); + const since = this.env.store.account(acct).itemsState; + if (!since) return; + const current = await this.env.jmap.takeState(this.itemType, acct); + if (!current || current === since) return; + const page = await this.env.jmap.changes(this.itemType, acct, since); + // Anything new since `since` is downloaded by the next attempt; `current` may predate it. + if (!page.created.length && !page.updated.length && !page.destroyed.length && !page.hasMoreChanges) { + this.setStates.set(acct, current); + } + } + + protected ifInState(acct: string): string | null { + if (!this.setStates.has(acct)) this.setStates.set(acct, this.env.store.account(acct).itemsState); + return this.setStates.get(acct) ?? null; + } + + /** New items get their uid and target written before anything is sent. */ + private async claimNewItems(): Promise { + const works: Work[] = []; + for (const held of await this.loadUploadItems()) { + const meta = held.kind.meta(held.local); + if (!meta.isNew || meta.pending || meta.deleted || !this.claimable(held)) continue; + const ref = `row:${meta.rowId}`; + const acct = this.claimAccount(held); + if (!acct) { + this.env.report.stats.skipped++; + this.env.report.itemError({ ref, side: 'upload', type: 'noTargetCollection' }); + continue; + } + let plan; + try { + plan = held.kind.planUpload(held.local, acct); + } catch (error) { + this.plannerFailed(acct, ref, 'upload', error); + continue; + } + if (plan.kind === 'skip') { + this.env.report.stats.skipped++; + this.env.report.itemError({ ref, side: 'upload', type: plan.reason }); + } else if (plan.kind === 'claim' || plan.kind === 'purge' || plan.kind === 'clean') { + works.push(this.planWork(acct, held, plan.kind, plan.ops)); + } + } + await this.env.writer.write(works); + } + + /** + * The deletion threshold: more than 50 local deletions and more than 20 % + * of the synced objects hold every deletion back (`tooManyDeletions`) + * unless the run overrides it; `discardLocalDeletions` restores them. + */ + private async deletionPolicy(items: Held[]): Promise { + const { count, synced, accounts } = await this.countDeletions(items); + if (count === 0) return true; + if (this.env.extras.discardLocalDeletions) { + await this.discardDeletions(items, accounts); + return false; + } + if (this.env.extras.overrideTooManyDeletions) return true; + const t = this.env.tuning; + if (count > t.deletionThresholdCount && count > synced * t.deletionThresholdRatio) { + this.env.report.tooManyDeletions = { + count, + threshold: Math.max(t.deletionThresholdCount, Math.floor(synced * t.deletionThresholdRatio)), + }; + return false; + } + return true; + } + + /** + * "Undo" in the too-many-deletions notification: the deleted rows go and + * the accounts get a reconcile marker, so the next runs download the + * objects again whatever their extras. The marker is stored first: a crash + * in between only asks the question again. + */ + private async discardDeletions(items: Held[], accounts: Set): Promise { + await this.env.writer.write([], this.tail([...accounts], (next) => { + for (const acct of accounts) accountOf(next, acct).reconcile = { from: null, phase: 'ids', position: 0, after: null }; + this.onDiscard(next, accounts); + })); + const works: Work[] = []; + for (const held of items) { + const meta = held.kind.meta(held.local); + if (!meta.deleted || meta.isNew) continue; + const acct = accountOfRef(meta.sourceId); + if (acct) works.push(this.purgeWork(acct, held, false)); + } + await this.env.writer.write(works); + this.env.report.note('local deletions discarded; the objects are downloaded again'); + } + + private async phase(acct: string, phase: 'create' | 'update' | 'delete', deletionsAllowed: boolean): Promise { + const items = (await this.loadUploadItems()).filter((held) => { + if (this.accountOfItem(held) !== acct) return false; + const m = held.kind.meta(held.local); + if (phase === 'create') return m.isNew && !m.deleted && !!m.pending; + if (phase === 'update') return !m.isNew && !m.deleted && m.dirty; + return m.deleted && (deletionsAllowed || m.isNew); + }); + this.noteUploadItems(acct, items); + const size = this.env.tuning.chunkSize; + for (let i = 0; i < items.length; i += size) { + await this.env.checkpoints.check(`upload:${phase}`); + // From a create's /set to its identity write nothing is interruptible; start only with time for it. + if (phase === 'create') this.env.checkpoints.requireBudget(this.env.tuning.createBudgetMs); + const ready = await this.prepare(acct, items.slice(i, i + size)); + const toSend = phase === 'create' ? await this.lookupBeforeCreate(acct, ready) : ready; + if (toSend.length) await this.send(acct, toSend); + } + } + + /** Items an upload phase looked at (contacts: their memberships are uploaded afterwards). */ + protected noteUploadItems(_acct: string, _items: Held[]): void {} + + /** Plans the items' uploads; plans that need no request are applied here. */ + protected async prepare(acct: string, items: Held[]): Promise { + await this.beforeUploadPlanning(items); + const works: Work[] = []; + const ready: Pending[] = []; + for (const held of items) { + const pending = await this.planItem(acct, held, works); + if (pending) ready.push(pending); + } + await this.env.writer.write(works); + return ready; + } + + protected async planItem(acct: string, held: Held, works: Work[]): Promise { + const { kind, local } = held; + const meta = kind.meta(local); + const ref = meta.sourceId ?? `row:${meta.rowId}`; + const report = this.env.report; + report.stats.entries++; + if (this.isStale(acct, meta.sourceId)) { + report.stats.skipped++; + report.itemError({ ref, side: 'upload', type: 'stale', description: 'Waits until the server version could be stored' }); + return null; + } + const fingerprint = kind.fingerprint(local); + if (meta.poison && isBackedOff(meta.poison, fingerprint, this.env.now())) { + report.stats.skipped++; + report.itemError({ ref, side: 'upload', type: meta.poison.type, description: meta.poison.description, retryAt: meta.poison.until }); + return null; + } + let plan; + try { + plan = kind.planUpload(local, acct); + } catch (error) { + this.plannerFailed(acct, ref, 'upload', error); + return null; + } + switch (plan.kind) { + case 'upload': { + const actions = await this.resolveDestroysByUid(acct, meta, plan.actions); + if (actions.length === 0) { + works.push(this.purgeWork(acct, held, false)); + return null; + } + return { held, meta, fingerprint, actions }; + } + case 'revert': + report.stats.skipped++; + works.push(this.planWork(acct, held, 'revert', plan.ops)); + if (plan.refetch) { + const id = idInAccount(meta.sourceId, acct); + if (id) this.markStale(acct, id); + } + return null; + case 'skip': + report.stats.skipped++; + report.itemError({ ref, side: 'upload', type: plan.reason }); + return null; + default: + works.push(this.planWork(acct, held, plan.kind, plan.ops)); + return null; + } + } + + /** A plan applied locally (clean, claim, purge, revert); replanned once more after an assert failure. */ + private planWork(acct: string, held: Held, kind: 'clean' | 'claim' | 'purge' | 'revert', group: OpGroup): Work { + return { + group, + replan: async () => { + const again = await this.reload(held.kind, held.local); + if (again === null) return null; + const plan = held.kind.planUpload(again, acct); + return plan.kind === kind && 'ops' in plan ? this.planWork(acct, { kind: held.kind, local: again }, kind, plan.ops) : null; + }, + }; + } + + /** The rows of an item whose deletion reached the server (or never had to). */ + protected purgeWork(acct: string, held: Held, uploaded: boolean): Work { + const meta = held.kind.meta(held.local); + const id = idInAccount(meta.sourceId, acct); + return { + group: held.kind.planLocalDelete(held.local), + applied: () => { + if (uploaded) this.env.report.stats.uploaded.deleted++; + if (id) this.onRemoved(held.kind, acct, id); + }, + replan: async () => { + const again = await this.reload(held.kind, held.local); + return again === null ? null : this.purgeWork(acct, { kind: held.kind, local: again }, uploaded); + }, + }; + } + + /** A destroy of "whatever carries this uid" (the create's outcome is unknown) becomes destroys of what the lookup finds. */ + private async resolveDestroysByUid(acct: string, meta: ItemMeta, actions: Action[]): Promise { + const out: Action[] = []; + for (const action of actions) { + if (action.kind !== 'destroy' || action.id !== null) { + out.push(action); + continue; + } + if (!action.uid) continue; + const collection = meta.pending ? meta.pending.target.slice(meta.pending.target.indexOf('/') + 1) : ''; + const found = await this.env.jmap.lookupUids(this.itemType, acct, [action.uid], this.lookupScope(acct, collection)); + for (const id of found.get(action.uid) ?? []) out.push({ ...action, id }); + } + return out; + } + + /** The uid lookup before a create: a hit is adopted (the row takes its identity and uploads as an update). */ + private async lookupBeforeCreate(acct: string, ready: Pending[]): Promise { + const scopes = new Map>(); + for (const p of ready) { + for (const a of p.actions) { + if (a.kind !== 'create') continue; + const scope = this.lookupScope(acct, a.collectionId); + let uids = scopes.get(scope); + if (!uids) scopes.set(scope, (uids = new Set())); + uids.add(a.uid); + } + } + const found = new Map(); + const keyOf = (collectionId: string, uid: string) => `${this.lookupScope(acct, collectionId) ?? ''}\u0000${uid}`; + for (const [scope, uids] of scopes) { + const hits = await this.env.jmap.lookupUids(this.itemType, acct, [...uids], scope); + for (const [uid, ids] of hits) found.set(`${scope ?? ''}\u0000${uid}`, ids[0]); + } + if (!found.size) return ready; + const out: Pending[] = []; + const works: Work[] = []; + for (const p of ready) { + const own = p.actions.find((a) => a.kind === 'create' && a.uid === p.meta.pending?.uid); + const ownId = own?.kind === 'create' ? found.get(keyOf(own.collectionId, own.uid)) : undefined; + if (ownId) { + works.push(...(await this.adoptWorks(acct, p, ownId))); + continue; + } + const actions = p.actions.filter((a) => a.kind !== 'create' || !found.has(keyOf(a.collectionId, a.uid))); + if (actions.length) out.push({ ...p, actions }); + } + await this.env.writer.write(works); + return out; + } + + /** + * The row takes the server object's identity and is merged as a dirty + * item (the planner adopts by pending uid). Only an object in the + * create's target collection can be the row's own earlier create: event + * uids are unique per account, and an unrelated event elsewhere with the + * same uid must not receive this row's content (the item is poisoned). + */ + private async adoptWorks(acct: string, p: Pending, id: string): Promise { + const { list } = await this.env.jmap.get(this.itemType, acct, [id], this.itemProperties); + const object = list[0]; + if (!object) return []; + const target = p.meta.pending?.target; + if (!target || !this.adoptionTargetMatches(acct, object, target)) { + return [this.poisonWork(p, { type: 'uidConflict', description: `Another object (${refOf(acct, id)}) has this uid` })]; + } + await this.beforePlanning(acct, [object]); + const work = this.downloadWork(p.held.kind, acct, object, p.held.local, null); + return work ? [work] : []; + } + + /** Sends the items' actions in `/set` calls and settles each item once all its actions have an answer. */ + protected async send(acct: string, pending: Pending[]): Promise { + const calls = this.packCalls(pending); + const objects = new Map(); + const outcomes = new Map(); + const remaining = new Map(pending.map((p) => [p, p.actions.length])); + const retry: Pending[] = []; + for (const call of calls) { + const response = await this.setCall(acct, call, objects); + const settled: Pending[] = []; + for (const entry of call.entries) { + const outcome = this.outcomeOf(response, entry.action, entry.key, call.args.sendSchedulingMessages); + // No answer for it at all: left as it is, the next run looks again. + if (!outcome) continue; + let list = outcomes.get(entry.pending); + if (!list) outcomes.set(entry.pending, (list = [])); + list.push(outcome); + const left = (remaining.get(entry.pending) ?? 1) - 1; + remaining.set(entry.pending, left); + if (left === 0) settled.push(entry.pending); + } + // Identities of this call's creates are written before anything else runs. + const works: Work[] = []; + for (const p of settled) works.push(...(await this.settle(acct, p, outcomes.get(p) ?? [], objects, retry))); + await this.env.writer.write(works); + } + if (retry.length) await this.send(acct, retry); + } + + private packCalls(pending: Pending[]): SetCallPlan[] { + const limits = this.env.jmap.limits(); + const maxObjects = Math.max(1, limits.maxObjectsInSet); + const maxCreates = Math.max(1, Math.min(this.env.tuning.maxCreatesPerSet, maxObjects)); + const maxBytes = Math.min(this.env.tuning.maxSetBytes, Math.max(1, limits.maxSizeRequest - 10_000)); + const calls: SetCallPlan[] = []; + const open = new Map(); + let creation = 0; + const fresh = (scheduling: boolean): SetCallPlan => ({ + args: { create: {}, update: {}, destroy: [], sendSchedulingMessages: scheduling }, + entries: [], + touched: new Set(), + updateIds: [], + creates: 0, + bytes: 0, + }); + for (const p of pending) { + for (const action of p.actions) { + // `sendSchedulingMessages` is a request-level flag: one value per call. + const scheduling = !p.noScheduling && !!action.sendSchedulingMessages; + const size = JSON.stringify(action).length; + const id = action.kind === 'create' ? null : action.id; + let call = open.get(scheduling); + const fits = + !!call && + call.entries.length + 1 <= maxObjects && + (action.kind !== 'create' || call.creates + 1 <= maxCreates) && + call.bytes + size <= maxBytes && + (id === null || !call.touched.has(id)); + if (!call || !fits) { + call = fresh(scheduling); + calls.push(call); + open.set(scheduling, call); + } + let key: string; + if (action.kind === 'create') { + key = `k${++creation}`; + call.args.create[key] = this.createObject(action); + call.creates++; + } else if (action.kind === 'update') { + key = action.id; + call.args.update[key] = action.patch; + call.updateIds.push(key); + } else { + key = action.id as string; + call.args.destroy.push(key); + } + call.touched.add(key); + call.bytes += size; + call.entries.push({ pending: p, action, key }); + } + } + return calls; + } + + private createObject(action: Extract): Record { + const object: Record = { ...(action.object as Record), uid: action.uid }; + if (!object[this.parentProperty]) object[this.parentProperty] = { [action.collectionId]: true }; + return object; + } + + /** + * One `/set` of the authority's type, guarded by `ifInState`; our + * `newState` guards the next one. `stateMismatch` (and Stalwart's + * `serverUnavailable`, a concurrent write that failed the whole call) + * throws StateMismatch: the phase downloads again and repeats. + */ + protected async setChecked( + acct: string, + args: SetArgs, + thenGet?: { ids: string[]; properties: readonly string[] }, + ): Promise<{ response: SetResponse; got: ServerObject[] | null }> { + let result; + try { + result = await this.env.jmap.set(this.itemType, acct, args, this.ifInState(acct), thenGet); + } catch (error) { + if (isMethodError(error, 'stateMismatch') || isMethodError(error, 'serverUnavailable')) throw new StateMismatch(acct); + throw error; + } + if (result.response.newState) { + this.setStates.set(acct, result.response.newState); + this.env.recordKnownState(acct, this.itemType, result.response.newState); + } + return result; + } + + private async setCall(acct: string, call: SetCallPlan, objects: Map): Promise { + const { response, got } = await this.setChecked( + acct, + { + create: call.args.create, + update: call.args.update, + destroy: call.args.destroy, + sendSchedulingMessages: call.args.sendSchedulingMessages, + }, + call.updateIds.length ? { ids: call.updateIds, properties: this.itemProperties } : undefined, + ); + for (const object of got ?? []) objects.set(object.id, object); + // Stalwart resolves no result reference to a /set: created objects are fetched in a request of their own. + const createdIds = Object.values(response.created ?? {}).map((c) => c?.id).filter((id): id is string => !!id); + if (createdIds.length) { + const { list } = await this.env.jmap.get(this.itemType, acct, createdIds, this.itemProperties); + for (const object of list) objects.set(object.id, object); + } + return response; + } + + private outcomeOf(response: SetResponse, action: Action, key: string, scheduling: boolean): Outcome | null { + if (action.kind === 'create') { + const created = response.created?.[key]; + if (created?.id) return { action, ok: true, id: created.id }; + const error = response.notCreated?.[key]; + return error ? { action, ok: false, error, scheduling } : null; + } + if (action.kind === 'update') { + if (response.updated && key in response.updated) return { action, ok: true, id: key }; + const error = response.notUpdated?.[key]; + return error ? { action, ok: false, error, scheduling } : null; + } + if (response.destroyed?.includes(key)) return { action, ok: true, id: key }; + const error = response.notDestroyed?.[key]; + return error ? { action, ok: false, error, scheduling } : null; + } + + /** What the server's answers mean for one item (docs/device-sync.md, "Uploads", SetErrors). */ + private async settle( + acct: string, + p: Pending, + outcomes: Outcome[], + objects: Map, + retry: Pending[], + ): Promise { + const failure = outcomes.find((o): o is Extract => !o.ok); + if (!failure) return this.succeeded(acct, p, outcomes, objects); + switch (classifySetError(failure.error, failure.action.kind)) { + case 'notFound': + if (failure.action.kind === 'destroy') { + // Gone already: the deletion is done. + const done = outcomes.map((o): Outcome => (o === failure ? { action: o.action, ok: true, id: (o.action as { id: string }).id } : o)); + return this.succeeded(acct, p, done, objects); + } + if (failure.action.kind === 'update') { + // Deleted on the server meanwhile: the server delete wins, the rows go. + const id = idInAccount(p.meta.sourceId, acct); + const work = id ? this.removeWork(acct, id, p.held, 'destroyed') : null; + return work ? [work] : []; + } + return [this.poisonWork(p, failure.error)]; + case 'forbidden': + if (failure.scheduling && !p.noScheduling) { + // Scheduling is off, the account has no calendar address, or no permission: store the change without it. + this.env.report.note('invitations not sent'); + retry.push({ ...p, noScheduling: true }); + return []; + } + if (failure.action.kind === 'create') return [this.poisonWork(p, failure.error)]; + return this.revertWorks(acct, p, failure.error); + case 'uidExists': + return this.adoptAfterUidError(acct, p, failure); + default: + return [this.poisonWork(p, failure.error)]; + } + } + + private async succeeded(acct: string, p: Pending, outcomes: Outcome[], objects: Map): Promise { + const ownUid = p.meta.pending?.uid; + let ownId = idInAccount(p.meta.sourceId, acct); + let created = false; + for (const o of outcomes) { + if (!o.ok || o.action.kind !== 'create') continue; + if (!ownId && o.action.uid === ownUid) { + ownId = o.id; + created = true; + } else { + let set = this.extraCreated.get(acct); + if (!set) this.extraCreated.set(acct, (set = new Set())); + set.add(o.id); + } + } + if (p.accept) return p.accept(objects); + const server = ownId ? objects.get(ownId) : undefined; + if (!p.meta.deleted && !server) { + // Accepted, but its new version could not be read: fetched again next run, nothing uploads for it meanwhile. + if (ownId) this.markStale(acct, ownId); + return []; + } + try { + return [p.meta.deleted ? this.purgeWork(acct, p.held, true) : this.acceptWork(acct, p, server as ServerObject, created)]; + } catch (error) { + // Uploaded, but the rows could not be planned: the shadow is behind, fetch it again. + this.plannerFailed(acct, p.meta.sourceId ?? `row:${p.meta.rowId}`, 'upload', error, ownId ?? undefined); + return []; + } + } + + /** + * The accepted version becomes the shadow and the baselines, DIRTY is + * cleared, behind the assert the upload was read under; edited again + * meanwhile, only identity, shadow and the uploaded baselines are written + * and DIRTY stays (the newer edit uploads next). + */ + protected acceptWork(acct: string, p: Pending, server: ServerObject, created: boolean): Work { + const { kind, local } = p.held; + const plan = kind.planAccepted(local, server, acct); + const extra = p.meta.poison ? [poisonOp(kind, p.meta.rowId, null)] : []; + const withExtra = (group: OpGroup): OpGroup => (extra.length ? { ref: group.ref, ops: [...group.ops, ...extra] } : group); + const counted = () => { + if (created) this.env.report.stats.uploaded.created++; + else this.env.report.stats.uploaded.updated++; + this.onAccepted(kind, acct, server); + }; + const failed = (reason: string, message: string) => { + this.markStale(acct, server.id); + this.env.report.itemError({ ref: refOf(acct, server.id), side: 'upload', type: reason, description: message }); + }; + return { + group: withExtra(plan.ops), + applied: counted, + replan: async () => ({ group: withExtra(plan.keepDirtyOps), applied: counted, failed }), + failed, + }; + } + + private poisonWork(p: Pending, error: SetErrorShape): Work { + const marker = nextMarker(p.meta.poison, p.fingerprint, error, this.env.now(), { + firstMs: this.env.tuning.poisonBackoffMs, + maxMs: this.env.tuning.poisonBackoffMaxMs, + }); + const ref = p.meta.sourceId ?? `row:${p.meta.rowId}`; + this.env.report.stats.skipped++; + this.env.report.itemError({ ref, side: 'upload', type: error.type, description: error.description, retryAt: marker.until }); + return { group: { ref, ops: [poisonOp(p.held.kind, p.meta.rowId, marker)] } }; + } + + /** Not allowed after all (read-only on the server): the rows are rewritten from the server's version. */ + private async revertWorks(acct: string, p: Pending, error: SetErrorShape): Promise { + const { kind, local } = p.held; + const ref = p.meta.sourceId ?? `row:${p.meta.rowId}`; + this.env.report.stats.skipped++; + this.env.report.itemError({ ref, side: 'upload', type: error.type, description: error.description }); + const id = idInAccount(p.meta.sourceId, acct); + const remove = kind.planLocalDelete(local); + const object = id ? (await this.env.jmap.get(this.itemType, acct, [id], this.itemProperties)).list[0] : undefined; + const objectKind = object ? this.kindOf(object) : null; + if (!object || !objectKind || !this.inSelection(acct, object)) { + return [{ group: remove, applied: () => (id ? this.onRemoved(kind, acct, id) : undefined) }]; + } + await this.beforePlanning(acct, [object]); + const insert = objectKind.planDownload(object, null, acct); + return [{ group: concatGroups(ref, remove, insert.ops), applied: () => this.onDownloaded(objectKind, acct, object) }]; + } + + /** + * `invalidProperties` on `uid`: the object exists (a lost response, or the + * client's automatic retry of the create). Cards name it in the error; + * events are looked up again with back-off (the uid index lags), else the + * next download adopts the row by uid. + */ + private async adoptAfterUidError(acct: string, p: Pending, failure: Extract): Promise { + const action = failure.action as Extract; + if (action.uid !== p.meta.pending?.uid) return []; + let id = existingIdFromUidError(failure.error); + if (!id) { + const scope = this.lookupScope(acct, action.collectionId); + for (const delay of [0, ...this.env.tuning.uidLookupDelaysMs]) { + if (delay) await this.env.sleep(delay); + const ids = (await this.env.jmap.lookupUids(this.itemType, acct, [action.uid], scope)).get(action.uid); + if (ids?.length) { + id = ids[0]; + break; + } + } + } + return id ? this.adoptWorks(acct, p, id) : []; + } + + /** Objects our uploads created for other rows come back through the download, which adopts them by uid. */ + private async adoptExtraCreated(acct: string): Promise { + const ids = [...(this.extraCreated.get(acct) ?? [])]; + this.extraCreated.delete(acct); + if (ids.length) await this.processIds(acct, ids, []); + } +} diff --git a/src/device-sync/engine/kinds.ts b/src/device-sync/engine/kinds.ts new file mode 100644 index 00000000..7ef6538f --- /dev/null +++ b/src/device-sync/engine/kinds.ts @@ -0,0 +1,95 @@ +/** + * The engine's view of one kind of local item (a contact, a group, an event + * master with its exceptions): what it needs to know about an item and the + * planner calls it makes for it. The contacts and calendar sync plug their + * planners in through this, so downloads, uploads, retries and the guards + * are written once (engine/item-sync.ts). + */ +import type { AcceptedPlan, DownloadPlan, OpGroup, PendingCreate, PoisonMarker, UploadPlan } from '../planner'; +import type { ProviderOp } from '../types'; + +export type KindName = 'contact' | 'group' | 'event'; + +/** A server object as `/get` returns it. */ +export interface ServerObject { + id: string; + uid?: string; + [property: string]: unknown; +} + +export interface ItemMeta { + rowId: number; + /** `/` of the server object; null until created (events: a pending marker is no identity). */ + sourceId: string | null; + pending: PendingCreate | null; + /** Local changes wait for an upload (events: the master or one of its exceptions). */ + dirty: boolean; + deleted: boolean; + /** Never reached the server: no identity (docs/device-sync.md, "Identity"). */ + isNew: boolean; + poison: PoisonMarker | null; + /** Collection keys the server object was in when last seen. */ + collections: string[]; +} + +/** A decoded local item and its kind. */ +export interface Held { + kind: Kind; + local: unknown; +} + +export interface Kind { + readonly name: KindName; + readonly table: 'raw_contacts' | 'groups' | 'events'; + /** The column holding the identity, for the "not inserted yet" guard of a download insert. */ + readonly identityColumn: 'sourceid' | '_sync_id'; + readonly poisonColumn: 'sync4' | 'sync_data5'; + meta(local: L): ItemMeta; + /** What an upload is computed from, for poison markers. */ + fingerprint(local: L): string; + /** Items by identity, one per ref. */ + loadByRefs(refs: readonly string[]): Promise>; + loadByRowIds(rowIds: readonly number[]): Promise; + /** Items without identity (new ones), for adoption by pending uid. */ + loadNew(): Promise; + planDownload(object: O, local: L | null, jmapAccountId: string): DownloadPlan; + planLocalDelete(local: L): OpGroup; + planBaselineHeal(local: L): OpGroup | null; + planUpload(local: L, jmapAccountId: string): UploadPlan; + planAccepted(local: L, server: O, jmapAccountId: string): AcceptedPlan; +} + +export function refOf(jmapAccountId: string, id: string): string { + return `${jmapAccountId}/${id}`; +} + +/** The id part of a `/` ref of `jmapAccountId`, else null. */ +export function idInAccount(ref: string | null, jmapAccountId: string): string | null { + if (!ref) return null; + const prefix = `${jmapAccountId}/`; + return ref.startsWith(prefix) && !ref.includes('#') ? ref.slice(prefix.length) : null; +} + +export function accountOfRef(ref: string | null): string | null { + if (!ref) return null; + const slash = ref.indexOf('/'); + return slash > 0 ? ref.slice(0, slash) : null; +} + +/** + * A download insert first asserts that no row has the identity yet: when a + * failed batch was committed up to a yield point, the retry must not insert + * the item twice. + */ +export function insertGuard(kind: Kind, ref: string): ProviderOp { + return { op: 'assert', table: kind.table, where: `${kind.identityColumn} = ?`, args: [ref], expectCount: 0 }; +} + +export function poisonOp(kind: Kind, rowId: number, marker: PoisonMarker | null): ProviderOp { + return { + op: 'update', + table: kind.table, + id: rowId, + values: { [kind.poisonColumn]: marker ? JSON.stringify(marker) : null }, + }; +} diff --git a/src/device-sync/engine/mutex.ts b/src/device-sync/engine/mutex.ts new file mode 100644 index 00000000..405a19cd --- /dev/null +++ b/src/device-sync/engine/mutex.ts @@ -0,0 +1,70 @@ +/** + * One sync run or app-side operation (turning sync off, sign-out) at a time + * per registry account and authority. Module level on purpose: a headless + * run and the UI share one JS runtime (docs/device-sync.md, "Architecture"), + * so a module-level lock covers both. + */ +import type { Authority } from '../types'; + +const tails = new Map>(); +const stopRequests = new Map(); + +export function lockKey(registryId: string, authority: Authority): string { + return `${registryId}\u0000${authority}`; +} + +/** + * Waits for the lock; resolves with its release function, or null when + * `waitMs` passed first (the place in the queue is then handed on). + */ +export function acquireLock(key: string, waitMs = Infinity): Promise<(() => void) | null> { + const previous = tails.get(key) ?? Promise.resolve(); + let pass!: () => void; + const mine = new Promise((resolve) => { + pass = resolve; + }); + const tail = previous.then(() => mine); + tails.set(key, tail); + void tail.then(() => { + if (tails.get(key) === tail) tails.delete(key); + }); + return new Promise((resolve) => { + let settled = false; + const timer = Number.isFinite(waitMs) + ? setTimeout(() => { + if (settled) return; + settled = true; + resolve(null); + void previous.then(pass); + }, Math.max(0, waitMs)) + : null; + void previous.then(() => { + if (settled) return; + settled = true; + if (timer) clearTimeout(timer); + let released = false; + resolve(() => { + if (released) return; + released = true; + pass(); + }); + }); + }); +} + +export function isLocked(key: string): boolean { + return tails.has(key); +} + +/** Asks the run holding the lock to stop at its next checkpoint (it reports `cancelled`). */ +export function requestStop(key: string): void { + stopRequests.set(key, (stopRequests.get(key) ?? 0) + 1); +} + +export function clearStopRequest(key: string): void { + stopRequests.delete(key); +} + +export function stopRequested(key: string): boolean { + return (stopRequests.get(key) ?? 0) > 0; +} diff --git a/src/device-sync/engine/poison.ts b/src/device-sync/engine/poison.ts new file mode 100644 index 00000000..cc1ffecc --- /dev/null +++ b/src/device-sync/engine/poison.ts @@ -0,0 +1,34 @@ +/** + * Poison markers (docs/device-sync.md, "Uploads", SetErrors): an item the + * server refused (`invalidProperties`, `invalidPatch`, `tooLarge`, + * `overQuota`, …) is not retried until its back-off ends (1 h, doubling to + * a day) or it changes. The engine writes the marker (SYNC4 / SYNC_DATA5) + * and checks it before planning an upload; the fingerprint covers what the + * item would upload from: its mapped rows, its flags and its shadow. + */ +import { jsonHash } from '../common/json'; +import type { PoisonMarker } from '../planner'; + +export function poisonFingerprint(content: unknown): string { + return jsonHash(content); +} + +/** True while the marker holds: same fingerprint and the back-off not over. */ +export function isBackedOff(marker: PoisonMarker | null, fingerprint: string, now: number): boolean { + return !!marker && marker.fp === fingerprint && marker.until > now; +} + +/** The next marker after a refusal: attempts count up while the item stays the same. */ +export function nextMarker( + previous: PoisonMarker | null, + fingerprint: string, + error: { type: string; description?: string }, + now: number, + backoff: { firstMs: number; maxMs: number }, +): PoisonMarker { + const n = previous && previous.fp === fingerprint ? previous.n + 1 : 1; + const delay = Math.min(backoff.firstMs * 2 ** (n - 1), backoff.maxMs); + const marker: PoisonMarker = { fp: fingerprint, type: error.type, n, until: now + delay }; + if (error.description) marker.description = error.description.slice(0, 500); + return marker; +} diff --git a/src/device-sync/engine/provider.ts b/src/device-sync/engine/provider.ts new file mode 100644 index 00000000..9fc7370b --- /dev/null +++ b/src/device-sync/engine/provider.ts @@ -0,0 +1,77 @@ +/** + * Reads through the provider port as row objects. Every query is scoped to + * our account by the native side; the SQL stays within the subset the fake + * provider evaluates (src/device-sync/__tests__/fakes/fake-sql.ts). + */ +import type { ProviderOp, ProviderPort, ProviderTable, Row } from '../types'; +import { RunAbort, isPermissionFailure } from './errors'; + +/** Host parameters per `IN (…)`: well under SQLite's 999. */ +const IN_CHUNK = 400; + +export class ProviderReader { + constructor(readonly port: ProviderPort) {} + + async rows(table: ProviderTable, columns: readonly string[], where?: string, args?: Array): Promise { + let result; + try { + result = await this.port.query({ table, columns: [...columns], ...(where ? { where, args: args ?? [] } : {}) }); + } catch (error) { + if (isPermissionFailure(error)) throw new RunAbort('permission', (error as Error).message); + throw error; + } + const { columns: names, rows } = result; + return rows.map((cells) => { + const row: Row = {}; + names.forEach((name, i) => { + row[name] = cells[i] ?? null; + }); + return row; + }); + } + + /** Rows whose `column` is one of `values` (in chunks), optionally narrowed by `extra`. */ + async rowsIn( + table: ProviderTable, + columns: readonly string[], + column: string, + values: ReadonlyArray, + extra?: { where: string; args?: Array }, + ): Promise { + const out: Row[] = []; + const unique = [...new Set(values)]; + for (let i = 0; i < unique.length; i += IN_CHUNK) { + const part = unique.slice(i, i + IN_CHUNK); + const inClause = `${column} IN (${part.map(() => '?').join(', ')})`; + const where = extra ? `(${inClause}) AND (${extra.where})` : inClause; + out.push(...(await this.rows(table, columns, where, [...part, ...(extra?.args ?? [])]))); + } + return out; + } + + async readSyncState(): Promise { + try { + return await this.port.readSyncState(); + } catch (error) { + if (isPermissionFailure(error)) throw new RunAbort('permission', (error as Error).message); + throw error; + } + } +} + +export function num(value: unknown): number { + return typeof value === 'number' ? value : Number(value ?? 0); +} + +export function flag(value: unknown): boolean { + return value !== null && value !== undefined && value !== '' && Number(value) !== 0; +} + +export function str(value: unknown): string | null { + return typeof value === 'string' && value ? value : value === null || value === undefined || value === '' ? null : String(value); +} + +/** True when an op group writes something (asserts alone change nothing). */ +export function hasWrites(ops: readonly ProviderOp[]): boolean { + return ops.some((op) => op.op !== 'assert'); +} diff --git a/src/device-sync/engine/report.ts b/src/device-sync/engine/report.ts new file mode 100644 index 00000000..6f810607 --- /dev/null +++ b/src/device-sync/engine/report.ts @@ -0,0 +1,95 @@ +/** + * The run report (`RunReport`, docs/device-sync.md "Headless payload, report + * and SyncResult"). Counts are objects actually written or sent, so a run + * that repeated a page without writing anything is no progress to + * SyncManager. + */ +import type { Authority, ItemError, RunOutcome, RunReport, RunStats, RunStatus } from '../types'; + +export function emptyStats(): RunStats { + return { + downloaded: { created: 0, updated: 0, deleted: 0 }, + uploaded: { created: 0, updated: 0, deleted: 0 }, + entries: 0, + skipped: 0, + }; +} + +export class ReportBuilder { + readonly stats = emptyStats(); + conflicts = 0; + private readonly errors: ItemError[] = []; + private errorCount = 0; + private readonly notes = new Set(); + tooManyDeletions?: { count: number; threshold: number }; + + constructor( + readonly runId: string, + readonly authority: Authority, + readonly startedAt: number, + private readonly maxItemErrors = 50, + ) {} + + itemError(error: ItemError): void { + this.errorCount++; + // One entry per item and side: a retried item is reported once, with its latest error. + const existing = this.errors.findIndex((e) => e.ref === error.ref && e.side === error.side); + if (existing >= 0) { + this.errors[existing] = error; + this.errorCount--; + return; + } + if (this.errors.length < this.maxItemErrors) this.errors.push(error); + } + + /** A note for the report's message ("invitations not sent", …). */ + note(text: string): void { + this.notes.add(text); + } + + get itemErrorCount(): number { + return this.errorCount; + } + + build( + outcome: RunOutcome, + now: number, + extra: { message?: string; delayUntil?: number; moreRecordsToGet?: boolean; noProgress?: boolean } = {}, + ): RunReport { + const stats: RunStats = JSON.parse(JSON.stringify(this.stats)); + if (extra.noProgress) { + stats.downloaded = { created: 0, updated: 0, deleted: 0 }; + stats.uploaded = { created: 0, updated: 0, deleted: 0 }; + } + const message = [extra.message, ...this.notes].filter(Boolean).join('; '); + const report: RunReport = { + v: 1, + runId: this.runId, + authority: this.authority, + outcome, + startedAt: this.startedAt, + durationMs: Math.max(0, now - this.startedAt), + stats, + conflicts: this.conflicts, + itemErrors: this.errors.slice(), + }; + if (message) report.message = message; + if (outcome === 'tooManyDeletions' && this.tooManyDeletions) report.tooManyDeletions = this.tooManyDeletions; + if (extra.delayUntil !== undefined) report.delayUntil = extra.delayUntil; + if (extra.moreRecordsToGet) report.moreRecordsToGet = true; + return report; + } +} + +export function statusOf(report: RunReport, itemErrorCount = report.itemErrors.length): RunStatus { + const status: RunStatus = { + at: report.startedAt + report.durationMs, + outcome: report.outcome, + durationMs: report.durationMs, + conflicts: report.conflicts, + itemErrors: itemErrorCount, + stats: report.stats, + }; + if (report.message) status.message = report.message; + return status; +} diff --git a/src/device-sync/engine/run.ts b/src/device-sync/engine/run.ts new file mode 100644 index 00000000..d037c332 --- /dev/null +++ b/src/device-sync/engine/run.ts @@ -0,0 +1,232 @@ +/** + * One sync run of one Android account and authority (docs/device-sync.md, + * "A sync run"): preflight, then the authority's phases, then the report. + * Never throws: every failure becomes an outcome. + */ +import { makeKeyMinter, uuidFrom } from '../common/ids'; +import { JmapCaller } from '../jmap/caller'; +import type { JmapConnection } from '../jmap/connection'; +import { accountsWithCapability } from '../jmap/session'; +import { + CALENDAR_AUTHORITY, + CONTACTS_AUTHORITY, + JMAP_CALENDARS, + JMAP_CONTACTS, + type Authority, + type RunExtras, + type RunPayload, + type RunReport, +} from '../types'; +import { BatchWriter } from './batch'; +import { CalendarSync } from './calendar-sync'; +import { Checkpoints } from './checkpoint'; +import { ContactsSync } from './contacts-sync'; +import type { RunEnv } from './context'; +import { DEFAULT_TUNING, type EngineDeps, type Tuning } from './deps'; +import { classifyFailure, RunAbort } from './errors'; +import { acquireLock, lockKey, stopRequested } from './mutex'; +import { ProviderReader, str } from './provider'; +import { ReportBuilder, statusOf } from './report'; +import { emptySyncState, parseSyncState, StateStore, type SyncState } from './sync-state'; +import type { ItemSync } from './item-sync'; + +export function tuningOf(deps: EngineDeps): Tuning { + return { ...DEFAULT_TUNING, ...deps.tuning }; +} + +const defaultYield = () => new Promise((resolve) => setTimeout(resolve, 0)); +const defaultSleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms)); + +function capabilityOf(authority: Authority): string { + return authority === CONTACTS_AUTHORITY ? JMAP_CONTACTS : JMAP_CALENDARS; +} + +function itemTypeOf(authority: Authority): string { + return authority === CONTACTS_AUTHORITY ? 'ContactCard' : 'CalendarEvent'; +} + +/** Rows waiting for an upload (dirty, deleted, new), or state work a run must do whatever its extras. */ +export async function hasLocalWork(reader: ProviderReader, authority: Authority, state: SyncState): Promise { + for (const account of Object.values(state.accounts)) { + if (account.reconcile || account.stale.length) return true; + } + if (authority === CALENDAR_AUTHORITY) { + const rows = await reader.rows('events', ['_id'], "dirty = 1 OR deleted = 1 OR _sync_id IS NULL OR _sync_id LIKE '~pending/%'"); + return rows.length > 0; + } + const contacts = await reader.rows('raw_contacts', ['_id'], 'dirty = 1 OR deleted = 1 OR sourceid IS NULL'); + if (contacts.length) return true; + const groups = await reader.rows('groups', ['_id', 'sourceid', 'dirty', 'deleted']); + const present = new Set(groups.map((g) => str(g.sourceid)).filter(Boolean)); + if (groups.some((g) => !str(g.sourceid) || Number(g.dirty) === 1 || Number(g.deleted) === 1)) return true; + // A group an app hard-deleted is only found by its absence. + return Object.values(state.accounts).some((a) => (a.groups ?? []).some((ref) => !present.has(ref))); +} + +export interface EnvOptions { + deps: EngineDeps; + tuning: Tuning; + mode: 'sync' | 'teardown'; + registryId: string; + accountName: string; + authority: Authority; + extras: RunExtras; + deadline: number; + isCancelled(): Promise; + reader: ProviderReader; + report: ReportBuilder; + stored: SyncState; + connection: JmapConnection; +} + +/** + * The run's environment once the preflight passed: the JMAP connection is + * open and its accounts known. Throws RunAbort('unsupported') when no + * account offers the authority's capability. + */ +export async function createEnv(options: EnvOptions): Promise<{ env: RunEnv; sync: ItemSync }> { + const { deps, tuning, registryId, authority, connection } = options; + const jmap = new JmapCaller(connection.port, () => deps.now()); + const accounts = accountsWithCapability(jmap.session(), capabilityOf(authority)); + if (!accounts.length) { + throw new RunAbort('unsupported', `The server offers no ${authority === CONTACTS_AUTHORITY ? 'contacts' : 'calendars'} for this account`); + } + const prefs = await deps.prefs(registryId); + const recordKnownState = (jmapAccountId: string, type: string, state: string) => { + try { + void Promise.resolve(deps.recordKnownState(registryId, jmapAccountId, type, state)).catch(() => undefined); + } catch { + // Only an echo hint for the app. + } + }; + const store = new StateStore(options.stored, { registryId, origin: connection.origin }, (before, after) => { + for (const [acct, account] of Object.entries(after.accounts)) { + const state = account.itemsState; + if (state && state !== before.accounts[acct]?.itemsState) recordKnownState(acct, itemTypeOf(authority), state); + } + }); + const checkpoints = new Checkpoints({ + now: () => deps.now(), + deadline: options.deadline, + marginMs: tuning.deadlineMarginMs, + echoAfterMs: tuning.echoAfterMs, + isCancelled: options.isCancelled, + yieldThread: deps.yieldThread ?? defaultYield, + }); + checkpoints.attach(jmap); + const mintKey = makeKeyMinter(deps.random); + const env: RunEnv = { + deps, + tuning, + mode: options.mode, + registryId, + accountName: options.accountName, + authority, + extras: options.extras, + reader: options.reader, + writer: new BatchWriter(options.reader.port, { + maxOps: tuning.maxBatchOps, + maxBytes: tuning.maxBatchBytes, + maxReplans: tuning.maxReplans, + }), + report: options.report, + checkpoints, + store, + jmap, + accounts, + prefs, + now: () => deps.now(), + mintKey: (taken) => mintKey(taken), + mintUid: () => uuidFrom(deps.random), + sleep: deps.sleep ?? defaultSleep, + log: (message, detail) => deps.log?.(message, detail), + recordKnownState, + }; + const sync = authority === CONTACTS_AUTHORITY ? new ContactsSync(env) : new CalendarSync(env); + return { env, sync }; +} + +/** + * Runs one sync and returns its report. Holds the (registry account, + * authority) lock for the whole run; a run that cannot get it before its + * deadline reports `cancelled` and asks for another sync. + */ +export async function runDeviceSync(payload: RunPayload, deps: EngineDeps): Promise { + const tuning = tuningOf(deps); + const report = new ReportBuilder(payload.runId, payload.authority, deps.now(), tuning.maxItemErrors); + const key = lockKey(payload.registryId, payload.authority); + const release = await acquireLock(key, Math.max(0, payload.deadline - deps.now() - tuning.deadlineMarginMs)); + let result: RunReport; + if (!release) { + result = report.build('cancelled', deps.now(), { message: 'Another sync of this account is still running', moreRecordsToGet: true }); + } else { + try { + result = await runLocked(payload, deps, tuning, report, key); + } finally { + release(); + } + } + try { + await deps.recordStatus(payload.registryId, payload.authority, statusOf(result, report.itemErrorCount)); + } catch (error) { + deps.log?.('recording the run status failed', error); + } + return result; +} + +async function runLocked(payload: RunPayload, deps: EngineDeps, tuning: Tuning, report: ReportBuilder, key: string): Promise { + const finish = (outcome: RunReport['outcome'], extra: Parameters[2] = {}) => + report.build(outcome, deps.now(), extra); + const extras = payload.extras ?? {}; + try { + if (!(await deps.isSyncEnabled(payload.accountName, payload.authority))) { + return finish('disabled', { message: 'Sync is off for this account' }); + } + const reader = new ProviderReader(deps.provider(payload.accountName, payload.authority)); + const { state: stored, readable } = parseSyncState(await reader.readSyncState()); + if (!readable) deps.log?.('unreadable SyncState: starting over with a full reconcile'); + if (stored.owner && stored.owner.registryId !== payload.registryId) { + return finish('internal', { message: 'The rows on this device were written for another account' }); + } + // Providers schedule an upload sync for every account 30 s after any app write. + if (extras.upload && !(await hasLocalWork(reader, payload.authority, stored))) return finish('ok'); + + const connection = await deps.jmap(payload.registryId); + if (stored.owner && stored.owner.origin !== connection.origin) { + return finish('internal', { message: 'The rows on this device were written for another server' }); + } + const { sync } = await createEnv({ + deps, + tuning, + mode: 'sync', + registryId: payload.registryId, + accountName: payload.accountName, + authority: payload.authority, + extras, + deadline: payload.deadline, + isCancelled: async () => stopRequested(key) || (await deps.isCancelled(payload.runId)), + reader, + report, + stored: readable ? stored : emptySyncState(), + connection, + }); + await sync.sync(); + return finish(report.tooManyDeletions ? 'tooManyDeletions' : 'ok'); + } catch (error) { + const failure = classifyFailure(error, deps.now()); + if (failure.authProblem) { + try { + await deps.notifyAuthProblem(payload.registryId, payload.accountName, payload.authority); + } catch (notifyError) { + deps.log?.('auth notification failed', notifyError); + } + } + if (failure.outcome === 'internal') deps.log?.('device sync failed', error); + return finish(failure.outcome, { + message: failure.message, + delayUntil: failure.delayUntil, + moreRecordsToGet: failure.moreRecordsToGet, + noProgress: failure.noProgress, + }); + } +} diff --git a/src/device-sync/engine/selection.ts b/src/device-sync/engine/selection.ts new file mode 100644 index 00000000..0072086c --- /dev/null +++ b/src/device-sync/engine/selection.ts @@ -0,0 +1,36 @@ +/** + * Which collections sync. The store keeps explicit choices only; a + * collection the user never touched follows the default: the personal + * account's collections on, shared accounts' off (docs/device-sync.md, + * "App integration"), except the webmail's allow-list address book. + */ +import { collectionKey } from '../common/ids'; +import { CONTACTS_AUTHORITY, type Authority } from '../types'; + +/** + * The address book the webmail keeps its allow-listed senders in. Mirrors + * TRUSTED_SENDERS_BOOK_NAME in src/stores/contacts-store.ts, which the pure + * engine cannot import. + */ +export const TRUSTED_SENDERS_BOOK_NAME = 'Trusted Senders'; + +/** + * Whether a collection the user never chose syncs: those of the personal + * account do, shared ones don't; nor does the "Trusted Senders" address + * book, a mail filter rather than people to call. + */ +export function collectionDefaultOn(personal: boolean, authority: Authority, name: string | null | undefined): boolean { + if (!personal) return false; + return !(authority === CONTACTS_AUTHORITY && name === TRUSTED_SENDERS_BOOK_NAME); +} + +/** An explicit choice wins; otherwise the default. */ +export function isCollectionSelected( + explicit: Readonly>, + account: { id: string; personal: boolean }, + authority: Authority, + collection: { id: string; name?: string | null }, +): boolean { + const choice = explicit[collectionKey(account.id, collection.id)]; + return typeof choice === 'boolean' ? choice : collectionDefaultOn(account.personal, authority, collection.name); +} diff --git a/src/device-sync/engine/sync-state.ts b/src/device-sync/engine/sync-state.ts new file mode 100644 index 00000000..f9df3106 --- /dev/null +++ b/src/device-sync/engine/sync-state.ts @@ -0,0 +1,188 @@ +/** + * The SyncState blob (docs/device-sync.md, "SyncState"): one per Android + * account and authority, versioned JSON the provider stores next to the + * rows. It is only ever written as the last op of a batch whose rows it + * describes, so a crash never leaves it ahead of them. + */ +import { clone } from '../common/json'; +import type { ProviderOp, ReminderOwner } from '../types'; + +export const SYNC_STATE_VERSION = 1; + +/** A full reconcile in progress. */ +export interface ReconcileMarker { + /** The item state taken before listing anything; null when a reconcile was asked for and has not started. */ + from: string | null; + phase: 'ids' | 'objects'; + /** Objects processed so far, in the reconcile's order. */ + position: number; + /** The last id processed in that (sorted) order: a resumed reconcile continues after it. */ + after: string | null; +} + +export interface AccountState { + /** AddressBook / Calendar state of the last collections read. */ + collectionsState: string | null; + /** ContactCard / CalendarEvent state the rows describe. */ + itemsState: string | null; + /** Collection keys the rows were written for. */ + selected: string[]; + /** Object ids whose rows could not be written; fetched again at the start of every run, nothing uploads for them meanwhile. */ + stale: string[]; + reconcile: ReconcileMarker | null; + /** Contacts: refs of the group cards present on the device (a hard-deleted group is found by its absence). */ + groups?: string[]; + /** Calendar: ids of calendars found to hold only tasks. */ + taskOnly?: string[]; +} + +export interface SyncState { + v: typeof SYNC_STATE_VERSION; + /** The registry account and server origin the rows belong to. */ + owner: { registryId: string; origin: string } | null; + accounts: Record; + /** Calendar: the zone floating events were written in. */ + deviceZone?: string | null; + /** Calendar: the zone a zone-change pass is moving floating events to. */ + deviceZonePending?: string | null; + /** Calendar: who the Reminders rows were written for (a change rewrites them). */ + reminderOwner?: ReminderOwner | null; +} + +export function emptySyncState(): SyncState { + return { v: SYNC_STATE_VERSION, owner: null, accounts: {} }; +} + +export function emptyAccountState(): AccountState { + return { collectionsState: null, itemsState: null, selected: [], stale: [], reconcile: null }; +} + +const strings = (value: unknown): string[] => + Array.isArray(value) ? value.filter((v): v is string => typeof v === 'string') : []; +const stringOrNull = (value: unknown): string | null => (typeof value === 'string' && value ? value : null); + +function parseReconcile(value: unknown): ReconcileMarker | null { + if (!value || typeof value !== 'object') return null; + const r = value as Record; + return { + from: stringOrNull(r.from), + phase: r.phase === 'objects' ? 'objects' : 'ids', + position: typeof r.position === 'number' && r.position >= 0 ? r.position : 0, + after: stringOrNull(r.after), + }; +} + +function parseAccount(value: unknown): AccountState { + const a = (value && typeof value === 'object' ? value : {}) as Record; + const out: AccountState = { + collectionsState: stringOrNull(a.collectionsState), + itemsState: stringOrNull(a.itemsState), + selected: strings(a.selected), + stale: strings(a.stale), + reconcile: parseReconcile(a.reconcile), + }; + if (a.groups !== undefined) out.groups = strings(a.groups); + if (a.taskOnly !== undefined) out.taskOnly = strings(a.taskOnly); + return out; +} + +/** + * The stored blob. `readable` is false when there was one that could not be + * understood (another version, garbage): the rows are then treated as ours + * and every account starts over with a full reconcile. + */ +export function parseSyncState(text: string | null): { state: SyncState; readable: boolean } { + if (!text) return { state: emptySyncState(), readable: true }; + let raw: unknown; + try { + raw = JSON.parse(text); + } catch { + return { state: emptySyncState(), readable: false }; + } + const r = (raw && typeof raw === 'object' ? raw : null) as Record | null; + if (!r || r.v !== SYNC_STATE_VERSION) return { state: emptySyncState(), readable: false }; + const owner = r.owner as Record | null | undefined; + const state: SyncState = { + v: SYNC_STATE_VERSION, + owner: + owner && typeof owner.registryId === 'string' && typeof owner.origin === 'string' + ? { registryId: owner.registryId, origin: owner.origin } + : null, + accounts: {}, + }; + if (r.accounts && typeof r.accounts === 'object') { + for (const [id, account] of Object.entries(r.accounts as Record)) { + state.accounts[id] = parseAccount(account); + } + } + if ('deviceZone' in r) state.deviceZone = stringOrNull(r.deviceZone); + if ('deviceZonePending' in r) state.deviceZonePending = stringOrNull(r.deviceZonePending); + if (r.reminderOwner === 'device' || r.reminderOwner === 'bulwark') state.reminderOwner = r.reminderOwner; + return { state, readable: true }; +} + +export function serializeSyncState(state: SyncState): string { + return JSON.stringify(state); +} + +/** The op that ends a batch and moves the stored state along with its rows. */ +export interface Tail { + op(): ProviderOp; + /** Called once the batch carrying `op()` applied. */ + applied(): void; +} + +/** + * The committed SyncState of a run. Changes are proposed as a tail: the + * batch writer calls `op()` right before sending the last batch (so the + * change sees what earlier batches decided, e.g. items put on `stale`), and + * the state is only adopted after that batch applied. + */ +export class StateStore { + private state: SyncState; + + constructor( + initial: SyncState, + private readonly owner: { registryId: string; origin: string } | null, + private readonly onCommit?: (before: SyncState, after: SyncState) => void, + ) { + this.state = clone(initial); + } + + get committed(): Readonly { + return this.state; + } + + account(jmapAccountId: string): Readonly { + return this.state.accounts[jmapAccountId] ?? emptyAccountState(); + } + + /** A tail applying `mutate` to a copy of the committed state (plus the owner). */ + tail(mutate: (next: SyncState) => void): Tail { + let next: SyncState | null = null; + return { + op: () => { + next = clone(this.state); + if (this.owner) next.owner = { ...this.owner }; + mutate(next); + return { op: 'syncState', value: serializeSyncState(next) }; + }, + applied: () => { + if (!next) return; + const before = this.state; + this.state = next; + this.onCommit?.(before, next); + }, + }; + } +} + +/** The account's entry in a state being built, created when missing. */ +export function accountOf(state: SyncState, jmapAccountId: string): AccountState { + let account = state.accounts[jmapAccountId]; + if (!account) { + account = emptyAccountState(); + state.accounts[jmapAccountId] = account; + } + return account; +} diff --git a/src/device-sync/engine/teardown.ts b/src/device-sync/engine/teardown.ts new file mode 100644 index 00000000..507280a9 --- /dev/null +++ b/src/device-sync/engine/teardown.ts @@ -0,0 +1,127 @@ +/** + * Turning device sync off for an authority, or signing out + * (docs/device-sync.md, "App integration", Lifecycle): under the run lock, + * the device's dirty, deleted and new items are uploaded first (bounded in + * time, failures ignored), then every row of the authority and its SyncState + * go. When changes are still waiting, nothing is deleted unless the user + * confirmed losing them (`force`). + */ +import { CALENDAR_AUTHORITY, type Authority, type ProviderOp } from '../types'; +import { BatchWriter } from './batch'; +import type { EngineDeps } from './deps'; +import { acquireLock, clearStopRequest, lockKey, requestStop } from './mutex'; +import { ProviderReader, str } from './provider'; +import { ReportBuilder } from './report'; +import { createEnv, tuningOf } from './run'; +import { parseSyncState } from './sync-state'; + +export interface TeardownOptions { + /** Upload waiting changes first (default true). */ + uploadFirst?: boolean; + /** Time for the upload (default 30 s). */ + timeoutMs?: number; + /** Delete even when changes could not be uploaded (the user confirmed). */ + force?: boolean; +} + +/** Items still waiting for an upload: dirty, deleted, new, and groups deleted by absence. */ +export async function countPending(reader: ProviderReader, authority: Authority): Promise { + const { state } = parseSyncState(await reader.readSyncState()); + if (authority === CALENDAR_AUTHORITY) { + const rows = await reader.rows( + 'events', + ['_id', 'original_id', 'original_sync_id'], + "dirty = 1 OR deleted = 1 OR _sync_id IS NULL OR _sync_id LIKE '~pending/%'", + ); + const masters = new Set(rows.map((r) => str(r.original_id) ?? str(r.original_sync_id) ?? `row:${String(r._id)}`)); + return masters.size; + } + const contacts = await reader.rows('raw_contacts', ['_id'], 'dirty = 1 OR deleted = 1 OR sourceid IS NULL'); + const groups = await reader.rows('groups', ['_id', 'sourceid', 'dirty', 'deleted']); + const present = new Set(groups.map((g) => str(g.sourceid)).filter(Boolean)); + const waitingGroups = groups.filter((g) => !str(g.sourceid) || Number(g.dirty) === 1 || Number(g.deleted) === 1).length; + const absent = Object.values(state.accounts).reduce((n, a) => n + (a.groups ?? []).filter((ref) => !present.has(ref)).length, 0); + return contacts.length + waitingGroups + absent; +} + +function deleteEverything(authority: Authority): ProviderOp[] { + // Sync-adapter deletes: raw contacts take their data rows along, calendar rows their events. + return authority === CALENDAR_AUTHORITY + ? [{ op: 'delete', table: 'calendars' }] + : [ + { op: 'delete', table: 'raw_contacts' }, + { op: 'delete', table: 'groups' }, + { op: 'delete', table: 'settings' }, + ]; +} + +export async function teardownAuthority( + deps: EngineDeps, + registryId: string, + accountName: string, + authority: Authority, + options: TeardownOptions = {}, +): Promise<{ pending: number }> { + const tuning = tuningOf(deps); + const key = lockKey(registryId, authority); + // A sync holding the lock stops at its next checkpoint instead of running to its deadline. + requestStop(key); + const release = await acquireLock(key); + clearStopRequest(key); + try { + const reader = new ProviderReader(deps.provider(accountName, authority)); + if (options.uploadFirst !== false) { + const started = deps.now(); + try { + const { state: stored, readable } = parseSyncState(await reader.readSyncState()); + // Rows written for another registry account are never uploaded to this one. + if (!stored.owner || stored.owner.registryId === registryId) { + const connection = await deps.jmap(registryId); + if (!stored.owner || stored.owner.origin === connection.origin) { + const report = new ReportBuilder('teardown', authority, started, tuning.maxItemErrors); + const { sync } = await createEnv({ + deps, + tuning: { ...tuning, deadlineMarginMs: 0, createBudgetMs: 0 }, + mode: 'teardown', + registryId, + accountName, + authority, + extras: {}, + deadline: started + (options.timeoutMs ?? 30_000), + isCancelled: async () => false, + reader, + report, + stored: readable ? stored : parseSyncState(null).state, + connection, + }); + await sync.uploadOnly(); + } + } + } catch (error) { + // Offline, signed out, out of time: whatever did not go up is counted below. + deps.log?.('upload before teardown stopped', error); + } + } + const pending = await countPending(reader, authority); + if (pending > 0 && !options.force) return { pending }; + const writer = new BatchWriter(reader.port, { + maxOps: tuning.maxBatchOps, + maxBytes: tuning.maxBatchBytes, + maxReplans: 0, + }); + await writer.write( + [ + { + group: { ref: 'teardown', ops: deleteEverything(authority) }, + failed: (reason, message) => { + throw new Error(`Could not remove the synced rows: ${reason} ${message}`); + }, + }, + ], + { op: () => ({ op: 'syncState', value: '' }), applied: () => undefined }, + ); + return { pending: 0 }; + } finally { + release?.(); + } +} diff --git a/src/device-sync/jmap/caller.ts b/src/device-sync/jmap/caller.ts new file mode 100644 index 00000000..52f34e08 --- /dev/null +++ b/src/device-sync/jmap/caller.ts @@ -0,0 +1,333 @@ +/** + * The engine's JMAP calls on one registry account's `JmapPort`: one request + * in flight at a time, the time of the last request (for the keep-alive + * echo), and the method helpers device sync needs, each sized by the + * session's limits (docs/device-sync.md, "A sync run" and "Limits"). + * + * Pure: the port is a detached JMAPClient on the device and the fake server + * in tests. + */ +import { requireMethodResult } from '../../api/jmap-result'; +import type { JMAPResponseBody } from '../../api/types'; +import { + JMAP_CALENDARS, + JMAP_CONTACTS, + JMAP_CORE, + type JmapCoreLimits, + type JmapInvocation, + type JmapPort, + type JmapResponse, + type JmapSessionView, +} from '../types'; +import type { PatchObject } from '../common/patch'; + +export type ItemType = 'ContactCard' | 'CalendarEvent'; +export type ContainerType = 'AddressBook' | 'Calendar'; +type DataType = ItemType | ContainerType; + +/** `/changes` pages: Stalwart caps at 5000, a small page keeps every page a quick checkpoint. */ +export const MAX_CHANGES = 256; +/** `/query` pages (Stalwart's `query_max_results`). */ +export const QUERY_PAGE = 5000; +/** Ids re-read at the start of the next `/query` page, so a deletion that shifts the list skips nothing. */ +const QUERY_OVERLAP = 100; + +export function capabilityOf(type: DataType): string { + return type === 'ContactCard' || type === 'AddressBook' ? JMAP_CONTACTS : JMAP_CALENDARS; +} + +function using(type: DataType): string[] { + return [JMAP_CORE, capabilityOf(type)]; +} + +export interface ChangesPage { + oldState: string; + newState: string; + hasMoreChanges: boolean; + created: string[]; + updated: string[]; + destroyed: string[]; +} + +export interface SetErrorShape { + type: string; + description?: string; + properties?: string[]; +} + +export interface SetResponse { + oldState?: string; + newState: string; + created?: Record>; + updated?: Record; + destroyed?: string[]; + notCreated?: Record; + notUpdated?: Record; + notDestroyed?: Record; +} + +export interface SetArgs { + create?: Record>; + update?: Record; + destroy?: string[]; + sendSchedulingMessages?: boolean; +} + +export class JmapCaller { + private chain: Promise = Promise.resolve(); + private lastRequestAt: number; + /** Requests sent, for tests and logs. */ + requestCount = 0; + + constructor( + readonly port: JmapPort, + private readonly now: () => number, + ) { + this.lastRequestAt = now(); + } + + session(): JmapSessionView { + return this.port.session(); + } + + limits(): JmapCoreLimits { + return this.port.limits(); + } + + /** Milliseconds since the last request started or ended. */ + idleMs(): number { + return this.now() - this.lastRequestAt; + } + + /** + * One request at a time: the server's `maxConcurrentRequests` is a per-user + * budget the UI shares, and a sync run has no reason to spend more than one. + */ + request(calls: JmapInvocation[], usingCaps: string[]): Promise { + const run = async () => { + this.lastRequestAt = this.now(); + this.requestCount++; + try { + return await this.port.request(calls, usingCaps); + } finally { + this.lastRequestAt = this.now(); + } + }; + const result = this.chain.then(run, run); + this.chain = result.catch(() => undefined); + return result; + } + + /** One method call; a method-level error throws `JMAPMethodError` with its `type`. */ + async call(method: string, args: Record, usingCaps: string[]): Promise { + const response = await this.request([[method, args, '0']], usingCaps); + return requireMethodResult(response as unknown as JMAPResponseBody, '0', method); + } + + /** Network traffic for SyncManager's "no traffic for 60 s" monitor during long local phases. */ + async echo(): Promise { + await this.request([['Core/echo', { ping: this.now() }, '0']], [JMAP_CORE]); + } + + /** Bytes of a blob (a blob-backed photo), in turn with the requests. */ + downloadBlob(accountId: string, blobId: string, type?: string): Promise { + const run = async () => { + this.lastRequestAt = this.now(); + try { + return await this.port.downloadBlob(accountId, blobId, type); + } finally { + this.lastRequestAt = this.now(); + } + }; + const result = this.chain.then(run, run); + this.chain = result.catch(() => undefined); + return result; + } + + // ── Reads ── + + /** + * Objects by id with explicit properties, in calls of at most + * `maxObjectsInGet` ids (never `ids: null`, which Stalwart silently caps). + */ + async get( + type: DataType, + accountId: string, + ids: readonly string[], + properties: readonly string[], + ): Promise<{ list: T[]; notFound: string[]; state: string | null }> { + const list: T[] = []; + const notFound: string[] = []; + let state: string | null = null; + const size = Math.max(1, this.limits().maxObjectsInGet); + for (let i = 0; i < ids.length; i += size) { + const body = await this.call<{ list?: T[]; notFound?: string[]; state?: string }>( + `${type}/get`, + { accountId, ids: ids.slice(i, i + size), properties: [...properties] }, + using(type), + ); + list.push(...(body.list ?? [])); + notFound.push(...(body.notFound ?? [])); + state = body.state ?? state; + } + return { list, notFound, state }; + } + + /** Every container of the account (there are few; Stalwart allows 250 per account). */ + async getContainers(type: ContainerType, accountId: string, properties: readonly string[]): Promise<{ list: T[]; state: string | null }> { + const body = await this.call<{ list?: T[]; state?: string }>( + `${type}/get`, + { accountId, ids: null, properties: [...properties] }, + using(type), + ); + return { list: body.list ?? [], state: body.state ?? null }; + } + + /** The current state of a type, taken before a full reconcile lists anything. */ + async takeState(type: ItemType, accountId: string): Promise { + const body = await this.call<{ state?: string }>(`${type}/get`, { accountId, ids: [], properties: ['id'] }, using(type)); + return body.state ?? ''; + } + + async changes(type: ItemType, accountId: string, sinceState: string, maxChanges = MAX_CHANGES): Promise { + const body = await this.call>( + `${type}/changes`, + { accountId, sinceState, maxChanges }, + using(type), + ); + return { + oldState: body.oldState ?? sinceState, + newState: body.newState ?? sinceState, + hasMoreChanges: Boolean(body.hasMoreChanges), + created: body.created ?? [], + updated: body.updated ?? [], + destroyed: body.destroyed ?? [], + }; + } + + /** + * Every id matching `filter`, paged by `position` in pages of at most 5000. + * Pages overlap a little: a deletion between two pages shifts the list and + * would otherwise skip an object. + */ + async queryAll(type: ItemType, accountId: string, filter: Record, pageSize = QUERY_PAGE): Promise { + const seen = new Set(); + let position = 0; + for (let round = 0; round < 1000; round++) { + const body = await this.call<{ ids?: string[]; total?: number; position?: number }>( + `${type}/query`, + { accountId, filter, position, limit: pageSize, calculateTotal: true }, + using(type), + ); + const ids = body.ids ?? []; + for (const id of ids) seen.add(id); + const total = typeof body.total === 'number' ? body.total : null; + const end = position + ids.length; + if (ids.length === 0) break; + // A server may cap the page below `pageSize`; the total says whether more follow. + if (total !== null ? end >= total : ids.length < pageSize) break; + position = end - Math.min(QUERY_OVERLAP, Math.floor(ids.length / 4)); + } + return [...seen]; + } + + /** The first `limit` ids matching `filter` (task-only calendar detection). */ + async queryFirst(type: ItemType, accountId: string, filter: Record, limit: number): Promise { + const body = await this.call<{ ids?: string[] }>(`${type}/query`, { accountId, filter, limit }, using(type)); + return body.ids ?? []; + } + + /** + * Ids of the objects carrying each uid, in `collectionId` when given + * (contact uids are unique per address book, event uids per account). The + * uid filter reads Stalwart's asynchronous search index, so a just-created + * object can be missing. + */ + async lookupUids( + type: ItemType, + accountId: string, + uids: readonly string[], + collectionId: string | null, + ): Promise> { + const found = new Map(); + if (uids.length === 0) return found; + const byUid: Record = + uids.length === 1 ? { uid: uids[0] } : { operator: 'OR', conditions: uids.map((uid) => ({ uid })) }; + const parentFilter = type === 'ContactCard' ? 'inAddressBook' : 'inCalendar'; + const filter = collectionId === null + ? byUid + : { operator: 'AND', conditions: [{ [parentFilter]: collectionId }, byUid] }; + const ids = await this.queryAll(type, accountId, filter); + if (ids.length === 0) return found; + const { list } = await this.get<{ id: string; uid?: string }>(type, accountId, ids, ['id', 'uid']); + for (const object of list) { + if (typeof object.uid !== 'string' || !uids.includes(object.uid)) continue; + found.set(object.uid, [...(found.get(object.uid) ?? []), object.id]); + } + return found; + } + + // ── Writes ── + + /** + * One `/set` call, guarded by `ifInState` when there is one; with + * `thenGet`, a `/get` of those ids in the same request (updates only: + * Stalwart resolves no result reference to a `/set`, and a create's id is + * only known from the response). A method-level error (`stateMismatch`, …) + * throws `JMAPMethodError`. + */ + async set( + type: ItemType, + accountId: string, + args: SetArgs, + ifInState: string | null, + thenGet?: { ids: string[]; properties: readonly string[] }, + ): Promise<{ response: SetResponse; got: T[] | null }> { + const setArgs: Record = { accountId }; + if (ifInState) setArgs.ifInState = ifInState; + if (args.create && Object.keys(args.create).length) setArgs.create = args.create; + if (args.update && Object.keys(args.update).length) setArgs.update = args.update; + if (args.destroy?.length) setArgs.destroy = args.destroy; + if (args.sendSchedulingMessages) setArgs.sendSchedulingMessages = true; + const calls: JmapInvocation[] = [[`${type}/set`, setArgs, 's']]; + const withGet = !!thenGet && thenGet.ids.length > 0 && thenGet.ids.length <= this.limits().maxObjectsInGet; + if (withGet) { + calls.push([`${type}/get`, { accountId, ids: thenGet!.ids, properties: [...thenGet!.properties] }, 'g']); + } + const res = (await this.request(calls, using(type))) as unknown as JMAPResponseBody; + const response = requireMethodResult(res, 's', `${type}/set`); + let got: T[] | null = null; + if (withGet) { + got = requireMethodResult<{ list?: T[] }>(res, 'g', `${type}/get`).list ?? []; + } else if (thenGet && thenGet.ids.length > 0) { + got = (await this.get(type, accountId, thenGet.ids, thenGet.properties)).list; + } + return { response, got }; + } +} + +// ── SetErrors ── + +/** + * What a SetError means for the item (docs/device-sync.md, "Uploads"): + * - `notFound`: gone on the server (an update: a remote delete; a destroy: done); + * - `forbidden`: not allowed (retried without scheduling messages, else reverted); + * - `uidExists`: `invalidProperties` on `uid`: the object exists already (a + * lost response, or the client's automatic retry of a create); + * - `poison`: the object itself is refused; back off until it changes. + */ +export type SetErrorClass = 'notFound' | 'forbidden' | 'uidExists' | 'poison'; + +export function classifySetError(error: SetErrorShape | undefined, op: 'create' | 'update' | 'destroy'): SetErrorClass { + const type = error?.type; + if (type === 'notFound') return 'notFound'; + if (type === 'forbidden') return 'forbidden'; + if (op === 'create' && type === 'invalidProperties' && error?.properties?.includes('uid')) return 'uidExists'; + return 'poison'; +} + +/** The existing card Stalwart names in a duplicate-uid error ("… already exists with id c5."). */ +export function existingIdFromUidError(error: SetErrorShape | undefined): string | null { + const m = /already exists with id ([A-Za-z0-9_-]+)/.exec(error?.description ?? ''); + return m ? m[1] : null; +} diff --git a/src/device-sync/jmap/client-port.ts b/src/device-sync/jmap/client-port.ts new file mode 100644 index 00000000..6fbe93f6 --- /dev/null +++ b/src/device-sync/jmap/client-port.ts @@ -0,0 +1,87 @@ +/** + * The JMAP side of a device sync run on the device: a `JmapPort` over a + * detached `JMAPClient` of one registry account. Never the UI's `jmapClient` + * singleton: a run can share the JS runtime with the UI (the headless task + * starts with `allowedInForeground`), and re-binding the singleton would send + * the UI's requests with another account's credentials (the widgets and the + * push task avoid it for the same reason). + * + * This file is the React Native boundary of `src/device-sync/jmap`; the rest + * of the folder is pure and runs against the fake server in tests. + */ +import { JMAPClient } from '../../api/jmap-client'; +import { JMAP_CORE, type JmapCoreLimits, type JmapPort, type JmapResponse, type JmapSessionView } from '../types'; +import { openWithAuthRebuild, type JmapConnection } from './connection'; + +/** RFC 8620 fallbacks when the server advertises no core limit. */ +const FALLBACK_LIMITS: JmapCoreLimits = { + maxObjectsInGet: 500, + maxObjectsInSet: 500, + maxCallsInRequest: 16, + maxConcurrentRequests: 4, + maxSizeRequest: 10_000_000, +}; + +function positive(value: unknown, fallback: number): number { + return typeof value === 'number' && Number.isFinite(value) && value > 0 ? value : fallback; +} + +export function coreLimits(capabilities: Record | undefined): JmapCoreLimits { + const core = (capabilities?.[JMAP_CORE] ?? {}) as Partial>; + return { + maxObjectsInGet: positive(core.maxObjectsInGet, FALLBACK_LIMITS.maxObjectsInGet), + maxObjectsInSet: positive(core.maxObjectsInSet, FALLBACK_LIMITS.maxObjectsInSet), + maxCallsInRequest: positive(core.maxCallsInRequest, FALLBACK_LIMITS.maxCallsInRequest), + maxConcurrentRequests: positive(core.maxConcurrentRequests, FALLBACK_LIMITS.maxConcurrentRequests), + maxSizeRequest: positive(core.maxSizeRequest, FALLBACK_LIMITS.maxSizeRequest), + }; +} + +function originOf(url: string | null): string { + const m = /^(https?:\/\/[^/?#]+)/i.exec(url ?? ''); + return m ? m[1].toLowerCase() : (url ?? ''); +} + +function sessionView(client: JMAPClient): JmapSessionView { + const session = client.currentSession; + if (!session) throw new Error('JMAP session not loaded'); + const accounts: JmapSessionView['accounts'] = {}; + for (const [id, account] of Object.entries(session.accounts ?? {})) { + accounts[id] = { + name: account.name, + isPersonal: account.isPersonal, + isReadOnly: account.isReadOnly, + accountCapabilities: account.accountCapabilities ?? {}, + }; + } + return { + username: session.username || client.username || '', + accounts, + primaryAccounts: session.primaryAccounts ?? {}, + capabilities: session.capabilities ?? {}, + }; +} + +function clientPort(client: JMAPClient): JmapPort { + return { + session: () => sessionView(client), + limits: () => coreLimits(client.currentSession?.capabilities), + request: async (calls, using) => (await client.request(calls, using)) as unknown as JmapResponse, + downloadBlob: async (accountId, blobId, type) => + new Uint8Array(await client.fetchBlobArrayBuffer(blobId, undefined, type, accountId)), + }; +} + +/** + * A port for the registry account `registryId` (`AccountEntry.id`). No + * stored credentials or rejected ones are retried once with a fresh client, + * then fail with an AuthenticationError; an unreachable server throws a + * NetworkError and touches nothing. + */ +export function openJmapPort(registryId: string): Promise { + return openWithAuthRebuild(async () => { + const client = new JMAPClient(); + if (!(await client.loadAccount(registryId))) return null; + return { port: clientPort(client), origin: originOf(client.serverUrl) }; + }); +} diff --git a/src/device-sync/jmap/connection.ts b/src/device-sync/jmap/connection.ts new file mode 100644 index 00000000..a36fd255 --- /dev/null +++ b/src/device-sync/jmap/connection.ts @@ -0,0 +1,62 @@ +/** + * A registry account's JMAP connection for one sync run, with the one + * rebuild the design allows when the server rejects the credentials: the UI + * (or another run) may have rotated the OAuth refresh token this client + * still holds, and a fresh client reads the rotated one from secure storage + * (docs/device-sync.md, "A sync run", preflight). + */ +import type { JmapPort } from '../types'; +import { authError, transportFailure } from './errors'; + +export interface JmapConnection { + port: JmapPort; + /** Origin of the server (`https://host[:port]`), recorded as the SyncState owner. */ + origin: string; +} + +/** Loads a client; null when the account has no usable stored credentials. */ +export type ConnectionLoader = () => Promise; + +async function loadOnce(load: ConnectionLoader): Promise { + try { + return await load(); + } catch (error) { + if (transportFailure(error) === 'auth') return null; + throw error; + } +} + +/** + * Opens the connection: a missing session or an AuthenticationError is + * retried once with a fresh client; still failing, it throws an + * AuthenticationError. Network trouble propagates as it came. Later + * requests rebuild once more the same way when the server answers 401. + */ +export async function openWithAuthRebuild(load: ConnectionLoader): Promise { + const first = await loadOnce(load); + const opened = first ?? (await load()); + if (!opened) throw authError('No usable credentials for this account'); + + let current = opened.port; + let rebuilt = !first; + const retryOnAuth = async (send: (port: JmapPort) => Promise): Promise => { + try { + return await send(current); + } catch (error) { + // A 401 comes back before the server ran anything, so replaying is safe. + if (transportFailure(error) !== 'auth' || rebuilt) throw error; + rebuilt = true; + const next = await load(); + if (!next) throw error; + current = next.port; + return send(current); + } + }; + const port: JmapPort = { + session: () => current.session(), + limits: () => current.limits(), + request: (calls, using) => retryOnAuth((p) => p.request(calls, using)), + downloadBlob: (accountId, blobId, type) => retryOnAuth((p) => p.downloadBlob(accountId, blobId, type)), + }; + return { port, origin: opened.origin }; +} diff --git a/src/device-sync/jmap/errors.ts b/src/device-sync/jmap/errors.ts new file mode 100644 index 00000000..faaeff72 --- /dev/null +++ b/src/device-sync/jmap/errors.ts @@ -0,0 +1,62 @@ +/** + * Transport failures of the JMAP side, recognised by `error.name` as + * jmap-client.ts sets it (AuthenticationError, NetworkError, + * RequestTimeoutError, RateLimitError), so the engine never imports the + * client: it stays pure and runs against the fake server in tests. + */ +import { JMAPMethodError } from '../../api/jmap-result'; + +export { JMAPMethodError }; + +/** + * - `auth`: the server rejected the credentials (a 401 even after a token refresh); + * - `network` / `timeout`: no response; the server may have applied the request; + * - `rateLimit`: a 429, with `retryAfterMs`; + * - `other`: anything else (a non-OK HTTP status, a body that is not JSON, a bug). + */ +export type TransportFailure = 'auth' | 'network' | 'timeout' | 'rateLimit' | 'other'; + +export function transportFailure(error: unknown): TransportFailure { + switch ((error as { name?: unknown } | null)?.name) { + case 'AuthenticationError': + case 'TotpRequiredError': + return 'auth'; + case 'NetworkError': + return 'network'; + case 'RequestTimeoutError': + return 'timeout'; + case 'RateLimitError': + return 'rateLimit'; + default: + return 'other'; + } +} + +/** Milliseconds a RateLimitError asks us to wait, or null. */ +export function retryAfterMs(error: unknown): number | null { + const ms = (error as { retryAfterMs?: unknown } | null)?.retryAfterMs; + return typeof ms === 'number' && Number.isFinite(ms) && ms >= 0 ? ms : null; +} + +/** A method-level error (`['error', {type}]`) of the given type, e.g. `cannotCalculateChanges`. */ +export function isMethodError(error: unknown, type?: string): error is JMAPMethodError { + return error instanceof JMAPMethodError && (type === undefined || error.type === type); +} + +/** + * jmap-client.ts turns Stalwart's HTTP 400 request-level limit error into a + * plain `JMAP request failed: 400 - {…limit…}`. The engine sizes every + * request by the session's limits, so one of those is a bug, not an outage. + */ +export function isRequestLimitError(error: unknown): boolean { + const message = (error as { message?: unknown } | null)?.message; + return typeof message === 'string' + && /JMAP request failed: 400/.test(message) + && message.includes('urn:ietf:params:jmap:error:limit'); +} + +export function authError(message: string): Error { + const error = new Error(message); + error.name = 'AuthenticationError'; + return error; +} diff --git a/src/device-sync/jmap/session.ts b/src/device-sync/jmap/session.ts new file mode 100644 index 00000000..caf93550 --- /dev/null +++ b/src/device-sync/jmap/session.ts @@ -0,0 +1,87 @@ +/** + * What device sync reads from a JMAP session: the accounts that hold address + * books or calendars, and the user's own calendar addresses. + */ +import { JMAP_CORE, type JmapSessionView } from '../types'; +import type { JmapCaller } from './caller'; + +const JMAP_SUBMISSION = 'urn:ietf:params:jmap:submission'; + +export interface CapabilityAccount { + id: string; + name: string; + /** The capability's primary account (the user's own). */ + primary: boolean; + /** Collections of personal accounts are selected by default, shared ones are not. */ + personal: boolean; + readOnly: boolean; +} + +/** + * The primary account of the capability (`primaryAccounts[capability]`) + * first, then every other account that advertises it (shared accounts). + */ +export function accountsWithCapability(session: JmapSessionView, capability: string): CapabilityAccount[] { + const out: CapabilityAccount[] = []; + const primary = session.primaryAccounts?.[capability]; + const add = (id: string, isPrimary: boolean) => { + const account = session.accounts?.[id]; + if (!account || out.some((a) => a.id === id)) return; + out.push({ + id, + name: account.name || id, + primary: isPrimary, + personal: isPrimary || account.isPersonal, + readOnly: !!account.isReadOnly, + }); + }; + if (primary) add(primary, true); + for (const [id, account] of Object.entries(session.accounts ?? {})) { + if (id !== primary && account.accountCapabilities && capability in account.accountCapabilities) add(id, false); + } + return out; +} + +export interface CalendarAddresses { + /** `OWNER_ACCOUNT` of our calendars: never empty (Etar crashes on a null owner). */ + ownerAccount: string; + /** Every address of the user, lowercase, without `mailto:`. */ + selfAddresses: string[]; +} + +function normalizeAddress(value: unknown): string | null { + if (typeof value !== 'string') return null; + const address = value.trim().replace(/^mailto:/i, '').toLowerCase(); + return address.includes('@') ? address : null; +} + +/** + * The login when it is an email address, else the first identity's address; + * aliases from identities count as "me" too (docs/device-sync.md, + * "Decisions"). Identities are read only when the session offers + * submission; a server without `Identity/get` leaves the login. + */ +export async function calendarAddresses(caller: JmapCaller, fallbackName: string): Promise { + const session = caller.session(); + const login = normalizeAddress(session.username); + const identities: string[] = []; + const submissionAccount = session.primaryAccounts?.[JMAP_SUBMISSION]; + if (submissionAccount) { + try { + const body = await caller.call<{ list?: Array<{ email?: string }> }>( + 'Identity/get', + { accountId: submissionAccount, properties: ['email'] }, + [JMAP_CORE, JMAP_SUBMISSION], + ); + for (const identity of body.list ?? []) { + const address = normalizeAddress(identity.email); + if (address) identities.push(address); + } + } catch { + // Without identities the login is the only address we know. + } + } + const selfAddresses = [...new Set([...(login ? [login] : []), ...identities])]; + const ownerAccount = login ?? identities[0] ?? (session.username || fallbackName).toLowerCase(); + return { ownerAccount, selfAddresses: selfAddresses.length ? selfAddresses : [ownerAccount] }; +} diff --git a/src/device-sync/planner.ts b/src/device-sync/planner.ts index ebf16a05..5f429838 100644 --- a/src/device-sync/planner.ts +++ b/src/device-sync/planner.ts @@ -182,6 +182,13 @@ export interface ContactsContext extends MapContext { nameForUid(uid: string): string | null; /** Group rows by SOURCE_ID, for memberships. */ groupRowIdBySourceId(sourceId: string): number | null; + /** + * SOURCE_IDs of the synced group cards whose `members` list this uid: the + * server keeps memberships on the group cards, so a contact's + * GroupMembership rows need the reverse lookup. The engine always provides + * it; without it the planner leaves memberships as they are. + */ + groupsOf?(uid: string): string[]; } export interface ContactsPlanner { @@ -332,7 +339,11 @@ export interface CalendarPlanner { /** Deletes the master and its exception rows (a sync-adapter delete does not cascade). */ planLocalDelete(local: LocalEvent): OpGroup; planBaselineHeal(local: LocalEvent): OpGroup | null; - /** May return several actions: a "this and following" split is a rule patch plus a create. */ + /** + * May return several actions. For a "this and following" split, the + * source's plan is the rule patch plus its pruned overrides; the new series + * is a row of its own and goes through claim and create by itself. + */ planUpload(local: LocalEvent, ctx: CalendarContext): UploadPlan; planAccepted(local: LocalEvent, server: CalendarEventWire, ctx: CalendarContext): AcceptedPlan; /** diff --git a/src/device-sync/task.ts b/src/device-sync/task.ts index 95e473d7..54d18436 100644 --- a/src/device-sync/task.ts +++ b/src/device-sync/task.ts @@ -1,41 +1,190 @@ /** * Entry of the BulwarkDeviceSync headless task, which the contacts and - * calendar sync adapters start for every sync (index.ts registers it). See - * docs/device-sync.md. + * calendar sync adapters start for every sync (index.ts registers it), and + * the teardown the app calls when sync is turned off or an account signs + * out. See docs/device-sync.md. * - * The run must always end with `finishRun`, called before the task's promise + * This is the React Native boundary of the engine: it wires the native + * module, a detached JMAPClient per registry account (never the UI's + * singleton) and the device-sync store into the pure engine. + * + * A run always ends with `finishRun`, called before the task's promise * settles: JS timers stop once the task is over, and a sync adapter waiting - * for a report that never comes holds its sync until the deadline. - * - * The sync engine is not wired in yet, so a run only reports that device - * sync is off; nothing in the app can turn it on at this point. + * for a report that never comes holds its sync until the deadline. Nothing + * here throws: every failure becomes a report. */ -import { NativeModules } from 'react-native'; -import type { DeviceSyncNativeModule, RunPayload, RunReport } from './types'; +import AsyncStorage from '@react-native-async-storage/async-storage'; +import { detectDeviceLocale, isSupportedLocale, translate, type LocaleCode } from '../i18n'; +import { randomBytes } from '../lib/random'; +import { accountSyncPrefs, useDeviceSyncStore, waitForDeviceSyncHydration } from '../stores/device-sync-store'; +import type { RandomSource } from './common/ids'; +import { + runDeviceSync, + teardownAuthority as teardownWithDeps, + type EngineDeps, + type SubscriptionCalendar, + type TeardownOptions, +} from './engine'; +import { emptyStats } from './engine/report'; +import { openJmapPort } from './jmap/client-port'; +import { createProviderPort, finishRun, getSyncSettings, isRunCancelled, showSyncProblem } from './native'; +import type { CalendarPlanner, ContactsPlanner } from './planner'; +import type { Authority, RunPayload, RunReport } from './types'; -export async function runDeviceSyncTask(payload: RunPayload): Promise { - const native = NativeModules.BulwarkDeviceSync as DeviceSyncNativeModule | undefined; - if (!native || !payload?.runId) return; - const report: RunReport = { +// Mirrors the persist names of the locale and feed-subscription stores; read +// straight from storage so a headless run loads neither store. +const LOCALE_STORAGE_KEY = 'webmail:locale:v1'; +const SUBSCRIPTIONS_STORAGE_KEY = 'calendar-subscriptions'; +/** Where "sign in again" leads (the accounts pane), as for a registry eviction. */ +const SIGN_IN_URI = 'bulwarkmobile://settings/account'; + +// The contacts and calendar planners land in the next +// commits. Until then a run fails before it touches a row. +function notWired(): never { + throw new Error('Device sync planners are not wired into this build yet'); +} +const contactsPlanner = new Proxy({}, { get: notWired }) as ContactsPlanner; +const calendarPlanner = new Proxy({}, { get: notWired }) as CalendarPlanner; + +/** Random numbers from the platform CSPRNG, buffered (uids and map keys of device-made items). */ +function secureRandom(): RandomSource { + let buffer: Uint8Array = new Uint8Array(0); + let offset = 0; + return () => { + if (offset + 4 > buffer.length) { + buffer = randomBytes(256); + offset = 0; + } + const n = ((buffer[offset] << 24) >>> 0) + (buffer[offset + 1] << 16) + (buffer[offset + 2] << 8) + buffer[offset + 3]; + offset += 4; + return n / 2 ** 32; + }; +} + +/** Feed-subscription calendars (read-only on the device), from the subscriptions store's storage. */ +async function subscriptionCalendars(): Promise { + try { + const raw = await AsyncStorage.getItem(SUBSCRIPTIONS_STORAGE_KEY); + const subscriptions = raw ? (JSON.parse(raw) as { state?: { subscriptions?: unknown } }).state?.subscriptions : null; + if (!Array.isArray(subscriptions)) return []; + return subscriptions + .filter((s): s is { calendarId: string; accountId?: unknown } => typeof s?.calendarId === 'string') + .map((s) => ({ calendarId: s.calendarId, jmapAccountId: typeof s.accountId === 'string' ? s.accountId : null })); + } catch { + return []; + } +} + +/** The app's language: the in-app override, else the device's. */ +async function appLocale(): Promise { + try { + const raw = await AsyncStorage.getItem(LOCALE_STORAGE_KEY); + const override = raw ? (JSON.parse(raw) as { override?: unknown }).override : null; + if (typeof override === 'string' && isSupportedLocale(override)) return override; + } catch { + // The device language it is. + } + return detectDeviceLocale(); +} + +function deviceZone(): string { + try { + return Intl.DateTimeFormat().resolvedOptions().timeZone || 'UTC'; + } catch { + return 'UTC'; + } +} + +async function notifyAuthProblem(_registryId: string, accountName: string): Promise { + const locale = await appLocale(); + await showSyncProblem( + accountName, + translate(locale, 'device_sync.auth_problem_title', 'Sign in again to keep syncing'), + translate( + locale, + 'device_sync.auth_problem_text', + 'The server no longer accepts the sign-in of {account}, so its contacts and calendars on this device stopped syncing. Sign in again to resume. Changes made on this device are kept.', + { account: accountName }, + ), + SIGN_IN_URI, + translate(locale, 'device_sync.problem_channel', 'Sync problems'), + ); +} + +export function createTaskDeps(): EngineDeps { + return { + provider: createProviderPort, + jmap: openJmapPort, + prefs: async (registryId) => { + await waitForDeviceSyncHydration(); + return accountSyncPrefs(registryId); + }, + planners: { contacts: contactsPlanner, calendar: calendarPlanner }, + now: () => Date.now(), + isCancelled: isRunCancelled, + random: secureRandom(), + subscriptionCalendars, + isSyncEnabled: async (accountName, authority) => (await getSyncSettings(accountName)).authorities?.[authority]?.automatic === true, + deviceZone, + // zustand 5 persists on every set(), hydrated or not: wait, or the stored preferences are overwritten. + recordStatus: async (registryId, authority, status) => { + await waitForDeviceSyncHydration(); + useDeviceSyncStore.getState().recordRunStatus(registryId, authority, status); + }, + recordKnownState: async (registryId, jmapAccountId, type, state) => { + await waitForDeviceSyncHydration(); + useDeviceSyncStore.getState().recordKnownState(registryId, jmapAccountId, type, state); + }, + notifyAuthProblem, + log: (message, detail) => { + console.warn(`[device-sync] ${message}`, detail instanceof Error ? detail.message : (detail ?? '')); + }, + }; +} + +function failureReport(payload: RunPayload, error: unknown): RunReport { + return { v: 1, runId: payload.runId, authority: payload.authority, - outcome: 'disabled', - message: 'Device sync is not available in this build', + outcome: 'internal', + message: error instanceof Error ? error.message : String(error), startedAt: Date.now(), durationMs: 0, - stats: { - downloaded: { created: 0, updated: 0, deleted: 0 }, - uploaded: { created: 0, updated: 0, deleted: 0 }, - entries: 0, - skipped: 0, - }, + stats: emptyStats(), conflicts: 0, itemErrors: [], }; +} + +export async function runDeviceSyncTask(payload: RunPayload): Promise { + if (!payload?.runId) return; + let report: RunReport; try { - await native.finishRun(payload.runId, JSON.stringify(report)); - } catch { + report = await runDeviceSync(payload, createTaskDeps()); + } catch (error) { + report = failureReport(payload, error); + } + try { + await finishRun(payload.runId, report); + } catch (error) { // The adapter ends the run on its own when the task finishes. + console.warn('[device-sync] finishRun failed', error instanceof Error ? error.message : error); } } + +/** + * Turning sync off for an authority (or signing out): uploads what the + * device changed (bounded by `timeoutMs`, default 30 s), then removes the + * authority's rows and SyncState. When changes are still waiting + * (`pending > 0`) nothing is deleted unless `force` is set, once the user + * confirmed losing them. Runs under the same lock as sync runs. + */ +export function teardownAuthority( + registryId: string, + accountName: string, + authority: Authority, + options?: TeardownOptions, +): Promise<{ pending: number }> { + return teardownWithDeps(createTaskDeps(), registryId, accountName, authority, options); +} From 3ed5fa53af12698bcb88e8e31c4069644b5dc033 Mon Sep 17 00:00:00 2001 From: Linus Rath <139418639+rathlinus@users.noreply.github.com> Date: Sun, 27 Sep 2026 04:21:30 +0200 Subject: [PATCH 08/87] feat: map calendars and events to android's calendar provider #34 The engine hands every event it syncs to a planner that turns it into CalendarContract rows, and device edits back into JMAP writes. This is that planner for calendars (`src/device-sync/calendar`), now used by the sync task: - Calendars become calendar rows with what the apps need (SYNC_EVENTS=1, an owner, colours, the reminders allowed); calendars of shared accounts carry the account's name. - Events keep their time zone, floating and all-day times, recurrence rules, excluded and changed instances, reminders, attendees and the user's own reply. A rule Android can't hold (RSCALE, SKIP, leap months) stays on the server untouched, and local times Stalwart can't resolve (a start in a repeated hour, an UNTIL in a skipped one) are written in a form it keeps. - Device edits upload as patches of what differs from the last synced state, never a whole event, and only what the device can show is ever sent. A single-instance edit or deletion becomes an override of its series; "this and following" caps the series and creates a new one. How Etar and Fossify save (a cancelled instance instead of an exclusion, attendee rows deleted and re-inserted) is read as the edit it stands for, not as a deletion. Invitations go out on the changes a calendar app would notify about. - Events in read-only calendars and subscribed feeds are put back when the device changes them; only the user's own reply goes up, where the server allows it. Tasks, task-only calendars and the birthday calendar never reach the device. - Reminders are written as Reminders rows when the calendar app is to remind the user, and left out when Bulwark does. Checked: 179 calendar tests (download, upload, exceptions, the edits Etar, Fossify and CONTENT_EXCEPTION_URI make, echoes that write nothing, time zone and DST edges), and `npm run typecheck`, `npm test` and `npm run i18n:check` on this commit. --- .../__tests__/calendar/calendars.test.ts | 114 ++++ .../__tests__/calendar/download.test.ts | 248 ++++++++ .../__tests__/calendar/duration.test.ts | 81 +++ .../__tests__/calendar/echo.test.ts | 66 ++ .../__tests__/calendar/edge-cases.test.ts | 85 +++ .../__tests__/calendar/exceptions.test.ts | 355 +++++++++++ src/device-sync/__tests__/calendar/harness.ts | 232 +++++++ .../__tests__/calendar/lifecycle.test.ts | 472 ++++++++++++++ .../__tests__/calendar/robustness.test.ts | 98 +++ .../__tests__/calendar/rrule.test.ts | 119 ++++ .../__tests__/calendar/upload.test.ts | 320 ++++++++++ .../__tests__/calendar/zoned-time.test.ts | 81 +++ .../__tests__/fakes/fake-provider.ts | 2 +- src/device-sync/android-columns.ts | 2 + src/device-sync/calendar/accepted.ts | 90 +++ src/device-sync/calendar/attendees.ts | 165 +++++ src/device-sync/calendar/calendars.ts | 141 +++++ src/device-sync/calendar/columns.ts | 263 ++++++++ src/device-sync/calendar/create.ts | 280 ++++++++ src/device-sync/calendar/decode.ts | 195 ++++++ src/device-sync/calendar/duration.ts | 177 ++++++ src/device-sync/calendar/edits.ts | 248 ++++++++ src/device-sync/calendar/exceptions.ts | 86 +++ src/device-sync/calendar/exdate.ts | 114 ++++ src/device-sync/calendar/image.ts | 195 ++++++ src/device-sync/calendar/merge.ts | 460 ++++++++++++++ src/device-sync/calendar/pairs.ts | 150 +++++ src/device-sync/calendar/planner.ts | 463 ++++++++++++++ src/device-sync/calendar/reminders.ts | 153 +++++ src/device-sync/calendar/rows.ts | 251 ++++++++ src/device-sync/calendar/rrule.ts | 245 +++++++ src/device-sync/calendar/scheduling.ts | 50 ++ src/device-sync/calendar/timing.ts | 191 ++++++ src/device-sync/calendar/units.ts | 103 +++ src/device-sync/calendar/upload.ts | 597 ++++++++++++++++++ src/device-sync/calendar/values.ts | 192 ++++++ src/device-sync/calendar/zoned-time.ts | 252 ++++++++ src/device-sync/task.ts | 8 +- 38 files changed, 7339 insertions(+), 5 deletions(-) create mode 100644 src/device-sync/__tests__/calendar/calendars.test.ts create mode 100644 src/device-sync/__tests__/calendar/download.test.ts create mode 100644 src/device-sync/__tests__/calendar/duration.test.ts create mode 100644 src/device-sync/__tests__/calendar/echo.test.ts create mode 100644 src/device-sync/__tests__/calendar/edge-cases.test.ts create mode 100644 src/device-sync/__tests__/calendar/exceptions.test.ts create mode 100644 src/device-sync/__tests__/calendar/harness.ts create mode 100644 src/device-sync/__tests__/calendar/lifecycle.test.ts create mode 100644 src/device-sync/__tests__/calendar/robustness.test.ts create mode 100644 src/device-sync/__tests__/calendar/rrule.test.ts create mode 100644 src/device-sync/__tests__/calendar/upload.test.ts create mode 100644 src/device-sync/__tests__/calendar/zoned-time.test.ts create mode 100644 src/device-sync/calendar/accepted.ts create mode 100644 src/device-sync/calendar/attendees.ts create mode 100644 src/device-sync/calendar/calendars.ts create mode 100644 src/device-sync/calendar/columns.ts create mode 100644 src/device-sync/calendar/create.ts create mode 100644 src/device-sync/calendar/decode.ts create mode 100644 src/device-sync/calendar/duration.ts create mode 100644 src/device-sync/calendar/edits.ts create mode 100644 src/device-sync/calendar/exceptions.ts create mode 100644 src/device-sync/calendar/exdate.ts create mode 100644 src/device-sync/calendar/image.ts create mode 100644 src/device-sync/calendar/merge.ts create mode 100644 src/device-sync/calendar/pairs.ts create mode 100644 src/device-sync/calendar/planner.ts create mode 100644 src/device-sync/calendar/reminders.ts create mode 100644 src/device-sync/calendar/rows.ts create mode 100644 src/device-sync/calendar/rrule.ts create mode 100644 src/device-sync/calendar/scheduling.ts create mode 100644 src/device-sync/calendar/timing.ts create mode 100644 src/device-sync/calendar/units.ts create mode 100644 src/device-sync/calendar/upload.ts create mode 100644 src/device-sync/calendar/values.ts create mode 100644 src/device-sync/calendar/zoned-time.ts diff --git a/src/device-sync/__tests__/calendar/calendars.test.ts b/src/device-sync/__tests__/calendar/calendars.test.ts new file mode 100644 index 00000000..98c82dc3 --- /dev/null +++ b/src/device-sync/__tests__/calendar/calendars.test.ts @@ -0,0 +1,114 @@ +import { describe, expect, it } from 'vitest'; +import { CalendarAccess, Calendars } from '../../android-columns'; +import { calendarPlanner } from '../../calendar/planner'; +import { CAN_PARTIALLY_UPDATE } from '../../calendar/columns'; +import { cssColorToArgb } from '../../calendar/values'; +import { BIRTHDAY_CALENDAR_ID } from '../../../lib/birthday-calendar'; +import type { CalendarLike } from '../../wire'; +import { ACCT, CALENDARS, Harness, ME } from './harness'; + +const personal = (calendar: CalendarLike, readOnly = false) => ({ jmapAccountId: ACCT, calendar, readOnly, accountName: ME }); + +describe('calendar planner: calendar rows', () => { + it('inserts every column the provider and the calendar apps need', async () => { + const h = await new Harness().setup(['b']); + const [row] = await h.calendars(); + expect(row.syncId).toBe(`${ACCT}/b`); + expect(row.cells).toMatchObject({ + [Calendars.NAME]: 'Personal', + [Calendars.CALENDAR_DISPLAY_NAME]: 'Personal', + [Calendars.CALENDAR_COLOR]: cssColorToArgb('#3b82f6'), + [Calendars.CALENDAR_ACCESS_LEVEL]: CalendarAccess.OWNER, + [Calendars.OWNER_ACCOUNT]: ME, + [Calendars.SYNC_EVENTS]: 1, + [Calendars.VISIBLE]: 1, + [Calendars.CALENDAR_TIME_ZONE]: 'Europe/Berlin', + [Calendars.ALLOWED_REMINDERS]: '1,2', + [Calendars.ALLOWED_AVAILABILITY]: '0,1', + [Calendars.ALLOWED_ATTENDEE_TYPES]: '0,1,2,3', + [Calendars.MAX_REMINDERS]: 5, + [Calendars.CAN_ORGANIZER_RESPOND]: 0, + [CAN_PARTIALLY_UPDATE]: 0, + }); + // CAL_SYNC1 would become the `feed` extra of every sync the provider requests for the calendar. + expect(h.fake.row('calendars', row.calendarRowId)).toMatchObject({ [Calendars.CAL_SYNC1]: null, [Calendars.DIRTY]: 0 }); + expect(row.shadow).toEqual(CALENDARS.b); + expect(row.flags).toEqual({}); + }); + + it('writes nothing when nothing changed, and only what changed otherwise', async () => { + const h = await new Harness().setup(['b']); + const local = await h.calendars(); + expect(calendarPlanner.planCalendars([personal(CALENDARS.b)], local, h.ctx)).toEqual({ groups: [], deleteCalendarRows: [] }); + const renamed = { ...CALENDARS.b, name: 'Private', color: 'tomato' } as CalendarLike; + const plan = calendarPlanner.planCalendars([personal(renamed)], local, h.ctx); + expect(plan.groups).toHaveLength(1); + const [op] = plan.groups[0].ops; + expect(op).toMatchObject({ op: 'update', table: 'calendars', id: local[0].calendarRowId, expectCount: 1 }); + expect(Object.keys((op as { values: object }).values).sort()).toEqual( + [Calendars.NAME, Calendars.CALENDAR_DISPLAY_NAME, Calendars.CALENDAR_COLOR, Calendars.CAL_SYNC2].sort(), + ); + // VISIBLE is the user's once the row exists. + expect((op as { values: object }).values).not.toHaveProperty(Calendars.VISIBLE); + }); + + it('keeps VISIBLE as the user set it', async () => { + const h = await new Harness().setup(['b']); + const [row] = await h.calendars(); + await h.apply({ ref: 'x', ops: [{ op: 'update', table: 'calendars', id: row.calendarRowId, values: { [Calendars.VISIBLE]: 0 } }] }); + expect(calendarPlanner.planCalendars([personal(CALENDARS.b)], await h.calendars(), h.ctx).groups).toEqual([]); + }); + + it('maps rights to access levels, and a writable read-only calendar is a subscribed feed', () => { + const level = (rights: CalendarLike['myRights'], readOnly = false) => { + const plan = calendarPlanner.planCalendars([personal({ id: 'z', name: 'Z', myRights: rights } as CalendarLike, readOnly)], [], new Harness().ctx); + const values = (plan.groups[0].ops[0] as { values: Record }).values; + return [values[Calendars.CALENDAR_ACCESS_LEVEL], JSON.parse(String(values[Calendars.CAL_SYNC3]))]; + }; + expect(level({ mayWriteAll: true, mayWriteOwn: true })).toEqual([CalendarAccess.OWNER, {}]); + expect(level({ mayWriteOwn: true })).toEqual([CalendarAccess.CONTRIBUTOR, {}]); + expect(level({ mayRSVP: true }, true)).toEqual([CalendarAccess.RESPOND, { readOnly: 'rights' }]); + expect(level({ mayReadItems: true }, true)).toEqual([CalendarAccess.READ, { readOnly: 'rights' }]); + expect(level({ mayWriteAll: true }, true)).toEqual([CalendarAccess.READ, { readOnly: 'subscription' }]); + }); + + it('offers no reminder editor when Bulwark owns reminders', async () => { + const h = await new Harness({ reminderOwner: 'bulwark' }).setup(['b']); + expect((await h.calendars())[0].cells[Calendars.MAX_REMINDERS]).toBe(0); + h.ctx = { ...h.ctx, reminderOwner: 'device' }; + const plan = calendarPlanner.planCalendars([personal(CALENDARS.b)], await h.calendars(), h.ctx); + expect((plan.groups[0].ops[0] as { values: object }).values).toEqual({ [Calendars.MAX_REMINDERS]: 5 }); + }); + + it('names shared accounts\' calendars after their account and colours calendars without a colour', () => { + const ctx = new Harness().ctx; + const plan = calendarPlanner.planCalendars( + [{ jmapAccountId: 'd', calendar: { id: 'x', name: 'Team', myRights: { mayWriteAll: true } } as CalendarLike, readOnly: false, accountName: 'Bob Builder' }], + [], + ctx, + ); + const values = (plan.groups[0].ops[0] as { values: Record }).values; + expect(values[Calendars._SYNC_ID]).toBe('d/x'); + expect(values[Calendars.CALENDAR_DISPLAY_NAME]).toBe('Team (Bob Builder)'); + expect(typeof values[Calendars.CALENDAR_COLOR]).toBe('number'); + expect(values[Calendars.OWNER_ACCOUNT]).toBe(ME); + }); + + it('deletes rows of deselected calendars and never syncs the birthday calendar', async () => { + const h = await new Harness().setup(['b', 'w']); + const local = await h.calendars(); + const plan = calendarPlanner.planCalendars( + [personal(CALENDARS.b), personal({ id: BIRTHDAY_CALENDAR_ID, name: 'Birthdays' } as CalendarLike)], + local, + h.ctx, + ); + expect(plan.groups).toEqual([]); + expect(plan.deleteCalendarRows).toEqual([local.find((c) => c.syncId === `${ACCT}/w`)!.calendarRowId]); + }); + + it('reads calendar rows defensively', () => { + const cal = calendarPlanner.decodeCalendar({ _id: '4', _sync_id: 'c/b', cal_sync2: 'garbage', cal_sync3: '{"readOnly":"subscription","taskOnly":true}' }); + expect(cal).toMatchObject({ calendarRowId: 4, syncId: 'c/b', shadow: null, flags: { readOnly: 'subscription', taskOnly: true } }); + expect(calendarPlanner.decodeCalendar({ _id: 5, _sync_id: null, cal_sync3: '[1]' }).flags).toEqual({}); + }); +}); diff --git a/src/device-sync/__tests__/calendar/download.test.ts b/src/device-sync/__tests__/calendar/download.test.ts new file mode 100644 index 00000000..ab00eea5 --- /dev/null +++ b/src/device-sync/__tests__/calendar/download.test.ts @@ -0,0 +1,248 @@ +import { describe, expect, it } from 'vitest'; +import { Attendees, Calendars, Events, Reminders } from '../../android-columns'; +import { calendarPlanner } from '../../calendar/planner'; +import { cssColorToArgb } from '../../calendar/values'; +import { ACCT, CALENDARS, Harness, ME, single, utc, weekly } from './harness'; +import type { CalendarEventWire } from '../../wire'; + +describe('calendar planner: downloads', () => { + it('writes a recurring event with its exception, exclusion, attendees and reminders', async () => { + const h = await new Harness().setup(); + const plan = await h.download(weekly()); + expect(plan.effect).toBe('insert'); + const master = h.masterRow('bl'); + expect(master).toMatchObject({ + [Events.TITLE]: 'Probe weekly', + [Events.DESCRIPTION]: 'desc', + [Events.EVENT_LOCATION]: 'Room 1', + [Events.STATUS]: 0, + [Events.AVAILABILITY]: 1, + [Events.ACCESS_LEVEL]: 2, + [Events.EVENT_COLOR]: cssColorToArgb('steelblue'), + [Events.DTSTART]: utc('2026-10-05T13:00:00Z'), + [Events.DTEND]: null, + [Events.DURATION]: 'P5400S', + [Events.EVENT_TIMEZONE]: 'America/New_York', + [Events.ALL_DAY]: 0, + [Events.RRULE]: 'FREQ=WEEKLY;COUNT=10;BYDAY=MO,WE', + [Events.EXDATE]: '20261012T130000Z', + [Events.RDATE]: null, + [Events.UID_2445]: 'probe-rec@example.org', + [Events.ORGANIZER]: ME, + [Events.HAS_ATTENDEE_DATA]: 1, + [Events.DIRTY]: 0, + }); + expect(JSON.parse(String(master[Events.SYNC_DATA1]))).toEqual(weekly()); + expect(master[Events.SELF_ATTENDEE_STATUS] ?? null).toBeNull(); + + const exceptions = h.fake.rows('events', `${Events.ORIGINAL_SYNC_ID} = ?`, [`${ACCT}/bl`]); + expect(exceptions).toHaveLength(1); + expect(exceptions[0]).toMatchObject({ + [Events._SYNC_ID]: `${ACCT}/bl#2026-10-07T09:00:00`, + [Events.SYNC_DATA2]: '2026-10-07T09:00:00', + [Events.ORIGINAL_ID]: master._id, + // Exact to the millisecond, or the provider shows the original instance as well. + [Events.ORIGINAL_INSTANCE_TIME]: utc('2026-10-07T13:00:00Z'), + [Events.ORIGINAL_ALL_DAY]: 0, + [Events.TITLE]: 'Moved Wednesday', + [Events.DTSTART]: utc('2026-10-07T15:00:00Z'), + [Events.DTEND]: utc('2026-10-07T16:30:00Z'), + [Events.DURATION]: null, + [Events.RRULE]: null, + [Events.STATUS]: 0, + }); + + const attendees = h.fake.rows('attendees', `${Attendees.EVENT_ID} = ?`, [Number(master._id)]); + expect(attendees.map((a) => [a[Attendees.ATTENDEE_EMAIL], a[Attendees.ATTENDEE_RELATIONSHIP], a[Attendees.ATTENDEE_STATUS]])).toEqual([ + [ME, 2, 1], + ['guest@example.net', 1, 3], + ]); + const reminders = h.fake.rows('reminders', `${Reminders.EVENT_ID} = ?`, [Number(master._id)]); + // Only the start-relative offset alert has a Reminders row; the others stay on the server. + expect(reminders.map((r) => [r[Reminders.MINUTES], r[Reminders.METHOD]])).toEqual([[15, 1]]); + }); + + it('writes nothing for an echo of what the rows already hold', async () => { + const h = await new Harness().setup(); + await h.download(weekly()); + const batches = h.fake.batches.length; + const plan = calendarPlanner.planDownload(weekly(), await h.local('bl'), h.ctx); + expect(plan).toMatchObject({ effect: 'none', writes: 0, conflicts: 0, stillDirty: false }); + expect(plan.ops.ops).toEqual([]); + expect(h.fake.batches.length).toBe(batches); + }); + + it('writes only what the server changed, behind asserts', async () => { + const h = await new Harness().setup(); + await h.download(weekly()); + const plan = await h.download({ ...weekly(), title: 'Renamed', updated: '2026-09-27T08:00:00Z' }); + expect(plan.effect).toBe('update'); + const ops = plan.ops.ops; + const firstWrite = ops.findIndex((op) => op.op !== 'assert'); + expect(ops.slice(0, firstWrite).every((op) => op.op === 'assert')).toBe(true); + expect(ops.slice(firstWrite).every((op) => op.op !== 'assert')).toBe(true); + const updates = ops.filter((op) => op.op === 'update'); + expect(updates).toHaveLength(1); + expect(Object.keys((updates[0] as { values: object }).values).sort()).toEqual([Events.SYNC_DATA1, Events.SYNC_DATA4, Events.TITLE].sort()); + expect(h.masterRow('bl')[Events.TITLE]).toBe('Renamed'); + }); + + it('updates only the shadow when the server changed a property the device does not show', async () => { + const h = await new Harness().setup(); + await h.download(weekly()); + const plan = await h.download({ ...weekly(), priority: 1, keywords: { other: true } }); + expect(plan).toMatchObject({ effect: 'update', writes: 1 }); + expect(JSON.parse(String(h.masterRow('bl')[Events.SYNC_DATA1])).priority).toBe(1); + }); + + it('keeps floating events on the wall clock of the device zone', async () => { + const h = await new Harness({ deviceZone: 'America/New_York' }).setup(); + await h.download(single({ id: 'f1', timeZone: null, start: '2026-10-06T14:00:00', duration: 'PT45M' })); + expect(h.masterRow('f1')).toMatchObject({ + [Events.EVENT_TIMEZONE]: 'America/New_York', + [Events.DTSTART]: utc('2026-10-06T18:00:00Z'), + [Events.DTEND]: utc('2026-10-06T18:45:00Z'), + }); + }); + + it('writes a timed event in its own zone, not the device one', async () => { + const h = await new Harness({ deviceZone: 'Europe/Berlin' }).setup(); + await h.download(single({ id: 'tk', timeZone: 'Asia/Tokyo', start: '2026-10-06T09:00:00' })); + expect(h.masterRow('tk')).toMatchObject({ [Events.EVENT_TIMEZONE]: 'Asia/Tokyo', [Events.DTSTART]: utc('2026-10-06T00:00:00Z') }); + }); + + it('writes all-day events as UTC midnights, multi-day and in weeks too', async () => { + const h = await new Harness({ deviceZone: 'Pacific/Auckland' }).setup(); + await h.download(single({ id: 'ad', start: '2026-10-08T00:00:00', timeZone: null, showWithoutTime: true, duration: 'P2D' })); + expect(h.masterRow('ad')).toMatchObject({ + [Events.ALL_DAY]: 1, + [Events.EVENT_TIMEZONE]: 'UTC', + [Events.DTSTART]: utc('2026-10-08T00:00:00Z'), + [Events.DTEND]: utc('2026-10-10T00:00:00Z'), + }); + // CalDAV DTENDs arrive as weeks. + await h.download(single({ id: 'wk', start: '2026-10-12T00:00:00', timeZone: null, showWithoutTime: true, duration: 'P1W' })); + expect(h.masterRow('wk')[Events.DTEND]).toBe(utc('2026-10-19T00:00:00Z')); + // A recurring all-day event gets a DURATION in days (the provider throws on seconds). + await h.download(single({ id: 'ar', start: '2026-10-12T00:00:00', timeZone: null, showWithoutTime: true, duration: 'P1D', recurrenceRule: { frequency: 'yearly' } })); + expect(h.masterRow('ar')).toMatchObject({ [Events.DURATION]: 'P1D', [Events.DTEND]: null, [Events.RRULE]: 'FREQ=YEARLY' }); + // A time part on an all-day duration rounds up. + await h.download(single({ id: 'ap', start: '2026-10-12T00:00:00', timeZone: null, showWithoutTime: true, duration: 'P1DT2H' })); + expect(h.masterRow('ap')[Events.DTEND]).toBe(utc('2026-10-14T00:00:00Z')); + }); + + it('writes the calendar defaults as reminders when the event uses them', async () => { + const h = await new Harness().setup(); + await h.download(single({ id: 'da', useDefaultAlerts: true, alerts: { own: { '@type': 'Alert', trigger: { '@type': 'OffsetTrigger', offset: '-PT1H' } } } })); + const master = h.masterRow('da'); + const reminders = h.fake.rows('reminders', `${Reminders.EVENT_ID} = ?`, [Number(master._id)]); + expect(reminders.map((r) => r[Reminders.MINUTES])).toEqual([10]); + }); + + it('writes no reminders when Bulwark owns them, and leaves the ones an app adds', async () => { + const h = await new Harness({ reminderOwner: 'bulwark' }).setup(); + const cal = (await h.calendars()).find((c) => c.syncId === `${ACCT}/b`)!; + expect(cal.cells[Calendars.MAX_REMINDERS]).toBe(0); + await h.download(weekly()); + const master = h.masterRow('bl'); + expect(h.fake.rows('reminders', `${Reminders.EVENT_ID} = ?`, [Number(master._id)])).toHaveLength(0); + h.fake.user.setReminders(Number(master._id), [{ minutes: 5 }]); + // Neither uploaded nor deleted: the edit only clears DIRTY. + const upload = calendarPlanner.planUpload((await h.local('bl'))!, h.ctx); + expect(upload.kind).toBe('clean'); + if (upload.kind === 'clean') await h.apply(upload.ops); + await h.download({ ...weekly(), title: 'Again' }); + expect(h.fake.rows('reminders', `${Reminders.EVENT_ID} = ?`, [Number(master._id)])).toHaveLength(1); + }); + + it('normalises the user\'s own address, under any alias, to OWNER_ACCOUNT', async () => { + const h = await new Harness().setup(); + const event = single({ + id: 'inv', + organizerCalendarAddress: 'mailto:boss@example.net', + participants: { + boss: { calendarAddress: 'mailto:boss@example.net', roles: { owner: true } }, + me: { calendarAddress: 'mailto:ALIAS@Example.org', roles: { attendee: true }, participationStatus: 'tentative' }, + room: { calendarAddress: 'mailto:room@example.net', kind: 'resource', roles: { attendee: true } }, + opt: { calendarAddress: 'mailto:opt@example.net', roles: { attendee: true, optional: true }, participationStatus: 'declined' }, + noMail: { name: 'Phone only', roles: { attendee: true } }, + }, + } as Partial); + await h.download(event); + const master = h.masterRow('inv'); + expect(master[Events.ORGANIZER]).toBe('boss@example.net'); + const rows = h.fake.rows('attendees', `${Attendees.EVENT_ID} = ?`, [Number(master._id)]); + expect(rows.map((a) => [a[Attendees.ATTENDEE_EMAIL], a[Attendees.ATTENDEE_RELATIONSHIP], a[Attendees.ATTENDEE_TYPE], a[Attendees.ATTENDEE_STATUS]])).toEqual([ + ['boss@example.net', 2, 1, 3], + [ME, 1, 1, 4], + ['room@example.net', 1, 3, 3], + ['opt@example.net', 1, 2, 2], + ]); + }); + + it('leaves ORGANIZER to the provider for events without participants', async () => { + const h = await new Harness().setup(); + const plan = await h.download(single()); + const insert = plan.ops.ops.find((op) => op.op === 'insert' && op.table === 'events') as { values: Record }; + expect(Events.ORGANIZER in insert.values).toBe(false); + expect(Events.SELF_ATTENDEE_STATUS in insert.values).toBe(false); + expect(h.masterRow('e1')[Events.ORGANIZER]).toBe(ME); + // The baseline predicted the provider's value, so there is nothing to heal and an echo writes nothing. + expect(calendarPlanner.planBaselineHeal((await h.local('e1'))!)).toBeNull(); + expect(calendarPlanner.planDownload(single(), await h.local('e1'), h.ctx).effect).toBe('none'); + }); + + it('never writes a NULL STATUS', async () => { + const h = await new Harness().setup(); + await h.download(single({ status: undefined })); + expect(h.masterRow('e1')[Events.STATUS]).toBe(1); + await h.download(single({ status: 'cancelled' })); + expect(h.masterRow('e1')[Events.STATUS]).toBe(2); + }); + + it('removes and adds exception rows as overrides come and go', async () => { + const h = await new Harness().setup(); + await h.download(weekly()); + const next = weekly(); + next.recurrenceOverrides = { + '2026-10-12T09:00:00': { excluded: true }, + '2026-10-14T09:00:00': { excluded: true }, + '2026-10-19T09:00:00': { title: 'Other' }, + }; + const plan = await h.download(next); + expect(plan.effect).toBe('update'); + const exceptions = h.fake.rows('events', `${Events.ORIGINAL_SYNC_ID} = ?`, [`${ACCT}/bl`]); + expect(exceptions.map((x) => x[Events.SYNC_DATA2])).toEqual(['2026-10-19T09:00:00']); + expect(exceptions[0][Events.TITLE]).toBe('Other'); + expect(h.masterRow('bl')[Events.EXDATE]).toBe('20261012T130000Z,20261014T130000Z'); + }); + + it('replaces the rows of a different object under a reused id', async () => { + const h = await new Harness().setup(); + await h.download(weekly()); + const other = single({ id: 'bl', uid: 'other-uid', title: 'Someone else' }); + const plan = await h.download(other); + expect(plan.effect).toBe('update'); + const events = await h.events(); + expect(events).toHaveLength(1); + expect(events[0]).toMatchObject({ syncId: `${ACCT}/bl`, exceptions: [] }); + expect(events[0].cells[Events.TITLE]).toBe('Someone else'); + }); + + it('refuses tasks', async () => { + const h = await new Harness().setup(); + expect(() => calendarPlanner.planDownload({ ...single(), '@type': 'Task' } as CalendarEventWire, null, h.ctx)).toThrow(/tasks/); + expect(() => calendarPlanner.planDownload({ ...single(), '@type': undefined, due: '2026-10-01T00:00:00' } as unknown as CalendarEventWire, null, h.ctx)).toThrow(/tasks/); + }); + + it('places an event in several calendars in the first selected one, and keeps it where it is', async () => { + const h = await new Harness().setup(); + await h.download(single({ calendarIds: { w: true, b: true } })); + expect(h.masterRow('e1')[Events.CALENDAR_ID]).toBe(h.rowIds.get('b')); + await h.download(single({ calendarIds: { w: true } })); + expect(h.masterRow('e1')[Events.CALENDAR_ID]).toBe(h.rowIds.get('w')); + await h.download(single({ calendarIds: { w: true, b: true }, title: 'x' })); + expect(h.masterRow('e1')[Events.CALENDAR_ID]).toBe(h.rowIds.get('w')); + expect(CALENDARS.w.name).toBe('Work'); + }); +}); diff --git a/src/device-sync/__tests__/calendar/duration.test.ts b/src/device-sync/__tests__/calendar/duration.test.ts new file mode 100644 index 00000000..e8affbc2 --- /dev/null +++ b/src/device-sync/__tests__/calendar/duration.test.ts @@ -0,0 +1,81 @@ +import { describe, expect, it } from 'vitest'; +import { + addDuration, + durationBetween, + durationTextSeconds, + formatAospDuration, + formatJsDuration, + formatJsDurationSeconds, + parseDuration, + sameDurationLength, +} from '../../calendar/duration'; +import { aospDurationValid } from '../fakes/fake-provider'; +import { localToUtc } from '../../calendar/zoned-time'; + +describe('calendar/duration', () => { + it('parses JSCalendar, iCalendar and AOSP durations', () => { + expect(parseDuration('PT1H30M')).toEqual({ negative: false, days: 0, seconds: 5400 }); + expect(parseDuration('P1W')).toEqual({ negative: false, days: 7, seconds: 0 }); + expect(parseDuration('P1W2D')).toEqual({ negative: false, days: 9, seconds: 0 }); + expect(parseDuration('P1DT2H')).toEqual({ negative: false, days: 1, seconds: 7200 }); + expect(parseDuration('-PT15M')).toEqual({ negative: true, days: 0, seconds: 900 }); + expect(parseDuration('+PT15M')).toEqual({ negative: false, days: 0, seconds: 900 }); + expect(parseDuration('PT1.5S')).toEqual({ negative: false, days: 0, seconds: 1.5 }); + expect(parseDuration('pt10m')).toEqual({ negative: false, days: 0, seconds: 600 }); + // AOSP: no T before seconds, M always minutes, Fossify's padded form. + expect(parseDuration('P3600S')).toEqual({ negative: false, days: 0, seconds: 3600 }); + expect(parseDuration('P0DT1H30M0S')).toEqual({ negative: false, days: 0, seconds: 5400 }); + for (const bad of ['P', 'PT', 'P1Y', 'P1.5D', '1H', '', 'PTH']) expect(parseDuration(bad)).toBeNull(); + expect(durationTextSeconds('P1W')).toBe(604_800); + }); + + it('writes the strict JSCalendar grammar', () => { + expect(formatJsDuration(0, 0)).toBe('PT0S'); + expect(formatJsDuration(2, 0)).toBe('P2D'); + expect(formatJsDuration(1, 3600)).toBe('P1DT1H'); + expect(formatJsDuration(0, 5400)).toBe('PT1H30M'); + // Minutes stay between hours and seconds. + expect(formatJsDuration(0, 3605)).toBe('PT1H0M5S'); + expect(formatJsDuration(0, 45)).toBe('PT45S'); + expect(formatJsDuration(0, 59.6)).toBe('PT1M'); + expect(formatJsDurationSeconds(90_000)).toBe('P1DT1H'); + expect(formatJsDuration(-1, -5)).toBe('PT0S'); + }); + + it('writes only what CalendarProvider accepts', () => { + expect(formatAospDuration(5400, false)).toBe('P5400S'); + expect(formatAospDuration(0, false)).toBe('P0S'); + expect(formatAospDuration(172_800, true)).toBe('P2D'); + // A time part on an all-day event is rounded up to whole days; never zero days. + expect(formatAospDuration(90_000, true)).toBe('P2D'); + expect(formatAospDuration(0, true)).toBe('P1D'); + for (const s of [0, 1, 59, 3600, 86_400, 1_000_000]) { + expect(aospDurationValid(formatAospDuration(s, false))).toBe(true); + expect(formatAospDuration(s, true)).toMatch(/^P\d+D$/); + } + }); + + it('counts whole days nominally in the zone and the rest exactly', () => { + const berlin = 'Europe/Berlin'; + // 23 h across the spring change are still one day. + const start = localToUtc('2026-03-28T12:00:00', berlin)!; + const end = localToUtc('2026-03-29T12:00:00', berlin)!; + expect(end - start).toBe(23 * 3_600_000); + expect(durationBetween(start, end, berlin)).toEqual({ negative: false, days: 1, seconds: 0 }); + expect(addDuration(start, { negative: false, days: 1, seconds: 0 }, berlin)).toBe(end); + expect(durationBetween(start, end + 1_800_000, berlin)).toEqual({ negative: false, days: 1, seconds: 1800 }); + expect(durationBetween(start, start + 5_400_000, berlin)).toEqual({ negative: false, days: 0, seconds: 5400 }); + expect(durationBetween(end, start, berlin)).toEqual({ negative: false, days: 0, seconds: 0 }); + }); + + it('keeps the second instant of a repeated hour when adding exact time', () => { + const second = Date.parse('2026-10-25T01:30:00Z'); // 02:30 CET, after the change + expect(addDuration(second, { negative: false, days: 0, seconds: 3600 }, 'Europe/Berlin')).toBe(second + 3_600_000); + }); + + it('compares lengths with a day as 24 hours', () => { + expect(sameDurationLength('PT24H', 'P1D')).toBe(true); + expect(sameDurationLength('P3600S', 'PT1H')).toBe(true); + expect(sameDurationLength('PT1H', 'PT2H')).toBe(false); + }); +}); diff --git a/src/device-sync/__tests__/calendar/echo.test.ts b/src/device-sync/__tests__/calendar/echo.test.ts new file mode 100644 index 00000000..daac115e --- /dev/null +++ b/src/device-sync/__tests__/calendar/echo.test.ts @@ -0,0 +1,66 @@ +/** + * Upload → server → accepted plan → the server's echo: after every kind of + * device edit, the rows must already be what the server's next download + * describes, so that echo writes nothing (docs/device-sync.md, invariant 4). + */ +import { describe, expect, it } from 'vitest'; +import { Attendees, Events } from '../../android-columns'; +import { calendarPlanner } from '../../calendar/planner'; +import type { CalendarEventWire } from '../../wire'; +import { ACCT, Harness, ME, serverApply, single, utc, weekly } from './harness'; + +async function roundTrip(event: CalendarEventWire, edit: (h: Harness, id: number) => void | Promise) { + const h = await new Harness().setup(); + await h.download(event); + const id = Number(h.masterRow(event.id)._id); + await edit(h, id); + const local = (await h.local(event.id))!; + const plan = calendarPlanner.planUpload(local, h.ctx); + if (plan.kind !== 'upload' || plan.actions[0].kind !== 'update') throw new Error(`expected an update: ${JSON.stringify(plan)}`); + const server = { ...serverApply(event, plan.actions[0].patch), updated: '2026-09-27T13:00:00Z' }; + await h.apply(calendarPlanner.planAccepted(local, server, h.ctx).ops); + const after = (await h.local(event.id))!; + return { h, id, server, after, patch: plan.actions[0].patch }; +} + +const EDITS: Array<[string, () => CalendarEventWire, (h: Harness, id: number) => void | Promise]> = [ + ['a title', weekly, (h, id) => h.fake.user.updateEvent(id, { [Events.TITLE]: 'T' })], + ['a start of a single event', () => single(), (h, id) => h.fake.user.updateEvent(id, { [Events.DTSTART]: utc('2026-10-06T11:00:00Z'), [Events.DTEND]: utc('2026-10-06T12:00:00Z') })], + ['a DST-ambiguous start (fixed-offset zone)', () => single(), (h, id) => h.fake.user.updateEvent(id, { [Events.DTSTART]: utc('2026-10-25T00:30:00Z'), [Events.DTEND]: utc('2026-10-25T01:30:00Z') })], + ['an "all events" move', weekly, (h, id) => h.fake.user.updateEvent(id, { [Events.DTSTART]: utc('2026-10-05T14:00:00Z') })], + ['a "this event" edit', weekly, (h, id) => void h.fake.user.insertException(id, utc('2026-10-14T13:00:00Z'), { [Events.TITLE]: 'Only this', [Events.DTSTART]: utc('2026-10-14T13:00:00Z'), [Events.DTEND]: utc('2026-10-14T14:30:00Z') })], + ['a cancelled instance', weekly, (h, id) => void h.fake.user.cancelInstance(id, utc('2026-10-19T13:00:00Z'), utc('2026-10-19T14:30:00Z'))], + ['a capped rule', weekly, (h, id) => h.fake.user.updateEvent(id, { [Events.RRULE]: 'FREQ=WEEKLY;UNTIL=20261010T000000Z;BYDAY=MO,WE' })], + ['an RSVP', weekly, (h, id) => h.fake.user.setAttendeeStatus(id, ME, 4)], + ['an added attendee', weekly, (h, id) => void h.fake.user.addAttendee(id, { [Attendees.ATTENDEE_EMAIL]: 'x@example.net', [Attendees.ATTENDEE_TYPE]: 1, [Attendees.ATTENDEE_STATUS]: 3, [Attendees.ATTENDEE_RELATIONSHIP]: 1 })], + ['reminders', weekly, (h, id) => h.fake.user.setReminders(id, [{ minutes: 5 }, { minutes: 1440, method: 2 }])], + ['a location on an event without one', () => single(), (h, id) => h.fake.user.updateEvent(id, { [Events.EVENT_LOCATION]: 'Hall' })], + ['an all-day event\'s dates', () => single({ id: 'a', start: '2026-10-08T00:00:00', timeZone: null, showWithoutTime: true, duration: 'P1D' }), (h, id) => h.fake.user.updateEvent(id, { [Events.DTSTART]: utc('2026-10-09T00:00:00Z'), [Events.DTEND]: utc('2026-10-12T00:00:00Z') })], +]; + +describe('calendar planner: accepted uploads leave nothing for the echo', () => { + it.each(EDITS)('%s', async (_name, make, edit) => { + const { h, server, after } = await roundTrip(make(), edit); + expect(after.dirty).toBe(false); + expect(after.exceptions.every((x) => !x.dirty)).toBe(true); + const echo = calendarPlanner.planDownload(server, after, h.ctx); + expect(echo).toMatchObject({ effect: 'none', writes: 0 }); + expect(calendarPlanner.planBaselineHeal(after)).toBeNull(); + }); + + it('re-keys the exception rows with the series after an "all events" move', async () => { + const { after, patch } = await roundTrip(weekly(), (h, id) => h.fake.user.updateEvent(id, { [Events.DTSTART]: utc('2026-10-05T14:00:00Z') })); + expect(Object.keys(patch.recurrenceOverrides as object).sort()).toEqual(['2026-10-07T10:00:00', '2026-10-12T10:00:00']); + expect(after.exceptions.map((x) => [x.syncId, x.cells[Events.ORIGINAL_INSTANCE_TIME]])).toEqual([ + [`${ACCT}/bl#2026-10-07T10:00:00`, utc('2026-10-07T14:00:00Z')], + ]); + }); + + it('drops the exception rows of overrides a "this and following" cut pruned', async () => { + const event = weekly(); + event.recurrenceRule = { frequency: 'weekly', byDay: [{ day: 'mo' }, { day: 'we' }] }; + event.recurrenceOverrides = { '2026-10-07T09:00:00': { title: 'Early' }, '2026-10-21T09:00:00': { title: 'Late' } }; + const { after } = await roundTrip(event, (h, id) => h.fake.user.updateEvent(id, { [Events.RRULE]: 'FREQ=WEEKLY;UNTIL=20261019T125959Z;BYDAY=MO,WE' })); + expect(after.exceptions.map((x) => x.recurrenceId)).toEqual(['2026-10-07T09:00:00']); + }); +}); diff --git a/src/device-sync/__tests__/calendar/edge-cases.test.ts b/src/device-sync/__tests__/calendar/edge-cases.test.ts new file mode 100644 index 00000000..b32cb6e4 --- /dev/null +++ b/src/device-sync/__tests__/calendar/edge-cases.test.ts @@ -0,0 +1,85 @@ +import { describe, expect, it } from 'vitest'; +import { Events } from '../../android-columns'; +import { calendarPlanner } from '../../calendar/planner'; +import type { CalendarEventWire } from '../../wire'; +import { ACCT, Harness, single, utc, weekly } from './harness'; + +async function synced(event: CalendarEventWire) { + const h = await new Harness().setup(); + await h.download(event); + return { h, id: Number(h.masterRow(event.id)._id) }; +} + +describe('calendar planner: edge cases', () => { + it('undoes a deletion in a read-only calendar by purging and refetching', async () => { + const { h, id } = await synced(single({ calendarIds: { r: true } })); + h.fake.user.deleteEvent(id); + const plan = calendarPlanner.planUpload((await h.local('e1'))!, h.ctx); + expect(plan).toMatchObject({ kind: 'revert', refetch: true }); + if (plan.kind === 'revert') await h.apply(plan.ops); + expect(await h.events()).toEqual([]); + }); + + it('treats an instance Stalwart holds without its series as read-only', async () => { + const { h, id } = await synced(single({ recurrenceId: '2026-10-06T12:00:00', recurrenceIdTimeZone: 'Europe/Berlin' })); + h.fake.user.updateEvent(id, { [Events.TITLE]: 'Mine' }); + const plan = calendarPlanner.planUpload((await h.local('e1'))!, h.ctx); + expect(plan.kind).toBe('revert'); + if (plan.kind === 'revert') await h.apply(plan.ops); + expect(h.row(id)).toMatchObject({ [Events.TITLE]: 'Lunch', [Events.DIRTY]: 0 }); + }); + + it('pairs Etar\'s move of a series with its copied exceptions and adopts them', async () => { + const { h, id } = await synced(weekly()); + h.fake.user.deleteEvent(id); + const fresh = h.fake.user.insertEvent(h.rowIds.get('w')!, { + [Events.TITLE]: 'Probe weekly', [Events.DESCRIPTION]: 'desc', [Events.EVENT_LOCATION]: 'Room 1', [Events.DTSTART]: utc('2026-10-05T13:00:00Z'), + [Events.DURATION]: 'P5400S', [Events.RRULE]: 'FREQ=WEEKLY;COUNT=10;BYDAY=MO,WE', [Events.EVENT_TIMEZONE]: 'America/New_York', + [Events.STATUS]: 0, [Events.AVAILABILITY]: 1, [Events.ACCESS_LEVEL]: 2, [Events.EVENT_COLOR]: h.row(id)[Events.EVENT_COLOR], + }); + for (const a of h.fake.rows('attendees', 'event_id = ?', [id])) { + h.fake.table('attendees').set(h.fake.allocId(), { ...a, _id: null, event_id: fresh }); + } + // Etar copies the exception too (API 30+). + h.fake.user.insertException(fresh, utc('2026-10-07T13:00:00Z'), { [Events.TITLE]: 'Moved Wednesday', [Events.DTSTART]: utc('2026-10-07T15:00:00Z'), [Events.DTEND]: utc('2026-10-07T16:30:00Z') }); + const events = await h.events(); + const pairs = calendarPlanner.planPairs(events.filter((e) => e.deleted), events.filter((e) => !e.syncId && !e.deleted), h.ctx); + expect(pairs.map((p) => p.actions)).toEqual([[{ kind: 'update', id: 'bl', patch: { 'calendarIds/b': null, 'calendarIds/w': true } }]]); + await h.apply(pairs[0].ops); + const after = (await h.local('bl'))!; + expect(after.eventId).toBe(fresh); + expect(after.exceptions.map((x) => [x.syncId, x.dirty])).toEqual([[`${ACCT}/bl#2026-10-07T09:00:00`, false]]); + // The old row and its own exception are gone. + expect((await h.events()).map((e) => e.eventId)).toEqual([fresh]); + }); + + it('compares descriptions over 1 KB by hash and uploads them only when edited', async () => { + const long = 'Lorem ipsum '.repeat(200); + const { h, id } = await synced(single({ description: long })); + const local = (await h.local('e1'))!; + expect(String(local.baseline!.cells[Events.DESCRIPTION])).toMatch(/^sha256:[0-9a-f]{64}$/); + h.fake.user.updateEvent(id, {}); + expect(calendarPlanner.planUpload((await h.local('e1'))!, h.ctx).kind).toBe('clean'); + h.fake.user.updateEvent(id, { [Events.DESCRIPTION]: `${long}!` }); + const plan = calendarPlanner.planUpload((await h.local('e1'))!, h.ctx); + expect(plan.kind === 'upload' && plan.actions[0].kind === 'update' && plan.actions[0].patch).toEqual({ description: `${long}!` }); + }); + + it('writes nothing into a split clone on download: it is a new event', async () => { + const event = single({ recurrenceRule: { frequency: 'daily' } }); + const { h, id } = await synced(event); + const clone = h.fake.user.splitViaUri(id, utc('2026-10-10T10:00:00Z'), '20261010T095959Z', {}); + const cloneEvent = (await h.events()).find((e) => e.eventId === clone)!; + expect(calendarPlanner.planDownload({ ...event, title: 'x' }, cloneEvent, h.ctx)).toMatchObject({ effect: 'none', stillDirty: true }); + }); + + it('keeps an exception row an app left without a description or attendees inheriting them', async () => { + const { h, id } = await synced(weekly()); + h.fake.user.insertException(id, utc('2026-10-14T13:00:00Z'), { [Events.DTSTART]: utc('2026-10-14T13:00:00Z'), [Events.DTEND]: utc('2026-10-14T14:30:00Z'), [Events.EVENT_LOCATION]: null }); + const plan = calendarPlanner.planUpload((await h.local('bl'))!, h.ctx); + if (plan.kind !== 'upload' || plan.actions[0].kind !== 'update') throw new Error(); + const override = plan.actions[0].patch['recurrenceOverrides/2026-10-14T09:00:00'] as Record; + // The series' description, location and attendees stay on the instance. + expect(override).toMatchObject({ description: 'desc', locations: weekly().locations, participants: weekly().participants }); + }); +}); diff --git a/src/device-sync/__tests__/calendar/exceptions.test.ts b/src/device-sync/__tests__/calendar/exceptions.test.ts new file mode 100644 index 00000000..540f73a6 --- /dev/null +++ b/src/device-sync/__tests__/calendar/exceptions.test.ts @@ -0,0 +1,355 @@ +import { describe, expect, it } from 'vitest'; +import { Attendees, Events } from '../../android-columns'; +import { calendarPlanner } from '../../calendar/planner'; +import type { LocalEvent, UploadPlan } from '../../planner'; +import type { CalendarEventWire } from '../../wire'; +import { ACCT, Harness, ME, serverApply, single, utc, weekly } from './harness'; + +type Plan = UploadPlan; + +function patchOf(plan: Plan) { + expect(plan.kind).toBe('upload'); + if (plan.kind !== 'upload' || plan.actions[0].kind !== 'update') throw new Error(JSON.stringify(plan)); + return plan.actions[0]; +} + +async function synced(event: CalendarEventWire) { + const h = await new Harness().setup(); + await h.download(event); + return { h, id: Number(h.masterRow(event.id)._id) }; +} + +/** Uploads, lets the server apply the patch, applies the accepted plan and returns the new server object. */ +async function accept(h: Harness, event: CalendarEventWire, local: LocalEvent, plan: Plan): Promise { + const action = patchOf(plan); + const server = serverApply(event, action.patch); + const accepted = calendarPlanner.planAccepted(local, server, h.ctx); + await h.apply(accepted.ops); + return server; +} + +const NO_FORBIDDEN = ['@type', 'organizerCalendarAddress', 'privacy', 'uid', 'recurrenceRule', 'recurrenceOverrides', 'recurrenceId', 'method', 'prodId', 'sentBy']; + +describe('calendar planner: recurrence edits made on the device', () => { + it('uploads an Etar "this event" edit as a new override carrying the master\'s details', async () => { + const { h, id } = await synced(weekly()); + h.fake.user.insertException(id, utc('2026-10-14T13:00:00Z'), { + [Events.TITLE]: 'Only this Wednesday', + [Events.DTSTART]: utc('2026-10-14T14:00:00Z'), + [Events.DTEND]: utc('2026-10-14T15:30:00Z'), + }); + const local = (await h.local('bl'))!; + expect(local.exceptions.find((x) => !x.cells[Events.SYNC_DATA2])?.recurrenceId).toBe('2026-10-14T09:00:00'); + const plan = calendarPlanner.planUpload(local, h.ctx); + const action = patchOf(plan); + expect(Object.keys(action.patch)).toEqual(['recurrenceOverrides/2026-10-14T09:00:00']); + const override = action.patch['recurrenceOverrides/2026-10-14T09:00:00'] as Record; + expect(override).toMatchObject({ + title: 'Only this Wednesday', + start: '2026-10-14T10:00:00', + duration: 'PT1H30M', + locations: weekly().locations, + participants: weekly().participants, + description: 'desc', + }); + for (const key of NO_FORBIDDEN) expect(override).not.toHaveProperty(key); + // The organizer changed an instance: attendees are told. + expect(action.sendSchedulingMessages).toBe(true); + + const server = await accept(h, weekly(), local, plan); + const after = (await h.local('bl'))!; + expect(after.dirty).toBe(false); + expect(after.exceptions.map((x) => [x.syncId, x.dirty])).toEqual([ + [`${ACCT}/bl#2026-10-07T09:00:00`, false], + [`${ACCT}/bl#2026-10-14T09:00:00`, false], + ]); + // The download of our own change writes nothing. + expect(calendarPlanner.planDownload(server, after, h.ctx).effect).toBe('none'); + }); + + it('sends the whole overrides map for the first override of a series', async () => { + const event = single({ recurrenceRule: { frequency: 'daily', count: 5 } }); + const { h, id } = await synced(event); + h.fake.user.insertException(id, utc('2026-10-07T10:00:00Z'), { [Events.TITLE]: 'Tuesday lunch', [Events.DTSTART]: utc('2026-10-07T10:00:00Z'), [Events.DTEND]: utc('2026-10-07T11:00:00Z') }); + const action = patchOf(calendarPlanner.planUpload((await h.local('e1'))!, h.ctx)); + expect(Object.keys(action.patch)).toEqual(['recurrenceOverrides']); + expect(action.patch.recurrenceOverrides).toEqual({ + '2026-10-07T12:00:00': { title: 'Tuesday lunch', start: '2026-10-07T12:00:00', duration: 'PT1H' }, + }); + }); + + it('uploads "delete this event" (a cancelled exception) as an exclusion and cleans up after', async () => { + const { h, id } = await synced(weekly()); + h.fake.user.cancelInstance(id, utc('2026-10-14T13:00:00Z'), utc('2026-10-14T14:30:00Z')); + const local = (await h.local('bl'))!; + const plan = calendarPlanner.planUpload(local, h.ctx); + expect(patchOf(plan).patch).toEqual({ 'recurrenceOverrides/2026-10-14T09:00:00': { excluded: true } }); + const server = await accept(h, weekly(), local, plan); + expect(h.masterRow('bl')[Events.EXDATE]).toBe('20261012T130000Z,20261014T130000Z'); + const after = (await h.local('bl'))!; + expect(after.exceptions.map((x) => x.recurrenceId)).toEqual(['2026-10-07T09:00:00']); + expect(calendarPlanner.planDownload(server, after, h.ctx).effect).toBe('none'); + }); + + it('excludes an instance whose synced exception row was deleted', async () => { + const { h } = await synced(weekly()); + const x = (await h.local('bl'))!.exceptions[0]; + h.fake.user.deleteEvent(x.eventId); + expect(h.row(x.eventId)).toMatchObject({ [Events.DELETED]: 1, [Events.DIRTY]: 1 }); + const local = (await h.local('bl'))!; + const plan = calendarPlanner.planUpload(local, h.ctx); + expect(patchOf(plan).patch).toEqual({ 'recurrenceOverrides/2026-10-07T09:00:00': { excluded: true } }); + await accept(h, weekly(), local, plan); + expect(h.row(x.eventId)).toBeUndefined(); + }); + + it('patches an existing override one level deep, never deeper', async () => { + const { h } = await synced(weekly()); + const x = (await h.local('bl'))!.exceptions[0]; + h.fake.user.updateEvent(x.eventId, { [Events.TITLE]: 'Moved again', [Events.EVENT_LOCATION]: 'Annex' }); + const action = patchOf(calendarPlanner.planUpload((await h.local('bl'))!, h.ctx)); + expect(action.patch).toEqual({ + 'recurrenceOverrides/2026-10-07T09:00:00/title': 'Moved again', + 'recurrenceOverrides/2026-10-07T09:00:00/locations': { locA: { name: 'Annex', '@type': 'Location' } }, + }); + }); + + it('answers one instance (CONTENT_EXCEPTION_URI) with the override\'s whole participants map', async () => { + const invite = weekly(); + invite.organizerCalendarAddress = 'mailto:boss@example.net'; + invite.participants = { + boss: { calendarAddress: 'mailto:boss@example.net', roles: { owner: true }, participationStatus: 'accepted' }, + me: { calendarAddress: 'mailto:usera@example.org', roles: { attendee: true }, participationStatus: 'needs-action' }, + other: { calendarAddress: 'mailto:other@example.net', roles: { attendee: true }, participationStatus: 'accepted' }, + }; + const { h, id } = await synced(invite); + h.fake.user.exceptionViaUri(id, utc('2026-10-19T13:00:00Z'), { [Events.SELF_ATTENDEE_STATUS]: 1, [Events.STATUS]: 1 }); + const action = patchOf(calendarPlanner.planUpload((await h.local('bl'))!, h.ctx)); + const override = action.patch['recurrenceOverrides/2026-10-19T09:00:00'] as Record; + expect(override.participants).toEqual({ ...invite.participants, me: { ...invite.participants!.me, participationStatus: 'accepted' } }); + // Etar's STATUS_CONFIRMED on the instance is not the attendee's to upload: the series' status is kept. + expect(override).toMatchObject({ status: 'tentative', start: '2026-10-19T09:00:00', duration: 'PT1H30M', title: 'Probe weekly' }); + for (const key of NO_FORBIDDEN) expect(override).not.toHaveProperty(key); + // An attendee's RSVP sends the REPLY. + expect(action.sendSchedulingMessages).toBe(true); + }); + + it('turns EXDATE entries an app adds or removes into exclusions and re-inclusions', async () => { + const { h, id } = await synced(weekly()); + h.fake.user.updateEvent(id, { [Events.EXDATE]: '20261012T130000Z\nAmerica/New_York;20261021T090000' }); + expect(patchOf(calendarPlanner.planUpload((await h.local('bl'))!, h.ctx)).patch).toEqual({ + 'recurrenceOverrides/2026-10-21T09:00:00': { excluded: true }, + }); + h.fake.user.updateEvent(id, { [Events.EXDATE]: null }); + expect(patchOf(calendarPlanner.planUpload((await h.local('bl'))!, h.ctx)).patch).toEqual({ + 'recurrenceOverrides/2026-10-12T09:00:00': null, + }); + }); + + it('uploads Etar\'s "this and following" as a capped rule, prunes later overrides, and creates the new series', async () => { + const event = weekly(); + event.recurrenceRule = { frequency: 'weekly', byDay: [{ day: 'mo' }, { day: 'we' }] }; + event.recurrenceOverrides = { + '2026-10-07T09:00:00': { title: 'Moved Wednesday', start: '2026-10-07T11:00:00' }, + '2026-10-21T09:00:00': { title: 'Later one' }, + '2026-10-26T09:00:00': { excluded: true }, + }; + const { h, id } = await synced(event); + // Split at Mon 2026-10-19 09:00 New York: UNTIL = the instance minus one second, in UTC. + h.fake.user.updateEvent(id, { [Events.RRULE]: 'FREQ=WEEKLY;UNTIL=20261019T125959Z;BYDAY=MO,WE' }); + const fresh = h.fake.user.insertEvent(h.rowIds.get('b')!, { + [Events.TITLE]: 'Probe weekly (new time)', + [Events.DTSTART]: utc('2026-10-19T14:00:00Z'), + [Events.DURATION]: 'P5400S', + [Events.RRULE]: 'FREQ=WEEKLY;BYDAY=MO,WE', + [Events.EVENT_TIMEZONE]: 'America/New_York', + [Events.STATUS]: 1, + }); + const action = patchOf(calendarPlanner.planUpload((await h.local('bl'))!, h.ctx)); + expect(action.patch).toEqual({ + recurrenceRule: { frequency: 'weekly', until: '2026-10-19T08:59:59', byDay: [{ day: 'mo' }, { day: 'we' }] }, + 'recurrenceOverrides/2026-10-21T09:00:00': null, + 'recurrenceOverrides/2026-10-26T09:00:00': null, + }); + + // The new series is a new event: claimed with a fresh uid, then created. + const newRow = await h.byRow(fresh); + const claim = calendarPlanner.planUpload(newRow, h.ctx); + expect(claim.kind).toBe('claim'); + if (claim.kind === 'claim') await h.apply(claim.ops); + const claimed = await h.byRow(fresh); + expect(claimed.syncId).toMatch(/^~pending\//); + expect(claimed.pending).toEqual({ uid: claimed.cells[Events.UID_2445], target: `${ACCT}/b` }); + const create = calendarPlanner.planUpload(claimed, h.ctx); + expect(create.kind).toBe('upload'); + if (create.kind !== 'upload' || create.actions[0].kind !== 'create') throw new Error('no create'); + expect(create.actions[0]).toMatchObject({ + kind: 'create', + uid: claimed.pending!.uid, + collectionId: 'b', + object: { + '@type': 'Event', + uid: claimed.pending!.uid, + calendarIds: { b: true }, + title: 'Probe weekly (new time)', + start: '2026-10-19T10:00:00', + timeZone: 'America/New_York', + duration: 'PT1H30M', + recurrenceRule: { frequency: 'weekly', byDay: [{ day: 'mo' }, { day: 'we' }] }, + }, + }); + }); + + it('prunes overrides after a series Etar capped with a lower COUNT, using the provider\'s LAST_DATE', async () => { + const { h, id } = await synced(weekly()); + // COUNT 10 → 3 (Mon 5, Wed 7, Mon 12); the provider computed LAST_DATE = last start + duration. + h.fake.user.updateEvent(id, { [Events.RRULE]: 'FREQ=WEEKLY;COUNT=3;BYDAY=MO,WE' }); + h.fake.table('events').get(id)![Events.LAST_DATE] = utc('2026-10-12T13:00:00Z') + 5_400_000; + const event = weekly(); + event.recurrenceOverrides!['2026-10-14T09:00:00'] = { title: 'x' }; + const local = (await h.local('bl'))!; + local.shadow = event; + expect(patchOf(calendarPlanner.planUpload(local, h.ctx)).patch).toEqual({ + recurrenceRule: { frequency: 'weekly', count: 3, byDay: [{ day: 'mo' }, { day: 'we' }] }, + 'recurrenceOverrides/2026-10-14T09:00:00': null, + }); + }); + + it('moves every recurrence id with an "all events" start change', async () => { + const { h, id } = await synced(weekly()); + h.fake.user.updateEvent(id, { [Events.DTSTART]: utc('2026-10-05T14:00:00Z') }); + const action = patchOf(calendarPlanner.planUpload((await h.local('bl'))!, h.ctx)); + expect(action.patch).toEqual({ + start: '2026-10-05T10:00:00', + recurrenceOverrides: { + '2026-10-12T10:00:00': { excluded: true }, + // Its own start was a move elsewhere: it stays. + '2026-10-07T10:00:00': { updated: '2026-09-26T22:57:38Z', title: 'Moved Wednesday', start: '2026-10-07T11:00:00' }, + }, + }); + }); + + it('resolves a CONTENT_EXCEPTION_URI split: the clone becomes a new event, the source uploads its capped rule', async () => { + const event = weekly(); + event.recurrenceRule = { frequency: 'weekly', byDay: [{ day: 'mo' }, { day: 'we' }] }; + event.recurrenceOverrides = { '2026-10-07T09:00:00': { title: 'Moved Wednesday', start: '2026-10-07T11:00:00' }, '2026-10-21T09:00:00': { title: 'Later' } }; + const { h, id } = await synced(event); + const clone = h.fake.user.splitViaUri(id, utc('2026-10-19T13:00:00Z'), '20261019T125959Z', { [Events.TITLE]: 'New series' }); + const all = await h.events(); + const source = all.find((e) => e.eventId === id)!; + const cloneEvent = all.find((e) => e.eventId === clone)!; + expect([source.split, cloneEvent.split]).toEqual(['source', 'clone']); + expect(source.dirty).toBe(false); + + // The source's rule was capped without DIRTY; it uploads as it stands. + expect(patchOf(calendarPlanner.planUpload(source, h.ctx)).patch).toEqual({ + recurrenceRule: { frequency: 'weekly', until: '2026-10-19T08:59:59', byDay: [{ day: 'mo' }, { day: 'we' }] }, + 'recurrenceOverrides/2026-10-21T09:00:00': null, + }); + + // The clone is claimed under a fresh uid, drops the source's sync data and marks the source dirty. + const claim = calendarPlanner.planUpload(cloneEvent, h.ctx); + expect(claim.kind).toBe('claim'); + if (claim.kind === 'claim') await h.apply(claim.ops); + const after = await h.events(); + const claimed = after.find((e) => e.eventId === clone)!; + expect(claimed.syncId).toMatch(/^~pending\//); + expect(claimed.cells[Events.UID_2445]).not.toBe(event.uid); + expect(claimed.shadow).toBeNull(); + expect(claimed.split).toBeUndefined(); + const sourceAfter = after.find((e) => e.eventId === id)!; + expect(sourceAfter.dirty).toBe(true); + // The source keeps its exceptions (and their link) whatever the provider's trigger did. + expect(sourceAfter.exceptions.map((x) => x.cells[Events.ORIGINAL_SYNC_ID])).toEqual([`${ACCT}/bl`, `${ACCT}/bl`]); + expect(patchOf(calendarPlanner.planUpload(sourceAfter, h.ctx)).patch.recurrenceRule).toMatchObject({ until: '2026-10-19T08:59:59' }); + }); + + it('keeps a local exception edit through a download that changed another instance', async () => { + const { h } = await synced(weekly()); + const x = (await h.local('bl'))!.exceptions[0]; + h.fake.user.updateEvent(x.eventId, { [Events.TITLE]: 'Local title' }); + const remote = weekly(); + remote.recurrenceOverrides!['2026-10-19T09:00:00'] = { title: 'Remote added' }; + const plan = await h.download(remote); + expect(plan).toMatchObject({ conflicts: 0, stillDirty: true }); + const after = (await h.local('bl'))!; + expect(after.exceptions.map((e) => [e.recurrenceId, e.cells[Events.TITLE], e.dirty])).toEqual([ + ['2026-10-07T09:00:00', 'Local title', true], + ['2026-10-19T09:00:00', 'Remote added', false], + ]); + expect(patchOf(calendarPlanner.planUpload(after, h.ctx)).patch).toEqual({ 'recurrenceOverrides/2026-10-07T09:00:00/title': 'Local title' }); + }); + + it('lets a server removal of an override win over a local edit of it', async () => { + const { h } = await synced(weekly()); + const x = (await h.local('bl'))!.exceptions[0]; + h.fake.user.updateEvent(x.eventId, { [Events.TITLE]: 'Local title' }); + const remote = weekly(); + delete remote.recurrenceOverrides!['2026-10-07T09:00:00']; + const plan = await h.download(remote); + expect(plan.conflicts).toBe(1); + expect(h.row(x.eventId)).toBeUndefined(); + }); + + it('lets a device deletion of an instance win over a server edit of it', async () => { + const { h } = await synced(weekly()); + const x = (await h.local('bl'))!.exceptions[0]; + h.fake.user.deleteEvent(x.eventId); + const remote = weekly(); + remote.recurrenceOverrides!['2026-10-07T09:00:00'] = { title: 'Remote title', start: '2026-10-07T11:00:00' }; + const plan = await h.download(remote); + expect(plan).toMatchObject({ conflicts: 0, stillDirty: true }); + expect(patchOf(calendarPlanner.planUpload((await h.local('bl'))!, h.ctx)).patch).toEqual({ + 'recurrenceOverrides/2026-10-07T09:00:00': { excluded: true }, + }); + }); + + it('writes an all-day series\' exceptions at UTC midnight and keys them by date', async () => { + const event = single({ + id: 'ad', + start: '2026-10-05T00:00:00', + timeZone: null, + showWithoutTime: true, + duration: 'P1D', + recurrenceRule: { frequency: 'daily', count: 5 }, + recurrenceOverrides: { '2026-10-06T00:00:00': { title: 'Day two' }, '2026-10-07T00:00:00': { excluded: true } }, + }); + const { h, id } = await synced(event); + expect(h.row(id)).toMatchObject({ [Events.EXDATE]: '20261007', [Events.DURATION]: 'P1D' }); + const [x] = (await h.local('ad'))!.exceptions; + expect(x.cells).toMatchObject({ [Events.ORIGINAL_INSTANCE_TIME]: utc('2026-10-06T00:00:00Z'), [Events.ORIGINAL_ALL_DAY]: 1, [Events.DTSTART]: utc('2026-10-06T00:00:00Z'), [Events.DTEND]: utc('2026-10-07T00:00:00Z') }); + // Etar cancels day four. + h.fake.user.cancelInstance(id, utc('2026-10-08T00:00:00Z'), utc('2026-10-09T00:00:00Z')); + expect(patchOf(calendarPlanner.planUpload((await h.local('ad'))!, h.ctx)).patch).toEqual({ 'recurrenceOverrides/2026-10-08T00:00:00': { excluded: true } }); + }); + + it('skips a series whose instance to change is a repeated wall time', async () => { + const event = single({ id: 'n', start: '2026-10-20T02:30:00', recurrenceRule: { frequency: 'daily' } }); + const { h, id } = await synced(event); + // Etar "this event" on the 25th: 02:30 happens twice that night in Berlin. + h.fake.user.insertException(id, utc('2026-10-25T00:30:00Z'), { [Events.TITLE]: 'x', [Events.DTSTART]: utc('2026-10-25T00:30:00Z'), [Events.DTEND]: utc('2026-10-25T01:30:00Z') }); + expect(calendarPlanner.planUpload((await h.local('n'))!, h.ctx)).toEqual({ kind: 'skip', reason: 'dstAmbiguous' }); + }); + + it('moves an UNTIL that falls into a DST gap or overlap one hour later', async () => { + const { h, id } = await synced(single({ recurrenceRule: { frequency: 'daily' } })); + // Until 2026-10-25 00:30Z = 02:30 CEST, a repeated wall time Stalwart would drop. + h.fake.user.updateEvent(id, { [Events.RRULE]: 'FREQ=DAILY;UNTIL=20261025T003000Z' }); + expect(patchOf(calendarPlanner.planUpload((await h.local('e1'))!, h.ctx)).patch).toEqual({ + recurrenceRule: { frequency: 'daily', until: '2026-10-25T03:30:00' }, + }); + }); + + it('keeps the organizer and other attendees on the instance a pointer RSVP would have dropped', async () => { + const { h, id } = await synced(weekly()); + const x = (await h.local('bl'))!.exceptions[0]; + h.fake.table('attendees').forEach((a) => { + if (a.event_id === x.eventId && a[Attendees.ATTENDEE_EMAIL] === ME) a[Attendees.ATTENDEE_STATUS] = 2; + }); + h.fake.user.updateEvent(x.eventId, {}); + const action = patchOf(calendarPlanner.planUpload((await h.local('bl'))!, h.ctx)); + expect(Object.keys(action.patch)).toEqual(['recurrenceOverrides/2026-10-07T09:00:00/participants']); + expect(Object.keys(action.patch['recurrenceOverrides/2026-10-07T09:00:00/participants'] as object)).toEqual(['me1', 'guestX']); + expect(id).toBeGreaterThan(0); + }); +}); diff --git a/src/device-sync/__tests__/calendar/harness.ts b/src/device-sync/__tests__/calendar/harness.ts new file mode 100644 index 00000000..e8f5e7af --- /dev/null +++ b/src/device-sync/__tests__/calendar/harness.ts @@ -0,0 +1,232 @@ +/** + * Test harness for the calendar planner: the fake CalendarProvider, a + * CalendarContext over a few server calendars, and helpers that plan, apply + * the op groups the way the engine does and read the rows back. + */ +import { expect } from 'vitest'; +import { FakeDeviceProviders } from '../fakes/fake-provider'; +import { CALENDAR_AUTHORITY, type ProviderPort, type Row } from '../../types'; +import type { CalendarContext, LocalCalendar, LocalEvent, OpGroup } from '../../planner'; +import type { CalendarEventWire, CalendarLike } from '../../wire'; +import { calendarPlanner } from '../../calendar/planner'; +import { applyPatch, type PatchObject } from '../../common/patch'; +import { makeKeyMinter, parseCollectionKey, uuidFrom } from '../../common/ids'; +import { Events } from '../../android-columns'; + +export const ACCOUNT = 'usera@example.org'; +export const ME = 'usera@example.org'; +export const ALIAS = 'alias@example.org'; +export const ACCT = 'c'; + +export function seeded(seed: number) { + return () => { + seed = (seed * 1103515245 + 12345) % 2147483648; + return seed / 2147483648; + }; +} + +export const CALENDARS: Record = { + b: { + id: 'b', + name: 'Personal', + color: '#3b82f6', + myRights: { mayReadItems: true, mayWriteAll: true, mayWriteOwn: true, mayRSVP: true, mayDelete: true }, + defaultAlertsWithTime: { d1: { '@type': 'Alert', trigger: { '@type': 'OffsetTrigger', offset: '-PT10M' }, action: 'display' } }, + defaultAlertsWithoutTime: { d2: { '@type': 'Alert', trigger: { '@type': 'OffsetTrigger', offset: '-PT15H' }, action: 'display' } }, + } as CalendarLike, + w: { id: 'w', name: 'Work', color: '#22c55e', myRights: { mayReadItems: true, mayWriteAll: true, mayWriteOwn: true, mayRSVP: true } } as CalendarLike, + r: { id: 'r', name: 'Team (read)', myRights: { mayReadItems: true, mayRSVP: true } } as CalendarLike, +}; + +export interface HarnessOptions { + deviceZone?: string; + reminderOwner?: 'device' | 'bulwark'; + readOnly?: string[]; + now?: number; +} + +export class Harness { + readonly fake = new FakeDeviceProviders(); + readonly port: ProviderPort = this.fake.port(ACCOUNT, CALENDAR_AUTHORITY); + readonly rowIds = new Map(); + ctx: CalendarContext; + + constructor(options: HarnessOptions = {}) { + const random = seeded(7); + const readOnly = new Set(options.readOnly ?? ['r']); + this.ctx = { + jmapAccountId: ACCT, + now: options.now ?? Date.parse('2026-09-27T12:00:00Z'), + mintKey: makeKeyMinter(random), + mintUid: () => uuidFrom(random), + deviceZone: options.deviceZone ?? 'Europe/Berlin', + ownerAccount: ME, + selfAddresses: [ME, ALIAS], + reminderOwner: options.reminderOwner ?? 'device', + calendar: (id) => CALENDARS[id], + calendarRowId: (id) => this.rowIds.get(id) ?? null, + calendarIdOfRow: (rowId) => { + for (const [id, row] of this.rowIds) if (row === rowId) return { jmapAccountId: ACCT, calendarId: id }; + return null; + }, + isReadOnly: (id) => readOnly.has(id), + isSelected: (key) => parseCollectionKey(key)?.accountId === ACCT, + }; + } + + /** Inserts the calendar rows through planCalendars. */ + async setup(ids: string[] = ['b', 'w', 'r']): Promise { + const plan = calendarPlanner.planCalendars( + ids.map((id) => ({ jmapAccountId: ACCT, calendar: CALENDARS[id], readOnly: this.ctx.isReadOnly(id), accountName: ME })), + [], + this.ctx, + ); + for (const group of plan.groups) await this.apply(group); + for (const cal of await this.calendars()) this.rowIds.set(parseCollectionKey(cal.syncId)!.id, cal.calendarRowId); + return this; + } + + async apply(group: OpGroup): Promise { + const result = await this.port.applyBatch(group.ops); + if (!result.ok) throw new Error(`batch ${group.ref} failed: ${result.reason} ${result.message}`); + } + + async tryApply(group: OpGroup) { + return this.port.applyBatch(group.ops); + } + + async query(table: 'events' | 'attendees' | 'reminders' | 'calendars', columns: readonly string[]): Promise { + const res = await this.port.query({ table, columns: [...columns] }); + return res.rows.map((cells) => Object.fromEntries(res.columns.map((c, i) => [c, cells[i]]))); + } + + async calendars(): Promise { + return (await this.query('calendars', calendarPlanner.calendarColumns)).map(calendarPlanner.decodeCalendar); + } + + async events(): Promise { + return calendarPlanner.decodeEvents( + await this.query('events', calendarPlanner.eventColumns), + await this.query('attendees', calendarPlanner.attendeeColumns), + await this.query('reminders', calendarPlanner.reminderColumns), + ); + } + + /** The local event with this server id (or null). */ + async local(id: string): Promise { + return (await this.events()).find((e) => e.syncId === `${ACCT}/${id}`) ?? null; + } + + async byRow(eventId: number): Promise { + const all = await this.events(); + const found = all.find((e) => e.eventId === eventId); + if (!found) throw new Error(`no event row ${eventId}`); + return found; + } + + /** Plans and applies a download of `event`, then heals baselines like the engine's read-back. */ + async download(event: CalendarEventWire) { + const local = await this.local(event.id); + const plan = calendarPlanner.planDownload(event, local, this.ctx); + if (plan.effect !== 'none') await this.apply(plan.ops); + await this.heal(event.id); + return plan; + } + + async heal(id: string): Promise { + const local = await this.local(id); + if (!local) return; + const heal = calendarPlanner.planBaselineHeal(local); + if (heal) await this.apply(heal); + } + + row(eventId: number): Row { + return this.fake.row('events', eventId)!; + } + + masterRow(id: string): Row { + const rows = this.fake.rows('events', `${Events._SYNC_ID} = ?`, [`${ACCT}/${id}`]); + expect(rows).toHaveLength(1); + return rows[0]; + } +} + +/** The server applying an accepted patch (Stalwart's normalisation of what matters here). */ +export function serverApply(event: CalendarEventWire, patch: PatchObject): CalendarEventWire { + const next = applyPatch(event, patch); + if (!next) throw new Error(`patch does not apply: ${JSON.stringify(patch)}`); + const overrides = next.recurrenceOverrides; + if (overrides) { + for (const [key, value] of Object.entries(overrides)) { + if (!value || typeof value !== 'object') continue; + if ((value as Record).excluded === true) { + overrides[key] = { excluded: true }; + continue; + } + for (const forbidden of ['@type', 'method', 'organizerCalendarAddress', 'privacy', 'prodId', 'recurrenceId', 'recurrenceIdTimeZone', 'sentBy', 'uid', 'recurrenceRule', 'recurrenceOverrides']) { + delete (value as Record)[forbidden]; + } + } + if (!Object.keys(overrides).length) delete next.recurrenceOverrides; + } + for (const map of ['participants', 'alerts', 'locations'] as const) { + if (next[map] && !Object.keys(next[map] as object).length) delete next[map]; + } + return next; +} + +/** The probe's weekly event as Stalwart returned it (scratchpad/probe/calendar.out.txt). */ +export function weekly(): CalendarEventWire { + return { + id: 'bl', + '@type': 'Event', + uid: 'probe-rec@example.org', + calendarIds: { b: true }, + title: 'Probe weekly', + description: 'desc', + start: '2026-10-05T09:00:00', + timeZone: 'America/New_York', + duration: 'PT1H30M', + status: 'tentative', + freeBusyStatus: 'free', + privacy: 'private', + color: 'steelblue', + keywords: { probe: true }, + priority: 5, + recurrenceRule: { frequency: 'weekly', count: 10, firstDayOfWeek: 'mo', byDay: [{ day: 'mo' }, { day: 'we' }] }, + recurrenceOverrides: { + '2026-10-12T09:00:00': { excluded: true }, + '2026-10-07T09:00:00': { updated: '2026-09-26T22:57:38Z', title: 'Moved Wednesday', start: '2026-10-07T11:00:00' }, + }, + locations: { locA: { name: 'Room 1', '@type': 'Location' } }, + participants: { + me1: { calendarAddress: 'mailto:usera@example.org', '@type': 'Participant', participationStatus: 'accepted', roles: { owner: true, attendee: true } }, + guestX: { calendarAddress: 'mailto:guest@example.net', '@type': 'Participant', participationStatus: 'needs-action', name: 'Guest', expectReply: true, roles: { attendee: true } }, + }, + organizerCalendarAddress: 'mailto:usera@example.org', + alerts: { + al1: { action: 'display', '@type': 'Alert', trigger: { '@type': 'OffsetTrigger', offset: '-PT15M', relativeTo: 'start' } }, + alEnd: { '@type': 'Alert', trigger: { '@type': 'OffsetTrigger', offset: 'PT5M', relativeTo: 'end' } }, + alAbs: { '@type': 'Alert', action: 'email', trigger: { '@type': 'AbsoluteTrigger', when: '2026-10-01T08:00:00Z' } }, + }, + updated: '2026-09-26T22:57:38Z', + isDraft: false, + } as CalendarEventWire; +} + +export function single(extra: Partial = {}): CalendarEventWire { + return { + id: 'e1', + '@type': 'Event', + uid: 'single-1@example.org', + calendarIds: { b: true }, + title: 'Lunch', + start: '2026-10-06T12:00:00', + timeZone: 'Europe/Berlin', + duration: 'PT1H', + updated: '2026-09-26T10:00:00Z', + ...extra, + } as CalendarEventWire; +} + +export const utc = (iso: string) => Date.parse(iso); diff --git a/src/device-sync/__tests__/calendar/lifecycle.test.ts b/src/device-sync/__tests__/calendar/lifecycle.test.ts new file mode 100644 index 00000000..994f2972 --- /dev/null +++ b/src/device-sync/__tests__/calendar/lifecycle.test.ts @@ -0,0 +1,472 @@ +import { describe, expect, it } from 'vitest'; +import { Attendees, Events, Reminders } from '../../android-columns'; +import { calendarPlanner } from '../../calendar/planner'; +import type { LocalEvent, UploadPlan } from '../../planner'; +import type { CalendarEventWire } from '../../wire'; +import { ACCT, Harness, ME, serverApply, single, utc, weekly } from './harness'; + +type Plan = UploadPlan; + +async function synced(event: CalendarEventWire, options = {}) { + const h = await new Harness(options).setup(); + await h.download(event); + return { h, id: Number(h.masterRow(event.id)._id) }; +} + +function actionsOf(plan: Plan) { + if (plan.kind !== 'upload') throw new Error(`expected an upload, got ${JSON.stringify(plan)}`); + return plan.actions; +} + +/** Claims a new row, then returns its create action. */ +async function claimAndCreate(h: Harness, eventId: number) { + const claim = calendarPlanner.planUpload(await h.byRow(eventId), h.ctx); + expect(claim.kind).toBe('claim'); + if (claim.kind === 'claim') await h.apply(claim.ops); + const local = await h.byRow(eventId); + const [create] = actionsOf(calendarPlanner.planUpload(local, h.ctx)); + if (create.kind !== 'create') throw new Error('not a create'); + return { local, create }; +} + +describe('calendar planner: new events', () => { + it('claims, creates and adopts a device event without writing anything on its echo', async () => { + const h = await new Harness().setup(); + const id = h.fake.user.insertEvent( + h.rowIds.get('b')!, + { [Events.TITLE]: 'Dentist', [Events.DTSTART]: utc('2026-10-09T08:00:00Z'), [Events.DTEND]: utc('2026-10-09T09:00:00Z'), [Events.EVENT_TIMEZONE]: 'Europe/Berlin', [Events.EVENT_LOCATION]: 'Main St 1', [Events.STATUS]: 1, [Events.ACCESS_LEVEL]: 2, [Events.AVAILABILITY]: 0 }, + { reminders: [{ [Reminders.MINUTES]: 30, [Reminders.METHOD]: 1 }] }, + ); + const { local, create } = await claimAndCreate(h, id); + expect(local.syncId).toBe(`~pending/${create.uid}`); + expect(create).toMatchObject({ collectionId: 'b', uid: local.pending!.uid }); + expect(create.sendSchedulingMessages).toBeUndefined(); + const object = create.object as Record; + expect(object).toMatchObject({ + '@type': 'Event', + uid: create.uid, + calendarIds: { b: true }, + title: 'Dentist', + start: '2026-10-09T10:00:00', + duration: 'PT1H', + timeZone: 'Europe/Berlin', + status: 'confirmed', + freeBusyStatus: 'busy', + privacy: 'private', + }); + expect(Object.values(object.locations as object)).toEqual([{ '@type': 'Location', name: 'Main St 1' }]); + expect(Object.values(object.alerts as object)).toEqual([{ '@type': 'Alert', trigger: { '@type': 'OffsetTrigger', offset: '-PT30M', relativeTo: 'start' }, action: 'display' }]); + + // The server stores it; the engine re-fetches it and applies the accepted plan. + const server = { ...(object as CalendarEventWire), id: 'n1', updated: '2026-09-27T12:00:00Z' } as CalendarEventWire; + const accepted = calendarPlanner.planAccepted(local, server, h.ctx); + await h.apply(accepted.ops); + const after = (await h.local('n1'))!; + expect(after).toMatchObject({ syncId: `${ACCT}/n1`, dirty: false, pending: null }); + expect(after.cells[Events.UID_2445]).toBe(create.uid); + expect(calendarPlanner.planDownload(server, after, h.ctx).effect).toBe('none'); + }); + + it('creates a new series with the exceptions and exclusions made before its upload', async () => { + const h = await new Harness().setup(); + const id = h.fake.user.insertEvent(h.rowIds.get('b')!, { + [Events.TITLE]: 'Standup', + [Events.DTSTART]: utc('2026-10-05T07:00:00Z'), + [Events.DURATION]: 'P900S', + [Events.RRULE]: 'FREQ=DAILY;COUNT=5', + [Events.EVENT_TIMEZONE]: 'Europe/Berlin', + [Events.STATUS]: 1, + }); + // Claimed first (the pending _SYNC_ID lets Etar offer "this event"). + const claim = calendarPlanner.planUpload(await h.byRow(id), h.ctx); + if (claim.kind === 'claim') await h.apply(claim.ops); + h.fake.user.insertException(id, utc('2026-10-06T07:00:00Z'), { [Events.TITLE]: 'Standup (late)', [Events.DTSTART]: utc('2026-10-06T08:00:00Z'), [Events.DTEND]: utc('2026-10-06T08:15:00Z') }); + h.fake.user.cancelInstance(id, utc('2026-10-07T07:00:00Z'), utc('2026-10-07T07:15:00Z')); + const local = await h.byRow(id); + expect(local.exceptions).toHaveLength(2); + const [create] = actionsOf(calendarPlanner.planUpload(local, h.ctx)); + if (create.kind !== 'create') throw new Error('not a create'); + expect(create.object.recurrenceRule).toEqual({ frequency: 'daily', count: 5 }); + expect(create.object.recurrenceOverrides).toEqual({ + // The series' details a new override needs come along (withNewOverrideDetails). + '2026-10-06T09:00:00': { title: 'Standup (late)', start: '2026-10-06T10:00:00', duration: 'PT15M', status: 'confirmed', freeBusyStatus: 'busy' }, + '2026-10-07T09:00:00': { excluded: true }, + }); + const server = { ...(create.object as CalendarEventWire), id: 's1' } as CalendarEventWire; + await h.apply(calendarPlanner.planAccepted(local, server, h.ctx).ops); + const after = (await h.local('s1'))!; + expect(after.exceptions.map((x) => [x.syncId, x.dirty])).toEqual([[`${ACCT}/s1#2026-10-06T09:00:00`, false]]); + expect(h.masterRow('s1')[Events.EXDATE]).toBe('20261007T070000Z'); + expect(calendarPlanner.planDownload(server, after, h.ctx).effect).toBe('none'); + }); + + it('invites the attendees of a new event as its organizer', async () => { + const h = await new Harness().setup(); + const id = h.fake.user.insertEvent( + h.rowIds.get('b')!, + { [Events.TITLE]: 'Review', [Events.DTSTART]: utc('2026-10-09T08:00:00Z'), [Events.DTEND]: utc('2026-10-09T09:00:00Z'), [Events.EVENT_TIMEZONE]: 'Europe/Berlin', [Events.STATUS]: 1 }, + { + attendees: [ + { [Attendees.ATTENDEE_EMAIL]: ME, [Attendees.ATTENDEE_RELATIONSHIP]: 2, [Attendees.ATTENDEE_TYPE]: 1, [Attendees.ATTENDEE_STATUS]: 1 }, + { [Attendees.ATTENDEE_EMAIL]: 'guest@example.net', [Attendees.ATTENDEE_NAME]: 'Guest', [Attendees.ATTENDEE_RELATIONSHIP]: 1, [Attendees.ATTENDEE_TYPE]: 1, [Attendees.ATTENDEE_STATUS]: 3 }, + ], + }, + ); + const { create } = await claimAndCreate(h, id); + expect(create.sendSchedulingMessages).toBe(true); + const participants = Object.values(create.object.participants ?? {}); + expect(participants).toEqual([ + { '@type': 'Participant', calendarAddress: `mailto:${ME}`, roles: { owner: true, attendee: true }, participationStatus: 'accepted' }, + { '@type': 'Participant', calendarAddress: 'mailto:guest@example.net', name: 'Guest', roles: { attendee: true }, participationStatus: 'needs-action', expectReply: true }, + ]); + expect(create.object.organizerCalendarAddress).toBe(`mailto:${ME}`); + }); + + it('gives a new event carrying another row\'s uid a fresh one', async () => { + const { h } = await synced(single()); + const copy = h.fake.user.insertEvent(h.rowIds.get('b')!, { + [Events.TITLE]: 'Copy', [Events.DTSTART]: utc('2026-10-09T08:00:00Z'), [Events.DTEND]: utc('2026-10-09T09:00:00Z'), [Events.EVENT_TIMEZONE]: 'UTC', [Events.UID_2445]: 'single-1@example.org', [Events.STATUS]: 1, + }); + const local = await h.byRow(copy); + expect(local.split).toBe('clone'); + const { create } = await claimAndCreate(h, copy); + expect(create.uid).not.toBe('single-1@example.org'); + // A uid nobody else carries is kept (an imported invitation keeps its UID). + const own = h.fake.user.insertEvent(h.rowIds.get('b')!, { + [Events.TITLE]: 'Imported', [Events.DTSTART]: utc('2026-10-09T08:00:00Z'), [Events.DTEND]: utc('2026-10-09T09:00:00Z'), [Events.EVENT_TIMEZONE]: 'UTC', [Events.UID_2445]: 'ics-uid@example.net', [Events.STATUS]: 1, + }); + expect((await claimAndCreate(h, own)).create.uid).toBe('ics-uid@example.net'); + }); + + it('adopts its own create found by uid on download, keeping later local edits', async () => { + const h = await new Harness().setup(); + const id = h.fake.user.insertEvent(h.rowIds.get('b')!, { [Events.TITLE]: 'Draft', [Events.DTSTART]: utc('2026-10-09T08:00:00Z'), [Events.DTEND]: utc('2026-10-09T09:00:00Z'), [Events.EVENT_TIMEZONE]: 'Europe/Berlin', [Events.STATUS]: 1 }); + const { create } = await claimAndCreate(h, id); + // The create reached the server, but the response was lost; meanwhile the user renamed it. + h.fake.user.updateEvent(id, { [Events.TITLE]: 'Final' }); + const server = { ...(create.object as CalendarEventWire), id: 'n7' } as CalendarEventWire; + const plan = calendarPlanner.planDownload(server, await h.byRow(id), h.ctx); + expect(plan).toMatchObject({ effect: 'update', stillDirty: true, conflicts: 0 }); + await h.apply(plan.ops); + const adopted = (await h.local('n7'))!; + expect(adopted).toMatchObject({ eventId: id, dirty: true, pending: null }); + const [update] = actionsOf(calendarPlanner.planUpload(adopted, h.ctx)); + expect(update).toMatchObject({ kind: 'update', id: 'n7', patch: { title: 'Final' } }); + }); + + it('removes a new event created in a read-only calendar', async () => { + const h = await new Harness().setup(); + const id = h.fake.user.insertEvent(h.rowIds.get('r')!, { [Events.TITLE]: 'x', [Events.DTSTART]: 0, [Events.DTEND]: 1000, [Events.EVENT_TIMEZONE]: 'UTC', [Events.STATUS]: 1 }); + const plan = calendarPlanner.planUpload(await h.byRow(id), h.ctx); + expect(plan.kind).toBe('revert'); + if (plan.kind === 'revert') await h.apply(plan.ops); + expect(h.row(id)).toBeUndefined(); + }); +}); + +describe('calendar planner: accepted uploads, merges and deletions', () => { + it('keeps a row edited again during its upload dirty, with the uploaded value as its baseline', async () => { + const { h, id } = await synced(single()); + h.fake.user.updateEvent(id, { [Events.TITLE]: 'First' }); + const local = (await h.local('e1'))!; + const [update] = actionsOf(calendarPlanner.planUpload(local, h.ctx)); + if (update.kind !== 'update') throw new Error(); + const server = serverApply(single(), update.patch); + const accepted = calendarPlanner.planAccepted(local, server, h.ctx); + // The user edits again before the clearing batch: its assert fails. + h.fake.beforeNextBatch(() => h.fake.user.updateEvent(id, { [Events.TITLE]: 'Second' })); + expect((await h.tryApply(accepted.ops)).ok).toBe(false); + await h.apply(accepted.keepDirtyOps); + const after = (await h.local('e1'))!; + expect(after).toMatchObject({ dirty: true }); + expect(after.cells[Events.TITLE]).toBe('Second'); + expect(after.shadow?.title).toBe('First'); + const [next] = actionsOf(calendarPlanner.planUpload(after, h.ctx)); + expect(next).toMatchObject({ patch: { title: 'Second' } }); + // A revert to the old value uploads too. + h.fake.user.updateEvent(id, { [Events.TITLE]: 'Lunch' }); + expect(actionsOf(calendarPlanner.planUpload((await h.local('e1'))!, h.ctx))[0]).toMatchObject({ patch: { title: 'Lunch' } }); + }); + + it('purges a cancelled instance whose exclusion was uploaded even when the series was edited meanwhile', async () => { + const { h, id } = await synced(weekly()); + const cancelled = h.fake.user.cancelInstance(id, utc('2026-10-19T13:00:00Z'), utc('2026-10-19T14:30:00Z')); + const local = (await h.local('bl'))!; + const [update] = actionsOf(calendarPlanner.planUpload(local, h.ctx)); + if (update.kind !== 'update') throw new Error(); + const server = serverApply(weekly(), update.patch); + const accepted = calendarPlanner.planAccepted(local, server, h.ctx); + h.fake.beforeNextBatch(() => h.fake.user.updateEvent(id, { [Events.TITLE]: 'Edited meanwhile' })); + expect((await h.tryApply(accepted.ops)).ok).toBe(false); + await h.apply(accepted.keepDirtyOps); + expect(h.row(cancelled)).toBeUndefined(); + const after = (await h.local('bl'))!; + const [next] = actionsOf(calendarPlanner.planUpload(after, h.ctx)); + // Only the newer edit: the exclusion is on the server already and stays one. + expect(next).toMatchObject({ kind: 'update', patch: { title: 'Edited meanwhile' } }); + }); + + it('fails a download write when the user edits the event meanwhile', async () => { + const { h, id } = await synced(weekly()); + const plan = calendarPlanner.planDownload({ ...weekly(), title: 'Server' }, await h.local('bl'), h.ctx); + h.fake.beforeNextBatch(() => h.fake.user.updateEvent(id, { [Events.DESCRIPTION]: 'mine' })); + const result = await h.tryApply(plan.ops); + expect(result).toMatchObject({ ok: false, reason: 'assert' }); + expect(h.row(id)[Events.TITLE]).toBe('Probe weekly'); + }); + + it('merges per unit: local and server changes to different fields both survive', async () => { + const { h, id } = await synced(weekly()); + h.fake.user.updateEvent(id, { [Events.TITLE]: 'Local title' }); + const plan = await h.download({ ...weekly(), description: 'Server description' }); + expect(plan).toMatchObject({ conflicts: 0, stillDirty: true }); + expect(h.row(id)).toMatchObject({ [Events.TITLE]: 'Local title', [Events.DESCRIPTION]: 'Server description', [Events.DIRTY]: 1 }); + expect(actionsOf(calendarPlanner.planUpload((await h.local('bl'))!, h.ctx))[0]).toMatchObject({ patch: { title: 'Local title' } }); + }); + + it('lets the server win a real conflict and counts it', async () => { + const { h, id } = await synced(weekly()); + h.fake.user.updateEvent(id, { [Events.TITLE]: 'Local title' }); + const plan = await h.download({ ...weekly(), title: 'Server title' }); + expect(plan).toMatchObject({ conflicts: 1, stillDirty: false }); + expect(h.row(id)).toMatchObject({ [Events.TITLE]: 'Server title', [Events.DIRTY]: 0 }); + }); + + it('treats the same change on both sides as converged', async () => { + const { h, id } = await synced(weekly()); + h.fake.user.updateEvent(id, { [Events.TITLE]: 'Same' }); + const plan = await h.download({ ...weekly(), title: 'Same' }); + expect(plan).toMatchObject({ conflicts: 0, stillDirty: false }); + expect(h.row(id)[Events.DIRTY]).toBe(0); + }); + + it('merges attendees by address and keeps the user\'s RSVP', async () => { + const invite = single({ + organizerCalendarAddress: 'mailto:boss@example.net', + participants: { + boss: { calendarAddress: 'mailto:boss@example.net', roles: { owner: true } }, + me: { calendarAddress: 'mailto:usera@example.org', roles: { attendee: true }, participationStatus: 'needs-action' }, + }, + } as Partial); + const { h, id } = await synced(invite); + h.fake.user.setAttendeeStatus(id, ME, 1); + const remote = { ...invite, participants: { ...invite.participants, new: { calendarAddress: 'mailto:new@example.net', roles: { attendee: true } } } } as CalendarEventWire; + const plan = await h.download(remote); + expect(plan).toMatchObject({ conflicts: 0, stillDirty: true }); + const rows = h.fake.rows('attendees', `${Attendees.EVENT_ID} = ?`, [id]); + expect(rows.map((a) => [a[Attendees.ATTENDEE_EMAIL], a[Attendees.ATTENDEE_STATUS]])).toEqual([ + ['boss@example.net', 3], + [ME, 1], + ['new@example.net', 3], + ]); + expect(actionsOf(calendarPlanner.planUpload((await h.local('e1'))!, h.ctx))[0]).toMatchObject({ + patch: { 'participants/me/participationStatus': 'accepted' }, + sendSchedulingMessages: true, + }); + }); + + it('destroys an event deleted on the device, with a CANCEL when others take part', async () => { + const { h, id } = await synced(weekly()); + h.fake.user.deleteEvent(id); + expect(actionsOf(calendarPlanner.planUpload((await h.local('bl'))!, h.ctx))).toEqual([ + { kind: 'destroy', id: 'bl', uid: 'probe-rec@example.org', sendSchedulingMessages: true }, + ]); + // Its exceptions go with it. + const local = (await h.local('bl'))!; + const group = calendarPlanner.planLocalDelete(local); + await h.apply(group); + expect(await h.events()).toEqual([]); + }); + + it('removes only the synced membership of an event that is in several calendars', async () => { + const h = await new Harness().setup(['b', 'w', 'r']); + const isSelected = h.ctx.isSelected; + h.ctx = { ...h.ctx, isSelected: (key) => key !== `${ACCT}/x` && isSelected(key) }; + await h.download(single({ calendarIds: { b: true, x: true } })); + h.fake.user.deleteEvent(Number(h.masterRow('e1')._id)); + expect(actionsOf(calendarPlanner.planUpload((await h.local('e1'))!, h.ctx))).toEqual([{ kind: 'update', id: 'e1', patch: { 'calendarIds/b': null } }]); + // The server keeps it elsewhere: the rows go. + const accepted = calendarPlanner.planAccepted((await h.local('e1'))!, single({ calendarIds: { x: true } }), h.ctx); + await h.apply(accepted.ops); + expect(await h.events()).toEqual([]); + }); + + it('destroys a deleted event whose create had an unknown outcome by its uid', async () => { + const h = await new Harness().setup(); + const id = h.fake.user.insertEvent(h.rowIds.get('b')!, { [Events.TITLE]: 'x', [Events.DTSTART]: 0, [Events.DTEND]: 1000, [Events.EVENT_TIMEZONE]: 'UTC', [Events.STATUS]: 1 }); + const { local } = await claimAndCreate(h, id); + h.fake.user.deleteEvent(id); + // With the pending _SYNC_ID the app delete was a soft delete the engine can act on. + expect(h.row(id)).toMatchObject({ [Events.DELETED]: 1 }); + expect(actionsOf(calendarPlanner.planUpload(await h.byRow(id), h.ctx))).toEqual([{ kind: 'destroy', id: null, uid: local.pending!.uid }]); + }); + + it('keeps a device deletion when the server edited the event meanwhile', async () => { + const { h, id } = await synced(single()); + h.fake.user.deleteEvent(id); + const plan = await h.download(single({ title: 'Server edit' })); + expect(plan).toMatchObject({ stillDirty: true, effect: 'update', writes: 1 }); + expect(h.row(id)).toMatchObject({ [Events.DELETED]: 1, [Events.TITLE]: 'Lunch' }); + expect(actionsOf(calendarPlanner.planUpload((await h.local('e1'))!, h.ctx))[0].kind).toBe('destroy'); + }); + + it('purges an exception row whose master is gone instead of uploading it', async () => { + const { h } = await synced(weekly()); + const local = (await h.local('bl'))!; + await h.apply({ ref: 'x', ops: [{ op: 'delete', table: 'events', id: local.eventId, expectCount: 1 }] }); + const [orphan] = await h.events(); + expect(orphan.cells[Events.ORIGINAL_SYNC_ID]).toBe(`${ACCT}/bl`); + const plan = calendarPlanner.planUpload(orphan, h.ctx); + expect(plan.kind).toBe('purge'); + if (plan.kind === 'purge') await h.apply(plan.ops); + expect(await h.events()).toEqual([]); + }); + + it('refuses tasks everywhere', async () => { + const { h } = await synced(single()); + const local = (await h.local('e1'))!; + const task = { ...single(), '@type': 'Task' } as CalendarEventWire; + expect(() => calendarPlanner.planAccepted(local, task, h.ctx)).toThrow(/tasks/); + expect(() => calendarPlanner.planUpload({ ...local, shadow: task }, h.ctx)).toThrow(/tasks/); + }); +}); + +describe('calendar planner: pairs', () => { + it('uploads Etar\'s move to another calendar (delete + insert) as a calendarIds swap', async () => { + const { h, id } = await synced(single()); + h.fake.user.deleteEvent(id); + const moved = h.fake.user.insertEvent(h.rowIds.get('w')!, { + [Events.TITLE]: 'Lunch', [Events.DTSTART]: utc('2026-10-06T10:00:00Z'), [Events.DTEND]: utc('2026-10-06T11:00:00Z'), [Events.EVENT_TIMEZONE]: 'Europe/Berlin', [Events.STATUS]: 1, [Events.AVAILABILITY]: 0, [Events.ACCESS_LEVEL]: 0, + }); + const events = await h.events(); + const pairs = calendarPlanner.planPairs(events.filter((e) => e.deleted), events.filter((e) => !e.syncId), h.ctx); + expect(pairs).toHaveLength(1); + expect(pairs[0].actions).toEqual([{ kind: 'update', id: 'e1', patch: { 'calendarIds/b': null, 'calendarIds/w': true } }]); + await h.apply(pairs[0].ops); + const after = await h.events(); + expect(after).toHaveLength(1); + expect(after[0]).toMatchObject({ eventId: moved, syncId: `${ACCT}/e1`, dirty: false }); + expect(after[0].shadow?.calendarIds).toEqual({ w: true }); + // The server's echo writes nothing. + expect(calendarPlanner.planDownload(single({ calendarIds: { w: true } }), after[0], h.ctx).effect).toBe('none'); + }); + + it('uploads a series turned into a single event by delete + insert as a rule removal', async () => { + const { h, id } = await synced(single({ recurrenceRule: { frequency: 'daily' } })); + h.fake.user.deleteEvent(id); + h.fake.user.insertEvent(h.rowIds.get('b')!, { + [Events.TITLE]: 'Lunch', [Events.DTSTART]: utc('2026-10-06T10:00:00Z'), [Events.DTEND]: utc('2026-10-06T11:00:00Z'), [Events.EVENT_TIMEZONE]: 'Europe/Berlin', [Events.STATUS]: 1, [Events.AVAILABILITY]: 0, [Events.ACCESS_LEVEL]: 0, + }); + const events = await h.events(); + const pairs = calendarPlanner.planPairs(events.filter((e) => e.deleted), events.filter((e) => !e.syncId), h.ctx); + expect(pairs.map((p) => p.actions)).toEqual([[{ kind: 'update', id: 'e1', patch: { recurrenceRule: null } }]]); + }); + + it('does not pair unrelated rows', async () => { + const { h, id } = await synced(single()); + h.fake.user.deleteEvent(id); + h.fake.user.insertEvent(h.rowIds.get('w')!, { [Events.TITLE]: 'Something else', [Events.DTSTART]: utc('2026-10-06T10:00:00Z'), [Events.DTEND]: utc('2026-10-06T11:00:00Z'), [Events.EVENT_TIMEZONE]: 'Europe/Berlin', [Events.STATUS]: 1 }); + const events = await h.events(); + expect(calendarPlanner.planPairs(events.filter((e) => e.deleted), events.filter((e) => !e.syncId), h.ctx)).toEqual([]); + }); +}); + +describe('calendar planner: baselines, zones and reminder owners', () => { + it('heals a clean row\'s baseline after provider normalisation, but never a series\' timing or rule', async () => { + const { h, id } = await synced(weekly()); + // Provider-side changes without DIRTY (as CONTENT_EXCEPTION_URI's split does to the rule). + const row = h.fake.table('events').get(id)!; + row[Events.ORGANIZER] = 'someone@example.net'; + row[Events.RRULE] = 'FREQ=WEEKLY;UNTIL=20261019T125959Z;BYDAY=MO,WE'; + const heal = calendarPlanner.planBaselineHeal((await h.local('bl'))!)!; + expect(heal).not.toBeNull(); + await h.apply(heal); + const baseline = (await h.local('bl'))!.baseline!; + expect(baseline.cells[Events.ORGANIZER]).toBe('someone@example.net'); + expect(baseline.cells[Events.RRULE]).toBe('FREQ=WEEKLY;COUNT=10;BYDAY=MO,WE'); + expect(calendarPlanner.planBaselineHeal((await h.local('bl'))!)).toBeNull(); + }); + + it('does not heal a row that turned dirty', async () => { + const { h, id } = await synced(single()); + h.fake.user.updateEvent(id, { [Events.TITLE]: 'x' }); + expect(calendarPlanner.planBaselineHeal((await h.local('e1'))!)).toBeNull(); + }); + + it('rewrites floating events for a new device zone, keeping their wall time', async () => { + const { h, id } = await synced(single({ id: 'f', timeZone: null, start: '2026-10-06T14:00:00' })); + expect(h.row(id)[Events.DTSTART]).toBe(utc('2026-10-06T12:00:00Z')); + h.ctx = { ...h.ctx, deviceZone: 'America/New_York' }; + const group = calendarPlanner.planZoneChange((await h.local('f'))!, 'Europe/Berlin', h.ctx)!; + await h.apply(group); + expect(h.row(id)).toMatchObject({ [Events.DTSTART]: utc('2026-10-06T18:00:00Z'), [Events.EVENT_TIMEZONE]: 'America/New_York' }); + expect(calendarPlanner.planZoneChange((await h.local('f'))!, 'Europe/Berlin', h.ctx)).toBeNull(); + // Events with a zone are not floating. + const t = await synced(single()); + expect(calendarPlanner.planZoneChange((await t.h.local('e1'))!, 'Europe/Berlin', { ...t.h.ctx, deviceZone: 'Asia/Tokyo' })).toBeNull(); + }); + + it('rewrites Reminders rows when the reminder owner changes, without server writes', async () => { + const { h, id } = await synced(weekly()); + expect(h.fake.rows('reminders', `${Reminders.EVENT_ID} = ?`, [id])).toHaveLength(1); + h.ctx = { ...h.ctx, reminderOwner: 'bulwark' }; + await h.apply(calendarPlanner.planReminderOwnerChange((await h.local('bl'))!, h.ctx)!); + expect(h.fake.rows('reminders', `${Reminders.EVENT_ID} = ?`, [id])).toHaveLength(0); + expect(calendarPlanner.planReminderOwnerChange((await h.local('bl'))!, h.ctx)).toBeNull(); + h.ctx = { ...h.ctx, reminderOwner: 'device' }; + await h.apply(calendarPlanner.planReminderOwnerChange((await h.local('bl'))!, h.ctx)!); + expect(h.fake.rows('reminders', `${Reminders.EVENT_ID} = ?`, [id]).map((r) => r[Reminders.MINUTES])).toEqual([15]); + // Back with the calendar app: an echo download writes nothing. + expect(calendarPlanner.planDownload(weekly(), await h.local('bl'), h.ctx).effect).toBe('none'); + }); +}); + +describe('calendar planner: decoding', () => { + it('tolerates garbage in sync columns and numbers read back as text', async () => { + const [event] = calendarPlanner.decodeEvents( + [ + { + _id: '5', calendar_id: '1', _sync_id: 'c/x', dirty: '1', deleted: '0', sync_data1: '{broken', sync_data3: '{"uid":1}', + sync_data4: '[]', sync_data5: 'nope', title: 'T', allDay: '0', dtstart: '1000', rrule: null, original_id: null, original_sync_id: null, + }, + ], + [], + [], + ); + expect(event).toMatchObject({ eventId: 5, calendarRowId: 1, syncId: 'c/x', dirty: true, deleted: false, shadow: null, pending: null, baseline: null, poison: null }); + }); + + it('groups exceptions by ORIGINAL_ID, else ORIGINAL_SYNC_ID, and returns orphans as masters', () => { + const events = calendarPlanner.decodeEvents( + [ + { _id: 1, calendar_id: 1, _sync_id: 'c/m', eventTimezone: 'Europe/Berlin', allDay: 0, dirty: 0, deleted: 0 }, + { _id: 2, calendar_id: 1, _sync_id: null, original_id: 1, originalInstanceTime: utc('2026-10-07T07:00:00Z'), dirty: 1, deleted: 0 }, + { _id: 3, calendar_id: 1, _sync_id: 'c/m#2026-10-08T09:00:00', original_sync_id: 'c/m', sync_data2: '2026-10-08T09:00:00', dirty: 0, deleted: 0 }, + { _id: 4, calendar_id: 1, _sync_id: null, original_sync_id: 'c/gone', originalInstanceTime: 0, dirty: 1, deleted: 0 }, + ], + [], + [], + ); + expect(events.map((e) => [e.eventId, e.exceptions.map((x) => [x.eventId, (x as { recurrenceId: string }).recurrenceId])])).toEqual([ + [1, [[2, '2026-10-07T09:00:00'], [3, '2026-10-08T09:00:00']]], + [4, []], + ]); + }); + + it('marks two masters sharing a _SYNC_ID as a split source and clone', () => { + const events = calendarPlanner.decodeEvents( + [ + { _id: 1, calendar_id: 1, _sync_id: 'c/m' }, + { _id: 9, calendar_id: 1, _sync_id: 'c/m' }, + { _id: 3, calendar_id: 1, _sync_id: 'c/n' }, + ], + [], + [], + ); + expect(events.map((e: LocalEvent) => [e.eventId, e.split])).toEqual([ + [1, 'source'], + [3, undefined], + [9, 'clone'], + ]); + }); +}); diff --git a/src/device-sync/__tests__/calendar/robustness.test.ts b/src/device-sync/__tests__/calendar/robustness.test.ts new file mode 100644 index 00000000..1abc6480 --- /dev/null +++ b/src/device-sync/__tests__/calendar/robustness.test.ts @@ -0,0 +1,98 @@ +import { describe, expect, it } from 'vitest'; +import { Attendees, Events, Reminders } from '../../android-columns'; +import { calendarPlanner } from '../../calendar/planner'; +import { Harness, single, utc, weekly } from './harness'; + +describe('calendar planner: what the real providers hand back', () => { + it('sees no edit in numbers read back as text', async () => { + const h = await new Harness().setup(); + await h.download(weekly()); + const id = Number(h.masterRow('bl')._id); + // Some provider columns come back through the bridge as text. + const row = h.fake.table('events').get(id)!; + for (const c of [Events.STATUS, Events.AVAILABILITY, Events.ACCESS_LEVEL, Events.DTSTART, Events.ALL_DAY, Events.EVENT_COLOR, Events.CALENDAR_ID]) { + row[c] = String(row[c]); + } + for (const a of h.fake.table('attendees').values()) { + if (a.event_id === id) for (const c of [Attendees.ATTENDEE_STATUS, Attendees.ATTENDEE_TYPE, Attendees.ATTENDEE_RELATIONSHIP]) a[c] = String(a[c]); + } + for (const r of h.fake.table('reminders').values()) { + if (r.event_id === id) for (const c of [Reminders.MINUTES, Reminders.METHOD]) r[c] = String(r[c]); + } + expect(calendarPlanner.planDownload(weekly(), await h.local('bl'), h.ctx).effect).toBe('none'); + h.fake.user.updateEvent(id, {}); + expect(calendarPlanner.planUpload((await h.local('bl'))!, h.ctx).kind).toBe('clean'); + }); + + it('asserts DIRTY as read, so a row inserted with a NULL DIRTY still takes a guarded write', async () => { + const h = await new Harness().setup(); + await h.download(single()); + const id = Number(h.masterRow('e1')._id); + h.fake.table('events').get(id)![Events.DIRTY] = null; + const local = (await h.local('e1'))!; + expect(local.dirty).toBe(false); + const plan = calendarPlanner.planDownload(single({ title: 'Changed' }), local, h.ctx); + const assert = plan.ops.ops.find((op) => op.op === 'assert' && 'id' in op && op.id === id) as { values: Record }; + expect(assert.values[Events.DIRTY]).toBeNull(); + await h.apply(plan.ops); + expect(h.row(id)[Events.TITLE]).toBe('Changed'); + }); + + it('writes DIRTY = 0 on every row it inserts', async () => { + const h = await new Harness().setup(); + const plan = calendarPlanner.planDownload(weekly(), null, h.ctx); + const inserts = plan.ops.ops.filter((op) => op.op === 'insert' && op.table === 'events') as Array<{ values: Record }>; + expect(inserts).toHaveLength(2); + for (const insert of inserts) expect(insert.values[Events.DIRTY]).toBe(0); + }); + + it('addresses attendee and reminder rows by id only', async () => { + const h = await new Harness().setup(); + await h.download(weekly()); + const plan = calendarPlanner.planDownload( + { ...weekly(), participants: { me1: weekly().participants!.me1 }, alerts: {} }, + await h.local('bl'), + h.ctx, + ); + for (const op of plan.ops.ops) { + if ((op.op === 'update' || op.op === 'delete') && (op.table === 'attendees' || op.table === 'reminders')) { + expect(op).toMatchObject({ expectCount: 1 }); + expect(typeof op.id).toBe('number'); + expect(op.where).toBeUndefined(); + } + if ('where' in op && op.where) expect(op.where).not.toMatch(/\b_id\b/); + } + await h.apply(plan.ops); + }); + + it('computes instance times exactly across a DST change of the series\' zone', async () => { + const h = await new Harness().setup(); + const event = weekly(); + event.recurrenceRule = { frequency: 'weekly', byDay: [{ day: 'mo' }] }; + event.recurrenceOverrides = { '2026-11-02T09:00:00': { title: 'After the change' } }; + await h.download(event); + const [x] = (await h.local('bl'))!.exceptions; + // New York left DST on 1 November: 09:00 is 14:00Z from then on, not 13:00Z. + expect(x.cells[Events.ORIGINAL_INSTANCE_TIME]).toBe(utc('2026-11-02T14:00:00Z')); + expect(x.cells[Events.DTSTART]).toBe(utc('2026-11-02T14:00:00Z')); + }); + + it('derives the recurrence id of an app\'s exception from its instance time in the master\'s zone', async () => { + const h = await new Harness().setup(); + const event = weekly(); + event.recurrenceRule = { frequency: 'weekly', byDay: [{ day: 'mo' }] }; + event.recurrenceOverrides = {}; + await h.download(event); + const id = Number(h.masterRow('bl')._id); + h.fake.user.insertException(id, utc('2026-11-02T14:00:00Z'), { [Events.TITLE]: 'x', [Events.DTSTART]: utc('2026-11-02T14:00:00Z'), [Events.DTEND]: utc('2026-11-02T15:30:00Z') }); + expect((await h.local('bl'))!.exceptions[0].recurrenceId).toBe('2026-11-02T09:00:00'); + }); + + it('ignores an EXRULE or RDATE an app wrote', async () => { + const h = await new Harness().setup(); + await h.download(weekly()); + const id = Number(h.masterRow('bl')._id); + h.fake.user.updateEvent(id, { [Events.EXRULE]: 'FREQ=MONTHLY' }); + expect(calendarPlanner.planUpload((await h.local('bl'))!, h.ctx).kind).toBe('clean'); + }); +}); diff --git a/src/device-sync/__tests__/calendar/rrule.test.ts b/src/device-sync/__tests__/calendar/rrule.test.ts new file mode 100644 index 00000000..3af02ed6 --- /dev/null +++ b/src/device-sync/__tests__/calendar/rrule.test.ts @@ -0,0 +1,119 @@ +import { describe, expect, it } from 'vitest'; +import type { RecurrenceRule } from '../../../api/types'; +import { + canonicalRRule, + isRuleRepresentable, + parseUntil, + ruleToRRule, + rruleToRule, + unrepresentableParts, + untilUtc, +} from '../../calendar/rrule'; +import { aospRruleValid } from '../fakes/fake-provider'; +import { exdateSet, formatExdate, parseExdate } from '../../calendar/exdate'; + +const timed = { allDay: false, zone: 'Europe/Berlin' }; +const allDay = { allDay: true, zone: 'UTC' }; + +describe('calendar/rrule', () => { + it.each<[string, RecurrenceRule, string]>([ + ['daily', { frequency: 'daily' }, 'FREQ=DAILY'], + ['interval', { frequency: 'weekly', interval: 2 }, 'FREQ=WEEKLY;INTERVAL=2'], + ['count', { frequency: 'daily', count: 10 }, 'FREQ=DAILY;COUNT=10'], + ['byDay', { frequency: 'weekly', byDay: [{ day: 'mo' }, { day: 'we' }] }, 'FREQ=WEEKLY;BYDAY=MO,WE'], + ['nth weekday', { frequency: 'monthly', byDay: [{ day: 'fr', nthOfPeriod: -1 }, { day: 'tu', nthOfPeriod: 2 }] }, 'FREQ=MONTHLY;BYDAY=-1FR,2TU'], + ['byMonthDay', { frequency: 'monthly', byMonthDay: [1, 15, -1] }, 'FREQ=MONTHLY;BYMONTHDAY=1,15,-1'], + ['byMonth', { frequency: 'yearly', byMonth: ['3', '10'] }, 'FREQ=YEARLY;BYMONTH=3,10'], + ['byYearDay', { frequency: 'yearly', byYearDay: [1, -1] }, 'FREQ=YEARLY;BYYEARDAY=1,-1'], + ['byWeekNo', { frequency: 'yearly', byWeekNo: [20], byDay: [{ day: 'mo' }] }, 'FREQ=YEARLY;BYDAY=MO;BYWEEKNO=20'], + ['byHour/Minute/Second', { frequency: 'daily', byHour: [9, 17], byMinute: [0, 30], bySecond: [0] }, 'FREQ=DAILY;BYSECOND=0;BYMINUTE=0,30;BYHOUR=9,17'], + ['bySetPosition', { frequency: 'monthly', byDay: [{ day: 'mo' }, { day: 'tu' }], bySetPosition: [-1] }, 'FREQ=MONTHLY;BYDAY=MO,TU;BYSETPOS=-1'], + ['wkst', { frequency: 'weekly', interval: 2, firstDayOfWeek: 'su' }, 'FREQ=WEEKLY;INTERVAL=2;WKST=SU'], + ])('writes and reads back %s', (_name, rule, rrule) => { + const written = ruleToRRule(rule, timed)!; + expect(written).toBe(rrule); + expect(aospRruleValid(written)).toBe(true); + expect(rruleToRule(written, timed)).toEqual(rule); + }); + + it('leaves out the defaults: INTERVAL=1, WKST=MO, rscale gregorian, skip omit', () => { + expect(ruleToRRule({ frequency: 'weekly', interval: 1, firstDayOfWeek: 'mo', rscale: 'gregorian', skip: 'omit' }, timed)).toBe('FREQ=WEEKLY'); + }); + + it('writes UNTIL as UTC for timed rules, converted from the event zone, and as a date for all-day rules', () => { + expect(ruleToRRule({ frequency: 'daily', until: '2026-10-30T09:00:00' }, timed)).toBe('FREQ=DAILY;UNTIL=20261030T080000Z'); + expect(ruleToRRule({ frequency: 'daily', until: '2026-07-30T09:00:00' }, timed)).toBe('FREQ=DAILY;UNTIL=20260730T070000Z'); + expect(ruleToRRule({ frequency: 'daily', until: '2026-10-30T23:59:59' }, allDay)).toBe('FREQ=DAILY;UNTIL=20261030'); + expect(rruleToRule('FREQ=DAILY;UNTIL=20261030T080000Z', timed)).toEqual({ frequency: 'daily', until: '2026-10-30T09:00:00' }); + expect(rruleToRule('FREQ=DAILY;UNTIL=20261030', allDay)).toEqual({ frequency: 'daily', until: '2026-10-30T00:00:00' }); + }); + + it('converts UNTIL across DST edges', () => { + // New York spring gap: 02:30 does not exist, UNTIL moves forward like the instance does. + expect(ruleToRRule({ frequency: 'daily', until: '2026-03-08T02:30:00' }, { allDay: false, zone: 'America/New_York' })).toBe( + 'FREQ=DAILY;UNTIL=20260308T073000Z', + ); + // Berlin autumn overlap: the first instant. + expect(ruleToRRule({ frequency: 'daily', until: '2026-10-25T02:30:00' }, timed)).toBe('FREQ=DAILY;UNTIL=20261025T003000Z'); + expect(untilUtc({ frequency: 'daily', until: '2026-10-25T02:30:00' }, timed)).toBe(Date.parse('2026-10-25T00:30:00Z')); + // Sydney's autumn overlap. + expect(ruleToRRule({ frequency: 'weekly', until: '2026-04-05T02:30:00' }, { allDay: false, zone: 'Australia/Sydney' })).toBe( + 'FREQ=WEEKLY;UNTIL=20260404T153000Z', + ); + }); + + it('reads rules the way device apps write them', () => { + expect(rruleToRule('rrule:freq=weekly;byday=mo,we;x-fossify=1', timed)).toEqual({ frequency: 'weekly', byDay: [{ day: 'mo' }, { day: 'we' }] }); + // Etar writes all-day UNTIL as a UTC date-time (the instance start minus 1 s). + expect(rruleToRule('FREQ=DAILY;UNTIL=20261015T235959Z', allDay)).toEqual({ frequency: 'daily', until: '2026-10-15T00:00:00' }); + // A floating UNTIL is UTC to AOSP, and so is a DATE on a timed rule. + expect(parseUntil('20261030T080000', timed)).toBe('2026-10-30T09:00:00'); + expect(parseUntil('20261030', timed)).toBe('2026-10-30T01:00:00'); + // COUNT wins over UNTIL next to it. + expect(rruleToRule('FREQ=DAILY;COUNT=5;UNTIL=20261030T080000Z', timed)).toEqual({ frequency: 'daily', count: 5 }); + // Several rules: the first one. + expect(rruleToRule('FREQ=DAILY\nFREQ=WEEKLY', timed)).toEqual({ frequency: 'daily' }); + expect(rruleToRule('INTERVAL=2', timed)).toBeNull(); + expect(rruleToRule('FREQ=FORTNIGHTLY', timed)).toBeNull(); + expect(rruleToRule(null, timed)).toBeNull(); + }); + + it('tells rules AOSP cannot hold', () => { + expect(unrepresentableParts({ frequency: 'yearly', rscale: 'chinese' })).toEqual(['rscale']); + expect(unrepresentableParts({ frequency: 'monthly', skip: 'forward' })).toEqual(['skip']); + expect(unrepresentableParts({ frequency: 'yearly', byMonth: ['5L'] })).toEqual(['byMonth']); + expect(isRuleRepresentable({ frequency: 'yearly', rscale: 'GREGORIAN', skip: 'omit' })).toBe(true); + // Never RSCALE or SKIP; a leap month is dropped, not written. + const written = ruleToRRule({ frequency: 'yearly', rscale: 'chinese', skip: 'forward', byMonth: ['5L', '6'] }, timed)!; + expect(written).toBe('FREQ=YEARLY;BYMONTH=6'); + expect(aospRruleValid(written)).toBe(true); + }); + + it('compares rules by meaning', () => { + expect(canonicalRRule('FREQ=WEEKLY;BYDAY=WE,MO;INTERVAL=1;WKST=MO')).toBe(canonicalRRule('freq=weekly;byday=mo,we')); + expect(canonicalRRule('FREQ=WEEKLY;BYDAY=MO')).not.toBe(canonicalRRule('FREQ=WEEKLY;BYDAY=TU')); + expect(canonicalRRule(null)).toBe(''); + }); +}); + +describe('calendar/exdate', () => { + it('writes excluded instances as UTC date-times, or dates for all-day events', () => { + expect(formatExdate(['2026-10-14T09:00:00', '2026-10-12T09:00:00'], timed)).toBe('20261012T070000Z,20261014T070000Z'); + expect(formatExdate(['2026-10-12T00:00:00'], allDay)).toBe('20261012'); + expect(formatExdate([], timed)).toBeNull(); + }); + + it('reads every form back into recurrence ids in the event zone', () => { + expect(parseExdate('20261012T070000Z,20261014T070000Z', timed)).toEqual(['2026-10-12T09:00:00', '2026-10-14T09:00:00']); + // A zone prefix, a floating time without one (UTC), several lines. + expect(parseExdate('Europe/Berlin;20261012T090000\n20261014T070000', timed)).toEqual(['2026-10-12T09:00:00', '2026-10-14T09:00:00']); + expect(parseExdate('20261012,20261013T000000Z', allDay)).toEqual(['2026-10-12T00:00:00', '2026-10-13T00:00:00']); + expect(parseExdate('garbage,20261012T070000Z', timed)).toEqual(['2026-10-12T09:00:00']); + expect(parseExdate(null, timed)).toEqual([]); + }); + + it('compares columns by the instants they name', () => { + expect([...exdateSet('Europe/Berlin;20261012T090000')]).toEqual(['20261012T070000Z']); + expect([...exdateSet('20261012T000000Z', true)]).toEqual(['20261012']); + }); +}); diff --git a/src/device-sync/__tests__/calendar/upload.test.ts b/src/device-sync/__tests__/calendar/upload.test.ts new file mode 100644 index 00000000..5e22fcfb --- /dev/null +++ b/src/device-sync/__tests__/calendar/upload.test.ts @@ -0,0 +1,320 @@ +import { describe, expect, it } from 'vitest'; +import { Attendees, Events, Reminders } from '../../android-columns'; +import { calendarPlanner } from '../../calendar/planner'; +import type { UploadPlan } from '../../planner'; +import type { CalendarEventWire } from '../../wire'; +import type { Row } from '../../types'; +import { cssColorToArgb } from '../../calendar/values'; +import { Harness, ME, serverApply, single, utc, weekly } from './harness'; + +type Plan = UploadPlan; + +function updateOf(plan: Plan) { + expect(plan.kind).toBe('upload'); + if (plan.kind !== 'upload') throw new Error('not an upload'); + expect(plan.actions).toHaveLength(1); + const action = plan.actions[0]; + if (action.kind !== 'update') throw new Error(`not an update: ${action.kind}`); + return action; +} + +async function synced(event: CalendarEventWire, options = {}) { + const h = await new Harness(options).setup(); + await h.download(event); + const id = Number(h.masterRow(event.id)._id); + return { h, id }; +} + +const FIXTURES: Array<[string, () => CalendarEventWire]> = [ + ['a recurring event with overrides', weekly], + ['a single timed event', () => single()], + ['a floating event', () => single({ id: 'f', timeZone: null })], + ['an all-day event', () => single({ id: 'a', start: '2026-10-08T00:00:00', timeZone: null, showWithoutTime: true, duration: 'P3D' })], + ['a recurring all-day event', () => single({ id: 'y', start: '2026-10-08T00:00:00', timeZone: null, showWithoutTime: true, duration: 'P1D', recurrenceRule: { frequency: 'yearly' } })], + ['an HTML description and a CSS colour name', () => single({ id: 'h', description: '

Hi there

', descriptionContentType: 'text/html', color: 'rebeccapurple' })], +]; + +describe('calendar planner: uploads of edited events', () => { + it.each(FIXTURES)('round-trips %s: rows untouched → clean', async (_name, make) => { + const event = make(); + const { h, id } = await synced(event); + // An app saving the event without changing anything. + h.fake.user.updateEvent(id, {}); + const plan = calendarPlanner.planUpload((await h.local(event.id))!, h.ctx); + expect(plan.kind).toBe('clean'); + if (plan.kind !== 'clean') return; + await h.apply(plan.ops); + expect(h.row(id)[Events.DIRTY]).toBe(0); + // And the next download of the same event writes nothing. + expect(calendarPlanner.planDownload(event, await h.local(event.id), h.ctx).effect).toBe('none'); + }); + + it.each<[string, Row, Record]>([ + ['title', { [Events.TITLE]: 'New title' }, { title: 'New title' }], + ['description', { [Events.DESCRIPTION]: 'New text' }, { description: 'New text' }], + ['a cleared description', { [Events.DESCRIPTION]: '' }, { description: null }], + ['location', { [Events.EVENT_LOCATION]: 'Room 2' }, { 'locations/locA/name': 'Room 2' }], + ['a cleared location', { [Events.EVENT_LOCATION]: null }, { 'locations/locA': null }], + ['status', { [Events.STATUS]: 1 }, { status: 'confirmed' }], + ['availability', { [Events.AVAILABILITY]: 0 }, { freeBusyStatus: 'busy' }], + ['tentative availability (busy)', { [Events.AVAILABILITY]: 2 }, { freeBusyStatus: 'busy' }], + ['privacy', { [Events.ACCESS_LEVEL]: 3 }, { privacy: 'public' }], + ['default privacy', { [Events.ACCESS_LEVEL]: 0 }, { privacy: null }], + ['colour', { [Events.EVENT_COLOR]: cssColorToArgb('#ff8800') }, { color: '#ff8800' }], + ['duration', { [Events.DURATION]: 'P7200S' }, { duration: 'PT2H' }], + ])('uploads one edited field of a series as a patch of only that field: %s', async (_name, values, patch) => { + const { h, id } = await synced(weekly()); + h.fake.user.updateEvent(id, values); + const action = updateOf(calendarPlanner.planUpload((await h.local('bl'))!, h.ctx)); + expect(action.id).toBe('bl'); + expect(action.patch).toEqual(patch); + }); + + it('adds the first location as a whole map', async () => { + const { h, id } = await synced(single()); + h.fake.user.updateEvent(id, { [Events.EVENT_LOCATION]: 'Café' }); + const action = updateOf(calendarPlanner.planUpload((await h.local('e1'))!, h.ctx)); + const [[key, value]] = Object.entries(action.patch.locations as Record); + expect(Object.keys(action.patch)).toEqual(['locations']); + expect(key).toMatch(/^b[0-9a-z]{8}$/); + expect(value).toEqual({ '@type': 'Location', name: 'Café' }); + }); + + it('uploads an HTML description as text/plain only when it was edited', async () => { + const { h, id } = await synced(single({ description: '

Hi there

', descriptionContentType: 'text/html' })); + expect(h.row(id)[Events.DESCRIPTION]).toBe('Hi there'); + h.fake.user.updateEvent(id, { [Events.DESCRIPTION]: 'Hi there!' }); + expect(updateOf(calendarPlanner.planUpload((await h.local('e1'))!, h.ctx)).patch).toEqual({ + description: 'Hi there!', + descriptionContentType: 'text/plain', + }); + }); + + it('never uploads an edit of a description that was written truncated', async () => { + const long = 'x'.repeat(70_000); + const { h, id } = await synced(single({ description: long })); + expect(String(h.row(id)[Events.DESCRIPTION]).length).toBe(65_536); + h.fake.user.updateEvent(id, { [Events.DESCRIPTION]: 'short now', [Events.TITLE]: 'T' }); + expect(updateOf(calendarPlanner.planUpload((await h.local('e1'))!, h.ctx)).patch).toEqual({ title: 'T' }); + }); + + it('moves a single event: start in its zone, the length unchanged', async () => { + const { h, id } = await synced(single()); + h.fake.user.updateEvent(id, { [Events.DTSTART]: utc('2026-10-06T12:30:00Z'), [Events.DTEND]: utc('2026-10-06T13:30:00Z') }); + expect(updateOf(calendarPlanner.planUpload((await h.local('e1'))!, h.ctx)).patch).toEqual({ start: '2026-10-06T14:30:00' }); + }); + + it('keeps a floating event floating, and takes a zone the user picked', async () => { + const { h, id } = await synced(single({ id: 'f', timeZone: null })); + h.fake.user.updateEvent(id, { [Events.DTSTART]: utc('2026-10-06T11:00:00Z'), [Events.DTEND]: utc('2026-10-06T12:00:00Z') }); + expect(updateOf(calendarPlanner.planUpload((await h.local('f'))!, h.ctx)).patch).toEqual({ start: '2026-10-06T13:00:00' }); + h.fake.user.updateEvent(id, { [Events.EVENT_TIMEZONE]: 'Asia/Tokyo' }); + expect(updateOf(calendarPlanner.planUpload((await h.local('f'))!, h.ctx)).patch).toEqual({ start: '2026-10-06T20:00:00', timeZone: 'Asia/Tokyo' }); + }); + + it('keeps all-day events dates and turns timed ones into all-day ones', async () => { + const { h, id } = await synced(single({ id: 'a', start: '2026-10-08T00:00:00', timeZone: null, showWithoutTime: true, duration: 'P1D' })); + h.fake.user.updateEvent(id, { [Events.DTSTART]: utc('2026-10-09T00:00:00Z'), [Events.DTEND]: utc('2026-10-11T00:00:00Z') }); + expect(updateOf(calendarPlanner.planUpload((await h.local('a'))!, h.ctx)).patch).toEqual({ start: '2026-10-09T00:00:00', duration: 'P2D' }); + + const t = await synced(single()); + t.h.fake.user.updateEvent(t.id, { + [Events.ALL_DAY]: 1, + [Events.EVENT_TIMEZONE]: 'UTC', + [Events.DTSTART]: utc('2026-10-06T00:00:00Z'), + [Events.DTEND]: utc('2026-10-07T00:00:00Z'), + }); + expect(updateOf(calendarPlanner.planUpload((await t.h.local('e1'))!, t.h.ctx)).patch).toEqual({ + start: '2026-10-06T00:00:00', + duration: 'P1D', + timeZone: null, + showWithoutTime: true, + }); + }); + + it('sends a single event whose new start is a repeated wall time in a fixed-offset zone', async () => { + const { h, id } = await synced(single()); + // 02:30 CEST on the night the clocks go back: Stalwart would drop it in Europe/Berlin. + h.fake.user.updateEvent(id, { [Events.DTSTART]: utc('2026-10-25T00:30:00Z'), [Events.DTEND]: utc('2026-10-25T01:30:00Z') }); + expect(updateOf(calendarPlanner.planUpload((await h.local('e1'))!, h.ctx)).patch).toEqual({ start: '2026-10-25T02:30:00', timeZone: 'Etc/GMT-2' }); + // The second 02:30 (CET) is the same wall time an hour later. + h.fake.user.updateEvent(id, { [Events.DTSTART]: utc('2026-10-25T01:30:00Z'), [Events.DTEND]: utc('2026-10-25T02:30:00Z') }); + expect(updateOf(calendarPlanner.planUpload((await h.local('e1'))!, h.ctx)).patch).toEqual({ start: '2026-10-25T02:30:00', timeZone: 'Etc/GMT-1' }); + }); + + it('skips a series whose new start is a repeated wall time', async () => { + const { h, id } = await synced(single({ recurrenceRule: { frequency: 'daily' } })); + h.fake.user.updateEvent(id, { [Events.DTSTART]: utc('2026-10-25T00:30:00Z') }); + expect(calendarPlanner.planUpload((await h.local('e1'))!, h.ctx)).toEqual({ kind: 'skip', reason: 'dstAmbiguous' }); + }); + + it('never uploads the timing of a rule Android cannot represent', async () => { + const event = single({ recurrenceRule: { frequency: 'yearly', rscale: 'chinese', byMonth: ['5L'] } }); + const { h, id } = await synced(event); + expect(h.row(id)[Events.RRULE]).toBe('FREQ=YEARLY'); + h.fake.user.updateEvent(id, { [Events.DTSTART]: utc('2026-10-06T11:00:00Z') }); + expect(calendarPlanner.planUpload((await h.local('e1'))!, h.ctx)).toEqual({ kind: 'skip', reason: 'ruleNotRepresentable' }); + h.fake.user.updateEvent(id, { [Events.TITLE]: 'Festival' }); + expect(updateOf(calendarPlanner.planUpload((await h.local('e1'))!, h.ctx)).patch).toEqual({ title: 'Festival' }); + }); + + it('uploads the user\'s RSVP with scheduling messages', async () => { + const event = single({ + organizerCalendarAddress: 'mailto:boss@example.net', + participants: { + boss: { calendarAddress: 'mailto:boss@example.net', roles: { owner: true } }, + me: { calendarAddress: 'mailto:usera@example.org', roles: { attendee: true }, participationStatus: 'needs-action' }, + }, + } as Partial); + const { h, id } = await synced(event); + h.fake.user.setAttendeeStatus(id, ME, 1); + const action = updateOf(calendarPlanner.planUpload((await h.local('e1'))!, h.ctx)); + expect(action).toMatchObject({ patch: { 'participants/me/participationStatus': 'accepted' }, sendSchedulingMessages: true }); + }); + + it('adds and removes attendees; the organizer notifies them', async () => { + const { h, id } = await synced(weekly()); + const guest = h.fake.rows('attendees', `${Attendees.EVENT_ID} = ? AND ${Attendees.ATTENDEE_EMAIL} = ?`, [id, 'guest@example.net'])[0]; + h.fake.user.removeAttendee(Number(guest._id)); + h.fake.user.addAttendee(id, { + [Attendees.ATTENDEE_EMAIL]: 'new@example.net', + [Attendees.ATTENDEE_NAME]: 'New', + [Attendees.ATTENDEE_TYPE]: 2, + [Attendees.ATTENDEE_RELATIONSHIP]: 1, + [Attendees.ATTENDEE_STATUS]: 3, + }); + const action = updateOf(calendarPlanner.planUpload((await h.local('bl'))!, h.ctx)); + expect(action.sendSchedulingMessages).toBe(true); + const added = Object.entries(action.patch).find(([k, v]) => k.startsWith('participants/') && v); + expect(added![1]).toEqual({ + '@type': 'Participant', + calendarAddress: 'mailto:new@example.net', + name: 'New', + roles: { attendee: true, optional: true }, + participationStatus: 'needs-action', + expectReply: true, + }); + expect(action.patch['participants/guestX']).toBeNull(); + }); + + it('sends the whole participants map for the first attendee', async () => { + const { h, id } = await synced(single()); + h.fake.user.addAttendee(id, { [Attendees.ATTENDEE_EMAIL]: 'guest@example.net', [Attendees.ATTENDEE_TYPE]: 1, [Attendees.ATTENDEE_STATUS]: 3 }); + const action = updateOf(calendarPlanner.planUpload((await h.local('e1'))!, h.ctx)); + expect(Object.keys(action.patch)).toEqual(['participants']); + expect(Object.values(action.patch.participants as object)).toHaveLength(1); + expect(action.sendSchedulingMessages).toBe(true); + }); + + it('replaces the reminders it can show, keeping the matching keys and the others', async () => { + const { h, id } = await synced(weekly()); + h.fake.user.setReminders(id, [{ minutes: 15 }, { minutes: 60, method: 2 }]); + const action = updateOf(calendarPlanner.planUpload((await h.local('bl'))!, h.ctx)); + const keys = Object.keys(action.patch); + expect(keys).toHaveLength(1); + expect(keys[0]).toMatch(/^alerts\/b[0-9a-z]{8}$/); + expect(action.patch[keys[0]]).toEqual({ '@type': 'Alert', trigger: { '@type': 'OffsetTrigger', offset: '-PT1H', relativeTo: 'start' }, action: 'email' }); + // Reminders alone notify nobody. + expect(action.sendSchedulingMessages).toBeUndefined(); + + h.fake.user.setReminders(id, []); + expect(updateOf(calendarPlanner.planUpload((await h.local('bl'))!, h.ctx)).patch).toEqual({ 'alerts/al1': null }); + }); + + it('switches off the calendar defaults when reminders of an event that used them change', async () => { + const { h, id } = await synced(single({ useDefaultAlerts: true })); + h.fake.user.setReminders(id, [{ minutes: 10 }, { minutes: 30 }]); + const patch = updateOf(calendarPlanner.planUpload((await h.local('e1'))!, h.ctx)).patch; + expect(patch.useDefaultAlerts).toBe(false); + const alerts = patch.alerts as Record; + // The default's key is kept for the reminder that matches it. + expect(alerts.d1.trigger.offset).toBe('-PT10M'); + expect(Object.values(alerts).map((a) => a.trigger.offset).sort()).toEqual(['-PT10M', '-PT30M']); + }); + + it('does not upload reminders when Bulwark owns them', async () => { + const { h, id } = await synced(weekly(), { reminderOwner: 'bulwark' }); + h.fake.user.setReminders(id, [{ minutes: 5 }]); + expect(calendarPlanner.planUpload((await h.local('bl'))!, h.ctx).kind).toBe('clean'); + }); + + it('ignores attendee types and relationships a lossy editor rewrote, and merges its rule part by part', async () => { + const event = weekly(); + event.recurrenceRule = { frequency: 'monthly', byDay: [{ day: 'mo' }, { day: 'tu' }], bySetPosition: [-1], count: 10 }; + const { h, id } = await synced(event); + const attendees = h.fake.rows('attendees', `${Attendees.EVENT_ID} = ?`, [id]).map((a) => ({ + [Attendees.ATTENDEE_EMAIL]: a[Attendees.ATTENDEE_EMAIL], + [Attendees.ATTENDEE_NAME]: a[Attendees.ATTENDEE_NAME], + [Attendees.ATTENDEE_RELATIONSHIP]: 1, + [Attendees.ATTENDEE_TYPE]: 1, + [Attendees.ATTENDEE_STATUS]: a[Attendees.ATTENDEE_STATUS], + })); + const reminders = h.fake.rows('reminders', `${Reminders.EVENT_ID} = ?`, [id]).map((r) => ({ minutes: Number(r[Reminders.MINUTES]) })); + // Fossify rebuilds the rule without BYSETPOS and changes the count. + h.fake.user.fossifySaveEvent(id, { [Events.RRULE]: 'FREQ=MONTHLY;COUNT=12;BYDAY=MO,TU' }, attendees, reminders); + h.fake.table('events').get(id)![Events.MUTATORS] = 'org.fossify.calendar'; + const action = updateOf(calendarPlanner.planUpload((await h.local('bl'))!, h.ctx)); + expect(action.patch).toEqual({ + recurrenceRule: { frequency: 'monthly', count: 12, byDay: [{ day: 'mo' }, { day: 'tu' }], bySetPosition: [-1] }, + }); + }); + + it('turns a series into a single event in place', async () => { + const { h, id } = await synced(weekly()); + h.fake.user.updateEvent(id, { [Events.RRULE]: null, [Events.DURATION]: null, [Events.DTEND]: utc('2026-10-05T14:30:00Z') }); + const action = updateOf(calendarPlanner.planUpload((await h.local('bl'))!, h.ctx)); + expect(action.patch).toEqual({ recurrenceRule: null, recurrenceOverrides: null }); + expect(action.sendSchedulingMessages).toBe(true); + }); + + it('moves an event between calendars of its account by swapping calendarIds', async () => { + const { h, id } = await synced(single()); + h.fake.user.updateEvent(id, { [Events.CALENDAR_ID]: h.rowIds.get('w')! }); + expect(updateOf(calendarPlanner.planUpload((await h.local('e1'))!, h.ctx)).patch).toEqual({ 'calendarIds/b': null, 'calendarIds/w': true }); + }); + + it('reverts edits in a read-only calendar, but lets an RSVP through', async () => { + const event = single({ + calendarIds: { r: true }, + organizerCalendarAddress: 'mailto:boss@example.net', + participants: { + boss: { calendarAddress: 'mailto:boss@example.net', roles: { owner: true } }, + me: { calendarAddress: 'mailto:usera@example.org', roles: { attendee: true }, participationStatus: 'needs-action' }, + }, + } as Partial); + const { h, id } = await synced(event); + h.fake.user.updateEvent(id, { [Events.TITLE]: 'Mine now' }); + const plan = calendarPlanner.planUpload((await h.local('e1'))!, h.ctx); + expect(plan.kind).toBe('revert'); + if (plan.kind === 'revert') await h.apply(plan.ops); + expect(h.row(id)).toMatchObject({ [Events.TITLE]: 'Lunch', [Events.DIRTY]: 0 }); + + h.fake.user.setAttendeeStatus(id, ME, 2); + expect(updateOf(calendarPlanner.planUpload((await h.local('e1'))!, h.ctx)).patch).toEqual({ 'participants/me/participationStatus': 'declined' }); + }); + + it('skips a poisoned item until it changes or the back-off ends', async () => { + const { h, id } = await synced(single()); + h.fake.user.updateEvent(id, { [Events.TITLE]: 'Bad' }); + const local = (await h.local('e1'))!; + const { poisonFingerprint, poisonOps } = await import('../../calendar/planner'); + const { nextMarker } = await import('../../engine/poison'); + // The marker as the engine writes it after an invalidProperties SetError. + const marker = nextMarker(null, poisonFingerprint(local), { type: 'invalidProperties' }, h.ctx.now, { firstMs: 3_600_000, maxMs: 86_400_000 }); + await h.apply(poisonOps(local, marker)); + expect(calendarPlanner.planUpload((await h.local('e1'))!, h.ctx)).toEqual({ kind: 'skip', reason: 'invalidProperties' }); + expect(calendarPlanner.planUpload((await h.local('e1'))!, { ...h.ctx, now: h.ctx.now + 7_200_000 }).kind).toBe('upload'); + h.fake.user.updateEvent(id, { [Events.TITLE]: 'Better' }); + expect(calendarPlanner.planUpload((await h.local('e1'))!, h.ctx).kind).toBe('upload'); + }); + + it('applies a patch the server accepts (round trip through serverApply)', async () => { + const { h, id } = await synced(weekly()); + h.fake.user.updateEvent(id, { [Events.TITLE]: 'Renamed', [Events.EVENT_LOCATION]: 'Hall' }); + const action = updateOf(calendarPlanner.planUpload((await h.local('bl'))!, h.ctx)); + const server = serverApply(weekly(), action.patch); + expect(server.title).toBe('Renamed'); + expect(server.locations).toEqual({ locA: { name: 'Hall', '@type': 'Location' } }); + expect(server.participants).toEqual(weekly().participants); + }); +}); diff --git a/src/device-sync/__tests__/calendar/zoned-time.test.ts b/src/device-sync/__tests__/calendar/zoned-time.test.ts new file mode 100644 index 00000000..59d2b7af --- /dev/null +++ b/src/device-sync/__tests__/calendar/zoned-time.test.ts @@ -0,0 +1,81 @@ +import { describe, expect, it } from 'vitest'; +import { + canonicalZone, + classifyLocal, + dateToUtcMidnight, + fixedOffsetZone, + isValidTimeZone, + localToUtc, + offsetAt, + parseLocalDateTime, + utcMidnightToDate, + utcToLocal, +} from '../../calendar/zoned-time'; + +const utc = (iso: string) => Date.parse(iso); + +describe('calendar/zoned-time', () => { + it('reads LocalDateTimes like Stalwart: fraction and suffix ignored, impossible dates refused', () => { + expect(parseLocalDateTime('2026-10-25T02:30:00')).toEqual({ year: 2026, month: 10, day: 25, hour: 2, minute: 30, second: 0 }); + expect(parseLocalDateTime('2026-10-25T02:30:00.250Z')).toMatchObject({ hour: 2, minute: 30, second: 0 }); + expect(parseLocalDateTime('2026-10-25')).toMatchObject({ hour: 0, minute: 0 }); + expect(parseLocalDateTime('2026-02-30T00:00:00')).toBeNull(); + expect(parseLocalDateTime('2026-01-01T24:00:00')).toBeNull(); + expect(parseLocalDateTime('garbage')).toBeNull(); + }); + + // R3 B1: the app's localDateTimeToInstant gets these wrong in one direction or the other. + it.each([ + ['America/New_York', '2026-03-08T02:30:00', 'gap', '2026-03-08T07:30:00Z'], + ['America/New_York', '2026-11-01T01:30:00', 'overlap', '2026-11-01T05:30:00Z'], + ['Europe/Berlin', '2026-03-29T02:30:00', 'gap', '2026-03-29T01:30:00Z'], + ['Europe/Berlin', '2026-10-25T02:30:00', 'overlap', '2026-10-25T00:30:00Z'], + ['Australia/Sydney', '2026-04-05T02:30:00', 'overlap', '2026-04-04T15:30:00Z'], + ['Australia/Sydney', '2026-10-04T02:30:00', 'gap', '2026-10-03T16:30:00Z'], + ['Australia/Lord_Howe', '2026-10-04T02:15:00', 'gap', '2026-10-03T15:45:00Z'], + ['Europe/Berlin', '2026-10-25T04:30:00', 'ok', '2026-10-25T03:30:00Z'], + ['Asia/Kolkata', '2026-06-01T10:00:00', 'ok', '2026-06-01T04:30:00Z'], + ])('resolves %s %s (%s) with RFC 5545 semantics', (zone, local, kind, expected) => { + expect(classifyLocal(local, zone)).toBe(kind); + expect(new Date(localToUtc(local, zone)!).toISOString().replace('.000', '')).toBe(expected); + }); + + it('moves a gap time forward by the gap and keeps the wall time of an overlap', () => { + expect(utcToLocal(localToUtc('2026-03-29T02:30:00', 'Europe/Berlin')!, 'Europe/Berlin')).toBe('2026-03-29T03:30:00'); + expect(utcToLocal(localToUtc('2026-03-08T02:30:00', 'America/New_York')!, 'America/New_York')).toBe('2026-03-08T03:30:00'); + // Both instants of the repeated hour read back as the same wall time. + expect(utcToLocal(utc('2026-10-25T00:30:00Z'), 'Europe/Berlin')).toBe('2026-10-25T02:30:00'); + expect(utcToLocal(utc('2026-10-25T01:30:00Z'), 'Europe/Berlin')).toBe('2026-10-25T02:30:00'); + }); + + it('does not depend on the zone the process runs in', () => { + // The device's own gap day is no special case: fields are parsed, not handed to Date. + expect(localToUtc('2026-03-29T02:30:00', 'UTC')).toBe(utc('2026-03-29T02:30:00Z')); + expect(offsetAt(utc('2026-07-01T00:00:00Z'), 'Europe/Berlin')).toBe(7_200_000); + expect(offsetAt(utc('2026-01-01T00:00:00Z'), 'America/New_York')).toBe(-18_000_000); + }); + + it('validates zones with Intl and names them canonically', () => { + expect(isValidTimeZone('Europe/Berlin')).toBe(true); + expect(isValidTimeZone('Mars/Olympus_Mons')).toBe(false); + expect(isValidTimeZone('')).toBe(false); + expect(isValidTimeZone(null)).toBe(false); + expect(canonicalZone('europe/berlin')).toBe('Europe/Berlin'); + expect(canonicalZone('US/Eastern')).toBe('America/New_York'); + }); + + it('names fixed-offset zones the Etc way, UTC for fractional offsets', () => { + expect(fixedOffsetZone(7_200_000)).toBe('Etc/GMT-2'); + expect(fixedOffsetZone(-18_000_000)).toBe('Etc/GMT+5'); + expect(fixedOffsetZone(0)).toBe('Etc/UTC'); + expect(fixedOffsetZone(19_800_000)).toBe('Etc/UTC'); + }); + + it('maps all-day dates to UTC midnight and back', () => { + expect(dateToUtcMidnight('2026-10-08T00:00:00')).toBe(utc('2026-10-08T00:00:00Z')); + expect(dateToUtcMidnight('2026-10-08T09:00:00')).toBe(utc('2026-10-08T00:00:00Z')); + expect(utcMidnightToDate(utc('2026-10-08T00:00:00Z'))).toBe('2026-10-08T00:00:00'); + // Date.UTC would read year 50 as 1950. + expect(new Date(dateToUtcMidnight('0050-01-01T00:00:00')!).toISOString()).toBe('0050-01-01T00:00:00.000Z'); + }); +}); diff --git a/src/device-sync/__tests__/fakes/fake-provider.ts b/src/device-sync/__tests__/fakes/fake-provider.ts index 3a611bb2..1acb2f6a 100644 --- a/src/device-sync/__tests__/fakes/fake-provider.ts +++ b/src/device-sync/__tests__/fakes/fake-provider.ts @@ -98,7 +98,7 @@ const COLUMNS: Record> = { data: new Set([...Object.values(Data), GroupMembership.GROUP_SOURCE_ID]), groups: new Set([...Object.values(Groups), 'auto_add', 'favorites']), settings: new Set(Object.values(ContactsSettings)), - calendars: new Set([...Object.values(Calendars), ...numbered('cal_sync', 4, 10), 'canPartiallyUpdate']), + calendars: new Set([...Object.values(Calendars), ...numbered('cal_sync', 4, 10)]), events: new Set([...Object.values(Events), ...numbered('sync_data', 7, 10), 'lastSynced']), attendees: new Set(Object.values(Attendees)), reminders: new Set(Object.values(Reminders)), diff --git a/src/device-sync/android-columns.ts b/src/device-sync/android-columns.ts index dc689821..e53eb262 100644 --- a/src/device-sync/android-columns.ts +++ b/src/device-sync/android-columns.ts @@ -231,6 +231,8 @@ export const Calendars = { MAX_REMINDERS: 'maxReminders', CAN_ORGANIZER_RESPOND: 'canOrganizerRespond', CAN_MODIFY_TIME_ZONE: 'canModifyTimeZone', + /** Must stay 0: with 1 the provider keeps a LAST_SYNCED copy of every event an app edits. */ + CAN_PARTIALLY_UPDATE: 'canPartiallyUpdate', IS_PRIMARY: 'isPrimary', CAL_SYNC1: 'cal_sync1', CAL_SYNC2: 'cal_sync2', diff --git a/src/device-sync/calendar/accepted.ts b/src/device-sync/calendar/accepted.ts new file mode 100644 index 00000000..f26c35f0 --- /dev/null +++ b/src/device-sync/calendar/accepted.ts @@ -0,0 +1,90 @@ +/** + * After the server accepted an upload (docs/device-sync.md, "Uploads", + * "Afterwards"): when the rows were edited again while the upload was under + * way, the clearing group's assert fails and `keepDirtyOps` apply instead. + * They write the identity and the new shadow, and set the baselines of the + * uploaded units to the values that were uploaded (the rows as read before + * the upload), leaving the rows and DIRTY alone: the newer edit, or a revert + * to the old value, uploads next time. + */ +import { Events } from '../android-columns'; +import type { CalendarContext, EventBaseline, LocalEvent, LocalEventRow, OpGroup } from '../planner'; +import type { CalendarEventWire } from '../wire'; +import { exceptionRef, objectRef } from '../common/ids'; +import { clone } from '../common/json'; +import { EXCEPTION_CELLS, MASTER_CELLS, encodeBaseline, makeBaseline } from './columns'; +import { COLUMN_UNITS } from './units'; + +export interface UploadedUnits { + /** Everything was uploaded (a create). */ + all: boolean; + master: Set; + /** By exception row id. */ + exceptions: Map>; + /** Exception rows whose instance is excluded on the server now. */ + removed: Set; +} + +function unitColumns(unit: string): readonly string[] { + return COLUMN_UNITS[unit] ?? (unit.startsWith('exdate:') ? [Events.EXDATE] : []); +} + +function uploadedBaseline(row: LocalEventRow, isException: boolean, units: Set | null, all: boolean): EventBaseline { + const uploaded = makeBaseline( + row.cells, + isException ? EXCEPTION_CELLS : MASTER_CELLS, + row.attendees.map((a) => a.cells), + row.reminders.map((r) => r.cells), + ); + if (all || !row.baseline) return uploaded; + const next = clone(row.baseline); + for (const unit of units ?? []) { + for (const c of unitColumns(unit)) next.cells[c] = uploaded.cells[c] ?? null; + if (unit === 'reminders') next.reminders = uploaded.reminders; + if (unit.startsWith('attendee')) next.attendees = uploaded.attendees; + } + return next; +} + +export function keepDirtyGroup(local: LocalEvent, server: CalendarEventWire, uploaded: UploadedUnits, ctx: CalendarContext): OpGroup { + const masterRef = objectRef(ctx.jmapAccountId, server.id); + const ops: OpGroup['ops'] = [ + { + op: 'update', + table: 'events', + id: local.eventId, + values: { + [Events._SYNC_ID]: masterRef, + [Events.UID_2445]: typeof server.uid === 'string' ? server.uid : null, + [Events.SYNC_DATA1]: JSON.stringify(server), + [Events.SYNC_DATA3]: null, + [Events.SYNC_DATA4]: encodeBaseline(uploadedBaseline(local, false, uploaded.master, uploaded.all)), + [Events.SYNC_DATA5]: null, + }, + expectCount: 1, + }, + ]; + for (const x of local.exceptions) { + // An instance deleted on the device is an exclusion on the server now: its row has done its job. + if (uploaded.removed.has(x.eventId)) { + ops.push({ op: 'delete', table: 'events', id: x.eventId, expectCount: 1 }); + continue; + } + const units = uploaded.exceptions.get(x.eventId); + if ((!uploaded.all && !units) || !x.recurrenceId) continue; + const all = uploaded.all || !!units?.has('new'); + ops.push({ + op: 'update', + table: 'events', + id: x.eventId, + values: { + [Events._SYNC_ID]: exceptionRef(masterRef, x.recurrenceId), + [Events.SYNC_DATA2]: x.recurrenceId, + [Events.ORIGINAL_SYNC_ID]: masterRef, + [Events.SYNC_DATA4]: encodeBaseline(uploadedBaseline(x, true, units ?? null, all)), + }, + expectCount: 1, + }); + } + return { ref: masterRef, ops }; +} diff --git a/src/device-sync/calendar/attendees.ts b/src/device-sync/calendar/attendees.ts new file mode 100644 index 00000000..0e748677 --- /dev/null +++ b/src/device-sync/calendar/attendees.ts @@ -0,0 +1,165 @@ +/** + * Participants ↔ Attendees rows (docs/device-sync.md, "Other properties"). + * + * Only participants with a `mailto:` calendar address become rows; the rest + * stay on the server untouched. Android decides who "me" is by comparing + * ATTENDEE_EMAIL with the calendar's OWNER_ACCOUNT case-sensitively, so the + * user's own address, under whichever alias, is written as exactly + * OWNER_ACCOUNT. Stalwart can list one address twice (the organizer derived + * from ORGANIZER plus its ATTENDEE entry); such entries become one row. + * Participant ids never reach the device: rows are matched to participants + * by address. + */ +import type { Participant } from '../../api/types'; +import { AttendeeRelationship, Attendees } from '../android-columns'; +import type { Row } from '../types'; +import { attendeeTypeToDevice, participationToDevice, relationshipToDevice } from './values'; + +export interface SelfContext { + /** OWNER_ACCOUNT of our calendars, lowercase, no `mailto:`. */ + ownerAccount: string; + /** Every address of the user, lowercase, no `mailto:`. */ + selfAddresses: string[]; +} + +type Participants = Record | null | undefined; + +/** The address of a `mailto:` calendar address (lowercased), else null. */ +export function mailtoAddress(p: Participant | null | undefined): string | null { + const addr = p?.calendarAddress; + if (typeof addr !== 'string') return null; + const m = /^mailto:(.+)$/i.exec(addr.trim()); + if (!m) return null; + let email = m[1].trim(); + try { + email = decodeURIComponent(email); + } catch { + // Keep it as written. + } + return email ? email.toLowerCase() : null; +} + +export function isSelfAddress(email: string | null | undefined, ctx: SelfContext): boolean { + if (!email) return false; + const e = email.trim().replace(/^mailto:/i, '').toLowerCase(); + return e === ctx.ownerAccount.toLowerCase() || ctx.selfAddresses.some((a) => a.toLowerCase() === e); +} + +/** The address as written to ATTENDEE_EMAIL / ORGANIZER: the user's own ones as OWNER_ACCOUNT. */ +export function deviceAddress(email: string, ctx: SelfContext): string { + return isSelfAddress(email, ctx) ? ctx.ownerAccount : email; +} + +/** A device address as the key rows and participants are matched by (the user's aliases are one key). */ +export function addressKey(email: unknown, ctx: SelfContext): string { + const e = String(email ?? '').trim().replace(/^mailto:/i, '').toLowerCase(); + return isSelfAddress(e, ctx) ? `self:${ctx.ownerAccount.toLowerCase()}` : e; +} + +interface AddressGroup { + key: string; + email: string; + ids: string[]; + preferred: string; +} + +/** + * Participants with a `mailto:` address grouped by address, in server + * order. The preferred entry of a group is the one that attends (Stalwart's + * organizer entry has no `attendee` role), then one with a status. + */ +export function addressGroups(participants: Participants, ctx: SelfContext): AddressGroup[] { + const groups = new Map(); + for (const [id, p] of Object.entries(participants ?? {})) { + const email = mailtoAddress(p); + if (!email) continue; + const key = addressKey(email, ctx); + const group = groups.get(key); + if (group) group.ids.push(id); + else groups.set(key, { key, email, ids: [id], preferred: id }); + } + const list = participants ?? {}; + for (const group of groups.values()) { + const score = (id: string) => (list[id]?.roles?.attendee ? 2 : 0) + (list[id]?.participationStatus ? 1 : 0); + group.preferred = group.ids.reduce((best, id) => (score(id) > score(best) ? id : best), group.ids[0]); + } + return [...groups.values()]; +} + +/** The organizer's address for ORGANIZER: `organizerCalendarAddress`, else an owner participant's. */ +export function organizerAddress( + event: { organizerCalendarAddress?: string | null; participants?: Participants }, + ctx: SelfContext, +): string | null { + const direct = event.organizerCalendarAddress + ? mailtoAddress({ calendarAddress: /^mailto:/i.test(event.organizerCalendarAddress) ? event.organizerCalendarAddress : `mailto:${event.organizerCalendarAddress}` }) + : null; + if (direct) return deviceAddress(direct, ctx); + for (const p of Object.values(event.participants ?? {})) { + if (p?.roles?.owner) { + const email = mailtoAddress(p); + if (email) return deviceAddress(email, ctx); + } + } + return null; +} + +/** Whether the event has participants that become Attendees rows. */ +export function hasMailtoParticipants(participants: Participants): boolean { + return Object.values(participants ?? {}).some((p) => mailtoAddress(p) !== null); +} + +/** Attendees rows for the participants, one per address, in server order. */ +export function attendeeRows( + event: { organizerCalendarAddress?: string | null; participants?: Participants }, + ctx: SelfContext, +): Row[] { + const participants = event.participants ?? {}; + const organizer = organizerAddress(event, ctx); + return addressGroups(participants, ctx).map((group) => { + const p = participants[group.preferred]; + const email = deviceAddress(group.email, ctx); + const isOrganizer = + group.ids.some((id) => participants[id]?.roles?.owner) || (organizer !== null && organizer.toLowerCase() === email.toLowerCase()); + const name = p.name ?? group.ids.map((id) => participants[id]?.name).find((n) => !!n) ?? null; + return { + [Attendees.ATTENDEE_EMAIL]: email, + [Attendees.ATTENDEE_NAME]: name || null, + [Attendees.ATTENDEE_RELATIONSHIP]: relationshipToDevice(isOrganizer), + [Attendees.ATTENDEE_TYPE]: attendeeTypeToDevice(p), + [Attendees.ATTENDEE_STATUS]: participationToDevice(p.participationStatus), + }; + }); +} + +/** The participant id a device address stands for (the group's preferred entry), or null. */ +export function participantIdFor(participants: Participants, email: unknown, ctx: SelfContext): string | null { + const key = addressKey(email, ctx); + return addressGroups(participants, ctx).find((g) => g.key === key)?.preferred ?? null; +} + +/** Whether an attendee row is the organizer's. */ +export function isOrganizerRow(row: Row): boolean { + return Number(row[Attendees.ATTENDEE_RELATIONSHIP]) === AttendeeRelationship.ORGANIZER; +} + +/** Whether anyone but the user takes part. */ +export function hasOtherParticipants(participants: Participants, ctx: SelfContext): boolean { + return Object.values(participants ?? {}).some((p) => { + const email = mailtoAddress(p); + return email !== null && !isSelfAddress(email, ctx); + }); +} + +/** + * Whether the user organizes the event: the organizer address or an owner + * participant is theirs. An event without an organizer counts too: once it + * has attendees Stalwart makes the account's first address its organizer. + */ +export function userIsOrganizer( + event: { organizerCalendarAddress?: string | null; participants?: Participants }, + ctx: SelfContext, +): boolean { + const organizer = organizerAddress(event, ctx); + return organizer ? isSelfAddress(organizer, ctx) : true; +} diff --git a/src/device-sync/calendar/calendars.ts b/src/device-sync/calendar/calendars.ts new file mode 100644 index 00000000..d1fe6302 --- /dev/null +++ b/src/device-sync/calendar/calendars.ts @@ -0,0 +1,141 @@ +/** + * JMAP calendars ↔ CalendarContract.Calendars rows (docs/device-sync.md, + * "Calendar rows"). + * + * Every column CalendarProvider and the calendar apps need is written: + * SYNC_EVENTS=1 (the default 0 hides every instance), a non-null + * OWNER_ACCOUNT (Etar crashes on NULL), CAN_PARTIALLY_UPDATE=0 (else the + * provider keeps LAST_SYNCED copies), MAX_REMINDERS by who owns reminders. + * VISIBLE is written on insert only: afterwards it is the user's. CAL_SYNC1 + * stays empty because the provider sends it as the `feed` extra of the sync + * it requests for such calendars. A row is updated only where it differs. + */ +import { CalendarAccess, Calendars } from '../android-columns'; +import type { CalendarContext, LocalCalendar, OpGroup } from '../planner'; +import type { Row, WriteRow } from '../types'; +import type { CalendarLike } from '../wire'; +import { collectionKey, parseCollectionKey } from '../common/ids'; +import { deepEqual, parseJsonColumn } from '../common/json'; +import { BIRTHDAY_CALENDAR_ID } from '../../lib/birthday-calendar'; +import { colors } from '../../theme/tokens'; +import { isSelfAddress } from './attendees'; +import { CAN_PARTIALLY_UPDATE } from './columns'; +import { cssColorToArgb } from './values'; +import { canonicalZone } from './zoned-time'; + +export interface SelectedCalendar { + jmapAccountId: string; + calendar: CalendarLike; + readOnly: boolean; + /** The JMAP account's name, appended to calendars of shared accounts. */ + accountName: string; +} + +type CalendarsContext = Omit; + +/** The app's palette colour for a calendar without one (`getCalendarColor` in lib/calendar-utils.ts). */ +const PALETTE = [ + colors.calendar.blue, + colors.calendar.green, + colors.calendar.purple, + colors.calendar.orange, + colors.calendar.red, + colors.calendar.pink, + colors.calendar.teal, + colors.calendar.indigo, +]; + +function paletteColor(id: string): string { + let hash = 0; + for (let i = 0; i < id.length; i++) hash = (hash * 31 + id.charCodeAt(i)) | 0; + return PALETTE[Math.abs(hash) % PALETTE.length]; +} + +/** + * CALENDAR_ACCESS_LEVEL from `myRights`: write all → OWNER, write own → + * CONTRIBUTOR (Etar's threshold for editing), RSVP → RESPOND, else READ. A + * read-only calendar whose rights allow writing is a subscribed feed: READ. + */ +export function accessLevel(calendar: CalendarLike, readOnly: boolean): { level: number; reason: 'rights' | 'subscription' | null } { + const rights = calendar.myRights; + const writable = !rights || !!rights.mayWriteAll || !!rights.mayWriteOwn; + if (readOnly && writable) return { level: CalendarAccess.READ, reason: 'subscription' }; + const level = !rights || rights.mayWriteAll + ? CalendarAccess.OWNER + : rights.mayWriteOwn + ? CalendarAccess.CONTRIBUTOR + : rights.mayRSVP + ? CalendarAccess.RESPOND + : CalendarAccess.READ; + return { level: readOnly && level >= CalendarAccess.CONTRIBUTOR ? CalendarAccess.RESPOND : level, reason: readOnly ? 'rights' : null }; +} + +/** The columns device sync keeps on a calendar row (VISIBLE aside). */ +export function calendarValues(selected: SelectedCalendar, ctx: CalendarsContext): WriteRow { + const { calendar } = selected; + const shared = !!selected.accountName && !isSelfAddress(selected.accountName, ctx); + const name = calendar.name || calendar.id; + const access = accessLevel(calendar, selected.readOnly); + const flags = access.reason ? { readOnly: access.reason } : {}; + return { + [Calendars._SYNC_ID]: collectionKey(selected.jmapAccountId, calendar.id), + [Calendars.NAME]: name, + [Calendars.CALENDAR_DISPLAY_NAME]: shared ? `${name} (${selected.accountName})` : name, + [Calendars.CALENDAR_COLOR]: cssColorToArgb(calendar.color) ?? cssColorToArgb(paletteColor(calendar.id))!, + [Calendars.CALENDAR_ACCESS_LEVEL]: access.level, + [Calendars.OWNER_ACCOUNT]: ctx.ownerAccount, + [Calendars.SYNC_EVENTS]: 1, + [Calendars.CALENDAR_TIME_ZONE]: canonicalZone(calendar.timeZone) ?? ctx.deviceZone, + [Calendars.ALLOWED_REMINDERS]: '1,2', + [Calendars.ALLOWED_AVAILABILITY]: '0,1', + [Calendars.ALLOWED_ATTENDEE_TYPES]: '0,1,2,3', + [Calendars.MAX_REMINDERS]: ctx.reminderOwner === 'bulwark' ? 0 : 5, + [Calendars.CAN_ORGANIZER_RESPOND]: 0, + [CAN_PARTIALLY_UPDATE]: 0, + [Calendars.CAL_SYNC2]: JSON.stringify(calendar), + [Calendars.CAL_SYNC3]: JSON.stringify(flags), + }; +} + +const JSON_COLUMNS = new Set([Calendars.CAL_SYNC2, Calendars.CAL_SYNC3]); + +function differs(column: string, have: Row[string] | undefined, want: WriteRow[string]): boolean { + if (JSON_COLUMNS.has(column)) return !deepEqual(parseJsonColumn(have), parseJsonColumn(want)); + if (typeof want === 'number') return Number(have) !== want || have === null || have === undefined; + return (have ?? null) !== (want ?? null); +} + +export function planCalendars( + selected: SelectedCalendar[], + local: LocalCalendar[], + ctx: CalendarsContext, +): { groups: OpGroup[]; deleteCalendarRows: number[] } { + const groups: OpGroup[] = []; + const rows = new Map(); + for (const row of local) if (row.syncId && !rows.has(row.syncId)) rows.set(row.syncId, row); + const wanted = new Set(); + for (const s of selected) { + // The birthday calendar is the app's own, generated from contacts. + if (s.calendar.id === BIRTHDAY_CALENDAR_ID) continue; + const key = collectionKey(s.jmapAccountId, s.calendar.id); + wanted.add(key); + const values = calendarValues(s, ctx); + const row = rows.get(key); + if (!row) { + // DIRTY is NULL after a sync-adapter insert unless written. + groups.push({ ref: key, ops: [{ op: 'insert', table: 'calendars', values: { ...values, [Calendars.VISIBLE]: 1, [Calendars.DIRTY]: 0 } }] }); + continue; + } + const changes: WriteRow = {}; + for (const [column, want] of Object.entries(values)) { + if (differs(column, row.cells[column], want)) changes[column] = want; + } + if (Object.keys(changes).length) { + groups.push({ ref: key, ops: [{ op: 'update', table: 'calendars', id: row.calendarRowId, values: changes, expectCount: 1 }] }); + } + } + const deleteCalendarRows = local + .filter((row) => row.syncId && parseCollectionKey(row.syncId) && !wanted.has(row.syncId)) + .map((row) => row.calendarRowId); + return { groups, deleteCalendarRows }; +} diff --git a/src/device-sync/calendar/columns.ts b/src/device-sync/calendar/columns.ts new file mode 100644 index 00000000..c887d253 --- /dev/null +++ b/src/device-sync/calendar/columns.ts @@ -0,0 +1,263 @@ +/** + * What the calendar planner reads and writes on each CalendarContract row, + * how a cell compares to another (an app's `''` is the provider's NULL, a + * `P3600S` is a `PT1H`) and the baseline format in SYNC_DATA4 + * (docs/device-sync.md, "Calendar columns" and "Baselines"). + */ +import { Attendees, Calendars, Events, Reminders } from '../android-columns'; +import type { Cell, Row } from '../types'; +import type { EventBaseline } from '../planner'; +import { parseJsonColumn, utf8 } from '../common/json'; +import { sha256Hex } from '../../lib/sha256'; +import { canonicalRRule } from './rrule'; +import { exdateSet } from './exdate'; +import { durationTextSeconds } from './duration'; + +/** Calendars.CAN_PARTIALLY_UPDATE, re-exported for the calendar modules. */ +export const CAN_PARTIALLY_UPDATE = Calendars.CAN_PARTIALLY_UPDATE; + +export const CALENDAR_COLUMNS = [ + Calendars._ID, + Calendars._SYNC_ID, + Calendars.NAME, + Calendars.CALENDAR_DISPLAY_NAME, + Calendars.CALENDAR_COLOR, + Calendars.CALENDAR_ACCESS_LEVEL, + Calendars.OWNER_ACCOUNT, + Calendars.VISIBLE, + Calendars.SYNC_EVENTS, + Calendars.CALENDAR_TIME_ZONE, + Calendars.ALLOWED_REMINDERS, + Calendars.ALLOWED_AVAILABILITY, + Calendars.ALLOWED_ATTENDEE_TYPES, + Calendars.MAX_REMINDERS, + Calendars.CAN_ORGANIZER_RESPOND, + CAN_PARTIALLY_UPDATE, + Calendars.CAL_SYNC2, + Calendars.CAL_SYNC3, +] as const; + +/** Event columns that map to JSCalendar (a master's projection). */ +export const MASTER_CELLS = [ + Events.CALENDAR_ID, + Events.TITLE, + Events.DESCRIPTION, + Events.EVENT_LOCATION, + Events.STATUS, + Events.AVAILABILITY, + Events.ACCESS_LEVEL, + Events.EVENT_COLOR, + Events.ORGANIZER, + Events.DTSTART, + Events.DTEND, + Events.DURATION, + Events.EVENT_TIMEZONE, + Events.ALL_DAY, + Events.RRULE, + Events.EXDATE, +] as const; + +/** An exception row's projection: no rule of its own, plus the instance it replaces. */ +export const EXCEPTION_CELLS = [ + Events.CALENDAR_ID, + Events.TITLE, + Events.DESCRIPTION, + Events.EVENT_LOCATION, + Events.STATUS, + Events.AVAILABILITY, + Events.ACCESS_LEVEL, + Events.EVENT_COLOR, + Events.ORGANIZER, + Events.DTSTART, + Events.DTEND, + Events.DURATION, + Events.EVENT_TIMEZONE, + Events.ALL_DAY, + Events.ORIGINAL_INSTANCE_TIME, + Events.ORIGINAL_ALL_DAY, +] as const; + +/** Read but never part of a projection: identity, sync state and provider-computed columns. */ +const EVENT_EXTRA_COLUMNS = [ + Events._ID, + Events._SYNC_ID, + Events.DIRTY, + Events.DELETED, + Events.MUTATORS, + Events.SYNC_DATA1, + Events.SYNC_DATA2, + Events.SYNC_DATA3, + Events.SYNC_DATA4, + Events.SYNC_DATA5, + Events.UID_2445, + Events.HAS_ATTENDEE_DATA, + Events.RDATE, + Events.EXRULE, + Events.LAST_DATE, + Events.ORIGINAL_ID, + Events.ORIGINAL_SYNC_ID, +] as const; + +export const EVENT_COLUMNS = [...new Set([...EVENT_EXTRA_COLUMNS, ...MASTER_CELLS, ...EXCEPTION_CELLS])]; + +export const ATTENDEE_CELLS = [ + Attendees.ATTENDEE_EMAIL, + Attendees.ATTENDEE_NAME, + Attendees.ATTENDEE_RELATIONSHIP, + Attendees.ATTENDEE_TYPE, + Attendees.ATTENDEE_STATUS, +] as const; +export const ATTENDEE_COLUMNS = [Attendees._ID, Attendees.EVENT_ID, ...ATTENDEE_CELLS]; + +export const REMINDER_CELLS = [Reminders.MINUTES, Reminders.METHOD] as const; +export const REMINDER_COLUMNS = [Reminders._ID, Reminders.EVENT_ID, ...REMINDER_CELLS]; + +// ─── Cells ────────────────────────────────────────────── + +const TEXT_COLUMNS = new Set([ + Events.TITLE, + Events.DESCRIPTION, + Events.EVENT_LOCATION, + Events.ORGANIZER, + Events.EVENT_TIMEZONE, + Attendees.ATTENDEE_EMAIL, + Attendees.ATTENDEE_NAME, +]); + +/** A cell as the bridge may hand it back: numbers stay numbers, `''` and NULL are one value for text. */ +export function normalCell(column: string, value: unknown): Cell { + if (value === undefined || value === null) return TEXT_COLUMNS.has(column) ? '' : null; + if (TEXT_COLUMNS.has(column)) return String(value); + if (typeof value === 'number') return value; + if (typeof value === 'string') { + if (value === '') return null; + const n = Number(value); + return Number.isFinite(n) && /^-?\d+(\.\d+)?$/.test(value.trim()) ? n : value; + } + return null; +} + +/** Whether two values of a column mean the same, the way device sync reads them. */ +export function sameCell(column: string, a: unknown, b: unknown): boolean { + if (column === Events.DURATION) { + const sa = durationTextSeconds(a); + const sb = durationTextSeconds(b); + if (sa !== null || sb !== null) return sa === sb; + return empty(a) === empty(b); + } + if (column === Events.RRULE) return canonicalRRule(a) === canonicalRRule(b); + if (column === Events.EXDATE) return sameSet(exdateSet(a), exdateSet(b)); + if (column === Events.ALL_DAY || column === Events.ORIGINAL_ALL_DAY) return truthy(a) === truthy(b); + if (isHashed(a) || isHashed(b)) return textDigest(a) === textDigest(b); + return normalCell(column, a) === normalCell(column, b); +} + +const empty = (v: unknown) => v === null || v === undefined || v === ''; +const truthy = (v: unknown) => Number(v ?? 0) === 1; + +function sameSet(a: Set, b: Set): boolean { + return a.size === b.size && [...a].every((x) => b.has(x)); +} + +// ─── Long text in baselines ───────────────────────────── + +const HASH_PREFIX = 'sha256:'; +const TRUNCATED_PREFIX = 'truncated:'; +const LONG_TEXT_BYTES = 1024; + +/** The largest description written to a row; longer ones are cut, marked in the baseline and never uploaded. */ +export const DESCRIPTION_LIMIT_BYTES = 65_536; + +function isHashed(v: unknown): boolean { + return typeof v === 'string' && (v.startsWith(HASH_PREFIX) || v.startsWith(TRUNCATED_PREFIX + HASH_PREFIX)); +} + +/** Text over 1 KB as `sha256:`, anything else unchanged. */ +export function baselineText(value: string): string { + return utf8(value).length > LONG_TEXT_BYTES ? HASH_PREFIX + sha256Hex(utf8(value)) : value; +} + +/** A truncated description's baseline: its hash, marked so a later edit is never uploaded. */ +export function truncatedBaseline(value: string): string { + return TRUNCATED_PREFIX + HASH_PREFIX + sha256Hex(utf8(value)); +} + +export function isTruncatedBaseline(value: unknown): boolean { + return typeof value === 'string' && value.startsWith(TRUNCATED_PREFIX); +} + +/** What a text cell or its baseline form is compared by: the hash for hashed or long text. */ +function textDigest(v: unknown): string { + if (typeof v !== 'string') return empty(v) ? '' : String(v); + if (v.startsWith(TRUNCATED_PREFIX)) return v.slice(TRUNCATED_PREFIX.length); + if (v.startsWith(HASH_PREFIX)) return v; + return utf8(v).length > LONG_TEXT_BYTES ? HASH_PREFIX + sha256Hex(utf8(v)) : v; +} + +/** Cuts text to at most `limit` UTF-8 bytes without splitting a character. */ +export function truncateUtf8(text: string, limit = DESCRIPTION_LIMIT_BYTES): { text: string; truncated: boolean } { + if (utf8(text).length <= limit) return { text, truncated: false }; + let lo = 0; + let hi = text.length; + while (lo < hi) { + const mid = Math.ceil((lo + hi) / 2); + if (utf8(text.slice(0, mid)).length <= limit) lo = mid; + else hi = mid - 1; + } + let end = lo; + // Never end on the first half of a surrogate pair. + if (end > 0 && /[\uD800-\uDBFF]/.test(text[end - 1])) end -= 1; + return { text: text.slice(0, end), truncated: true }; +} + +// ─── Baselines (SYNC_DATA4) ───────────────────────────── + +/** + * The columns of a row, each in its normal form (numbers as numbers even when + * the provider handed them back as text), so a read-back never differs from + * a baseline by type alone. + */ +function normalRow(row: Row, columns: readonly string[]): Row { + const out: Row = {}; + for (const c of columns) { + const v = row[c] ?? null; + out[c] = v === null || TEXT_COLUMNS.has(c) || typeof v !== 'string' ? v : normalCell(c, v); + } + return out; +} + +/** A baseline from cells as stored: normal forms, long text hashed, a truncated description marked. */ +export function makeBaseline( + cells: Row, + columns: readonly string[], + attendees: Row[], + reminders: Row[], + truncatedDescription = false, +): EventBaseline { + const out = normalRow(cells, columns); + for (const c of columns) { + const v = out[c]; + if (c === Events.DESCRIPTION && truncatedDescription && typeof v === 'string') out[c] = truncatedBaseline(v); + else if (typeof v === 'string' && !isHashed(v)) out[c] = baselineText(v); + } + return { + cells: out, + attendees: attendees.map((a) => normalRow(a, ATTENDEE_CELLS)), + reminders: reminders.map((r) => normalRow(r, REMINDER_CELLS)), + }; +} + +export function encodeBaseline(baseline: EventBaseline): string { + return JSON.stringify({ v: 1, ...baseline }); +} + +/** SYNC_DATA4 read defensively: another app may have written anything there. */ +export function decodeBaseline(text: unknown): EventBaseline | null { + const parsed = parseJsonColumn>(text); + if (!parsed || typeof parsed !== 'object' || Array.isArray(parsed)) return null; + const cells = parsed.cells; + if (!cells || typeof cells !== 'object' || Array.isArray(cells)) return null; + const rows = (v: unknown): Row[] => + Array.isArray(v) ? v.filter((r): r is Row => !!r && typeof r === 'object' && !Array.isArray(r)) : []; + return { cells: cells as Row, attendees: rows(parsed.attendees), reminders: rows(parsed.reminders) }; +} diff --git a/src/device-sync/calendar/create.ts b/src/device-sync/calendar/create.ts new file mode 100644 index 00000000..bad4ec4f --- /dev/null +++ b/src/device-sync/calendar/create.ts @@ -0,0 +1,280 @@ +/** + * New and deleted events (docs/device-sync.md, "Uploads" and "Identity"). + * + * A new master row is claimed before anything is sent: the uid and target go + * into SYNC_DATA3, UID_2445 and `_SYNC_ID = ~pending/`, so an app delete + * becomes a soft delete the engine can act on and a retried create is found + * by uid. The row keeps a uid it carries unless another row does too (a + * CONTENT_EXCEPTION_URI split clones UID_2445). The create then sends the + * whole event, with the exceptions and exclusions made before its upload as + * overrides. A deleted event is destroyed, or loses only its synced calendar + * memberships when it is in others as well. + */ +import type { Participant } from '../../api/types'; +import { AttendeeType, Attendees, Events } from '../android-columns'; +import type { CalendarContext, LocalEvent, LocalException, OpGroup, UploadAction } from '../planner'; +import type { Row } from '../types'; +import type { CalendarEventWire } from '../wire'; +import { collectionKey, parseCollectionKey, parseObjectRef, pendingRef, pendingUidOf } from '../common/ids'; +import type { PatchObject } from '../common/patch'; +import { isSelfAddress } from './attendees'; +import { applyAttendeeEdits, attendeeEdits, descriptionValue, locationChanges, reminderChanges, titleValue } from './edits'; +import { exdateEntryKey } from './exdate'; +import type { Override } from './exceptions'; +import { isRemovedInstance } from './merge'; +import { rruleParts, type RuleZone } from './rrule'; +import { sendsSchedulingMessages } from './scheduling'; +import { fixedOffsetTiming, rowTiming } from './timing'; +import { exdateEntries, sideOfRow } from './units'; +import { SKIP, SkipUpload, deviceRule, newOverride, put } from './upload'; +import { argbToCss, availabilityFromDevice, participationFromDevice, privacyFromDevice, statusFromDevice } from './values'; +import { canonicalZone, classifyLocal } from './zoned-time'; + +// ─── Creates ──────────────────────────────────────────── + +export interface CreateComputation { + object: Partial; + sendSchedulingMessages: boolean; +} + +/** The event a new master row (with its exceptions and EXDATE) creates. Throws `SkipUpload`. */ +export function computeCreate(local: LocalEvent, uid: string, calendarId: string, ctx: CalendarContext): CreateComputation { + const cells = local.cells; + const recurring = !!rruleParts(cells[Events.RRULE]); + let t = rowTiming(cells, { server: null, baselineZone: null, recurring, deviceZone: ctx.deviceZone }); + if (!t) throw new SkipUpload('noStart'); + if (t.ambiguous) { + if (recurring) throw new SkipUpload(SKIP.dstAmbiguous); + t = fixedOffsetTiming(t); + } + const object: Record = { + '@type': 'Event', + uid, + calendarIds: { [calendarId]: true }, + title: titleValue(cells), + start: t.start, + duration: t.duration, + status: statusFromDevice(cells[Events.STATUS]), + freeBusyStatus: availabilityFromDevice(cells[Events.AVAILABILITY]), + }; + if (t.showWithoutTime) object.showWithoutTime = true; + else object.timeZone = t.timeZone; + const description = descriptionValue(cells); + if (description) object.description = description; + const privacy = privacyFromDevice(cells[Events.ACCESS_LEVEL]); + if (privacy) object.privacy = privacy; + const color = argbToCss(cells[Events.EVENT_COLOR]); + if (color) object.color = color; + const location = locationChanges({}, cells[Events.EVENT_LOCATION], ctx.mintKey).locations; + if (location) object.locations = location; + + const { participants, organizer } = createParticipants(local, ctx); + if (participants) { + object.participants = participants; + object.organizerCalendarAddress = `mailto:${organizer}`; + } + if (ctx.reminderOwner === 'device' && local.reminders.length) { + const alerts = reminderChanges({}, undefined, local.reminders.map((r) => r.cells), ctx.mintKey).alerts; + if (Object.keys(alerts).length) object.alerts = alerts; + } + + const zone: RuleZone = t.showWithoutTime ? { allDay: true, zone: 'UTC' } : { allDay: false, zone: canonicalZone(t.timeZone) ?? 'UTC' }; + const rule = recurring ? deviceRule(cells[Events.RRULE], null, null, zone, false) : null; + if (rule) { + object.recurrenceRule = rule; + const overrides: Record = {}; + const checked = (key: string | null) => { + if (key && !zone.allDay && classifyLocal(key, zone.zone) === 'overlap') throw new SkipUpload(SKIP.dstAmbiguous); + return key; + }; + for (const entry of exdateEntries(sideOfRow(local))) { + const key = checked(exdateEntryKey(entry, zone)); + if (key) overrides[key] = { excluded: true }; + } + for (const x of local.exceptions) { + const key = checked(x.recurrenceId); + if (!key) continue; + overrides[key] = isRemovedInstance(x) ? { excluded: true } : createOverride(x, object as CalendarEventWire, key, local, ctx); + } + if (Object.keys(overrides).length) object.recurrenceOverrides = overrides; + } + return { + object: object as Partial, + sendSchedulingMessages: sendsSchedulingMessages(object as CalendarEventWire, { kind: 'create' }, ctx), + }; +} + +/** An exception of a series that is created with it: what the exception row says differently from its master row. */ +function createOverride(x: LocalException, master: CalendarEventWire, key: string, local: LocalEvent, ctx: CalendarContext): Override { + const values: Record = {}; + let t = rowTiming(x.cells, { + server: master, + baselineZone: String(local.cells[Events.EVENT_TIMEZONE] ?? ''), + recurring: false, + deviceZone: ctx.deviceZone, + }); + if (t) { + if (t.ambiguous) throw new SkipUpload(SKIP.dstAmbiguous); + if (t.showWithoutTime !== !!master.showWithoutTime) t = { ...t, showWithoutTime: !!master.showWithoutTime }; + values.start = t.start; + values.duration = t.duration; + } + const cells = x.cells; + // What the exception row leaves empty is one its app did not model: it comes from the series. + const differs = (column: string) => { + const v = cells[column]; + return v !== null && v !== undefined && v !== '' && String(v) !== String(local.cells[column] ?? ''); + }; + if (differs(Events.TITLE)) values.title = titleValue(cells); + if (differs(Events.DESCRIPTION)) values.description = descriptionValue(cells); + if (differs(Events.EVENT_LOCATION)) values.locations = locationChanges(master, cells[Events.EVENT_LOCATION], ctx.mintKey).locations; + if (differs(Events.STATUS)) values.status = statusFromDevice(cells[Events.STATUS]); + if (differs(Events.AVAILABILITY)) values.freeBusyStatus = availabilityFromDevice(cells[Events.AVAILABILITY]); + if (differs(Events.EVENT_COLOR)) values.color = argbToCss(cells[Events.EVENT_COLOR]); + if (x.attendees.length) { + const edits = attendeeEdits(x.attendees.map((a) => a.cells), local.attendees.map((a) => a.cells), ctx, false); + if (edits.length) values.participants = applyAttendeeEdits(master.participants, edits, ctx, ctx.mintKey, false).participants; + } + return newOverride(master, key, values); +} + +/** + * Participants of a new event from its attendee rows: the user as organizer + * (unless ORGANIZER names someone else) and everyone else as attendees. + * None when the user is alone. + */ +function createParticipants(local: LocalEvent, ctx: CalendarContext): { participants: Record | null; organizer: string } { + const rows = local.attendees.map((a) => a.cells); + const isSelf = (row: Row) => isSelfAddress(String(row[Attendees.ATTENDEE_EMAIL] ?? ''), ctx); + const organizerCell = String(local.cells[Events.ORGANIZER] ?? '').trim(); + const organizer = organizerCell && !isSelfAddress(organizerCell, ctx) ? organizerCell : ctx.ownerAccount; + const guests = rows.filter((r) => !isSelf(r)); + if (!guests.length) return { participants: null, organizer }; + const participants: Record = {}; + const self = rows.find(isSelf); + const iOrganize = isSelfAddress(organizer, ctx); + if (iOrganize || self) { + const status = self ? participationFromDevice(self[Attendees.ATTENDEE_STATUS]) : null; + participants[ctx.mintKey(Object.keys(participants))] = { + '@type': 'Participant', + calendarAddress: `mailto:${ctx.ownerAccount}`, + roles: iOrganize ? { owner: true, attendee: true } : { attendee: true }, + participationStatus: status ?? (iOrganize ? 'accepted' : 'needs-action'), + }; + } + for (const row of guests) { + const email = String(row[Attendees.ATTENDEE_EMAIL]).trim(); + const p: Participant = { + '@type': 'Participant', + calendarAddress: `mailto:${email}`, + roles: email.toLowerCase() === organizer.toLowerCase() ? { owner: true } : { attendee: true }, + participationStatus: 'needs-action', + expectReply: true, + }; + const name = row[Attendees.ATTENDEE_NAME]; + if (typeof name === 'string' && name.trim()) p.name = name.trim(); + if (Number(row[Attendees.ATTENDEE_TYPE]) === AttendeeType.OPTIONAL) p.roles = { ...p.roles, optional: true }; + if (Number(row[Attendees.ATTENDEE_TYPE]) === AttendeeType.RESOURCE) p.kind = 'resource'; + participants[ctx.mintKey(Object.keys(participants))] = p; + } + return { participants, organizer }; +} + +// ─── Claims ───────────────────────────────────────────── + +/** + * The claim of a new row (see the header). A split clone also drops the + * source's shadow and baseline it was cloned with, points the source's + * exceptions back at the source (the provider's trigger may have followed the + * `_SYNC_ID` change) and marks the source dirty, so the source's capped rule + * uploads even when this group is applied before the source is planned. + */ +export function claimOps(local: LocalEvent, uid: string, target: string): OpGroup { + const ops: OpGroup['ops'] = [ + { + op: 'assert', + table: 'events', + id: local.eventId, + values: { [Events._SYNC_ID]: local.syncId, [Events.DELETED]: local.cells[Events.DELETED] ?? null }, + expectCount: 1, + }, + ]; + const values: Row = { + [Events._SYNC_ID]: pendingRef(uid), + [Events.UID_2445]: uid, + [Events.SYNC_DATA3]: JSON.stringify({ uid, target }), + }; + const sharedSyncId = local.split === 'clone' && parseObjectRef(local.syncId) ? local.syncId : null; + if (local.split === 'clone') { + Object.assign(values, { [Events.SYNC_DATA1]: null, [Events.SYNC_DATA2]: null, [Events.SYNC_DATA4]: null, [Events.SYNC_DATA5]: null }); + } + ops.push({ op: 'update', table: 'events', id: local.eventId, values, expectCount: 1 }); + if (sharedSyncId) { + ops.push({ + op: 'update', + table: 'events', + where: `${Events.ORIGINAL_SYNC_ID} = ? AND (${Events.ORIGINAL_ID} IS NULL OR ${Events.ORIGINAL_ID} != ?)`, + args: [pendingRef(uid), local.eventId], + values: { [Events.ORIGINAL_SYNC_ID]: sharedSyncId }, + }); + // The clone no longer carries the shared `_SYNC_ID` here, so this reaches the source master(s) only. + ops.push({ + op: 'update', + table: 'events', + where: `${Events._SYNC_ID} = ? AND ${Events.ORIGINAL_ID} IS NULL`, + args: [sharedSyncId], + values: { [Events.DIRTY]: 1 }, + }); + } + return { ref: `row:${local.eventId}`, ops }; +} + +/** The uid of a new row's complete, consistent claim, or null when it has to be claimed (again). */ +export function claimedUid(local: LocalEvent): string | null { + if (local.split === 'clone' || !local.pending) return null; + const uid = local.pending.uid; + if (pendingUidOf(local.syncId) !== uid || local.cells[Events.UID_2445] !== uid) return null; + return uid; +} + +/** The uid a claim writes: the row's own unless it is a clone of another row's. */ +export function claimUid(local: LocalEvent, ctx: CalendarContext): string { + if (local.split === 'clone') return ctx.mintUid(); + const own = local.cells[Events.UID_2445]; + if (typeof own === 'string' && own.trim()) return own; + return pendingUidOf(local.syncId) ?? ctx.mintUid(); +} + +/** Where a new row's create goes: the target fixed at its claim, else the calendar of its row. */ +export function createTarget(local: LocalEvent, ctx: CalendarContext): { key: string; calendarId: string } | null { + if (local.pending && local.split !== 'clone') { + const parsed = parseCollectionKey(local.pending.target); + if (parsed) return { key: local.pending.target, calendarId: parsed.id }; + } + const cal = ctx.calendarIdOfRow(local.calendarRowId); + return cal ? { key: collectionKey(cal.jmapAccountId, cal.calendarId), calendarId: cal.calendarId } : null; +} + +// ─── Deletions ────────────────────────────────────────── + +/** + * A deleted event with identity: an object in several calendars loses only + * the selected memberships; otherwise it is destroyed, with scheduling + * messages when others take part. + */ +export function deleteActions(local: LocalEvent, ctx: CalendarContext): UploadAction[] { + const ref = parseObjectRef(local.syncId)!; + const shadow = local.shadow; + if (shadow) { + const memberships = Object.entries(shadow.calendarIds ?? {}).filter(([, on]) => on).map(([id]) => id); + const selected = memberships.filter((id) => ctx.isSelected(collectionKey(ctx.jmapAccountId, id))); + if (selected.length && selected.length < memberships.length) { + const patch: PatchObject = {}; + for (const id of selected) put(patch, ['calendarIds', id], null); + return [{ kind: 'update', id: ref.id, patch }]; + } + } + const action: UploadAction = { kind: 'destroy', id: ref.id, uid: typeof shadow?.uid === 'string' ? shadow.uid : null }; + if (shadow && sendsSchedulingMessages(shadow, { kind: 'destroy' }, ctx)) action.sendSchedulingMessages = true; + return [action]; +} diff --git a/src/device-sync/calendar/decode.ts b/src/device-sync/calendar/decode.ts new file mode 100644 index 00000000..d162ecf5 --- /dev/null +++ b/src/device-sync/calendar/decode.ts @@ -0,0 +1,195 @@ +/** + * Provider rows → the planner's view of local events and calendars + * (docs/device-sync.md, "Device data model"). Every sync column is read + * defensively: another app could have written anything there, and a value + * that doesn't parse counts as absent. + * + * Exception rows are grouped under their master by ORIGINAL_ID, else by + * ORIGINAL_SYNC_ID. Two masters sharing a `_SYNC_ID` are a + * CONTENT_EXCEPTION_URI split (the provider clones `_SYNC_ID`, SYNC_DATA* + * and UID_2445 and caps the old rule without DIRTY): the newer row is marked + * `clone`, the older `source`. A new master whose UID_2445 another row + * already carries is marked `clone` too, so its create gets a fresh uid + * instead of adopting the other row's server object. + */ +import { Attendees, Calendars, Events, Reminders } from '../android-columns'; +import type { LocalAttendee, LocalCalendar, LocalEvent, LocalEventRow, LocalException, LocalReminder, PendingCreate, PoisonMarker } from '../planner'; +import type { Row } from '../types'; +import type { CalendarEventWire, CalendarLike } from '../wire'; +import { parseJsonColumn } from '../common/json'; +import { parseObjectRef, pendingUidOf } from '../common/ids'; +import { decodeBaseline } from './columns'; +import { instanceTimeToKey } from './exceptions'; +import { canonicalZone, parseLocalDateTime } from './zoned-time'; + +const has = (v: unknown) => v !== null && v !== undefined && v !== ''; +const num = (v: unknown) => (has(v) && Number.isFinite(Number(v)) ? Number(v) : null); +const str = (v: unknown) => (typeof v === 'string' && v ? v : typeof v === 'number' ? String(v) : null); + +export function decodePending(text: unknown): PendingCreate | null { + const p = parseJsonColumn>(text); + if (!p || typeof p.uid !== 'string' || !p.uid || typeof p.target !== 'string' || !p.target) return null; + return { uid: p.uid, target: p.target }; +} + +export function decodePoison(text: unknown): PoisonMarker | null { + const p = parseJsonColumn>(text); + if (!p || typeof p.fp !== 'string' || typeof p.type !== 'string') return null; + const n = Number(p.n); + const until = Number(p.until); + return { + fp: p.fp, + type: p.type, + ...(typeof p.description === 'string' ? { description: p.description } : {}), + n: Number.isFinite(n) ? n : 1, + until: Number.isFinite(until) ? until : 0, + }; +} + +export function decodeShadow(text: unknown): CalendarEventWire | null { + const e = parseJsonColumn(text); + return e && typeof e === 'object' && !Array.isArray(e) && typeof e.id === 'string' ? e : null; +} + +/** Whether a row is an exception (ORIGINAL_ID or ORIGINAL_SYNC_ID set): it never uploads as an event of its own. */ +export function isExceptionRow(cells: Row): boolean { + return has(cells[Events.ORIGINAL_ID]) || has(cells[Events.ORIGINAL_SYNC_ID]); +} + +function decodeRow(row: Row, attendees: LocalAttendee[], reminders: LocalReminder[]): LocalEventRow { + const cells: Row = { ...row }; + for (const c of [Events.SYNC_DATA1, Events.SYNC_DATA3, Events.SYNC_DATA4, Events.SYNC_DATA5]) delete cells[c]; + return { + eventId: Number(row[Events._ID]), + calendarRowId: Number(row[Events.CALENDAR_ID]), + syncId: str(row[Events._SYNC_ID]), + dirty: Number(row[Events.DIRTY] ?? 0) === 1, + deleted: Number(row[Events.DELETED] ?? 0) === 1, + cells, + attendees, + reminders, + baseline: decodeBaseline(row[Events.SYNC_DATA4]), + pending: decodePending(row[Events.SYNC_DATA3]), + poison: decodePoison(row[Events.SYNC_DATA5]), + mutators: str(row[Events.MUTATORS]), + }; +} + +/** The recurrence id of an exception row: SYNC_DATA2 on our rows, else ORIGINAL_INSTANCE_TIME in the master's zone. */ +export function exceptionKey(row: LocalEventRow, master: LocalEventRow | null): string | null { + const stored = row.cells[Events.SYNC_DATA2]; + if (typeof stored === 'string' && parseLocalDateTime(stored)) return stored; + const oit = num(row.cells[Events.ORIGINAL_INSTANCE_TIME]); + if (oit === null) return null; + const source = master ?? row; + const allDay = Number(source.cells[Events.ALL_DAY] ?? row.cells[Events.ORIGINAL_ALL_DAY] ?? 0) === 1; + const zone = canonicalZone(source.cells[Events.EVENT_TIMEZONE]) ?? 'UTC'; + return instanceTimeToKey(oit, allDay, allDay ? 'UTC' : zone); +} + +export function decodeEvents(events: Row[], attendees: Row[], reminders: Row[]): LocalEvent[] { + const attendeesOf = new Map(); + for (const a of attendees) { + const id = Number(a[Attendees.EVENT_ID]); + const list = attendeesOf.get(id) ?? []; + const cells: Row = { ...a }; + delete cells[Attendees._ID]; + delete cells[Attendees.EVENT_ID]; + list.push({ id: Number(a[Attendees._ID]), cells }); + attendeesOf.set(id, list); + } + const remindersOf = new Map(); + for (const r of reminders) { + const id = Number(r[Reminders.EVENT_ID]); + const list = remindersOf.get(id) ?? []; + const cells: Row = { ...r }; + delete cells[Reminders._ID]; + delete cells[Reminders.EVENT_ID]; + list.push({ id: Number(r[Reminders._ID]), cells }); + remindersOf.set(id, list); + } + + const masters: LocalEvent[] = []; + const exceptionRows: LocalEventRow[] = []; + for (const row of [...events].sort((a, b) => Number(a[Events._ID]) - Number(b[Events._ID]))) { + const id = Number(row[Events._ID]); + const decoded = decodeRow(row, attendeesOf.get(id) ?? [], remindersOf.get(id) ?? []); + if (isExceptionRow(decoded.cells)) { + exceptionRows.push(decoded); + } else { + masters.push({ ...decoded, shadow: decodeShadow(row[Events.SYNC_DATA1]), exceptions: [] }); + } + } + + const byId = new Map(masters.map((m) => [m.eventId, m])); + const bySyncId = new Map(); + for (const m of masters) { + if (!m.syncId) continue; + const list = bySyncId.get(m.syncId) ?? []; + list.push(m); + bySyncId.set(m.syncId, list); + } + + const orphans: LocalEvent[] = []; + for (const row of exceptionRows) { + const originalId = num(row.cells[Events.ORIGINAL_ID]); + let master = originalId !== null ? byId.get(originalId) : undefined; + if (!master) { + const candidates = bySyncId.get(String(row.cells[Events.ORIGINAL_SYNC_ID] ?? '')) ?? []; + master = candidates.find((m) => m.calendarRowId === row.calendarRowId) ?? candidates[0]; + } + if (!master) { + orphans.push({ ...row, shadow: null, exceptions: [] }); + continue; + } + const exception: LocalException = { + ...row, + recurrenceId: exceptionKey(row, master), + originalInstanceTime: num(row.cells[Events.ORIGINAL_INSTANCE_TIME]), + }; + master.exceptions.push(exception); + } + + // A CONTENT_EXCEPTION_URI split: the oldest row keeps the identity, the others are new events. + for (const group of bySyncId.values()) { + if (group.length < 2) continue; + group.sort((a, b) => a.eventId - b.eventId); + group[0].split = 'source'; + for (const clone of group.slice(1)) clone.split = 'clone'; + } + // A new row carrying a uid another row uses: create it under a fresh uid. + const uidCount = new Map(); + const uidsOf = (m: LocalEvent) => { + const uids = new Set(); + const uid = str(m.cells[Events.UID_2445]); + if (uid) uids.add(uid); + const pendingUid = pendingUidOf(m.syncId) ?? m.pending?.uid; + if (pendingUid) uids.add(pendingUid); + return uids; + }; + for (const m of masters) for (const uid of uidsOf(m)) uidCount.set(uid, (uidCount.get(uid) ?? 0) + 1); + for (const m of masters) { + if (m.split || m.syncId !== null) continue; + if ([...uidsOf(m)].some((uid) => (uidCount.get(uid) ?? 0) > 1)) m.split = 'clone'; + } + + return [...masters, ...orphans]; +} + +export function decodeCalendar(row: Row): LocalCalendar { + const flags = parseJsonColumn>(row[Calendars.CAL_SYNC3]); + const shadow = parseJsonColumn(row[Calendars.CAL_SYNC2]); + const readOnly = flags?.readOnly === 'rights' || flags?.readOnly === 'subscription' ? flags.readOnly : undefined; + return { + calendarRowId: Number(row[Calendars._ID]), + syncId: str(row[Calendars._SYNC_ID]), + cells: { ...row }, + shadow: shadow && typeof shadow === 'object' && !Array.isArray(shadow) && typeof shadow.id === 'string' ? shadow : null, + flags: flags && typeof flags === 'object' ? { ...(readOnly ? { readOnly } : {}), ...(flags.taskOnly === true ? { taskOnly: true } : {}) } : null, + }; +} + +/** Whether a master has no server identity yet (no `_SYNC_ID`, or a pending one). */ +export function isNewMaster(local: LocalEventRow): boolean { + return parseObjectRef(local.syncId) === null; +} diff --git a/src/device-sync/calendar/duration.ts b/src/device-sync/calendar/duration.ts new file mode 100644 index 00000000..3f1f9682 --- /dev/null +++ b/src/device-sync/calendar/duration.ts @@ -0,0 +1,177 @@ +/** + * Durations in the three dialects device sync meets (docs/device-sync.md, + * "Timing"): + * + * - JSCalendar (RFC 8984 §1.4.6) on the server: weeks, days and a time part + * with fractional seconds; `P1W` arrives from CalDAV DTENDs. The app's own + * parsers ignore weeks (`P1W` → 0) and signs, so none of them is reused. + * - AOSP's calendarcommon2 Duration in the DURATION column: `[+-]P` then + * ``, `T` ignored, `M` always minutes, no fractions and + * no `Y`. All-day rows must say `PD`: CalendarProvider rewrites `PS` + * with `Integer.parseInt` and throws on anything else ending in `S`. + * - Whole days are nominal (a calendar day in the event's zone), the rest is + * exact time. + */ +import { + addToWall, + canonicalZone, + resolveWallClock, + wallClockAt, + wallMs, + type WallClock, +} from './zoned-time'; + +export interface Duration { + negative: boolean; + /** Whole nominal days (weeks included). */ + days: number; + /** The exact part in seconds (may be fractional). */ + seconds: number; +} + +const DAY_S = 86_400; + +/** + * Parses any duration either side writes: RFC 8984/5545 text, signed or not, + * weeks and days, `M` as minutes, a fraction on the seconds, lower case, and + * AOSP's forms (`P3600S`, `P1DT0S`, `P0DT1H30M0S`). Null for anything else + * (years, months before the time part, an empty `P`). + */ +export function parseDuration(text: unknown): Duration | null { + if (typeof text !== 'string') return null; + const m = /^([+-])?P(.*)$/.exec(text.trim().toUpperCase()); + if (!m || !m[2]) return null; + let days = 0; + let seconds = 0; + let any = false; + const token = /^(?:T|(\d+(?:[.,]\d+)?)([WDHMS]))/; + let rest = m[2]; + while (rest) { + const t = token.exec(rest); + if (!t) return null; + rest = rest.slice(t[0].length); + if (t[0] === 'T') continue; + const unit = t[2]; + const fractional = /[.,]/.test(t[1]); + if (fractional && unit !== 'S') return null; + const n = Number(t[1].replace(',', '.')); + any = true; + if (unit === 'W') days += n * 7; + else if (unit === 'D') days += n; + else if (unit === 'H') seconds += n * 3600; + else if (unit === 'M') seconds += n * 60; + else seconds += n; + } + if (!any) return null; + return { negative: m[1] === '-', days, seconds }; +} + +/** Total seconds with a day counted as 24 hours, signed: CalendarProvider's arithmetic. */ +export function durationSeconds(d: Duration): number { + const total = d.days * DAY_S + d.seconds; + return d.negative ? -total : total; +} + +/** Seconds of a duration text, or null when it can't be read. */ +export function durationTextSeconds(text: unknown): number | null { + const d = parseDuration(text); + return d ? durationSeconds(d) : null; +} + +/** + * A JSCalendar Duration in the strict grammar: `PD`, `THMS` with + * minutes kept between hours and seconds (`PT1H0M5S`; `PT1H5S` is not + * valid), whole seconds, never weeks, `PT0S` for zero. Negative input is + * treated as zero: an event's duration is unsigned. + */ +export function formatJsDuration(days: number, seconds: number): string { + const d = Math.max(0, Math.floor(days)); + let s = Math.max(0, Math.round(seconds)); + let out = 'P'; + if (d > 0) out += `${d}D`; + if (s > 0) { + const h = Math.floor(s / 3600); + s -= h * 3600; + const min = Math.floor(s / 60); + s -= min * 60; + out += 'T'; + if (h > 0) out += `${h}H`; + if (min > 0 || (h > 0 && s > 0)) out += `${min}M`; + if (s > 0) out += `${s}S`; + } + return out === 'P' ? 'PT0S' : out; +} + +/** A fixed number of seconds as a JSCalendar duration, with whole days split off as nominal days. */ +export function formatJsDurationSeconds(totalSeconds: number): string { + const s = Math.max(0, Math.round(totalSeconds)); + return formatJsDuration(Math.floor(s / DAY_S), s % DAY_S); +} + +/** + * The DURATION column: `PD` for all-day rows (at least one day; a time + * part is rounded up), `PS` for timed rows. Never a fraction, a + * `Y`, a sign or weeks. + */ +export function formatAospDuration(totalSeconds: number, allDay: boolean): string { + if (allDay) return `P${Math.max(1, Math.ceil(totalSeconds / DAY_S))}D`; + return `P${Math.max(0, Math.round(totalSeconds))}S`; +} + +/** Whole days of an all-day duration on the device: at least one, a time part rounded up. */ +export function allDayDays(d: Duration | null): number { + if (!d || d.negative) return 1; + return Math.max(1, Math.ceil((d.days * DAY_S + d.seconds) / DAY_S)); +} + +/** + * The end of a duration that starts at `startUtc`: whole days in the zone's + * wall clock, then exact seconds. With no whole days the start instant is + * used as is, so a start that is the second instant of a repeated hour does + * not collapse onto the first one. + */ +export function addDuration(startUtc: number, d: Duration, zone: string): number { + const days = d.negative ? 0 : d.days; + const seconds = d.negative ? 0 : d.seconds; + if (days === 0) return startUtc + Math.round(seconds * 1000); + const z = canonicalZone(zone) ?? 'UTC'; + const wall = wallClockAt(startUtc, z); + const shifted = resolveWallClock(addToWall(wall, days), z).utc + (startUtc % 1000 + 1000) % 1000; + return shifted + Math.round(seconds * 1000); +} + +/** + * DTSTART/DTEND as a duration with whole days nominal in `zone` and the rest + * exact: 00:00 to 00:00 the next day is `P1D` even across a DST change. + */ +export function durationBetween(startUtc: number, endUtc: number, zone: string): Duration { + if (!(endUtc > startUtc)) return { negative: false, days: 0, seconds: 0 }; + const z = canonicalZone(zone) ?? 'UTC'; + const sw = wallClockAt(startUtc, z); + const ew = wallClockAt(endUtc, z); + let days = Math.round((dateMs(ew) - dateMs(sw)) / 86_400_000); + if (timeOfDay(ew) < timeOfDay(sw)) days -= 1; + days = Math.max(0, days); + let anchor = days > 0 ? addDuration(startUtc, { negative: false, days, seconds: 0 }, z) : startUtc; + while (days > 0 && anchor > endUtc) { + days -= 1; + anchor = days > 0 ? addDuration(startUtc, { negative: false, days, seconds: 0 }, z) : startUtc; + } + return { negative: false, days, seconds: Math.max(0, (endUtc - anchor) / 1000) }; +} + +function dateMs(w: WallClock): number { + return wallMs({ ...w, hour: 0, minute: 0, second: 0 }); +} + +function timeOfDay(w: WallClock): number { + return w.hour * 3600 + w.minute * 60 + w.second; +} + +/** Equal lengths with a day counted as 24 hours (`PT24H` and `P1D` are the same to CalendarProvider). */ +export function sameDurationLength(a: unknown, b: unknown): boolean { + const sa = durationTextSeconds(a); + const sb = durationTextSeconds(b); + if (sa === null || sb === null) return (a ?? null) === (b ?? null); + return sa === sb; +} diff --git a/src/device-sync/calendar/edits.ts b/src/device-sync/calendar/edits.ts new file mode 100644 index 00000000..2a0eef2f --- /dev/null +++ b/src/device-sync/calendar/edits.ts @@ -0,0 +1,248 @@ +/** + * Device edits of attendees, reminders and the location, turned into + * JSCalendar values (docs/device-sync.md, "Other properties"). + * + * - Attendees: the user's own status is their RSVP; an added row is a new + * participant (`roles: {attendee: true}`, `needs-action`, `expectReply`); + * a removed row deletes its participants. Other people's statuses, the + * organizer relationship and, for lossy editors, the attendee type are not + * the device's to change. + * - Reminders replace the alerts a Reminders row can hold, keeping the keys + * of those still there; the others stay. + * - The location edits the first location's name, adds one, or deletes it. + * + * Changes come back as paths (raw member names) so the caller escapes them + * once, with `ptr`, wherever the patch is rooted. + */ +import type { Alert, Participant } from '../../api/types'; +import { Attendees, AttendeeType, Events } from '../android-columns'; +import type { Row } from '../types'; +import type { CalendarLike, CalendarEventWire } from '../wire'; +import { clone } from '../common/json'; +import { addressGroups, isSelfAddress, participantIdFor, type SelfContext } from './attendees'; +import { attendeeDiffers, attendeesByKey } from './units'; +import { effectiveAlerts, remindersToAlerts } from './reminders'; +import { participationFromDevice } from './values'; + +export type MintKey = (taken?: Iterable) => string; + +/** One change: a path of raw member names below some root, and the value (null removes). */ +export interface PathChange { + path: string[]; + value: unknown; +} + +export interface AttendeeEdit { + kind: 'add' | 'remove' | 'change'; + key: string; + row?: Row; + baseline?: Row; +} + +/** What the device changed in the attendee rows since `baselineRows`. */ +export function attendeeEdits(currentRows: Row[], baselineRows: Row[], self: SelfContext, lossy: boolean): AttendeeEdit[] { + const cur = attendeesByKey(currentRows, self); + const bl = attendeesByKey(baselineRows, self); + const edits: AttendeeEdit[] = []; + for (const key of new Set([...bl.keys(), ...cur.keys()])) { + const row = cur.get(key); + const baseline = bl.get(key); + if (row && !baseline) edits.push({ kind: 'add', key, row }); + else if (!row && baseline) edits.push({ kind: 'remove', key, baseline }); + else if (attendeeDiffers(row, baseline, lossy)) edits.push({ kind: 'change', key, row, baseline }); + } + return edits; +} + +/** Whether an attendee address key is the user's. */ +export function isSelfKey(key: string): boolean { + return key.startsWith('self:'); +} + +/** A participant for an attendee row an app added. */ +export function newParticipant(row: Row, self: SelfContext): Participant { + const email = String(row[Attendees.ATTENDEE_EMAIL] ?? '').trim(); + const p: Participant = { + '@type': 'Participant', + calendarAddress: `mailto:${email}`, + roles: { attendee: true }, + participationStatus: 'needs-action', + expectReply: true, + }; + const name = row[Attendees.ATTENDEE_NAME]; + if (typeof name === 'string' && name.trim()) p.name = name.trim(); + const type = Number(row[Attendees.ATTENDEE_TYPE]); + if (type === AttendeeType.OPTIONAL) p.roles = { attendee: true, optional: true }; + if (type === AttendeeType.RESOURCE) p.kind = 'resource'; + if (isSelfAddress(email, self)) { + const status = participationFromDevice(row[Attendees.ATTENDEE_STATUS]); + if (status) p.participationStatus = status; + delete p.expectReply; + } + return p; +} + +export interface AttendeeChanges { + /** Changes below the participants map (`[id]`, `[id, 'participationStatus']`, …). */ + changes: PathChange[]; + /** The whole map after the edits. */ + participants: Record; + /** The user's own status changed. */ + rsvp: boolean; + /** Someone was added or removed, or a name or type changed. */ + others: boolean; +} + +/** Applies attendee edits to a participants map (not changed in place). */ +export function applyAttendeeEdits( + participants: Record | null | undefined, + edits: AttendeeEdit[], + self: SelfContext, + mintKey: MintKey, + lossy: boolean, +): AttendeeChanges { + const map = clone(participants ?? {}) as Record; + const changes: PathChange[] = []; + let rsvp = false; + let others = false; + const groups = addressGroups(map, self); + const idsOf = (key: string) => groups.find((g) => g.key === key)?.ids ?? []; + const setStatus = (id: string, row: Row, baseline?: Row) => { + if (baseline && Number(row[Attendees.ATTENDEE_STATUS]) === Number(baseline[Attendees.ATTENDEE_STATUS])) return; + const status = participationFromDevice(row[Attendees.ATTENDEE_STATUS]); + if (!status || map[id].participationStatus === status) return; + map[id] = { ...map[id], participationStatus: status }; + changes.push({ path: [id, 'participationStatus'], value: status }); + rsvp = true; + }; + for (const edit of edits) { + const selfRow = isSelfKey(edit.key); + if (edit.kind === 'add') { + const existing = participantIdFor(map, edit.row![Attendees.ATTENDEE_EMAIL], self); + if (existing) { + // The server has this address already (added meanwhile): only the user's own status counts. + if (selfRow) setStatus(existing, edit.row!); + continue; + } + // An app adding the owner's row next to new guests: Stalwart adds the organizer itself. + if (selfRow) continue; + const id = mintKey(Object.keys(map)); + map[id] = newParticipant(edit.row!, self); + changes.push({ path: [id], value: map[id] }); + others = true; + } else if (edit.kind === 'remove') { + if (selfRow) continue; + for (const id of idsOf(edit.key)) { + delete map[id]; + changes.push({ path: [id], value: null }); + others = true; + } + } else { + const id = participantIdFor(map, edit.row![Attendees.ATTENDEE_EMAIL], self); + if (!id) continue; + if (selfRow) setStatus(id, edit.row!, edit.baseline); + const name = edit.row![Attendees.ATTENDEE_NAME]; + if (String(name ?? '') !== String(edit.baseline![Attendees.ATTENDEE_NAME] ?? '')) { + if (typeof name === 'string' && name.trim()) { + map[id] = { ...map[id], name: name.trim() }; + changes.push({ path: [id, 'name'], value: name.trim() }); + } else if (map[id].name !== undefined) { + const { name: _gone, ...rest } = map[id]; + map[id] = rest; + changes.push({ path: [id, 'name'], value: null }); + } + others = true; + } + const type = Number(edit.row![Attendees.ATTENDEE_TYPE]); + if (!lossy && type !== Number(edit.baseline![Attendees.ATTENDEE_TYPE])) { + const roles = { ...(map[id].roles ?? { attendee: true }) }; + if (type === AttendeeType.OPTIONAL) roles.optional = true; + else delete roles.optional; + map[id] = { ...map[id], roles }; + changes.push({ path: [id, 'roles'], value: roles }); + if (type === AttendeeType.RESOURCE && map[id].kind !== 'resource') { + map[id] = { ...map[id], kind: 'resource' }; + changes.push({ path: [id, 'kind'], value: 'resource' }); + } else if (type !== AttendeeType.RESOURCE && map[id].kind === 'resource') { + const { kind: _gone, ...rest } = map[id]; + map[id] = rest; + changes.push({ path: [id, 'kind'], value: null }); + } + others = true; + } + } + } + return { changes, participants: map, rsvp, others }; +} + +export interface AlertChanges { + /** Changes relative to the event: `['alerts', k]`, `['useDefaultAlerts']` or the whole `['alerts']`. */ + changes: PathChange[]; + /** The alerts in force afterwards. */ + alerts: Record; +} + +/** + * The device's reminders as alert changes. With `useDefaultAlerts` the + * calendar's defaults were in force: the event switches to explicit alerts + * (built from them). An event without alerts gets the whole map, since a + * pointer below a missing property fails. + */ +export function reminderChanges( + event: Pick, + calendar: CalendarLike | undefined, + reminders: Row[], + mintKey: MintKey, +): AlertChanges { + const current = effectiveAlerts(event, calendar); + const { alerts, removed, added } = remindersToAlerts(current, reminders, mintKey); + const changes: PathChange[] = []; + if (event.useDefaultAlerts) { + changes.push({ path: ['useDefaultAlerts'], value: false }); + changes.push({ path: ['alerts'], value: Object.keys(alerts).length ? alerts : null }); + } else if (!Object.keys(event.alerts ?? {}).length) { + if (Object.keys(added).length) changes.push({ path: ['alerts'], value: alerts }); + } else { + for (const key of removed) changes.push({ path: ['alerts', key], value: null }); + for (const [key, alert] of Object.entries(added)) changes.push({ path: ['alerts', key], value: alert }); + } + return { changes, alerts }; +} + +/** The location edit, as changes relative to the event and as the resulting map. */ +export function locationChanges( + event: Pick, + name: unknown, + mintKey: MintKey, +): { changes: PathChange[]; locations: Record | null } { + const locations = clone((event.locations ?? {}) as Record>); + const first = Object.keys(locations)[0]; + const text = typeof name === 'string' ? name.trim() : ''; + const changes: PathChange[] = []; + if (!text) { + if (first) { + delete locations[first]; + changes.push({ path: ['locations', first], value: null }); + } + } else if (first) { + locations[first] = { ...locations[first], name: text }; + changes.push({ path: ['locations', first, 'name'], value: text }); + } else { + const key = mintKey(); + locations[key] = { '@type': 'Location', name: text }; + changes.push({ path: ['locations'], value: { [key]: locations[key] } }); + } + return { changes, locations: Object.keys(locations).length ? locations : null }; +} + +/** The title cell as JSCalendar text. */ +export function titleValue(cells: Row): string { + const t = cells[Events.TITLE]; + return typeof t === 'string' ? t : t === null || t === undefined ? '' : String(t); +} + +/** The description cell as JSCalendar text; null removes it. */ +export function descriptionValue(cells: Row): string | null { + const d = cells[Events.DESCRIPTION]; + return typeof d === 'string' && d ? d : null; +} diff --git a/src/device-sync/calendar/exceptions.ts b/src/device-sync/calendar/exceptions.ts new file mode 100644 index 00000000..9c5386df --- /dev/null +++ b/src/device-sync/calendar/exceptions.ts @@ -0,0 +1,86 @@ +/** + * Recurrence overrides and the exception rows they become + * (docs/device-sync.md, "Recurrence"). + * + * An override value is a PatchObject against the master (RFC 8984 §4.3.4), + * so an exception row shows master ⊕ override, applied pointer by pointer + * (the app's own expansion shallow-merges and turns + * `participants/p1/participationStatus` into a junk key, R3 B10). Stalwart + * drops a handful of properties from every override it stores; they are + * never sent in one. + */ +import type { CalendarEventWire } from '../wire'; +import { applyPatch, ptrSegments } from '../common/patch'; +import { clone } from '../common/json'; +import { dateToUtcMidnight, localToUtc, utcMidnightToDate, utcToLocal } from './zoned-time'; + +/** Properties calcard removes from an override (`remove_forbidden_override_patches`). */ +export const FORBIDDEN_OVERRIDE_KEYS = [ + '@type', + 'method', + 'organizerCalendarAddress', + 'privacy', + 'prodId', + 'recurrenceId', + 'recurrenceIdTimeZone', + 'sentBy', + 'uid', + 'recurrenceRule', + 'recurrenceOverrides', +] as const; + +const FORBIDDEN = new Set(FORBIDDEN_OVERRIDE_KEYS); + +/** Properties of the master that never belong to an instance. */ +const SERIES_ONLY = ['recurrenceRule', 'recurrenceRules', 'recurrenceOverrides', 'excludedRecurrenceRule', 'excludedRecurrenceRules']; + +export type Override = Record; + +export function isExcluded(override: unknown): boolean { + return !!override && typeof override === 'object' && (override as Override).excluded === true; +} + +/** An override without what Stalwart would drop from it (top-level and pointer keys alike). */ +export function withoutForbiddenKeys(override: Override): Override { + const out: Override = {}; + for (const [key, value] of Object.entries(override)) { + if (FORBIDDEN.has(ptrSegments(key)[0])) continue; + out[key] = value; + } + return out; +} + +/** + * One instance of a series: the master without its recurrence, patched with + * the override (pointer by pointer; a pointer whose parent is missing is + * skipped rather than failing the instance), starting at the recurrence id + * unless the override moves it. + */ +export function instanceOf(master: CalendarEventWire, key: string, override?: Override | null): CalendarEventWire { + let instance = clone(master) as Record; + for (const name of SERIES_ONLY) delete instance[name]; + instance.start = key; + for (const [pointer, value] of Object.entries(override ?? {})) { + if (pointer === 'excluded' || pointer === 'updated') continue; + if (FORBIDDEN.has(ptrSegments(pointer)[0])) continue; + const patched = applyPatch(instance, { [pointer]: value }); + if (patched) instance = patched; + } + return instance as CalendarEventWire; +} + +/** ORIGINAL_INSTANCE_TIME of a recurrence id: its instant in the master's zone, UTC midnight for all-day series. */ +export function keyToInstanceTime(key: string, allDay: boolean, zone: string): number | null { + return allDay ? dateToUtcMidnight(key) : localToUtc(key, zone); +} + +/** The recurrence id of an ORIGINAL_INSTANCE_TIME (the inverse of `keyToInstanceTime`). */ +export function instanceTimeToKey(originalInstanceTime: number, allDay: boolean, zone: string): string { + return allDay ? utcMidnightToDate(originalInstanceTime) : utcToLocal(originalInstanceTime, zone); +} + +/** The overrides of an event as a plain map (Stalwart omits the property when there are none). */ +export function overridesOf(event: CalendarEventWire | null | undefined): Record { + const map = event?.recurrenceOverrides; + return map && typeof map === 'object' ? (map as Record) : {}; +} diff --git a/src/device-sync/calendar/exdate.ts b/src/device-sync/calendar/exdate.ts new file mode 100644 index 00000000..c2a4358f --- /dev/null +++ b/src/device-sync/calendar/exdate.ts @@ -0,0 +1,114 @@ +/** + * The EXDATE column ↔ excluded recurrence overrides (docs/device-sync.md, + * "Recurrence"). AOSP's RecurrenceSet reads lines split by `\n`, each + * `[;][,…]` with a bare Olson id as prefix; dates are + * `YYYYMMDD`, `YYYYMMDDTHHMMSS` or `…Z`, and a line without a prefix is UTC, + * not the event's zone. Device sync writes one line without a prefix: UTC + * date-times for timed events, dates for all-day ones. + */ +import { + canonicalZone, + dateToUtcMidnight, + formatDate, + localToUtc, + resolveWallClock, + utcMidnightToDate, + utcToLocal, + wallFromMs, + type WallClock, +} from './zoned-time'; +import type { RuleZone } from './rrule'; + +const compactUtc = (utc: number) => { + const w = wallFromMs(utc); + const p = (n: number) => String(n).padStart(2, '0'); + return `${formatDate(w).replace(/-/g, '')}T${p(w.hour)}${p(w.minute)}${p(w.second)}Z`; +}; + +/** One EXDATE entry for a recurrence id (a LocalDateTime in the event's zone). */ +export function exdateEntry(key: string, ctx: RuleZone): string | null { + if (ctx.allDay) { + const midnight = dateToUtcMidnight(key); + return midnight === null ? null : formatDate(wallFromMs(midnight)).replace(/-/g, ''); + } + const utc = localToUtc(key, ctx.zone); + return utc === null ? null : compactUtc(utc); +} + +/** The EXDATE column for a set of recurrence ids, sorted; null for none. */ +export function formatExdate(keys: Iterable, ctx: RuleZone): string | null { + const entries = [...new Set([...keys].map((k) => exdateEntry(k, ctx)).filter((e): e is string => !!e))].sort(); + return entries.length ? entries.join(',') : null; +} + +interface ParsedEntry { + /** The instant (timed) or UTC midnight of the date. */ + utc: number; + dateOnly: boolean; +} + +function parseEntry(text: string, zone: string | null): ParsedEntry | null { + const m = /^(\d{4})(\d{2})(\d{2})(?:T(\d{2})(\d{2})(\d{2})(Z)?)?$/i.exec(text.trim()); + if (!m) return null; + const wall: WallClock = { + year: Number(m[1]), + month: Number(m[2]), + day: Number(m[3]), + hour: Number(m[4] ?? 0), + minute: Number(m[5] ?? 0), + second: Number(m[6] ?? 0), + }; + const dateOnly = m[4] === undefined; + if (m[7] || !zone) return { utc: resolveWallClock(wall, 'UTC').utc, dateOnly }; + return { utc: resolveWallClock(wall, zone).utc, dateOnly }; +} + +/** Every entry of an EXDATE (or RDATE) column as an instant; garbage is skipped. */ +export function exdateInstants(text: unknown): ParsedEntry[] { + if (typeof text !== 'string' || !text.trim()) return []; + const out: ParsedEntry[] = []; + for (const line of text.split(/\r?\n/)) { + if (!line.trim()) continue; + const semi = line.indexOf(';'); + const zone = semi >= 0 ? canonicalZone(line.slice(0, semi).trim()) : null; + const dates = semi >= 0 ? line.slice(semi + 1) : line; + for (const d of dates.split(',')) { + const entry = parseEntry(d, zone); + if (entry) out.push(entry); + } + } + return out; +} + +/** + * The recurrence ids an EXDATE column excludes, in the event's zone + * (all-day: the date at midnight). A timed entry's key is its instant in the + * zone; a date-only entry of a timed event is AOSP's UTC midnight of it. + */ +export function parseExdate(text: unknown, ctx: RuleZone): string[] { + const keys = new Set(); + for (const entry of exdateInstants(text)) { + keys.add(ctx.allDay ? utcMidnightToDate(entry.utc) : utcToLocal(entry.utc, ctx.zone)); + } + return [...keys].sort(); +} + +/** + * The entries as a set of normalised instants (`YYYYMMDDTHHMMSSZ`, or + * `YYYYMMDD` for dates and for every entry of an all-day event): the + * column's meaning independent of how an app spelled it. + */ +export function exdateSet(text: unknown, allDay = false): Set { + const out = new Set(); + for (const entry of exdateInstants(text)) { + out.add(entry.dateOnly || allDay ? formatDate(wallFromMs(entry.utc)).replace(/-/g, '') : compactUtc(entry.utc)); + } + return out; +} + +/** The recurrence id of one normalised entry (see `exdateSet`). */ +export function exdateEntryKey(entry: string, ctx: RuleZone): string | null { + const parsed = parseEntry(entry, null); + if (!parsed) return null; + return ctx.allDay ? utcMidnightToDate(parsed.utc) : utcToLocal(parsed.utc, ctx.zone); +} diff --git a/src/device-sync/calendar/image.ts b/src/device-sync/calendar/image.ts new file mode 100644 index 00000000..039a4ed2 --- /dev/null +++ b/src/device-sync/calendar/image.ts @@ -0,0 +1,195 @@ +/** + * A server event as the rows it becomes on the device: the master row's + * mapped cells, its attendees and reminders, and one exception row per + * override that is not an exclusion (docs/device-sync.md, "Calendar + * mapping"). The cells are what CalendarProvider stores after the write: + * all-day times at UTC midnight, DURATION only on recurring masters, the + * organizer the provider would fill in, STATUS never NULL. Predicting the + * provider keeps echo downloads free of writes. + */ +import { Events } from '../android-columns'; +import type { CalendarContext } from '../planner'; +import type { Row } from '../types'; +import type { CalendarEventWire } from '../wire'; +import { htmlToPlainText } from '../../lib/compose-html'; +import { attendeeRows, hasMailtoParticipants, organizerAddress } from './attendees'; +import { truncateUtf8 } from './columns'; +import { formatExdate } from './exdate'; +import { instanceOf, isExcluded, keyToInstanceTime, overridesOf } from './exceptions'; +import { effectiveAlerts, reminderRows } from './reminders'; +import { isRuleRepresentable, ruleToRRule } from './rrule'; +import { eventZone, timingCells } from './timing'; +import { + availabilityToDevice, + cssColorToArgb, + privacyToDevice, + statusToDevice, +} from './values'; +import { parseLocalDateTime } from './zoned-time'; + +export type ImageContext = Pick< + CalendarContext, + 'deviceZone' | 'ownerAccount' | 'selfAddresses' | 'reminderOwner' | 'calendar' +>; + +export interface RowImage { + cells: Row; + attendees: Row[]; + /** Null when Bulwark owns reminders: Reminders rows are then left alone. */ + reminders: Row[] | null; + /** Whether the event has `mailto:` participants (ORGANIZER is written only then). */ + hasParticipants: boolean; + truncatedDescription: boolean; +} + +export interface ExceptionImage extends RowImage { + key: string; +} + +export interface EventImage { + master: RowImage; + /** Exception rows by recurrence id. */ + exceptions: Map; + /** Recurrence ids of `{excluded: true}` overrides (the EXDATE column). */ + excluded: Set; + zone: string; + allDay: boolean; + floating: boolean; + recurring: boolean; + /** False when the rule needs RSCALE, SKIP or leap months: shown approximately, timing never uploaded. */ + representable: boolean; +} + +/** DESCRIPTION: HTML turned to text (and uploaded as text/plain only when edited), cut at 64 KB. */ +export function descriptionCell(event: CalendarEventWire): { text: string | null; truncated: boolean } { + let text = typeof event.description === 'string' ? event.description : ''; + if (text && String(event.descriptionContentType ?? '').toLowerCase().startsWith('text/html')) text = htmlToPlainText(text); + if (!text) return { text: null, truncated: false }; + const cut = truncateUtf8(text); + return { text: cut.text, truncated: cut.truncated }; +} + +/** The first location's name, in server order. */ +export function locationCell(event: CalendarEventWire): string | null { + const first = Object.values(event.locations ?? {})[0]; + const name = first && typeof first === 'object' ? first.name : undefined; + return typeof name === 'string' && name ? name : null; +} + +function rowImage( + event: CalendarEventWire, + timing: Row, + calendarRowId: number, + calendarId: string, + ctx: ImageContext, +): RowImage { + const description = descriptionCell(event); + const hasParticipants = hasMailtoParticipants(event.participants); + const cells: Row = { + [Events.CALENDAR_ID]: calendarRowId, + [Events.TITLE]: typeof event.title === 'string' && event.title ? event.title : null, + [Events.DESCRIPTION]: description.text, + [Events.EVENT_LOCATION]: locationCell(event), + [Events.STATUS]: statusToDevice(event.status), + [Events.AVAILABILITY]: availabilityToDevice(event.freeBusyStatus), + [Events.ACCESS_LEVEL]: privacyToDevice(event.privacy), + [Events.EVENT_COLOR]: cssColorToArgb(event.color), + [Events.ORGANIZER]: (hasParticipants ? organizerAddress(event, ctx) : null) ?? ctx.ownerAccount, + ...timing, + }; + const reminders = + ctx.reminderOwner === 'device' + ? reminderRows(effectiveAlerts(event, ctx.calendar(calendarId))) + : null; + return { + cells, + attendees: hasParticipants ? attendeeRows(event, ctx) : [], + reminders, + hasParticipants, + truncatedDescription: description.truncated, + }; +} + +/** + * The rows of a server event in the calendar row `calendarRowId` (server + * calendar `calendarId`, whose default alerts apply). Null when the event's + * start can't be read. + */ +export function eventImage( + event: CalendarEventWire, + calendarRowId: number, + calendarId: string, + ctx: ImageContext, +): EventImage | null { + const { zone, allDay, floating } = eventZone(event, ctx.deviceZone); + const rule = event.recurrenceRule ?? null; + const rrule = rule ? ruleToRRule(rule, { allDay, zone }) : null; + const recurring = rrule !== null; + const timing = timingCells(event, recurring, ctx.deviceZone); + if (!timing) return null; + const overrides = recurring ? overridesOf(event) : {}; + const excluded = new Set(); + const exceptions = new Map(); + for (const [key, override] of Object.entries(overrides)) { + if (!parseLocalDateTime(key) || !override || typeof override !== 'object') continue; + if (isExcluded(override)) { + excluded.add(key); + continue; + } + const originalInstanceTime = keyToInstanceTime(key, allDay, zone); + if (originalInstanceTime === null) continue; + const instance = instanceOf(event, key, override); + // An instance keeps the series' kind of time; only its start and length move. + const instanceTiming = timingCells( + { ...instance, showWithoutTime: event.showWithoutTime, timeZone: instance.timeZone ?? event.timeZone }, + false, + ctx.deviceZone, + ); + if (!instanceTiming) continue; + const image = rowImage(instance, instanceTiming, calendarRowId, calendarId, ctx); + image.cells[Events.ORIGINAL_INSTANCE_TIME] = originalInstanceTime; + image.cells[Events.ORIGINAL_ALL_DAY] = allDay ? 1 : 0; + exceptions.set(key, { ...image, key }); + } + const master = rowImage(event, timing, calendarRowId, calendarId, ctx); + master.cells[Events.RRULE] = rrule; + master.cells[Events.EXDATE] = recurring ? formatExdate(excluded, { allDay, zone }) : null; + return { + master, + exceptions, + excluded, + zone, + allDay, + floating, + recurring, + representable: isRuleRepresentable(rule), + }; +} + +/** + * The row an instance of the series would have without an exception row: + * the master at that recurrence id. The baseline of an exception an app + * creates, and what its changes are measured against. + */ +export function plainInstanceImage( + master: CalendarEventWire, + key: string, + calendarRowId: number, + calendarId: string, + ctx: ImageContext, +): ExceptionImage | null { + const { zone, allDay } = eventZone(master, ctx.deviceZone); + const override = overridesOf(master)[key]; + const instance = instanceOf(master, key, override && !isExcluded(override) ? override : null); + const timing = timingCells( + { ...instance, showWithoutTime: master.showWithoutTime, timeZone: instance.timeZone ?? master.timeZone }, + false, + ctx.deviceZone, + ); + const originalInstanceTime = keyToInstanceTime(key, allDay, zone); + if (!timing || originalInstanceTime === null) return null; + const image = rowImage(instance, timing, calendarRowId, calendarId, ctx); + image.cells[Events.ORIGINAL_INSTANCE_TIME] = originalInstanceTime; + image.cells[Events.ORIGINAL_ALL_DAY] = allDay ? 1 : 0; + return { ...image, key }; +} diff --git a/src/device-sync/calendar/merge.ts b/src/device-sync/calendar/merge.ts new file mode 100644 index 00000000..53715e5a --- /dev/null +++ b/src/device-sync/calendar/merge.ts @@ -0,0 +1,460 @@ +/** + * A server event into its rows (docs/device-sync.md, "Change detection and + * merge rules"). One writer serves downloads, accepted uploads, reverts and + * zone changes: + * + * - a clean row is rewritten to the image of the server event (only cells + * that differ are written, so an echo writes nothing); + * - a dirty row is merged per unit against its baseline (what the device + * changed) and the old shadow's image (what the server changed): the server + * wins real conflicts, local changes the server did not touch stay dirty + * for the upload; + * - exception rows are matched to overrides by recurrence id and merged the + * same way; a device deletion of an instance wins over a server edit, a + * server removal wins over a device edit. + * + * Every write into an existing item is guarded: the group first asserts + * DIRTY and the projection the plan was made from (with attendees and + * reminders when the item is dirty) and the number of exception rows. + */ +import { Events } from '../android-columns'; +import type { CalendarContext, EventBaseline, LocalEvent, LocalEventRow, LocalException } from '../planner'; +import type { Row } from '../types'; +import type { CalendarEventWire } from '../wire'; +import { collectionKey, exceptionRef, objectRef } from '../common/ids'; +import { EventStatus } from '../android-columns'; +import type { SelfContext } from './attendees'; +import { eventImage, plainInstanceImage, type EventImage, type ExceptionImage, type RowImage } from './image'; +import { + GroupBuilder, + assertExceptionCount, + assertRow, + deleteRow, + insertRow, + updateRow, + type RowState, +} from './rows'; +import { + COLUMN_UNITS, + EXCEPTION_COLUMN_UNITS, + MASTER_COLUMN_UNITS, + attendeeDiffers, + attendeesByKey, + baselineOfImage, + columnUnitDiffers, + exdateEntries, + isLossyEditor, + remindersDiffer, + sideOfBaseline, + sideOfImage, + sideOfRow, +} from './units'; + +export interface CalendarTarget { + calendarRowId: number; + calendarId: string; +} + +/** + * The calendar row an event lives in: the row it is already in while the + * event still belongs to that calendar, else its first selected calendar by + * id (docs/device-sync.md, "Decisions and limitations"). + */ +export function pickCalendar( + event: CalendarEventWire, + local: LocalEventRow | null, + ctx: Pick, +): CalendarTarget | null { + const ids = Object.entries(event.calendarIds ?? {}) + .filter(([, on]) => on) + .map(([id]) => id) + .sort(); + const candidates = ids.filter((id) => ctx.calendarRowId(id) !== null && ctx.isSelected(collectionKey(ctx.jmapAccountId, id))); + if (local) { + const current = ctx.calendarIdOfRow(local.calendarRowId); + if (current && current.jmapAccountId === ctx.jmapAccountId && candidates.includes(current.calendarId)) { + return { calendarRowId: local.calendarRowId, calendarId: current.calendarId }; + } + } + if (!candidates.length) return null; + return { calendarRowId: ctx.calendarRowId(candidates[0])!, calendarId: candidates[0] }; +} + +// ─── One row ──────────────────────────────────────────── + +export interface RowMerge { + cells: Row; + attendees: Row[]; + reminders: Row[] | null; + baseline: EventBaseline; + /** Units whose local change survived (the row stays dirty). */ + kept: string[]; + conflicts: number; +} + +export interface RowMergeInput { + isException: boolean; + current: LocalEventRow; + /** What the device last wrote (SYNC_DATA4), or an image standing in for it. */ + baseline: EventBaseline; + /** The old server state's image of this row. */ + base: RowImage; + /** The new server state's image of this row. */ + remote: RowImage; + /** Whether the row's differences from its baseline are edits (DIRTY). */ + dirty: boolean; + /** Units counted as edited even on a clean row (a split source's rule). */ + forced?: ReadonlySet; + /** An editor that rewrites attendee types (Fossify). */ + lossy?: boolean; + self: SelfContext; +} + +/** Per-unit merge of one row (see the header): what to write, the new baseline, what stays dirty. */ +export function mergeRow(input: RowMergeInput): RowMerge { + const { current, baseline, base, remote, dirty } = input; + const forced = input.forced ?? new Set(); + const cur = sideOfRow(current); + const bl = sideOfBaseline(baseline); + const bs = sideOfImage(base); + const rm = sideOfImage(remote); + const cells: Row = { ...remote.cells }; + const remoteBaseline = baselineOfImage(remote, input.isException); + const baseCells: Row = { ...remoteBaseline.cells }; + const kept: string[] = []; + let conflicts = 0; + + const decide = (unit: string, local: boolean, remoteChanged: boolean, same: boolean): 'local' | 'remote' => { + if (!local) return 'remote'; + if (!remoteChanged) { + kept.push(unit); + return 'local'; + } + if (!same) conflicts += 1; + return 'remote'; + }; + + for (const unit of input.isException ? EXCEPTION_COLUMN_UNITS : MASTER_COLUMN_UNITS) { + const local = (dirty || forced.has(unit)) && columnUnitDiffers(unit, cur, bl); + const choice = decide(unit, local, columnUnitDiffers(unit, bs, rm), !columnUnitDiffers(unit, cur, rm)); + if (choice === 'local') { + for (const c of COLUMN_UNITS[unit]) { + cells[c] = current.cells[c] ?? null; + baseCells[c] = baseline.cells[c] ?? null; + } + } + } + + if (!input.isException) { + const curE = exdateEntries(cur); + const blE = exdateEntries(bl); + const bsE = exdateEntries(bs); + const rmE = exdateEntries(rm); + const final = new Set(); + const finalBaseline = new Set(); + for (const e of new Set([...curE, ...blE, ...bsE, ...rmE])) { + const choice = decide(`exdate:${e}`, dirty && curE.has(e) !== blE.has(e), bsE.has(e) !== rmE.has(e), curE.has(e) === rmE.has(e)); + if (choice === 'remote') { + if (rmE.has(e)) { + final.add(e); + finalBaseline.add(e); + } + } else { + if (curE.has(e)) final.add(e); + if (blE.has(e)) finalBaseline.add(e); + } + } + const recurring = !!cells[Events.RRULE]; + // Our own column text when nothing local is kept, so an echo compares equal character for character. + cells[Events.EXDATE] = !recurring || !final.size ? null : sameEntries(final, rmE) ? remote.cells[Events.EXDATE] : [...final].sort().join(','); + baseCells[Events.EXDATE] = !finalBaseline.size ? null : [...finalBaseline].sort().join(','); + } + + const curA = attendeesByKey(cur.attendees, input.self); + const blA = attendeesByKey(bl.attendees, input.self); + const bsA = attendeesByKey(bs.attendees, input.self); + const rmA = attendeesByKey(rm.attendees, input.self); + const attendees: Row[] = []; + const baselineAttendees: Row[] = []; + for (const key of new Set([...rmA.keys(), ...curA.keys(), ...blA.keys(), ...bsA.keys()])) { + const local = dirty && attendeeDiffers(curA.get(key), blA.get(key), input.lossy); + const choice = decide( + `attendee:${key}`, + local, + attendeeDiffers(bsA.get(key), rmA.get(key)), + !attendeeDiffers(curA.get(key), rmA.get(key), input.lossy), + ); + if (choice === 'remote') { + const r = rmA.get(key); + if (r) { + attendees.push(r); + baselineAttendees.push(r); + } + } else { + const c = curA.get(key); + const b = blA.get(key); + if (c) attendees.push(c); + if (b) baselineAttendees.push(b); + } + } + + let reminders: Row[] | null = null; + let baselineReminders: Row[]; + if (remote.reminders === null) { + // Bulwark owns reminders: the rows are an app's business, neither written nor uploaded. + baselineReminders = current.reminders.map((r) => r.cells); + } else { + const local = dirty && remindersDiffer(cur, bl); + const choice = decide('reminders', local, remindersDiffer(bs, rm), !remindersDiffer(cur, rm)); + reminders = choice === 'remote' ? remote.reminders : current.reminders.map((r) => r.cells); + baselineReminders = choice === 'remote' ? remote.reminders : bl.reminders ?? []; + } + + return { + cells, + attendees, + reminders, + baseline: { cells: baseCells, attendees: baselineAttendees, reminders: baselineReminders }, + kept, + conflicts, + }; +} + +/** Exception rows the provider links to the master by ORIGINAL_ID (what the count assert counts). */ +export function linkedExceptionCount(local: LocalEvent): number { + return local.exceptions.filter((x) => Number(x.cells[Events.ORIGINAL_ID]) === local.eventId).length; +} + +function sameEntries(a: Set, b: Set): boolean { + return a.size === b.size && [...a].every((e) => b.has(e)); +} + +// ─── A whole event ────────────────────────────────────── + +export type WriteMode = + /** A download: dirty rows are merged per unit. */ + | 'merge' + /** After an accepted upload, a revert or a zone change: every row becomes the image, DIRTY cleared. */ + | 'overwrite'; + +export interface EventWriteResult { + group: GroupBuilder; + conflicts: number; + stillDirty: boolean; +} + +/** Whether an exception row stands for "this instance was deleted" (a DELETED row or a new cancellation). */ +export function isRemovedInstance(row: LocalEventRow): boolean { + if (row.deleted) return true; + if (Number(row.cells[Events.STATUS]) !== EventStatus.CANCELED) return false; + // A cancellation the row already had when device sync wrote it is the server's own status. + return !row.baseline || Number(row.baseline.cells[Events.STATUS]) !== EventStatus.CANCELED; +} + +export interface EventWriteOptions { + mode: WriteMode; + ref: string; + ctx: CalendarContext; + /** The old server state the rows were written from; null: adopt (the rows' changes are measured against `event`). */ + base: CalendarEventWire | null; + /** SYNC_DATA3 to write; undefined leaves it. */ + pending?: null; + /** SYNC_DATA5 to write; undefined leaves it. */ + clearPoison?: boolean; +} + +/** + * The ops that bring a local event (master and exceptions) to `event`, + * merged per `options.mode`. `image` is `event`'s image in its calendar. + */ +export function writeEvent( + event: CalendarEventWire, + image: EventImage, + target: CalendarTarget, + local: LocalEvent, + options: EventWriteOptions, +): EventWriteResult { + const { ctx } = options; + const group = new GroupBuilder(options.ref); + const overwrite = options.mode === 'overwrite'; + const masterRef = objectRef(ctx.jmapAccountId, event.id); + const uid = typeof event.uid === 'string' && event.uid ? event.uid : null; + const baseEvent = options.base ?? event; + const baseTarget = options.base ? { calendarRowId: local.calendarRowId, calendarId: ctx.calendarIdOfRow(local.calendarRowId)?.calendarId ?? target.calendarId } : target; + const baseImage = options.base ? eventImage(baseEvent, baseTarget.calendarRowId, baseTarget.calendarId, ctx) ?? image : image; + const lossy = isLossyEditor(local.mutators); + const itemDirty = local.dirty || local.exceptions.some((x) => x.dirty || x.deleted); + + // Asserts first: the rows as read. + assertRow(group, local, false, itemDirty); + for (const x of local.exceptions) assertRow(group, x, true, itemDirty); + assertExceptionCount(group, local.eventId, linkedExceptionCount(local)); + + let conflicts = 0; + let stillDirty = false; + + const masterMerge = mergeRow({ + isException: false, + current: local, + baseline: options.base && local.baseline ? local.baseline : baselineOfImage(options.base ? baseImage.master : image.master, false), + base: baseImage.master, + remote: image.master, + dirty: !overwrite && local.dirty, + // A CONTENT_EXCEPTION_URI split capped this rule without DIRTY. + forced: !overwrite && local.split === 'source' ? new Set(['rule']) : undefined, + lossy, + self: ctx, + }); + conflicts += masterMerge.conflicts; + if (masterMerge.kept.length) stillDirty = true; + const masterState: RowState = { + isException: false, + cells: masterMerge.cells, + attendees: masterMerge.attendees, + reminders: masterMerge.reminders, + baseline: masterMerge.baseline, + syncId: masterRef, + uid, + shadow: event, + ...(options.pending === null ? { pending: null } : {}), + ...(options.clearPoison ? { poison: null } : {}), + clearDirty: overwrite || (local.dirty && !masterMerge.kept.length), + hasParticipants: image.master.hasParticipants, + }; + + const deletes: LocalException[] = []; + const updates: Array<{ row: LocalEventRow; state: RowState }> = []; + const inserts: RowState[] = []; + + const byKey = new Map(); + for (const x of local.exceptions) { + if (!x.recurrenceId) continue; + const list = byKey.get(x.recurrenceId) ?? []; + list.push(x); + byKey.set(x.recurrenceId, list); + } + const keys = new Set([...byKey.keys(), ...image.exceptions.keys(), ...baseImage.exceptions.keys()]); + const finalCalendarRowId = Number(masterMerge.cells[Events.CALENDAR_ID]); + + const exceptionState = (key: string, merged: { cells: Row; attendees: Row[]; reminders: Row[] | null; baseline: EventBaseline }, dirtyLeft: boolean, row: LocalEventRow | null, ours: boolean): RowState => ({ + isException: true, + cells: { ...merged.cells, [Events.CALENDAR_ID]: finalCalendarRowId }, + attendees: merged.attendees, + reminders: merged.reminders, + baseline: { ...merged.baseline, cells: { ...merged.baseline.cells, [Events.CALENDAR_ID]: finalCalendarRowId } }, + syncId: ours ? exceptionRef(masterRef, key) : row?.syncId ?? null, + uid, + recurrenceKey: ours ? key : ((row?.cells[Events.SYNC_DATA2] as string | null | undefined) ?? null), + originalSyncId: masterRef, + clearDirty: !!row?.dirty && !dirtyLeft, + hasParticipants: merged.attendees.length > 0, + }); + + for (const key of [...keys].sort()) { + const rows = byKey.get(key) ?? []; + const [row, ...duplicates] = rows; + // Two rows for one instance (an app's own duplicate): the extra ones go when clean. + for (const extra of duplicates) { + if (overwrite || !(extra.dirty || extra.deleted)) deletes.push(extra); + else stillDirty = true; + } + const r = image.exceptions.get(key); + const b = baseImage.exceptions.get(key); + if (!row) { + if (r) inserts.push(imageState(r, finalCalendarRowId, masterRef, uid)); + continue; + } + if (overwrite && row.deleted) { + // Its exclusion was uploaded (or is being reverted): a soft-deleted row is purged either way. + deletes.push(row); + if (r) inserts.push(imageState(r, finalCalendarRowId, masterRef, uid)); + continue; + } + const rowDirty = !overwrite && (row.dirty || row.deleted); + if (!rowDirty) { + if (r) updates.push({ row, state: exceptionState(key, { ...r, baseline: baselineOfImage(r, true) }, false, row, true) }); + else deletes.push(row); + continue; + } + if (isRemovedInstance(row)) { + // The device deleted this instance: that wins over a server edit and uploads as an exclusion. + stillDirty = true; + continue; + } + if ((b && !r) || image.excluded.has(key)) { + // The server removed or excluded the instance's override: the server delete wins. + deletes.push(row); + conflicts += 1; + continue; + } + const remoteSide = r ?? plainInstanceImage(event, key, finalCalendarRowId, target.calendarId, ctx); + const baseSide = b ?? plainInstanceImage(baseEvent, key, baseTarget.calendarRowId, baseTarget.calendarId, ctx) ?? remoteSide; + if (!remoteSide || !baseSide) { + stillDirty = true; + continue; + } + const merged = mergeRow({ + isException: true, + current: row, + baseline: row.baseline ?? baselineOfImage(baseSide, true), + base: baseSide, + remote: remoteSide, + dirty: true, + lossy, + self: ctx, + }); + conflicts += merged.conflicts; + // A local exception the server has no override for keeps its local identity until it is uploaded. + const left = merged.kept.length > 0 || !r; + if (left) stillDirty = true; + updates.push({ row, state: exceptionState(key, merged, left, row, !!r) }); + } + + for (const row of deletes) deleteRow(group, row.eventId); + updateRow(group, local, masterState); + for (const { row, state } of updates) updateRow(group, row, state); + for (const state of inserts) insertRow(group, state, { id: local.eventId }); + + return { group, conflicts, stillDirty: !overwrite && (stillDirty || local.deleted) }; +} + +/** The row an exception image becomes, written clean. */ +export function imageState(x: ExceptionImage, calendarRowId: number, masterRef: string, uid: string | null): RowState { + const cells = { ...x.cells, [Events.CALENDAR_ID]: calendarRowId }; + const baseline = baselineOfImage({ ...x, cells }, true); + return { + isException: true, + cells, + attendees: x.attendees, + reminders: x.reminders, + baseline, + syncId: exceptionRef(masterRef, x.key), + uid, + recurrenceKey: x.key, + originalSyncId: masterRef, + hasParticipants: x.hasParticipants, + }; +} + +/** A new event's rows: the master and one row per exception, attendees and reminders included. */ +export function insertEvent(event: CalendarEventWire, image: EventImage, ctx: CalendarContext, ref: string): GroupBuilder { + const group = new GroupBuilder(ref); + const masterRef = objectRef(ctx.jmapAccountId, event.id); + const uid = typeof event.uid === 'string' && event.uid ? event.uid : null; + const master = insertRow(group, { + isException: false, + cells: image.master.cells, + attendees: image.master.attendees, + reminders: image.master.reminders, + baseline: baselineOfImage(image.master, false), + syncId: masterRef, + uid, + shadow: event, + hasParticipants: image.master.hasParticipants, + }); + for (const key of [...image.exceptions.keys()].sort()) { + const x = image.exceptions.get(key)!; + insertRow(group, imageState(x, Number(image.master.cells[Events.CALENDAR_ID]), masterRef, uid), { ref: master }); + } + return group; +} diff --git a/src/device-sync/calendar/pairs.ts b/src/device-sync/calendar/pairs.ts new file mode 100644 index 00000000..8f4eeb81 --- /dev/null +++ b/src/device-sync/calendar/pairs.ts @@ -0,0 +1,150 @@ +/** + * Deletions and inserts that are one edit (docs/device-sync.md, "Uploads", + * "Pairs"). Etar moves an event to another calendar, and turns a series into + * a single event, by deleting the row and inserting a copy. A deleted row + * and a new row of the same run whose projections match apart from the + * calendar (or the rule) upload as a patch of the existing object, so its + * uid, participants and history stay; the new row then takes the old one's + * identity. Across JMAP accounts nothing pairs: that move is a create in the + * target account and a destroy in the source. + */ +import { Attendees, Events } from '../android-columns'; +import type { CalendarContext, LocalEvent, LocalEventRow, OpGroup } from '../planner'; +import type { Row } from '../types'; +import type { CalendarEventWire } from '../wire'; +import { exceptionRef, parseObjectRef } from '../common/ids'; +import { applyPatch, type PatchObject } from '../common/patch'; +import { EXCEPTION_CELLS, MASTER_CELLS, encodeBaseline, makeBaseline } from './columns'; +import { durationTextSeconds } from './duration'; +import { isExcluded, overridesOf } from './exceptions'; +import { GroupBuilder, assertRow, deleteRow, updateRow } from './rows'; +import { rruleParts } from './rrule'; +import { sendsSchedulingMessages } from './scheduling'; +import { put } from './upload'; + +/** What must be equal for two rows to be one event (the calendar and the rule aside). */ +const PAIR_COLUMNS = [ + Events.TITLE, + Events.DESCRIPTION, + Events.EVENT_LOCATION, + Events.STATUS, + Events.AVAILABILITY, + Events.ACCESS_LEVEL, + Events.EVENT_COLOR, + Events.DTSTART, + Events.ALL_DAY, +]; + +const num = (v: unknown) => (v === null || v === undefined || v === '' ? null : Number(v)); + +function lengthMs(cells: Row): number | null { + const dtstart = num(cells[Events.DTSTART]); + const dtend = num(cells[Events.DTEND]); + if (dtstart !== null && dtend !== null) return dtend - dtstart; + const seconds = durationTextSeconds(cells[Events.DURATION]); + return seconds === null ? null : seconds * 1000; +} + +const attendeeSet = (row: LocalEventRow) => + row.attendees.map((a) => String(a.cells[Attendees.ATTENDEE_EMAIL] ?? '').toLowerCase()).sort().join(','); + +export interface PairKind { + /** The same event in another calendar. */ + move: boolean; + /** A series turned into a single event. */ + unrecur: boolean; +} + +/** How a deleted and a new row are one edit, or null. */ +export function pairKind(deleted: LocalEvent, fresh: LocalEvent): PairKind | null { + for (const c of PAIR_COLUMNS) { + if (String(deleted.cells[c] ?? '') !== String(fresh.cells[c] ?? '')) return null; + } + const allDay = Number(fresh.cells[Events.ALL_DAY] ?? 0) === 1; + if (!allDay && String(deleted.cells[Events.EVENT_TIMEZONE] ?? '') !== String(fresh.cells[Events.EVENT_TIMEZONE] ?? '')) return null; + if (lengthMs(deleted.cells) !== lengthMs(fresh.cells)) return null; + if (attendeeSet(deleted) !== attendeeSet(fresh)) return null; + const move = deleted.calendarRowId !== fresh.calendarRowId; + const deletedRule = rruleParts(deleted.cells[Events.RRULE]); + const freshRule = rruleParts(fresh.cells[Events.RRULE]); + const unrecur = !!deletedRule && !freshRule; + if (!unrecur && String(deleted.cells[Events.RRULE] ?? '') !== String(fresh.cells[Events.RRULE] ?? '')) return null; + return move || unrecur ? { move, unrecur } : null; +} + +/** The patch a pair uploads: the calendar membership swapped, the rule and its overrides removed. */ +export function pairPatch( + deleted: LocalEvent, + fresh: LocalEvent, + kind: PairKind, + ctx: CalendarContext, +): { patch: PatchObject; sendSchedulingMessages: boolean } | null { + const shadow = deleted.shadow; + if (!parseObjectRef(deleted.syncId) || !shadow) return null; + const patch: PatchObject = {}; + if (kind.move) { + const from = ctx.calendarIdOfRow(deleted.calendarRowId); + const to = ctx.calendarIdOfRow(fresh.calendarRowId); + if (!from || !to || from.jmapAccountId !== ctx.jmapAccountId || to.jmapAccountId !== ctx.jmapAccountId) return null; + put(patch, ['calendarIds', from.calendarId], null); + put(patch, ['calendarIds', to.calendarId], true); + } + if (kind.unrecur) { + put(patch, ['recurrenceRule'], null); + if (Object.keys(overridesOf(shadow)).length) put(patch, ['recurrenceOverrides'], null); + } + // A move alone tells nobody; a series ending its recurrence does. + const scheduling = kind.unrecur && sendsSchedulingMessages(shadow, { kind: 'update', notifying: true, rsvp: false, addsAttendees: false }, ctx); + return { patch, sendSchedulingMessages: scheduling }; +} + +/** + * Applied once the pair's patch is accepted: the old row and its exception + * rows go, the new row takes the identity (its copied exception rows the + * overrides that still exist), and the shadow is the old one with the patch + * applied until the next download brings the server's. + */ +export function pairOps(deleted: LocalEvent, fresh: LocalEvent, patch: PatchObject): OpGroup { + const syncId = deleted.syncId!; + const shadow: CalendarEventWire = applyPatch(deleted.shadow!, patch) ?? deleted.shadow!; + const group = new GroupBuilder(syncId); + assertRow(group, deleted, false, false); + assertRow(group, fresh, false, true); + for (const x of fresh.exceptions) assertRow(group, x, true, true); + for (const x of deleted.exceptions) deleteRow(group, x.eventId); + deleteRow(group, deleted.eventId); + const children = (row: LocalEventRow) => ({ attendees: row.attendees.map((a) => a.cells), reminders: row.reminders.map((r) => r.cells) }); + const cells: Row = {}; + for (const c of MASTER_CELLS) cells[c] = fresh.cells[c] ?? null; + updateRow( + group, + { ...fresh, shadow: null }, + { + isException: false, + cells, + ...children(fresh), + baseline: makeBaseline(fresh.cells, MASTER_CELLS, children(fresh).attendees, children(fresh).reminders), + syncId, + uid: typeof shadow.uid === 'string' ? shadow.uid : null, + shadow, + pending: null, + clearDirty: true, + hasParticipants: fresh.attendees.length > 0, + }, + ); + const overrides = overridesOf(shadow); + for (const x of fresh.exceptions) { + const key = x.recurrenceId; + const values: Row = { [Events.ORIGINAL_SYNC_ID]: syncId }; + if (key && overrides[key] && !isExcluded(overrides[key])) { + Object.assign(values, { + [Events._SYNC_ID]: exceptionRef(syncId, key), + [Events.SYNC_DATA2]: key, + [Events.SYNC_DATA4]: encodeBaseline(makeBaseline(x.cells, EXCEPTION_CELLS, children(x).attendees, children(x).reminders)), + [Events.DIRTY]: 0, + }); + } + group.write({ op: 'update', table: 'events', id: x.eventId, values, expectCount: 1 }); + } + return group.build(); +} diff --git a/src/device-sync/calendar/planner.ts b/src/device-sync/calendar/planner.ts new file mode 100644 index 00000000..31e11f68 --- /dev/null +++ b/src/device-sync/calendar/planner.ts @@ -0,0 +1,463 @@ +/** + * The calendar planner (docs/device-sync.md, "Calendar mapping"): JSCalendar + * events and calendars ↔ CalendarContract rows, pure. The engine owns all + * I/O; this module turns rows and server objects into op groups and upload + * actions, following the contract in ../planner.ts. + * + * Tasks never reach CalendarContract: the engine filters them, and every + * entry point here refuses one as well. + */ +import { Events } from '../android-columns'; +import type { + AcceptedPlan, + CalendarContext, + CalendarPlanner, + DownloadPlan, + LocalEvent, + LocalEventRow, + OpGroup, + PoisonMarker, + UploadAction, + UploadPlan, +} from '../planner'; +import type { CalendarEventWire } from '../wire'; +import { objectRef, parseObjectRef, pendingUidOf } from '../common/ids'; +import { deepEqual, jsonHash } from '../common/json'; +import { isTaskLikeObject } from '../../lib/calendar-component-detection'; +import { keepDirtyGroup, type UploadedUnits } from './accepted'; +import { ATTENDEE_COLUMNS, CALENDAR_COLUMNS, EVENT_COLUMNS, EXCEPTION_CELLS, MASTER_CELLS, REMINDER_COLUMNS, isTruncatedBaseline, makeBaseline } from './columns'; +import { planCalendars } from './calendars'; +import { claimOps, claimUid, claimedUid, computeCreate, createTarget, deleteActions } from './create'; +import { decodeCalendar, decodeEvents, isExceptionRow, isNewMaster } from './decode'; +import { isExcluded, overridesOf } from './exceptions'; +import { eventImage } from './image'; +import { insertEvent, isRemovedInstance, linkedExceptionCount, pickCalendar, writeEvent } from './merge'; +import { pairKind, pairOps, pairPatch } from './pairs'; +import { GroupBuilder, assertExceptionCount, assertRow, deleteRow, writesOf } from './rows'; +import { eventZone } from './timing'; +import { columnUnitDiffers, remindersDiffer, sideOfBaseline, sideOfRow } from './units'; +import { SKIP, SkipUpload, computeUpdate, type UploadComputation } from './upload'; + +type Plan = UploadPlan; + +function refuseTask(event: CalendarEventWire | null | undefined): void { + if (event && isTaskLikeObject(event)) { + throw new Error('calendar planner: tasks are not synced to the device (filter them before planning)'); + } +} + +function refOf(local: LocalEventRow | null, event?: CalendarEventWire, jmapAccountId?: string): string { + if (local?.syncId) return local.syncId; + if (event && jmapAccountId) return objectRef(jmapAccountId, event.id); + return local ? `row:${local.eventId}` : 'new'; +} + +// ─── Downloads ────────────────────────────────────────── + +function planDownload(event: CalendarEventWire, local: LocalEvent | null, ctx: CalendarContext): DownloadPlan { + refuseTask(event); + refuseTask(local?.shadow); + const ref = refOf(local, event, ctx.jmapAccountId); + const none = (stillDirty: boolean): DownloadPlan => ({ ops: { ref, ops: [] }, conflicts: 0, stillDirty, effect: 'none', writes: 0 }); + const target = pickCalendar(event, local, ctx); + if (!target) throw new Error(`calendar planner: ${ref} is in no synced calendar (the engine removes such rows)`); + const image = eventImage(event, target.calendarRowId, target.calendarId, ctx); + // Stalwart drops a start it can't resolve (a DST gap or overlap): nothing to show. + if (!image) return none(!!local && (local.dirty || local.deleted)); + + if (!local) { + const ops = insertEvent(event, image, ctx, ref).build(); + return { ops, conflicts: 0, stillDirty: false, effect: 'insert', writes: writesOf(ops) }; + } + // A split clone only looks like this object: it is a new event and uploads as one. The source row is the object's. + if (local.split === 'clone') return none(true); + + // A different object under a reused id: its rows are replaced, never patched. + const uidChanged = !!local.shadow && typeof local.shadow.uid === 'string' && typeof event.uid === 'string' && local.shadow.uid !== event.uid; + if (uidChanged) { + const group = new GroupBuilder(ref); + for (const x of local.exceptions) deleteRow(group, x.eventId); + deleteRow(group, local.eventId); + const inserted = insertEvent(event, image, ctx, ref).build().ops; + const offset = group.writeCount; + for (const op of inserted) { + if (op.op === 'insert' && op.refs) { + const refs = Object.fromEntries(Object.entries(op.refs).map(([k, v]) => [k, v + offset])); + group.write({ ...op, refs }); + } else group.write(op); + } + const ops = group.build(); + return { ops, conflicts: 0, stillDirty: false, effect: 'update', writes: writesOf(ops) }; + } + + if (local.deleted) { + // Deleted on the device: that wins over a server edit, so only the shadow follows, for the destroy. + if (!local.shadow || deepEqual(local.shadow, event)) return none(true); + const group = new GroupBuilder(ref); + group.assert({ op: 'assert', table: 'events', id: local.eventId, values: { [Events.DELETED]: 1, [Events._SYNC_ID]: local.syncId }, expectCount: 1 }); + group.write({ op: 'update', table: 'events', id: local.eventId, values: { [Events.SYNC_DATA1]: JSON.stringify(event) }, expectCount: 1 }); + const ops = group.build(); + return { ops, conflicts: 0, stillDirty: true, effect: 'update', writes: writesOf(ops) }; + } + + const itemDirty = local.dirty || local.split === 'source' || local.exceptions.some((x) => x.dirty || x.deleted); + const result = writeEvent(event, image, target, local, { + mode: 'merge', + ref, + ctx, + // No shadow: our own create whose identity was never written. Adopt it; its rows count as edits of `event`. + base: local.shadow, + ...(local.shadow ? {} : { pending: null }), + clearPoison: !itemDirty, + }); + const ops = result.group.build(); + const writes = writesOf(ops); + return { + ops: writes ? ops : { ref, ops: [] }, + conflicts: result.conflicts, + stillDirty: result.stillDirty, + effect: writes ? 'update' : 'none', + writes, + }; +} + +function planLocalDelete(local: LocalEvent): OpGroup { + const group = new GroupBuilder(refOf(local)); + assertExceptionCount(group, local.eventId, linkedExceptionCount(local)); + for (const x of local.exceptions) deleteRow(group, x.eventId); + deleteRow(group, local.eventId); + return group.build(); +} + +/** + * Rewrites the baselines of clean rows that drifted from them (the provider + * normalised a write, or a crash hit between a chunk and its read-back), + * behind DIRTY=0. A recurring master's timing and rule are never healed: a + * CONTENT_EXCEPTION_URI split changes them without DIRTY. + */ +function planBaselineHeal(local: LocalEvent): OpGroup | null { + if (local.split || local.deleted || isExceptionRow(local.cells)) return null; + const group = new GroupBuilder(refOf(local)); + const heals: Array<{ row: LocalEventRow; isException: boolean; baseline: ReturnType }> = []; + const consider = (row: LocalEventRow, isException: boolean) => { + if (row.dirty || row.deleted) return; + const columns = isException ? EXCEPTION_CELLS : MASTER_CELLS; + const truncated = isTruncatedBaseline(row.baseline?.cells[Events.DESCRIPTION]) && !columnUnitDiffers('description', sideOfRow(row), sideOfBaseline(row.baseline!)); + const next = makeBaseline(row.cells, columns, row.attendees.map((a) => a.cells), row.reminders.map((r) => r.cells), truncated); + if (truncated) next.cells[Events.DESCRIPTION] = row.baseline!.cells[Events.DESCRIPTION]; + const recurring = !isException && (!!row.cells[Events.RRULE] || !!row.baseline?.cells[Events.RRULE]); + if (recurring && row.baseline) { + for (const c of [Events.DTSTART, Events.DTEND, Events.DURATION, Events.EVENT_TIMEZONE, Events.ALL_DAY, Events.RRULE]) { + next.cells[c] = row.baseline.cells[c] ?? null; + } + } + if (row.baseline && deepEqual(next, row.baseline)) return; + heals.push({ row, isException, baseline: next }); + }; + consider(local, false); + for (const x of local.exceptions) consider(x, true); + if (!heals.length) return null; + for (const h of heals) assertRow(group, h.row, h.isException, false); + for (const h of heals) { + group.write({ op: 'update', table: 'events', id: h.row.eventId, values: { [Events.SYNC_DATA4]: JSON.stringify({ v: 1, ...h.baseline }) }, expectCount: 1 }); + } + return group.build(); +} + +// ─── Uploads ──────────────────────────────────────────── + +/** + * The fingerprint a poison marker records for an event: what it would upload + * from (its flags, shadow, rows, attendees, reminders and exceptions), so the + * back-off holds while nothing changes and lifts when the user edits it. The + * engine writes the markers and checks them before planning + * (engine/calendar-sync.ts); this is the same content, so an item handed in + * anyway is still skipped. + */ +export function poisonFingerprint(e: LocalEvent): string { + return jsonHash({ + deleted: e.deleted, + shadow: e.shadow, + cells: e.cells, + attendees: e.attendees.map((a) => JSON.stringify(a.cells)).sort(), + reminders: e.reminders.map((r) => JSON.stringify(r.cells)).sort(), + exceptions: e.exceptions + .map((x) => JSON.stringify([x.deleted, x.cells, x.attendees.map((a) => a.cells), x.reminders.map((r) => r.cells)])) + .sort(), + }); +} + +/** Writes a poison marker (SYNC_DATA5) on the master. */ +export function poisonOps(local: LocalEvent, marker: PoisonMarker): OpGroup { + return { + ref: refOf(local), + ops: [{ op: 'update', table: 'events', id: local.eventId, values: { [Events.SYNC_DATA5]: JSON.stringify(marker) }, expectCount: 1 }], + }; +} + +function activePoison(local: LocalEvent, ctx: CalendarContext): PoisonMarker | null { + const p = local.poison; + if (!p || ctx.now >= p.until) return null; + return p.fp === poisonFingerprint(local) ? p : null; +} + +/** Deletes a master and its exceptions (a purge, a revert that refetches). */ +function purgeGroup(local: LocalEvent): OpGroup { + return planLocalDelete(local); +} + +/** Rows rewritten from the shadow, DIRTY cleared: a read-only event's device edits are undone. */ +function revertGroup(local: LocalEvent, ctx: CalendarContext): OpGroup | null { + const shadow = local.shadow; + if (!shadow) return null; + const target = pickCalendar(shadow, local, ctx) ?? (() => { + const cal = ctx.calendarIdOfRow(local.calendarRowId); + return cal ? { calendarRowId: local.calendarRowId, calendarId: cal.calendarId } : null; + })(); + if (!target) return null; + const image = eventImage(shadow, target.calendarRowId, target.calendarId, ctx); + if (!image) return null; + return writeEvent(shadow, image, target, local, { mode: 'overwrite', ref: refOf(local), ctx, base: shadow }).group.build(); +} + +/** Clears DIRTY where nothing uploadable changed, taking the rows as they are as the new baselines. */ +function cleanGroup(local: LocalEvent, computation: UploadComputation | null): OpGroup { + const group = new GroupBuilder(refOf(local)); + assertRow(group, local, false, true); + for (const x of local.exceptions) assertRow(group, x, true, true); + const settle = (row: LocalEventRow, isException: boolean) => { + if (!row.dirty) return; + const truncated = isTruncatedBaseline(row.baseline?.cells[Events.DESCRIPTION]); + const baseline = makeBaseline(row.cells, isException ? EXCEPTION_CELLS : MASTER_CELLS, row.attendees.map((a) => a.cells), row.reminders.map((r) => r.cells), truncated); + group.write({ op: 'update', table: 'events', id: row.eventId, values: { [Events.DIRTY]: 0, [Events.SYNC_DATA4]: JSON.stringify({ v: 1, ...baseline }) }, expectCount: 1 }); + }; + settle(local, false); + const overrides = overridesOf(local.shadow); + for (const x of local.exceptions) { + const excludedAlready = !!x.recurrenceId && isExcluded(overrides[x.recurrenceId]); + if ((computation?.removedExceptions.has(x.eventId) || isRemovedInstance(x)) && excludedAlready) { + deleteRow(group, x.eventId); + continue; + } + settle(x, true); + } + return group.build(); +} + +function planUpload(local: LocalEvent, ctx: CalendarContext): Plan { + refuseTask(local.shadow); + // An exception row whose master is gone never uploads as an event of its own. + if (isExceptionRow(local.cells)) return { kind: 'purge', ops: purgeGroup(local) }; + const calendar = ctx.calendarIdOfRow(local.calendarRowId); + if (!calendar) return { kind: 'skip', reason: SKIP.notOurCalendar }; + const isNew = local.split === 'clone' || (!local.shadow && isNewMaster(local)); + + if (local.deleted) { + if (isNew) { + // Never created (or its outcome unknown): destroy whatever carries the pending uid, else just purge. + const uid = local.split === 'clone' ? null : local.pending?.uid ?? pendingUidOf(local.syncId); + if (!uid) return { kind: 'purge', ops: purgeGroup(local) }; + return { kind: 'upload', actions: [{ kind: 'destroy', id: null, uid }] }; + } + if (!parseObjectRef(local.syncId)) return { kind: 'purge', ops: purgeGroup(local) }; + if (ctx.isReadOnly(calendar.calendarId)) return { kind: 'revert', ops: purgeGroup(local), refetch: true }; + return { kind: 'upload', actions: deleteActions(local, ctx) }; + } + + if (isNew) { + if (ctx.isReadOnly(calendar.calendarId)) return { kind: 'revert', ops: purgeGroup(local) }; + const uid = claimedUid(local); + const target = createTarget(local, ctx); + if (!target) return { kind: 'skip', reason: SKIP.notOurCalendar }; + if (!uid) return { kind: 'claim', ops: claimOps(local, claimUid(local, ctx), target.key) }; + const poison = activePoison(local, ctx); + if (poison) return { kind: 'skip', reason: poison.type }; + try { + const create = computeCreate(local, uid, target.calendarId, ctx); + const action: UploadAction = { kind: 'create', uid, collectionId: target.calendarId, object: create.object }; + if (create.sendSchedulingMessages) action.sendSchedulingMessages = true; + return { kind: 'upload', actions: [action] }; + } catch (e) { + if (e instanceof SkipUpload) return { kind: 'skip', reason: e.reason }; + throw e; + } + } + + const ref = parseObjectRef(local.syncId); + const shadow = local.shadow; + if (!ref || !shadow) return { kind: 'skip', reason: 'noShadow' }; + const poison = activePoison(local, ctx); + if (poison) return { kind: 'skip', reason: poison.type }; + // An instance Stalwart holds without its series can't be changed on its own. + if (shadow.recurrenceId && !shadow.recurrenceRule) { + const ops = revertGroup(local, ctx); + return ops ? { kind: 'revert', ops } : { kind: 'skip', reason: SKIP.instanceOnly }; + } + + let computation: UploadComputation; + try { + computation = computeUpdate(local, ctx); + } catch (e) { + if (e instanceof SkipUpload) return { kind: 'skip', reason: e.reason }; + throw e; + } + const keys = Object.keys(computation.patch); + const readOnly = ctx.isReadOnly(calendar.calendarId); + const movedToReadOnly = keys.some((k) => { + const m = /^calendarIds\/(.+)$/.exec(k); + return m && computation.patch[k] === true && ctx.isReadOnly(m[1]); + }); + if ((readOnly && keys.length) || movedToReadOnly) { + // RSVP is the one change a calendar without write rights may take. + const rsvpOnly = keys.length > 0 && keys.every((k) => /^participants\/[^/]+\/participationStatus$/.test(k)); + if (!(rsvpOnly && ctx.calendar(calendar.calendarId)?.myRights?.mayRSVP && !movedToReadOnly)) { + const ops = revertGroup(local, ctx); + return ops ? { kind: 'revert', ops } : { kind: 'skip', reason: 'readOnly' }; + } + } + if (!keys.length) { + if (computation.dropped.some((u) => u === 'timing' || u === 'rule')) return { kind: 'skip', reason: SKIP.ruleNotRepresentable }; + return { kind: 'clean', ops: cleanGroup(local, computation) }; + } + const action: UploadAction = { kind: 'update', id: ref.id, patch: computation.patch }; + if (computation.sendSchedulingMessages) action.sendSchedulingMessages = true; + return { kind: 'upload', actions: [action] }; +} + +function planAccepted(local: LocalEvent, server: CalendarEventWire, ctx: CalendarContext): AcceptedPlan { + refuseTask(server); + const target = pickCalendar(server, local, ctx); + if (!target) { + // Moved out of every synced calendar (a membership removal): the rows go. + const ops = purgeGroup(local); + return { ops, keepDirtyOps: ops }; + } + const wasNew = local.split === 'clone' || (!local.shadow && isNewMaster(local)); + let uploaded: UploadedUnits; + if (wasNew || !local.shadow) { + uploaded = { all: true, master: new Set(), exceptions: new Map(), removed: new Set(local.exceptions.filter(isRemovedInstance).map((x) => x.eventId)) }; + } else { + try { + const c = computeUpdate(local, ctx); + uploaded = { all: false, master: c.masterUnits, exceptions: c.exceptionUnits, removed: c.removedExceptions }; + } catch { + uploaded = { all: false, master: new Set(), exceptions: new Map(), removed: new Set() }; + } + } + const keepDirtyOps = keepDirtyGroup(local, server, uploaded, ctx); + const image = eventImage(server, target.calendarRowId, target.calendarId, ctx); + if (!image) return { ops: keepDirtyOps, keepDirtyOps }; + const ops = writeEvent(server, image, target, local, { + mode: 'overwrite', + ref: objectRef(ctx.jmapAccountId, server.id), + ctx, + base: local.shadow ?? server, + pending: null, + clearPoison: true, + }).group.build(); + return { ops, keepDirtyOps }; +} + +function planPairs( + deleted: LocalEvent[], + fresh: LocalEvent[], + ctx: CalendarContext, +): Array<{ deleted: LocalEvent; fresh: LocalEvent; actions: UploadAction[]; ops: OpGroup }> { + const pairs: Array<{ deleted: LocalEvent; fresh: LocalEvent; actions: UploadAction[]; ops: OpGroup }> = []; + const used = new Set(); + const inAccount = (row: LocalEventRow) => ctx.calendarIdOfRow(row.calendarRowId)?.jmapAccountId === ctx.jmapAccountId; + for (const d of deleted) { + const ref = parseObjectRef(d.syncId); + if (!ref || !d.shadow || !d.deleted || d.split || !inAccount(d)) continue; + refuseTask(d.shadow); + for (const f of fresh) { + if (used.has(f.eventId) || f.syncId !== null || f.pending || f.deleted || isExceptionRow(f.cells) || !inAccount(f)) continue; + const kind = pairKind(d, f); + if (!kind) continue; + const pair = pairPatch(d, f, kind, ctx); + if (!pair) continue; + const action: UploadAction = { kind: 'update', id: ref.id, patch: pair.patch }; + if (pair.sendSchedulingMessages) action.sendSchedulingMessages = true; + pairs.push({ deleted: d, fresh: f, actions: [action], ops: pairOps(d, f, pair.patch) }); + used.add(f.eventId); + break; + } + } + return pairs; +} + +// ─── Device zone and reminder owner ───────────────────── + +function planZoneChange(local: LocalEvent, previousZone: string, ctx: CalendarContext): OpGroup | null { + const shadow = local.shadow; + if (!shadow || local.split || local.deleted || local.dirty || local.exceptions.some((x) => x.dirty || x.deleted)) return null; + refuseTask(shadow); + if (previousZone === ctx.deviceZone || !eventZone(shadow, ctx.deviceZone).floating) return null; + const target = pickCalendar(shadow, local, ctx); + if (!target) return null; + const image = eventImage(shadow, target.calendarRowId, target.calendarId, ctx); + if (!image) return null; + const ops = writeEvent(shadow, image, target, local, { mode: 'overwrite', ref: refOf(local), ctx, base: shadow }).group.build(); + return writesOf(ops) ? ops : null; +} + +function planReminderOwnerChange(local: LocalEvent, ctx: CalendarContext): OpGroup | null { + const shadow = local.shadow; + if (!shadow || local.deleted || isExceptionRow(local.cells)) return null; + refuseTask(shadow); + const target = pickCalendar(shadow, local, ctx); + if (!target) return null; + const image = eventImage(shadow, target.calendarRowId, target.calendarId, { ...ctx, reminderOwner: 'device' }); + if (!image) return null; + const rows: Array<{ row: LocalEventRow; isException: boolean; wanted: LocalEventRow['cells'][] }> = []; + const add = (row: LocalEventRow, isException: boolean, wanted: LocalEventRow['cells'][] | null) => { + if (!wanted) return; + // A reminder edit that has not reached the server yet is not overwritten. + if (row.dirty && row.baseline && remindersDiffer(sideOfRow(row), sideOfBaseline(row.baseline))) return; + rows.push({ row, isException, wanted: ctx.reminderOwner === 'device' ? wanted : [] }); + }; + add(local, false, image.master.reminders); + for (const x of local.exceptions) { + if (x.deleted || !x.recurrenceId) continue; + const xi = image.exceptions.get(x.recurrenceId); + add(x, true, xi ? xi.reminders : null); + } + const changing = rows.filter(({ row, wanted }) => { + const have = row.reminders.map((r) => r.cells); + return have.length !== wanted.length || remindersDiffer({ cells: {}, attendees: [], reminders: have }, { cells: {}, attendees: [], reminders: wanted }); + }); + if (!changing.length) return null; + const group = new GroupBuilder(refOf(local)); + for (const { row, isException } of changing) assertRow(group, row, isException, true); + for (const { row, isException, wanted } of changing) { + for (const r of row.reminders) group.write({ op: 'delete', table: 'reminders', id: r.id, expectCount: 1 }); + for (const w of wanted) group.write({ op: 'insert', table: 'reminders', values: { ...w, event_id: row.eventId } }); + const baseline = row.baseline ?? makeBaseline(row.cells, isException ? EXCEPTION_CELLS : MASTER_CELLS, row.attendees.map((a) => a.cells), []); + group.write({ + op: 'update', + table: 'events', + id: row.eventId, + values: { [Events.SYNC_DATA4]: JSON.stringify({ v: 1, ...baseline, reminders: wanted }) }, + expectCount: 1, + }); + } + return group.build(); +} + +export const calendarPlanner: CalendarPlanner = { + calendarColumns: CALENDAR_COLUMNS, + eventColumns: EVENT_COLUMNS, + attendeeColumns: ATTENDEE_COLUMNS, + reminderColumns: REMINDER_COLUMNS, + decodeCalendar, + decodeEvents, + planCalendars, + planDownload, + planLocalDelete, + planBaselineHeal, + planUpload, + planAccepted, + planPairs, + planZoneChange, + planReminderOwnerChange, +}; diff --git a/src/device-sync/calendar/reminders.ts b/src/device-sync/calendar/reminders.ts new file mode 100644 index 00000000..73d418f1 --- /dev/null +++ b/src/device-sync/calendar/reminders.ts @@ -0,0 +1,153 @@ +/** + * Alerts ↔ Reminders rows (docs/device-sync.md, "Other properties" and + * "Reminder owner"). + * + * A Reminders row is "minutes before the start" plus a method, so only an + * offset alert relative to the start (or with no `relativeTo`) that fires at + * or before it is shown; alerts relative to the end, absolute alerts and + * alerts after the start stay on the server. The app's + * `offsetToMinutesBefore` rejects the valid `+PT15M` (and fractions), so + * offsets are read with this module's own duration parser. + */ +import type { Alert } from '../../api/types'; +import { Reminders } from '../android-columns'; +import type { CalendarLike } from '../wire'; +import type { Row } from '../types'; +import { minutesBeforeToOffset } from '../../lib/calendar-alerts'; +import { durationSeconds, parseDuration } from './duration'; +import { reminderActionFromDevice, reminderMethodToDevice } from './values'; + +export interface ReminderValue { + minutes: number; + /** Reminders.METHOD, normalised: EMAIL (2) or ALERT (1). */ + method: number; +} + +type Alerts = Record | null | undefined; + +/** + * The alerts that are in force: with `useDefaultAlerts` the calendar's + * defaults for timed or all-day events (and the event's own `alerts` are + * ignored, as the server and the app's scheduler do), else the event's own. + */ +export function effectiveAlerts( + event: { useDefaultAlerts?: boolean | null; alerts?: Alerts; showWithoutTime?: boolean | null }, + calendar: CalendarLike | undefined, +): Record { + if (event.useDefaultAlerts) { + const defaults = event.showWithoutTime ? calendar?.defaultAlertsWithoutTime : calendar?.defaultAlertsWithTime; + return { ...(defaults ?? {}) }; + } + return { ...(event.alerts ?? {}) }; +} + +/** The reminder an alert shows as, or null when a Reminders row cannot hold it. */ +export function alertReminder(alert: Alert | null | undefined): ReminderValue | null { + const trigger = alert?.trigger; + if (!trigger || typeof trigger !== 'object') return null; + const type = trigger['@type']; + if (type !== undefined && type !== 'OffsetTrigger') return null; + if (trigger.offset === undefined || trigger.when !== undefined) return null; + if (trigger.relativeTo !== undefined && trigger.relativeTo !== null && trigger.relativeTo !== 'start') return null; + const d = parseDuration(trigger.offset); + if (!d) return null; + const seconds = durationSeconds(d); + if (seconds > 0) return null; + return { minutes: Math.round(-seconds / 60), method: reminderMethodToDevice(alert.action) }; +} + +export function isRepresentableAlert(alert: Alert | null | undefined): boolean { + return alertReminder(alert) !== null; +} + +export function reminderRow(value: ReminderValue): Row { + return { [Reminders.MINUTES]: value.minutes, [Reminders.METHOD]: value.method }; +} + +/** Reminders rows for the alerts that have one, in the alerts' order. */ +export function reminderRows(alerts: Alerts): Row[] { + return Object.values(alerts ?? {}) + .map(alertReminder) + .filter((r): r is ReminderValue => r !== null) + .map(reminderRow); +} + +/** + * A device row as a reminder, or null for one that means nothing to the + * server (MINUTES = -1 is "the app's default"). + */ +export function rowReminder(row: Row): ReminderValue | null { + const minutes = Number(row[Reminders.MINUTES]); + if (!Number.isInteger(minutes) || minutes < 0) return null; + return { minutes, method: reminderMethodToDevice(reminderActionFromDevice(row[Reminders.METHOD])) }; +} + +const keyOf = (r: ReminderValue) => `${r.minutes}/${r.method}`; + +/** Two reminder lists as multisets of (minutes, method). */ +export function sameReminders(a: Row[], b: Row[]): boolean { + const count = (rows: Row[]) => { + const m = new Map(); + for (const r of rows) { + const v = rowReminder(r); + if (v) m.set(keyOf(v), (m.get(keyOf(v)) ?? 0) + 1); + } + return m; + }; + const ca = count(a); + const cb = count(b); + if (ca.size !== cb.size) return false; + for (const [k, n] of ca) if (cb.get(k) !== n) return false; + return true; +} + +/** A new alert for a device reminder, shaped like the app's reminder picker writes them. */ +export function reminderAlert(value: ReminderValue): Alert { + return { + '@type': 'Alert', + trigger: { '@type': 'OffsetTrigger', offset: minutesBeforeToOffset(value.minutes), relativeTo: 'start' }, + action: reminderActionFromDevice(value.method), + }; +} + +/** + * The alerts after the device's reminders replaced the representable ones: + * existing alerts whose (minutes, method) a reminder still has keep their + * key and value, the others go, unmatched reminders are added under minted + * keys, and alerts a Reminders row can't hold stay as they are. + */ +export function remindersToAlerts( + current: Record, + reminders: Row[], + mintKey: (taken: Iterable) => string, +): { alerts: Record; removed: string[]; added: Record } { + const wanted = reminders.map(rowReminder).filter((r): r is ReminderValue => r !== null); + const pool = new Map(); + for (const r of wanted) pool.set(keyOf(r), (pool.get(keyOf(r)) ?? 0) + 1); + const alerts: Record = {}; + const removed: string[] = []; + for (const [key, alert] of Object.entries(current)) { + const r = alertReminder(alert); + if (!r) { + alerts[key] = alert; + continue; + } + const left = pool.get(keyOf(r)) ?? 0; + if (left > 0) { + pool.set(keyOf(r), left - 1); + alerts[key] = alert; + } else { + removed.push(key); + } + } + const added: Record = {}; + for (const r of wanted) { + const left = pool.get(keyOf(r)) ?? 0; + if (left <= 0) continue; + pool.set(keyOf(r), left - 1); + const key = mintKey([...Object.keys(current), ...Object.keys(added)]); + added[key] = reminderAlert(r); + alerts[key] = added[key]; + } + return { alerts, removed, added }; +} diff --git a/src/device-sync/calendar/rows.ts b/src/device-sync/calendar/rows.ts new file mode 100644 index 00000000..8c877a6c --- /dev/null +++ b/src/device-sync/calendar/rows.ts @@ -0,0 +1,251 @@ +/** + * Writing event rows: the op group builder and the translation of "this row + * should hold that" into the fewest provider ops (docs/device-sync.md, + * "Merge rules" and "Uploads"). + * + * A group is atomic and its asserts come first, so the builder refuses an + * assert after the first write; `refs` are indexes into the group. Rows are + * addressed by id with `expectCount: 1`, exception rows are deleted by id + * only (a sync-adapter delete never cascades to them), and a write that + * would not change a cell is left out, so an echo writes nothing. + */ +import { Attendees, Events, Reminders } from '../android-columns'; +import type { EventBaseline, LocalEventRow, OpGroup, PendingCreate, PoisonMarker } from '../planner'; +import type { ProviderOp, Row, WriteRow } from '../types'; +import type { CalendarEventWire } from '../wire'; +import { deepEqual } from '../common/json'; +import { ATTENDEE_CELLS, MASTER_CELLS, EXCEPTION_CELLS, REMINDER_CELLS, encodeBaseline, sameCell } from './columns'; +import { rowReminder } from './reminders'; + +export class GroupBuilder { + private readonly asserts: ProviderOp[] = []; + private readonly writes: ProviderOp[] = []; + + constructor(readonly ref: string) {} + + assert(op: Extract): void { + if (this.writes.length) throw new Error('calendar planner: an assert after a write'); + this.asserts.push(op); + } + + /** Adds a write and returns its index in the group (for `refs`). */ + write(op: ProviderOp): number { + this.writes.push(op); + return this.asserts.length + this.writes.length - 1; + } + + get writeCount(): number { + return this.writes.length; + } + + build(): OpGroup { + return { ref: this.ref, ops: [...this.asserts, ...this.writes] }; + } +} + +/** Columns asserted for a row: sync state plus its projection. */ +const ASSERT_EXTRA = [Events.DIRTY, Events.DELETED, Events._SYNC_ID] as const; + +export function projectionColumns(isException: boolean): readonly string[] { + return isException ? EXCEPTION_CELLS : MASTER_CELLS; +} + +/** + * Asserts that a row still holds what was read: DIRTY, DELETED, `_SYNC_ID` + * and its mapped cells, and with `children` its attendees and reminders + * (each row's cells plus their number, so an added one fails too). DIRTY + * alone guards a clean row against app edits (every app write to an event, + * its attendees or reminders sets it); a dirty row needs the whole + * projection. + */ +export function assertRow(group: GroupBuilder, row: LocalEventRow, isException: boolean, children: boolean): void { + const values: Row = {}; + // As read: a row a sync adapter inserted without DIRTY holds NULL there, not 0. + values[Events.DIRTY] = row.cells[Events.DIRTY] ?? null; + values[Events.DELETED] = row.cells[Events.DELETED] ?? null; + values[Events._SYNC_ID] = row.syncId; + for (const c of projectionColumns(isException)) values[c] = row.cells[c] ?? null; + for (const c of ASSERT_EXTRA) if (!(c in values)) values[c] = row.cells[c] ?? null; + group.assert({ op: 'assert', table: 'events', id: row.eventId, values, expectCount: 1 }); + if (!children) return; + for (const a of row.attendees) { + group.assert({ op: 'assert', table: 'attendees', id: a.id, values: pick(a.cells, ATTENDEE_CELLS), expectCount: 1 }); + } + group.assert({ op: 'assert', table: 'attendees', where: `${Attendees.EVENT_ID} = ?`, args: [row.eventId], expectCount: row.attendees.length }); + for (const r of row.reminders) { + group.assert({ op: 'assert', table: 'reminders', id: r.id, values: pick(r.cells, REMINDER_CELLS), expectCount: 1 }); + } + group.assert({ op: 'assert', table: 'reminders', where: `${Reminders.EVENT_ID} = ?`, args: [row.eventId], expectCount: row.reminders.length }); +} + +/** Asserts the number of exception rows of a master, so one an app adds meanwhile fails the group. */ +export function assertExceptionCount(group: GroupBuilder, masterId: number, count: number): void { + group.assert({ op: 'assert', table: 'events', where: `${Events.ORIGINAL_ID} = ?`, args: [masterId], expectCount: count }); +} + +function pick(row: Row, columns: readonly string[]): Row { + const out: Row = {}; + for (const c of columns) out[c] = row[c] ?? null; + return out; +} + +/** What a row should hold after the group: content, sync columns, children. */ +export interface RowState { + isException: boolean; + /** Mapped cells (the projection). */ + cells: Row; + attendees: Row[]; + /** Null: Reminders rows are left as they are. */ + reminders: Row[] | null; + baseline: EventBaseline; + syncId: string | null; + uid: string | null; + /** Masters: SYNC_DATA1; undefined leaves it. */ + shadow?: CalendarEventWire | null; + /** Exceptions: SYNC_DATA2 (the recurrence id). */ + recurrenceKey?: string | null; + /** SYNC_DATA3; undefined leaves it. */ + pending?: PendingCreate | null; + /** SYNC_DATA5; undefined leaves it. */ + poison?: PoisonMarker | null; + /** Exceptions: ORIGINAL_SYNC_ID. */ + originalSyncId?: string | null; + /** Written as DIRTY=0 when set; DIRTY is otherwise left to the provider. */ + clearDirty?: boolean; + /** ORGANIZER is written on insert only when the event has participants (the provider fills the owner otherwise). */ + hasParticipants: boolean; +} + +function syncValues(state: RowState): WriteRow { + const values: WriteRow = {}; + values[Events._SYNC_ID] = state.syncId; + values[Events.UID_2445] = state.uid; + if (state.shadow !== undefined) values[Events.SYNC_DATA1] = state.shadow ? JSON.stringify(state.shadow) : null; + if (state.recurrenceKey !== undefined) values[Events.SYNC_DATA2] = state.recurrenceKey; + if (state.pending !== undefined) values[Events.SYNC_DATA3] = state.pending ? JSON.stringify(state.pending) : null; + values[Events.SYNC_DATA4] = encodeBaseline(state.baseline); + if (state.poison !== undefined) values[Events.SYNC_DATA5] = state.poison ? JSON.stringify(state.poison) : null; + if (state.originalSyncId !== undefined) values[Events.ORIGINAL_SYNC_ID] = state.originalSyncId; + return values; +} + +/** + * Inserts a row and its attendees and reminders. `originalId` links an + * exception to its master: an id, or the index of the master's insert in + * this group. + */ +export function insertRow(group: GroupBuilder, state: RowState, originalId?: { id: number } | { ref: number }): number { + // DIRTY is NULL after a sync-adapter insert unless written, and later asserts expect 0. + const values: WriteRow = { ...syncValues(state), [Events.HAS_ATTENDEE_DATA]: 1, [Events.DIRTY]: 0 }; + for (const [c, v] of Object.entries(state.cells)) { + if (c === Events.ORGANIZER && !state.hasParticipants) continue; + values[c] = v; + } + const refs: Record = {}; + if (originalId && 'id' in originalId) values[Events.ORIGINAL_ID] = originalId.id; + if (originalId && 'ref' in originalId) refs[Events.ORIGINAL_ID] = originalId.ref; + const index = group.write({ op: 'insert', table: 'events', values, ...(Object.keys(refs).length ? { refs } : {}) }); + for (const a of state.attendees) group.write({ op: 'insert', table: 'attendees', values: { ...a }, refs: { [Attendees.EVENT_ID]: index } }); + for (const r of state.reminders ?? []) group.write({ op: 'insert', table: 'reminders', values: { ...r }, refs: { [Reminders.EVENT_ID]: index } }); + return index; +} + +export function deleteRow(group: GroupBuilder, eventId: number): void { + group.write({ op: 'delete', table: 'events', id: eventId, expectCount: 1 }); +} + +const sameJson = (a: unknown, b: unknown) => deepEqual(a ?? null, b ?? null); + +/** + * Updates an existing row to `state`, writing only cells and sync columns + * that differ, then its attendees (matched by address) and reminders + * (matched by minutes and method). Returns the number of write ops. + */ +export function updateRow(group: GroupBuilder, current: LocalEventRow & { shadow?: CalendarEventWire | null }, state: RowState): number { + const before = group.writeCount; + const values: WriteRow = {}; + for (const [c, v] of Object.entries(state.cells)) { + const have = current.cells[c] ?? null; + const want = v ?? null; + // Same meaning (`''` and NULL, `P3600S` and `PT1H`) is no reason to write. + if (have === want || sameCell(c, have, want)) continue; + values[c] = want; + } + // Timing columns go together: the provider keeps DTEND and DURATION apart by what is written. + const timing = [Events.DTSTART, Events.DTEND, Events.DURATION, Events.EVENT_TIMEZONE, Events.ALL_DAY, Events.RRULE]; + if (timing.some((c) => c in values)) for (const c of timing) if (c in state.cells) values[c] = state.cells[c]; + if ((current.syncId ?? null) !== state.syncId) values[Events._SYNC_ID] = state.syncId; + if ((current.cells[Events.UID_2445] ?? null) !== state.uid) values[Events.UID_2445] = state.uid; + if (state.shadow !== undefined && !sameJson(current.shadow, state.shadow)) { + values[Events.SYNC_DATA1] = state.shadow ? JSON.stringify(state.shadow) : null; + } + if (state.recurrenceKey !== undefined && (current.cells[Events.SYNC_DATA2] ?? null) !== state.recurrenceKey) { + values[Events.SYNC_DATA2] = state.recurrenceKey; + } + if (state.pending !== undefined && !sameJson(current.pending, state.pending)) { + values[Events.SYNC_DATA3] = state.pending ? JSON.stringify(state.pending) : null; + } + if (!sameJson(current.baseline, state.baseline)) values[Events.SYNC_DATA4] = encodeBaseline(state.baseline); + if (state.poison !== undefined && !sameJson(current.poison, state.poison)) { + values[Events.SYNC_DATA5] = state.poison ? JSON.stringify(state.poison) : null; + } + if (state.originalSyncId !== undefined && (current.cells[Events.ORIGINAL_SYNC_ID] ?? null) !== state.originalSyncId) { + values[Events.ORIGINAL_SYNC_ID] = state.originalSyncId; + } + if (state.clearDirty && current.dirty) values[Events.DIRTY] = 0; + if (Object.keys(values).length) group.write({ op: 'update', table: 'events', id: current.eventId, values, expectCount: 1 }); + + writeAttendees(group, current, state.attendees); + if (state.reminders !== null) writeReminders(group, current, state.reminders); + return group.writeCount - before; +} + +const emailKey = (row: Row) => String(row[Attendees.ATTENDEE_EMAIL] ?? '').trim().toLowerCase(); + +function writeAttendees(group: GroupBuilder, current: LocalEventRow, wanted: Row[]): void { + const unmatched = [...current.attendees]; + const inserts: Row[] = []; + const updates: Array<{ id: number; values: Row }> = []; + for (const want of wanted) { + const i = unmatched.findIndex((a) => emailKey(a.cells) === emailKey(want)); + if (i < 0) { + inserts.push(want); + continue; + } + const [have] = unmatched.splice(i, 1); + const values: Row = {}; + for (const c of ATTENDEE_CELLS) { + const v = want[c] ?? null; + const h = have.cells[c] ?? null; + if (h === v) continue; + // The email's case matters: Android finds "me" by comparing it with OWNER_ACCOUNT exactly. + if (c !== Attendees.ATTENDEE_EMAIL && sameCell(c, h, v)) continue; + values[c] = v; + } + if (Object.keys(values).length) updates.push({ id: have.id, values }); + } + for (const gone of unmatched) group.write({ op: 'delete', table: 'attendees', id: gone.id, expectCount: 1 }); + for (const u of updates) group.write({ op: 'update', table: 'attendees', id: u.id, values: u.values, expectCount: 1 }); + for (const add of inserts) group.write({ op: 'insert', table: 'attendees', values: { ...add, [Attendees.EVENT_ID]: current.eventId } }); +} + +function writeReminders(group: GroupBuilder, current: LocalEventRow, wanted: Row[]): void { + const key = (row: Row) => { + const r = rowReminder(row); + return r ? `${r.minutes}/${r.method}` : `raw:${String(row[Reminders.MINUTES])}/${String(row[Reminders.METHOD])}`; + }; + const unmatched = [...current.reminders]; + const inserts: Row[] = []; + for (const want of wanted) { + const i = unmatched.findIndex((r) => key(r.cells) === key(want)); + if (i < 0) inserts.push(want); + else unmatched.splice(i, 1); + } + for (const gone of unmatched) group.write({ op: 'delete', table: 'reminders', id: gone.id, expectCount: 1 }); + for (const add of inserts) group.write({ op: 'insert', table: 'reminders', values: { ...add, [Reminders.EVENT_ID]: current.eventId } }); +} + +/** Counts the provider writes of a group (asserts are not writes). */ +export function writesOf(group: OpGroup): number { + return group.ops.filter((op) => op.op === 'insert' || op.op === 'update' || op.op === 'delete').length; +} diff --git a/src/device-sync/calendar/rrule.ts b/src/device-sync/calendar/rrule.ts new file mode 100644 index 00000000..66c5973c --- /dev/null +++ b/src/device-sync/calendar/rrule.ts @@ -0,0 +1,245 @@ +/** + * JSCalendar `recurrenceRule` ↔ the RRULE column (docs/device-sync.md, + * "Recurrence"). + * + * AOSP's EventRecurrence rejects every part it does not know except `X-…`, + * so RSCALE and SKIP are never written (the #805 failure); a rule that needs + * them (a non-Gregorian `rscale`, a `skip` other than `omit`, a leap month + * such as `5L`) is written without them and reported as not representable. + * UNTIL is UTC (`…Z`) for timed events and a DATE for all-day ones, because + * AOSP reads a floating UNTIL as UTC anyway. Rules written by device apps + * are read tolerantly: any case, an `RRULE:` prefix, `X-` parts, UNTIL as a + * DATE, a floating or a UTC date-time (Etar writes UTC date-times even for + * all-day events), COUNT together with UNTIL. + */ +import type { RecurrenceRule } from '../../api/types'; +import { + canonicalZone, + dateToUtcMidnight, + formatDate, + localToUtc, + parseLocalDateTime, + utcToLocal, + wallFromMs, + wallMs, + type WallClock, +} from './zoned-time'; + +export interface RuleZone { + allDay: boolean; + /** The event's zone for timed events (the device zone for floating ones). */ + zone: string; +} + +const FREQUENCIES = ['yearly', 'monthly', 'weekly', 'daily', 'hourly', 'minutely', 'secondly']; +const WEEKDAYS = ['mo', 'tu', 'we', 'th', 'fr', 'sa', 'su']; + +/** Parts EventRecurrence knows, in the order they are written. */ +const PART_ORDER = [ + 'FREQ', 'UNTIL', 'COUNT', 'INTERVAL', 'BYSECOND', 'BYMINUTE', 'BYHOUR', 'BYDAY', 'BYMONTHDAY', + 'BYYEARDAY', 'BYWEEKNO', 'BYMONTH', 'BYSETPOS', 'WKST', +] as const; +const KNOWN_PARTS = new Set(PART_ORDER); + +/** Why a rule can only be shown approximately; empty when it is representable. */ +export function unrepresentableParts(rule: RecurrenceRule | null | undefined): string[] { + if (!rule) return []; + const out: string[] = []; + if (rule.rscale && rule.rscale.toLowerCase() !== 'gregorian') out.push('rscale'); + if (rule.skip && rule.skip.toLowerCase() !== 'omit') out.push('skip'); + if ((rule.byMonth ?? []).some((m) => /l$/i.test(String(m)))) out.push('byMonth'); + if (!FREQUENCIES.includes(String(rule.frequency ?? '').toLowerCase())) out.push('frequency'); + return out; +} + +export function isRuleRepresentable(rule: RecurrenceRule | null | undefined): boolean { + return unrepresentableParts(rule).length === 0; +} + +const ints = (list: unknown): number[] => + Array.isArray(list) ? list.filter((n): n is number => Number.isInteger(n)) : []; + +/** + * The RRULE column for a rule, or null when it cannot be written at all (no + * known frequency). Unrepresentable parts are left out (see the header). + */ +export function ruleToRRule(rule: RecurrenceRule | null | undefined, ctx: RuleZone): string | null { + if (!rule) return null; + const freq = String(rule.frequency ?? '').toLowerCase(); + if (!FREQUENCIES.includes(freq)) return null; + const parts: Array<[string, string]> = [['FREQ', freq.toUpperCase()]]; + if (Number.isInteger(rule.count) && (rule.count as number) > 0) { + parts.push(['COUNT', String(rule.count)]); + } else if (rule.until) { + const until = formatUntil(rule.until, ctx); + if (until) parts.push(['UNTIL', until]); + } + if (Number.isInteger(rule.interval) && (rule.interval as number) > 1) parts.push(['INTERVAL', String(rule.interval)]); + const list = (name: string, values: Array) => { + if (values.length) parts.push([name, values.join(',')]); + }; + list('BYSECOND', ints(rule.bySecond)); + list('BYMINUTE', ints(rule.byMinute)); + list('BYHOUR', ints(rule.byHour)); + list( + 'BYDAY', + (rule.byDay ?? []) + .filter((d) => d && WEEKDAYS.includes(String(d.day).toLowerCase())) + .map((d) => `${Number.isInteger(d.nthOfPeriod) && d.nthOfPeriod !== 0 ? d.nthOfPeriod : ''}${String(d.day).toUpperCase()}`), + ); + list('BYMONTHDAY', ints(rule.byMonthDay)); + list('BYYEARDAY', ints(rule.byYearDay)); + list('BYWEEKNO', ints(rule.byWeekNo)); + list('BYMONTH', (rule.byMonth ?? []).map(String).filter((m) => /^\d{1,2}$/.test(m)).map(Number)); + list('BYSETPOS', ints(rule.bySetPosition)); + const wkst = String(rule.firstDayOfWeek ?? '').toLowerCase(); + if (WEEKDAYS.includes(wkst) && wkst !== 'mo') parts.push(['WKST', wkst.toUpperCase()]); + return parts.map(([k, v]) => `${k}=${v}`).join(';'); +} + +const compact = (w: WallClock, withTime: boolean) => { + const date = formatDate(w).replace(/-/g, ''); + if (!withTime) return date; + const p = (n: number) => String(n).padStart(2, '0'); + return `${date}T${p(w.hour)}${p(w.minute)}${p(w.second)}`; +}; + +/** UNTIL: the date for all-day rules, else the instant of the LocalDateTime in the event's zone, in UTC. */ +export function formatUntil(until: string, ctx: RuleZone): string | null { + const wall = parseLocalDateTime(until); + if (!wall) return null; + if (ctx.allDay) return compact(wall, false); + const utc = localToUtc(until, ctx.zone); + return utc === null ? null : `${compact(wallFromMs(utc), true)}Z`; +} + +/** + * The instant an UNTIL value stands for, the way AOSP's RecurrenceProcessor + * reads it: a DATE is midnight UTC, a floating date-time is UTC as well. + */ +export function untilInstant(value: string): number | null { + const m = /^(\d{4})(\d{2})(\d{2})(?:T(\d{2})(\d{2})(\d{2})Z?)?$/i.exec(value.trim()); + if (!m) return null; + return wallMs({ + year: Number(m[1]), + month: Number(m[2]), + day: Number(m[3]), + hour: Number(m[4] ?? 0), + minute: Number(m[5] ?? 0), + second: Number(m[6] ?? 0), + }); +} + +/** UNTIL as JSCalendar's LocalDateTime: the UTC date for all-day rules, the instant in the event's zone otherwise. */ +export function parseUntil(value: string, ctx: RuleZone): string | null { + const utc = untilInstant(value); + if (utc === null) return null; + if (ctx.allDay) return `${formatDate(wallFromMs(utc))}T00:00:00`; + return utcToLocal(utc, canonicalZone(ctx.zone) ?? 'UTC'); +} + +/** + * The parts of an RRULE column as a map (upper-case names and values), from + * its first rule, or null when there is no rule. `X-` parts are dropped and + * so are names EventRecurrence would reject. + */ +export function rruleParts(text: unknown): Map | null { + if (typeof text !== 'string') return null; + const first = text.split(/\r?\n/).map((l) => l.trim()).find(Boolean); + if (!first) return null; + const parts = new Map(); + for (const raw of first.replace(/^RRULE:/i, '').split(';')) { + const eq = raw.indexOf('='); + if (eq <= 0) continue; + const name = raw.slice(0, eq).trim().toUpperCase(); + if (name.startsWith('X-') || !KNOWN_PARTS.has(name) || parts.has(name)) continue; + parts.set(name, raw.slice(eq + 1).trim().toUpperCase()); + } + return parts.has('FREQ') ? parts : null; +} + +/** + * A comparable form of an RRULE column: parts in a fixed order, list values + * sorted, defaults (INTERVAL=1, WKST=MO) dropped. Two texts with the same + * meaning compare equal, so an app that rewrites a rule unchanged (Fossify + * rebuilds it on every save) is not an edit. Empty for no rule. + */ +export function canonicalRRule(text: unknown): string { + const parts = rruleParts(text); + if (!parts) return ''; + if (parts.get('INTERVAL') === '1') parts.delete('INTERVAL'); + if (parts.get('WKST') === 'MO') parts.delete('WKST'); + return PART_ORDER.filter((p) => parts.has(p)) + .map((p) => { + const v = parts.get(p)!; + return `${p}=${p.startsWith('BY') ? v.split(',').sort().join(',') : v}`; + }) + .join(';'); +} + +const numberList = (v: string | undefined) => + v === undefined ? [] : v.split(',').map((n) => Number(n)).filter((n) => Number.isInteger(n)); + +/** + * A rule from an RRULE column, or null when there is none or its FREQ is + * unknown. A COUNT wins over an UNTIL next to it (RFC 5545 forbids both; + * AOSP only warns). + */ +export function rruleToRule(text: unknown, ctx: RuleZone): RecurrenceRule | null { + const parts = rruleParts(text); + if (!parts) return null; + return partsToRule(parts, ctx); +} + +export function partsToRule(parts: Map, ctx: RuleZone): RecurrenceRule | null { + const frequency = (parts.get('FREQ') ?? '').toLowerCase(); + if (!FREQUENCIES.includes(frequency)) return null; + const rule: RecurrenceRule = { frequency }; + const interval = Number(parts.get('INTERVAL')); + if (Number.isInteger(interval) && interval > 1) rule.interval = interval; + const count = Number(parts.get('COUNT')); + if (parts.has('COUNT') && Number.isInteger(count) && count > 0) { + rule.count = count; + } else if (parts.has('UNTIL')) { + const until = parseUntil(parts.get('UNTIL')!, ctx); + if (until) rule.until = until; + } + const byDay = (parts.get('BYDAY') ?? '') + .split(',') + .map((d) => /^([+-]?\d{1,2})?(MO|TU|WE|TH|FR|SA|SU)$/.exec(d.trim())) + .filter((m): m is RegExpExecArray => !!m) + .map((m) => (m[1] && Number(m[1]) !== 0 ? { day: m[2].toLowerCase(), nthOfPeriod: Number(m[1]) } : { day: m[2].toLowerCase() })); + if (byDay.length) rule.byDay = byDay; + const lists: Array<[keyof RecurrenceRule, string]> = [ + ['byMonthDay', 'BYMONTHDAY'], + ['byYearDay', 'BYYEARDAY'], + ['byWeekNo', 'BYWEEKNO'], + ['byHour', 'BYHOUR'], + ['byMinute', 'BYMINUTE'], + ['bySecond', 'BYSECOND'], + ['bySetPosition', 'BYSETPOS'], + ]; + for (const [key, part] of lists) { + const values = numberList(parts.get(part)); + if (values.length) (rule as unknown as Record)[key] = values; + } + const byMonth = numberList(parts.get('BYMONTH')).filter((m) => m >= 1 && m <= 12).map(String); + if (byMonth.length) rule.byMonth = byMonth; + const wkst = (parts.get('WKST') ?? '').toLowerCase(); + if (WEEKDAYS.includes(wkst) && wkst !== 'mo') rule.firstDayOfWeek = wkst; + return rule; +} + +/** + * Where a rule stops, for pruning overrides after a "this and following" + * cut: the UNTIL instant, or null when the rule is unbounded or bounded by + * COUNT (the caller then asks CalendarProvider's LAST_DATE). + */ +export function untilUtc(rule: RecurrenceRule | null | undefined, ctx: RuleZone): number | null { + if (!rule?.until || rule.count) return null; + if (ctx.allDay) { + const midnight = dateToUtcMidnight(rule.until); + return midnight === null ? null : midnight + 86_399_000; + } + return localToUtc(rule.until, ctx.zone); +} diff --git a/src/device-sync/calendar/scheduling.ts b/src/device-sync/calendar/scheduling.ts new file mode 100644 index 00000000..f311ef50 --- /dev/null +++ b/src/device-sync/calendar/scheduling.ts @@ -0,0 +1,50 @@ +/** + * When a device change asks Stalwart to send iTIP messages + * (docs/device-sync.md, "Scheduling messages"). Stalwart schedules only when + * a `/set` says so, while CalDAV clients get implicit scheduling on every + * change; device sync follows CalDAV for the changes a calendar app would + * notify about: + * + * - the organizer, with others taking part: a change to the title, + * description, timing, rule, overrides, location, status or attendees, a + * create, and a destroy (CANCEL); reminders, colour, availability and + * privacy alone send nothing; + * - an attendee: their own RSVP, and a destroy (a declining REPLY); their + * other edits are stored for them only. + * + * Events in the past need no rule here: Stalwart sends nothing for them. + */ +import type { Participant } from '../../api/types'; +import { hasOtherParticipants, userIsOrganizer, type SelfContext } from './attendees'; + +/** Master units whose change the organizer tells the attendees about. */ +export const NOTIFYING_UNITS = ['title', 'description', 'timing', 'rule', 'location', 'status'] as const; + +type Scheduled = { organizerCalendarAddress?: string | null; participants?: Record | null }; + +export type SchedulingChange = + | { + kind: 'update'; + /** A notifying unit, an override or an attendee changed. */ + notifying: boolean; + /** The user's own participation status changed. */ + rsvp: boolean; + /** Attendees were added (the event may have had no one else before). */ + addsAttendees: boolean; + } + | { kind: 'create' } + | { kind: 'destroy' }; + +/** Whether the write carries `sendSchedulingMessages: true`, for `event` as it is before (or, for a create, as it is sent). */ +export function sendsSchedulingMessages(event: Scheduled, change: SchedulingChange, ctx: SelfContext): boolean { + const others = hasOtherParticipants(event.participants, ctx); + switch (change.kind) { + case 'destroy': + return others; + case 'create': + return others && userIsOrganizer(event, ctx); + case 'update': + if (userIsOrganizer(event, ctx)) return (others || change.addsAttendees) && change.notifying; + return change.rsvp; + } +} diff --git a/src/device-sync/calendar/timing.ts b/src/device-sync/calendar/timing.ts new file mode 100644 index 00000000..502104a4 --- /dev/null +++ b/src/device-sync/calendar/timing.ts @@ -0,0 +1,191 @@ +/** + * An event's timing ↔ DTSTART, DTEND, DURATION, EVENT_TIMEZONE and ALL_DAY + * (docs/device-sync.md, "Timing" and "DST edges"). + * + * - Timed events are written at the instant of `start` in `timeZone`; + * floating events (no zone) in the device zone, which the row's + * EVENT_TIMEZONE then records. + * - All-day events are UTC midnight of the date with EVENT_TIMEZONE `UTC`, + * the only form CalendarProvider accepts. + * - Single events get DTEND (days nominal in the zone), recurring masters a + * DURATION (`PD` all-day, `PS` timed), exceptions DTEND. + * + * Uploads go the other way only for a changed timing: the start is read back + * in the event's zone (floating stays floating, all-day stays a date), and a + * start whose wall time repeats in the zone is flagged, because Stalwart + * drops such a start. + */ +import { Events } from '../android-columns'; +import type { Row } from '../types'; +import { + allDayDays, + addDuration, + durationBetween, + durationSeconds, + formatAospDuration, + formatJsDuration, + parseDuration, + type Duration, +} from './duration'; +import { + DAY_MS, + canonicalZone, + classifyLocal, + dateToUtcMidnight, + fixedOffsetZone, + localToUtc, + offsetAt, + utcMidnightToDate, + utcToLocal, +} from './zoned-time'; + +export interface TimingSource { + start?: string | null; + duration?: string | null; + timeZone?: string | null; + showWithoutTime?: boolean | null; +} + +export interface EventZone { + /** The zone timed values are computed in; `UTC` for all-day events. */ + zone: string; + allDay: boolean; + /** No usable `timeZone`: written in the device zone. */ + floating: boolean; +} + +export function eventZone(event: TimingSource, deviceZone: string): EventZone { + if (event.showWithoutTime) return { zone: 'UTC', allDay: true, floating: false }; + const zone = canonicalZone(event.timeZone); + if (zone) return { zone, allDay: false, floating: false }; + return { zone: canonicalZone(deviceZone) ?? 'UTC', allDay: false, floating: true }; +} + +/** The timing cells of an event (or of one instance, `recurring` false). Null when `start` is unreadable. */ +export function timingCells(event: TimingSource, recurring: boolean, deviceZone: string): Row | null { + const { zone, allDay } = eventZone(event, deviceZone); + const duration = parseDuration(event.duration ?? 'PT0S') ?? { negative: false, days: 0, seconds: 0 }; + if (allDay) { + const dtstart = dateToUtcMidnight(event.start ?? ''); + if (dtstart === null) return null; + const days = allDayDays(duration); + return { + [Events.DTSTART]: dtstart, + [Events.DTEND]: recurring ? null : dtstart + days * DAY_MS, + [Events.DURATION]: recurring ? `P${days}D` : null, + [Events.EVENT_TIMEZONE]: 'UTC', + [Events.ALL_DAY]: 1, + }; + } + const dtstart = localToUtc(event.start ?? '', zone); + if (dtstart === null) return null; + const nonNegative: Duration = duration.negative ? { negative: false, days: 0, seconds: 0 } : duration; + return { + [Events.DTSTART]: dtstart, + [Events.DTEND]: recurring ? null : addDuration(dtstart, nonNegative, zone), + [Events.DURATION]: recurring ? formatAospDuration(durationSeconds(nonNegative), false) : null, + [Events.EVENT_TIMEZONE]: zone, + [Events.ALL_DAY]: 0, + }; +} + +/** The timing a row holds, as JSCalendar properties. */ +export interface RowTiming { + start: string; + duration: string; + /** The zone the start is in; null for floating and for all-day events without one. */ + timeZone: string | null; + showWithoutTime: boolean; + /** Whether `start` names a repeated wall time in `timeZone` (Stalwart drops those). */ + ambiguous: boolean; + /** The instant of the start, for fixed-offset fallbacks. */ + dtstart: number; +} + +export interface RowTimingContext { + /** The event's timing on the server (the shadow), for what an edit left alone. */ + server: TimingSource | null; + /** EVENT_TIMEZONE as last written by device sync (the baseline), to tell a zone change from none. */ + baselineZone: string | null; + recurring: boolean; + deviceZone: string; +} + +const truthy = (v: unknown) => Number(v ?? 0) === 1; + +/** + * JSCalendar timing from a row: all-day rows give a date and whole days; + * timed rows give the start in the event's zone and the duration with whole + * days nominal. A floating event stays floating while its row keeps the + * zone it was written in; a zone the user picked (or an all-day event made + * timed) becomes the event's `timeZone`. An EVENT_TIMEZONE Intl doesn't know + * is read as Android reads it, as GMT. + */ +export function rowTiming(cells: Row, ctx: RowTimingContext): RowTiming | null { + const dtstart = Number(cells[Events.DTSTART]); + if (cells[Events.DTSTART] === null || cells[Events.DTSTART] === undefined || !Number.isFinite(dtstart)) return null; + const allDay = truthy(cells[Events.ALL_DAY]); + const lengthOf = (zone: string): Duration => { + if (ctx.recurring || cells[Events.DTEND] === null || cells[Events.DTEND] === undefined) { + const d = parseDuration(cells[Events.DURATION]); + if (!d || d.negative) return { negative: false, days: 0, seconds: 0 }; + const s = durationSeconds(d); + return { negative: false, days: Math.floor(s / 86_400), seconds: s % 86_400 }; + } + return durationBetween(dtstart, Number(cells[Events.DTEND]), zone); + }; + if (allDay) { + const d = lengthOf('UTC'); + const days = allDayDays(d); + const keepZone = ctx.server?.showWithoutTime ? (ctx.server.timeZone ?? null) : null; + return { + start: utcMidnightToDate(dtstart), + duration: formatJsDuration(days, 0), + timeZone: keepZone, + showWithoutTime: true, + ambiguous: false, + dtstart, + }; + } + const rowZoneRaw = String(cells[Events.EVENT_TIMEZONE] ?? ''); + const rowZone = canonicalZone(rowZoneRaw); + const zoneUnchanged = ctx.baselineZone !== null && rowZoneRaw === ctx.baselineZone; + const server = ctx.server; + const serverTimed = !!server && !server.showWithoutTime; + let timeZone: string | null; + let computeIn: string; + if (serverTimed && zoneUnchanged && !canonicalZone(server!.timeZone)) { + // Floating on the server (or a zone Intl doesn't know): shown in the zone it was written in. + timeZone = server!.timeZone ?? null; + computeIn = rowZone ?? 'UTC'; + } else if (serverTimed && zoneUnchanged && server!.timeZone) { + timeZone = server!.timeZone; + computeIn = canonicalZone(server!.timeZone)!; + } else if (rowZone) { + timeZone = rowZone; + computeIn = rowZone; + } else { + timeZone = 'Etc/UTC'; + computeIn = 'UTC'; + } + const start = utcToLocal(dtstart, computeIn); + const d = lengthOf(computeIn); + return { + start, + duration: formatJsDuration(d.days, d.seconds), + timeZone, + showWithoutTime: false, + ambiguous: timeZone !== null && classifyLocal(start, computeIn) === 'overlap', + dtstart, + }; +} + +/** + * The same instant and wall time in a fixed-offset zone, for a start that is + * a repeated wall time in its own zone (non-recurring events only). + */ +export function fixedOffsetTiming(timing: RowTiming): RowTiming { + const zone = fixedOffsetZone(offsetAt(timing.dtstart, timing.timeZone ? canonicalZone(timing.timeZone) ?? 'UTC' : 'UTC')); + const computeIn = canonicalZone(zone) ?? 'UTC'; + return { ...timing, timeZone: zone, start: utcToLocal(timing.dtstart, computeIn), ambiguous: false }; +} diff --git a/src/device-sync/calendar/units.ts b/src/device-sync/calendar/units.ts new file mode 100644 index 00000000..b397857e --- /dev/null +++ b/src/device-sync/calendar/units.ts @@ -0,0 +1,103 @@ +/** + * The units changes are detected, merged and uploaded in (docs/device-sync.md, + * "Units"): groups of related columns, each attendee (by address), the + * reminder set, and for a master each EXDATE entry. Exception rows are + * compared with the same units, recursively. + */ +import { Attendees, Events } from '../android-columns'; +import type { EventBaseline, LocalEventRow } from '../planner'; +import type { Row } from '../types'; +import { addressKey, type SelfContext } from './attendees'; +import { MASTER_CELLS, EXCEPTION_CELLS, makeBaseline, sameCell } from './columns'; +import { exdateSet } from './exdate'; +import { sameReminders } from './reminders'; +import type { RowImage } from './image'; + +export const COLUMN_UNITS: Record = { + title: [Events.TITLE], + description: [Events.DESCRIPTION], + location: [Events.EVENT_LOCATION], + timing: [Events.DTSTART, Events.DTEND, Events.DURATION, Events.EVENT_TIMEZONE, Events.ALL_DAY], + rule: [Events.RRULE], + status: [Events.STATUS], + availability: [Events.AVAILABILITY], + privacy: [Events.ACCESS_LEVEL], + color: [Events.EVENT_COLOR], + calendar: [Events.CALENDAR_ID], +}; + +export const MASTER_COLUMN_UNITS = Object.keys(COLUMN_UNITS); +/** An exception has no rule of its own and lives in its master's calendar. */ +export const EXCEPTION_COLUMN_UNITS = MASTER_COLUMN_UNITS.filter((u) => u !== 'rule' && u !== 'calendar'); + +/** Apps that rewrite what they don't model (docs/device-sync.md, "Lossy calendar editors"). */ +const LOSSY_EDITORS = ['org.fossify.calendar', 'com.simplemobiletools.calendar.pro', 'com.simplemobiletools.calendar']; + +/** Whether MUTATORS names an editor whose attendee types and rule parts can't be trusted. */ +export function isLossyEditor(mutators: string | null | undefined): boolean { + if (!mutators) return false; + return mutators.split(',').some((m) => LOSSY_EDITORS.includes(m.trim())); +} + +/** One side of a comparison: a row as read, a baseline, or an image. */ +export interface Side { + cells: Row; + attendees: Row[]; + /** Null: not managed (Bulwark owns reminders). */ + reminders: Row[] | null; +} + +export function sideOfRow(row: LocalEventRow): Side { + return { cells: row.cells, attendees: row.attendees.map((a) => a.cells), reminders: row.reminders.map((r) => r.cells) }; +} + +export function sideOfBaseline(b: EventBaseline): Side { + return { cells: b.cells, attendees: b.attendees, reminders: b.reminders }; +} + +export function sideOfImage(image: RowImage): Side { + return { cells: image.cells, attendees: image.attendees, reminders: image.reminders }; +} + +export function baselineOfImage(image: RowImage, isException: boolean): EventBaseline { + return makeBaseline( + image.cells, + isException ? EXCEPTION_CELLS : MASTER_CELLS, + image.attendees, + image.reminders ?? [], + image.truncatedDescription, + ); +} + +export function columnUnitDiffers(unit: string, a: Side, b: Side): boolean { + return (COLUMN_UNITS[unit] ?? []).some((c) => !sameCell(c, a.cells[c], b.cells[c])); +} + +/** Attendee rows by address key (the user's aliases are one key). */ +export function attendeesByKey(rows: Row[], ctx: SelfContext): Map { + const out = new Map(); + for (const row of rows) { + const key = addressKey(row[Attendees.ATTENDEE_EMAIL], ctx); + if (key && !out.has(key)) out.set(key, row); + } + return out; +} + +const ATTENDEE_COMPARED = [Attendees.ATTENDEE_NAME, Attendees.ATTENDEE_RELATIONSHIP, Attendees.ATTENDEE_TYPE, Attendees.ATTENDEE_STATUS]; +/** What a lossy editor rewrites on every save and so never counts as an edit. */ +const LOSSY_IGNORED = new Set([Attendees.ATTENDEE_RELATIONSHIP, Attendees.ATTENDEE_TYPE]); + +export function attendeeDiffers(a: Row | undefined, b: Row | undefined, lossy = false): boolean { + if (!a || !b) return !!a !== !!b; + return ATTENDEE_COMPARED.some((c) => !(lossy && LOSSY_IGNORED.has(c)) && !sameCell(c, a[c], b[c])); +} + +export function remindersDiffer(a: Side, b: Side): boolean { + if (a.reminders === null || b.reminders === null) return false; + return !sameReminders(a.reminders, b.reminders); +} + +/** EXDATE entries of a side, normalised (see `exdateSet`). */ +export function exdateEntries(side: Side): Set { + return exdateSet(side.cells[Events.EXDATE], Number(side.cells[Events.ALL_DAY] ?? 0) === 1); +} diff --git a/src/device-sync/calendar/upload.ts b/src/device-sync/calendar/upload.ts new file mode 100644 index 00000000..d312f3b8 --- /dev/null +++ b/src/device-sync/calendar/upload.ts @@ -0,0 +1,597 @@ +/** + * Device changes → JMAP writes (docs/device-sync.md, "Uploads", "Device-side + * changes" and "Scheduling messages"). + * + * A dirty event uploads a patch built per unit from what differs from its + * baseline, against the current shadow: pointers where Stalwart takes them, + * a whole map where it needs one (the first participant, alert or override). + * Exception rows upload only as overrides of their master: a new override + * carries the master's details (Stalwart stores a bare one as a separate + * event otherwise) without the properties Stalwart drops from overrides; an + * existing one is patched one level deep, never deeper, because a deeper + * pointer becomes a partial override that loses the other attendees. + * + * What Android can't represent is never sent: timing and rule of events + * whose rule needs RSCALE, SKIP or leap months; a truncated description. + * Stalwart drops local times it can't resolve, so a start in a repeated hour + * goes out in a fixed-offset zone (single events) or not at all (series: + * `dstAmbiguous`), and an UNTIL in a skipped or repeated hour moves an hour + * later. + */ +import type { RecurrenceRule } from '../../api/types'; +import { EventStatus, Events } from '../android-columns'; +import type { CalendarContext, LocalEvent, LocalEventRow, LocalException } from '../planner'; +import type { Row } from '../types'; +import type { CalendarEventWire } from '../wire'; +import { clone } from '../common/json'; +import { ptr, setInPatch, type PatchObject } from '../common/patch'; +import { withNewOverrideDetails } from '../../lib/recurrence-instances'; +import { userIsOrganizer } from './attendees'; +import { isTruncatedBaseline } from './columns'; +import { durationTextSeconds, sameDurationLength } from './duration'; +import { + applyAttendeeEdits, + attendeeEdits, + descriptionValue, + locationChanges, + reminderChanges, + titleValue, + type PathChange, +} from './edits'; +import { exdateEntryKey } from './exdate'; +import { + FORBIDDEN_OVERRIDE_KEYS, + instanceOf, + isExcluded, + keyToInstanceTime, + overridesOf, + withoutForbiddenKeys, + type Override, +} from './exceptions'; +import { eventImage, plainInstanceImage } from './image'; +import { isRemovedInstance } from './merge'; +import { isRuleRepresentable, partsToRule, ruleToRRule, rruleParts, untilUtc, type RuleZone } from './rrule'; +import { NOTIFYING_UNITS, sendsSchedulingMessages } from './scheduling'; +import { fixedOffsetTiming, rowTiming, type RowTiming } from './timing'; +import { + EXCEPTION_COLUMN_UNITS, + MASTER_COLUMN_UNITS, + baselineOfImage, + columnUnitDiffers, + exdateEntries, + isLossyEditor, + remindersDiffer, + sideOfBaseline, + sideOfRow, + type Side, +} from './units'; +import { argbToCss, availabilityFromDevice, privacyFromDevice, statusFromDevice } from './values'; +import { + canonicalZone, + classifyLocal, + formatLocalDateTime, + parseLocalDateTime, + resolveWallClock, + sameLocalDateTime, + utcToLocal, + wallFromMs, + wallMs, +} from './zoned-time'; + +/** Why an item is not uploaded now. */ +export const SKIP = { + dstAmbiguous: 'dstAmbiguous', + ruleNotRepresentable: 'ruleNotRepresentable', + instanceOnly: 'instanceOnly', + notOurCalendar: 'notOurCalendar', +} as const; + +export class SkipUpload extends Error { + constructor(readonly reason: string) { + super(reason); + } +} + +const num = (v: unknown) => (v === null || v === undefined || v === '' ? null : Number.isFinite(Number(v)) ? Number(v) : null); + +/** The zone and all-day flag the series' LocalDateTimes (start, keys, UNTIL) are in, as written to the rows. */ +export function seriesZone(local: LocalEventRow, shadow: CalendarEventWire | null, deviceZone: string): RuleZone { + const allDay = shadow ? !!shadow.showWithoutTime : Number(local.cells[Events.ALL_DAY] ?? 0) === 1; + if (allDay) return { allDay: true, zone: 'UTC' }; + const zone = + canonicalZone(shadow?.timeZone) ?? + canonicalZone(local.baseline?.cells[Events.EVENT_TIMEZONE]) ?? + canonicalZone(local.cells[Events.EVENT_TIMEZONE]) ?? + canonicalZone(deviceZone) ?? + 'UTC'; + return { allDay: false, zone }; +} + +/** A wall time Stalwart would drop (skipped or repeated) moved an hour later. */ +function safeUntil(until: string, zone: RuleZone): string { + if (zone.allDay) return until; + const kind = classifyLocal(until, zone.zone); + if (kind !== 'gap' && kind !== 'overlap') return until; + const wall = parseLocalDateTime(until)!; + return formatLocalDateTime(wallFromMs(wallMs(wall) + 3_600_000)); +} + +/** Parts Fossify Calendar's rule editor models; others it drops, and they stay. */ +const LOSSY_RULE_PARTS = ['FREQ', 'INTERVAL', 'COUNT', 'UNTIL', 'BYDAY', 'BYMONTHDAY', 'BYMONTH']; + +/** + * The rule a device RRULE stands for. A lossy editor's rule is applied part + * by part onto the server's rule: parts it models and changed are taken, + * parts it dropped stay (docs/device-sync.md, "Lossy calendar editors"). + */ +export function deviceRule( + rrule: unknown, + baselineRRule: unknown, + server: RecurrenceRule | null | undefined, + zone: RuleZone, + lossy: boolean, +): RecurrenceRule | null { + const parts = rruleParts(rrule); + if (!parts) return null; + let merged = parts; + if (lossy && server) { + const before = rruleParts(baselineRRule) ?? new Map(); + const serverParts = rruleParts(ruleText(server, zone)) ?? new Map(); + merged = new Map(serverParts); + for (const part of LOSSY_RULE_PARTS) { + const now = parts.get(part); + if (now !== undefined && now !== before.get(part)) { + merged.set(part, now); + if (part === 'COUNT') merged.delete('UNTIL'); + if (part === 'UNTIL') merged.delete('COUNT'); + } + } + } + const rule = partsToRule(merged, zone); + if (rule?.until) rule.until = safeUntil(rule.until, zone); + return rule; +} + +/** The server's rule in RRULE parts, so both sides of a lossy merge compare in one dialect. */ +function ruleText(rule: RecurrenceRule, zone: RuleZone): string { + return ruleToRRule(rule, zone) ?? ''; +} + +// ─── Patches for a dirty event ────────────────────────── + +export interface UploadComputation { + patch: PatchObject; + /** Master units in the patch. */ + masterUnits: Set; + /** Exception rows (by id) whose changes are in the patch. */ + exceptionUnits: Map>; + /** Exception rows whose instance is excluded by the patch, or already was. */ + removedExceptions: Set; + sendSchedulingMessages: boolean; + /** Units changed on the device that can't be uploaded (reported when nothing else is). */ + dropped: string[]; +} + +interface OverrideWork { + /** Whole override values (or null) by recurrence id. */ + whole: Map; + /** One-level properties of existing overrides by recurrence id. */ + props: Map>; +} + +/** Sets a raw path in a patch: escaped once, never overlapping a pointer already there. */ +export function put(patch: PatchObject, path: string[], value: unknown): void { + setInPatch(patch, ptr(...path), value); +} + +function putAll(patch: PatchObject, root: string[], changes: PathChange[]): void { + for (const c of changes) put(patch, [...root, ...c.path], c.value); +} + +/** + * The patch for a dirty event with identity (see the header). Throws + * `SkipUpload` for an item that must not be uploaded now. + */ +export function computeUpdate(local: LocalEvent, ctx: CalendarContext): UploadComputation { + const shadow = local.shadow!; + const calendar = ctx.calendarIdOfRow(local.calendarRowId); + const calendarId = calendar?.calendarId ?? Object.keys(shadow.calendarIds ?? {})[0] ?? ''; + const image = eventImage(shadow, local.calendarRowId, calendarId, ctx); + const baseline = local.baseline ?? (image ? baselineOfImage(image.master, false) : null); + const zone = seriesZone(local, shadow, ctx.deviceZone); + const lossy = isLossyEditor(local.mutators); + const patch: PatchObject = {}; + const masterUnits = new Set(); + const exceptionUnits = new Map>(); + const removedExceptions = new Set(); + const dropped: string[] = []; + const overrides: OverrideWork = { whole: new Map(), props: new Map() }; + const representable = isRuleRepresentable(shadow.recurrenceRule) && !shadow.recurrenceId; + const serverOverrides = overridesOf(shadow); + let rsvp = false; + let attendeeOthers = false; + let overridesChanged = false; + let ruleRemoved = false; + let rekey: number | null = null; + + const cur = sideOfRow(local); + const bl: Side | null = baseline ? sideOfBaseline(baseline) : null; + const changed = (unit: string) => { + if (!bl) return false; + const forced = local.split === 'source' && unit === 'rule'; + return (local.dirty || forced) && columnUnitDiffers(unit, cur, bl); + }; + + for (const unit of MASTER_COLUMN_UNITS) { + if (!changed(unit)) continue; + const cells = local.cells; + switch (unit) { + case 'title': + put(patch, ['title'], titleValue(cells)); + break; + case 'description': + if (isTruncatedBaseline(baseline!.cells[Events.DESCRIPTION])) { + dropped.push(unit); + continue; + } + put(patch, ['description'], descriptionValue(cells)); + if (String(shadow.descriptionContentType ?? '').toLowerCase().startsWith('text/html')) { + put(patch, ['descriptionContentType'], 'text/plain'); + } + break; + case 'location': + putAll(patch, [], locationChanges(shadow, cells[Events.EVENT_LOCATION], ctx.mintKey).changes); + break; + case 'status': + put(patch, ['status'], statusFromDevice(cells[Events.STATUS])); + break; + case 'availability': + put(patch, ['freeBusyStatus'], availabilityFromDevice(cells[Events.AVAILABILITY])); + break; + case 'privacy': + put(patch, ['privacy'], privacyFromDevice(cells[Events.ACCESS_LEVEL])); + break; + case 'color': + put(patch, ['color'], argbToCss(cells[Events.EVENT_COLOR])); + break; + case 'calendar': { + const to = ctx.calendarIdOfRow(Number(cells[Events.CALENDAR_ID])); + const from = ctx.calendarIdOfRow(Number(baseline!.cells[Events.CALENDAR_ID])); + if (!to || !from || to.jmapAccountId !== ctx.jmapAccountId || from.jmapAccountId !== ctx.jmapAccountId) { + dropped.push(unit); + continue; + } + put(patch, ['calendarIds', from.calendarId], null); + put(patch, ['calendarIds', to.calendarId], true); + break; + } + case 'timing': { + if (!representable) { + dropped.push(unit); + continue; + } + const recurring = !!cells[Events.RRULE]; + let t = rowTiming(cells, { + server: shadow, + baselineZone: baseline!.cells[Events.EVENT_TIMEZONE] === null ? null : String(baseline!.cells[Events.EVENT_TIMEZONE] ?? ''), + recurring, + deviceZone: ctx.deviceZone, + }); + if (!t) { + dropped.push(unit); + continue; + } + if (t.ambiguous) { + if (recurring) throw new SkipUpload(SKIP.dstAmbiguous); + t = fixedOffsetTiming(t); + } + applyTiming(patch, [], t, shadow); + if (recurring && shadow.recurrenceRule && !sameLocalDateTime(t.start, shadow.start)) { + rekey = wallMs(parseLocalDateTime(t.start)!) - wallMs(parseLocalDateTime(shadow.start)!); + } + break; + } + case 'rule': { + if (!representable) { + dropped.push(unit); + continue; + } + const ruleZone: RuleZone = { allDay: Number(cells[Events.ALL_DAY] ?? 0) === 1, zone: zone.zone }; + const rule = deviceRule(cells[Events.RRULE], baseline!.cells[Events.RRULE], shadow.recurrenceRule, ruleZone, lossy); + if (!rule) { + put(patch, ['recurrenceRule'], null); + if (Object.keys(serverOverrides).length) put(patch, ['recurrenceOverrides'], null); + ruleRemoved = true; + break; + } + put(patch, ['recurrenceRule'], rule); + // "This and following": overrides after the new end go. + const end = seriesEnd(rule, local, ruleZone); + if (end !== null) { + for (const key of Object.keys(serverOverrides)) { + const at = keyToInstanceTime(key, ruleZone.allDay, ruleZone.zone); + if (at !== null && at > end) { + overrides.whole.set(key, null); + overridesChanged = true; + } + } + } + break; + } + } + masterUnits.add(unit); + } + + // EXDATE entries added or removed on the device. + if (bl && local.dirty && !ruleRemoved) { + const now = exdateEntries(cur); + const before = exdateEntries(bl); + for (const entry of new Set([...now, ...before])) { + if (now.has(entry) === before.has(entry)) continue; + const key = exdateEntryKey(entry, zone); + if (!key) continue; + if (!zone.allDay && classifyLocal(key, zone.zone) === 'overlap') throw new SkipUpload(SKIP.dstAmbiguous); + if (now.has(entry) && !isExcluded(serverOverrides[key])) overrides.whole.set(key, { excluded: true }); + else if (!now.has(entry) && isExcluded(serverOverrides[key])) overrides.whole.set(key, null); + else continue; + overridesChanged = true; + masterUnits.add(`exdate:${entry}`); + } + } + + // Attendees. + if (bl && local.dirty) { + const edits = attendeeEdits(cur.attendees, bl.attendees, ctx, lossy); + if (edits.length) { + const result = applyAttendeeEdits(shadow.participants, edits, ctx, ctx.mintKey, lossy); + if (result.changes.length) { + if (!Object.keys(shadow.participants ?? {}).length) put(patch, ['participants'], result.participants); + else putAll(patch, ['participants'], result.changes); + rsvp = rsvp || result.rsvp; + attendeeOthers = attendeeOthers || result.others; + for (const e of edits) masterUnits.add(`attendee:${e.key}`); + } + } + } + + // Reminders, only when the calendar app owns them. + if (bl && local.dirty && ctx.reminderOwner === 'device' && remindersDiffer(cur, bl)) { + putAll(patch, [], reminderChanges(shadow, ctx.calendar(calendarId), cur.reminders!, ctx.mintKey).changes); + masterUnits.add('reminders'); + } + + // Exception rows. + if (!ruleRemoved && shadow.recurrenceRule) { + for (const x of local.exceptions) { + const units = exceptionChanges(x, local, shadow, calendarId, zone, overrides, removedExceptions, ctx, lossy); + if (units.units.size) { + exceptionUnits.set(x.eventId, units.units); + overridesChanged = true; + rsvp = rsvp || units.rsvp; + } + } + } + + if (!ruleRemoved) materializeOverrides(patch, shadow, overrides, rekey, zone); + + const notifying = NOTIFYING_UNITS.some((u) => masterUnits.has(u)) || overridesChanged || attendeeOthers; + const sendSchedulingMessages = sendsSchedulingMessages(shadow, { kind: 'update', notifying, rsvp, addsAttendees: attendeeOthers }, ctx); + + return { patch, masterUnits, exceptionUnits, removedExceptions, sendSchedulingMessages, dropped }; +} + +/** Timing properties that differ from the server's. */ +function applyTiming(patch: PatchObject, root: string[], t: RowTiming, server: { start?: string; duration?: string; timeZone?: string | null; showWithoutTime?: boolean | null }): void { + if (!sameLocalDateTime(t.start, server.start)) put(patch, [...root, 'start'], t.start); + if (!sameDurationLength(t.duration, server.duration ?? 'PT0S')) put(patch, [...root, 'duration'], t.duration); + if (root.length === 0 && (t.timeZone ?? null) !== (server.timeZone ?? null)) put(patch, ['timeZone'], t.timeZone); + if (root.length === 0 && !!t.showWithoutTime !== !!server.showWithoutTime) put(patch, ['showWithoutTime'], t.showWithoutTime); +} + +/** + * Where the capped series ends: the UNTIL instant, or for a COUNT the start + * of the last instance as CalendarProvider computed it (LAST_DATE minus the + * duration). Null when unknown or unbounded. + */ +function seriesEnd(rule: RecurrenceRule, local: LocalEventRow, zone: RuleZone): number | null { + if (rule.until && !rule.count) return untilUtc(rule, zone); + if (!rule.count) return null; + const lastDate = num(local.cells[Events.LAST_DATE]); + const seconds = durationTextSeconds(local.cells[Events.DURATION]); + if (lastDate === null || seconds === null) return null; + return lastDate - seconds * 1000; +} + +function exceptionChanges( + x: LocalException, + master: LocalEvent, + shadow: CalendarEventWire, + calendarId: string, + zone: RuleZone, + work: OverrideWork, + removed: Set, + ctx: CalendarContext, + lossy: boolean, +): { units: Set; rsvp: boolean } { + const units = new Set(); + const key = x.recurrenceId; + if (!key) return { units, rsvp: false }; + const ours = typeof x.cells[Events.SYNC_DATA2] === 'string' && !!x.cells[Events.SYNC_DATA2]; + if (!ours && !zone.allDay && classifyLocal(key, zone.zone) === 'overlap') throw new SkipUpload(SKIP.dstAmbiguous); + const server = overridesOf(shadow)[key]; + const cancelled = Number(x.cells[Events.STATUS]) === EventStatus.CANCELED; + if (isRemovedInstance(x) || (cancelled && isExcluded(server))) { + removed.add(x.eventId); + if (!isExcluded(server)) { + work.whole.set(key, { excluded: true }); + units.add('removed'); + } + return { units, rsvp: false }; + } + if (!x.dirty) return { units, rsvp: false }; + const plain = plainInstanceImage(shadow, key, master.calendarRowId, calendarId, ctx); + if (!plain) return { units, rsvp: false }; + const existing = !!server && !isExcluded(server); + const instance = instanceOf(shadow, key, existing ? server : null); + // An existing override changes against what device sync last wrote; a new one against the plain instance. + const reference: Side = existing && x.baseline ? sideOfBaseline(x.baseline) : { cells: plain.cells, attendees: plain.attendees, reminders: plain.reminders }; + const cur = sideOfRow(x); + const values: Record = {}; + let rsvp = false; + const organizer = userIsOrganizer(shadow, ctx); + + for (const unit of EXCEPTION_COLUMN_UNITS) { + if (unit === 'privacy') continue; // Stalwart drops privacy from overrides. + // Etar answers one instance with STATUS_CONFIRMED on it; an attendee may not change an event's status. + if (unit === 'status' && !organizer) continue; + if (unit !== 'timing' && !columnUnitDiffers(unit, cur, reference)) continue; + // A new exception row holds what its app copied: an empty value there is one it did not model, not a deletion. + if (!existing && INHERITED_WHEN_EMPTY.has(unit) && isEmptyUnit(unit, x.cells)) continue; + const cells = x.cells; + switch (unit) { + case 'title': + values.title = titleValue(cells); + break; + case 'description': + if (x.baseline && isTruncatedBaseline(x.baseline.cells[Events.DESCRIPTION])) continue; + values.description = descriptionValue(cells); + break; + case 'location': + values.locations = locationChanges(instance, cells[Events.EVENT_LOCATION], ctx.mintKey).locations; + break; + case 'status': + values.status = statusFromDevice(cells[Events.STATUS]); + break; + case 'availability': + values.freeBusyStatus = availabilityFromDevice(cells[Events.AVAILABILITY]); + break; + case 'color': + values.color = argbToCss(cells[Events.EVENT_COLOR]); + break; + case 'timing': { + const moved = columnUnitDiffers('timing', cur, reference); + if (existing && !moved) continue; + let t = rowTiming(cells, { + server: { ...instance, showWithoutTime: shadow.showWithoutTime, timeZone: instance.timeZone ?? shadow.timeZone }, + baselineZone: String(reference.cells[Events.EVENT_TIMEZONE] ?? ''), + recurring: false, + deviceZone: ctx.deviceZone, + }); + if (!t) continue; + if (t.ambiguous) throw new SkipUpload(SKIP.dstAmbiguous); + if (t.showWithoutTime !== !!shadow.showWithoutTime) t = { ...t, showWithoutTime: !!shadow.showWithoutTime }; + if (existing) { + if (!sameLocalDateTime(t.start, instance.start)) values.start = t.start; + if (!sameDurationLength(t.duration, instance.duration ?? 'PT0S')) values.duration = t.duration; + if (!('start' in values) && !('duration' in values)) continue; + } else { + // A new override pins its start and length, as the app's own occurrence edits do. + values.start = t.start; + values.duration = t.duration; + if (!moved) continue; + } + break; + } + } + units.add(unit); + } + + // A new exception row without attendees or reminders did not copy them: the series' apply. + const edits = existing || cur.attendees.length ? attendeeEdits(cur.attendees, reference.attendees, ctx, lossy) : []; + if (edits.length) { + const result = applyAttendeeEdits(instance.participants, edits, ctx, ctx.mintKey, lossy); + if (result.changes.length) { + values.participants = result.participants; + rsvp = result.rsvp; + units.add('attendees'); + } + } + if (ctx.reminderOwner === 'device' && (existing || cur.reminders!.length) && remindersDiffer(cur, reference)) { + values.alerts = reminderChanges(instance, ctx.calendar(calendarId), cur.reminders!, ctx.mintKey).alerts; + units.add('reminders'); + } + + if (existing) { + if (units.size) work.props.set(key, { ...(work.props.get(key) ?? {}), ...values }); + return { units, rsvp }; + } + // A new override: nothing but its pinned time is still an override (an app saving the instance unchanged). + if (!units.size) units.add('new'); + work.whole.set(key, newOverride(shadow, key, values)); + return { units, rsvp }; +} + +/** Units a new exception row inherits from its series when it leaves them empty. */ +const INHERITED_WHEN_EMPTY = new Set(['title', 'description', 'location', 'color']); + +function isEmptyUnit(unit: string, cells: Row): boolean { + const column = { title: Events.TITLE, description: Events.DESCRIPTION, location: Events.EVENT_LOCATION, color: Events.EVENT_COLOR }[unit]; + const v = column ? cells[column] : null; + return v === null || v === undefined || v === ''; +} + +/** + * A new override: the changes plus the details Stalwart needs on it (title, + * locations, participants, sequence, …; `withNewOverrideDetails`), minus what + * Stalwart drops from overrides. The master's own alerts are not copied + * while it uses the calendar's defaults: they are not in force. + */ +export function newOverride(master: CalendarEventWire, key: string, values: Record): Override { + const occurrence = { + ...(master as unknown as Record), + recurrenceId: key, + recurrenceOverrides: master.recurrenceOverrides ?? {}, + start: undefined, + baseEventId: undefined, + }; + const withDetails = withNewOverrideDetails(occurrence as never, values as never) as Record; + if (master.useDefaultAlerts && !('alerts' in values)) delete withDetails.alerts; + const out = withoutForbiddenKeys(withDetails); + for (const name of FORBIDDEN_OVERRIDE_KEYS) delete out[name]; + return out; +} + +function materializeOverrides(patch: PatchObject, shadow: CalendarEventWire, work: OverrideWork, rekey: number | null, zone: RuleZone): void { + const server = overridesOf(shadow); + if (rekey !== null && Object.keys(server).length) { + // "All events" moved the series: every recurrence id moves by the same wall-clock offset. + const map: Record = clone(server); + for (const [key, value] of work.whole) { + if (value === null) delete map[key]; + else map[key] = value; + } + for (const [key, props] of work.props) if (map[key]) map[key] = { ...map[key], ...props }; + const shifted: Record = {}; + for (const [key, value] of Object.entries(map)) { + const moved = shiftLocal(key, rekey, zone); + const next = clone(value); + if (!isExcluded(next) && typeof next.start === 'string' && sameLocalDateTime(next.start, key)) next.start = shiftLocal(next.start, rekey, zone); + shifted[moved] = next; + } + put(patch, ['recurrenceOverrides'], Object.keys(shifted).length ? shifted : null); + return; + } + if (!Object.keys(server).length) { + // The first override: a pointer below a missing map fails, so the whole map goes. + const whole: Record = {}; + for (const [key, value] of work.whole) if (value) whole[key] = value; + if (Object.keys(whole).length) put(patch, ['recurrenceOverrides'], whole); + return; + } + for (const [key, value] of work.whole) put(patch, ['recurrenceOverrides', key], value); + for (const [key, props] of work.props) for (const [name, value] of Object.entries(props)) put(patch, ['recurrenceOverrides', key, name], value); +} + +/** A LocalDateTime moved by wall-clock milliseconds; a result in a skipped hour moves forward like the instance does. */ +function shiftLocal(value: string, deltaMs: number, zone: RuleZone): string { + const wall = parseLocalDateTime(value); + if (!wall) return value; + const moved = wallFromMs(wallMs(wall) + deltaMs); + const text = formatLocalDateTime(moved); + if (zone.allDay) return text; + const kind = classifyLocal(text, zone.zone); + if (kind === 'overlap') throw new SkipUpload(SKIP.dstAmbiguous); + if (kind === 'gap') return utcToLocal(resolveWallClock(moved, canonicalZone(zone.zone) ?? 'UTC').utc, zone.zone); + return text; +} + diff --git a/src/device-sync/calendar/values.ts b/src/device-sync/calendar/values.ts new file mode 100644 index 00000000..402d0639 --- /dev/null +++ b/src/device-sync/calendar/values.ts @@ -0,0 +1,192 @@ +/** + * Value tables between JSCalendar and CalendarContract (docs/device-sync.md, + * "Other properties"): status, availability, privacy, colours and the + * attendee enums. Pure lookups, both directions. + */ +import { + AttendeeRelationship, + AttendeeStatus, + AttendeeType, + EventAccess, + EventAvailability, + EventStatus, + ReminderMethod, +} from '../android-columns'; + +// ─── Event status, availability, privacy ───────────────── + +/** `status` → STATUS; absent (JSCalendar's default is confirmed) → CONFIRMED. Never null: the provider throws on it. */ +export function statusToDevice(status: unknown): number { + switch (String(status ?? '').toLowerCase()) { + case 'tentative': + return EventStatus.TENTATIVE; + case 'cancelled': + return EventStatus.CANCELED; + default: + return EventStatus.CONFIRMED; + } +} + +export function statusFromDevice(value: unknown): 'confirmed' | 'tentative' | 'cancelled' { + const n = Number(value); + if (n === EventStatus.TENTATIVE) return 'tentative'; + if (n === EventStatus.CANCELED) return 'cancelled'; + return 'confirmed'; +} + +/** `freeBusyStatus` → AVAILABILITY; JSCalendar's default is busy. */ +export function availabilityToDevice(freeBusy: unknown): number { + return String(freeBusy ?? '').toLowerCase() === 'free' ? EventAvailability.FREE : EventAvailability.BUSY; +} + +/** AVAILABILITY → `freeBusyStatus`; TENTATIVE has no JSCalendar value and uploads as busy. */ +export function availabilityFromDevice(value: unknown): 'free' | 'busy' { + return Number(value) === EventAvailability.FREE ? 'free' : 'busy'; +} + +/** `privacy` → ACCESS_LEVEL: absent DEFAULT, public PUBLIC, private PRIVATE, secret CONFIDENTIAL. */ +export function privacyToDevice(privacy: unknown): number { + switch (String(privacy ?? '').toLowerCase()) { + case 'public': + return EventAccess.PUBLIC; + case 'private': + return EventAccess.PRIVATE; + case 'secret': + return EventAccess.CONFIDENTIAL; + default: + return EventAccess.DEFAULT; + } +} + +/** ACCESS_LEVEL → `privacy`; DEFAULT removes the property. */ +export function privacyFromDevice(value: unknown): 'public' | 'private' | 'secret' | null { + switch (Number(value)) { + case EventAccess.PUBLIC: + return 'public'; + case EventAccess.PRIVATE: + return 'private'; + case EventAccess.CONFIDENTIAL: + return 'secret'; + default: + return null; + } +} + +// ─── Attendees ────────────────────────────────────────── + +/** participationStatus → ATTENDEE_STATUS: accepted 1, declined 2, needs-action (the default) 3, tentative 4, else NONE. */ +export function participationToDevice(status: unknown): number { + switch (String(status ?? 'needs-action').toLowerCase()) { + case 'accepted': + return AttendeeStatus.ACCEPTED; + case 'declined': + return AttendeeStatus.DECLINED; + case 'needs-action': + return AttendeeStatus.INVITED; + case 'tentative': + return AttendeeStatus.TENTATIVE; + default: + return AttendeeStatus.NONE; + } +} + +/** ATTENDEE_STATUS → participationStatus; NONE has no JSCalendar value. */ +export function participationFromDevice(value: unknown): string | null { + switch (Number(value)) { + case AttendeeStatus.ACCEPTED: + return 'accepted'; + case AttendeeStatus.DECLINED: + return 'declined'; + case AttendeeStatus.INVITED: + return 'needs-action'; + case AttendeeStatus.TENTATIVE: + return 'tentative'; + default: + return null; + } +} + +/** ATTENDEE_TYPE: the `optional` role → OPTIONAL, kind resource/location → RESOURCE, else REQUIRED. */ +export function attendeeTypeToDevice(participant: { roles?: Record | null; kind?: string | null }): number { + const kind = String(participant.kind ?? '').toLowerCase(); + if (kind === 'resource' || kind === 'location') return AttendeeType.RESOURCE; + if (participant.roles?.optional) return AttendeeType.OPTIONAL; + return AttendeeType.REQUIRED; +} + +export function relationshipToDevice(isOrganizer: boolean): number { + return isOrganizer ? AttendeeRelationship.ORGANIZER : AttendeeRelationship.ATTENDEE; +} + +// ─── Reminders ────────────────────────────────────────── + +/** Reminders.METHOD for an alert action: email → EMAIL, anything else → ALERT. */ +export function reminderMethodToDevice(action: unknown): number { + return String(action ?? '').toLowerCase() === 'email' ? ReminderMethod.EMAIL : ReminderMethod.ALERT; +} + +/** The alert action of a METHOD: EMAIL → email; DEFAULT, ALERT, ALARM and SMS all fire a notification. */ +export function reminderActionFromDevice(method: unknown): 'email' | 'display' { + return Number(method) === ReminderMethod.EMAIL ? 'email' : 'display'; +} + +// ─── Colours ──────────────────────────────────────────── + +/** CSS named colours (CSS Color Module Level 4), as `name:rrggbb`. */ +const NAMED_COLORS = Object.fromEntries( + ( + 'aliceblue:f0f8ff antiquewhite:faebd7 aqua:00ffff aquamarine:7fffd4 azure:f0ffff beige:f5f5dc bisque:ffe4c4 ' + + 'black:000000 blanchedalmond:ffebcd blue:0000ff blueviolet:8a2be2 brown:a52a2a burlywood:deb887 ' + + 'cadetblue:5f9ea0 chartreuse:7fff00 chocolate:d2691e coral:ff7f50 cornflowerblue:6495ed cornsilk:fff8dc ' + + 'crimson:dc143c cyan:00ffff darkblue:00008b darkcyan:008b8b darkgoldenrod:b8860b darkgray:a9a9a9 ' + + 'darkgreen:006400 darkgrey:a9a9a9 darkkhaki:bdb76b darkmagenta:8b008b darkolivegreen:556b2f ' + + 'darkorange:ff8c00 darkorchid:9932cc darkred:8b0000 darksalmon:e9967a darkseagreen:8fbc8f ' + + 'darkslateblue:483d8b darkslategray:2f4f4f darkslategrey:2f4f4f darkturquoise:00ced1 darkviolet:9400d3 ' + + 'deeppink:ff1493 deepskyblue:00bfff dimgray:696969 dimgrey:696969 dodgerblue:1e90ff firebrick:b22222 ' + + 'floralwhite:fffaf0 forestgreen:228b22 fuchsia:ff00ff gainsboro:dcdcdc ghostwhite:f8f8ff gold:ffd700 ' + + 'goldenrod:daa520 gray:808080 green:008000 greenyellow:adff2f grey:808080 honeydew:f0fff0 hotpink:ff69b4 ' + + 'indianred:cd5c5c indigo:4b0082 ivory:fffff0 khaki:f0e68c lavender:e6e6fa lavenderblush:fff0f5 ' + + 'lawngreen:7cfc00 lemonchiffon:fffacd lightblue:add8e6 lightcoral:f08080 lightcyan:e0ffff ' + + 'lightgoldenrodyellow:fafad2 lightgray:d3d3d3 lightgreen:90ee90 lightgrey:d3d3d3 lightpink:ffb6c1 ' + + 'lightsalmon:ffa07a lightseagreen:20b2aa lightskyblue:87cefa lightslategray:778899 lightslategrey:778899 ' + + 'lightsteelblue:b0c4de lightyellow:ffffe0 lime:00ff00 limegreen:32cd32 linen:faf0e6 magenta:ff00ff ' + + 'maroon:800000 mediumaquamarine:66cdaa mediumblue:0000cd mediumorchid:ba55d3 mediumpurple:9370db ' + + 'mediumseagreen:3cb371 mediumslateblue:7b68ee mediumspringgreen:00fa9a mediumturquoise:48d1cc ' + + 'mediumvioletred:c71585 midnightblue:191970 mintcream:f5fffa mistyrose:ffe4e1 moccasin:ffe4b5 ' + + 'navajowhite:ffdead navy:000080 oldlace:fdf5e6 olive:808000 olivedrab:6b8e23 orange:ffa500 ' + + 'orangered:ff4500 orchid:da70d6 palegoldenrod:eee8aa palegreen:98fb98 paleturquoise:afeeee ' + + 'palevioletred:db7093 papayawhip:ffefd5 peachpuff:ffdab9 peru:cd853f pink:ffc0cb plum:dda0dd ' + + 'powderblue:b0e0e6 purple:800080 rebeccapurple:663399 red:ff0000 rosybrown:bc8f8f royalblue:4169e1 ' + + 'saddlebrown:8b4513 salmon:fa8072 sandybrown:f4a460 seagreen:2e8b57 seashell:fff5ee sienna:a0522d ' + + 'silver:c0c0c0 skyblue:87ceeb slateblue:6a5acd slategray:708090 slategrey:708090 snow:fffafa ' + + 'springgreen:00ff7f steelblue:4682b4 tan:d2b48c teal:008080 thistle:d8bfd8 tomato:ff6347 ' + + 'turquoise:40e0d0 violet:ee82ee wheat:f5deb3 white:ffffff whitesmoke:f5f5f5 yellow:ffff00 ' + + 'yellowgreen:9acd32' + ) + .split(' ') + .map((pair) => pair.split(':') as [string, string]), +); + +/** + * A CSS colour (a name, `#rgb` or `#rrggbb`) as the signed 32-bit ARGB int + * the provider stores, fully opaque; null for anything else. + */ +export function cssColorToArgb(color: unknown): number | null { + if (typeof color !== 'string') return null; + const c = color.trim().toLowerCase(); + let hex = NAMED_COLORS[c] ?? null; + if (!hex) { + const m = /^#([0-9a-f]{3}|[0-9a-f]{6})$/.exec(c); + if (!m) return null; + hex = m[1].length === 3 ? m[1].split('').map((ch) => ch + ch).join('') : m[1]; + } + return (0xff000000 | parseInt(hex, 16)) | 0; +} + +/** An ARGB int as `#rrggbb` (the alpha is dropped: JSCalendar colours are opaque). */ +export function argbToCss(value: unknown): string | null { + if (value === null || value === undefined || value === '') return null; + const n = Number(value); + if (!Number.isFinite(n)) return null; + return `#${((n >>> 0) & 0xffffff).toString(16).padStart(6, '0')}`; +} diff --git a/src/device-sync/calendar/zoned-time.ts b/src/device-sync/calendar/zoned-time.ts new file mode 100644 index 00000000..a2f56ff3 --- /dev/null +++ b/src/device-sync/calendar/zoned-time.ts @@ -0,0 +1,252 @@ +/** + * JSCalendar LocalDateTimes in an IANA zone ↔ UTC instants, for mapping + * events to CalendarProvider's millisecond columns (docs/device-sync.md, + * "Timing" and "DST edges"). Only `Intl` is used, so it behaves the same on + * Hermes and in vitest, whatever zone the process runs in. + * + * RFC 5545 §3.3.5 decides the transition hours: a wall time in a DST gap is + * read with the offset before the gap (it moves forward by the gap), and a + * wall time in an overlap is the earlier of its two instants. The app's + * `localDateTimeToInstant` (lib/calendar-timezone.ts) gets one of the two + * wrong depending on the sign of the zone's offset, and its module imports + * the settings store, so it is not reused here. + */ + +export interface WallClock { + year: number; + month: number; + day: number; + hour: number; + minute: number; + second: number; +} + +export type LocalTimeKind = 'ok' | 'gap' | 'overlap'; + +export const DAY_MS = 86_400_000; +const HOUR_MS = 3_600_000; + +const LOCAL_DATE_TIME = /^(\d{4})-(\d{2})-(\d{2})(?:T(\d{2}):(\d{2})(?::(\d{2})(?:[.,]\d+)?)?)?/; + +/** + * The fields of a LocalDateTime (`2026-10-25T02:30:00`) or a date + * (`2026-10-25`, read as midnight). Like Stalwart, a fraction and a trailing + * `Z` or offset are ignored. Null for anything that is not a real date-time. + */ +export function parseLocalDateTime(value: unknown): WallClock | null { + if (typeof value !== 'string') return null; + const m = LOCAL_DATE_TIME.exec(value.trim()); + if (!m) return null; + const wall: WallClock = { + year: Number(m[1]), + month: Number(m[2]), + day: Number(m[3]), + hour: Number(m[4] ?? 0), + minute: Number(m[5] ?? 0), + second: Number(m[6] ?? 0), + }; + if (wall.hour > 23 || wall.minute > 59 || wall.second > 59) return null; + // Date arithmetic rolls 2026-02-30 over into March; a real date survives the round trip. + const back = wallFromMs(wallMs(wall)); + return back.month === wall.month && back.day === wall.day ? wall : null; +} + +const pad = (n: number, len = 2) => String(n).padStart(len, '0'); + +export function formatLocalDateTime(wall: WallClock): string { + return `${pad(wall.year, 4)}-${pad(wall.month)}-${pad(wall.day)}T${pad(wall.hour)}:${pad(wall.minute)}:${pad(wall.second)}`; +} + +/** `YYYY-MM-DD` of a wall clock. */ +export function formatDate(wall: WallClock): string { + return `${pad(wall.year, 4)}-${pad(wall.month)}-${pad(wall.day)}`; +} + +/** The wall clock read as if it were UTC, in milliseconds (years below 100 included). */ +export function wallMs(wall: WallClock): number { + const d = new Date(0); + d.setUTCFullYear(wall.year, wall.month - 1, wall.day); + d.setUTCHours(wall.hour, wall.minute, wall.second, 0); + return d.getTime(); +} + +export function wallFromMs(ms: number): WallClock { + const d = new Date(ms); + return { + year: d.getUTCFullYear(), + month: d.getUTCMonth() + 1, + day: d.getUTCDate(), + hour: d.getUTCHours(), + minute: d.getUTCMinutes(), + second: d.getUTCSeconds(), + }; +} + +/** Wall-clock arithmetic: calendar days, then an exact amount of wall time. */ +export function addToWall(wall: WallClock, days: number, ms = 0): WallClock { + return wallFromMs(wallMs(wall) + days * DAY_MS + ms); +} + +// ─── Zones ────────────────────────────────────────────── + +const zoneCache = new Map(); + +/** + * The zone as Intl names it (canonical case, aliases resolved: `US/Eastern` + * → `America/New_York`, `Etc/UTC` → `UTC`), or null when Intl does not know + * it. Android's TimeZone silently turns an unknown id into GMT and Stalwart + * turns one into a floating event, so every zone is checked before either + * write. + */ +export function canonicalZone(zone: unknown): string | null { + if (typeof zone !== 'string' || !zone) return null; + let canonical = zoneCache.get(zone); + if (canonical === undefined) { + try { + canonical = new Intl.DateTimeFormat('en-US', { timeZone: zone }).resolvedOptions().timeZone ?? zone; + } catch { + canonical = null; + } + zoneCache.set(zone, canonical); + } + return canonical; +} + +export function isValidTimeZone(zone: unknown): zone is string { + return canonicalZone(zone) !== null; +} + +const formatters = new Map(); + +function formatter(zone: string): Intl.DateTimeFormat { + let f = formatters.get(zone); + if (!f) { + f = new Intl.DateTimeFormat('en-US', { + timeZone: zone, + hourCycle: 'h23', + year: 'numeric', + month: 'numeric', + day: 'numeric', + hour: 'numeric', + minute: 'numeric', + second: 'numeric', + }); + formatters.set(zone, f); + } + return f; +} + +/** Wall-clock fields of an instant in `zone` (whole seconds). */ +export function wallClockAt(utcMs: number, zone: string): WallClock { + const fields: Record = {}; + for (const part of formatter(zone).formatToParts(new Date(floorSecond(utcMs)))) { + if (part.type !== 'literal') fields[part.type] = Number(part.value); + } + return { + year: fields.year, + month: fields.month, + day: fields.day, + // Some engines still say "24" at midnight despite h23. + hour: fields.hour === 24 ? 0 : fields.hour, + minute: fields.minute, + second: fields.second, + }; +} + +function floorSecond(ms: number): number { + return Math.floor(ms / 1000) * 1000; +} + +/** The zone's UTC offset at an instant in ms, east positive. */ +export function offsetAt(utcMs: number, zone: string): number { + const at = floorSecond(utcMs); + return wallMs(wallClockAt(at, zone)) - at; +} + +export interface ResolvedLocalTime { + /** The instant per RFC 5545: gap → the offset before it; overlap → the earlier one. */ + utc: number; + kind: LocalTimeKind; +} + +/** + * The instant of a wall clock in `zone`, and whether the wall time is + * normal, skipped (gap) or repeated (overlap). Assumes at most one offset + * change within a day of the wall time, which holds for every real zone. + */ +export function resolveWallClock(wall: WallClock, zone: string): ResolvedLocalTime { + const w = wallMs(wall); + const before = offsetAt(w - DAY_MS, zone); + const after = offsetAt(w + DAY_MS, zone); + const tBefore = w - before; + const tAfter = w - after; + const okBefore = offsetAt(tBefore, zone) === before; + const okAfter = offsetAt(tAfter, zone) === after; + if (okBefore && okAfter) { + return tBefore === tAfter ? { utc: tBefore, kind: 'ok' } : { utc: Math.min(tBefore, tAfter), kind: 'overlap' }; + } + if (okBefore) return { utc: tBefore, kind: 'ok' }; + if (okAfter) return { utc: tAfter, kind: 'ok' }; + return { utc: tBefore, kind: 'gap' }; +} + +/** A LocalDateTime in `zone` as a UTC instant (RFC 5545 semantics), or null when unreadable. */ +export function localToUtc(value: string, zone: string): number | null { + const wall = parseLocalDateTime(value); + const z = canonicalZone(zone); + if (!wall || !z) return null; + return resolveWallClock(wall, z).utc; +} + +/** Whether a LocalDateTime is an ordinary, a skipped or a repeated wall time in `zone`. */ +export function classifyLocal(value: string, zone: string): LocalTimeKind | null { + const wall = parseLocalDateTime(value); + const z = canonicalZone(zone); + if (!wall || !z) return null; + return resolveWallClock(wall, z).kind; +} + +/** The instant as a LocalDateTime in `zone`. Both instants of an overlap give the same text. */ +export function utcToLocal(utcMs: number, zone: string): string { + return formatLocalDateTime(wallClockAt(utcMs, canonicalZone(zone) ?? 'UTC')); +} + +/** UTC midnight of a LocalDateTime's date: how CalendarProvider stores all-day times. */ +export function dateToUtcMidnight(value: string): number | null { + const wall = parseLocalDateTime(value); + return wall ? wallMs({ ...wall, hour: 0, minute: 0, second: 0 }) : null; +} + +/** The UTC date of an instant as a LocalDateTime at midnight (`2026-10-08T00:00:00`). */ +export function utcMidnightToDate(utcMs: number): string { + return `${formatDate(wallFromMs(utcMs))}T00:00:00`; +} + +/** + * A fixed-offset zone for an instant: `Etc/GMT-2` for UTC+2 (the Etc names + * flip the sign), `Etc/UTC` when the offset is zero or not a whole number of + * hours. Stalwart drops a start whose local time is repeated in its zone; the + * same instant and wall time in a fixed-offset zone survive (checked on the + * live server). + */ +export function fixedOffsetZone(offsetMs: number): string { + const hours = offsetMs / HOUR_MS; + if (!Number.isInteger(hours) || hours === 0 || hours < -12 || hours > 14) return 'Etc/UTC'; + return `Etc/GMT${hours > 0 ? '-' : '+'}${Math.abs(hours)}`; +} + +/** Compares two LocalDateTimes by their fields (a fraction or suffix is ignored). */ +export function sameLocalDateTime(a: unknown, b: unknown): boolean { + const wa = parseLocalDateTime(a); + const wb = parseLocalDateTime(b); + if (!wa || !wb) return wa === wb && a === b; + return wallMs(wa) === wallMs(wb); +} + +/** Orders LocalDateTimes (ISO text compares chronologically once normalised). */ +export function compareLocal(a: string, b: string): number { + const wa = parseLocalDateTime(a); + const wb = parseLocalDateTime(b); + if (!wa || !wb) return a < b ? -1 : a > b ? 1 : 0; + return wallMs(wa) - wallMs(wb); +} diff --git a/src/device-sync/task.ts b/src/device-sync/task.ts index 54d18436..6a1149d9 100644 --- a/src/device-sync/task.ts +++ b/src/device-sync/task.ts @@ -25,10 +25,11 @@ import { type SubscriptionCalendar, type TeardownOptions, } from './engine'; +import { calendarPlanner } from './calendar/planner'; import { emptyStats } from './engine/report'; import { openJmapPort } from './jmap/client-port'; import { createProviderPort, finishRun, getSyncSettings, isRunCancelled, showSyncProblem } from './native'; -import type { CalendarPlanner, ContactsPlanner } from './planner'; +import type { ContactsPlanner } from './planner'; import type { Authority, RunPayload, RunReport } from './types'; // Mirrors the persist names of the locale and feed-subscription stores; read @@ -38,13 +39,12 @@ const SUBSCRIPTIONS_STORAGE_KEY = 'calendar-subscriptions'; /** Where "sign in again" leads (the accounts pane), as for a registry eviction. */ const SIGN_IN_URI = 'bulwarkmobile://settings/account'; -// The contacts and calendar planners land in the next -// commits. Until then a run fails before it touches a row. +// The contacts planner lands in the next +// commit. Until then a run fails before it touches a row. function notWired(): never { throw new Error('Device sync planners are not wired into this build yet'); } const contactsPlanner = new Proxy({}, { get: notWired }) as ContactsPlanner; -const calendarPlanner = new Proxy({}, { get: notWired }) as CalendarPlanner; /** Random numbers from the platform CSPRNG, buffered (uids and map keys of device-made items). */ function secureRandom(): RandomSource { From a948738506db0be975438d0f6cd5761f2c8f74f9 Mon Sep 17 00:00:00 2001 From: Linus Rath <139418639+rathlinus@users.noreply.github.com> Date: Sun, 27 Sep 2026 04:23:22 +0200 Subject: [PATCH 09/87] feat: map contacts and groups to android's contacts provider #34 This is the planner for contacts (`src/device-sync/contacts`), the last piece the sync task was waiting for: it turns JSContact cards into raw contacts and data rows, and device edits back into JMAP writes. - Names, nicknames, organizations and titles, emails, phones, postal addresses, websites, birthdays and anniversaries (also without a year), relations, notes and photos map both ways, with their types and custom labels. Group cards become groups, and their members become group memberships. - Device edits upload as patches of the fields whose rows changed; a whole map goes up only where Stalwart needs one. Whatever a card holds that Android can't show (pronouns, online services, keywords, a second note, an address's coordinates) is never touched, so an edit on the phone leaves it intact on the server. - Rows are matched to their card's entries by the keys they carry and, where an editor re-inserted them without (Fossify saves that way), by their values. An entry the editor didn't show is never read as deleted, and the keys Stalwart assigns by position to cards from other clients are followed when they shift. - Contacts in read-only address books are put back when the device changes them. Text beyond what the provider stores is cut on the device and never uploaded. Checked: 151 contacts tests (mappings both ways, one test per edited field, Fossify and AOSP editing, re-keyed cards, merges, groups, read-only books, a full lifecycle), the engine's tests with both real planners, and `npm run typecheck`, `npm test` and `npm run i18n:check` on this commit. --- .../__tests__/contacts/accepted.test.ts | 95 +++++ .../__tests__/contacts/download.test.ts | 243 ++++++++++++ .../__tests__/contacts/fixtures.ts | 142 +++++++ .../__tests__/contacts/groups.test.ts | 203 ++++++++++ src/device-sync/__tests__/contacts/harness.ts | 234 ++++++++++++ .../__tests__/contacts/lifecycle.test.ts | 123 ++++++ .../__tests__/contacts/merge.test.ts | 164 ++++++++ .../__tests__/contacts/units.test.ts | 155 ++++++++ .../__tests__/contacts/upload-editors.test.ts | 358 ++++++++++++++++++ .../__tests__/contacts/upload-fields.test.ts | 229 +++++++++++ .../__tests__/engine/real-planners.test.ts | 221 +++++++++++ src/device-sync/contacts/cells.ts | 76 ++++ src/device-sync/contacts/columns.ts | 113 ++++++ src/device-sync/contacts/diff.ts | 61 +++ src/device-sync/contacts/download.ts | 340 +++++++++++++++++ src/device-sync/contacts/entries.ts | 348 +++++++++++++++++ src/device-sync/contacts/groups.ts | 252 ++++++++++++ src/device-sync/contacts/keys.ts | 65 ++++ src/device-sync/contacts/matching.ts | 131 +++++++ src/device-sync/contacts/members.ts | 149 ++++++++ src/device-sync/contacts/name.ts | 213 +++++++++++ src/device-sync/contacts/ops.ts | 75 ++++ src/device-sync/contacts/photo.ts | 91 +++++ src/device-sync/contacts/planner.ts | 78 ++++ src/device-sync/contacts/postal.ts | 129 +++++++ src/device-sync/contacts/project.ts | 328 ++++++++++++++++ src/device-sync/contacts/types-map.ts | 223 +++++++++++ src/device-sync/contacts/upload.ts | 292 ++++++++++++++ src/device-sync/task.ts | 9 +- 29 files changed, 5132 insertions(+), 8 deletions(-) create mode 100644 src/device-sync/__tests__/contacts/accepted.test.ts create mode 100644 src/device-sync/__tests__/contacts/download.test.ts create mode 100644 src/device-sync/__tests__/contacts/fixtures.ts create mode 100644 src/device-sync/__tests__/contacts/groups.test.ts create mode 100644 src/device-sync/__tests__/contacts/harness.ts create mode 100644 src/device-sync/__tests__/contacts/lifecycle.test.ts create mode 100644 src/device-sync/__tests__/contacts/merge.test.ts create mode 100644 src/device-sync/__tests__/contacts/units.test.ts create mode 100644 src/device-sync/__tests__/contacts/upload-editors.test.ts create mode 100644 src/device-sync/__tests__/contacts/upload-fields.test.ts create mode 100644 src/device-sync/__tests__/engine/real-planners.test.ts create mode 100644 src/device-sync/contacts/cells.ts create mode 100644 src/device-sync/contacts/columns.ts create mode 100644 src/device-sync/contacts/diff.ts create mode 100644 src/device-sync/contacts/download.ts create mode 100644 src/device-sync/contacts/entries.ts create mode 100644 src/device-sync/contacts/groups.ts create mode 100644 src/device-sync/contacts/keys.ts create mode 100644 src/device-sync/contacts/matching.ts create mode 100644 src/device-sync/contacts/members.ts create mode 100644 src/device-sync/contacts/name.ts create mode 100644 src/device-sync/contacts/ops.ts create mode 100644 src/device-sync/contacts/photo.ts create mode 100644 src/device-sync/contacts/planner.ts create mode 100644 src/device-sync/contacts/postal.ts create mode 100644 src/device-sync/contacts/project.ts create mode 100644 src/device-sync/contacts/types-map.ts create mode 100644 src/device-sync/contacts/upload.ts diff --git a/src/device-sync/__tests__/contacts/accepted.test.ts b/src/device-sync/__tests__/contacts/accepted.test.ts new file mode 100644 index 00000000..3635346f --- /dev/null +++ b/src/device-sync/__tests__/contacts/accepted.test.ts @@ -0,0 +1,95 @@ +import { describe, expect, it } from 'vitest'; +import { RawContacts } from '../../android-columns'; +import { sha256Hex } from '../../../lib/sha256'; +import { base64ToBytes } from '../../contacts/photo'; +import { appleCard, JPEG2, probeCard } from './fixtures'; +import { Harness } from './harness'; + +async function patchOf(h: Harness, rawId: number) { + const plan = h.planner.planUpload(await h.contact(rawId), h.ctx); + return plan.kind === 'upload' && plan.actions[0].kind === 'update' ? plan.actions[0].patch : plan.kind; +} + +describe('contacts planAccepted', () => { + it('writes the new shadow and baselines and clears DIRTY behind the VERSION read before the upload', async () => { + const h = new Harness(); + const { id, rawId } = await h.seed(appleCard()); + const email = h.rowsByKey(rawId)['emails:k1']._id as number; + h.device.user.updateData(email, { data1: 'new@example.com' }); + const local = await h.contact(rawId); + const plan = h.planner.planUpload(local, h.ctx); + if (plan.kind !== 'upload') throw new Error(plan.kind); + h.send(plan.actions); + const accepted = h.planner.planAccepted(local, h.card(id), h.ctx); + expect(accepted.ops.ops[0]).toEqual({ op: 'assert', table: 'raw_contacts', id: rawId, values: { version: local.version }, expectCount: 1 }); + await h.applyOk(accepted.ops); + const after = await h.contact(rawId); + expect(after.dirty).toBe(false); + expect(after.shadow?.emails).toEqual(h.card(id).emails); + expect(after.rows.find((r) => r.id === email)?.baseline?.data1).toBe('new@example.com'); + expect(h.planner.planDownload(h.card(id), after, h.ctx).effect).toBe('none'); + expect(h.planner.planBaselineHeal(after)).toBeNull(); + }); + + it('keeps DIRTY and uploads only the newer edit when the contact was edited during the upload', async () => { + const h = new Harness(); + const { rawId } = await h.seed(appleCard()); + const rows = h.rowsByKey(rawId); + h.device.user.updateData(rows['emails:k1']._id as number, { data1: 'first@example.com' }); + await h.upload(rawId, () => h.device.user.updateData(rows['phones:k1']._id as number, { data1: '+1 555 0142' })); + expect(h.dirty(rawId)).toBe(true); + expect(await patchOf(h, rawId)).toEqual({ 'phones/k1/number': '+1 555 0142' }); + }); + + it('uploads a second edit of the same field made during the upload, and a revert to the old value', async () => { + const h = new Harness(); + const { rawId } = await h.seed(appleCard()); + const email = h.rowsByKey(rawId)['emails:k1']._id as number; + h.device.user.updateData(email, { data1: 'a1@example.com' }); + await h.upload(rawId, () => h.device.user.updateData(email, { data1: 'a2@example.com' })); + expect(await patchOf(h, rawId)).toEqual({ 'emails/k1/address': 'a2@example.com' }); + + const h2 = new Harness(); + const seeded = await h2.seed(appleCard()); + const email2 = h2.rowsByKey(seeded.rawId)['emails:k1']._id as number; + h2.device.user.updateData(email2, { data1: 'a1@example.com' }); + await h2.upload(seeded.rawId, () => h2.device.user.updateData(email2, { data1: 'anna@example.com' })); + expect(await patchOf(h2, seeded.rawId)).toEqual({ 'emails/k1/address': 'anna@example.com' }); + }); + + it('writes the identity of a create even when the contact changed meanwhile', async () => { + const h = new Harness(); + const rawId = h.device.user.insertContact('usera@example.org', [ + { mimetype: 'vnd.android.cursor.item/name', data1: 'Late Edit', data2: 'Late', data3: 'Edit' }, + ]); + await h.upload(rawId, () => h.device.user.star(rawId)); + const local = await h.contact(rawId); + expect(local.sourceId).toMatch(/^c\//); + expect(local.pending).toBeNull(); + }); + + it('keeps the device photo after uploading it and records the server hash', async () => { + const h = new Harness(); + const { id, rawId } = await h.seed(probeCard()); + h.device.user.setPhoto(rawId, JPEG2); + h.devicePhotos.set(rawId, JPEG2); + const fileId = h.rowsByKey(rawId)['media:ph'].data14; + await h.upload(rawId); + const photo = h.rowsByKey(rawId)['media:ph']; + expect(photo.data14).toBe(fileId); + expect(photo.data_sync2).toBe(`sha256:${sha256Hex(base64ToBytes(JPEG2)!)}`); + expect(h.dirty(rawId)).toBe(false); + const local = await h.contact(rawId); + expect(h.planner.planDownload(h.card(id), local, h.ctx).effect).toBe('none'); + expect(h.planner.planBaselineHeal(local)).toBeNull(); + }); + + it('clears the poison marker and the pending create once accepted', async () => { + const h = new Harness(); + const { rawId } = await h.seed(appleCard()); + await h.applyOk({ ref: 'x', ops: [{ op: 'update', table: 'raw_contacts', id: rawId, values: { [RawContacts.SYNC4]: JSON.stringify({ fp: 'old', type: 'invalidProperties', n: 1, until: 1 }) } }] }); + h.device.user.updateData(h.rowsByKey(rawId)['emails:k1']._id as number, { data1: 'z@example.com' }); + await h.upload(rawId); + expect(h.rawContact(rawId)[RawContacts.SYNC4]).toBeNull(); + }); +}); diff --git a/src/device-sync/__tests__/contacts/download.test.ts b/src/device-sync/__tests__/contacts/download.test.ts new file mode 100644 index 00000000..3bdf68f2 --- /dev/null +++ b/src/device-sync/__tests__/contacts/download.test.ts @@ -0,0 +1,243 @@ +import { describe, expect, it } from 'vitest'; +import { Data, MimeType, RawContacts } from '../../android-columns'; +import { sha256Hex } from '../../../lib/sha256'; +import { base64ToBytes } from '../../contacts/photo'; +import type { Row } from '../../types'; +import { appleCard, googleCard, JPEG, probeCard } from './fixtures'; +import { Harness, JMAP } from './harness'; + +/** Cells as text, so the fixtures don't depend on how the provider types a column. */ +function cells(row: Row | undefined, columns: string[]): Record { + if (!row) throw new Error('missing row'); + return Object.fromEntries(columns.map((c) => [c, row[c] === null || row[c] === undefined ? null : String(row[c])])); +} + +const NAME = ['data1', 'data2', 'data3', 'data4', 'data5', 'data6', 'data7', 'data8', 'data9']; +const TYPED = ['data1', 'data2', 'data3']; +const PRIMARY = ['data1', 'data2', 'data3', 'is_primary']; +const POSTAL = ['data1', 'data2', 'data3', 'data4', 'data5', 'data6', 'data7', 'data8', 'data9', 'data10']; +const ORG = ['data1', 'data4', 'data5', 'data6']; + +const photoHash = (b64: string) => `sha256:${sha256Hex(base64ToBytes(b64)!)}`; + +describe('contacts download: golden rows', () => { + it('writes the probe card as one row per entry with its keys', async () => { + const h = new Harness(); + h.names.set('urn:uuid:other-person', 'Other Person'); + const { id, rawId } = await h.seed(probeCard()); + const rows = h.rowsByKey(rawId); + expect(Object.keys(rows).sort()).toEqual([ + 'addresses:home', 'anniversaries:bday', 'anniversaries:wed', 'emails:work1', 'emails:x/y', 'emails:zz9', + 'links:l1', 'media:ph', 'name', 'nicknames:nk', 'notes:n1', 'organizations:o1|titles:t1,r1', 'phones:fax7', + 'phones:p-a', 'relatedTo:urn:uuid:other-person', + ]); + expect(cells(rows.name, NAME)).toEqual({ + data1: 'Dr. Probe Person', data2: 'Probe', data3: 'Person', data4: 'Dr.', data5: null, data6: null, + data7: 'Proob', data8: null, data9: null, + }); + expect(cells(rows['nicknames:nk'], ['data1', 'data2'])).toEqual({ data1: 'Probby', data2: '1' }); + expect(cells(rows['emails:work1'], PRIMARY)).toEqual({ data1: 'probe@work.example', data2: '0', data3: 'Office', is_primary: '1' }); + expect(cells(rows['emails:x/y'], PRIMARY)).toEqual({ data1: 'slash@example.org', data2: '3', data3: null, is_primary: '0' }); + expect(cells(rows['emails:zz9'], PRIMARY)).toEqual({ data1: 'private@example.org', data2: '1', data3: null, is_primary: '0' }); + expect(cells(rows['phones:p-a'], TYPED)).toEqual({ data1: '+49 30 1234', data2: '2', data3: null }); + expect(cells(rows['phones:fax7'], TYPED)).toEqual({ data1: '+49 30 9999', data2: '4', data3: null }); + expect(cells(rows['addresses:home'], POSTAL)).toEqual({ + data1: 'Main St 12, 10115 Berlin, Germany', data2: '1', data3: null, data4: 'Main St 12 4b', data5: null, + data6: null, data7: 'Berlin', data8: null, data9: '10115', data10: 'Germany', + }); + expect(cells(rows['organizations:o1|titles:t1,r1'], ORG)).toEqual({ data1: 'ACME', data4: 'Chief Prober', data5: 'R&D', data6: 'Lead' }); + expect(cells(rows['links:l1'], TYPED)).toEqual({ data1: 'https://probe.example', data2: '5', data3: null }); + expect(cells(rows['anniversaries:bday'], TYPED)).toEqual({ data1: '--03-14', data2: '3', data3: null }); + expect(cells(rows['anniversaries:wed'], TYPED)).toEqual({ data1: '2010-06-01', data2: '1', data3: null }); + expect(cells(rows['relatedTo:urn:uuid:other-person'], TYPED)).toEqual({ data1: 'Other Person', data2: '6', data3: null }); + expect(cells(rows['notes:n1'], ['data1'])).toEqual({ data1: 'first note' }); + expect(rows['media:ph'].data_sync2).toBe(photoHash(JPEG)); + expect(h.device.photo(Number(rows['media:ph'].data14))).toBe(JPEG); + + const raw = h.rawContact(rawId); + expect(raw).toMatchObject({ + [RawContacts.SOURCE_ID]: `${JMAP}/${id}`, + [RawContacts.SYNC1]: `${JMAP}/${h.book}`, + [RawContacts.SYNC3]: null, + [RawContacts.DIRTY]: 0, + }); + expect(Number(raw[RawContacts.RAW_CONTACT_IS_READ_ONLY])).toBe(0); + }); + + it('keeps the photo out of the shadow: its data URI becomes the hash of its bytes', async () => { + const h = new Harness(); + const { id, rawId } = await h.seed(probeCard()); + const shadow = JSON.parse(h.rawContact(rawId)[RawContacts.SYNC2] as string); + expect(shadow.media.ph.uri).toBe(photoHash(JPEG)); + expect({ ...shadow, media: undefined, '~memberOf': undefined }).toEqual({ ...h.card(id), media: undefined }); + }); + + it('records each row as stored, so a clean contact needs no baseline heal', async () => { + const h = new Harness(); + for (const card of [probeCard(), appleCard(), googleCard(), { name: { full: 'Madonna' } }]) { + const { rawId } = await h.seed(card); + const local = await h.contact(rawId); + expect(h.planner.planBaselineHeal(local)).toBeNull(); + for (const row of local.rows) if (row.mimetype !== MimeType.PHOTO) expect(row.baseline).not.toBeNull(); + } + }); + + it('maps an Apple card with positional keys, filling the formatted address itself', async () => { + const h = new Harness(); + const { rawId } = await h.seed(appleCard()); + const rows = h.rowsByKey(rawId); + expect(cells(rows['emails:k1'], PRIMARY)).toEqual({ data1: 'anna@example.com', data2: '1', data3: null, is_primary: '1' }); + expect(cells(rows['emails:k2'], PRIMARY)).toEqual({ data1: 'anna@work.example', data2: '2', data3: null, is_primary: '0' }); + expect(cells(rows['phones:k1'], TYPED)).toMatchObject({ data2: '2' }); + expect(cells(rows['phones:k2'], TYPED)).toMatchObject({ data2: '3' }); + expect(cells(rows['phones:k3'], TYPED)).toMatchObject({ data2: '4' }); + expect(cells(rows['addresses:k1'], POSTAL)).toEqual({ + data1: '1 Infinite Loop, 95014 Cupertino, CA, USA', data2: '2', data3: null, data4: '1 Infinite Loop', data5: null, + data6: null, data7: 'Cupertino', data8: 'CA', data9: '95014', data10: 'USA', + }); + expect(cells(rows['anniversaries:k1'], TYPED)).toEqual({ data1: '1980-05-04', data2: '3', data3: null }); + expect(Object.keys(rows).some((k) => k.startsWith('onlineServices'))).toBe(false); + }); + + it('maps a Google card: custom display name, legacy cell phone, labelled link, department list, relation by name', async () => { + const h = new Harness(); + const { rawId } = await h.seed(googleCard()); + const rows = h.rowsByKey(rawId); + expect(cells(rows.name, NAME)).toEqual({ + data1: 'Groß, Günther', data2: 'Günther', data3: 'Groß', data4: 'Prof.', data5: 'Karl', data6: 'PhD', + data7: null, data8: null, data9: null, + }); + expect(cells(rows['nicknames:k1'], ['data1'])).toEqual({ data1: 'Gü' }); + expect(cells(rows['nicknames:k2'], ['data1'])).toEqual({ data1: 'GKG' }); + expect(cells(rows['phones:k1'], TYPED)).toEqual({ data1: '0171 2345678', data2: '2', data3: null }); + expect(cells(rows['links:k1'], TYPED)).toEqual({ data1: 'https://example.org/~guenther', data2: '0', data3: 'profile' }); + expect(cells(rows['organizations:k1|titles:k1,'], ORG)).toEqual({ data1: 'Uni Beispiel', data4: 'Professor', data5: 'Informatik, AG Sync', data6: null }); + expect(cells(rows['relatedTo:Maria Groß'], TYPED)).toEqual({ data1: 'Maria Groß', data2: '14', data3: null }); + expect(cells(rows['anniversaries:k1'], TYPED)).toEqual({ data1: '--08-15', data2: '1', data3: null }); + }); + + it('writes a name with only `full` as the display name and lets the provider split it', async () => { + const h = new Harness(); + const { rawId } = await h.seed({ name: { full: 'Madonna' } }); + expect(cells(h.rowsByKey(rawId).name, ['data1', 'data2', 'data3'])).toEqual({ data1: 'Madonna', data2: 'Madonna', data3: null }); + }); + + it('writes text longer than the provider keeps already cut, and never uploads the cut column', async () => { + const h = new Harness(); + const long = 'x'.repeat(12_000); + const { id, rawId } = await h.seed({ name: { full: 'Long Note' }, notes: { n1: { note: long } }, phones: { p1: { number: '1'.repeat(1200) } } }); + const rows = h.rowsByKey(rawId); + expect((rows['notes:n1'].data1 as string).length).toBe(10 * 1024); + expect((rows['phones:p1'].data1 as string).length).toBe(1000); + expect(h.planner.planDownload(h.card(id), await h.contact(rawId), h.ctx).effect).toBe('none'); + h.device.user.updateData(rows['notes:n1']._id as number, { data1: `${rows['notes:n1'].data1 as string}!` }); + expect(h.planner.planUpload(await h.contact(rawId), h.ctx).kind).toBe('clean'); + }); + + it('marks a card of read-only books read-only', async () => { + const h = new Harness(); + h.readOnlyBooks.add(h.book); + const { rawId } = await h.seed(appleCard()); + expect(Number(h.rawContact(rawId)[RawContacts.RAW_CONTACT_IS_READ_ONLY])).toBe(1); + }); +}); + +describe('contacts download: echoes and updates', () => { + it.each([['probe', probeCard], ['apple', appleCard], ['google', googleCard]])('writes nothing for an unchanged %s card', async (_n, make) => { + const h = new Harness(); + const { id, rawId } = await h.seed(make()); + const plan = h.planner.planDownload(h.card(id), await h.contact(rawId), h.ctx); + expect(plan).toMatchObject({ effect: 'none', writes: 0, conflicts: 0 }); + expect(plan.ops.ops).toEqual([]); + }); + + it('writes nothing when the shadow itself comes back (a re-plan after a group change)', async () => { + const h = new Harness(); + const { rawId } = await h.seed(probeCard()); + const local = await h.contact(rawId); + const plan = h.planner.planDownload(local.shadow!, local, h.ctx); + expect(plan.effect).toBe('none'); + }); + + it('rewrites only the row of the entry the server changed, behind VERSION and DIRTY', async () => { + const h = new Harness(); + const { id, rawId } = await h.seed(appleCard()); + h.server.serverUpdate('ContactCard', JMAP, id, { 'emails/k2/address': 'anna@new-work.example' }); + const local = await h.contact(rawId); + const plan = h.planner.planDownload(h.card(id), local, h.ctx); + const [assert, raw, ...data] = plan.ops.ops; + expect(assert).toEqual({ op: 'assert', table: 'raw_contacts', id: rawId, values: { version: local.version, dirty: 0 }, expectCount: 1 }); + expect(raw).toMatchObject({ op: 'update', table: 'raw_contacts', values: { [RawContacts.SYNC2]: expect.any(String) } }); + expect(data).toHaveLength(1); + expect(data[0]).toMatchObject({ op: 'update', table: 'data', expectCount: 1, values: { data1: 'anna@new-work.example', data_sync1: 'emails:k2' } }); + await h.applyOk(plan.ops); + expect(h.rowsByKey(rawId)['emails:k2'].data1).toBe('anna@new-work.example'); + expect(h.dirty(rawId)).toBe(false); + }); + + it('inserts entries the server added and deletes the ones it removed', async () => { + const h = new Harness(); + const { id, rawId } = await h.seed(appleCard()); + h.server.serverUpdate('ContactCard', JMAP, id, { 'phones/k3': null, 'links': { k1: { uri: 'https://anna.example' } } }); + await h.download(id); + const rows = h.rowsByKey(rawId); + expect(rows['phones:k3']).toBeUndefined(); + expect(cells(rows['links:k1'], TYPED)).toEqual({ data1: 'https://anna.example', data2: '7', data3: null }); + expect(h.planner.planDownload(h.card(id), await h.contact(rawId), h.ctx).effect).toBe('none'); + }); + + it('follows keys Stalwart shifted (a CardDAV insert at the front) by content, with no rewrite of the rows', async () => { + const h = new Harness(); + const { id, rawId } = await h.seed(appleCard()); + const before = h.rowsByKey(rawId); + h.server.serverUpdate('ContactCard', JMAP, id, { + emails: { + k1: { address: 'first@example.com' }, + k2: { address: 'anna@example.com', contexts: { private: true }, pref: 1 }, + k3: { address: 'anna@work.example', contexts: { work: true } }, + }, + }); + const plan = h.planner.planDownload(h.card(id), await h.contact(rawId), h.ctx); + const data = plan.ops.ops.filter((o) => o.op !== 'assert' && o.op !== 'syncState' && o.table === 'data'); + expect(data).toHaveLength(3); + const renames = data.filter((o) => o.op === 'update'); + for (const r of renames) expect(Object.keys((r as { values: object }).values).sort()).toEqual(['data_sync1', 'data_sync3']); + await h.applyOk(plan.ops); + const rows = h.rowsByKey(rawId); + expect(rows['emails:k2']._id).toBe(before['emails:k1']._id); + expect(rows['emails:k3']._id).toBe(before['emails:k2']._id); + expect(rows['emails:k1'].data1).toBe('first@example.com'); + }); + + it('fails as a whole when the user edits the contact between read and write', async () => { + const h = new Harness(); + const { id, rawId } = await h.seed(appleCard()); + h.server.serverUpdate('ContactCard', JMAP, id, { 'notes/k1/note': 'from the server' }); + const plan = h.planner.planDownload(h.card(id), await h.contact(rawId), h.ctx); + const phone = h.rowsByKey(rawId)['phones:k1']._id as number; + h.device.beforeNextBatch(() => h.device.user.updateData(phone, { [Data.DATA1]: '+1 555 0199' })); + const res = await h.apply(plan.ops); + expect(res).toMatchObject({ ok: false, reason: 'assert' }); + expect(h.rowsByKey(rawId)['notes:k1'].data1).toBe('Met at WWDC'); + }); + + it('adopts its own create whose identity was never written, keeping the local rows', async () => { + const h = new Harness(); + const rawId = h.device.user.insertContact('usera@example.org', [ + { [Data.MIMETYPE]: MimeType.STRUCTURED_NAME, [Data.DATA1]: 'New Person', [Data.DATA2]: 'New', [Data.DATA3]: 'Person' }, + { [Data.MIMETYPE]: MimeType.EMAIL, [Data.DATA1]: 'new@example.org', [Data.DATA2]: 1 }, + ]); + const plan = await h.planUpload(rawId); + if (plan.kind !== 'upload' || plan.actions[0].kind !== 'create') throw new Error(`expected a create, got ${plan.kind}`); + const cardId = h.send(plan.actions)!; + // The response was lost: the next download finds the card and the pending uid. + const local = await h.contact(rawId); + const adopt = h.planner.planDownload(h.card(cardId), local, h.ctx); + expect(adopt.effect).toBe('update'); + expect(adopt.stillDirty).toBe(true); + await h.applyOk(adopt.ops); + const after = await h.contact(rawId); + expect(after).toMatchObject({ sourceId: `${JMAP}/${cardId}`, pending: null, dirty: true }); + expect(h.planner.planUpload(after, h.ctx).kind).toBe('clean'); + }); +}); diff --git a/src/device-sync/__tests__/contacts/fixtures.ts b/src/device-sync/__tests__/contacts/fixtures.ts new file mode 100644 index 00000000..ba39dadd --- /dev/null +++ b/src/device-sync/__tests__/contacts/fixtures.ts @@ -0,0 +1,142 @@ +/** + * Server cards for the contacts planner tests. `probeCard` is what the local + * Stalwart returned for a card created over JMAP with client keys (live probe + * output); the others are shaped like cards synced in over CardDAV from + * Apple and Google clients, whose keys Stalwart generates by position. + */ + +/** A 1×1 JPEG. */ +export const JPEG = '/9j/4AAQSkZJRgABAQEASABIAAD/2wBDAP//////////////////////////////////////////////////////////////////////////////////////wgALCAABAAEBAREA/8QAFBABAAAAAAAAAAAAAAAAAAAAAP/aAAgBAQABPxA='; +/** Another picture's bytes (any base64 does for the fakes). */ +export const JPEG2 = '/9j/4AAQSkZJRgABAgAAAQABAAD/2wBDAAgGBgcGBQgHBwcJCQgKDBQNDAsLDBkSEw8UHRofHh0aHBwgJC4nICIsIxwcKDcpLDAxNDQ0Hyc5PTgyPC4zNDL/wAALCAABAAEBAREA/8QAFAABAAAAAAAAAAAAAAAAAAAACf/EABQQAQAAAAAAAAAAAAAAAAAAAAD/2gAIAQEAAD8AKp//2Q=='; + +export function probeCard(): Record { + return { + uid: 'urn:uuid:probe-1790463389675', + kind: 'individual', + name: { + full: 'Dr. Probe Person', + phoneticSystem: 'ipa', + components: [ + { kind: 'title', value: 'Dr.' }, + { kind: 'given', value: 'Probe', phonetic: 'Proob' }, + { kind: 'surname', value: 'Person' }, + ], + isOrdered: true, + }, + nicknames: { nk: { name: 'Probby' } }, + emails: { + work1: { address: 'probe@work.example', contexts: { work: true }, pref: 1, label: 'Office' }, + 'x/y': { address: 'slash@example.org' }, + zz9: { address: 'private@example.org', contexts: { private: true } }, + }, + phones: { + 'p-a': { number: '+49 30 1234', contexts: { private: true }, features: { voice: true, mobile: true } }, + fax7: { number: '+49 30 9999', contexts: { work: true }, features: { fax: true } }, + }, + addresses: { + home: { + contexts: { private: true }, + coordinates: 'geo:52.5,13.4', + timeZone: 'Europe/Berlin', + countryCode: 'DE', + full: 'Main St 12, 10115 Berlin, Germany', + components: [ + { kind: 'name', value: 'Main St' }, + { kind: 'separator', value: ' ' }, + { kind: 'number', value: '12' }, + { kind: 'locality', value: 'Berlin' }, + { kind: 'postcode', value: '10115' }, + { kind: 'country', value: 'Germany' }, + { kind: 'apartment', value: '4b' }, + ], + isOrdered: true, + }, + }, + organizations: { o1: { name: 'ACME', units: [{ name: 'R&D' }] } }, + titles: { + t1: { name: 'Chief Prober', kind: 'title', organizationId: 'o1' }, + r1: { name: 'Lead', kind: 'role', organizationId: 'o1' }, + }, + links: { l1: { uri: 'https://probe.example', contexts: { work: true } } }, + onlineServices: { im1: { uri: 'xmpp:probe@example.org', service: 'XMPP' } }, + anniversaries: { + bday: { date: { '@type': 'PartialDate', month: 3, day: 14 }, kind: 'birth' }, + wed: { date: { '@type': 'PartialDate', year: 2010, month: 6, day: 1 }, kind: 'wedding' }, + }, + relatedTo: { 'urn:uuid:other-person': { relation: { friend: true } } }, + notes: { n1: { note: 'first note' }, n2: { note: 'second note' } }, + media: { ph: { uri: `data:image/jpeg;base64,${JPEG}`, kind: 'photo', mediaType: 'image/jpeg' } }, + keywords: { vip: true }, + personalInfo: { pi1: { value: 'probing', kind: 'hobby' } }, + preferredLanguages: { lang1: { language: 'de', pref: 1 } }, + speakToAs: { grammaticalGender: 'neuter' }, + }; +} + +/** Apple Contacts over CardDAV: no PROP-IDs, so `k1…kN`; an address without `full`. */ +export function appleCard(): Record { + return { + uid: 'urn:uuid:5F2C9B1E-APPLE', + kind: 'individual', + prodId: '-//Apple Inc.//iPhone OS 18.0//EN', + name: { + components: [{ kind: 'given', value: 'Anna' }, { kind: 'surname', value: 'Apfel' }], + isOrdered: true, + full: 'Anna Apfel', + }, + emails: { + k1: { address: 'anna@example.com', contexts: { private: true }, pref: 1 }, + k2: { address: 'anna@work.example', contexts: { work: true } }, + }, + phones: { + k1: { number: '+1 555 0100', features: { mobile: true, voice: true } }, + k2: { number: '+1 555 0101', contexts: { work: true }, features: { voice: true } }, + k3: { number: '+1 555 0102', contexts: { work: true }, features: { fax: true } }, + }, + addresses: { + k1: { + components: [ + { kind: 'name', value: '1 Infinite Loop' }, + { kind: 'locality', value: 'Cupertino' }, + { kind: 'region', value: 'CA' }, + { kind: 'postcode', value: '95014' }, + { kind: 'country', value: 'USA' }, + ], + isOrdered: true, + contexts: { work: true }, + }, + }, + anniversaries: { k1: { kind: 'birth', date: { '@type': 'PartialDate', year: 1980, month: 5, day: 4 } } }, + notes: { k1: { note: 'Met at WWDC' } }, + onlineServices: { k1: { service: 'Skype', user: 'anna.apfel' } }, + keywords: { friends: true }, + }; +} + +/** Google Contacts over CardDAV: a legacy `cell` feature, a custom display name, two nicknames. */ +export function googleCard(): Record { + return { + uid: 'google-8a7d2f', + kind: 'individual', + name: { + components: [ + { kind: 'title', value: 'Prof.' }, + { kind: 'given', value: 'Günther' }, + { kind: 'given2', value: 'Karl' }, + { kind: 'surname', value: 'Groß' }, + { kind: 'credential', value: 'PhD' }, + ], + isOrdered: true, + full: 'Groß, Günther', + }, + nicknames: { k1: { name: 'Gü' }, k2: { name: 'GKG' } }, + phones: { k1: { number: '0171 2345678', features: { cell: true } } }, + links: { k1: { uri: 'https://example.org/~guenther', label: 'profile' } }, + organizations: { k1: { name: 'Uni Beispiel', units: [{ name: 'Informatik' }, { name: 'AG Sync' }] } }, + titles: { k1: { name: 'Professor', kind: 'title', organizationId: 'k1' } }, + relatedTo: { 'Maria Groß': { relation: { spouse: true } } }, + anniversaries: { k1: { kind: 'wedding', date: { '@type': 'PartialDate', month: 8, day: 15 } } }, + speakToAs: { grammaticalGender: 'masculine' }, + }; +} diff --git a/src/device-sync/__tests__/contacts/groups.test.ts b/src/device-sync/__tests__/contacts/groups.test.ts new file mode 100644 index 00000000..74ffb7dd --- /dev/null +++ b/src/device-sync/__tests__/contacts/groups.test.ts @@ -0,0 +1,203 @@ +import { describe, expect, it } from 'vitest'; +import { Data, Groups, MimeType } from '../../android-columns'; +import type { ContactCardWire, OpGroup, UploadPlan } from '../../planner'; +import { appleCard, googleCard } from './fixtures'; +import { Harness, JMAP } from './harness'; + +function actionsOf(plan: UploadPlan) { + if (plan.kind !== 'upload') throw new Error(`expected an upload, got ${plan.kind}`); + return plan.actions; +} + +function opsOf(plan: UploadPlan): OpGroup { + if (plan.kind === 'upload' || plan.kind === 'skip') throw new Error(`expected ops, got ${plan.kind}`); + return plan.ops; +} + +async function localGroup(h: Harness, groupId: string) { + const g = (await h.groups()).find((x) => x.sourceId === `${JMAP}/${groupId}`); + if (!g) throw new Error(`no group row for ${groupId}`); + return g; +} + +/** A group card holding Anna (apple card), both on the device. */ +async function withGroup(members: Record = { [appleCard().uid as string]: true }) { + const h = new Harness(); + const gid = h.addCard({ uid: 'urn:uuid:group-friends', kind: 'group', name: { full: 'Friends' }, members }); + await h.download(gid); + const { id, rawId } = await h.seed(appleCard()); + return { h, gid, id, rawId }; +} + +describe('contacts groups: download', () => { + it('writes a group card as a visible group row, and nothing for its echo', async () => { + const { h, gid } = await withGroup(); + const row = h.device.rows('groups', 'sourceid = ?', [`${JMAP}/${gid}`])[0]; + expect(row).toMatchObject({ [Groups.TITLE]: 'Friends', [Groups.GROUP_VISIBLE]: 1, [Groups.DIRTY]: 0 }); + const plan = h.planner.planGroupDownload(h.card(gid), await localGroup(h, gid), h.ctx); + expect(plan).toMatchObject({ effect: 'none', ops: { ops: [] } }); + }); + + it('writes one membership row per group card that lists the contact', async () => { + const { h, gid, rawId } = await withGroup(); + expect(h.membershipGroups(rawId)).toEqual([`${JMAP}/${gid}`]); + const local = await h.contact(rawId); + expect(local.rows.find((r) => r.mimetype === MimeType.GROUP_MEMBERSHIP)?.key).toBe(`members:${JMAP}/${gid}`); + expect(local.shadow?.['~memberOf']).toEqual([`${JMAP}/${gid}`]); + }); + + it('leaves memberships alone when the engine has no group index', async () => { + const h = new Harness(); + h.groupIndex = false; + const gid = h.addCard({ uid: 'urn:uuid:g', kind: 'group', name: { full: 'G' }, members: { [appleCard().uid as string]: true } }); + await h.download(gid); + const { rawId } = await h.seed(appleCard()); + expect(h.membershipGroups(rawId)).toEqual([]); + expect((await h.contact(rawId)).shadow?.['~memberOf']).toBeUndefined(); + }); + + it('follows the group card when members change on the server (a re-plan of the unchanged contact)', async () => { + const { h, gid, rawId } = await withGroup(); + h.server.serverUpdate('ContactCard', JMAP, gid, { [`members/${appleCard().uid as string}`]: null }); + await h.download(gid); + let local = await h.contact(rawId); + await h.applyOk(h.planner.planDownload(local.shadow!, local, h.ctx).ops); + expect(h.membershipGroups(rawId)).toEqual([]); + h.server.serverUpdate('ContactCard', JMAP, gid, { members: { [appleCard().uid as string]: true } }); + await h.download(gid); + local = await h.contact(rawId); + await h.applyOk(h.planner.planDownload(local.shadow!, local, h.ctx).ops); + expect(h.membershipGroups(rawId)).toEqual([`${JMAP}/${gid}`]); + }); + + it('lets the server win when both sides renamed the group', async () => { + const { h, gid } = await withGroup(); + const g = await localGroup(h, gid); + h.device.user.updateGroup(g.groupId, { [Groups.TITLE]: 'Mates' }); + h.server.serverUpdate('ContactCard', JMAP, gid, { 'name/full': 'Pals' }); + const plan = h.planner.planGroupDownload(h.card(gid), await localGroup(h, gid), h.ctx); + expect(plan.conflicts).toBe(1); + await h.applyOk(plan.ops); + expect((await localGroup(h, gid)).title).toBe('Pals'); + }); +}); + +describe('contacts groups: membership edits', () => { + it('adds the contact to a group card with `members/: true`, keeping it dirty until the server has it', async () => { + const h = new Harness(); + const other = h.addCard({ uid: 'urn:uuid:g2', kind: 'group', name: { full: 'Work' }, members: { 'urn:uuid:someone': true } }); + await h.download(other); + const { rawId } = await h.seed(appleCard()); + const g = await localGroup(h, other); + h.device.user.addToGroup(rawId, g.groupId); + expect((await h.upload(rawId)).kind).toBe('clean'); + expect(h.dirty(rawId)).toBe(true); + const actions = h.planner.planMembershipUploads([await h.contact(rawId)], await h.groups(), h.ctx); + expect(actions).toEqual([{ kind: 'update', id: other, patch: { [`members/${appleCard().uid as string}`]: true } }]); + h.send(actions); + const accepted = h.planner.planGroupAccepted(g, h.card(other), h.ctx); + await h.applyOk(accepted.ops); + await h.upload(rawId); + expect(h.dirty(rawId)).toBe(false); + expect(h.planner.planMembershipUploads([await h.contact(rawId)], await h.groups(), h.ctx)).toEqual([]); + }); + + it('sends the whole map for the first member of an empty group', async () => { + const h = new Harness(); + const empty = h.addCard({ uid: 'urn:uuid:g3', kind: 'group', name: { full: 'Empty' } }); + await h.download(empty); + const { rawId } = await h.seed(appleCard()); + h.device.user.addToGroup(rawId, (await localGroup(h, empty)).groupId); + const actions = h.planner.planMembershipUploads([await h.contact(rawId)], await h.groups(), h.ctx); + expect(actions).toEqual([{ kind: 'update', id: empty, patch: { members: { [appleCard().uid as string]: true } } }]); + }); + + it('removes the contact with `members/: null` when its membership row was deleted in place', async () => { + const { h, gid, rawId } = await withGroup({ [appleCard().uid as string]: true, 'urn:uuid:someone': true }); + const membership = (await h.contact(rawId)).rows.find((r) => r.mimetype === MimeType.GROUP_MEMBERSHIP)!; + h.device.user.deleteData(membership.id); + const actions = h.planner.planMembershipUploads([await h.contact(rawId)], await h.groups(), h.ctx); + expect(actions).toEqual([{ kind: 'update', id: gid, patch: { [`members/${appleCard().uid as string}`]: null } }]); + }); + + it('uploads nothing when an editor re-inserted the same memberships without keys', async () => { + const { h, rawId } = await withGroup(); + const rows = h.device.rows('data', 'raw_contact_id = ?', [rawId]).filter((r) => r[Data.MIMETYPE] === MimeType.GROUP_MEMBERSHIP); + h.device.user.fossifySave(rawId, null, rows.map((r) => ({ [Data.MIMETYPE]: MimeType.GROUP_MEMBERSHIP, data1: r.data1 }))); + expect(h.planner.planMembershipUploads([await h.contact(rawId)], await h.groups(), h.ctx)).toEqual([]); + }); + + it('does not take a server-side addition for a local removal while the contact is dirty', async () => { + const { h, gid } = await withGroup({}); + const { rawId } = await h.seed(googleCard()); + h.device.user.updateData(h.rowsByKey(rawId)['phones:k1']._id as number, { data1: '0171 999' }); + h.server.serverUpdate('ContactCard', JMAP, gid, { members: { [googleCard().uid as string]: true } }); + await h.download(gid); + const local = await h.contact(rawId); + await h.applyOk(h.planner.planDownload(local.shadow!, local, h.ctx).ops); + expect(h.membershipGroups(rawId)).toEqual([`${JMAP}/${gid}`]); + expect(h.planner.planMembershipUploads([await h.contact(rawId)], await h.groups(), h.ctx)).toEqual([]); + }); +}); + +describe('contacts groups: device edits of groups', () => { + it('uploads a rename as `name/full`', async () => { + const { h, gid } = await withGroup(); + const g = await localGroup(h, gid); + h.device.user.updateGroup(g.groupId, { [Groups.TITLE]: 'Close friends' }); + const plan = h.planner.planGroupUpload(await localGroup(h, gid), h.ctx); + expect(plan).toEqual({ kind: 'upload', actions: [{ kind: 'update', id: gid, patch: { 'name/full': 'Close friends' } }] }); + h.send(actionsOf(plan)); + const accepted = h.planner.planGroupAccepted(await localGroup(h, gid), h.card(gid), h.ctx); + await h.applyOk(accepted.ops); + expect(await localGroup(h, gid)).toMatchObject({ dirty: false, title: 'Close friends' }); + expect(h.planner.planGroupDownload(h.card(gid), await localGroup(h, gid), h.ctx).effect).toBe('none'); + }); + + it('claims and creates a group made on the device', async () => { + const h = new Harness(); + const groupId = h.device.user.insertGroup('usera@example.org', 'Book club'); + const decode = async () => (await h.groups()).find((g) => g.groupId === groupId)!; + const claim = h.planner.planGroupUpload(await decode(), h.ctx); + expect(claim.kind).toBe('claim'); + await h.applyOk(opsOf(claim)); + const pending = (await decode()).pending!; + const plan = h.planner.planGroupUpload(await decode(), h.ctx); + expect(plan).toEqual({ + kind: 'upload', + actions: [{ + kind: 'create', uid: pending.uid, collectionId: h.book, + object: { '@type': 'Card', version: '1.0', uid: pending.uid, kind: 'group', addressBookIds: { [h.book]: true }, name: { full: 'Book club' } }, + }], + }); + const id = h.send(actionsOf(plan))!; + await h.applyOk(h.planner.planGroupAccepted(await decode(), h.card(id), h.ctx).ops); + expect(await decode()).toMatchObject({ sourceId: `${JMAP}/${id}`, dirty: false, pending: null }); + }); + + it('destroys a group deleted on the device', async () => { + const { h, gid } = await withGroup(); + h.device.user.deleteGroup((await localGroup(h, gid)).groupId); + expect(h.planner.planGroupUpload(await localGroup(h, gid), h.ctx)).toEqual({ + kind: 'upload', actions: [{ kind: 'destroy', id: gid, uid: 'urn:uuid:group-friends' }], + }); + }); + + it('reverts a rename of a read-only group', async () => { + const { h, gid } = await withGroup(); + h.readOnlyBooks.add(h.book); + const g = await localGroup(h, gid); + h.device.user.updateGroup(g.groupId, { [Groups.TITLE]: 'Mine now' }); + const plan = h.planner.planGroupUpload(await localGroup(h, gid), h.ctx); + expect(plan.kind).toBe('revert'); + await h.applyOk(opsOf(plan)); + expect(await localGroup(h, gid)).toMatchObject({ title: 'Friends', dirty: false }); + }); + + it('deletes the group row for a server delete', async () => { + const { h, gid, rawId } = await withGroup(); + await h.applyOk(h.planner.planGroupLocalDelete(await localGroup(h, gid))); + expect(h.device.rows('groups', 'sourceid = ?', [`${JMAP}/${gid}`])).toEqual([]); + expect(h.membershipGroups(rawId)).toEqual([]); + }); +}); diff --git a/src/device-sync/__tests__/contacts/harness.ts b/src/device-sync/__tests__/contacts/harness.ts new file mode 100644 index 00000000..095a1c9b --- /dev/null +++ b/src/device-sync/__tests__/contacts/harness.ts @@ -0,0 +1,234 @@ +/** + * Test rig for the contacts planner: the fake device providers, the fake + * Stalwart, and a ContactsContext wired between them. Plans are applied the + * way the engine applies them (one op group per batch); uploads go through + * the fake server's patch handling and come back through its normalisation. + */ +import { GroupMembership, Groups, RawContacts } from '../../android-columns'; +import { makeKeyMinter, parseObjectRef, uuidFrom } from '../../common/ids'; +import { contactsPlanner } from '../../contacts/planner'; +import type { + ContactCardWire, + ContactsContext, + LocalContact, + LocalGroup, + OpGroup, + UploadAction, + UploadPlan, +} from '../../planner'; +import { CONTACTS_AUTHORITY, type BatchResult, type ProviderPort, type Row } from '../../types'; +import { FakeJmapServer } from '../fakes/fake-jmap-server'; +import { FakeDeviceProviders } from '../fakes/fake-provider'; + +export const ACCOUNT = 'usera@example.org'; +export const JMAP = 'c'; + +export function seeded(seed = 7): () => number { + return () => { + seed = (seed * 1103515245 + 12345) % 2147483648; + return seed / 2147483648; + }; +} + +function rowsOf(res: { columns: string[]; rows: unknown[][] }): Row[] { + return res.rows.map((cells) => Object.fromEntries(res.columns.map((c, i) => [c, cells[i]])) as Row); +} + +export class Harness { + readonly device = new FakeDeviceProviders(); + readonly port: ProviderPort = this.device.port(ACCOUNT, CONTACTS_AUTHORITY); + readonly server = new FakeJmapServer(); + readonly book: string; + readonly planner = contactsPlanner; + /** Base64 of blob-backed photos by blobId, and the device photo per raw contact. */ + readonly blobs = new Map(); + readonly devicePhotos = new Map(); + readonly names = new Map(); + readonly readOnlyBooks = new Set(); + readonly unselected = new Set(); + /** When false the context has no `groupsOf` (memberships not synced). */ + groupIndex = true; + readonly ctx: ContactsContext; + + constructor() { + this.server.addAccount(JMAP, { name: ACCOUNT }); + this.book = this.server.addAddressBook(JMAP, { name: 'Personal' }); + const random = seeded(); + const mintKey = makeKeyMinter(random); + const self = this; + this.ctx = { + jmapAccountId: JMAP, + now: 1_800_000_000_000, + mintKey, + mintUid: () => `urn:uuid:${uuidFrom(random)}`, + selectedCollections: (card) => + Object.keys(card.addressBookIds ?? {}).map((id) => `${JMAP}/${id}`).filter((k) => !self.unselected.has(k)), + isReadOnly: (card) => Object.keys(card.addressBookIds ?? {}).every((id) => self.readOnlyBooks.has(id)), + isSelected: (key) => !self.unselected.has(key), + createTarget: () => `${JMAP}/${self.book}`, + photoBytes: (entry) => (entry.blobId ? self.blobs.get(entry.blobId) ?? null : null), + devicePhoto: (id) => self.devicePhotos.get(id) ?? null, + nameForUid: (uid) => self.names.get(uid) ?? null, + groupRowIdBySourceId: (sourceId) => { + const row = self.device.rows('groups', 'sourceid = ?', [sourceId])[0]; + return row ? Number(row[Groups._ID]) : null; + }, + groupsOf(uid: string) { + return self.server + .all('ContactCard', JMAP) + .filter((c) => c.kind === 'group' && (c.members as Record | undefined)?.[uid] === true) + .map((c) => `${JMAP}/${c.id as string}`); + }, + }; + } + + /** The context as the engine may build it without a group index. */ + context(): ContactsContext { + if (this.groupIndex) return this.ctx; + const { groupsOf: _drop, ...rest } = this.ctx; + return rest as ContactsContext; + } + + async apply(group: OpGroup): Promise { + if (!group.ops.length) return { ok: true, results: [] }; + return this.port.applyBatch(group.ops); + } + + async applyOk(group: OpGroup): Promise { + const res = await this.apply(group); + if (!res.ok) throw new Error(`batch failed: ${res.reason} ${res.message}`); + } + + async contacts(): Promise { + const raw = rowsOf(await this.port.query({ table: 'raw_contacts', columns: [...this.planner.rawContactColumns] })); + const data = rowsOf(await this.port.query({ table: 'data', columns: [...this.planner.dataColumns] })); + return raw.map((r) => this.planner.decodeContact(r, data.filter((d) => d.raw_contact_id === r._id))); + } + + async contact(rawContactId: number): Promise { + const found = (await this.contacts()).find((c) => c.rawContactId === rawContactId); + if (!found) throw new Error(`no raw contact ${rawContactId}`); + return found; + } + + async bySource(cardId: string): Promise { + return (await this.contacts()).find((c) => c.sourceId === `${JMAP}/${cardId}`) ?? null; + } + + async groups(): Promise { + const rows = rowsOf(await this.port.query({ table: 'groups', columns: [...this.planner.groupColumns] })); + return rows.map((r) => this.planner.decodeGroup(r)); + } + + card(id: string): ContactCardWire { + const card = this.server.get('ContactCard', JMAP, id); + if (!card) throw new Error(`no card ${id}`); + return card as ContactCardWire; + } + + addCard(card: Record): string { + return this.server.addCard(JMAP, { addressBookIds: { [this.book]: true }, ...card }); + } + + /** Downloads a server card into its local contact (or a new one); returns the raw contact id. */ + async download(cardId: string): Promise { + const card = this.card(cardId); + const local = await this.bySource(cardId); + const plan = card.kind === 'group' + ? this.planner.planGroupDownload(card, (await this.groups()).find((g) => g.sourceId === `${JMAP}/${cardId}`) ?? null, this.context()) + : this.planner.planDownload(card, local, this.context()); + await this.applyOk(plan.ops); + if (card.kind === 'group') return -1; + const after = await this.bySource(cardId); + if (!after) throw new Error('download wrote no contact'); + const heal = this.planner.planBaselineHeal(after); + if (heal) await this.applyOk(heal); + return after.rawContactId; + } + + /** A card on the server and on the device, clean. */ + async seed(card: Record): Promise<{ id: string; rawId: number }> { + const id = this.addCard(card); + return { id, rawId: await this.download(id) }; + } + + /** Sends upload actions to the fake server the way the engine would; returns the touched card id. */ + send(actions: UploadAction[]): string | null { + let touched: string | null = null; + for (const a of actions) { + if (a.kind === 'update') { + this.server.serverUpdate('ContactCard', JMAP, a.id, a.patch); + touched = a.id; + } else if (a.kind === 'create') { + touched = this.server.addCard(JMAP, a.object as Record); + } else if (a.id) { + this.server.serverDestroy('ContactCard', JMAP, a.id); + } + } + return touched; + } + + /** Plans an upload for a contact, claiming it first when it is new. */ + async planUpload(rawContactId: number): Promise> { + let plan = this.planner.planUpload(await this.contact(rawContactId), this.context()); + if (plan.kind === 'claim') { + await this.applyOk(plan.ops); + plan = this.planner.planUpload(await this.contact(rawContactId), this.context()); + } + return plan; + } + + /** + * One upload round for a contact: plan, send, re-read the card, apply the + * accepted plan (or its keep-dirty fallback). Returns the plan. + */ + async upload(rawContactId: number, between?: () => void): Promise> { + const plan = await this.planUpload(rawContactId); + const local = await this.contact(rawContactId); + if (plan.kind === 'upload') { + const touched = this.send(plan.actions); + between?.(); + if (local.deleted) { + const accepted = this.planner.planAccepted(local, {} as ContactCardWire, this.context()); + await this.applyOk(accepted.ops); + return plan; + } + const server = this.card(touched!); + const accepted = this.planner.planAccepted(local, server, this.context()); + const res = await this.apply(accepted.ops); + if (!res.ok) await this.applyOk(accepted.keepDirtyOps); + } else if (plan.kind !== 'skip') { + between?.(); + await this.applyOk(plan.ops); + } + return plan; + } + + cardIdOf(local: LocalContact): string { + return parseObjectRef(local.sourceId)!.id; + } + + /** Data rows of a raw contact as the provider holds them, keyed by DATA_SYNC1 (or `row:`). */ + rowsByKey(rawContactId: number): Record { + const out: Record = {}; + for (const r of this.device.rows('data', 'raw_contact_id = ?', [rawContactId])) { + out[(r.data_sync1 as string | null) ?? `row:${r._id}`] = r; + } + return out; + } + + rawContact(rawContactId: number): Row { + return this.device.row('raw_contacts', rawContactId)!; + } + + dirty(rawContactId: number): boolean { + return Number(this.rawContact(rawContactId)[RawContacts.DIRTY]) === 1; + } + + membershipGroups(rawContactId: number): string[] { + return this.device + .rows('data', 'raw_contact_id = ? AND mimetype = ?', [rawContactId, 'vnd.android.cursor.item/group_membership']) + .map((r) => r[GroupMembership.GROUP_SOURCE_ID] as string) + .sort(); + } +} diff --git a/src/device-sync/__tests__/contacts/lifecycle.test.ts b/src/device-sync/__tests__/contacts/lifecycle.test.ts new file mode 100644 index 00000000..94c414ff --- /dev/null +++ b/src/device-sync/__tests__/contacts/lifecycle.test.ts @@ -0,0 +1,123 @@ +import { describe, expect, it } from 'vitest'; +import { Data, MimeType } from '../../android-columns'; +import type { LocalContact } from '../../planner'; +import { JPEG, probeCard } from './fixtures'; +import { Harness, JMAP } from './harness'; + +/** The engine's view after every write: a clean contact whose rows the server card describes. */ +async function expectSettled(h: Harness, id: string, rawId: number): Promise { + const local = await h.contact(rawId); + expect(local.dirty).toBe(false); + expect(h.planner.planDownload(h.card(id), local, h.ctx).effect).toBe('none'); + expect(h.planner.planBaselineHeal(local)).toBeNull(); + return local; +} + +describe('contacts lifecycle', () => { + it('settles after device edits, an editor rewrite and server edits in turn', async () => { + const h = new Harness(); + const { id, rawId } = await h.seed(probeCard()); + let rows = h.rowsByKey(rawId); + + // Several fields at once in AOSP Contacts. + h.device.user.updateData(rows.name._id as number, { data2: 'Probert', data1: 'Dr. Probert Person' }); + h.device.user.updateData(rows['emails:zz9']._id as number, { data1: 'home@example.org' }); + h.device.user.updateData(rows['addresses:home']._id as number, { data9: '10117' }); + h.device.user.insertData(rawId, { [Data.MIMETYPE]: MimeType.PHONE, data1: '+49 30 7777', data2: 3 }); + await h.upload(rawId); + const card = h.card(id); + expect(card.name).toMatchObject({ full: 'Dr. Probert Person' }); + expect(card.emails!.zz9.address).toBe('home@example.org'); + expect(card.addresses!.home.components).toContainEqual({ kind: 'postcode', value: '10117' }); + expect(Object.values(card.phones!).map((p) => p.number)).toContain('+49 30 7777'); + expect(card.onlineServices).toEqual(probeCard().onlineServices); + expect(card.notes!.n2).toEqual({ note: 'second note' }); + await expectSettled(h, id, rawId); + + // Fossify rewrites everything but the name; nothing uploads, the rows come back whole. + rows = h.rowsByKey(rawId); + h.device.user.fossifySave(rawId, null, Object.values(rows) + .filter((r) => r[Data.MIMETYPE] === MimeType.EMAIL || r[Data.MIMETYPE] === MimeType.PHONE) + .map((r) => ({ [Data.MIMETYPE]: r[Data.MIMETYPE], data1: r.data1, data2: r.data2 }))); + expect((await h.upload(rawId)).kind).toBe('clean'); + await expectSettled(h, id, rawId); + expect(h.rowsByKey(rawId)['organizations:o1|titles:t1,r1'].data5).toBe('R&D'); + + // The server edits and removes things. + h.server.serverUpdate('ContactCard', JMAP, id, { 'nicknames/nk/name': 'Probo', 'links/l1': null, 'addresses/home/components': [{ kind: 'locality', value: 'Hamburg' }] }); + await h.download(id); + await expectSettled(h, id, rawId); + rows = h.rowsByKey(rawId); + expect(rows['nicknames:nk'].data1).toBe('Probo'); + expect(rows['links:l1']).toBeUndefined(); + expect(rows['addresses:home']).toMatchObject({ data7: 'Hamburg', data4: null }); + }); + + it('follows Stalwart re-keying the whole card by position, without rewriting a row', async () => { + const h = new Harness(); + const { id, rawId } = await h.seed(probeCard()); + const before = h.rowsByKey(rawId); + h.server.rekeyPositionally(JMAP, id, 'ContactCard'); + const plan = h.planner.planDownload(h.card(id), await h.contact(rawId), h.ctx); + for (const op of plan.ops.ops) { + if (op.op === 'update' && op.table === 'data') expect(Object.keys(op.values).every((c) => c.startsWith('data_sync'))).toBe(true); + expect(op.op === 'insert' || op.op === 'delete').toBe(false); + } + await h.applyOk(plan.ops); + const after = h.rowsByKey(rawId); + expect(after['organizations:k1|titles:k1,k2']._id).toBe(before['organizations:o1|titles:t1,r1']._id); + expect(after['emails:k2']._id).toBe(before['emails:x/y']._id); + await expectSettled(h, id, rawId); + + // The next device edit patches the entry under its new key. + h.device.user.updateData(after['emails:k2']._id as number, { data1: 'slash3@example.org' }); + const upload = h.planner.planUpload(await h.contact(rawId), h.ctx); + expect(upload).toMatchObject({ kind: 'upload', actions: [{ patch: { 'emails/k2/address': 'slash3@example.org' } }] }); + }); + + it('writes a blob-backed photo from the bytes the engine fetched, and nothing for its echo', async () => { + const h = new Harness(); + h.blobs.set('B1', JPEG); + const { id, rawId } = await h.seed({ name: { full: 'Blob Photo' }, media: { p: { kind: 'photo', blobId: 'B1', mediaType: 'image/jpeg' } } }); + const photo = h.rowsByKey(rawId)['media:p']; + expect(h.device.photo(Number(photo.data14))).toBe(JPEG); + expect((await h.contact(rawId)).shadow?.media).toEqual({ p: { kind: 'photo', blobId: 'B1', mediaType: 'image/jpeg' } }); + await expectSettled(h, id, rawId); + }); + + it('keeps the photo row when the photo bytes could not be fetched, and never reads it as deleted', async () => { + const h = new Harness(); + h.blobs.set('B1', JPEG); + const { id, rawId } = await h.seed({ name: { full: 'Blob Photo' }, emails: { e1: { address: 'a@example.org' } }, media: { p: { kind: 'photo', blobId: 'B1' } } }); + const photoRow = h.rowsByKey(rawId)['media:p']._id; + h.blobs.delete('B1'); + h.server.serverUpdate('ContactCard', JMAP, id, { 'emails/e1/address': 'b@example.org' }); + await h.download(id); + expect(h.rowsByKey(rawId)['media:p']?._id).toBe(photoRow); + h.device.user.updateData(h.rowsByKey(rawId)['emails:e1']._id as number, { data1: 'c@example.org' }); + expect(h.planner.planUpload(await h.contact(rawId), h.ctx)).toMatchObject({ kind: 'upload', actions: [{ patch: { 'emails/e1/address': 'c@example.org' } }] }); + const plan = h.planner.planUpload(await h.contact(rawId), h.ctx); + expect(plan.kind === 'upload' && Object.keys((plan.actions[0] as { patch: object }).patch)).toEqual(['emails/e1/address']); + }); + + it('heals a baseline the provider stored differently from the prediction', async () => { + const h = new Harness(); + const { id, rawId } = await h.seed({ name: { full: 'Dr. Jane Q. Public' } }); + const name = h.rowsByKey(rawId).name; + // The real NameSplitter knows titles and middle names; the prediction does not. + await h.applyOk({ ref: 'split', ops: [{ op: 'update', table: 'data', id: name._id as number, values: { data4: 'Dr.', data2: 'Jane', data5: 'Q.', data3: 'Public' } }] }); + const heal = h.planner.planBaselineHeal(await h.contact(rawId)); + expect(heal?.ops.map((o) => o.op)).toEqual(['assert', 'update']); + await h.applyOk(heal!); + await expectSettled(h, id, rawId); + h.device.user.star(rawId); + expect((await h.upload(rawId)).kind).toBe('clean'); + }); + + it('does not heal a dirty contact: its edit must still count', async () => { + const h = new Harness(); + const { rawId } = await h.seed(probeCard()); + h.device.user.updateData(h.rowsByKey(rawId)['notes:n1']._id as number, { data1: 'edited' }); + expect(h.planner.planBaselineHeal(await h.contact(rawId))).toBeNull(); + }); +}); diff --git a/src/device-sync/__tests__/contacts/merge.test.ts b/src/device-sync/__tests__/contacts/merge.test.ts new file mode 100644 index 00000000..709f3d92 --- /dev/null +++ b/src/device-sync/__tests__/contacts/merge.test.ts @@ -0,0 +1,164 @@ +import { describe, expect, it } from 'vitest'; +import { Data, MimeType } from '../../android-columns'; +import type { ContactCardWire } from '../../planner'; +import { appleCard, googleCard } from './fixtures'; +import { Harness, JMAP } from './harness'; + +/** A seeded card, edited on the device (in place, like AOSP) and on the server. */ +async function bothEdited( + make: () => Record, + device: (h: Harness, rawId: number) => void, + server: Record, +) { + const h = new Harness(); + const { id, rawId } = await h.seed(make()); + device(h, rawId); + h.server.serverUpdate('ContactCard', JMAP, id, server); + const plan = h.planner.planDownload(h.card(id), await h.contact(rawId), h.ctx); + await h.applyOk(plan.ops); + return { h, id, rawId, plan }; +} + +const edit = (key: string, values: Record) => (h: Harness, rawId: number) => + h.device.user.updateData(h.rowsByKey(rawId)[key]._id as number, values); + +async function patchOf(h: Harness, rawId: number) { + const plan = h.planner.planUpload(await h.contact(rawId), h.ctx); + return plan.kind === 'upload' && plan.actions[0].kind === 'update' ? plan.actions[0].patch : plan.kind; +} + +describe('contacts merge: a dirty contact meets a new server version', () => { + it('lets the server win a real conflict and counts it', async () => { + const { h, rawId, plan } = await bothEdited(appleCard, edit('emails:k1', { data1: 'mine@example.com' }), { 'emails/k1/address': 'theirs@example.com' }); + expect(plan).toMatchObject({ conflicts: 1, stillDirty: false, effect: 'update' }); + expect(h.rowsByKey(rawId)['emails:k1'].data1).toBe('theirs@example.com'); + expect(h.dirty(rawId)).toBe(true); + expect(await patchOf(h, rawId)).toBe('clean'); + }); + + it('keeps a local edit of one unit and takes the server edit of another', async () => { + const { h, rawId, plan } = await bothEdited(appleCard, edit('phones:k1', { data1: '+1 555 0199' }), { 'emails/k2/address': 'anna@new.example' }); + expect(plan).toMatchObject({ conflicts: 0, stillDirty: true }); + expect(h.rowsByKey(rawId)['emails:k2'].data1).toBe('anna@new.example'); + expect(await patchOf(h, rawId)).toEqual({ 'phones/k1/number': '+1 555 0199' }); + }); + + it('treats the same change on both sides as converged', async () => { + const { h, rawId, plan } = await bothEdited(appleCard, edit('emails:k1', { data1: 'same@example.com' }), { 'emails/k1/address': 'same@example.com' }); + expect(plan).toMatchObject({ conflicts: 0, stillDirty: false }); + expect(await patchOf(h, rawId)).toBe('clean'); + }); + + it('does not count a server change to a sub-field Android does not show', async () => { + const { h, id, rawId, plan } = await bothEdited(appleCard, edit('emails:k1', { data1: 'mine@example.com' }), { 'emails/k1/contexts': { private: true, school: true } }); + expect(plan.conflicts).toBe(0); + expect(await patchOf(h, rawId)).toEqual({ 'emails/k1/address': 'mine@example.com' }); + await h.upload(rawId); + expect(h.card(id).emails!.k1).toMatchObject({ address: 'mine@example.com', contexts: { private: true, school: true } }); + }); + + it('deletes a row the server deleted, over a local edit (counted)', async () => { + const { h, rawId, plan } = await bothEdited(appleCard, edit('phones:k3', { data1: '+1 555 0177' }), { 'phones/k3': null }); + expect(plan.conflicts).toBe(1); + expect(h.rowsByKey(rawId)['phones:k3']).toBeUndefined(); + }); + + it('puts back an entry deleted on the device that the server changed (counted)', async () => { + const { h, rawId, plan } = await bothEdited( + appleCard, + (hh, raw) => hh.device.user.deleteData(hh.rowsByKey(raw)['phones:k3']._id as number), + { 'phones/k3/number': '+1 555 0188' }, + ); + expect(plan.conflicts).toBe(1); + expect(h.rowsByKey(rawId)['phones:k3'].data1).toBe('+1 555 0188'); + }); + + it('keeps a device deletion of an entry the server left alone', async () => { + const { h, rawId, plan } = await bothEdited( + appleCard, + (hh, raw) => hh.device.user.deleteData(hh.rowsByKey(raw)['phones:k3']._id as number), + { 'emails/k2/address': 'anna@new.example' }, + ); + expect(plan).toMatchObject({ conflicts: 0, stillDirty: true }); + const patch = await patchOf(h, rawId) as Record; + expect(Object.keys(patch)).toEqual(['phones']); + expect(Object.keys(patch.phones as object)).toEqual(['k1', 'k2']); + }); + + it('matches an entry both sides added instead of duplicating it', async () => { + const { h, rawId, plan } = await bothEdited( + appleCard, + (hh, raw) => hh.device.user.insertData(raw, { [Data.MIMETYPE]: MimeType.EMAIL, data1: 'both@example.com', data2: 3 }), + { 'emails/k3': { address: 'both@example.com' } }, + ); + expect(plan.conflicts).toBe(0); + const emails = h.device.rows('data', 'raw_contact_id = ? AND mimetype = ?', [rawId, MimeType.EMAIL]); + expect(emails.filter((r) => r.data1 === 'both@example.com')).toHaveLength(1); + expect(await patchOf(h, rawId)).toBe('clean'); + }); + + it('inserts what the server added next to a pending local edit', async () => { + const { h, rawId, plan } = await bothEdited(appleCard, edit('phones:k1', { data1: '+1 555 0199' }), { 'emails/k3': { address: 'third@example.com' } }); + expect(plan.stillDirty).toBe(true); + expect(h.rowsByKey(rawId)['emails:k3'].data1).toBe('third@example.com'); + expect(await patchOf(h, rawId)).toEqual({ 'phones/k1/number': '+1 555 0199' }); + }); + + it('follows shifted keys and uploads the local edit under the new key', async () => { + const { h, rawId, plan } = await bothEdited(appleCard, edit('emails:k2', { data1: 'anna@work2.example' }), { + emails: { + k1: { address: 'first@example.com' }, + k2: { address: 'anna@example.com', contexts: { private: true }, pref: 1 }, + k3: { address: 'anna@work.example', contexts: { work: true } }, + }, + }); + expect(plan.conflicts).toBe(0); + const rows = h.rowsByKey(rawId); + expect(rows['emails:k3'].data1).toBe('anna@work2.example'); + expect(rows['emails:k1'].data1).toBe('first@example.com'); + expect(await patchOf(h, rawId)).toEqual({ 'emails/k3/address': 'anna@work2.example' }); + }); + + it('merges the name as one unit: a typed name on the server beats a local component edit', async () => { + const { h, rawId, plan } = await bothEdited(appleCard, edit('name', { data2: 'Annie', data1: 'Annie Apfel' }), { 'name/full': 'A. Apfel' }); + expect(plan.conflicts).toBe(1); + expect(h.rowsByKey(rawId).name).toMatchObject({ data1: 'A. Apfel', data2: 'Anna' }); + }); + + it('never gives keys to a kind an editor rewrote, so a nickname it could not show is not deleted later', async () => { + const h = new Harness(); + const { id, rawId } = await h.seed(googleCard()); + // Fossify keeps one nickname, re-inserted without its key. + h.device.user.fossifySave(rawId, null, [{ [Data.MIMETYPE]: MimeType.NICKNAME, data1: 'Gü' }]); + h.server.serverUpdate('ContactCard', JMAP, id, { 'nicknames/k1/name': 'Guenther' }); + const plan = h.planner.planDownload(h.card(id), await h.contact(rawId), h.ctx); + await h.applyOk(plan.ops); + const nick = h.device.rows('data', 'raw_contact_id = ? AND mimetype = ?', [rawId, MimeType.NICKNAME]); + expect(nick.map((r) => [r.data1, r.data_sync1])).toEqual([['Guenther', null]]); + const patch = await patchOf(h, rawId); + expect(typeof patch === 'object' && Object.keys(patch).some((k) => k.startsWith('nicknames'))).toBe(false); + }); + + it('leaves a contact deleted on the device alone: the delete wins and uploads', async () => { + const h = new Harness(); + const { id, rawId } = await h.seed(appleCard()); + h.device.user.deleteContact(rawId); + h.server.serverUpdate('ContactCard', JMAP, id, { 'emails/k1/address': 'x@example.com' }); + const plan = h.planner.planDownload(h.card(id), await h.contact(rawId), h.ctx); + expect(plan).toMatchObject({ effect: 'none', stillDirty: true, ops: { ops: [] } }); + expect(h.planner.planUpload(await h.contact(rawId), h.ctx)).toMatchObject({ kind: 'upload', actions: [{ kind: 'destroy', id }] }); + }); + + it('replaces the rows of a different object behind the same id (another uid)', async () => { + const h = new Harness(); + const { id, rawId } = await h.seed(appleCard()); + edit('emails:k1', { data1: 'mine@example.com' })(h, rawId); + const other = { ...(googleCard() as object), id, addressBookIds: { [h.book]: true } } as unknown as ContactCardWire; + const plan = h.planner.planDownload(other, await h.contact(rawId), h.ctx); + await h.applyOk(plan.ops); + const rows = h.rowsByKey(rawId); + expect(rows['emails:k1']).toBeUndefined(); + expect(rows['nicknames:k1'].data1).toBe('Gü'); + expect(h.dirty(rawId)).toBe(false); + }); +}); diff --git a/src/device-sync/__tests__/contacts/units.test.ts b/src/device-sync/__tests__/contacts/units.test.ts new file mode 100644 index 00000000..497dc2c1 --- /dev/null +++ b/src/device-sync/__tests__/contacts/units.test.ts @@ -0,0 +1,155 @@ +import { describe, expect, it } from 'vitest'; +import { Data, MimeType, PhoneType, RelationType } from '../../android-columns'; +import { contactsPlanner } from '../../contacts/planner'; +import { entryKey, orgKey, parseEntryKey, parseOrgKey } from '../../contacts/keys'; +import { editPostalComponents, postalCells } from '../../contacts/postal'; +import { base64ToBytes, dataUriBase64, shadowOf } from '../../contacts/photo'; +import { fullIsDerived, nameCells } from '../../contacts/name'; +import { phoneFlags, phoneType, relationName, relationType } from '../../contacts/types-map'; +import type { ContactCardWire } from '../../planner'; +import { JPEG } from './fixtures'; + +describe('contacts decode', () => { + it('queries the columns the planner reads', () => { + expect(contactsPlanner.rawContactColumns).toEqual(expect.arrayContaining([ + '_id', 'sourceid', 'version', 'dirty', 'deleted', 'sync1', 'sync2', 'sync3', 'sync4', 'raw_contact_is_read_only', + ])); + expect(contactsPlanner.dataColumns).toEqual(expect.arrayContaining([ + '_id', 'raw_contact_id', 'mimetype', 'is_primary', 'data1', 'data14', 'data_sync1', 'data_sync2', 'data_sync3', 'group_sourceid', + ])); + expect(contactsPlanner.dataColumns).not.toContain('data15'); + expect(contactsPlanner.groupColumns).toEqual(expect.arrayContaining(['_id', 'sourceid', 'version', 'dirty', 'deleted', 'title', 'sync2', 'sync3', 'sync4'])); + }); + + it('reads numbers given as text (data columns are TEXT) and tolerates what other apps left in sync columns', () => { + const local = contactsPlanner.decodeContact( + { _id: '5', sourceid: 'c/e1', version: '3', dirty: '1', deleted: null, sync1: 'c/b,garbage,,c/x', sync2: '{"not":"a card"}', sync3: '{"uid":1}', sync4: 'nope', raw_contact_is_read_only: 0 }, + [ + { _id: '9', raw_contact_id: '5', mimetype: MimeType.EMAIL, data1: 'a@b', data2: '2', data_sync1: 'emails:e1', data_sync2: '', data_sync3: '[1]' }, + { _id: 10, raw_contact_id: 6, mimetype: MimeType.NOTE, data1: 'another contact' }, + ], + ); + expect(local).toMatchObject({ + rawContactId: 5, sourceId: 'c/e1', version: 3, dirty: true, deleted: false, readOnly: false, + collections: ['c/b', 'c/x'], shadow: null, pending: null, poison: null, + }); + expect(local.rows).toEqual([{ id: 9, mimetype: MimeType.EMAIL, cells: { data1: 'a@b', data2: '2' }, key: 'emails:e1', photoHash: null, baseline: null }]); + }); + + it('decodes a pending create, a poison marker and a group', () => { + const local = contactsPlanner.decodeContact({ + _id: 1, sourceid: null, version: 1, dirty: 1, deleted: 0, + sync3: JSON.stringify({ uid: 'urn:uuid:x', target: 'c/b' }), + sync4: JSON.stringify({ fp: 'f', type: 'invalidProperties', n: 2, until: 5 }), + }, []); + expect(local.pending).toEqual({ uid: 'urn:uuid:x', target: 'c/b' }); + expect(local.poison).toEqual({ fp: 'f', type: 'invalidProperties', n: 2, until: 5 }); + expect(contactsPlanner.decodeGroup({ _id: 4, sourceid: 'c/g', version: 2, dirty: 0, deleted: 1, title: 'T', sync2: '{"id":"g","kind":"group"}' })) + .toMatchObject({ groupId: 4, sourceId: 'c/g', deleted: true, title: 'T', shadow: { id: 'g', kind: 'group' } }); + }); +}); + +describe('contacts keys', () => { + it('round-trips entry keys with any character', () => { + for (const key of ['work1', 'x/y', 'a,b|c%d', 'urn:uuid:1', 'k:1']) { + expect(parseEntryKey(entryKey('emails', key))).toEqual({ map: 'emails', key }); + } + expect(parseEntryKey('name')).toBeNull(); + expect(parseEntryKey('garbage')).toBeNull(); + }); + + it('round-trips organization keys, titles positional as title then role', () => { + expect(orgKey({ org: 'o1', title: 't1', role: 'r1' })).toBe('organizations:o1|titles:t1,r1'); + expect(orgKey({ org: null, title: null, role: 'r1' })).toBe('titles:,r1'); + expect(orgKey({ org: 'o1', title: null, role: null })).toBe('organizations:o1'); + for (const parts of [{ org: 'o|1', title: 't,1', role: null }, { org: null, title: 't1', role: 'r1' }, { org: 'o', title: null, role: null }]) { + expect(parseOrgKey(orgKey(parts))).toEqual(parts); + } + expect(parseOrgKey('emails:e1')).toBeNull(); + }); +}); + +describe('contacts type tables', () => { + it.each([ + [{ features: { mobile: true }, contexts: { work: true } }, PhoneType.WORK_MOBILE], + [{ features: { cell: true } }, PhoneType.MOBILE], + [{ features: { fax: true }, contexts: { private: true } }, PhoneType.FAX_HOME], + [{ features: { fax: true } }, PhoneType.OTHER_FAX], + [{ features: { pager: true }, contexts: { work: true } }, PhoneType.WORK_PAGER], + [{ features: { textphone: true } }, PhoneType.TTY_TDD], + [{ features: { 'main-number': true } }, PhoneType.MAIN], + [{ features: { voice: true }, contexts: { work: true } }, PhoneType.WORK], + [{ contexts: { private: true } }, PhoneType.HOME], + [{ features: { video: true } }, PhoneType.OTHER], + [{ features: { mobile: true }, label: 'Boat' }, PhoneType.CUSTOM], + ])('phone %j → TYPE %i', (entry, type) => { + expect(phoneType(entry).type).toBe(type); + }); + + it('reads the phone table right to left, OTHER meaning no features', () => { + expect(phoneFlags(PhoneType.OTHER, null)).toEqual({ contexts: [], features: [], label: null }); + expect(phoneFlags(PhoneType.FAX_WORK, null)).toEqual({ contexts: ['work'], features: ['fax'], label: null }); + expect(phoneFlags(PhoneType.CAR, null)).toEqual({ contexts: [], features: ['voice'], label: 'car' }); + }); + + it('maps relations both ways, device-only types to the nearest RFC type', () => { + expect(relationType({ friend: true })).toEqual({ type: RelationType.FRIEND, label: null }); + expect(relationType({ 'co-worker': true })).toEqual({ type: RelationType.CUSTOM, label: 'co-worker' }); + expect(relationType({ agent: true })).toEqual({ type: RelationType.ASSISTANT, label: null }); + expect(relationName(RelationType.BROTHER, null)).toBe('sibling'); + expect(relationName(RelationType.REFERRED_BY, null)).toBe('contact'); + expect(relationName(RelationType.CUSTOM, 'Emergency')).toBe('emergency'); + }); +}); + +describe('contacts name and postal helpers', () => { + it('fills the family name from surname and surname2, the suffix from the first of its kinds, and a missing `full` with deriveFullName', () => { + // deriveFullName (the app's) leaves out surname2 and credential. + expect(nameCells({ components: [{ kind: 'surname', value: 'García' }, { kind: 'surname2', value: 'López' }, { kind: 'credential', value: 'MD' }] })) + .toMatchObject({ data3: 'García López', data6: 'MD', data1: 'García' }); + }); + + it('knows a derived `full` in the app’s and in Stalwart’s form', () => { + const components = [{ kind: 'given', value: 'A' }, { kind: 'surname', value: 'B' }, { kind: 'surname2', value: 'C' }]; + expect(fullIsDerived({ components, full: 'A B' })).toBe(true); + expect(fullIsDerived({ components, full: 'A B C' })).toBe(true); + expect(fullIsDerived({ components, full: 'B, A' })).toBe(false); + }); + + it('keeps separators between street parts and falls back to the legacy flat fields', () => { + expect(postalCells({ street: 'Old Rd 1', locality: 'Town' })).toMatchObject({ data4: 'Old Rd 1', data7: 'Town' }); + const address = { components: [{ kind: 'number', value: '12' }, { kind: 'separator', value: '-' }, { kind: 'name', value: 'Rue X' }] }; + expect(postalCells(address).data4).toBe('12-Rue X'); + expect(editPostalComponents(address, { data4: 'Rue Y 3' }, new Set(['data4']))).toEqual([{ kind: 'name', value: 'Rue Y 3' }]); + }); +}); + +describe('contacts photos', () => { + it('decodes base64 and data URIs, also without a media type', () => { + expect(base64ToBytes('AAECAw==')).toEqual(new Uint8Array([0, 1, 2, 3])); + expect(base64ToBytes('!!')).toBeNull(); + expect(dataUriBase64(`data:image/jpeg;base64,${JPEG}`)).toBe(JPEG); + expect(dataUriBase64(`data:base64,${JPEG}`)).toBe(JPEG); + expect(dataUriBase64('https://example.org/p.jpg')).toBeNull(); + }); + + it('keeps hashes, not bytes, in the shadow and leaves blob-backed entries alone', () => { + const card = { id: 'e1', media: { a: { kind: 'photo', uri: `data:image/jpeg;base64,${JPEG}` }, b: { kind: 'logo', blobId: 'B1' } } } as unknown as ContactCardWire; + const shadow = shadowOf(card); + expect((shadow.media as Record).a.uri).toMatch(/^sha256:[0-9a-f]{64}$/); + expect((shadow.media as Record).b).toEqual({ kind: 'logo', blobId: 'B1' }); + expect((card.media as Record).a.uri).toContain('base64'); + }); +}); + +describe('contacts rows of kinds the planner does not map', () => { + it('ignores them on download and upload', async () => { + const { Harness } = await import('./harness'); + const { appleCard } = await import('./fixtures'); + const h = new Harness(); + const { id, rawId } = await h.seed(appleCard()); + h.device.user.insertData(rawId, { [Data.MIMETYPE]: 'vnd.android.cursor.item/im', data1: 'anna@jabber.example' }); + expect(h.planner.planUpload(await h.contact(rawId), h.ctx).kind).toBe('clean'); + expect(h.planner.planDownload(h.card(id), await h.contact(rawId), h.ctx).effect).toBe('none'); + }); +}); diff --git a/src/device-sync/__tests__/contacts/upload-editors.test.ts b/src/device-sync/__tests__/contacts/upload-editors.test.ts new file mode 100644 index 00000000..9bf3fe16 --- /dev/null +++ b/src/device-sync/__tests__/contacts/upload-editors.test.ts @@ -0,0 +1,358 @@ +import { describe, expect, it } from 'vitest'; +import { Data, MimeType, RawContacts } from '../../android-columns'; +import { contactFingerprint } from '../../contacts/planner'; +import type { Row } from '../../types'; +import { appleCard, googleCard, JPEG2, probeCard } from './fixtures'; +import { ACCOUNT, Harness, JMAP } from './harness'; + +/** + * The rows Fossify Contacts writes back on save: every kind it models + * re-inserted without DATA_SYNC, IS_PRIMARY or the columns it has no field + * for (a department, a job description, a website's type), one nickname and + * one organization only. + */ +function fossifyRows(h: Harness, rawId: number, edit: (rows: Row[]) => void = () => {}): Row[] { + const out: Row[] = []; + let nickname = false; + let organization = false; + for (const r of h.device.rows('data', 'raw_contact_id = ?', [rawId])) { + const pick = (...columns: string[]) => { + const row: Row = { [Data.MIMETYPE]: r[Data.MIMETYPE] }; + for (const c of columns) row[c] = r[c]; + out.push(row); + }; + switch (r[Data.MIMETYPE]) { + case MimeType.NICKNAME: if (!nickname) pick('data1'); nickname = true; break; + case MimeType.PHONE: case MimeType.EMAIL: pick('data1', 'data2', 'data3'); break; + case MimeType.STRUCTURED_POSTAL: pick('data1', 'data2'); break; + case MimeType.EVENT: pick('data1', 'data2'); break; + case MimeType.NOTE: pick('data1'); break; + case MimeType.ORGANIZATION: if (!organization) pick('data1', 'data4'); organization = true; break; + case MimeType.WEBSITE: out.push({ [Data.MIMETYPE]: MimeType.WEBSITE, data1: r.data1, data2: 1 }); break; + case MimeType.GROUP_MEMBERSHIP: pick('data1'); break; + } + } + edit(out); + return out; +} + +async function uploadPlan(h: Harness, rawId: number) { + return h.planner.planUpload(await h.contact(rawId), h.ctx); +} + +async function patchOf(h: Harness, rawId: number) { + const plan = await uploadPlan(h, rawId); + if (plan.kind !== 'upload' || plan.actions[0].kind !== 'update') throw new Error(`expected an update, got ${plan.kind}`); + return plan.actions[0].patch; +} + +describe('contacts upload: Fossify re-inserts rows', () => { + it('uploads nothing for a save without changes, then restores what it dropped', async () => { + const h = new Harness(); + const { rawId } = await h.seed(probeCard()); + h.device.user.fossifySave(rawId, null, fossifyRows(h, rawId)); + const plan = await uploadPlan(h, rawId); + expect(plan.kind).toBe('clean'); + await h.upload(rawId); + const rows = h.rowsByKey(rawId); + expect(rows['organizations:o1|titles:t1,r1']).toMatchObject({ data1: 'ACME', data4: 'Chief Prober', data5: 'R&D', data6: 'Lead' }); + expect(Number(rows['links:l1'].data2)).toBe(5); + expect(Number(rows['emails:work1'].is_primary)).toBe(1); + expect(h.dirty(rawId)).toBe(false); + expect(h.planner.planBaselineHeal(await h.contact(rawId))).toBeNull(); + }); + + it('never clears the department it drops (a value-matched row has no baseline)', async () => { + const h = new Harness(); + const { rawId } = await h.seed(probeCard()); + h.device.user.fossifySave(rawId, null, fossifyRows(h, rawId, (rows) => { + rows.find((r) => r[Data.MIMETYPE] === MimeType.ORGANIZATION)!.data4 = 'Chief Tester'; + })); + expect(await patchOf(h, rawId)).toEqual({ 'titles/t1/name': 'Chief Tester' }); + }); + + it('uploads the one email it changed, matching the others by value', async () => { + const h = new Harness(); + const { rawId } = await h.seed(appleCard()); + h.device.user.fossifySave(rawId, null, fossifyRows(h, rawId, (rows) => { + rows.find((r) => r.data1 === 'anna@work.example')!.data1 = 'anna@new.example'; + })); + expect(await patchOf(h, rawId)).toEqual({ 'emails/k2/address': 'anna@new.example' }); + }); + + it('keeps the second nickname it could not show, and puts it back on the device', async () => { + const h = new Harness(); + const { id, rawId } = await h.seed(googleCard()); + h.device.user.fossifySave(rawId, null, fossifyRows(h, rawId)); + expect((await uploadPlan(h, rawId)).kind).toBe('clean'); + await h.upload(rawId); + expect(h.rowsByKey(rawId)['nicknames:k2'].data1).toBe('GKG'); + expect(h.card(id).nicknames).toEqual(googleCard().nicknames); + }); + + it('does not delete an entry it rewrote away (a documented limitation)', async () => { + const h = new Harness(); + const { rawId } = await h.seed(appleCard()); + h.device.user.fossifySave(rawId, null, fossifyRows(h, rawId, (rows) => { + rows.splice(rows.findIndex((r) => r.data1 === 'anna@work.example'), 1); + })); + expect((await uploadPlan(h, rawId)).kind).toBe('clean'); + }); +}); + +describe('contacts upload: deletions (in-place editor)', () => { + it('resends the map minus the entry, its other entries as the server has them', async () => { + const h = new Harness(); + const { id, rawId } = await h.seed(probeCard()); + h.device.user.deleteData(h.rowsByKey(rawId)['emails:zz9']._id as number); + const { emails } = h.card(id) as { emails: Record }; + expect(await patchOf(h, rawId)).toEqual({ emails: { work1: emails.work1, 'x/y': emails['x/y'] } }); + }); + + it('deletes an address with a null (clean on Stalwart)', async () => { + const h = new Harness(); + const { rawId } = await h.seed(probeCard()); + h.device.user.deleteData(h.rowsByKey(rawId)['addresses:home']._id as number); + expect(await patchOf(h, rawId)).toEqual({ 'addresses/home': null }); + }); + + it('keeps the notes after the first when the note is deleted', async () => { + const h = new Harness(); + const { rawId } = await h.seed(probeCard()); + h.device.user.deleteData(h.rowsByKey(rawId)['notes:n1']._id as number); + expect(await patchOf(h, rawId)).toEqual({ notes: { n2: { note: 'second note' } } }); + }); + + it('removes a map whose last entry went, when other rows show an in-place editor', async () => { + const h = new Harness(); + const { rawId } = await h.seed(probeCard()); + h.device.user.deleteData(h.rowsByKey(rawId)['nicknames:nk']._id as number); + expect(await patchOf(h, rawId)).toEqual({ nicknames: null }); + }); + + it('deletes an organization with its titles', async () => { + const h = new Harness(); + const { rawId } = await h.seed(probeCard()); + h.device.user.deleteData(h.rowsByKey(rawId)['organizations:o1|titles:t1,r1']._id as number); + expect(await patchOf(h, rawId)).toEqual({ organizations: null, titles: null }); + }); + + it('deletes the photo', async () => { + const h = new Harness(); + const { rawId } = await h.seed(probeCard()); + h.device.user.deleteData(h.rowsByKey(rawId)['media:ph']._id as number); + expect(await patchOf(h, rawId)).toEqual({ media: null }); + }); + + it('deletes a relation by resending relatedTo (Stalwart ignores a null there)', async () => { + const h = new Harness(); + const { rawId } = await h.seed(probeCard()); + h.device.user.deleteData(h.rowsByKey(rawId)['relatedTo:urn:uuid:other-person']._id as number); + expect(await patchOf(h, rawId)).toEqual({ relatedTo: null }); + }); + + it('folds an edit into a map it resends', async () => { + const h = new Harness(); + const { id, rawId } = await h.seed(probeCard()); + const rows = h.rowsByKey(rawId); + h.device.user.deleteData(rows['emails:zz9']._id as number); + h.device.user.updateData(rows['emails:work1']._id as number, { data1: 'new@work.example' }); + const { emails } = h.card(id) as unknown as { emails: Record> }; + expect(await patchOf(h, rawId)).toEqual({ emails: { work1: { ...emails.work1, address: 'new@work.example' }, 'x/y': emails['x/y'] } }); + }); +}); + +describe('contacts upload: additions', () => { + it('adds an entry to an existing map under a new key', async () => { + const h = new Harness(); + const { rawId } = await h.seed(probeCard()); + h.device.user.insertData(rawId, { [Data.MIMETYPE]: MimeType.EMAIL, data1: 'added@example.org', data2: 2 }); + const patch = await patchOf(h, rawId); + const [pointer] = Object.keys(patch); + expect(pointer).toMatch(/^emails\/b[0-9a-z]{8}$/); + expect(patch[pointer]).toEqual({ address: 'added@example.org', contexts: { work: true } }); + }); + + it('sends the whole map for the first entry of a map', async () => { + const h = new Harness(); + const { rawId } = await h.seed(appleCard()); + h.device.user.insertData(rawId, { [Data.MIMETYPE]: MimeType.WEBSITE, data1: 'https://anna.example', data2: 7 }); + const patch = await patchOf(h, rawId); + expect(Object.keys(patch)).toEqual(['links']); + expect(Object.values(patch.links as object)).toEqual([{ uri: 'https://anna.example' }]); + }); + + it('adds a work phone with its features and contexts', async () => { + const h = new Harness(); + const { rawId } = await h.seed(probeCard()); + h.device.user.insertData(rawId, { [Data.MIMETYPE]: MimeType.PHONE, data1: '+49 89 1', data2: 3 }); + expect(Object.values(await patchOf(h, rawId))).toEqual([{ number: '+49 89 1', features: { voice: true }, contexts: { work: true } }]); + }); + + it('adds an organization and a title that points at it', async () => { + const h = new Harness(); + const { rawId } = await h.seed(appleCard()); + h.device.user.insertData(rawId, { [Data.MIMETYPE]: MimeType.ORGANIZATION, data1: 'Apple', data4: 'Engineer' }); + const patch = await patchOf(h, rawId) as { organizations: Record; titles: Record> }; + const [orgKey] = Object.keys(patch.organizations); + expect(patch.organizations[orgKey]).toEqual({ name: 'Apple' }); + expect(Object.values(patch.titles)).toEqual([{ name: 'Engineer', kind: 'title', organizationId: orgKey }]); + }); + + it('adds a relation keyed by its name', async () => { + const h = new Harness(); + const { rawId } = await h.seed(probeCard()); + h.device.user.insertData(rawId, { [Data.MIMETYPE]: MimeType.RELATION, data1: 'Jane Doe', data2: 14 }); + expect(await patchOf(h, rawId)).toEqual({ 'relatedTo/Jane Doe': { relation: { spouse: true } } }); + }); + + it('adds a photo from the device as a JPEG data URI', async () => { + const h = new Harness(); + const { rawId } = await h.seed(appleCard()); + h.device.user.setPhoto(rawId, JPEG2); + h.devicePhotos.set(rawId, JPEG2); + const patch = await patchOf(h, rawId); + expect(Object.values(patch.media as object)).toEqual([{ kind: 'photo', uri: `data:image/jpeg;base64,${JPEG2}`, mediaType: 'image/jpeg' }]); + }); + + it('replaces a changed photo in place', async () => { + const h = new Harness(); + const { rawId } = await h.seed(probeCard()); + h.device.user.setPhoto(rawId, JPEG2); + h.devicePhotos.set(rawId, JPEG2); + expect(await patchOf(h, rawId)).toEqual({ 'media/ph/uri': `data:image/jpeg;base64,${JPEG2}`, 'media/ph/mediaType': 'image/jpeg' }); + }); +}); + +describe('contacts upload: new contacts', () => { + function insertNew(h: Harness): number { + return h.device.user.insertContact(ACCOUNT, [ + { [Data.MIMETYPE]: MimeType.STRUCTURED_NAME, data1: 'Neu Person', data2: 'Neu', data3: 'Person' }, + { [Data.MIMETYPE]: MimeType.EMAIL, data1: 'neu@example.org', data2: 1, is_primary: 1 }, + { [Data.MIMETYPE]: MimeType.PHONE, data1: '+49 1', data2: 2 }, + { [Data.MIMETYPE]: MimeType.STRUCTURED_POSTAL, data1: 'Weg 1, 12345 Ort', data2: 1, data4: 'Weg 1', data7: 'Ort', data9: '12345' }, + { [Data.MIMETYPE]: MimeType.EVENT, data1: '1990-01-02', data2: 3 }, + { [Data.MIMETYPE]: MimeType.NOTE, data1: 'hello' }, + { [Data.MIMETYPE]: MimeType.NICKNAME, data1: 'Neuling' }, + ]); + } + + it('claims a uid and target first, then creates the card from its rows', async () => { + const h = new Harness(); + const rawId = insertNew(h); + const claim = h.planner.planUpload(await h.contact(rawId), h.ctx); + expect(claim.kind).toBe('claim'); + await h.applyOk((claim as { ops: import('../../planner').OpGroup }).ops); + const pending = (await h.contact(rawId)).pending!; + expect(pending.target).toBe(`${JMAP}/${h.book}`); + const plan = h.planner.planUpload(await h.contact(rawId), h.ctx); + if (plan.kind !== 'upload' || plan.actions[0].kind !== 'create') throw new Error(plan.kind); + const { object, uid, collectionId } = plan.actions[0]; + expect(uid).toBe(pending.uid); + expect(collectionId).toBe(h.book); + const values = (map: unknown) => Object.values(map as object); + expect(object).toMatchObject({ '@type': 'Card', version: '1.0', uid: pending.uid, addressBookIds: { [h.book]: true } }); + expect(object.name).toEqual({ components: [{ kind: 'given', value: 'Neu' }, { kind: 'surname', value: 'Person' }], full: 'Neu Person' }); + expect(values(object.emails)).toEqual([{ address: 'neu@example.org', contexts: { private: true }, pref: 1 }]); + expect(values(object.phones)).toEqual([{ number: '+49 1', features: { mobile: true } }]); + expect(values(object.addresses)).toEqual([{ + components: [{ kind: 'name', value: 'Weg 1' }, { kind: 'locality', value: 'Ort' }, { kind: 'postcode', value: '12345' }], + full: 'Weg 1, 12345 Ort', + contexts: { private: true }, + }]); + expect(values(object.anniversaries)).toEqual([{ kind: 'birth', date: { '@type': 'PartialDate', year: 1990, month: 1, day: 2 } }]); + expect(values(object.notes)).toEqual([{ note: 'hello' }]); + expect(values(object.nicknames)).toEqual([{ name: 'Neuling' }]); + expect(Object.keys(object).sort()).toEqual([ + '@type', 'addressBookIds', 'addresses', 'anniversaries', 'emails', 'name', 'nicknames', 'notes', 'phones', 'uid', 'version', + ]); + }); + + it('takes the identity after the create and writes nothing when the card comes back', async () => { + const h = new Harness(); + const rawId = insertNew(h); + await h.upload(rawId); + const local = await h.contact(rawId); + expect(local).toMatchObject({ dirty: false, pending: null }); + expect(local.sourceId).toMatch(/^c\//); + expect(local.rows.every((r) => r.key !== null)).toBe(true); + const plan = h.planner.planDownload(h.card(h.cardIdOf(local)), local, h.ctx); + expect(plan.effect).toBe('none'); + }); + + it('is skipped when no writable book is selected', async () => { + const h = new Harness(); + const rawId = insertNew(h); + const plan = h.planner.planUpload(await h.contact(rawId), { ...h.ctx, createTarget: () => null }); + expect(plan).toEqual({ kind: 'skip', reason: 'noWritableAddressBook' }); + }); +}); + +describe('contacts upload: deleted contacts', () => { + it('destroys the card, then purges the rows', async () => { + const h = new Harness(); + const { id, rawId } = await h.seed(appleCard()); + h.device.user.deleteContact(rawId); + const plan = await uploadPlan(h, rawId); + expect(plan).toEqual({ kind: 'upload', actions: [{ kind: 'destroy', id, uid: appleCard().uid }] }); + await h.upload(rawId); + expect(h.device.row('raw_contacts', rawId)).toBeUndefined(); + expect(h.server.get('ContactCard', JMAP, id)).toBeUndefined(); + }); + + it('only leaves the synced book of a card that is also in an unsynced one', async () => { + const h = new Harness(); + const other = h.server.addAddressBook(JMAP, { name: 'Other' }); + h.unselected.add(`${JMAP}/${other}`); + const { rawId } = await h.seed({ ...appleCard(), addressBookIds: { [h.book]: true, [other]: true } }); + h.device.user.deleteContact(rawId); + expect(await patchOf(h, rawId)).toEqual({ [`addressBookIds/${h.book}`]: null }); + }); + + it('purges a new contact that never reached the server, and destroys a claimed one by uid', async () => { + const h = new Harness(); + const fresh = h.device.user.insertContact(ACCOUNT, [{ [Data.MIMETYPE]: MimeType.NOTE, data1: 'x' }]); + h.device.user.deleteContact(fresh); + expect((await uploadPlan(h, fresh)).kind).toBe('purge'); + const claimed = h.device.user.insertContact(ACCOUNT, [{ [Data.MIMETYPE]: MimeType.NOTE, data1: 'y' }]); + await h.planUpload(claimed); + const uid = (await h.contact(claimed)).pending!.uid; + h.device.user.deleteContact(claimed); + expect(await uploadPlan(h, claimed)).toEqual({ kind: 'upload', actions: [{ kind: 'destroy', id: null, uid }] }); + }); +}); + +describe('contacts upload: read-only and poisoned contacts', () => { + it('reverts an edit of a read-only contact from the shadow', async () => { + const h = new Harness(); + h.readOnlyBooks.add(h.book); + const { rawId } = await h.seed(appleCard()); + h.device.user.updateData(h.rowsByKey(rawId)['emails:k1']._id as number, { data1: 'hacked@example.org' }); + const plan = await uploadPlan(h, rawId); + expect(plan.kind).toBe('revert'); + await h.upload(rawId); + expect(h.rowsByKey(rawId)['emails:k1'].data1).toBe('anna@example.com'); + expect(h.dirty(rawId)).toBe(false); + }); + + it('purges a deleted read-only contact and asks for it again', async () => { + const h = new Harness(); + h.readOnlyBooks.add(h.book); + const { rawId } = await h.seed(appleCard()); + h.device.user.deleteContact(rawId); + expect(await uploadPlan(h, rawId)).toMatchObject({ kind: 'revert', refetch: true }); + }); + + it('skips a poisoned contact until it changes or the back-off ends', async () => { + const h = new Harness(); + const { rawId } = await h.seed(appleCard()); + const email = h.rowsByKey(rawId)['emails:k1']._id as number; + h.device.user.updateData(email, { data1: 'bad@example.org' }); + const poison = { fp: contactFingerprint(await h.contact(rawId)), type: 'invalidProperties', n: 1, until: h.ctx.now + 3_600_000 }; + await h.applyOk({ ref: 'x', ops: [{ op: 'update', table: 'raw_contacts', id: rawId, values: { [RawContacts.SYNC4]: JSON.stringify(poison) } }] }); + expect(await uploadPlan(h, rawId)).toEqual({ kind: 'skip', reason: 'poisoned:invalidProperties' }); + expect(h.planner.planUpload(await h.contact(rawId), { ...h.ctx, now: poison.until + 1 }).kind).toBe('upload'); + h.device.user.updateData(email, { data1: 'good@example.org' }); + expect((await uploadPlan(h, rawId)).kind).toBe('upload'); + }); +}); diff --git a/src/device-sync/__tests__/contacts/upload-fields.test.ts b/src/device-sync/__tests__/contacts/upload-fields.test.ts new file mode 100644 index 00000000..62fb106d --- /dev/null +++ b/src/device-sync/__tests__/contacts/upload-fields.test.ts @@ -0,0 +1,229 @@ +import { describe, expect, it } from 'vitest'; +import { Data, MimeType } from '../../android-columns'; +import { deepEqual } from '../../common/json'; +import { applyPatch, assertPatchWellFormed, type PatchObject } from '../../common/patch'; +import type { ContactCardWire } from '../../planner'; +import type { Row } from '../../types'; +import { appleCard, googleCard, probeCard } from './fixtures'; +import { Harness } from './harness'; + +/** Card data Android has no column for: every patch must leave it as it was. */ +const UNMAPPED = [ + 'onlineServices', 'speakToAs', 'keywords', 'personalInfo', 'preferredLanguages', 'notes/n2', 'name/phoneticSystem', + 'addresses/home/coordinates', 'addresses/home/timeZone', 'addresses/home/countryCode', 'kind', 'uid', +]; + +function at(card: unknown, path: string): unknown { + return path.split('/').reduce((v, k) => (v && typeof v === 'object' ? (v as Record)[k] : undefined), card); +} + +/** Seeds a card, edits one row in place like AOSP Contacts, and returns the upload patch. */ +async function editedPatch(make: () => Record, key: string, values: Row): Promise<{ patch: PatchObject; card: ContactCardWire }> { + const h = new Harness(); + const { id, rawId } = await h.seed(make()); + const row = h.rowsByKey(rawId)[key]; + if (!row) throw new Error(`no row ${key}`); + h.device.user.updateData(row._id as number, values); + const plan = h.planner.planUpload(await h.contact(rawId), h.ctx); + if (plan.kind !== 'upload' || plan.actions[0].kind !== 'update') throw new Error(`expected an update, got ${plan.kind}`); + return { patch: plan.actions[0].patch, card: h.card(id) }; +} + +const probeEdits: Array<[string, string, Row, PatchObject]> = [ + ['email address', 'emails:work1', { data1: 'new@work.example' }, { 'emails/work1/address': 'new@work.example' }], + ['email type (home → work)', 'emails:zz9', { data2: 2 }, { 'emails/zz9/contexts': { work: true } }], + ['email custom label', 'emails:work1', { data3: 'Desk' }, { 'emails/work1/label': 'Desk' }], + ['a key with a slash', 'emails:x/y', { data1: 'slash2@example.org' }, { 'emails/x~1y/address': 'slash2@example.org' }], + ['phone number', 'phones:p-a', { data1: '+49 30 5555' }, { 'phones/p-a/number': '+49 30 5555' }], + ['phone type (mobile → work mobile)', 'phones:p-a', { data2: 17 }, { 'phones/p-a/contexts': { work: true }, 'phones/p-a/features': { mobile: true } }], + ['nickname', 'nicknames:nk', { data1: 'Probster' }, { 'nicknames/nk/name': 'Probster' }], + ['street', 'addresses:home', { data4: 'Hauptstr. 1' }, { + 'addresses/home/components': [ + { kind: 'name', value: 'Hauptstr. 1' }, + { kind: 'locality', value: 'Berlin' }, + { kind: 'postcode', value: '10115' }, + { kind: 'country', value: 'Germany' }, + ], + }], + ['city', 'addresses:home', { data7: 'Potsdam' }, { + 'addresses/home/components': [ + { kind: 'name', value: 'Main St' }, { kind: 'separator', value: ' ' }, { kind: 'number', value: '12' }, + { kind: 'locality', value: 'Potsdam' }, { kind: 'postcode', value: '10115' }, { kind: 'country', value: 'Germany' }, + { kind: 'apartment', value: '4b' }, + ], + }], + ['region, which the address did not have', 'addresses:home', { data8: 'Berlin' }, { + 'addresses/home/components': [ + { kind: 'name', value: 'Main St' }, { kind: 'separator', value: ' ' }, { kind: 'number', value: '12' }, + { kind: 'locality', value: 'Berlin' }, { kind: 'region', value: 'Berlin' }, { kind: 'postcode', value: '10115' }, + { kind: 'country', value: 'Germany' }, { kind: 'apartment', value: '4b' }, + ], + }], + ['formatted address', 'addresses:home', { data1: 'Main St 12\n10115 Berlin' }, { 'addresses/home/full': 'Main St 12\n10115 Berlin' }], + ['address type (home → work)', 'addresses:home', { data2: 2 }, { 'addresses/home/contexts': { work: true } }], + ['company', 'organizations:o1|titles:t1,r1', { data1: 'ACME Corp' }, { 'organizations/o1/name': 'ACME Corp' }], + ['department', 'organizations:o1|titles:t1,r1', { data5: 'QA, Tools' }, { 'organizations/o1/units': [{ name: 'QA' }, { name: 'Tools' }] }], + ['job title', 'organizations:o1|titles:t1,r1', { data4: 'Chief Tester' }, { 'titles/t1/name': 'Chief Tester' }], + ['job description (role)', 'organizations:o1|titles:t1,r1', { data6: 'Mentor' }, { 'titles/r1/name': 'Mentor' }], + ['website', 'links:l1', { data1: 'https://probe2.example' }, { 'links/l1/uri': 'https://probe2.example' }], + ['website type (work → home)', 'links:l1', { data2: 4 }, { 'links/l1/contexts': { private: true } }], + ['birthday date', 'anniversaries:bday', { data1: '--03-15' }, { 'anniversaries/bday/date': { '@type': 'PartialDate', month: 3, day: 15 } }], + ['anniversary with a year', 'anniversaries:wed', { data1: '2011-06-01' }, { 'anniversaries/wed/date': { '@type': 'PartialDate', year: 2011, month: 6, day: 1 } }], + ['anniversary type (birthday → other)', 'anniversaries:bday', { data2: 2 }, { 'anniversaries/bday/kind': 'other' }], + ['relation type (friend → spouse)', 'relatedTo:urn:uuid:other-person', { data2: 14 }, { 'relatedTo/urn:uuid:other-person/relation': { spouse: true } }], + ['relation type without an RFC twin (brother)', 'relatedTo:urn:uuid:other-person', { data2: 2 }, { 'relatedTo/urn:uuid:other-person/relation': { sibling: true } }], + ['note', 'notes:n1', { data1: 'edited note' }, { 'notes/n1/note': 'edited note' }], +]; + +describe('contacts upload: one edited field, one pointer', () => { + it.each(probeEdits)('%s', async (_name, key, values, expected) => { + const { patch, card } = await editedPatch(probeCard, key, values); + expect(patch).toEqual(expected); + assertPatchWellFormed(patch); + const applied = applyPatch(card, patch); + expect(applied).not.toBeNull(); + for (const path of UNMAPPED) expect(deepEqual(at(applied, path), at(card, path))).toBe(true); + }); + + it('moves IS_PRIMARY as pref', async () => { + const h = new Harness(); + const { rawId } = await h.seed(probeCard()); + const rows = h.rowsByKey(rawId); + h.device.user.updateData(rows['emails:x/y']._id as number, { is_primary: 1 }); + h.device.user.updateData(rows['emails:work1']._id as number, { is_primary: 0 }); + const plan = h.planner.planUpload(await h.contact(rawId), h.ctx); + expect(plan).toMatchObject({ kind: 'upload', actions: [{ kind: 'update', patch: { 'emails/x~1y/pref': 1, 'emails/work1/pref': null } }] }); + }); + + it('renames a relation by moving it to the new key (relatedTo is keyed by value)', async () => { + const { patch } = await editedPatch(probeCard, 'relatedTo:urn:uuid:other-person', { data1: 'Jane Roe' }); + expect(patch).toEqual({ relatedTo: { 'Jane Roe': { relation: { friend: true } } } }); + }); + + it('keeps a free-text date the server cannot take on the device only', async () => { + const h = new Harness(); + const { rawId } = await h.seed(probeCard()); + h.device.user.updateData(h.rowsByKey(rawId)['anniversaries:bday']._id as number, { data1: 'next spring' }); + expect(h.planner.planUpload(await h.contact(rawId), h.ctx).kind).toBe('clean'); + }); +}); + +describe('contacts upload: the name', () => { + const NAME_KEY = 'name'; + + it('re-derives a derived `full` when a component changes (and the provider recomputed the display name)', async () => { + const { patch } = await editedPatch(probeCard, NAME_KEY, { data2: 'Probert', data1: 'Dr. Probert Person' }); + expect(patch).toEqual({ + 'name/components': [ + { kind: 'title', value: 'Dr.' }, + { kind: 'given', value: 'Probert', phonetic: 'Proob' }, + { kind: 'surname', value: 'Person' }, + ], + 'name/full': 'Dr. Probert Person', + }); + }); + + it('uploads a typed display name as `full` alone', async () => { + const { patch } = await editedPatch(probeCard, NAME_KEY, { data1: 'Probe P.' }); + expect(patch).toEqual({ 'name/full': 'Probe P.' }); + }); + + it('leaves a custom `full` alone when only components change', async () => { + const { patch } = await editedPatch(googleCard, NAME_KEY, { data3: 'Gross', data1: 'Prof. Günther Karl Gross, PhD' }); + expect(patch).toEqual({ + 'name/components': [ + { kind: 'title', value: 'Prof.' }, + { kind: 'given', value: 'Günther' }, + { kind: 'given2', value: 'Karl' }, + { kind: 'surname', value: 'Gross' }, + { kind: 'credential', value: 'PhD' }, + ], + }); + }); + + it('adds a component the name did not have in display order', async () => { + const { patch } = await editedPatch(appleCard, NAME_KEY, { data5: 'Maria', data1: 'Anna Maria Apfel' }); + expect(patch).toEqual({ + 'name/components': [{ kind: 'given', value: 'Anna' }, { kind: 'given2', value: 'Maria' }, { kind: 'surname', value: 'Apfel' }], + 'name/full': 'Anna Maria Apfel', + }); + }); + + it('removes a component whose column was emptied', async () => { + const { patch } = await editedPatch(probeCard, NAME_KEY, { data4: null, data1: 'Probe Person' }); + expect(patch['name/components']).toEqual([{ kind: 'given', value: 'Probe', phonetic: 'Proob' }, { kind: 'surname', value: 'Person' }]); + expect(patch['name/full']).toBe('Probe Person'); + }); + + it('uploads a phonetic name as the component phonetic', async () => { + const { patch } = await editedPatch(probeCard, NAME_KEY, { data9: 'Pörson' }); + expect(patch).toEqual({ + 'name/components': [ + { kind: 'title', value: 'Dr.' }, + { kind: 'given', value: 'Probe', phonetic: 'Proob' }, + { kind: 'surname', value: 'Person', phonetic: 'Pörson' }, + ], + }); + }); + + it('uploads a retyped name the server holds as `full` alone whole: `full` and every component', async () => { + const h = new Harness(); + const { id, rawId } = await h.seed({ name: { full: 'Ada' } }); + // A typed display name: the provider splits it (the fake does so when no component is given). + h.device.user.updateData(h.rowsByKey(rawId).name._id as number, { data1: 'Ada King', data2: null, data3: null, data4: null, data5: null, data6: null }); + expect(h.rowsByKey(rawId).name).toMatchObject({ data1: 'Ada King', data2: 'Ada', data3: 'King' }); + const plan = h.planner.planUpload(await h.contact(rawId), h.ctx); + expect(plan).toMatchObject({ + kind: 'upload', + actions: [{ patch: { 'name/components': [{ kind: 'given', value: 'Ada' }, { kind: 'surname', value: 'King' }], 'name/full': 'Ada King' } }], + }); + await h.upload(rawId); + expect(h.card(id).name).toMatchObject({ full: 'Ada King', components: [{ kind: 'given', value: 'Ada' }, { kind: 'surname', value: 'King' }] }); + const local = await h.contact(rawId); + expect(local.dirty).toBe(false); + expect(h.rowsByKey(rawId).name).toMatchObject({ data1: 'Ada King', data2: 'Ada', data3: 'King' }); + expect(h.planner.planDownload(h.card(id), local, h.ctx).effect).toBe('none'); + }); + + it('re-derives a derived `full` when a retyped display name adds a component', async () => { + const h = new Harness(); + const { id, rawId } = await h.seed({ name: { components: [{ kind: 'given', value: 'Ada' }], full: 'Ada' } }); + h.device.user.updateData(h.rowsByKey(rawId).name._id as number, { data1: 'Ada King', data2: null, data3: null, data4: null, data5: null, data6: null }); + await h.upload(rawId); + expect(h.card(id).name).toMatchObject({ full: 'Ada King', components: [{ kind: 'given', value: 'Ada' }, { kind: 'surname', value: 'King' }] }); + }); + + it('drops surname2 only when the family name changed', async () => { + const card = () => ({ name: { components: [{ kind: 'given', value: 'Ana' }, { kind: 'surname', value: 'García' }, { kind: 'surname2', value: 'López' }], full: 'Ana García López' } }); + const h = new Harness(); + const { rawId } = await h.seed(card()); + expect(h.rowsByKey(rawId).name.data3).toBe('García López'); + const { patch } = await editedPatch(card, NAME_KEY, { data3: 'García Pérez', data1: 'Ana García Pérez' }); + expect(patch['name/components']).toEqual([{ kind: 'given', value: 'Ana' }, { kind: 'surname', value: 'García Pérez' }]); + }); +}); + +describe('contacts upload: nothing mapped changed', () => { + it.each([['probe', probeCard], ['apple', appleCard], ['google', googleCard]])('clears DIRTY without a request for a starred %s card', async (_n, make) => { + const h = new Harness(); + const { rawId } = await h.seed(make()); + const before = JSON.stringify(h.device.rows('data', 'raw_contact_id = ?', [rawId])); + h.device.user.star(rawId); + const plan = h.planner.planUpload(await h.contact(rawId), h.ctx); + expect(plan.kind).toBe('clean'); + if (plan.kind !== 'clean') return; + expect(plan.ops.ops.filter((o) => o.op !== 'assert' && o.op !== 'syncState' && o.table === 'data')).toEqual([]); + await h.applyOk(plan.ops); + expect(h.dirty(rawId)).toBe(false); + expect(JSON.stringify(h.device.rows('data', 'raw_contact_id = ?', [rawId]))).toBe(before); + }); + + it('never uploads IS_PRIMARY of a row an editor re-inserted', async () => { + const h = new Harness(); + const { rawId } = await h.seed(appleCard()); + const rows = h.rowsByKey(rawId); + h.device.user.deleteData(rows['emails:k1']._id as number); + h.device.user.insertData(rawId, { [Data.MIMETYPE]: MimeType.EMAIL, [Data.DATA1]: 'anna@example.com', [Data.DATA2]: 1, is_primary: 0 }); + expect(h.planner.planUpload(await h.contact(rawId), h.ctx).kind).toBe('clean'); + }); +}); diff --git a/src/device-sync/__tests__/engine/real-planners.test.ts b/src/device-sync/__tests__/engine/real-planners.test.ts new file mode 100644 index 00000000..7c3c00c6 --- /dev/null +++ b/src/device-sync/__tests__/engine/real-planners.test.ts @@ -0,0 +1,221 @@ +// The engine with the real contacts and calendar planners (not the toy +// ones): the contract between them holds end to end. Mapping details are +// the planners' own tests; these check what the engine guarantees. + +import { describe, expect, it } from 'vitest'; +import { Data, MimeType } from '../../android-columns'; +import { calendarPlanner } from '../../calendar/planner'; +import { contactsPlanner } from '../../contacts/planner'; +import { + addDeviceContact, + addServerCards, + CALENDAR_AUTHORITY, + createHarness, + rowWrites, + type Harness, +} from './harness'; + +function real(options: Parameters[0] = {}): Harness { + const h = createHarness(options); + h.deps.planners = { contacts: contactsPlanner, calendar: calendarPlanner }; + return h; +} + +/** A device edit of the display name only, as AOSP Contacts makes it (the provider fills the components). */ +function retypeName(h: Harness, rawContactId: number, name: string): void { + const row = h.nameDataRow(rawContactId)!; + h.device.user.updateData(Number(row._id), { [Data.DATA1]: name }); +} + +function serverUids(h: Harness): string[] { + return h.server.all('ContactCard', 'a').map((c) => String(c.uid)); +} + +describe('device sync engine with the real planners', () => { + it('syncs contacts both ways and writes nothing for its own echo', async () => { + const h = real(); + const [, grace] = addServerCards(h, ['Ada Lovelace', 'Grace Hopper']); + expect((await h.run()).outcome).toBe('ok'); + expect(h.contacts().map((c) => c.name).sort()).toEqual(['Ada Lovelace', 'Grace Hopper']); + + addDeviceContact(h, 'Hedy Lamarr'); + h.device.user.deleteContact(h.contactNamed('Grace Hopper')!.id); + const up = await h.run(); + expect(up).toMatchObject({ outcome: 'ok', itemErrors: [] }); + expect(up.stats.uploaded).toMatchObject({ created: 1, deleted: 1 }); + expect(h.server.get('ContactCard', 'a', grace)).toBeUndefined(); + expect(h.contacts().every((c) => c.sourceId && !c.dirty)).toBe(true); + expect(new Set(serverUids(h)).size).toBe(2); + + const before = h.batches.log.length; + const echo = await h.run(); + expect(echo.outcome).toBe('ok'); + expect(rowWrites(h.batches.log.slice(before))).toEqual([]); + }); + + it('converges with the real contacts planner after a crash at any checkpoint', async () => { + const setup = async () => { + const h = real(); + addServerCards(h, ['Ada Lovelace', 'Grace Hopper', 'Linus Torvalds']); + await h.run(); + retypeName(h, h.contactNamed('Grace Hopper')!.id, 'Grace B. Hopper'); + addDeviceContact(h, 'Hedy Lamarr'); + h.device.user.deleteContact(h.contactNamed('Linus Torvalds')!.id); + h.server.serverUpdate('ContactCard', 'a', h.contactNamed('Ada Lovelace')!.sourceId!.split('/')[1], { 'name/full': 'Ada King' }); + addServerCards(h, ['Margaret Hamilton']); + h.checkpoints.count = 0; + return h; + }; + const probe = await setup(); + await probe.run(); + const final = probe.contacts().map((c) => c.name).sort(); + const finalServer = probe.serverNames(); + const checkpoints = probe.checkpoints.count; + + for (let n = 1; n <= checkpoints; n++) { + const h = await setup(); + h.checkpoints.crashAt = n; + await h.run(); + h.checkpoints.crashAt = null; + const report = await h.run(); + expect(report.outcome, `crash at checkpoint ${n}`).toBe('ok'); + expect(h.contacts().map((c) => c.name).sort(), `device after crash ${n}`).toEqual(final); + expect(h.serverNames(), `server after crash ${n}`).toEqual(finalServer); + expect(h.contacts().every((c) => !c.dirty && !c.deleted)).toBe(true); + expect(new Set(serverUids(h)).size).toBe(serverUids(h).length); + } + }); + + it('adopts a lost create by uid with the real contacts planner', async () => { + const h = real(); + addServerCards(h, ['Ada Lovelace']); + await h.run(); + h.server.setUidIndexLag(3); + const id = addDeviceContact(h, 'Hedy Lamarr'); + h.server.applyThenLoseResponse({ match: 'ContactCard/set' }); + + expect((await h.run()).outcome).toBe('io'); + expect((await h.run()).outcome).toBe('ok'); + + expect(h.serverNames()).toEqual(['Ada Lovelace', 'Hedy Lamarr']); + expect(h.contacts()).toHaveLength(2); + expect(h.contacts().find((c) => c.id === id)).toMatchObject({ dirty: false, sourceId: expect.stringMatching(/^a\//) }); + expect(h.device.rows('data').filter((d) => d.mimetype === MimeType.STRUCTURED_NAME)).toHaveLength(2); + }); + + it("follows a group's members when only the group card changed on the server", async () => { + const h = real(); + const [ada, grace] = addServerCards(h, ['Ada Lovelace', 'Grace Hopper']); + const uid = (id: string) => h.server.get('ContactCard', 'a', id)!.uid as string; + const group = h.server.addCard('a', { uid: 'g1', kind: 'group', name: { full: 'Friends' }, members: { [uid(ada)]: true } }); + const members = () => + h.device + .rows('data') + .filter((d) => d.mimetype === MimeType.GROUP_MEMBERSHIP) + .map((d) => h.contacts().find((c) => c.id === Number(d.raw_contact_id))?.name) + .sort(); + expect((await h.run()).outcome).toBe('ok'); + expect(members()).toEqual(['Ada Lovelace']); + + h.server.serverUpdate('ContactCard', 'a', group, { members: { [uid(grace)]: true } }); + expect((await h.run()).outcome).toBe('ok'); + + expect(members()).toEqual(['Grace Hopper']); + expect(h.contacts().every((c) => !c.dirty)).toBe(true); + }); + + it('creates a device event that carries the uid of a synced event (an app copied it) under a fresh uid', async () => { + const h = real(); + const standup = h.server.addEvent('a', { + uid: 'standup-uid', + title: 'Standup', + start: '2026-09-28T09:00:00', + duration: 'PT15M', + timeZone: 'Europe/Berlin', + calendarIds: { [h.calendar]: true }, + }); + await h.run(CALENDAR_AUTHORITY); + const rowId = Number(h.device.rows('calendars')[0]._id); + + const copy = h.device.user.insertEvent(rowId, { + title: 'Standup (copy)', + dtstart: Date.UTC(2026, 8, 29, 7), + dtend: Date.UTC(2026, 8, 29, 7, 15), + eventTimezone: 'Europe/Berlin', + uid2445: 'standup-uid', + }); + const report = await h.run(CALENDAR_AUTHORITY); + + expect(report).toMatchObject({ outcome: 'ok', itemErrors: [], stats: { uploaded: { created: 1 } } }); + expect(h.server.get('CalendarEvent', 'a', standup)).toMatchObject({ title: 'Standup', uid: 'standup-uid' }); + const created = h.server.all('CalendarEvent', 'a').find((e) => e.title === 'Standup (copy)')!; + expect(created.uid).not.toBe('standup-uid'); + expect(h.events().find((e) => Number(e._id) === copy)).toMatchObject({ _sync_id: `a/${created.id}`, uid2445: created.uid }); + expect(h.events().every((e) => Number(e.dirty ?? 0) === 0)).toBe(true); + }); + + it('uploads "this and following" in one run: the capped rule and the new series', async () => { + const h = real(); + const series = h.server.addEvent('a', { + uid: 'series-uid', + title: 'Standup', + start: '2026-09-28T09:00:00', + duration: 'PT15M', + timeZone: 'Europe/Berlin', + recurrenceRule: { '@type': 'RecurrenceRule', frequency: 'daily' }, + calendarIds: { [h.calendar]: true }, + }); + await h.run(CALENDAR_AUTHORITY); + const master = h.events()[0]; + expect(master).toMatchObject({ rrule: 'FREQ=DAILY' }); + const rowId = Number(master.calendar_id); + + // Etar: the master's rule ends before the split, a new series starts there (it copies the uid). + h.device.user.updateEvent(Number(master._id), { rrule: 'FREQ=DAILY;UNTIL=20260930T235959Z' }); + const later = h.device.user.insertEvent(rowId, { + title: 'Standup', + dtstart: Date.UTC(2026, 9, 1, 7), + duration: 'P900S', + rrule: 'FREQ=DAILY', + eventTimezone: 'Europe/Berlin', + uid2445: 'series-uid', + }); + const report = await h.run(CALENDAR_AUTHORITY); + + expect(report).toMatchObject({ outcome: 'ok', itemErrors: [], stats: { uploaded: { created: 1, updated: 1 } } }); + const capped = h.server.get('CalendarEvent', 'a', series)!; + expect(capped.recurrenceRule).toMatchObject({ frequency: 'daily', until: expect.any(String) }); + const next = h.server.all('CalendarEvent', 'a').find((e) => e.id !== series)!; + expect(next).toMatchObject({ start: '2026-10-01T09:00:00', timeZone: 'Europe/Berlin', recurrenceRule: { frequency: 'daily' } }); + expect(next.uid).not.toBe('series-uid'); + expect(h.events().find((e) => Number(e._id) === later)).toMatchObject({ _sync_id: `a/${next.id}` }); + expect(h.events().every((e) => Number(e.dirty ?? 0) === 0)).toBe(true); + }); + + it('syncs events both ways and writes nothing for its own echo', async () => { + const h = real(); + const standup = h.server.addEvent('a', { + uid: 'e1', + title: 'Standup', + start: '2026-09-28T09:00:00', + duration: 'PT15M', + timeZone: 'Europe/Berlin', + calendarIds: { [h.calendar]: true }, + }); + expect((await h.run(CALENDAR_AUTHORITY)).outcome).toBe('ok'); + const rowId = Number(h.device.rows('calendars')[0]._id); + + h.device.user.insertEvent(rowId, { title: 'Lunch', dtstart: Date.UTC(2026, 8, 28, 10), dtend: Date.UTC(2026, 8, 28, 11), eventTimezone: 'Europe/Berlin' }); + h.device.user.updateEvent(Number(h.events().find((e) => e.title === 'Standup')!._id), { title: 'Daily standup' }); + const up = await h.run(CALENDAR_AUTHORITY); + + expect(up).toMatchObject({ outcome: 'ok', itemErrors: [], stats: { uploaded: { created: 1, updated: 1 } } }); + expect(h.server.get('CalendarEvent', 'a', standup)).toMatchObject({ title: 'Daily standup' }); + expect(h.server.all('CalendarEvent', 'a').find((e) => e.title === 'Lunch')).toMatchObject({ start: '2026-09-28T12:00:00', timeZone: 'Europe/Berlin' }); + expect(h.events().every((e) => Number(e.dirty ?? 0) === 0 && String(e._sync_id).startsWith('a/'))).toBe(true); + + const before = h.batches.log.length; + expect((await h.run(CALENDAR_AUTHORITY)).outcome).toBe('ok'); + expect(rowWrites(h.batches.log.slice(before))).toEqual([]); + }); +}); diff --git a/src/device-sync/contacts/cells.ts b/src/device-sync/contacts/cells.ts new file mode 100644 index 00000000..daa2ee3c --- /dev/null +++ b/src/device-sync/contacts/cells.ts @@ -0,0 +1,76 @@ +/** + * Data-row cells as the contacts planner compares them. The bridge hands + * numbers back as numbers or text, "" and NULL mean the same (the design's + * rule for names, and what editors do to every field), and IS_PRIMARY 0 is + * the same as unset. Baselines (DATA_SYNC3) keep text over 1 KB as its hash. + */ +import { sha256Hex } from '../../lib/sha256'; +import { Data } from '../android-columns'; +import { parseJsonColumn, utf8 } from '../common/json'; +import type { Cell, Row } from '../types'; + +const LONG_TEXT = 1024; +const HASH_PREFIX = '#sha256:'; + +/** Comparable text of a cell; null for NULL, "" and anything that is not a cell. */ +export function cellText(value: unknown, column?: string): string | null { + let text: string | null = null; + if (typeof value === 'string') text = value === '' ? null : value; + else if (typeof value === 'number' && Number.isFinite(value)) text = String(value); + if (column === Data.IS_PRIMARY && text === '0') return null; + return text; +} + +export function sameCell(a: unknown, b: unknown, column?: string): boolean { + return cellText(a, column) === cellText(b, column); +} + +function baselineValue(value: unknown, column: string): string | null { + const text = cellText(value, column); + if (text !== null && text.length > LONG_TEXT) return `${HASH_PREFIX}${sha256Hex(utf8(text))}`; + return text; +} + +/** DATA_SYNC3 of a row: its mapped cells, long text hashed. */ +export function encodeBaseline(cells: Row, columns: readonly string[]): string { + const out: Record = {}; + for (const c of columns) if (c in cells) out[c] = baselineValue(cells[c], c); + return JSON.stringify(out); +} + +/** A baseline read back from DATA_SYNC3; null when another app left something else there. */ +export function decodeBaseline(text: unknown): Row | null { + const parsed = parseJsonColumn(text); + if (!parsed || typeof parsed !== 'object' || Array.isArray(parsed)) return null; + const out: Row = {}; + for (const [k, v] of Object.entries(parsed as Record)) { + if (v !== null && typeof v !== 'string' && typeof v !== 'number') return null; + out[k] = v as Cell; + } + return out; +} + +/** Whether a cell still holds what the baseline recorded for it. */ +function matchesBaseline(value: unknown, baseline: Row, column: string): boolean { + const recorded = cellText(baseline[column], column); + if (recorded !== null && recorded.startsWith(HASH_PREFIX)) return baselineValue(value, column) === recorded; + return cellText(value, column) === recorded; +} + +/** Mapped columns that differ from the baseline; columns the baseline never recorded count as unchanged. */ +export function changedFromBaseline(cells: Row, baseline: Row, columns: readonly string[]): string[] { + return columns.filter((c) => c in baseline && !matchesBaseline(cells[c], baseline, c)); +} + +/** The cell as text for building card values; null when blank. */ +export function text(value: unknown): string | null { + const t = cellText(value); + return t === null || t.trim() === '' ? null : t; +} + +export function num(value: unknown): number | null { + const t = cellText(value); + if (t === null) return null; + const n = Number(t); + return Number.isFinite(n) ? n : null; +} diff --git a/src/device-sync/contacts/columns.ts b/src/device-sync/contacts/columns.ts new file mode 100644 index 00000000..87701327 --- /dev/null +++ b/src/device-sync/contacts/columns.ts @@ -0,0 +1,113 @@ +/** + * The columns the engine queries for contacts, and the decoders that turn + * rows into `LocalContact`/`LocalGroup`. Sync columns are read defensively: + * any app holding WRITE_CONTACTS can write them, so what does not parse is + * treated as absent. + */ +import { Data, GroupMembership, Groups, RawContacts } from '../android-columns'; +import { parseCollectionKey } from '../common/ids'; +import { parseJsonColumn } from '../common/json'; +import type { ContactCardWire, LocalContact, LocalDataRow, LocalGroup, PendingCreate, PoisonMarker } from '../planner'; +import type { Row } from '../types'; +import { cellText, decodeBaseline, num } from './cells'; + +const DATA_CELLS = [ + Data.IS_PRIMARY, Data.DATA1, Data.DATA2, Data.DATA3, Data.DATA4, Data.DATA5, Data.DATA6, Data.DATA7, + Data.DATA8, Data.DATA9, Data.DATA10, Data.DATA11, Data.DATA12, Data.DATA13, Data.DATA14, + GroupMembership.GROUP_SOURCE_ID, +]; + +export const RAW_CONTACT_COLUMNS: readonly string[] = [ + RawContacts._ID, RawContacts.SOURCE_ID, RawContacts.VERSION, RawContacts.DIRTY, RawContacts.DELETED, + RawContacts.SYNC1, RawContacts.SYNC2, RawContacts.SYNC3, RawContacts.SYNC4, RawContacts.RAW_CONTACT_IS_READ_ONLY, +]; + +export const DATA_COLUMNS: readonly string[] = [ + Data._ID, Data.RAW_CONTACT_ID, Data.MIMETYPE, ...DATA_CELLS, Data.DATA_SYNC1, Data.DATA_SYNC2, Data.DATA_SYNC3, +]; + +export const GROUP_COLUMNS: readonly string[] = [ + Groups._ID, Groups.SOURCE_ID, Groups.VERSION, Groups.DIRTY, Groups.DELETED, Groups.TITLE, + Groups.SYNC2, Groups.SYNC3, Groups.SYNC4, +]; + +const flag = (v: unknown) => num(v) === 1; + +function stringOrNull(v: unknown): string | null { + return typeof v === 'string' && v !== '' ? v : null; +} + +export function decodeShadow(v: unknown): ContactCardWire | null { + const card = parseJsonColumn(v); + return card && typeof card === 'object' && !Array.isArray(card) && typeof (card as { id?: unknown }).id === 'string' + ? (card as ContactCardWire) + : null; +} + +export function decodePending(v: unknown): PendingCreate | null { + const p = parseJsonColumn>(v); + if (!p || typeof p !== 'object' || typeof p.uid !== 'string' || !p.uid || typeof p.target !== 'string') return null; + return parseCollectionKey(p.target) ? { uid: p.uid, target: p.target } : null; +} + +export function decodePoison(v: unknown): PoisonMarker | null { + const p = parseJsonColumn>(v); + if (!p || typeof p !== 'object' || typeof p.fp !== 'string' || typeof p.type !== 'string') return null; + if (typeof p.n !== 'number' || typeof p.until !== 'number') return null; + return { + fp: p.fp, + type: p.type, + n: p.n, + until: p.until, + ...(typeof p.description === 'string' ? { description: p.description } : {}), + }; +} + +function decodeDataRow(row: Row): LocalDataRow { + const cells: Row = {}; + for (const c of DATA_CELLS) if (c in row) cells[c] = row[c]; + return { + id: Number(row[Data._ID]), + mimetype: String(row[Data.MIMETYPE] ?? ''), + cells, + key: stringOrNull(row[Data.DATA_SYNC1]), + photoHash: stringOrNull(row[Data.DATA_SYNC2]), + baseline: decodeBaseline(row[Data.DATA_SYNC3]), + }; +} + +export function decodeContact(rawContact: Row, data: Row[]): LocalContact { + const id = Number(rawContact[RawContacts._ID]); + const own = data.filter((d) => !(Data.RAW_CONTACT_ID in d) || Number(d[Data.RAW_CONTACT_ID]) === id); + const collections = (cellText(rawContact[RawContacts.SYNC1]) ?? '') + .split(',') + .map((s) => s.trim()) + .filter((s) => parseCollectionKey(s)); + return { + rawContactId: id, + sourceId: stringOrNull(rawContact[RawContacts.SOURCE_ID]), + version: num(rawContact[RawContacts.VERSION]) ?? 0, + dirty: flag(rawContact[RawContacts.DIRTY]), + deleted: flag(rawContact[RawContacts.DELETED]), + readOnly: flag(rawContact[RawContacts.RAW_CONTACT_IS_READ_ONLY]), + collections, + shadow: decodeShadow(rawContact[RawContacts.SYNC2]), + pending: decodePending(rawContact[RawContacts.SYNC3]), + poison: decodePoison(rawContact[RawContacts.SYNC4]), + rows: own.map(decodeDataRow), + }; +} + +export function decodeGroup(group: Row): LocalGroup { + return { + groupId: Number(group[Groups._ID]), + sourceId: stringOrNull(group[Groups.SOURCE_ID]), + version: num(group[Groups.VERSION]) ?? 0, + dirty: flag(group[Groups.DIRTY]), + deleted: flag(group[Groups.DELETED]), + title: typeof group[Groups.TITLE] === 'string' ? (group[Groups.TITLE] as string) : null, + shadow: decodeShadow(group[Groups.SYNC2]), + pending: decodePending(group[Groups.SYNC3]), + poison: decodePoison(group[Groups.SYNC4]), + }; +} diff --git a/src/device-sync/contacts/diff.ts b/src/device-sync/contacts/diff.ts new file mode 100644 index 00000000..e0dac5d5 --- /dev/null +++ b/src/device-sync/contacts/diff.ts @@ -0,0 +1,61 @@ +/** + * What changed in a unit, on either side (docs/device-sync.md, "Row + * matching"). A row matched by its key is compared column by column with its + * baseline, so an emptied column is a real deletion. A row matched by value + * has no baseline: its values count only where they are non-empty and differ + * from the shadow's projection, TYPE and LABEL only for phones, emails, + * addresses and events, and IS_PRIMARY never. + */ +import { CommonKind, Data, MimeType, StructuredPostal as SP } from '../android-columns'; +import type { Row } from '../types'; +import { changedFromBaseline, sameCell, text } from './cells'; +import type { Match } from './matching'; +import { POSTAL_PART_COLUMNS } from './postal'; +import { SPECS, type Unit } from './project'; + +/** + * A postal row whose parts are only its formatted address copied into STREET: + * the provider's split of an editor that wrote FORMATTED_ADDRESS alone. + */ +function providerSplitPostal(cells: Row): boolean { + const street = text(cells[SP.STREET]); + return street !== null + && street === text(cells[SP.FORMATTED_ADDRESS]) + && POSTAL_PART_COLUMNS.every((c) => c === SP.STREET || text(cells[c]) === null); +} + +export function localChanges(m: Match): string[] { + const { row, unit } = m; + const changed = rawChanges(m).filter((c) => !unit.truncated?.includes(c)); + if (row.mimetype === MimeType.STRUCTURED_POSTAL && providerSplitPostal(row.cells)) { + // A multi-part formatted address in STREET was the provider's split, not a typed street. + const split = m.how !== 'key' || !row.baseline || /[,\n]/.test(text(row.cells[SP.FORMATTED_ADDRESS]) ?? ''); + if (split) return changed.filter((c) => !(POSTAL_PART_COLUMNS as readonly string[]).includes(c)); + } + return changed; +} + +function rawChanges(m: Match): string[] { + const spec = SPECS[m.row.mimetype]; + const { row, unit } = m; + if (m.how === 'key' && row.baseline) return changedFromBaseline(row.cells, row.baseline, spec.columns); + if (row.mimetype === MimeType.PHOTO) { + // A photo row without our baseline was put there by an editor: its picture counts as new. + return m.how === 'key' ? [] : [Data.DATA14]; + } + const out = spec.valueColumns.filter((c) => text(row.cells[c]) !== null && !sameCell(row.cells[c], unit.cells[c], c)); + if (spec.typed && text(row.cells[CommonKind.TYPE]) !== null && !sameCell(row.cells[CommonKind.TYPE], unit.cells[CommonKind.TYPE])) { + out.push(CommonKind.TYPE, CommonKind.LABEL); + } else if (spec.typed && text(row.cells[CommonKind.LABEL]) !== null && !sameCell(row.cells[CommonKind.LABEL], unit.cells[CommonKind.LABEL])) { + out.push(CommonKind.LABEL); + } + return out; +} + +/** Whether two units show the same on the device (sub-fields Android can't hold don't count). */ +export function sameUnit(a: Unit, b: Unit): boolean { + if (a.mimetype !== b.mimetype) return false; + // A photo whose bytes could not be had compares as unchanged. + if (a.mimetype === MimeType.PHOTO) return a.photoHash === undefined || b.photoHash === undefined || a.photoHash === b.photoHash; + return SPECS[a.mimetype].columns.every((c) => sameCell(a.cells[c], b.cells[c], c)); +} diff --git a/src/device-sync/contacts/download.ts b/src/device-sync/contacts/download.ts new file mode 100644 index 00000000..4cf57042 --- /dev/null +++ b/src/device-sync/contacts/download.ts @@ -0,0 +1,340 @@ +/** + * Server cards into rows (docs/device-sync.md, "Change detection and merge + * rules"): inserts, writes that make a contact's rows the card's (a clean + * contact, an accepted upload, a revert), and the per-unit merge of a dirty + * contact with a new server version. + * + * Rows are matched to the shadow's units after its keys were mapped to the + * new version's (Stalwart re-keys cards written over CardDAV). A clean write + * then touches only rows that differ, so an echo of our own upload writes + * nothing. A merge writes only units the server changed, and never gives a + * key to a row of a kind an editor rewrote (all its rows keyless): a keyed + * row there would later make the entries the editor could not show look + * deleted. + */ +import { Data, MimeType, RawContacts } from '../android-columns'; +import { canonicalJson } from '../common/json'; +import { objectRef } from '../common/ids'; +import type { ContactCardWire, ContactsContext, DownloadPlan, LocalContact, LocalDataRow } from '../planner'; +import type { ProviderOp, WriteRow } from '../types'; +import { changedFromBaseline, encodeBaseline, sameCell } from './cells'; +import { localChanges, sameUnit } from './diff'; +import { deletionsInferred, inPlaceEvidence, matchKind, reconcileKeys, type KindMatch } from './matching'; +import { planMemberships, remoteMemberships, shadowMemberOf, withMemberOf, withoutMemberOf } from './members'; +import { assertContact, deleteData, insertUnit, updateData, updateRawContact, updateUnit } from './ops'; +import { shadowOf } from './photo'; +import { KIND_ORDER, projectCard, rowMatchesUnit, SPECS, type Unit } from './project'; + +const ENTRY_KINDS = KIND_ORDER.filter((k) => k !== MimeType.GROUP_MEMBERSHIP); + +type Ctx = ContactsContext; + +export function cardUnits(card: ContactCardWire | null, ctx: Pick): Unit[] { + return card ? projectCard(withoutMemberOf(card), ctx, { memberOf: null }) : []; +} + +/** Raw contact columns that describe the server card: collections, shadow, read-only flag. */ +function cardColumns(card: ContactCardWire, shadow: ContactCardWire, ctx: Ctx): WriteRow { + return { + [RawContacts.SYNC1]: ctx.selectedCollections(card).join(',') || null, + [RawContacts.SYNC2]: JSON.stringify(shadow), + [RawContacts.RAW_CONTACT_IS_READ_ONLY]: ctx.isReadOnly(card) ? 1 : 0, + }; +} + +/** Only the raw contact columns that differ from what the local item holds. */ +function changedRawColumns(local: LocalContact, values: WriteRow): WriteRow { + const current: Record = { + [RawContacts.SOURCE_ID]: local.sourceId, + [RawContacts.SYNC1]: local.collections.join(',') || null, + [RawContacts.SYNC2]: local.shadow ? JSON.stringify(local.shadow) : null, + [RawContacts.SYNC3]: local.pending ? JSON.stringify(local.pending) : null, + [RawContacts.SYNC4]: local.poison ? JSON.stringify(local.poison) : null, + [RawContacts.RAW_CONTACT_IS_READ_ONLY]: local.readOnly ? 1 : 0, + [RawContacts.DIRTY]: local.dirty ? 1 : 0, + }; + const out: WriteRow = {}; + for (const [column, value] of Object.entries(values)) { + if (column === RawContacts.SYNC2) { + const before = local.shadow ? canonicalJson(local.shadow) : null; + const after = typeof value === 'string' ? canonicalJson(JSON.parse(value)) : null; + if (before !== after) out[column] = value; + } else if (!sameCell(current[column], value as unknown)) { + out[column] = value; + } + } + return out; +} + +/** Row keys after the server re-keyed: mapped, or marked gone so they can't meet a new entry's key. */ +function renameRows(rows: LocalDataRow[], base: Unit[], keyMap: Map): LocalDataRow[] { + const baseKeys = new Set(base.map((u) => u.key)); + return rows.map((r) => { + if (!r.key) return r; + const key = keyMap.get(r.key) ?? (baseKeys.has(r.key) ? `~gone:${r.key}` : r.key); + return key === r.key ? r : { ...r, key }; + }); +} + +function renameUnits(base: Unit[], keyMap: Map): Unit[] { + return base.map((u) => ({ ...u, key: keyMap.get(u.key) ?? `~gone:${u.key}` })); +} + +interface RowPlan { + ops: ProviderOp[]; + conflicts: number; + stillDirty: boolean; + evidence: boolean; +} + +/** + * Makes the rows of every entry kind the card's: rows matched to a unit are + * rewritten when they differ (or only get their key and baseline), rows no + * unit accounts for go, units without a row are inserted. + */ +function cleanRows(local: LocalContact, remote: Unit[], keyMap: Map, base: Unit[], accepted: boolean): RowPlan { + const ops: ProviderOp[] = []; + const rows = renameRows(local.rows, base, keyMap); + const kinds = ENTRY_KINDS.map((k) => matchKind(k, rows, remote)); + for (const kind of kinds) { + for (const m of kind.matched) { + const original = local.rows.find((r) => r.id === m.row.id)!; + const { unit } = m; + const columns = SPECS[unit.mimetype].columns; + if (unit.mimetype === MimeType.PHOTO) { + if (accepted && unit.photoHash && original.photoHash !== unit.photoHash && localChanges({ ...m, row: original }).length) { + // The device's own photo just uploaded: record the server's hash, keep the picture. + // (A photo that could not be sent keeps its old baseline, so it still counts as changed.) + ops.push(updateData(m.row.id, { + [Data.DATA_SYNC1]: unit.key, + [Data.DATA_SYNC2]: unit.photoHash ?? null, + [Data.DATA_SYNC3]: encodeBaseline(m.row.cells, columns), + })); + } else if (original.photoHash !== unit.photoHash && unit.extra) { + ops.push(updateUnit(m.row.id, unit, true)); + } else if (original.key !== unit.key) { + ops.push(updateData(m.row.id, { [Data.DATA_SYNC1]: unit.key })); + } + continue; + } + if (!rowMatchesUnit(unit, m.row.cells, sameCell)) ops.push(updateUnit(m.row.id, unit, true)); + else if (original.key !== unit.key || !original.baseline || changedFromBaseline(m.row.cells, original.baseline, columns).length) { + // Same content: only the key and the baseline (what the provider holds now) are written. + ops.push(updateData(m.row.id, { [Data.DATA_SYNC1]: unit.key, [Data.DATA_SYNC3]: encodeBaseline(m.row.cells, columns) })); + } + } + for (const row of kind.fresh) ops.push(deleteData(row.id)); + for (const unit of kind.missing) { + if (unit.mimetype === MimeType.PHOTO && !unit.extra) continue; + ops.push(insertUnit(local.rawContactId, unit, true)); + } + } + return { ops, conflicts: 0, stillDirty: false, evidence: inPlaceEvidence(kinds) }; +} + +/** Per-unit merge of a dirty contact's rows with a new server version; the server wins real conflicts. */ +function mergeRows(local: LocalContact, base: Unit[], remote: Unit[], keyMap: Map): RowPlan { + const ops: ProviderOp[] = []; + let conflicts = 0; + let stillDirty = false; + const rows = renameRows(local.rows, base, keyMap); + const renamedBase = renameUnits(base, keyMap); + const kinds: KindMatch[] = ENTRY_KINDS.map((k) => matchKind(k, rows, renamedBase)); + const evidence = inPlaceEvidence(kinds); + const mapped = new Set(keyMap.values()); + + for (const kind of kinds) { + const remoteOf = new Map(remote.filter((u) => u.mimetype === kind.mimetype).map((u) => [u.key, u])); + const hasKeyed = kind.matched.some((m) => m.how === 'key'); + const hasKeyless = kind.fresh.length > 0 || kind.matched.some((m) => m.how !== 'key'); + const keyed = hasKeyed || !hasKeyless; + const deletions = kind.mimetype !== MimeType.STRUCTURED_NAME && deletionsInferred(kind, evidence); + const write = (row: LocalDataRow, unit: Unit) => { + if (unit.mimetype === MimeType.PHOTO && !unit.extra) return; + ops.push(updateUnit(row.id, unit, keyed)); + }; + + for (const m of kind.matched) { + const original = local.rows.find((r) => r.id === m.row.id)!; + const remoteUnit = remoteOf.get(m.unit.key); + const changedHere = localChanges({ ...m, row: original }).length > 0; + if (!remoteUnit) { + // Removed on the server: the delete wins, over a local edit too. + if (changedHere) conflicts++; + ops.push(deleteData(m.row.id)); + } else if (sameUnit(m.unit, remoteUnit)) { + if (changedHere) stillDirty = true; + if (m.how === 'key' && original.key !== remoteUnit.key) ops.push(updateData(m.row.id, { [Data.DATA_SYNC1]: remoteUnit.key })); + } else if (!changedHere) { + write(m.row, remoteUnit); + } else if (remoteUnit.mimetype !== MimeType.PHOTO && rowMatchesUnit(remoteUnit, m.row.cells, sameCell)) { + // Both sides made the same change: converged, new baseline. + if (keyed) { + ops.push(updateData(m.row.id, { [Data.DATA_SYNC1]: remoteUnit.key, [Data.DATA_SYNC3]: encodeBaseline(m.row.cells, SPECS[kind.mimetype].columns) })); + } + } else { + conflicts++; + write(m.row, remoteUnit); + } + } + + for (const unit of kind.missing) { + const remoteUnit = remoteOf.get(unit.key); + if (!deletions) continue; + if (!remoteUnit) continue; + if (sameUnit(unit, remoteUnit)) stillDirty = true; + else if (!(remoteUnit.mimetype === MimeType.PHOTO && !remoteUnit.extra)) { + conflicts++; + ops.push(insertUnit(local.rawContactId, remoteUnit, keyed)); + } + } + + // Entries the server added: a new local row with the same value converged with it. + const fresh = [...kind.fresh]; + const spec = SPECS[kind.mimetype]; + for (const unit of remoteOf.values()) { + if (mapped.has(unit.key)) continue; + const p = spec.primary(unit.cells); + const twin = p === null ? -1 : fresh.findIndex((r) => spec.primary(r.cells) === p); + if (twin >= 0) { + const [row] = fresh.splice(twin, 1); + if (keyed) ops.push(updateUnit(row.id, unit, true)); + } else if (!(unit.mimetype === MimeType.PHOTO && !unit.extra)) { + ops.push(insertUnit(local.rawContactId, unit, keyed)); + } + } + if (fresh.length) stillDirty = true; + } + return { ops, conflicts, stillDirty, evidence }; +} + +export interface CleanWriteOptions { + /** DIRTY the assert expects; null asserts VERSION only (an accepted upload, a revert). */ + assertDirty: boolean | null; + /** Clear DIRTY (unless membership changes still wait for the server). */ + clearDirty: boolean; + /** Extra raw contact columns: identity, SYNC3/SYNC4 resets, DELETED. */ + raw?: WriteRow; + /** The rows' changes were just uploaded (planAccepted): a changed photo is the server's now. */ + accepted?: boolean; +} + +/** + * The ops that make a local contact's rows, shadow and raw columns the + * server card's, behind the VERSION assert. Membership changes that the + * server does not have yet stay (and keep the contact dirty). + */ +export function cleanWrite(local: LocalContact, card: ContactCardWire, ctx: Ctx, options: CleanWriteOptions) { + const server = withoutMemberOf(card); + const base = cardUnits(local.shadow, ctx); + const remote = cardUnits(server, ctx); + const keyMap = reconcileKeys(base, remote); + const rows = cleanRows(local, remote, keyMap, base, options.accepted ?? false); + const members = planMemberships({ + rows: local.rows, + base: shadowMemberOf(local.shadow), + remote: remoteMemberships(server, ctx), + evidence: rows.evidence, + restore: true, + parent: local.rawContactId, + groupRowIdBySourceId: (g) => ctx.groupRowIdBySourceId(g), + }); + const shadow = withMemberOf(shadowOf(server), members.memberOf); + const raw = changedRawColumns(local, { + ...cardColumns(server, shadow, ctx), + ...(options.raw ?? {}), + ...(options.clearDirty && !members.pending ? { [RawContacts.DIRTY]: 0 } : {}), + }); + const dataOps = [...rows.ops, ...members.ops]; + const ops: ProviderOp[] = []; + if (Object.keys(raw).length) ops.push(updateRawContact(local.rawContactId, raw)); + ops.push(...dataOps); + return { + ops: ops.length ? [assertContact(local, options.assertDirty), ...ops] : [], + writes: ops.length, + pendingMembers: members.pending, + }; +} + +function insertPlan(card: ContactCardWire, ctx: Ctx): DownloadPlan { + const identity = objectRef(ctx.jmapAccountId, card.id); + const units = cardUnits(card, ctx); + const remote = remoteMemberships(card, ctx); + const members = planMemberships({ + rows: [], + base: remote === null ? null : [], + remote, + evidence: false, + restore: true, + parent: { ref: 0 }, + groupRowIdBySourceId: (g) => ctx.groupRowIdBySourceId(g), + }); + const shadow = withMemberOf(shadowOf(card), members.memberOf); + const ops: ProviderOp[] = [ + { op: 'insert', table: 'raw_contacts', values: { [RawContacts.SOURCE_ID]: identity, [RawContacts.DIRTY]: 0, ...cardColumns(card, shadow, ctx) } }, + ...units.filter((u) => u.mimetype !== MimeType.PHOTO || u.extra).map((u) => insertUnit({ ref: 0 }, u, true)), + ...members.ops, + ]; + return { ops: { ref: identity, ops }, conflicts: 0, stillDirty: false, effect: 'insert', writes: ops.length }; +} + +export function planContactDownload(card: ContactCardWire, local: LocalContact | null, ctx: Ctx): DownloadPlan { + const server = withoutMemberOf(card); + const identity = objectRef(ctx.jmapAccountId, server.id); + if (!local) return insertPlan(server, ctx); + const none = (stillDirty: boolean): DownloadPlan => ({ ops: { ref: identity, ops: [] }, conflicts: 0, stillDirty, effect: 'none', writes: 0 }); + // Deleted on the device: the delete wins and uploads as a destroy. + if (local.deleted) return none(true); + + if (!local.sourceId) { + // Our own create whose identity was never written: adopt it. The rows stay + // as they are and upload against the card as their new shadow. + const members = shadowMemberOf(local.shadow) ?? []; + const shadow = withMemberOf(shadowOf(server), remoteMemberships(server, ctx) === null ? null : members); + const ops = [ + assertContact(local, local.dirty), + updateRawContact(local.rawContactId, { [RawContacts.SOURCE_ID]: identity, [RawContacts.SYNC3]: null, ...cardColumns(server, shadow, ctx) }), + ]; + return { ops: { ref: identity, ops }, conflicts: 0, stillDirty: true, effect: 'update', writes: 1 }; + } + + const uidChanged = typeof local.shadow?.uid === 'string' && typeof server.uid === 'string' && local.shadow!.uid !== server.uid; + if (!local.dirty || uidChanged) { + // A different object behind the same id is replaced, never merged into. + const target = uidChanged ? { ...local, shadow: null, rows: local.rows.map((r) => ({ ...r, key: null })) } : local; + const write = cleanWrite(target, server, ctx, { assertDirty: local.dirty, clearDirty: uidChanged }); + return { + ops: { ref: identity, ops: write.ops }, + conflicts: 0, + stillDirty: write.pendingMembers, + effect: write.writes ? 'update' : 'none', + writes: write.writes, + }; + } + + const base = cardUnits(local.shadow, ctx); + const remote = cardUnits(server, ctx); + const keyMap = reconcileKeys(base, remote); + const rows = mergeRows(local, base, remote, keyMap); + const members = planMemberships({ + rows: local.rows, + base: shadowMemberOf(local.shadow), + remote: remoteMemberships(server, ctx), + evidence: rows.evidence, + restore: false, + parent: local.rawContactId, + groupRowIdBySourceId: (g) => ctx.groupRowIdBySourceId(g), + }); + const shadow = withMemberOf(shadowOf(server), members.memberOf); + const raw = changedRawColumns(local, cardColumns(server, shadow, ctx)); + const ops: ProviderOp[] = []; + if (Object.keys(raw).length) ops.push(updateRawContact(local.rawContactId, raw)); + ops.push(...rows.ops, ...members.ops); + return { + ops: { ref: identity, ops: ops.length ? [assertContact(local, true), ...ops] : [] }, + conflicts: rows.conflicts, + stillDirty: rows.stillDirty || members.pending, + effect: ops.length ? 'update' : 'none', + writes: ops.length, + }; +} diff --git a/src/device-sync/contacts/entries.ts b/src/device-sync/contacts/entries.ts new file mode 100644 index 00000000..cc0444c1 --- /dev/null +++ b/src/device-sync/contacts/entries.ts @@ -0,0 +1,348 @@ +/** + * Rows back to JSContact: the sub-field edits of a changed unit, and the + * entries a new row stands for. Only what the row can say is touched; a + * phone's other features, an entry's other contexts and every property + * Android has no column for stay as the server has them. + */ +import type { ContactAddress, ContactTitle } from '../../api/types'; +import { stringToPartialDate } from '../../lib/contact-utils'; +import { + CommonKind, + ContactEventType, + Data, + EmailType, + MimeType, + Organization as Org, + PhoneType, + PostalType, + RelationType, + StructuredPostal as SP, + WebsiteType, +} from '../android-columns'; +import type { ContactCardWire } from '../planner'; +import type { Row } from '../types'; +import { num, text } from './cells'; +import { parseEntryKey, parseOrgKey } from './keys'; +import { editPostalComponents, postalComponentsFromRow, POSTAL_PART_COLUMNS } from './postal'; +import { + emailFlags, + eventKind, + PHONE_TYPE_FEATURES, + phoneFlags, + postalFlags, + relationName, + relationTypesOf, + TYPED_CONTEXTS, + websiteFlags, + type EntryFlags, +} from './types-map'; + +/** A pointer (raw member names) and the value it should get; null removes. */ +export interface Edit { + path: string[]; + value: unknown; +} + +/** An entry a row adds (or, for a relation, a key it moves to). */ +export interface NewEntry { + map: string; + /** Fixed key (relatedTo is keyed by its value); otherwise a new key is minted. */ + key?: string; + /** Reference into another new entry's key, e.g. a title's organization. */ + value: Record; + /** Entries created together share a group id so titles can point at their organization. */ + role?: 'organization' | 'title'; +} + +type Entry = Record; + +const flagsOf = (value: unknown): Record => + value && typeof value === 'object' && !Array.isArray(value) ? { ...(value as Record) } : {}; + +/** Replaces the flags a TYPE speaks for; null when nothing is left. */ +function swapFlags(current: unknown, owned: string[], wanted: string[]): Record | null { + const out = flagsOf(current); + for (const k of owned) delete out[k]; + for (const k of wanted) out[k] = true; + return Object.keys(out).length ? out : null; +} + +function same(a: unknown, b: unknown): boolean { + return JSON.stringify(a ?? null) === JSON.stringify(b ?? null); +} + +function pushIf(out: Edit[], entry: Entry, path: string[], field: string, value: unknown): void { + if (!same(entry[field], value)) out.push({ path: [...path, field], value }); +} + +/** + * A new TYPE replaces the contexts and features it speaks for; a new LABEL of + * the same custom TYPE is only the label (the contexts it hides stay). + */ +function typeEdits(out: Edit[], entry: Entry, path: string[], flags: EntryFlags, changed: ReadonlySet, custom: boolean): void { + if (changed.has(CommonKind.TYPE)) { + pushIf(out, entry, path, 'contexts', swapFlags(entry.contexts, TYPED_CONTEXTS, flags.contexts)); + if (flags.features) pushIf(out, entry, path, 'features', swapFlags(entry.features, PHONE_TYPE_FEATURES, flags.features)); + pushIf(out, entry, path, 'label', flags.label); + } else if (changed.has(CommonKind.LABEL) && custom) { + pushIf(out, entry, path, 'label', flags.label); + } +} + +function prefEdit(out: Edit[], entry: Entry, path: string[], cells: Row): void { + const primary = num(cells[Data.IS_PRIMARY]) === 1; + if (primary && entry.pref !== 1) out.push({ path: [...path, 'pref'], value: 1 }); + else if (!primary && entry.pref === 1) out.push({ path: [...path, 'pref'], value: null }); +} + +const typeOf = (cells: Row) => num(cells[CommonKind.TYPE]); +const labelOf = (cells: Row) => text(cells[CommonKind.LABEL]); + +function mapEntry(card: ContactCardWire, map: string, key: string): Entry | null { + const m = card[map]; + const e = m && typeof m === 'object' ? (m as Record)[key] : undefined; + return e && typeof e === 'object' ? (e as Entry) : null; +} + +function partialDate(value: unknown): Record | null { + const t = text(value); + const pd = t ? stringToPartialDate(t) : null; + return pd ? { '@type': 'PartialDate', ...pd } : null; +} + +function splitUnits(value: string | null): Array<{ name: string }> | null { + const units = (value ?? '').split(',').map((s) => s.trim()).filter(Boolean); + return units.length ? units.map((name) => ({ name })) : null; +} + +export interface UnitEdits { + edits: Edit[]; + /** Entries to add (a company typed into a title-only row creates its organization). */ + added: NewEntry[]; + /** Entries this unit no longer has (a relation moved to another key, a title emptied). */ + removed: Array<{ map: string; key: string }>; +} + +/** + * Edits for a changed unit. `changed` are the row's changed columns, `key` + * the unit's DATA_SYNC1 key in the shadow, `baseCells` the shadow's + * projection of it (the TYPE a relation had), `byKey` whether the row was + * matched by its key. + */ +export function unitEdits( + mimetype: string, + key: string, + card: ContactCardWire, + cells: Row, + baseCells: Row, + changed: ReadonlySet, + shownName: string | null, + byKey: boolean, +): UnitEdits { + const edits: Edit[] = []; + const added: NewEntry[] = []; + const removed: Array<{ map: string; key: string }> = []; + const result = { edits, added, removed }; + if (mimetype === MimeType.ORGANIZATION) return orgEdits(key, card, cells, changed, result); + const parsed = parseEntryKey(key); + if (!parsed) return result; + const entry = mapEntry(card, parsed.map, parsed.key); + if (!entry) return result; + const path = [parsed.map, parsed.key]; + const typeChanged = changed.has(CommonKind.TYPE) || changed.has(CommonKind.LABEL); + switch (mimetype) { + case MimeType.NICKNAME: + if (changed.has(Data.DATA1)) pushIf(edits, entry, path, 'name', text(cells[Data.DATA1])); + break; + case MimeType.EMAIL: + if (changed.has(Data.DATA1)) pushIf(edits, entry, path, 'address', text(cells[Data.DATA1])); + if (typeChanged) typeEdits(edits, entry, path, emailFlags(typeOf(cells), labelOf(cells)), changed, typeOf(cells) === EmailType.CUSTOM); + if (changed.has(Data.IS_PRIMARY)) prefEdit(edits, entry, path, cells); + break; + case MimeType.PHONE: + if (changed.has(Data.DATA1)) pushIf(edits, entry, path, 'number', text(cells[Data.DATA1])); + if (typeChanged) typeEdits(edits, entry, path, phoneFlags(typeOf(cells), labelOf(cells)), changed, typeOf(cells) === PhoneType.CUSTOM); + if (changed.has(Data.IS_PRIMARY)) prefEdit(edits, entry, path, cells); + break; + case MimeType.STRUCTURED_POSTAL: { + const parts = new Set(POSTAL_PART_COLUMNS.filter((c) => changed.has(c))); + if (parts.size) { + const comps = editPostalComponents(entry as ContactAddress, cells, parts); + pushIf(edits, entry, path, 'components', comps.length ? comps : null); + } + if (changed.has(SP.FORMATTED_ADDRESS)) pushIf(edits, entry, path, 'full', text(cells[SP.FORMATTED_ADDRESS])); + if (typeChanged) typeEdits(edits, entry, path, postalFlags(typeOf(cells), labelOf(cells)), changed, typeOf(cells) === PostalType.CUSTOM); + break; + } + case MimeType.WEBSITE: + if (changed.has(Data.DATA1)) pushIf(edits, entry, path, 'uri', text(cells[Data.DATA1])); + if (typeChanged) typeEdits(edits, entry, path, websiteFlags(typeOf(cells), labelOf(cells)), changed, typeOf(cells) === WebsiteType.CUSTOM); + break; + case MimeType.EVENT: + if (changed.has(Data.DATA1)) { + const date = partialDate(cells[Data.DATA1]); + // Free text Stalwart can't take stays on the device only. + if (date) pushIf(edits, entry, path, 'date', date); + } + if (changed.has(CommonKind.TYPE) || (changed.has(CommonKind.LABEL) && typeOf(cells) === ContactEventType.CUSTOM)) { + pushIf(edits, entry, path, 'kind', eventKind(typeOf(cells), labelOf(cells))); + } + break; + case MimeType.RELATION: { + const name = text(cells[Data.DATA1]); + // Only a row we wrote can be renamed: a re-inserted row showing another name may just + // show a related card that was renamed since. + const renamed = byKey && changed.has(Data.DATA1) && name !== null && name !== shownName && name !== parsed.key; + const relationChanged = changed.has(CommonKind.TYPE) || (changed.has(CommonKind.LABEL) && typeOf(cells) === RelationType.CUSTOM); + const relation = relationChanged + ? swapFlags(entry.relation, relationTypesOf(typeOf(baseCells), labelOf(baseCells)), relationTypes(cells)) ?? {} + : flagsOf(entry.relation); + if (renamed) { + // relatedTo is keyed by its value: another name is another entry. + removed.push({ map: 'relatedTo', key: parsed.key }); + added.push({ map: 'relatedTo', key: name!, value: { ...entry, relation } }); + } else if (relationChanged) { + pushIf(edits, entry, path, 'relation', relation); + } + break; + } + case MimeType.NOTE: + if (changed.has(Data.DATA1)) pushIf(edits, entry, path, 'note', text(cells[Data.DATA1])); + break; + } + return result; +} + +function relationTypes(cells: Row): string[] { + const name = relationName(typeOf(cells), labelOf(cells)); + return name ? [name] : []; +} + +function orgEdits(key: string, card: ContactCardWire, cells: Row, changed: ReadonlySet, out: UnitEdits): UnitEdits { + const parts = parseOrgKey(key); + if (!parts) return out; + const org = parts.org ? mapEntry(card, 'organizations', parts.org) : null; + const orgPath = parts.org ? ['organizations', parts.org] : null; + const company = text(cells[Org.COMPANY]); + const department = text(cells[Org.DEPARTMENT]); + let newOrg: NewEntry | null = null; + if (org && orgPath) { + if (changed.has(Org.COMPANY)) pushIf(out.edits, org, orgPath, 'name', company); + if (changed.has(Org.DEPARTMENT)) pushIf(out.edits, org, orgPath, 'units', splitUnits(department)); + } else if ((changed.has(Org.COMPANY) && company) || (changed.has(Org.DEPARTMENT) && department)) { + const value: Record = {}; + if (company) value.name = company; + const units = splitUnits(department); + if (units) value.units = units; + newOrg = { map: 'organizations', value, role: 'organization' }; + out.added.push(newOrg); + } + const titleEdit = (column: string, titleKey: string | null, kind: 'title' | 'role') => { + const title = titleKey ? mapEntry(card, 'titles', titleKey) : null; + const value = text(cells[column]); + if (title && titleKey) { + if (changed.has(column)) { + if (value === null) out.removed.push({ map: 'titles', key: titleKey }); + else pushIf(out.edits, title, ['titles', titleKey], 'name', value); + } + if (newOrg) out.edits.push({ path: ['titles', titleKey, 'organizationId'], value: newOrg }); + } else if (changed.has(column) && value !== null) { + const t: Record = { name: value, kind }; + if (parts.org) t.organizationId = parts.org; + out.added.push({ map: 'titles', value: t, role: 'title' }); + } + }; + titleEdit(Org.TITLE, parts.title, 'title'); + titleEdit(Org.JOB_DESCRIPTION, parts.role, 'role'); + // A new organization's titles point at its minted key (resolved by the patch builder). + if (newOrg) for (const e of out.added) if (e.role === 'title') e.value.organizationId = newOrg; + return out; +} + +/** The entries a new row adds. */ +export function newEntries(mimetype: string, cells: Row, shownName?: string | null): NewEntry[] { + const typeFlags = (flags: EntryFlags, withFeatures = false): Record => { + const out: Record = {}; + const contexts = swapFlags(undefined, [], flags.contexts); + if (contexts) out.contexts = contexts; + if (withFeatures) { + const features = swapFlags(undefined, [], flags.features ?? []); + if (features) out.features = features; + } + if (flags.label) out.label = flags.label; + return out; + }; + const primary = num(cells[Data.IS_PRIMARY]) === 1 ? { pref: 1 } : {}; + const value = text(cells[Data.DATA1]); + switch (mimetype) { + case MimeType.NICKNAME: + return value ? [{ map: 'nicknames', value: { name: value } }] : []; + case MimeType.EMAIL: + return value ? [{ map: 'emails', value: { address: value, ...typeFlags(emailFlags(typeOf(cells), labelOf(cells))), ...primary } }] : []; + case MimeType.PHONE: + return value ? [{ map: 'phones', value: { number: value, ...typeFlags(phoneFlags(typeOf(cells), labelOf(cells)), true), ...primary } }] : []; + case MimeType.STRUCTURED_POSTAL: { + const address: Record = {}; + const comps = postalComponentsFromRow(cells); + if (comps.length) address.components = comps; + const full = text(cells[SP.FORMATTED_ADDRESS]); + if (full) address.full = full; + if (!comps.length && !full) return []; + return [{ map: 'addresses', value: { ...address, ...typeFlags(postalFlags(typeOf(cells), labelOf(cells))) } }]; + } + case MimeType.ORGANIZATION: { + const out: NewEntry[] = []; + const company = text(cells[Org.COMPANY]); + const units = splitUnits(text(cells[Org.DEPARTMENT])); + let org: NewEntry | null = null; + if (company || units) { + const v: Record = {}; + if (company) v.name = company; + if (units) v.units = units; + org = { map: 'organizations', value: v, role: 'organization' }; + out.push(org); + } + const title = (column: string, kind: ContactTitle['kind']) => { + const name = text(cells[column]); + if (!name) return; + const v: Record = { name, kind }; + if (org) v.organizationId = org; + out.push({ map: 'titles', value: v, role: 'title' }); + }; + title(Org.TITLE, 'title'); + title(Org.JOB_DESCRIPTION, 'role'); + return out; + } + case MimeType.WEBSITE: + return value ? [{ map: 'links', value: { uri: value, ...typeFlags(websiteFlags(typeOf(cells), labelOf(cells))) } }] : []; + case MimeType.EVENT: { + const date = partialDate(cells[Data.DATA1]); + return date ? [{ map: 'anniversaries', value: { kind: eventKind(typeOf(cells), labelOf(cells)), date } }] : []; + } + case MimeType.RELATION: { + if (!value || value === shownName) return []; + const type = relationName(typeOf(cells), labelOf(cells)); + return [{ map: 'relatedTo', key: value, value: { relation: type ? { [type]: true } : {} } }]; + } + case MimeType.NOTE: + return value ? [{ map: 'notes', value: { note: value } }] : []; + default: + return []; + } +} + +/** Keys (map, key) a unit stands for, for deleting it. */ +export function unitEntries(mimetype: string, key: string): Array<{ map: string; key: string }> { + if (mimetype === MimeType.ORGANIZATION) { + const parts = parseOrgKey(key); + if (!parts) return []; + const out: Array<{ map: string; key: string }> = []; + if (parts.org) out.push({ map: 'organizations', key: parts.org }); + if (parts.title) out.push({ map: 'titles', key: parts.title }); + if (parts.role) out.push({ map: 'titles', key: parts.role }); + return out; + } + const parsed = parseEntryKey(key); + return parsed ? [parsed] : []; +} + diff --git a/src/device-sync/contacts/groups.ts b/src/device-sync/contacts/groups.ts new file mode 100644 index 00000000..6e9d8dae --- /dev/null +++ b/src/device-sync/contacts/groups.ts @@ -0,0 +1,252 @@ +/** + * Group cards (`kind: "group"`) ↔ Groups rows, and membership edits made on + * the device ↔ the group cards' `members` (docs/device-sync.md, "Contacts + * mapping", "Groups"). Group rows are written before the contacts' plans run, + * so memberships can name them. `members` takes `members/: null` as a + * clean delete; the first member of a group without any is sent as the whole + * map. + */ +import type { ContactName } from '../../api/types'; +import { deriveFullName } from '../../lib/contact-utils'; +import { Groups } from '../android-columns'; +import { collectionKey, objectRef, parseCollectionKey, parseObjectRef } from '../common/ids'; +import { canonicalJson, jsonHash } from '../common/json'; +import { ptr, type PatchObject } from '../common/patch'; +import type { + AcceptedPlan, + ContactCardWire, + ContactsContext, + DownloadPlan, + LocalContact, + LocalGroup, + OpGroup, + UploadAction, + UploadPlan, +} from '../planner'; +import type { ProviderOp, WriteRow } from '../types'; +import { sameCell } from './cells'; +import { cardUnits } from './download'; +import { inPlaceEvidence, matchKind } from './matching'; +import { planMemberships, shadowMemberOf } from './members'; +import { deleteWhereId } from './ops'; +import { shadowOf } from './photo'; +import { KIND_ORDER } from './project'; + +type Ctx = ContactsContext; + +/** The title a group card shows. */ +function groupTitle(card: ContactCardWire | null): string | null { + const name = card?.name as ContactName | undefined; + if (typeof name?.full === 'string' && name.full.trim()) return name.full; + return deriveFullName(name?.components) || null; +} + +const ref = (local: LocalGroup) => local.sourceId ?? `group:${local.groupId}`; + +function assertGroup(local: LocalGroup, dirty: boolean | null): ProviderOp { + const values: Record = { [Groups.VERSION]: local.version }; + if (dirty !== null) values[Groups.DIRTY] = dirty ? 1 : 0; + return { op: 'assert', table: 'groups', id: local.groupId, values, expectCount: 1 }; +} + +const updateGroup = (local: LocalGroup, values: WriteRow): ProviderOp => ({ + op: 'update', table: 'groups', id: local.groupId, values, expectCount: 1, +}); + +function cardColumns(card: ContactCardWire, ctx: Ctx): WriteRow { + return { + [Groups.SYNC2]: JSON.stringify(shadowOf(card)), + [Groups.GROUP_IS_READ_ONLY]: ctx.isReadOnly(card) ? 1 : 0, + }; +} + +/** + * Only the values that differ from what the local group holds. The read-only + * flag is not decoded (LocalGroup has no field for it), so it only goes along + * with other writes, which keeps an echo free of writes. + */ +function changed(local: LocalGroup, values: WriteRow): WriteRow { + const out: WriteRow = {}; + for (const [column, value] of Object.entries(values)) { + if (column === Groups.SYNC2) { + const before = local.shadow ? canonicalJson(local.shadow) : null; + if (typeof value !== 'string' || before !== canonicalJson(JSON.parse(value))) out[column] = value; + } else if (column === Groups.TITLE) { + if (!sameCell(local.title, value as unknown)) out[column] = value; + } else if (column !== Groups.GROUP_IS_READ_ONLY) { + out[column] = value; + } + } + if (Object.keys(out).length && Groups.GROUP_IS_READ_ONLY in values) out[Groups.GROUP_IS_READ_ONLY] = values[Groups.GROUP_IS_READ_ONLY]; + return out; +} + +export function planGroupDownload(card: ContactCardWire, local: LocalGroup | null, ctx: Ctx): DownloadPlan { + const identity = objectRef(ctx.jmapAccountId, card.id); + const remoteTitle = groupTitle(card); + const plan = (ops: ProviderOp[], effect: DownloadPlan['effect'], conflicts = 0, stillDirty = false): DownloadPlan => ({ + ops: { ref: identity, ops }, conflicts, stillDirty, effect, writes: ops.filter((o) => o.op !== 'assert').length, + }); + if (!local) { + return plan([{ + op: 'insert', + table: 'groups', + values: { [Groups.SOURCE_ID]: identity, [Groups.TITLE]: remoteTitle ?? '', [Groups.GROUP_VISIBLE]: 1, [Groups.DIRTY]: 0, ...cardColumns(card, ctx) }, + }], 'insert'); + } + if (local.deleted) return plan([], 'none', 0, true); + if (!local.sourceId) { + // Our own create whose identity was never written. + return plan([ + assertGroup(local, local.dirty), + updateGroup(local, { [Groups.SOURCE_ID]: identity, [Groups.SYNC3]: null, ...cardColumns(card, ctx) }), + ], 'update', 0, true); + } + const base = groupTitle(local.shadow); + const localChanged = local.dirty && !sameCell(local.title, base); + const remoteChanged = !sameCell(remoteTitle, base); + let conflicts = 0; + const values: WriteRow = { ...cardColumns(card, ctx) }; + if (remoteChanged && (!localChanged || !sameCell(local.title, remoteTitle))) { + if (localChanged) conflicts++; + values[Groups.TITLE] = remoteTitle ?? ''; + } + const diff = changed(local, values); + if (!Object.keys(diff).length) return plan([], 'none', 0, localChanged && !remoteChanged); + return plan([assertGroup(local, local.dirty), updateGroup(local, diff)], 'update', conflicts, localChanged && !remoteChanged); +} + +export function planGroupLocalDelete(local: LocalGroup): OpGroup { + return { ref: ref(local), ops: [deleteWhereId('groups', local.groupId)] }; +} + +/** A poison marker's `fp` for a group, as the engine takes it (engine/poison.ts). */ +export function groupFingerprint(local: LocalGroup): string { + return jsonHash({ deleted: local.deleted, title: local.title, shadow: local.shadow }); +} + +export function planGroupUpload(local: LocalGroup, ctx: Ctx): UploadPlan { + const group: OpGroup = { ref: ref(local), ops: [] }; + if (!local.deleted && !local.dirty && local.sourceId) return { kind: 'clean', ops: group }; + const p = local.poison; + if (p && p.until > ctx.now && p.fp === groupFingerprint(local)) return { kind: 'skip', reason: `poisoned:${p.type}` }; + const parsed = parseObjectRef(local.sourceId); + const readOnly = !!local.shadow && ctx.isReadOnly(local.shadow); + + if (local.deleted) { + if (parsed) { + if (readOnly) return { kind: 'revert', ops: { ...group, ops: [deleteWhereId('groups', local.groupId)] }, refetch: true }; + const books = Object.entries(local.shadow?.addressBookIds ?? {}).filter(([, on]) => on).map(([id]) => id); + const selected = books.filter((id) => ctx.isSelected(collectionKey(parsed.accountId, id))); + if (selected.length && selected.length < books.length) { + const patch: PatchObject = {}; + for (const id of selected) patch[ptr('addressBookIds', id)] = null; + return { kind: 'upload', actions: [{ kind: 'update', id: parsed.id, patch }] }; + } + return { kind: 'upload', actions: [{ kind: 'destroy', id: parsed.id, uid: typeof local.shadow?.uid === 'string' ? local.shadow.uid : null }] }; + } + if (local.pending) return { kind: 'upload', actions: [{ kind: 'destroy', id: null, uid: local.pending.uid }] }; + return { kind: 'purge', ops: { ...group, ops: [deleteWhereId('groups', local.groupId)] } }; + } + if (local.sourceId && !parsed) return { kind: 'skip', reason: 'invalidSourceId' }; + + const title = local.title?.trim() ? local.title : null; + if (!local.sourceId) { + if (!local.pending) { + const target = ctx.createTarget(); + if (!target) return { kind: 'skip', reason: 'noWritableAddressBook' }; + const pending = { uid: ctx.mintUid(), target }; + return { kind: 'claim', ops: { ...group, ops: [assertGroup(local, null), updateGroup(local, { [Groups.SYNC3]: JSON.stringify(pending) })] } }; + } + const target = parseCollectionKey(local.pending.target); + if (!target) return { kind: 'skip', reason: 'invalidPendingTarget' }; + const object: Partial = { + '@type': 'Card', version: '1.0', uid: local.pending.uid, kind: 'group', addressBookIds: { [target.id]: true }, + ...(title ? { name: { full: title } } : {}), + }; + return { kind: 'upload', actions: [{ kind: 'create', uid: local.pending.uid, collectionId: target.id, object }] }; + } + + const shadowTitle = groupTitle(local.shadow); + if (readOnly) { + return { kind: 'revert', ops: { ...group, ops: [assertGroup(local, null), updateGroup(local, { [Groups.TITLE]: shadowTitle ?? '', [Groups.DIRTY]: 0 })] } }; + } + if (sameCell(title, shadowTitle) || title === null) { + return { kind: 'clean', ops: { ...group, ops: [assertGroup(local, true), updateGroup(local, { [Groups.DIRTY]: 0 })] } }; + } + const patch: PatchObject = local.shadow?.name && typeof local.shadow.name === 'object' ? { 'name/full': title } : { name: { full: title } }; + return { kind: 'upload', actions: [{ kind: 'update', id: parsed!.id, patch }] }; +} + +export function planGroupAccepted(local: LocalGroup, server: ContactCardWire, ctx: Ctx): AcceptedPlan { + if (local.deleted) { + const purge = planGroupLocalDelete(local); + return { ops: purge, keepDirtyOps: purge }; + } + const account = parseObjectRef(local.sourceId)?.accountId ?? parseCollectionKey(local.pending?.target)?.accountId ?? ctx.jmapAccountId; + const identity = objectRef(account, server.id); + const raw: WriteRow = { [Groups.SOURCE_ID]: identity, [Groups.SYNC3]: null, [Groups.SYNC4]: null, ...cardColumns(server, ctx) }; + const title = groupTitle(server); + const ops: WriteRow = { ...raw, [Groups.DIRTY]: 0 }; + if (!sameCell(local.title, title)) ops[Groups.TITLE] = title ?? ''; + return { + ops: { ref: identity, ops: [assertGroup(local, null), updateGroup(local, ops)] }, + keepDirtyOps: { ref: identity, ops: [updateGroup(local, raw)] }, + }; +} + +/** + * Membership edits of the given contacts as patches of the group cards' + * `members`. A contact's edits are its rows against the base kept in its + * shadow; what the group shadows already show is not sent again. + */ +export function planMembershipUploads(contacts: LocalContact[], groups: LocalGroup[], ctx: Ctx): UploadAction[] { + const bySource = new Map(groups.filter((g) => g.sourceId).map((g) => [g.sourceId!, g])); + const members = (g: LocalGroup) => { + const m = g.shadow?.members; + return m && typeof m === 'object' ? (m as Record) : {}; + }; + const edits = new Map>(); + for (const contact of contacts) { + if (contact.deleted) continue; + const uid = typeof contact.shadow?.uid === 'string' ? contact.shadow.uid : contact.pending?.uid; + const account = parseObjectRef(contact.sourceId)?.accountId; + if (!uid || !account) continue; + const remote = groups + .filter((g) => g.sourceId && !g.deleted && members(g)[uid] === true) + .map((g) => g.sourceId!); + const base = contact.shadow ? shadowMemberOf(contact.shadow) : []; + if (base === null) continue; + const units = cardUnits(contact.shadow, ctx); + const evidence = inPlaceEvidence(KIND_ORDER.map((k) => matchKind(k, contact.rows, units))); + const plan = planMemberships({ + rows: contact.rows, base, remote, evidence, restore: false, parent: contact.rawContactId, + groupRowIdBySourceId: () => null, + }); + const put = (gsid: string, on: boolean) => { + const g = bySource.get(gsid); + const gref = parseObjectRef(gsid); + if (!g || g.deleted || !g.shadow || !gref || gref.accountId !== account) return; + if (ctx.isReadOnly(g.shadow)) return; + if (!edits.has(g)) edits.set(g, new Map()); + edits.get(g)!.set(uid, on); + }; + for (const g of plan.added) put(g, true); + for (const g of plan.removed) put(g, false); + } + const actions: UploadAction[] = []; + for (const [g, changes] of edits) { + const current = members(g); + const patch: PatchObject = {}; + if (!Object.keys(current).length) { + const whole: Record = {}; + for (const [uid, on] of changes) if (on) whole[uid] = true; + if (!Object.keys(whole).length) continue; + patch.members = whole; + } else { + for (const [uid, on] of changes) patch[ptr('members', uid)] = on ? true : null; + } + actions.push({ kind: 'update', id: parseObjectRef(g.sourceId)!.id, patch }); + } + return actions; +} diff --git a/src/device-sync/contacts/keys.ts b/src/device-sync/contacts/keys.ts new file mode 100644 index 00000000..547af4b2 --- /dev/null +++ b/src/device-sync/contacts/keys.ts @@ -0,0 +1,65 @@ +/** + * DATA_SYNC1 of a contacts data row: which card entry (or entries) the row + * was written from. Keys are hints, not identity (docs/device-sync.md, "Row + * matching"): editors that re-insert rows drop them. + * + * name the StructuredName row + * emails:work1 one entry of a map + * organizations:o1|titles:t1,r1 an organization with its title and role key + * titles:,r1 a role without an organization + * relatedTo:urn:uuid:… relatedTo is keyed by its value + * members:/ a group membership + * + * Map keys may hold any character; `%`, `|` and `,` are escaped so the + * organization form stays parseable. + */ + +export const NAME_KEY = 'name'; + +const escapeKey = (key: string) => key.replace(/%/g, '%25').replace(/\|/g, '%7C').replace(/,/g, '%2C'); + +function unescapeKey(text: string): string | null { + try { + return decodeURIComponent(text); + } catch { + return null; + } +} + +export function entryKey(map: string, key: string): string { + return `${map}:${escapeKey(key)}`; +} + +/** `{ map, key }` of a simple entry key; null for the name, organizations and anything unreadable. */ +export function parseEntryKey(text: string | null): { map: string; key: string } | null { + if (!text) return null; + const colon = text.indexOf(':'); + if (colon <= 0) return null; + const map = text.slice(0, colon); + if (map === 'organizations' || map === 'titles') return null; + const key = unescapeKey(text.slice(colon + 1)); + return key ? { map, key } : null; +} + +export interface OrgKeyParts { + org: string | null; + title: string | null; + role: string | null; +} + +export function orgKey({ org, title, role }: OrgKeyParts): string { + const titles = title || role ? `titles:${title ? escapeKey(title) : ''},${role ? escapeKey(role) : ''}` : ''; + if (org === null) return titles; + return titles ? `organizations:${escapeKey(org)}|${titles}` : `organizations:${escapeKey(org)}`; +} + +export function parseOrgKey(text: string | null): OrgKeyParts | null { + if (!text) return null; + const m = /^(?:organizations:([^|]*))?(?:\|?titles:([^,]*),([^,]*))?$/.exec(text); + if (!m || (m[1] === undefined && m[2] === undefined)) return null; + const part = (s: string | undefined) => (s ? unescapeKey(s) : null); + const org = part(m[1]); + if (m[1] !== undefined && !org) return null; + const parts = { org, title: part(m[2]), role: part(m[3]) }; + return parts.org || parts.title || parts.role ? parts : null; +} diff --git a/src/device-sync/contacts/matching.ts b/src/device-sync/contacts/matching.ts new file mode 100644 index 00000000..b1e321e0 --- /dev/null +++ b/src/device-sync/contacts/matching.ts @@ -0,0 +1,131 @@ +/** + * Row matching and key reconciliation (docs/device-sync.md, "Row matching" + * and "Key reconciliation"). Keys are hints: Fossify re-inserts every row but + * the name on each save, without DATA_SYNC columns, dropping what it does not + * model; AOSP edits in place. Stalwart regenerates `k1…kN` keys by position + * for cards written over CardDAV, so keys can shift under us. + */ +import type { LocalDataRow } from '../planner'; +import { isEmptyRow, SPECS, type Unit } from './project'; + +export interface Match { + row: LocalDataRow; + unit: Unit; + how: 'key' | 'value' | 'pair'; +} + +export interface KindMatch { + mimetype: string; + matched: Match[]; + /** Rows no unit accounts for: new entries. */ + fresh: LocalDataRow[]; + /** Units without a row. */ + missing: Unit[]; + /** Non-empty rows of the kind. */ + rows: LocalDataRow[]; +} + +/** + * Matches one kind's non-empty rows to units: by key, then by the kind's + * primary value, then (only when the counts agree) pairwise in order. + */ +export function matchKind(mimetype: string, rows: LocalDataRow[], units: Unit[]): KindMatch { + const spec = SPECS[mimetype]; + const live = rows.filter((r) => r.mimetype === mimetype && !isEmptyRow(mimetype, r.cells)); + const kindUnits = units.filter((u) => u.mimetype === mimetype); + const matched: Match[] = []; + const takenUnits = new Set(); + const takenRows = new Set(); + const byKey = new Map(kindUnits.map((u) => [u.key, u])); + + for (const row of live) { + const u = row.key ? byKey.get(row.key) : undefined; + if (u && !takenUnits.has(u)) { + matched.push({ row, unit: u, how: 'key' }); + takenUnits.add(u); + takenRows.add(row); + } + } + for (const row of live) { + if (takenRows.has(row)) continue; + const p = spec.primary(row.cells); + if (p === null) continue; + const u = kindUnits.find((x) => !takenUnits.has(x) && spec.primary(x.cells) === p); + if (u) { + matched.push({ row, unit: u, how: 'value' }); + takenUnits.add(u); + takenRows.add(row); + } + } + const restRows = live.filter((r) => !takenRows.has(r)); + const restUnits = kindUnits.filter((u) => !takenUnits.has(u)); + if (restRows.length && restRows.length === restUnits.length) { + restRows.forEach((row, i) => matched.push({ row, unit: restUnits[i], how: 'pair' })); + return { mimetype, matched, fresh: [], missing: [], rows: live }; + } + return { mimetype, matched, fresh: restRows, missing: restUnits, rows: live }; +} + +/** + * Whether the units of a kind that lost their rows were deleted on the + * device. Only when the kind still has rows matched by key (an in-place + * editor removed one); when every row is keyless an editor rewrote them all + * and a missing entry may just be one it could not show. A kind left without + * any row is judged by the rest of the contact: keyed rows of kinds that + * re-inserting editors rewrite mean an in-place editor did it. + */ +export function deletionsInferred(kind: KindMatch, inPlaceEvidence: boolean): boolean { + const spec = SPECS[kind.mimetype]; + if (spec.inPlaceOnly) return true; + if (kind.matched.some((m) => m.how === 'key')) return true; + return kind.rows.length === 0 && inPlaceEvidence; +} + +/** Keyed rows of rewritten kinds: evidence that the last editor worked in place. */ +export function inPlaceEvidence(kinds: KindMatch[]): boolean { + return kinds.some((k) => SPECS[k.mimetype].rewritten && k.matched.some((m) => m.how === 'key')); +} + +/** + * Maps the keys of the shadow's units to the new server version's keys by + * content: equal entries first, then equal primary values, then an + * unchanged key (an entry edited in place). Unmapped units were removed on + * the server; unmapped new units were added. + */ +export function reconcileKeys(base: Unit[], remote: Unit[]): Map { + const out = new Map(); + const kinds = new Set([...base, ...remote].map((u) => u.mimetype)); + for (const mimetype of kinds) { + const spec = SPECS[mimetype]; + const b = base.filter((u) => u.mimetype === mimetype); + const r = remote.filter((u) => u.mimetype === mimetype); + const takenB = new Set(); + const takenR = new Set(); + const link = (x: Unit, y: Unit) => { + out.set(x.key, y.key); + takenB.add(x); + takenR.add(y); + }; + for (const x of b) { + const y = r.find((u) => !takenR.has(u) && u.key === x.key && u.source === x.source); + if (y) link(x, y); + } + for (const x of b) { + if (takenB.has(x)) continue; + const y = r.find((u) => !takenR.has(u) && u.source === x.source); + if (y) link(x, y); + } + for (const x of b) { + if (takenB.has(x)) continue; + const p = spec.primary(x.cells); + const y = p === null ? undefined : r.find((u) => !takenR.has(u) && spec.primary(u.cells) === p); + if (y) link(x, y); + } + for (const x of b) { + if (takenB.has(x)) continue; + const y = r.find((u) => !takenR.has(u) && u.key === x.key); + if (y) link(x, y); + } + } + return out; +} diff --git a/src/device-sync/contacts/members.ts b/src/device-sync/contacts/members.ts new file mode 100644 index 00000000..0a07983b --- /dev/null +++ b/src/device-sync/contacts/members.ts @@ -0,0 +1,149 @@ +/** + * Group memberships of a contact: one GroupMembership row per group card + * whose `members` holds the card's uid (docs/device-sync.md, "Contacts + * mapping"). The server keeps them on the group cards, so the rows are + * compared with three sets: + * + * - base: the memberships the rows held after our last write, kept in the + * contact's shadow under `~memberOf` (the card itself has no such field); + * - local: the rows now; + * - remote: the group cards that list the uid now (`groupsOf`, from the + * engine's group shadows). + * + * Per group, a local change wins where the server did not change, the server + * wins elsewhere; both sides can only flip a membership the same way, so + * memberships never conflict. A local change stays in the base until the + * server has it, which keeps it visible to `planMembershipUploads`. + */ +import { Data, GroupMembership, MimeType } from '../android-columns'; +import type { ContactCardWire, ContactsContext, LocalDataRow } from '../planner'; +import type { ProviderOp } from '../types'; +import { text } from './cells'; +import { entryKey } from './keys'; +import { deleteData, insertUnit, updateData } from './ops'; +import { MEMBER_OF, type Unit } from './project'; + +/** Remote memberships of a card, or null when the engine can't tell (memberships are then left alone). */ +export function remoteMemberships(card: ContactCardWire, ctx: ContactsContext): string[] | null { + if (typeof ctx.groupsOf !== 'function' || typeof card.uid !== 'string' || !card.uid) return null; + return [...new Set(ctx.groupsOf(card.uid).filter((g) => typeof g === 'string'))].sort(); +} + +export function shadowMemberOf(shadow: ContactCardWire | null): string[] | null { + const v = shadow?.[MEMBER_OF]; + return Array.isArray(v) && v.every((g) => typeof g === 'string') ? [...v] : null; +} + +/** A card without the `~memberOf` a shadow carries. */ +export function withoutMemberOf(card: ContactCardWire): ContactCardWire { + if (!(MEMBER_OF in card)) return card; + const { [MEMBER_OF]: _drop, ...rest } = card; + return rest as ContactCardWire; +} + +export function withMemberOf(shadow: ContactCardWire, memberOf: string[] | null): ContactCardWire { + const out = withoutMemberOf(shadow); + return memberOf === null ? out : { ...out, [MEMBER_OF]: [...memberOf].sort() }; +} + +function membershipUnit(gsid: string, groupRowId: number): Unit { + return { + mimetype: MimeType.GROUP_MEMBERSHIP, + key: entryKey('members', gsid), + cells: { [GroupMembership.GROUP_SOURCE_ID]: gsid }, + extra: { [GroupMembership.GROUP_ROW_ID]: groupRowId }, + source: JSON.stringify(gsid), + }; +} + +/** The group SOURCE_ID a membership row points at; null for a group not yet on the server. */ +export function rowGroup(row: LocalDataRow): string | null { + return text(row.cells[GroupMembership.GROUP_SOURCE_ID]); +} + +export interface MemberPlan { + ops: ProviderOp[]; + writes: number; + /** The new base (`~memberOf`); null when memberships are not synced. */ + memberOf: string[] | null; + /** Local membership changes the server does not have yet. */ + pending: boolean; + /** Groups the device added the contact to, or removed it from, that the server does not show yet. */ + added: string[]; + removed: string[]; +} + +export interface MemberInput { + rows: LocalDataRow[]; + base: string[] | null; + remote: string[] | null; + /** Deleted rows count as removals (keyed rows of the kind, or in-place evidence). */ + evidence: boolean; + /** Clean writes put back memberships an editor did not show; merges leave them. */ + restore: boolean; + parent: number | { ref: number }; + groupRowIdBySourceId(sourceId: string): number | null; +} + +export function planMemberships(input: MemberInput): MemberPlan { + const live = input.rows.filter((r) => r.mimetype === MimeType.GROUP_MEMBERSHIP); + if (input.remote === null) { + // Without the engine's group index memberships are not synced: nothing is written or kept pending. + return { ops: [], writes: 0, memberOf: input.base, pending: false, added: [], removed: [] }; + } + const ops: ProviderOp[] = []; + const rowOf = new Map(); + let unsynced = false; + for (const r of live) { + const g = rowGroup(r); + if (g === null) unsynced = true; + else if (!rowOf.has(g)) rowOf.set(g, r); + } + const keyed = live.some((r) => r.key !== null && r.key === entryKey('members', rowGroup(r) ?? '')); + const deletions = keyed || (live.length === 0 && input.evidence); + const base = new Set(input.base ?? rowOf.keys()); + const remote = new Set(input.remote); + const after = new Set(); + const pendingAdded = new Set(); + const pendingRemoved = new Set(); + + const keep = (g: string, row: LocalDataRow) => { + after.add(g); + const key = entryKey('members', g); + if (row.key !== key) ops.push(updateData(row.id, { [Data.DATA_SYNC1]: key })); + }; + const insert = (g: string) => { + const groupRowId = input.groupRowIdBySourceId(g); + if (groupRowId === null) return; + ops.push(insertUnit(input.parent, membershipUnit(g, groupRowId), true)); + after.add(g); + }; + + for (const g of [...new Set([...base, ...remote, ...rowOf.keys()])].sort()) { + const inBase = base.has(g); + const inRemote = remote.has(g); + const row = rowOf.get(g); + const change = inBase ? (!row && deletions ? 'removed' : null) : row ? 'added' : null; + if (change === 'added') { + keep(g, row!); + if (!inRemote) pendingAdded.add(g); + } else if (change === 'removed') { + if (inRemote) pendingRemoved.add(g); + } else if (row) { + if (inRemote) keep(g, row); + else ops.push(deleteData(row.id)); + } else if (inRemote && (!inBase || input.restore)) { + insert(g); + } + } + const memberOf = [...after].filter((g) => !pendingAdded.has(g)); + for (const g of pendingRemoved) memberOf.push(g); + return { + ops, + writes: ops.length, + memberOf: memberOf.sort(), + pending: unsynced || pendingAdded.size > 0 || pendingRemoved.size > 0, + added: [...pendingAdded], + removed: [...pendingRemoved], + }; +} diff --git a/src/device-sync/contacts/name.ts b/src/device-sync/contacts/name.ts new file mode 100644 index 00000000..7704e7b8 --- /dev/null +++ b/src/device-sync/contacts/name.ts @@ -0,0 +1,213 @@ +/** + * `name` ↔ the StructuredName row (docs/device-sync.md, "Name"). + * + * Download writes DISPLAY_NAME and the components together, so the provider + * leaves both alone; a name with `full` only is split by the provider, which + * the baseline predicts roughly and the baseline heal corrects. Upload + * replaces the first component of each edited column's kinds, keeps every + * other component, and sends the whole `name/components` array (JSON Pointer + * can't address array elements). `name/full` goes along when the display name + * was typed, or re-derived when only components changed and the server's + * `full` was the derived one (Stalwart never re-derives it by itself). + */ +import type { ContactName, NameComponent } from '../../api/types'; +import { deriveFullName } from '../../lib/contact-utils'; +import { StructuredName as SN } from '../android-columns'; +import { clone } from '../common/json'; +import type { Row } from '../types'; +import { text } from './cells'; + +type Component = NameComponent & { phonetic?: string }; + +export const NAME_COLUMNS = [ + SN.DISPLAY_NAME, SN.GIVEN_NAME, SN.FAMILY_NAME, SN.PREFIX, SN.MIDDLE_NAME, SN.SUFFIX, + SN.PHONETIC_GIVEN_NAME, SN.PHONETIC_MIDDLE_NAME, SN.PHONETIC_FAMILY_NAME, +] as const; + +/** The component columns the provider splits a lone display name into, or joins into one. */ +const NAME_PART_COLUMNS = [SN.PREFIX, SN.GIVEN_NAME, SN.MIDDLE_NAME, SN.FAMILY_NAME, SN.SUFFIX] as const; + +interface ColumnSpec { + column: string; + kinds: string[]; + /** Rank in display order, for placing a component the name did not have. */ + rank: number; +} + +const PARTS: ColumnSpec[] = [ + { column: SN.PREFIX, kinds: ['title', 'prefix'], rank: 0 }, + { column: SN.GIVEN_NAME, kinds: ['given'], rank: 1 }, + { column: SN.MIDDLE_NAME, kinds: ['given2', 'middle', 'additional'], rank: 2 }, + { column: SN.FAMILY_NAME, kinds: ['surname'], rank: 3 }, + { column: SN.SUFFIX, kinds: ['generation', 'credential', 'suffix'], rank: 5 }, +]; + +const PHONETICS: Array<{ column: string; part: string }> = [ + { column: SN.PHONETIC_GIVEN_NAME, part: SN.GIVEN_NAME }, + { column: SN.PHONETIC_MIDDLE_NAME, part: SN.MIDDLE_NAME }, + { column: SN.PHONETIC_FAMILY_NAME, part: SN.FAMILY_NAME }, +]; + +const RANK: Record = { surname2: 4 }; +for (const p of PARTS) for (const k of p.kinds) RANK[k] = p.rank; + +function components(name: ContactName | undefined): Component[] { + return Array.isArray(name?.components) + ? (name!.components as Component[]).filter((c) => c && typeof c.kind === 'string') + : []; +} + +const firstOf = (comps: Component[], kinds: string[]) => + comps.find((c) => kinds.includes(c.kind) && typeof c.value === 'string'); + +/** StructuredName cells of a card's name; null when there is no name to show. */ +export function nameCells(name: ContactName | undefined): Row | null { + if (!name || typeof name !== 'object') return null; + const comps = components(name); + const cells: Row = {}; + for (const p of PARTS) cells[p.column] = text(firstOf(comps, p.kinds)?.value); + const surname2 = comps.filter((c) => c.kind === 'surname2').map((c) => text(c.value)).filter(Boolean); + if (surname2.length) cells[SN.FAMILY_NAME] = [cells[SN.FAMILY_NAME], ...surname2].filter(Boolean).join(' '); + for (const ph of PHONETICS) { + const spec = PARTS.find((p) => p.column === ph.part)!; + cells[ph.column] = text(firstOf(comps, spec.kinds)?.phonetic); + } + const full = typeof name.full === 'string' && name.full.trim() ? name.full : deriveFullName(comps); + cells[SN.DISPLAY_NAME] = text(full); + return Object.values(cells).some((v) => v !== null) ? cells : null; +} + +/** + * What ContactsProvider stores for written name cells: a display name without + * components is split into them (roughly as its NameSplitter does for simple + * names; the baseline heal corrects the rest). + */ +export function predictStoredName(cells: Row): Row { + const out = { ...cells }; + const display = text(cells[SN.DISPLAY_NAME]); + if (display && NAME_PART_COLUMNS.every((c) => text(cells[c]) === null)) { + const words = display.trim().split(/\s+/); + out[SN.GIVEN_NAME] = words[0] ?? null; + out[SN.FAMILY_NAME] = words.length > 1 ? words.slice(1).join(' ') : null; + } + return out; +} + +/** Whether a stored name row is what writing `wanted` gives (the provider's split allowed). */ +export function nameMatches(wanted: Row, stored: Row, same: (a: unknown, b: unknown) => boolean): boolean { + const split = NAME_PART_COLUMNS.every((c) => text(wanted[c]) === null); + return NAME_COLUMNS.every((c) => (split && (NAME_PART_COLUMNS as readonly string[]).includes(c)) || same(wanted[c], stored[c])); +} + +const squash = (s: string) => s.replace(/\s+/g, ' ').trim(); + +/** The display names the provider (or an editor) computes from the components. */ +function joins(cells: Row): string[] { + const [prefix, given, middle, family, suffix] = NAME_PART_COLUMNS.map((c) => text(cells[c])); + const western = [prefix, given, middle, family].filter(Boolean).join(' '); + return [ + [prefix, given, middle, family, suffix].filter(Boolean).join(' '), + suffix ? `${western}, ${suffix}` : western, + [prefix, family, middle, given, suffix].filter(Boolean).join(''), + [prefix, family, given, middle].filter(Boolean).join(' ') + (suffix ? `, ${suffix}` : ''), + ].map(squash); +} + +const NAME_KINDS = new Set(['surname', 'given', 'given2', 'title', 'credential', 'surname2', 'generation']); + +/** `full` as Stalwart derives it (components in order, separators verbatim). */ +function serverDerivedFull(name: ContactName): string { + const sep = typeof name.defaultSeparator === 'string' && name.defaultSeparator ? name.defaultSeparator : ' '; + let full = ''; + let afterValue = false; + for (const c of components(name)) { + if (typeof c.value !== 'string') continue; + if (c.kind === 'separator') { + full += c.value; + afterValue = false; + } else if (NAME_KINDS.has(c.kind)) { + if (afterValue) full += sep; + full += c.value; + afterValue = true; + } + } + return full; +} + +/** Whether the server's `full` only repeats its components (so it follows them). */ +export function fullIsDerived(name: ContactName | undefined): boolean { + if (!name) return true; + const full = typeof name.full === 'string' ? squash(name.full) : ''; + if (!full) return true; + const comps = components(name); + return full === squash(deriveFullName(comps)) || full === squash(serverDerivedFull(name)); +} + +function place(comps: Component[], component: Component): void { + const rank = RANK[component.kind] ?? 99; + const at = comps.findIndex((c) => (RANK[c.kind] ?? -1) > rank); + if (at < 0) comps.push(component); + else comps.splice(at, 0, component); +} + +/** The server's components with the edited columns applied. */ +function editComponents(name: ContactName | undefined, cells: Row, changed: ReadonlySet): Component[] { + const comps = clone(components(name)); + for (const p of PARTS) { + if (!changed.has(p.column)) continue; + const value = text(cells[p.column]); + if (p.column === SN.FAMILY_NAME) { + for (let i = comps.length - 1; i >= 0; i--) if (comps[i].kind === 'surname2') comps.splice(i, 1); + } + const target = firstOf(comps, p.kinds); + if (target && value !== null) target.value = value; + else if (target) comps.splice(comps.indexOf(target), 1); + else if (value !== null) place(comps, { kind: p.kinds[0], value }); + } + for (const ph of PHONETICS) { + if (!changed.has(ph.column)) continue; + const spec = PARTS.find((p) => p.column === ph.part)!; + const value = text(cells[ph.column]); + const target = firstOf(comps, spec.kinds); + if (target && value !== null) target.phonetic = value; + else if (target) delete target.phonetic; + else if (value !== null) place(comps, { kind: spec.kinds[0], value: '', phonetic: value }); + } + return comps; +} + +/** + * The name part of a patch for edited name columns: `name/components`, + * `name/full`, or the whole `name` when the card has none. + */ +export function namePatch(name: ContactName | undefined, cells: Row, changed: ReadonlySet): Record { + const out: Record = {}; + const partsChanged = [...PARTS.map((p) => p.column), ...PHONETICS.map((p) => p.column)].some((c) => changed.has(c)); + const displayChanged = changed.has(SN.DISPLAY_NAME); + if (!partsChanged && !displayChanged) return out; + const display = text(cells[SN.DISPLAY_NAME]); + if (name && typeof name === 'object' && components(name).length === 0) { + // The server holds this name as `full` alone, so the device's components are the + // provider's split of it: the name goes up whole from the row. + const all = editComponents(undefined, cells, new Set(NAME_COLUMNS)); + const full = display ?? (deriveFullName(all) || null); + if (all.length) out['name/components'] = all; + if (full !== (typeof name.full === 'string' ? name.full : null)) out['name/full'] = full; + return out; + } + const comps = partsChanged ? editComponents(name, cells, changed) : components(name); + let full: string | null | undefined; + if (displayChanged && display && !joins(cells).includes(squash(display))) full = display; + else if (fullIsDerived(name)) full = deriveFullName(comps) || display || null; + const hasName = !!name && typeof name === 'object'; + if (!hasName) { + const whole: Record = {}; + if (comps.length) whole.components = comps; + if (full) whole.full = full; + if (Object.keys(whole).length) out.name = whole; + return out; + } + if (partsChanged) out['name/components'] = comps.length ? comps : null; + if (full !== undefined && full !== (typeof name!.full === 'string' ? name!.full : null)) out['name/full'] = full; + return out; +} diff --git a/src/device-sync/contacts/ops.ts b/src/device-sync/contacts/ops.ts new file mode 100644 index 00000000..063b7b0b --- /dev/null +++ b/src/device-sync/contacts/ops.ts @@ -0,0 +1,75 @@ +/** + * Provider ops the contacts planner emits. Every write into an existing item + * is guarded: the group asserts the VERSION (and DIRTY) the plan was made + * from, and ops that address one row by id expect exactly one row + * (docs/device-sync.md, "Merge rules"). + */ +import { Data, GroupMembership, MimeType, RawContacts } from '../android-columns'; +import type { LocalContact } from '../planner'; +import type { ProviderOp, WriteRow } from '../types'; +import { encodeBaseline } from './cells'; +import { predictStored, SPECS, type Unit } from './project'; + +export function assertContact(local: LocalContact, dirty: boolean | null): ProviderOp { + const values: Record = { [RawContacts.VERSION]: local.version }; + if (dirty !== null) values[RawContacts.DIRTY] = dirty ? 1 : 0; + return { op: 'assert', table: 'raw_contacts', id: local.rawContactId, values, expectCount: 1 }; +} + +export function updateRawContact(id: number, values: WriteRow): ProviderOp { + return { op: 'update', table: 'raw_contacts', id, values, expectCount: 1 }; +} + +/** The cells a unit writes (a membership names its group by row id: GROUP_SOURCE_ID is read-only here). */ +function unitValues(u: Unit): WriteRow { + const out: WriteRow = {}; + for (const [column, value] of Object.entries(u.cells)) { + if (column !== GroupMembership.GROUP_SOURCE_ID) out[column] = value; + } + return { ...out, ...(u.extra ?? {}) }; +} + +/** DATA_SYNC1..3 for a row written from a unit: key, photo hash and the predicted stored cells. */ +export function syncValues(u: Unit): WriteRow { + const out: WriteRow = { + [Data.DATA_SYNC1]: u.key, + [Data.DATA_SYNC3]: encodeBaseline(predictStored(u), SPECS[u.mimetype].columns), + }; + if (u.mimetype === MimeType.PHOTO) out[Data.DATA_SYNC2] = u.photoHash ?? null; + return out; +} + +/** Inserts a unit's row; `parent` is the raw contact id, or `{ ref }` to an insert earlier in the group. */ +export function insertUnit(parent: number | { ref: number }, u: Unit, keyed: boolean): ProviderOp { + const values: WriteRow = { [Data.MIMETYPE]: u.mimetype, ...unitValues(u), ...(keyed ? syncValues(u) : {}) }; + if (typeof parent === 'number') { + values[Data.RAW_CONTACT_ID] = parent; + return { op: 'insert', table: 'data', values }; + } + return { op: 'insert', table: 'data', values, refs: { [Data.RAW_CONTACT_ID]: parent.ref } }; +} + +/** Rewrites a row from a unit; photo bytes only go when the hash differs. */ +export function updateUnit(rowId: number, u: Unit, keyed: boolean, withBytes = true): ProviderOp { + const values = unitValues(u); + if (!withBytes) delete values[Data.DATA15]; + return { op: 'update', table: 'data', id: rowId, values: { ...values, ...(keyed ? syncValues(u) : {}) }, expectCount: 1 }; +} + +export function updateData(rowId: number, values: WriteRow): ProviderOp { + return { op: 'update', table: 'data', id: rowId, values, expectCount: 1 }; +} + +export function deleteData(rowId: number): ProviderOp { + return { op: 'delete', table: 'data', id: rowId, expectCount: 1 }; +} + +/** A delete that tolerates a row that is already gone (purges, server deletes). */ +export function deleteWhereId(table: 'raw_contacts' | 'groups' | 'data', id: number): ProviderOp { + return { op: 'delete', table, where: '_id = ?', args: [id] }; +} + +/** A write that tolerates a row that is already gone (keep-dirty baselines of rows edited meanwhile). */ +export function updateWhereId(table: 'data', id: number, values: WriteRow): ProviderOp { + return { op: 'update', table, where: '_id = ?', args: [id], values }; +} diff --git a/src/device-sync/contacts/photo.ts b/src/device-sync/contacts/photo.ts new file mode 100644 index 00000000..05eb171e --- /dev/null +++ b/src/device-sync/contacts/photo.ts @@ -0,0 +1,91 @@ +/** + * The contact photo: the card's first `media` entry of kind `photo` + * (docs/device-sync.md, "Contacts mapping"). Rows get the full-size bytes + * and DATA_SYNC2 the hash of what was applied; the shadow keeps the hash + * instead of the bytes (`sha256:`), so a raw contact row stays small. + */ +import type { ContactMedia } from '../../api/types'; +import { normalizeContactPhotoUri } from '../../lib/contact-utils'; +import { sha256Hex } from '../../lib/sha256'; +import { clone } from '../common/json'; +import type { ContactCardWire, ContactsContext } from '../planner'; + +const HASH_URI = 'sha256:'; +const B64 = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/'; +const B64_INDEX = new Map([...B64].map((c, i) => [c, i])); + +/** Bytes of standard or URL-safe base64 (padding optional); null for anything else. */ +export function base64ToBytes(b64: string): Uint8Array | null { + const clean = b64.replace(/\s+/g, '').replace(/-/g, '+').replace(/_/g, '/').replace(/=+$/, ''); + if (clean.length % 4 === 1) return null; + const out = new Uint8Array(Math.floor((clean.length * 3) / 4)); + let bits = 0; + let value = 0; + let o = 0; + for (const ch of clean) { + const v = B64_INDEX.get(ch); + if (v === undefined) return null; + value = (value << 6) | v; + bits += 6; + if (bits >= 8) { + bits -= 8; + out[o++] = (value >> bits) & 0xff; + } + } + return out.subarray(0, o); +} + +/** The base64 payload of a `data:…;base64,` URI (Stalwart may leave out the media type, #307). */ +export function dataUriBase64(uri: string): string | null { + const m = /^data:[^,]*;base64,(.*)$/is.exec(normalizeContactPhotoUri(uri.trim())); + return m ? m[1].replace(/\s+/g, '') : null; +} + +export function hashBase64(b64: string): string | null { + const bytes = base64ToBytes(b64); + return bytes ? `${HASH_URI}${sha256Hex(bytes)}` : null; +} + +/** The photo entry the device shows: the first `kind: "photo"` entry in map order. */ +export function photoEntry(card: ContactCardWire): { key: string; entry: ContactMedia } | null { + const media = card.media; + if (!media || typeof media !== 'object') return null; + for (const [key, entry] of Object.entries(media)) { + if (entry && typeof entry === 'object' && entry.kind === 'photo' && (entry.uri || (entry as { blobId?: unknown }).blobId)) { + return { key, entry }; + } + } + return null; +} + +/** Bytes (base64) and hash of a photo entry; bytes are null for a shadow's hash-only entry. */ +export function photoPayload(entry: ContactMedia, ctx: Pick): { b64: string | null; hash: string } | null { + const uri = typeof entry.uri === 'string' ? entry.uri : ''; + if (uri.startsWith(HASH_URI)) return { b64: null, hash: uri }; + const b64 = dataUriBase64(uri) ?? ctx.photoBytes(entry as { uri?: string; blobId?: string; mediaType?: string }); + const hash = b64 ? hashBase64(b64) : null; + return b64 && hash ? { b64, hash } : null; +} + +/** The shadow of a card: every `data:` photo URI replaced by the hash of its bytes. */ +export function shadowOf(card: ContactCardWire): ContactCardWire { + const out = clone(card); + const media = out.media; + if (media && typeof media === 'object') { + for (const entry of Object.values(media)) { + if (!entry || typeof entry.uri !== 'string') continue; + const b64 = dataUriBase64(entry.uri); + const hash = b64 ? hashBase64(b64) : null; + if (hash) entry.uri = hash; + } + } + return out; +} + +export function isHashUri(uri: unknown): boolean { + return typeof uri === 'string' && uri.startsWith(HASH_URI); +} + +export function jpegDataUri(b64: string): string { + return `data:image/jpeg;base64,${b64}`; +} diff --git a/src/device-sync/contacts/planner.ts b/src/device-sync/contacts/planner.ts new file mode 100644 index 00000000..f7c79f2d --- /dev/null +++ b/src/device-sync/contacts/planner.ts @@ -0,0 +1,78 @@ +/** + * The contacts planner (docs/device-sync.md, "Contacts mapping" and "Change + * detection and merge rules"): JSContact cards ↔ ContactsContract rows for + * individual and org cards, group cards ↔ Groups rows. Pure: the engine does + * every query, batch and request. + * + * Memberships are synced only when the context has `groupsOf`. + */ +import { Data, MimeType } from '../android-columns'; +import type { ContactsPlanner, LocalContact, OpGroup } from '../planner'; +import type { ProviderOp } from '../types'; +import { changedFromBaseline, encodeBaseline } from './cells'; +import { DATA_COLUMNS, decodeContact, decodeGroup, GROUP_COLUMNS, RAW_CONTACT_COLUMNS } from './columns'; +import { cardUnits, planContactDownload } from './download'; +import { + groupFingerprint, + planGroupAccepted, + planGroupDownload, + planGroupLocalDelete, + planGroupUpload, + planMembershipUploads, +} from './groups'; +import { entryKey } from './keys'; +import { rowGroup } from './members'; +import { assertContact, deleteWhereId, updateData } from './ops'; +import { SPECS } from './project'; +import { contactFingerprint, planContactAccepted, planContactUpload } from './upload'; + +export { contactFingerprint, groupFingerprint }; + +/** Enough context to project a shadow for its keys. */ +const OFFLINE = { nameForUid: () => null, photoBytes: () => null, groupRowIdBySourceId: () => null }; + +function planLocalDelete(local: LocalContact): OpGroup { + return { ref: local.sourceId ?? `row:${local.rawContactId}`, ops: [deleteWhereId('raw_contacts', local.rawContactId)] }; +} + +/** + * Baselines of a clean contact's rows that no longer describe the rows (the + * provider normalised a write, or the read-back after a chunk never ran): + * rewritten to the rows' current cells. Photo rows get the file id the + * provider assigned. + */ +function planBaselineHeal(local: LocalContact): OpGroup | null { + if (local.dirty || local.deleted || !local.sourceId) return null; + const known = new Set(cardUnits(local.shadow, OFFLINE).map((u) => u.key)); + const ops: ProviderOp[] = []; + for (const row of local.rows) { + const spec = SPECS[row.mimetype]; + if (!spec || !row.key) continue; + const ours = row.mimetype === MimeType.GROUP_MEMBERSHIP ? row.key === entryKey('members', rowGroup(row) ?? '') : known.has(row.key); + if (!ours) continue; + const stale = !row.baseline + || changedFromBaseline(row.cells, row.baseline, spec.columns).length > 0 + || spec.columns.some((c) => !(c in row.baseline!)); + if (stale) ops.push(updateData(row.id, { [Data.DATA_SYNC3]: encodeBaseline(row.cells, spec.columns) })); + } + if (!ops.length) return null; + return { ref: local.sourceId, ops: [assertContact(local, false), ...ops] }; +} + +export const contactsPlanner: ContactsPlanner = { + rawContactColumns: RAW_CONTACT_COLUMNS, + dataColumns: DATA_COLUMNS, + groupColumns: GROUP_COLUMNS, + decodeContact, + decodeGroup, + planDownload: planContactDownload, + planLocalDelete, + planBaselineHeal, + planUpload: planContactUpload, + planAccepted: planContactAccepted, + planGroupDownload, + planGroupLocalDelete, + planGroupUpload, + planGroupAccepted, + planMembershipUploads, +}; diff --git a/src/device-sync/contacts/postal.ts b/src/device-sync/contacts/postal.ts new file mode 100644 index 00000000..fd7d590e --- /dev/null +++ b/src/device-sync/contacts/postal.ts @@ -0,0 +1,129 @@ +/** + * `addresses/` ↔ a StructuredPostal row (docs/device-sync.md, "Postal + * components"). Each column joins the components of its kinds in order; an + * edited column replaces all components it was built from with one + * component of its first kind and keeps the rest (separators between other + * components, unknown kinds, phonetics). + */ +import type { AddressComponent, ContactAddress } from '../../api/types'; +import { StructuredPostal as SP } from '../android-columns'; +import { clone } from '../common/json'; +import type { Row } from '../types'; +import { text } from './cells'; + +export const POSTAL_PART_COLUMNS = [SP.STREET, SP.POBOX, SP.NEIGHBORHOOD, SP.CITY, SP.REGION, SP.POSTCODE, SP.COUNTRY] as const; + +const PARTS: Array<{ column: string; kinds: string[]; legacy?: keyof ContactAddress }> = [ + { column: SP.STREET, kinds: ['name', 'number', 'building', 'floor', 'apartment', 'room', 'block', 'direction', 'landmark'], legacy: 'street' }, + { column: SP.POBOX, kinds: ['postOfficeBox'] }, + { column: SP.NEIGHBORHOOD, kinds: ['district', 'subdistrict'] }, + { column: SP.CITY, kinds: ['locality'], legacy: 'locality' }, + { column: SP.REGION, kinds: ['region'], legacy: 'region' }, + { column: SP.POSTCODE, kinds: ['postcode'], legacy: 'postcode' }, + { column: SP.COUNTRY, kinds: ['country'], legacy: 'country' }, +]; + +type Component = AddressComponent; + +const RANK: Record = {}; +PARTS.forEach((p, i) => p.kinds.forEach((k) => (RANK[k] = i))); + +function components(address: ContactAddress): Component[] { + if (Array.isArray(address.components) && address.components.length) { + return address.components.filter((c) => c && typeof c.kind === 'string'); + } + // Legacy flat fields are read when there are no components (never written). + const out: Component[] = []; + for (const p of PARTS) { + const v = p.legacy ? address[p.legacy] : undefined; + if (typeof v === 'string' && v.trim()) out.push({ kind: p.kinds[0], value: v }); + } + return out; +} + +/** Joins the components of `kinds` in order; a separator between two of them is kept verbatim. */ +function joinKinds(comps: Component[], kinds: string[], defaultSeparator: string): string | null { + let out = ''; + let prevWasPart = false; + for (let i = 0; i < comps.length; i++) { + const c = comps[i]; + if (!kinds.includes(c.kind) || typeof c.value !== 'string' || !c.value.trim()) { + if (c.kind !== 'separator') prevWasPart = false; + continue; + } + if (out) { + const prev = comps[i - 1]; + out += prevWasPart && prev?.kind === 'separator' && typeof prev.value === 'string' ? prev.value : defaultSeparator; + } + out += c.value; + prevWasPart = true; + } + return text(out); +} + +/** The single-line address written when the card has no `full`, so the provider leaves the row alone. */ +function joinPostal(cells: Row): string | null { + const [street, pobox, neighborhood, city, region, postcode, country] = POSTAL_PART_COLUMNS.map((c) => text(cells[c])); + const place = [postcode, city].filter(Boolean).join(' '); + return text([street, pobox, neighborhood, place, region, country].filter(Boolean).join(', ')); +} + +/** Postal cells (without TYPE/LABEL) of an address. */ +export function postalCells(address: ContactAddress): Row { + const comps = components(address); + const sep = typeof address.defaultSeparator === 'string' && address.defaultSeparator ? address.defaultSeparator : ' '; + const cells: Row = {}; + for (const p of PARTS) cells[p.column] = joinKinds(comps, p.kinds, p.column === SP.STREET ? sep : ', '); + const full = typeof address.full === 'string' && address.full.trim() ? address.full + : typeof address.fullAddress === 'string' && address.fullAddress.trim() ? address.fullAddress + : joinPostal(cells); + cells[SP.FORMATTED_ADDRESS] = text(full); + return cells; +} + +/** What the provider stores: a formatted address without components is copied into STREET. */ +export function predictStoredPostal(cells: Row): Row { + const out = { ...cells }; + if (text(cells[SP.FORMATTED_ADDRESS]) && POSTAL_PART_COLUMNS.every((c) => text(cells[c]) === null)) { + out[SP.STREET] = cells[SP.FORMATTED_ADDRESS]; + } + return out; +} + +/** The address's components with the edited columns applied (a list for `addresses//components`). */ +export function editPostalComponents(address: ContactAddress, cells: Row, changed: ReadonlySet): Component[] { + const comps = clone(components(address)); + for (const p of PARTS) { + if (!changed.has(p.column)) continue; + const value = text(cells[p.column]); + const first = comps.findIndex((c) => p.kinds.includes(c.kind)); + const drop = new Set(); + comps.forEach((c, i) => { + if (p.kinds.includes(c.kind)) drop.add(i); + }); + // Separators only between removed parts (or at their edge) go with them. + comps.forEach((c, i) => { + if (c.kind !== 'separator') return; + const before = i > 0 && drop.has(i - 1); + const after = i < comps.length - 1 && drop.has(i + 1); + if ((before && (after || i === comps.length - 1)) || (after && i === 0)) drop.add(i); + }); + const kept = comps.filter((_, i) => !drop.has(i)); + if (value !== null) { + let at = comps.slice(0, Math.max(first, 0)).filter((_, i) => !drop.has(i)).length; + if (first < 0) { + // A part the address did not have goes before the first part that follows it in display order. + const later = kept.findIndex((c) => (RANK[c.kind] ?? -1) > (RANK[p.kinds[0]] ?? 0)); + at = later < 0 ? kept.length : later; + } + kept.splice(at, 0, { kind: p.kinds[0], value }); + } + comps.splice(0, comps.length, ...kept); + } + return comps; +} + +/** Components for a new address from its row. */ +export function postalComponentsFromRow(cells: Row): Component[] { + return editPostalComponents({}, cells, new Set(POSTAL_PART_COLUMNS)); +} diff --git a/src/device-sync/contacts/project.ts b/src/device-sync/contacts/project.ts new file mode 100644 index 00000000..b548fd13 --- /dev/null +++ b/src/device-sync/contacts/project.ts @@ -0,0 +1,328 @@ +/** + * A card as the data rows the device should hold for it: one `Unit` per row, + * with its DATA_SYNC1 key, its mapped cells and whatever else the write needs + * (docs/device-sync.md, "Contacts mapping"). Units are what changes are + * detected, merged and uploaded by. + */ +import type { + ContactAddress, + ContactAnniversary, + ContactEmail, + ContactLink, + ContactNickname, + ContactNote, + ContactOrganization, + ContactPhone, + ContactRelation, + ContactTitle, +} from '../../api/types'; +import { partialDateToString } from '../../lib/contact-utils'; +import { + CommonKind, + Data, + GroupMembership, + MimeType, + NicknameType, + Organization as Org, + StructuredPostal as SP, +} from '../android-columns'; +import { canonicalJson } from '../common/json'; +import type { ContactCardWire, ContactsContext } from '../planner'; +import type { Row, WriteRow } from '../types'; +import { cellText, text } from './cells'; +import { NAME_KEY, entryKey, orgKey } from './keys'; +import { NAME_COLUMNS, nameCells, nameMatches, predictStoredName } from './name'; +import { photoEntry, photoPayload } from './photo'; +import { POSTAL_PART_COLUMNS, postalCells, predictStoredPostal } from './postal'; +import { emailType, eventType, phoneType, postalType, relationType, websiteType } from './types-map'; + +/** The group SOURCE_IDs the rows' memberships were written from, kept in the shadow (see `memberOf`). */ +export const MEMBER_OF = '~memberOf'; + +export interface KindSpec { + mimetype: string; + /** Mapped columns: recorded in the baseline, compared for changes. */ + columns: readonly string[]; + /** Columns a row matched by value may upload when non-empty and different (not TYPE/LABEL/IS_PRIMARY). */ + valueColumns: readonly string[]; + /** Whether a row matched by value uploads its TYPE/LABEL too (phones, emails, addresses, events). */ + typed: boolean; + /** Value-match key of a row; null when the row has nothing to match by. */ + primary(cells: Row): string | null; + /** Editors that re-insert rows (Fossify) rewrite this kind on every save. */ + rewritten: boolean; + /** Missing rows of this kind are always deletions: no editor re-inserts it. */ + inPlaceOnly?: boolean; +} + +const low = (v: unknown) => text(v)?.trim().toLowerCase() ?? null; +const squash = (v: unknown) => low(v)?.replace(/[\s,]+/g, ' ').trim() || null; + +const TYPED = [CommonKind.TYPE, CommonKind.LABEL]; + +export const SPECS: Record = { + [MimeType.STRUCTURED_NAME]: { + mimetype: MimeType.STRUCTURED_NAME, columns: NAME_COLUMNS, valueColumns: NAME_COLUMNS, typed: false, + primary: () => NAME_KEY, rewritten: false, + }, + [MimeType.NICKNAME]: { + mimetype: MimeType.NICKNAME, columns: [Data.DATA1], valueColumns: [Data.DATA1], typed: false, + primary: (c) => low(c[Data.DATA1]), rewritten: true, + }, + [MimeType.EMAIL]: { + mimetype: MimeType.EMAIL, columns: [Data.DATA1, ...TYPED, Data.IS_PRIMARY], valueColumns: [Data.DATA1], typed: true, + primary: (c) => low(c[Data.DATA1]), rewritten: true, + }, + [MimeType.PHONE]: { + mimetype: MimeType.PHONE, columns: [Data.DATA1, ...TYPED, Data.IS_PRIMARY], valueColumns: [Data.DATA1], typed: true, + primary: (c) => { + const digits = text(c[Data.DATA1])?.replace(/\D/g, ''); + return digits || low(c[Data.DATA1]); + }, + rewritten: true, + }, + [MimeType.STRUCTURED_POSTAL]: { + mimetype: MimeType.STRUCTURED_POSTAL, + columns: [SP.FORMATTED_ADDRESS, ...TYPED, ...POSTAL_PART_COLUMNS], + valueColumns: [SP.FORMATTED_ADDRESS, ...POSTAL_PART_COLUMNS], + typed: true, + primary: (c) => squash(c[SP.FORMATTED_ADDRESS]) ?? squash(POSTAL_PART_COLUMNS.map((k) => text(c[k])).filter(Boolean).join(' ')), + rewritten: true, + }, + [MimeType.ORGANIZATION]: { + mimetype: MimeType.ORGANIZATION, + columns: [Org.COMPANY, Org.TITLE, Org.DEPARTMENT, Org.JOB_DESCRIPTION], + valueColumns: [Org.COMPANY, Org.TITLE, Org.DEPARTMENT, Org.JOB_DESCRIPTION], + typed: false, + primary: (c) => low(c[Org.COMPANY]) ?? low(c[Org.TITLE]) ?? low(c[Org.JOB_DESCRIPTION]), + rewritten: true, + }, + [MimeType.WEBSITE]: { + mimetype: MimeType.WEBSITE, columns: [Data.DATA1, ...TYPED], valueColumns: [Data.DATA1], typed: false, + primary: (c) => low(c[Data.DATA1])?.replace(/\/+$/, '') ?? null, rewritten: true, + }, + [MimeType.EVENT]: { + mimetype: MimeType.EVENT, columns: [Data.DATA1, ...TYPED], valueColumns: [Data.DATA1], typed: true, + primary: (c) => (text(c[Data.DATA1]) ? `${text(c[Data.DATA1])!.trim()}|${cellText(c[Data.DATA2]) ?? ''}` : null), + rewritten: true, + }, + [MimeType.RELATION]: { + mimetype: MimeType.RELATION, columns: [Data.DATA1, ...TYPED], valueColumns: [Data.DATA1], typed: false, + primary: (c) => low(c[Data.DATA1]), rewritten: false, + }, + [MimeType.NOTE]: { + mimetype: MimeType.NOTE, columns: [Data.DATA1], valueColumns: [Data.DATA1], typed: false, + primary: (c) => text(c[Data.DATA1])?.trim() ?? null, rewritten: true, + }, + [MimeType.PHOTO]: { + mimetype: MimeType.PHOTO, columns: [Data.DATA14], valueColumns: [], typed: false, + primary: () => 'photo', rewritten: false, inPlaceOnly: true, + }, + [MimeType.GROUP_MEMBERSHIP]: { + mimetype: MimeType.GROUP_MEMBERSHIP, columns: [GroupMembership.GROUP_SOURCE_ID], valueColumns: [], typed: false, + primary: (c) => text(c[GroupMembership.GROUP_SOURCE_ID]), rewritten: true, + }, +}; + +/** Kinds in the order rows are written and compared. */ +export const KIND_ORDER = [ + MimeType.STRUCTURED_NAME, MimeType.NICKNAME, MimeType.EMAIL, MimeType.PHONE, MimeType.STRUCTURED_POSTAL, + MimeType.ORGANIZATION, MimeType.WEBSITE, MimeType.EVENT, MimeType.RELATION, MimeType.NOTE, MimeType.PHOTO, + MimeType.GROUP_MEMBERSHIP, +]; + +/** One data row the card calls for. */ +export interface Unit { + mimetype: string; + /** DATA_SYNC1. */ + key: string; + /** The mapped cells, as written. */ + cells: Row; + /** Written but not mapped: a nickname's TYPE, photo bytes, a membership's group row id. */ + extra?: WriteRow; + /** Photos: the hash of the server photo (DATA_SYNC2); unknown when its bytes are not at hand. */ + photoHash?: string; + /** Canonical JSON of the card entries the row comes from, for key reconciliation. */ + source: string; + /** Columns cut to the provider's limit: the device never held all of them, so they never upload. */ + truncated?: string[]; +} + +export interface ProjectOptions { + /** Group SOURCE_IDs whose `members` hold the card's uid; null leaves memberships alone. */ + memberOf: string[] | null; +} + +const entries = (map: unknown): Array<[string, T]> => + map && typeof map === 'object' && !Array.isArray(map) + ? Object.entries(map as Record).filter(([, v]) => v && typeof v === 'object') + : []; + +/** ContactsProvider cuts text columns to 10 KiB and phone numbers to 1000 characters, and keeps the cut. */ +const MAX_TEXT = 10 * 1024; +const MAX_NUMBER = 1000; + +function unit(mimetype: string, key: string, cells: Row, source: unknown, extra?: WriteRow): Unit { + const out: Unit = { mimetype, key, cells, extra, source: canonicalJson(source) }; + for (const [column, value] of Object.entries(cells)) { + const max = mimetype === MimeType.PHONE && column === Data.DATA1 ? MAX_NUMBER : MAX_TEXT; + if (typeof value === 'string' && value.length > max) { + cells[column] = value.slice(0, max); + (out.truncated ??= []).push(column); + } + } + return out; +} + +/** The first entry of a pref-ordered map that is preferred (`pref: 1`) marks IS_PRIMARY. */ +function primaryKey(list: Array<[string, T]>): string | null { + return list.find(([, e]) => e.pref === 1)?.[0] ?? null; +} + +function organizationUnits(card: ContactCardWire): Unit[] { + const orgs = entries(card.organizations); + const titles = entries(card.titles).filter(([, t]) => text(t.name)); + const used = new Set(); + const pick = (org: string | null, role: boolean) => { + const hit = titles.find(([k, t]) => !used.has(k) && (t.organizationId ?? null) === org && (t.kind === 'role') === role); + if (hit) used.add(hit[0]); + return hit ?? null; + }; + const out: Unit[] = []; + const push = (org: [string, ContactOrganization] | null, title: [string, ContactTitle] | null, role: [string, ContactTitle] | null) => { + const units = Array.isArray(org?.[1].units) + ? org![1].units!.map((u) => text(u?.name)).filter(Boolean).join(', ') + : null; + const cells: Row = { + [Org.COMPANY]: text(org?.[1].name), + [Org.TITLE]: text(title?.[1].name), + [Org.DEPARTMENT]: text(units), + [Org.JOB_DESCRIPTION]: text(role?.[1].name), + }; + if (Object.values(cells).every((v) => v === null)) return; + const key = orgKey({ org: org?.[0] ?? null, title: title?.[0] ?? null, role: role?.[0] ?? null }); + out.push(unit(MimeType.ORGANIZATION, key, cells, [org?.[1] ?? null, title?.[1] ?? null, role?.[1] ?? null])); + }; + for (const org of orgs) { + const orgId = org[0]; + push(org, pick(orgId, false), pick(orgId, true)); + } + // Titles of no (or an unknown) organization get rows of their own. + for (const t of titles) { + if (used.has(t[0])) continue; + used.add(t[0]); + if (t[1].kind === 'role') push(null, null, t); + else push(null, t, null); + } + return out; +} + +/** Every data row a card maps to, in `KIND_ORDER`. */ +export function projectCard(card: ContactCardWire, ctx: Pick, options: ProjectOptions): Unit[] { + const out: Unit[] = []; + const name = nameCells(card.name); + if (name) out.push(unit(MimeType.STRUCTURED_NAME, NAME_KEY, name, card.name)); + + for (const [k, e] of entries(card.nicknames)) { + if (!text(e.name)) continue; + out.push(unit(MimeType.NICKNAME, entryKey('nicknames', k), { [Data.DATA1]: text(e.name) }, e, { [Data.DATA2]: NicknameType.DEFAULT })); + } + + const emails = entries(card.emails).filter(([, e]) => text(e.address)); + const primaryEmail = primaryKey(emails); + for (const [k, e] of emails) { + const t = emailType(e); + out.push(unit(MimeType.EMAIL, entryKey('emails', k), { + [Data.DATA1]: text(e.address), [Data.DATA2]: t.type, [Data.DATA3]: t.label, [Data.IS_PRIMARY]: k === primaryEmail ? 1 : 0, + }, e)); + } + + const phones = entries(card.phones).filter(([, e]) => text(e.number)); + const primaryPhone = primaryKey(phones); + for (const [k, e] of phones) { + const t = phoneType(e); + out.push(unit(MimeType.PHONE, entryKey('phones', k), { + [Data.DATA1]: text(e.number), [Data.DATA2]: t.type, [Data.DATA3]: t.label, [Data.IS_PRIMARY]: k === primaryPhone ? 1 : 0, + }, e)); + } + + for (const [k, e] of entries(card.addresses)) { + const cells = postalCells(e); + if (Object.values(cells).every((v) => v === null)) continue; + const t = postalType(e); + out.push(unit(MimeType.STRUCTURED_POSTAL, entryKey('addresses', k), { ...cells, [SP.TYPE]: t.type, [SP.LABEL]: t.label }, e)); + } + + out.push(...organizationUnits(card)); + + for (const [k, e] of entries(card.links)) { + if (!text(e.uri)) continue; + const t = websiteType(e); + out.push(unit(MimeType.WEBSITE, entryKey('links', k), { [Data.DATA1]: text(e.uri), [Data.DATA2]: t.type, [Data.DATA3]: t.label }, e)); + } + + for (const [k, e] of entries(card.anniversaries)) { + const date = text(partialDateToString(e.date)); + if (!date) continue; + const t = eventType(e.kind); + out.push(unit(MimeType.EVENT, entryKey('anniversaries', k), { [Data.DATA1]: date, [Data.DATA2]: t.type, [Data.DATA3]: t.label }, e)); + } + + for (const [uri, e] of entries(card.relatedTo)) { + if (!uri.trim()) continue; + const shown = (uri.startsWith('urn:uuid:') ? ctx.nameForUid(uri) : null) ?? uri; + const t = relationType(e.relation); + out.push(unit(MimeType.RELATION, entryKey('relatedTo', uri), { [Data.DATA1]: shown, [Data.DATA2]: t.type, [Data.DATA3]: t.label }, e)); + } + + const note = entries(card.notes).find(([, e]) => text(e.note)); + if (note) out.push(unit(MimeType.NOTE, entryKey('notes', note[0]), { [Data.DATA1]: text(note[1].note) }, note[1])); + + const photo = photoEntry(card); + if (photo) { + // Without its bytes (a blob the engine could not fetch) the photo still counts, so an + // existing row is kept, not taken for a deletion; it just can't be written. + const payload = photoPayload(photo.entry, ctx); + const u = unit(MimeType.PHOTO, entryKey('media', photo.key), {}, payload?.hash ?? photo.entry); + if (payload) u.photoHash = payload.hash; + if (payload?.b64) u.extra = { [Data.DATA15]: { b64: payload.b64 } }; + out.push(u); + } + + for (const gsid of options.memberOf ?? []) { + const groupRowId = ctx.groupRowIdBySourceId(gsid); + // A membership naming an unknown group would make the provider create an empty one. + if (groupRowId === null) continue; + out.push(unit(MimeType.GROUP_MEMBERSHIP, entryKey('members', gsid), { [GroupMembership.GROUP_SOURCE_ID]: gsid }, gsid, { + [GroupMembership.GROUP_ROW_ID]: groupRowId, + })); + } + return out; +} + +/** What the provider stores for a unit's cells (it splits a lone display name or address). */ +export function predictStored(u: Pick): Row { + if (u.mimetype === MimeType.STRUCTURED_NAME) return predictStoredName(u.cells); + if (u.mimetype === MimeType.STRUCTURED_POSTAL) return predictStoredPostal(u.cells); + return u.cells; +} + +/** Whether a stored row is what writing the unit gives, allowing for the provider's normalisation. */ +export function rowMatchesUnit(u: Pick, stored: Row, same: (a: unknown, b: unknown, c?: string) => boolean): boolean { + if (u.mimetype === MimeType.STRUCTURED_NAME) return nameMatches(u.cells, stored, same); + const spec = SPECS[u.mimetype]; + if (u.mimetype === MimeType.STRUCTURED_POSTAL && POSTAL_PART_COLUMNS.every((c) => text(u.cells[c]) === null)) { + return spec.columns.filter((c) => !(POSTAL_PART_COLUMNS as readonly string[]).includes(c)).every((c) => same(u.cells[c], stored[c], c)); + } + if (u.mimetype === MimeType.PHOTO) return true; + return spec.columns.every((c) => same(u.cells[c], stored[c], c)); +} + +/** Whether a row has nothing in it (an editor's blank nickname or note). */ +export function isEmptyRow(mimetype: string, cells: Row): boolean { + const spec = SPECS[mimetype]; + if (!spec) return true; + if (mimetype === MimeType.PHOTO) return cellText(cells[Data.DATA14]) === null; + if (mimetype === MimeType.GROUP_MEMBERSHIP) return cellText(cells[GroupMembership.GROUP_SOURCE_ID]) === null && cellText(cells[Data.DATA1]) === null; + return spec.valueColumns.every((c) => text(cells[c]) === null); +} diff --git a/src/device-sync/contacts/types-map.ts b/src/device-sync/contacts/types-map.ts new file mode 100644 index 00000000..bd39d86c --- /dev/null +++ b/src/device-sync/contacts/types-map.ts @@ -0,0 +1,223 @@ +/** + * JSContact contexts, features, kinds and relation types ↔ the TYPE/LABEL + * columns of Android's typed data kinds (docs/device-sync.md, "Contacts + * mapping"). Uploads replace only the vocabulary a TYPE stands for and keep + * the rest (a phone's `text` feature, a `school` context). + */ +import { + ContactEventType, + EmailType, + PhoneType, + PostalType, + RelationType, + WebsiteType, +} from '../android-columns'; + +const on = (flags: unknown, name: string) => + !!flags && typeof flags === 'object' && (flags as Record)[name] === true; + +/** TYPE + LABEL as the row holds them. */ +export interface TypeCells { + type: number; + label: string | null; +} + +/** What a TYPE says about an entry: the contexts, features and label to upload. */ +export interface EntryFlags { + contexts: string[]; + features?: string[]; + /** The label a custom type carries; null removes the entry's label. */ + label: string | null; +} + +function labelled(label: unknown, type: number, custom: number): TypeCells { + return typeof label === 'string' && label.trim() ? { type: custom, label } : { type, label: null }; +} + +// ─── Emails ──────────────────────────────────────────── + +export function emailType(entry: { contexts?: unknown; label?: unknown }): TypeCells { + const type = on(entry.contexts, 'private') ? EmailType.HOME : on(entry.contexts, 'work') ? EmailType.WORK : EmailType.OTHER; + return labelled(entry.label, type, EmailType.CUSTOM); +} + +export function emailFlags(type: number | null, label: string | null): EntryFlags { + switch (type) { + case EmailType.HOME: return { contexts: ['private'], label: null }; + case EmailType.WORK: return { contexts: ['work'], label: null }; + case EmailType.MOBILE: return { contexts: [], label: 'mobile' }; + case EmailType.CUSTOM: return { contexts: [], label }; + default: return { contexts: [], label: null }; + } +} + +// ─── Phones ──────────────────────────────────────────── + +/** First match wins, as in the design's table; `cell` is read as `mobile`. */ +export function phoneType(entry: { features?: unknown; contexts?: unknown; label?: unknown }): TypeCells { + const f = (n: string) => on(entry.features, n); + const c = (n: string) => on(entry.contexts, n); + const mobile = f('mobile') || f('cell'); + let type: number = PhoneType.OTHER; + if (mobile && c('work')) type = PhoneType.WORK_MOBILE; + else if (mobile) type = PhoneType.MOBILE; + else if (f('fax') && c('work')) type = PhoneType.FAX_WORK; + else if (f('fax') && c('private')) type = PhoneType.FAX_HOME; + else if (f('fax')) type = PhoneType.OTHER_FAX; + else if (f('pager') && c('work')) type = PhoneType.WORK_PAGER; + else if (f('pager')) type = PhoneType.PAGER; + else if (f('textphone')) type = PhoneType.TTY_TDD; + else if (f('main-number')) type = PhoneType.MAIN; + else if (c('work')) type = PhoneType.WORK; + else if (c('private')) type = PhoneType.HOME; + return labelled(entry.label, type, PhoneType.CUSTOM); +} + +/** The features a phone TYPE stands for; uploads swap these and keep the others. */ +export const PHONE_TYPE_FEATURES = ['mobile', 'cell', 'fax', 'pager', 'textphone', 'main-number', 'voice']; + +const PHONE_LABELS: Record = { + [PhoneType.CALLBACK]: 'callback', + [PhoneType.CAR]: 'car', + [PhoneType.ISDN]: 'isdn', + [PhoneType.RADIO]: 'radio', + [PhoneType.TELEX]: 'telex', + [PhoneType.ASSISTANT]: 'assistant', +}; + +export function phoneFlags(type: number | null, label: string | null): EntryFlags { + switch (type) { + case PhoneType.WORK_MOBILE: return { contexts: ['work'], features: ['mobile'], label: null }; + case PhoneType.MOBILE: return { contexts: [], features: ['mobile'], label: null }; + case PhoneType.FAX_WORK: return { contexts: ['work'], features: ['fax'], label: null }; + case PhoneType.FAX_HOME: return { contexts: ['private'], features: ['fax'], label: null }; + case PhoneType.OTHER_FAX: return { contexts: [], features: ['fax'], label: null }; + case PhoneType.WORK_PAGER: return { contexts: ['work'], features: ['pager'], label: null }; + case PhoneType.PAGER: return { contexts: [], features: ['pager'], label: null }; + case PhoneType.TTY_TDD: return { contexts: [], features: ['textphone'], label: null }; + case PhoneType.MAIN: return { contexts: [], features: ['main-number'], label: null }; + case PhoneType.COMPANY_MAIN: return { contexts: ['work'], features: ['main-number'], label: null }; + case PhoneType.WORK: return { contexts: ['work'], features: ['voice'], label: null }; + case PhoneType.HOME: return { contexts: ['private'], features: ['voice'], label: null }; + case PhoneType.MMS: return { contexts: [], features: ['text'], label: null }; + case PhoneType.CUSTOM: return { contexts: [], features: [], label }; + case null: + case PhoneType.OTHER: return { contexts: [], features: [], label: null }; + default: return { contexts: [], features: ['voice'], label: PHONE_LABELS[type] ?? null }; + } +} + +// ─── Addresses and websites ──────────────────────────── + +export function postalType(entry: { contexts?: unknown; label?: unknown }): TypeCells { + const type = on(entry.contexts, 'private') ? PostalType.HOME : on(entry.contexts, 'work') ? PostalType.WORK : PostalType.OTHER; + return labelled(entry.label, type, PostalType.CUSTOM); +} + +export function postalFlags(type: number | null, label: string | null): EntryFlags { + switch (type) { + case PostalType.HOME: return { contexts: ['private'], label: null }; + case PostalType.WORK: return { contexts: ['work'], label: null }; + case PostalType.CUSTOM: return { contexts: [], label }; + default: return { contexts: [], label: null }; + } +} + +export function websiteType(entry: { contexts?: unknown; label?: unknown }): TypeCells { + const type = on(entry.contexts, 'private') ? WebsiteType.HOME : on(entry.contexts, 'work') ? WebsiteType.WORK : WebsiteType.OTHER; + return labelled(entry.label, type, WebsiteType.CUSTOM); +} + +const WEBSITE_LABELS: Record = { + [WebsiteType.HOMEPAGE]: 'homepage', + [WebsiteType.BLOG]: 'blog', + [WebsiteType.PROFILE]: 'profile', + [WebsiteType.FTP]: 'ftp', +}; + +export function websiteFlags(type: number | null, label: string | null): EntryFlags { + switch (type) { + case WebsiteType.HOME: return { contexts: ['private'], label: null }; + case WebsiteType.WORK: return { contexts: ['work'], label: null }; + case WebsiteType.CUSTOM: return { contexts: [], label }; + case null: + case WebsiteType.OTHER: return { contexts: [], label: null }; + default: return { contexts: [], label: WEBSITE_LABELS[type] ?? null }; + } +} + +/** Contexts the TYPEs above speak for; others are kept on upload. */ +export const TYPED_CONTEXTS = ['private', 'work']; + +// ─── Anniversaries ───────────────────────────────────── + +export function eventType(kind: unknown): TypeCells { + switch (kind) { + case 'birth': return { type: ContactEventType.BIRTHDAY, label: null }; + case 'wedding': return { type: ContactEventType.ANNIVERSARY, label: null }; + case 'other': return { type: ContactEventType.OTHER, label: null }; + default: return { type: ContactEventType.CUSTOM, label: typeof kind === 'string' && kind ? kind : null }; + } +} + +/** The anniversary kind for a device TYPE; a custom label becomes the kind itself. */ +export function eventKind(type: number | null, label: string | null): string { + switch (type) { + case ContactEventType.BIRTHDAY: return 'birth'; + case ContactEventType.ANNIVERSARY: return 'wedding'; + case ContactEventType.CUSTOM: return label?.trim() ? label.trim().toLowerCase() : 'other'; + default: return 'other'; + } +} + +// ─── Relations ───────────────────────────────────────── + +/** RFC 9553 relation types in the order they pick the device TYPE. */ +const RELATION_TO_TYPE: Array<[string, number]> = [ + ['friend', RelationType.FRIEND], + ['spouse', RelationType.SPOUSE], + ['child', RelationType.CHILD], + ['parent', RelationType.PARENT], + ['sibling', RelationType.RELATIVE], + ['kin', RelationType.RELATIVE], + ['agent', RelationType.ASSISTANT], +]; + +export function relationType(relation: unknown): TypeCells { + const types = relation && typeof relation === 'object' + ? Object.keys(relation as object).filter((k) => on(relation, k)) + : []; + for (const [name, type] of RELATION_TO_TYPE) if (types.includes(name)) return { type, label: null }; + return { type: RelationType.CUSTOM, label: types[0] ?? null }; +} + +/** Device types without an RFC 9553 twin upload as the nearest one. */ +const TYPE_TO_RELATION: Record = { + [RelationType.SPOUSE]: 'spouse', + [RelationType.CHILD]: 'child', + [RelationType.PARENT]: 'parent', + [RelationType.FRIEND]: 'friend', + [RelationType.RELATIVE]: 'kin', + [RelationType.ASSISTANT]: 'agent', + [RelationType.BROTHER]: 'sibling', + [RelationType.SISTER]: 'sibling', + [RelationType.MOTHER]: 'parent', + [RelationType.FATHER]: 'parent', + [RelationType.MANAGER]: 'colleague', + [RelationType.DOMESTIC_PARTNER]: 'spouse', + [RelationType.PARTNER]: 'spouse', + [RelationType.REFERRED_BY]: 'contact', +}; + +export function relationName(type: number | null, label: string | null): string | null { + if (type === RelationType.CUSTOM || type === null) return label?.trim() ? label.trim().toLowerCase() : null; + return TYPE_TO_RELATION[type] ?? null; +} + +/** The relation types a device TYPE covers, removed when the TYPE changes. */ +export function relationTypesOf(type: number | null, label: string | null): string[] { + const covered = RELATION_TO_TYPE.filter(([, t]) => t === type).map(([name]) => name); + const own = relationName(type, label); + if (own && !covered.includes(own)) covered.push(own); + return covered; +} diff --git a/src/device-sync/contacts/upload.ts b/src/device-sync/contacts/upload.ts new file mode 100644 index 00000000..b9d221dc --- /dev/null +++ b/src/device-sync/contacts/upload.ts @@ -0,0 +1,292 @@ +/** + * Device changes to the server (docs/device-sync.md, "Uploads"): per changed + * unit a patch of the sub-fields whose columns changed; a whole map only + * where Stalwart needs one (the first entry of a map, and deletions, which + * resend the map from the fresh shadow minus the entry, except `addresses`, + * where `addresses/: null` is clean); never a property Android can't + * represent. New contacts are claimed (a uid and a target book, written + * before anything is sent) and then created. + */ +import { Data, MimeType, RawContacts } from '../android-columns'; +import { collectionKey, parseCollectionKey, parseObjectRef, objectRef } from '../common/ids'; +import { jsonHash } from '../common/json'; +import { applyPatch, ptr, setInPatch, type PatchObject } from '../common/patch'; +import type { + AcceptedPlan, + ContactCardWire, + ContactsContext, + LocalContact, + OpGroup, + UploadAction, + UploadPlan, +} from '../planner'; +import type { ProviderOp } from '../types'; +import { encodeBaseline } from './cells'; +import { localChanges } from './diff'; +import { cardUnits, cleanWrite } from './download'; +import { newEntries, unitEdits, unitEntries, type Edit, type NewEntry } from './entries'; +import { parseEntryKey } from './keys'; +import { deletionsInferred, inPlaceEvidence, matchKind, type KindMatch } from './matching'; +import { shadowMemberOf, withMemberOf, withoutMemberOf } from './members'; +import { NAME_COLUMNS, namePatch } from './name'; +import { assertContact, deleteWhereId, updateRawContact, updateWhereId } from './ops'; +import { isHashUri, jpegDataUri, shadowOf } from './photo'; +import { KIND_ORDER, SPECS } from './project'; + +type Ctx = ContactsContext; + +const ENTRY_KINDS = KIND_ORDER.filter((k) => k !== MimeType.GROUP_MEMBERSHIP); + +/** + * What a poison marker's `fp` is taken of: the engine writes the markers + * (engine/poison.ts) with this same content, so a skip here agrees with them. + */ +export function contactFingerprint(local: LocalContact): string { + return jsonHash({ + deleted: local.deleted, + shadow: local.shadow, + rows: local.rows.map((r) => JSON.stringify([r.mimetype, r.cells])).sort(), + }); +} + +function backedOff(local: LocalContact, ctx: Ctx, actions?: UploadAction[]): boolean { + const p = local.poison; + if (!p || !(p.until > ctx.now)) return false; + return p.fp === contactFingerprint(local) || (actions !== undefined && p.fp === jsonHash(actions)); +} + +interface Changes { + edits: Edit[]; + added: NewEntry[]; + removed: Array<{ map: string; key: string }>; + name: Record; + /** Rows whose keyed units uploaded, for the keep-dirty baselines. */ + uploadedRows: number[]; +} + +/** Everything the rows changed against a shadow (an empty card for a new contact). */ +function collectChanges(local: LocalContact, shadow: ContactCardWire, ctx: Ctx): Changes { + const base = cardUnits(shadow, ctx); + const kinds: KindMatch[] = ENTRY_KINDS.map((k) => matchKind(k, local.rows, base)); + const evidence = inPlaceEvidence(kinds); + const out: Changes = { edits: [], added: [], removed: [], name: {}, uploadedRows: [] }; + for (const kind of kinds) { + if (kind.mimetype === MimeType.STRUCTURED_NAME) { + const m = kind.matched[0]; + const row = m?.row ?? kind.fresh[0]; + if (!row) continue; + const changed = m ? localChanges(m) : [...NAME_COLUMNS]; + out.name = namePatch(m ? shadow.name : undefined, row.cells, new Set(changed)); + if (m && changed.length) out.uploadedRows.push(row.id); + continue; + } + if (kind.mimetype === MimeType.PHOTO) { + photoChanges(local, kind, evidence, ctx, out); + continue; + } + for (const m of kind.matched) { + const changed = localChanges(m); + if (!changed.length) continue; + const shown = kind.mimetype === MimeType.RELATION ? (m.unit.cells[Data.DATA1] as string | null) : null; + const e = unitEdits(kind.mimetype, m.unit.key, shadow, m.row.cells, m.unit.cells, new Set(changed), shown, m.how === 'key'); + out.edits.push(...e.edits); + out.added.push(...e.added); + out.removed.push(...e.removed); + if (m.how === 'key') out.uploadedRows.push(m.row.id); + } + for (const row of kind.fresh) out.added.push(...newEntries(kind.mimetype, row.cells)); + if (deletionsInferred(kind, evidence)) { + for (const unit of kind.missing) out.removed.push(...unitEntries(kind.mimetype, unit.key)); + } + } + return out; +} + +function photoChanges(local: LocalContact, kind: KindMatch, evidence: boolean, ctx: Ctx, out: Changes): void { + const m = kind.matched[0]; + const row = m?.row ?? kind.fresh[0]; + if (row && (!m || localChanges(m).length)) { + const jpeg = ctx.devicePhoto(local.rawContactId); + if (!jpeg) return; + const uri = jpegDataUri(jpeg); + if (m) { + const key = parseEntryKey(m.unit.key)!.key; + out.edits.push({ path: ['media', key, 'uri'], value: uri }, { path: ['media', key, 'mediaType'], value: 'image/jpeg' }); + out.uploadedRows.push(row.id); + } else { + out.added.push({ map: 'media', value: { kind: 'photo', uri, mediaType: 'image/jpeg' } }); + } + } else if (!row && deletionsInferred(kind, evidence)) { + for (const unit of kind.missing) out.removed.push(...unitEntries(kind.mimetype, unit.key)); + } +} + +const mapOf = (card: ContactCardWire, map: string): Record | null => { + const v = card[map]; + return v && typeof v === 'object' && !Array.isArray(v) ? (v as Record) : null; +}; + +/** The PatchObject for a set of changes against the shadow. */ +function assemble(changes: Changes, shadow: ContactCardWire, ctx: Ctx): PatchObject { + const patch: PatchObject = {}; + for (const [pointer, value] of Object.entries(changes.name)) setInPatch(patch, pointer, value); + + // Keys for new entries; titles may point at an organization minted here. + const minted = new Map(); + const taken = new Map>(); + const takenOf = (map: string) => { + if (!taken.has(map)) taken.set(map, new Set(Object.keys(mapOf(shadow, map) ?? {}))); + return taken.get(map)!; + }; + const added = changes.added.filter((e) => { + const keys = takenOf(e.map); + if (e.key !== undefined) { + if (keys.has(e.key) && !changes.removed.some((r) => r.map === e.map && r.key === e.key)) return false; + minted.set(e, e.key); + } else { + minted.set(e, ctx.mintKey(keys)); + } + keys.add(minted.get(e)!); + return true; + }); + const resolve = (value: unknown): unknown => (value && typeof value === 'object' && minted.has(value as NewEntry) ? minted.get(value as NewEntry) : value); + const entryValue = (e: NewEntry) => { + const v: Record = {}; + for (const [k, x] of Object.entries(e.value)) v[k] = resolve(x); + return v; + }; + + const maps = new Set([...added.map((e) => e.map), ...changes.removed.map((r) => r.map)]); + for (const map of maps) { + const current = mapOf(shadow, map); + const removed = changes.removed.filter((r) => r.map === map).map((r) => r.key); + const adds = added.filter((e) => e.map === map); + const remaining = Object.keys(current ?? {}).filter((k) => !removed.includes(k)); + // A map with a placeholder photo URI (the shadow keeps hashes) can't be resent whole. + const resendable = map !== 'media' || remaining.every((k) => !isHashUri((current![k] as { uri?: unknown })?.uri)); + const empty = !current || !Object.keys(current).length; + if (!empty && (map === 'addresses' || (removed.length && !resendable))) { + for (const key of removed) setInPatch(patch, ptr(map, key), null); + for (const e of adds) setInPatch(patch, ptr(map, minted.get(e)!), entryValue(e)); + continue; + } + if (removed.length || empty) { + const whole: Record = {}; + for (const k of remaining) whole[k] = current![k]; + for (const e of adds) whole[minted.get(e)!] = entryValue(e); + setInPatch(patch, ptr(map), Object.keys(whole).length ? whole : null); + continue; + } + for (const e of adds) setInPatch(patch, ptr(map, minted.get(e)!), entryValue(e)); + } + + for (const edit of changes.edits) { + const [map, key] = edit.path; + if (changes.removed.some((r) => r.map === map && r.key === key)) continue; + setInPatch(patch, ptr(...edit.path), resolve(edit.value)); + } + return patch; +} + +/** A new contact's card from its rows: only RFC 9553 properties. */ +export function createObject(local: LocalContact, uid: string, bookId: string, ctx: Ctx): Partial { + const empty = {} as ContactCardWire; + const changes = collectChanges(local, empty, ctx); + changes.removed = []; + const body = applyPatch(empty, assemble(changes, empty, ctx)) ?? {}; + return { '@type': 'Card', version: '1.0', ...body, uid, addressBookIds: { [bookId]: true } }; +} + +/** The patch for a dirty contact against its shadow; empty when nothing mapped changed. */ +function updatePatch(local: LocalContact, ctx: Ctx): PatchObject { + const shadow = withoutMemberOf(local.shadow!); + return assemble(collectChanges(local, shadow, ctx), shadow, ctx); +} + +const group = (local: LocalContact, ops: ProviderOp[]): OpGroup => ({ ref: local.sourceId ?? `row:${local.rawContactId}`, ops }); + +function deletedPlan(local: LocalContact, ctx: Ctx): UploadPlan { + const purge = (): OpGroup => group(local, [deleteWhereId('raw_contacts', local.rawContactId)]); + const ref = parseObjectRef(local.sourceId); + if (ref) { + if (local.readOnly) return { kind: 'revert', ops: purge(), refetch: true }; + const books = Object.entries(local.shadow?.addressBookIds ?? {}).filter(([, on]) => on).map(([id]) => id); + const selected = books.filter((id) => ctx.isSelected(collectionKey(ref.accountId, id))); + // A card that is also in books this device does not sync only leaves the synced ones. + if (selected.length && selected.length < books.length) { + const patch: PatchObject = {}; + for (const id of selected) patch[ptr('addressBookIds', id)] = null; + return { kind: 'upload', actions: [{ kind: 'update', id: ref.id, patch }] }; + } + return { kind: 'upload', actions: [{ kind: 'destroy', id: ref.id, uid: typeof local.shadow?.uid === 'string' ? local.shadow.uid : null }] }; + } + if (local.pending) return { kind: 'upload', actions: [{ kind: 'destroy', id: null, uid: local.pending.uid }] }; + return { kind: 'purge', ops: purge() }; +} + +export function planContactUpload(local: LocalContact, ctx: Ctx): UploadPlan { + if (!local.deleted && !local.dirty && local.sourceId) return { kind: 'clean', ops: group(local, []) }; + if (backedOff(local, ctx)) return { kind: 'skip', reason: `poisoned:${local.poison!.type}` }; + if (local.deleted) return deletedPlan(local, ctx); + if (local.sourceId && !parseObjectRef(local.sourceId)) return { kind: 'skip', reason: 'invalidSourceId' }; + + if (!local.sourceId) { + if (!local.pending) { + const target = ctx.createTarget(); + if (!target) return { kind: 'skip', reason: 'noWritableAddressBook' }; + const pending = { uid: ctx.mintUid(), target }; + return { kind: 'claim', ops: group(local, [assertContact(local, null), updateRawContact(local.rawContactId, { [RawContacts.SYNC3]: JSON.stringify(pending) })]) }; + } + const target = parseCollectionKey(local.pending.target); + if (!target) return { kind: 'skip', reason: 'invalidPendingTarget' }; + const object = createObject(local, local.pending.uid, target.id, ctx); + const actions: UploadAction[] = [{ kind: 'create', uid: local.pending.uid, collectionId: target.id, object }]; + return { kind: 'upload', actions }; + } + + if (!local.shadow) return { kind: 'skip', reason: 'noShadow' }; + if (local.readOnly) { + const write = cleanWrite(local, local.shadow, ctx, { assertDirty: null, clearDirty: true }); + return { kind: 'revert', ops: group(local, write.ops.length ? write.ops : [assertContact(local, null)]) }; + } + const patch = updatePatch(local, ctx); + if (!Object.keys(patch).length) { + // Nothing mapped changed (a star, a ringtone, an edit the server can't take): the rows + // become the shadow's again and DIRTY is cleared behind the VERSION assert. + const write = cleanWrite(local, local.shadow, ctx, { assertDirty: true, clearDirty: true }); + return { kind: 'clean', ops: group(local, write.ops.length ? write.ops : [assertContact(local, true)]) }; + } + const actions: UploadAction[] = [{ kind: 'update', id: parseObjectRef(local.sourceId)!.id, patch }]; + if (backedOff(local, ctx, actions)) return { kind: 'skip', reason: `poisoned:${local.poison!.type}` }; + return { kind: 'upload', actions }; +} + +/** The JMAP account a local contact belongs to. */ +function accountOf(local: LocalContact, ctx: Ctx): string { + return parseObjectRef(local.sourceId)?.accountId ?? parseCollectionKey(local.pending?.target)?.accountId ?? ctx.jmapAccountId; +} + +export function planContactAccepted(local: LocalContact, server: ContactCardWire, ctx: Ctx): AcceptedPlan { + if (local.deleted) { + const purge = group(local, [deleteWhereId('raw_contacts', local.rawContactId)]); + return { ops: purge, keepDirtyOps: purge }; + } + const identity = objectRef(accountOf(local, ctx), server.id); + const raw = { [RawContacts.SOURCE_ID]: identity, [RawContacts.SYNC3]: null, [RawContacts.SYNC4]: null }; + const write = cleanWrite(local, server, ctx, { assertDirty: null, clearDirty: true, accepted: true, raw }); + const ops = group(local, write.ops.length ? write.ops : [assertContact(local, null)]); + ops.ref = identity; + + // Edited again during the upload: identity and shadow, and the baselines of + // the uploaded units set to what was uploaded; rows and DIRTY stay. + const changes = local.shadow ? collectChanges(local, withoutMemberOf(local.shadow), ctx) : null; + const shadow = withMemberOf(shadowOf(withoutMemberOf(server)), shadowMemberOf(local.shadow) ?? (local.shadow ? null : [])); + const keep: ProviderOp[] = [ + updateRawContact(local.rawContactId, { ...raw, [RawContacts.SYNC2]: JSON.stringify(shadow) }), + ]; + for (const id of changes?.uploadedRows ?? []) { + const row = local.rows.find((r) => r.id === id)!; + keep.push(updateWhereId('data', id, { [Data.DATA_SYNC3]: encodeBaseline(row.cells, SPECS[row.mimetype].columns) })); + } + return { ops, keepDirtyOps: { ref: identity, ops: keep } }; +} diff --git a/src/device-sync/task.ts b/src/device-sync/task.ts index 6a1149d9..4500875b 100644 --- a/src/device-sync/task.ts +++ b/src/device-sync/task.ts @@ -26,10 +26,10 @@ import { type TeardownOptions, } from './engine'; import { calendarPlanner } from './calendar/planner'; +import { contactsPlanner } from './contacts/planner'; import { emptyStats } from './engine/report'; import { openJmapPort } from './jmap/client-port'; import { createProviderPort, finishRun, getSyncSettings, isRunCancelled, showSyncProblem } from './native'; -import type { ContactsPlanner } from './planner'; import type { Authority, RunPayload, RunReport } from './types'; // Mirrors the persist names of the locale and feed-subscription stores; read @@ -39,13 +39,6 @@ const SUBSCRIPTIONS_STORAGE_KEY = 'calendar-subscriptions'; /** Where "sign in again" leads (the accounts pane), as for a registry eviction. */ const SIGN_IN_URI = 'bulwarkmobile://settings/account'; -// The contacts planner lands in the next -// commit. Until then a run fails before it touches a row. -function notWired(): never { - throw new Error('Device sync planners are not wired into this build yet'); -} -const contactsPlanner = new Proxy({}, { get: notWired }) as ContactsPlanner; - /** Random numbers from the platform CSPRNG, buffered (uids and map keys of device-made items). */ function secureRandom(): RandomSource { let buffer: Uint8Array = new Uint8Array(0); From e67e2408407f946fb5d5bac675fc80890cc16fb3 Mon Sep 17 00:00:00 2001 From: Linus Rath <139418639+rathlinus@users.noreply.github.com> Date: Sun, 27 Sep 2026 04:36:51 +0200 Subject: [PATCH 10/87] feat: ask android to sync when contacts or calendars change #34 With device sync on, Android should hear about a change as soon as the app does, not only at the next periodic sync. This adds what tells it: - The app's own contact and calendar edits, StateChanges on the live stream for `AddressBook`/`ContactCard` and `Calendar`/`CalendarEvent`, and the app coming back to the foreground request a sync of that account and authority. Requests within 5 s fold into one, and a StateChange the engine caused itself (its own upload echoing back) is skipped. - While the app is closed, the account's push subscription carries those types too. The FCM and UnifiedPush services hand every push to the native router from f743ab0, which requests the syncs, and start the mail push task only when a push has `EmailDelivery`; the task also returns early for such pushes, so a contact change can never show the "newest unread mail" notification. - At launch, on foreground and when Android's accounts change, the app checks Android's accounts against its own: an account removed in Android Settings shows sync off and is not recreated, and one whose app account is gone stops syncing and posts "Sign in again", keeping its data. The functions that turn sync on and off (`src/device-sync/app/lifecycle.ts`) come with this; the settings that call them land in a later commit, so nothing here runs before an account has device sync on. Checked: the new store, lifecycle, trigger and push tests, `npm run typecheck`, `npm test` and `npm run i18n:check` on this commit, and the debug build. --- App.tsx | 4 + .../bulwarkmobile/BulwarkMessagingService.kt | 7 +- .../BulwarkUnifiedPushService.kt | 6 +- locales/rn/en.json | 106 ++- .../__tests__/app/lifecycle.test.ts | 485 ++++++++++++++ .../__tests__/app/triggers.test.ts | 285 ++++++++ src/device-sync/app/available.ts | 17 + src/device-sync/app/confirm.ts | 40 ++ src/device-sync/app/lifecycle.ts | 622 ++++++++++++++++++ src/device-sync/app/permissions.ts | 54 ++ src/device-sync/app/prefs.ts | 80 +++ src/device-sync/app/request.ts | 92 +++ src/device-sync/app/triggers.ts | 119 ++++ .../__tests__/push-background-task.test.ts | 53 ++ src/lib/__tests__/push-notifications.test.ts | 133 ++++ src/lib/platform-capabilities.ts | 12 + src/lib/push-background-task.ts | 18 +- src/lib/push-notifications.ts | 83 ++- .../__tests__/device-sync-store.test.ts | 268 ++++++++ src/stores/calendar-store.ts | 18 +- src/stores/contacts-store.ts | 13 + src/stores/device-sync-store.ts | 290 ++++++-- 22 files changed, 2742 insertions(+), 63 deletions(-) create mode 100644 src/device-sync/__tests__/app/lifecycle.test.ts create mode 100644 src/device-sync/__tests__/app/triggers.test.ts create mode 100644 src/device-sync/app/available.ts create mode 100644 src/device-sync/app/confirm.ts create mode 100644 src/device-sync/app/lifecycle.ts create mode 100644 src/device-sync/app/permissions.ts create mode 100644 src/device-sync/app/prefs.ts create mode 100644 src/device-sync/app/request.ts create mode 100644 src/device-sync/app/triggers.ts create mode 100644 src/stores/__tests__/device-sync-store.test.ts diff --git a/App.tsx b/App.tsx index 31b0ddc7..1eba8d4d 100644 --- a/App.tsx +++ b/App.tsx @@ -18,6 +18,7 @@ import { startCalendarReminderTapHandling, } from './src/lib/calendar-notifications'; import { startLivenessMonitor } from './src/lib/connection-liveness'; +import { startDeviceSyncTriggers } from './src/device-sync/app/triggers'; import { setPendingCalendarOpen, type CalendarReminderTarget, @@ -399,6 +400,9 @@ export default function App() { // Calendar reminders are kept scheduled from launch, not only once the // Calendar tab has been opened. startCalendarNotificationSync(); + // Device sync (Android, #34): live changes, the app's own edits and the + // foreground ask Android to sync; accounts are checked against Android's. + startDeviceSyncTriggers(); return useNetworkStore.getState().init(); }, []); diff --git a/android/app/src/main/java/com/anonymous/bulwarkmobile/BulwarkMessagingService.kt b/android/app/src/main/java/com/anonymous/bulwarkmobile/BulwarkMessagingService.kt index 41c553db..3959e5b6 100644 --- a/android/app/src/main/java/com/anonymous/bulwarkmobile/BulwarkMessagingService.kt +++ b/android/app/src/main/java/com/anonymous/bulwarkmobile/BulwarkMessagingService.kt @@ -11,6 +11,7 @@ import android.os.Bundle import android.os.PowerManager import android.util.Log import androidx.core.app.NotificationCompat +import com.anonymous.bulwarkmobile.sync.DeviceSyncPushRouter import com.facebook.react.bridge.Arguments import com.google.firebase.messaging.FirebaseMessagingService import com.google.firebase.messaging.RemoteMessage @@ -26,12 +27,16 @@ class BulwarkMessagingService : FirebaseMessagingService() { val data = message.data + // Contact and calendar changes become syncs of the Android accounts + // they feed (device sync, #34); only mail needs the push task. + val mail = DeviceSyncPushRouter.route(applicationContext, data) + // Hand off to JS via a headless task only when the app isn't already // running in the foreground - HeadlessJsTaskContext throws if started // while foreground. When the app is open, the main JS instance already // receives JMAP push directly, so the fcm:message event below is // sufficient for it to refresh state. - if (!isAppInForeground()) { + if (mail && !isAppInForeground()) { startHeadlessTask(applicationContext, data) } diff --git a/android/app/src/main/java/com/anonymous/bulwarkmobile/BulwarkUnifiedPushService.kt b/android/app/src/main/java/com/anonymous/bulwarkmobile/BulwarkUnifiedPushService.kt index 81f9f196..5fac3d44 100644 --- a/android/app/src/main/java/com/anonymous/bulwarkmobile/BulwarkUnifiedPushService.kt +++ b/android/app/src/main/java/com/anonymous/bulwarkmobile/BulwarkUnifiedPushService.kt @@ -1,6 +1,7 @@ package com.anonymous.bulwarkmobile import android.content.Context +import com.anonymous.bulwarkmobile.sync.DeviceSyncPushRouter import com.facebook.react.bridge.Arguments import org.json.JSONArray import org.json.JSONObject @@ -31,7 +32,10 @@ class BulwarkUnifiedPushService : PushService() { BulwarkMessagingService.ensureChannel(this) val data = parsePayload(message.content) ?: return - if (!BulwarkMessagingService.isAppInForeground()) { + // Same routing as FCM: contact and calendar changes request device + // syncs, and only mail starts the push task. + val mail = DeviceSyncPushRouter.route(applicationContext, data) + if (mail && !BulwarkMessagingService.isAppInForeground()) { BulwarkMessagingService.startHeadlessTask(applicationContext, data) } diff --git a/locales/rn/en.json b/locales/rn/en.json index b880e06a..ff5d09ef 100644 --- a/locales/rn/en.json +++ b/locales/rn/en.json @@ -93,6 +93,8 @@ "until_date": "Until {date}" }, "settings": { + "device_sync_desc": "Show your calendars in the Calendar app and other apps on this device. Changes sync both ways.", + "device_sync_title": "Sync to this device", "show_week_numbers_mobile_desc": "Display week numbers in the month view.", "time_zone": "Time zone", "time_zone_auto": "Device time zone", @@ -255,7 +257,14 @@ "device_sync": { "auth_problem_text": "The server no longer accepts the sign-in of {account}, so its contacts and calendars on this device stopped syncing. Sign in again to resume. Changes made on this device are kept.", "auth_problem_title": "Sign in again to keep syncing", - "problem_channel": "Sync problems" + "problem_channel": "Sync problems", + "sign_out_anyway": "Sign out", + "sign_out_pending_message": "{count, plural, one {# change made on this device hasn't reached the server. It is lost if you sign out now.} other {# changes made on this device haven't reached the server. They are lost if you sign out now.}}", + "sign_out_pending_title": "Sign out anyway?", + "sign_out_unknown_message": "Changes made on this device may not have reached the server. They are lost if you sign out now.", + "signed_out_text": "{account} was signed out, so its contacts and calendars on this device no longer sync. Sign in again to resume. Changes made on this device are kept.", + "signed_out_title": "Sign in to keep syncing", + "signing_out": "Removing synced contacts and calendars from this device…" }, "email_composer": { "attach_camera": "Camera", @@ -613,11 +622,106 @@ "book_count": "{count, plural, one {# contact} other {# contacts}}", "default_book": "Default address book", "description_mobile": "Display preferences and address-book tools.", + "device_sync_desc": "Show your address books in the Contacts app and other apps on this device. Changes sync both ways.", + "device_sync_title": "Sync to this device", "export_description_count": "{count, plural, one {Export # contact as a single vCard file.} other {Export # contacts as a single vCard file.}}", "export_empty": "No contacts to export", "import_description_mobile": "Import contacts from a vCard (.vcf) file with duplicate detection.", "manage_description_mobile": "Create, rename, and remove the address books that organize your contacts." }, + "device_sync": { + "also_synced_by": "Also synced for {accounts}, so it shows twice on this device.", + "android_settings": "Android account settings", + "blocked_calendar": "Android no longer asks for access to calendars. Allow it in the app's settings, then turn sync on again.", + "blocked_contacts": "Android no longer asks for access to contacts. Allow it in the app's settings, then turn sync on again.", + "blocked_title": "Access is turned off", + "books_default": "Your own address books sync; shared ones only when you pick them.", + "books_hint": "Your own address books sync unless you turn them off; shared ones only when you turn them on.", + "books_label": "Address books", + "calendars_default": "Your own calendars sync; shared ones only when you pick them.", + "calendars_hint": "Your own calendars sync unless you turn them off; shared ones only when you turn them on. Task lists stay in Bulwark.", + "calendars_label": "Calendars", + "choose": "Choose", + "choose_books_title": "Address books to sync", + "choose_calendars_title": "Calendars to sync", + "collections_count": "{selected} of {total} sync", + "continue": "Continue", + "denied_calendar": "Without access to calendars, Bulwark cannot put your calendars on this device.", + "denied_contacts": "Without access to contacts, Bulwark cannot put your address books on this device.", + "denied_title": "Sync stays off", + "failed_title": "Sync could not be turned on", + "grant_access": "Grant access", + "interval_15m": "Every 15 minutes", + "interval_1h": "Every hour", + "interval_30m": "Every 30 minutes", + "interval_6h": "Every 6 hours", + "interval_desc": "How often Android syncs on its own. Edits in Bulwark and pushed changes sync sooner.", + "interval_label": "Sync interval", + "interval_manual": "No regular sync", + "keep_syncing": "Keep syncing", + "list_error_auth": "The server did not accept the sign-in.", + "list_error_network": "Could not reach the server.", + "list_error_server": "The server could not list them.", + "list_error_signed_out": "This account is not signed in on this device.", + "master_off": "Auto-sync is off on this device, so syncs only run when you tap Sync now.", + "new_contacts_default": "Default address book", + "new_contacts_desc": "The address book for contacts created in other apps on this device.", + "new_contacts_label": "New contacts go to", + "no_books": "This account has no address books.", + "no_calendars": "This account has no calendars to sync.", + "ok": "OK", + "open_app_settings": "Open app settings", + "own_collections": "Your own", + "rationale_calendar": "To show your calendars in the Calendar app, Bulwark needs access to the calendars on this device. It only changes the calendars of its own account.", + "rationale_contacts": "To show your address books in the Contacts app, Bulwark needs access to the contacts on this device. It only changes the contacts of its own account.", + "rationale_title_calendar": "Allow access to calendars", + "rationale_title_contacts": "Allow access to contacts", + "read_only": "Read-only", + "reminder_owner_bulwark": "Bulwark", + "reminder_owner_device": "Calendar app", + "reminder_question_message": "Synced events can remind you from the calendar app on this device or from Bulwark. Pick one so you are not reminded twice. You can change it later.", + "reminder_question_title": "Who should remind you?", + "reminders_desc": "Who reminds you of synced events. The other stays quiet, so you are not reminded twice.", + "reminders_label": "Reminders", + "reminders_nobody": "Bulwark's event notifications are off, so nothing reminds you of synced events.", + "remove": "Remove", + "remove_from_device": "Remove from this device", + "remove_message": "The account's contacts and calendars are removed from this device. Changes made here that haven't reached the server are lost.", + "remove_title": "Remove from this device?", + "review_deletions": "Review deletions", + "shared_by": "Shared by {name}", + "sign_in_again": "Sign in again", + "status_auth": "The server no longer accepts the sign-in. Sign in again to keep syncing.", + "status_cancelled": "The last sync stopped early. It continues next time.", + "status_failed": "The last sync failed: {message}", + "status_failed_plain": "The last sync failed.", + "status_io": "Could not reach the server {when}. Android tries again.", + "status_item_errors": "{count, plural, one {# item could not be synced.} other {# items could not be synced.}}", + "status_off": "Off", + "status_paused": "Paused in Android settings", + "status_permission_calendar": "Access to calendars was turned off, so nothing syncs.", + "status_permission_contacts": "Access to contacts was turned off, so nothing syncs.", + "status_removed": "Removed in Android settings. Turn on to add the account again.", + "status_safety_abort": "Stopped: the server returned nothing while this device still holds items. Check the server, then sync again.", + "status_synced": "Synced {when}", + "status_syncing": "Syncing…", + "status_too_many_deletions": "Many items were deleted on this device. They are not deleted on the server until you confirm.", + "status_unsupported_calendar": "The server offers no calendars for this account.", + "status_unsupported_contacts": "The server offers no address books for this account.", + "status_waiting": "Waiting for the first sync", + "status_waiting_next": "Waiting for the next sync", + "suspended_desc": "Signed out. Its contacts and calendars stay on this device, with the changes made here, and sync again once you sign in.", + "switch_label": "Sync {account} to this device", + "sync_now": "Sync now", + "turn_off_anyway": "Turn off", + "turn_off_failed": "The synced data could not be removed from this device. Sync is paused; try again later.", + "turn_off_failed_title": "Sync could not be turned off", + "turn_off_pending": "{count, plural, one {# change made on this device hasn't reached the server yet. It is lost if you turn sync off now.} other {# changes made on this device haven't reached the server yet. They are lost if you turn sync off now.}}", + "turn_off_title": "Turn off anyway?", + "turn_off_unknown": "Bulwark could not check whether changes made on this device reached the server. They are lost if you turn sync off now.", + "turning_off": "Uploading changes, then removing the data from this device…", + "turning_on": "Setting up…" + }, "downloads": { "attachment_template": { "description_mobile": "Template used when saving or sharing an attachment." diff --git a/src/device-sync/__tests__/app/lifecycle.test.ts b/src/device-sync/__tests__/app/lifecycle.test.ts new file mode 100644 index 00000000..d328abe6 --- /dev/null +++ b/src/device-sync/__tests__/app/lifecycle.test.ts @@ -0,0 +1,485 @@ +import { describe, it, expect, vi, beforeEach } from 'vitest'; +import AsyncStorage from '@react-native-async-storage/async-storage'; +import { + CALENDAR_AUTHORITY, + CONTACTS_AUTHORITY, + type Authority, +} from '../../types'; + +// A fake of the native module: Android's accounts of our type and their sync +// settings, with every call logged in order. +const h = vi.hoisted(() => { + type Settings = { automatic: boolean; periodicSeconds: number }; + const state = { + available: true, + permitted: true, + master: true, + accounts: [] as Array<{ name: string; registryId: string | null; sync: Record }>, + log: [] as string[], + failSetSyncEnabled: false, + }; + const find = (name: string) => state.accounts.find((a) => a.name === name); + const settingsOf = (name: string, authority: string) => { + const account = find(name); + if (!account) return { automatic: false, periodicSeconds: 0 }; + account.sync[authority] ??= { automatic: false, periodicSeconds: 0 }; + return account.sync[authority]; + }; + const teardown = async ( + _registryId: string, + name: string, + authority: string, + options?: { uploadFirst?: boolean; timeoutMs?: number; force?: boolean }, + ) => { + state.log.push(`teardown ${name} ${authority} ${JSON.stringify(options ?? {})}`); + return { pending: 0 }; + }; + return { state, find, settingsOf, teardown }; +}); + +vi.mock('../../native', () => ({ + isDeviceSyncAvailable: () => h.state.available, + listAndroidAccounts: vi.fn(async () => h.state.accounts.map((a) => ({ name: a.name, registryId: a.registryId }))), + ensureAndroidAccount: vi.fn(async (name: string, registryId: string) => { + h.state.log.push(`ensure ${name}`); + const existing = h.find(name); + if (existing && existing.registryId !== registryId) throw new Error('conflict'); + if (existing) return false; + h.state.accounts.push({ name, registryId, sync: {} }); + return true; + }), + removeAndroidAccount: vi.fn(async (name: string) => { + h.state.log.push(`remove ${name}`); + h.state.accounts = h.state.accounts.filter((a) => a.name !== name); + return true; + }), + getSyncSettings: vi.fn(async (name: string) => ({ + masterAutomatic: h.state.master, + authorities: { + 'com.android.contacts': { syncable: 1, active: false, pending: false, ...h.settingsOf(name, 'com.android.contacts') }, + 'com.android.calendar': { syncable: 1, active: false, pending: false, ...h.settingsOf(name, 'com.android.calendar') }, + }, + })), + setSyncEnabled: vi.fn(async (name: string, authority: string, enabled: boolean) => { + if (h.state.failSetSyncEnabled) throw new Error('boom'); + h.state.log.push(`auto ${name} ${authority} ${enabled}`); + h.settingsOf(name, authority).automatic = enabled; + }), + setPeriodicSync: vi.fn(async (name: string, authority: string, seconds: number) => { + h.state.log.push(`periodic ${name} ${authority} ${seconds}`); + h.settingsOf(name, authority).periodicSeconds = seconds; + }), + requestSync: vi.fn(async (name: string, authority: string, options?: Record) => { + h.state.log.push(`sync ${name} ${authority} ${JSON.stringify(options ?? {})}`); + }), + openAccountSettings: vi.fn(async () => undefined), + setPushRoutes: vi.fn(async () => undefined), + showSyncProblem: vi.fn(async (name: string) => { h.state.log.push(`problem ${name}`); }), + clearSyncProblem: vi.fn(async (name: string) => { h.state.log.push(`clear ${name}`); }), + onAccountsChanged: vi.fn(() => () => undefined), +})); + +vi.mock('../../task', () => ({ teardownAuthority: vi.fn(h.teardown) })); + +vi.mock('../../app/permissions', () => ({ + hasSyncPermissions: vi.fn(async () => h.state.permitted), +})); + +vi.mock('../../../lib/push-notifications', () => ({ + refreshPushSubscriptionTypes: vi.fn(async () => undefined), +})); + +import * as native from '../../native'; +import * as task from '../../task'; +import { refreshPushSubscriptionTypes } from '../../../lib/push-notifications'; +import { useAccountStore, type AccountEntry } from '../../../stores/account-store'; +import { + accountDeviceSync, + useDeviceSyncStore, + waitForDeviceSyncHydration, +} from '../../../stores/device-sync-store'; +import { + changeReminderOwner, + changeSyncInterval, + deviceSyncSignedIn, + disableDeviceSync, + enableDeviceSync, + preferredAndroidAccountName, + reconcileDeviceAccounts, + releaseDeviceSyncBeforeSignOut, + removeSuspendedAccount, + resumeDeviceSync, + TEARDOWN_TIMEOUT_MS, +} from '../../app/lifecycle'; + +const ALICE = 'alice@mail.example.org'; +const ALICE2 = 'alice@other.example.net'; +const BOB = 'bob@mail.example.org'; + +function entry(id: string, username: string): AccountEntry { + return { + id, + serverUrl: `https://${id.split('@').pop()}`, + username, + displayName: username, + email: `${username}@example.org`, + avatarColor: '#000', + lastLoginAt: 0, + isConnected: true, + hasError: false, + isDefault: false, + }; +} + +async function ready(registry: AccountEntry[]): Promise { + await waitForDeviceSyncHydration(); + if (!useAccountStore.persist.hasHydrated()) { + await new Promise((resolve) => { + const off = useAccountStore.persist.onFinishHydration(() => { off(); resolve(); }); + }); + } + useAccountStore.setState({ accounts: registry, activeAccountId: registry[0]?.id ?? null }); +} + +const teardown = vi.mocked(task.teardownAuthority); +const setSyncEnabled = vi.mocked(native.setSyncEnabled); + +beforeEach(async () => { + vi.clearAllMocks(); + h.state.available = true; + h.state.permitted = true; + h.state.master = true; + h.state.accounts = []; + h.state.log = []; + h.state.failSetSyncEnabled = false; + teardown.mockImplementation(h.teardown); + await AsyncStorage.clear(); + useDeviceSyncStore.setState({ accounts: {} }); + await ready([entry(ALICE, 'alice'), entry(BOB, 'bob')]); +}); + +describe('enable', () => { + it('creates the Android account under the login, turns automatic and periodic sync on and syncs', async () => { + useDeviceSyncStore.getState().setIntervalSeconds(ALICE, CONTACTS_AUTHORITY, 1800); + const outcome = await enableDeviceSync(ALICE, CONTACTS_AUTHORITY); + expect(outcome).toEqual({ kind: 'enabled', accountName: 'alice' }); + expect(h.state.log).toEqual([ + 'ensure alice', + `auto alice ${CONTACTS_AUTHORITY} true`, + `periodic alice ${CONTACTS_AUTHORITY} 1800`, + `sync alice ${CONTACTS_AUTHORITY} {"manual":true}`, + ]); + const stored = accountDeviceSync(ALICE); + expect(stored.androidAccountName).toBe('alice'); + expect(stored.enabled).toEqual({ [CONTACTS_AUTHORITY]: true }); + await vi.waitFor(() => expect(refreshPushSubscriptionTypes).toHaveBeenCalledWith(ALICE)); + }); + + it('uses the default hourly interval', async () => { + await enableDeviceSync(ALICE, CALENDAR_AUTHORITY); + expect(h.state.log).toContain(`periodic alice ${CALENDAR_AUTHORITY} 3600`); + }); + + it('does nothing without the runtime permissions', async () => { + h.state.permitted = false; + expect(await enableDeviceSync(ALICE, CONTACTS_AUTHORITY)).toEqual({ kind: 'permission' }); + expect(h.state.log).toEqual([]); + expect(accountDeviceSync(ALICE).enabled).toBeUndefined(); + }); + + it('reuses the Android account of the app account', async () => { + h.state.accounts.push({ name: 'alice', registryId: ALICE, sync: {} }); + await enableDeviceSync(ALICE, CALENDAR_AUTHORITY); + expect(h.state.log.filter((l) => l.startsWith('ensure'))).toEqual([]); + expect(accountDeviceSync(ALICE).androidAccountName).toBe('alice'); + }); + + it('names the account after its registry id when another app account has the login', async () => { + await ready([entry(ALICE, 'alice'), entry(ALICE2, 'alice')]); + h.state.accounts.push({ name: 'alice', registryId: ALICE, sync: {} }); + expect(preferredAndroidAccountName(ALICE2, await native.listAndroidAccounts())).toBe(ALICE2); + const outcome = await enableDeviceSync(ALICE2, CONTACTS_AUTHORITY); + expect(outcome).toEqual({ kind: 'enabled', accountName: ALICE2 }); + }); + + it('falls back to the registry id when the login turns out to be taken', async () => { + const list = vi.mocked(native.listAndroidAccounts); + // A stale list: the clash only shows when the account is added. + list.mockResolvedValueOnce([]); + h.state.accounts.push({ name: 'alice', registryId: ALICE2, sync: {} }); + const outcome = await enableDeviceSync(ALICE, CONTACTS_AUTHORITY); + expect(outcome).toEqual({ kind: 'enabled', accountName: ALICE }); + }); + + it('leaves no account behind when turning sync on fails', async () => { + h.state.failSetSyncEnabled = true; + const outcome = await enableDeviceSync(ALICE, CONTACTS_AUTHORITY); + expect(outcome.kind).toBe('failed'); + expect(h.state.accounts).toEqual([]); + expect(accountDeviceSync(ALICE).enabled).toBeUndefined(); + }); + + it('keeps the account of the other authority when turning one on fails', async () => { + await enableDeviceSync(ALICE, CONTACTS_AUTHORITY); + vi.mocked(native.setPeriodicSync).mockRejectedValueOnce(new Error('boom')); + expect((await enableDeviceSync(ALICE, CALENDAR_AUTHORITY)).kind).toBe('failed'); + expect(h.find('alice')?.sync[CALENDAR_AUTHORITY].automatic).toBe(false); + expect(h.find('alice')?.sync[CONTACTS_AUTHORITY].automatic).toBe(true); + expect(accountDeviceSync(ALICE)).toMatchObject({ + androidAccountName: 'alice', + enabled: { [CONTACTS_AUTHORITY]: true }, + }); + }); + + it('is a no-op where device sync is not available', async () => { + h.state.available = false; + expect((await enableDeviceSync(ALICE, CONTACTS_AUTHORITY)).kind).toBe('failed'); + expect(h.state.log).toEqual([]); + }); +}); + +describe('disable', () => { + beforeEach(async () => { + await enableDeviceSync(ALICE, CONTACTS_AUTHORITY); + await enableDeviceSync(ALICE, CALENDAR_AUTHORITY); + await changeReminderOwner(ALICE, 'bulwark'); + h.state.log = []; + }); + + it('turns automatic sync off before the teardown uploads and deletes', async () => { + const result = await disableDeviceSync(ALICE, CALENDAR_AUTHORITY); + expect(result).toEqual({ done: true, pending: 0 }); + expect(h.state.log).toEqual([ + `auto alice ${CALENDAR_AUTHORITY} false`, + `teardown alice ${CALENDAR_AUTHORITY} {"uploadFirst":true,"timeoutMs":${TEARDOWN_TIMEOUT_MS}}`, + ]); + const stored = accountDeviceSync(ALICE); + expect(stored.enabled).toEqual({ [CONTACTS_AUTHORITY]: true, [CALENDAR_AUTHORITY]: false }); + // Contacts still sync: the account stays; the reminder question comes again next time. + expect(h.find('alice')).toBeDefined(); + expect(stored.reminderOwner).toBeNull(); + }); + + it('removes the Android account once both authorities are off', async () => { + await disableDeviceSync(ALICE, CALENDAR_AUTHORITY); + await disableDeviceSync(ALICE, CONTACTS_AUTHORITY); + expect(h.state.accounts).toEqual([]); + expect(h.state.log).toContain('remove alice'); + expect(accountDeviceSync(ALICE).androidAccountName).toBeUndefined(); + }); + + it('deletes nothing and reports the waiting changes when they could not be uploaded', async () => { + teardown.mockResolvedValueOnce({ pending: 3 }); + const result = await disableDeviceSync(ALICE, CONTACTS_AUTHORITY); + expect(result).toEqual({ done: false, pending: 3 }); + expect(accountDeviceSync(ALICE).enabled?.[CONTACTS_AUTHORITY]).toBe(true); + + // The user keeps syncing: automatic sync comes back. + h.state.log = []; + await resumeDeviceSync(ALICE, CONTACTS_AUTHORITY); + expect(h.state.log).toEqual([ + `auto alice ${CONTACTS_AUTHORITY} true`, + `sync alice ${CONTACTS_AUTHORITY} {"manual":true}`, + ]); + }); + + it('deletes on the user\'s word with force', async () => { + teardown.mockResolvedValueOnce({ pending: 2 }); + await disableDeviceSync(ALICE, CONTACTS_AUTHORITY); + h.state.log = []; + const result = await disableDeviceSync(ALICE, CONTACTS_AUTHORITY, { force: true }); + expect(result.done).toBe(true); + expect(h.state.log).toContain(`teardown alice ${CONTACTS_AUTHORITY} {"force":true,"timeoutMs":5000}`); + expect(accountDeviceSync(ALICE).enabled?.[CONTACTS_AUTHORITY]).toBe(false); + }); + + it('reports a failed teardown as unknown changes', async () => { + const warn = vi.spyOn(console, 'warn').mockImplementation(() => undefined); + teardown.mockRejectedValueOnce(new Error('provider gone')); + expect(await disableDeviceSync(ALICE, CONTACTS_AUTHORITY)).toEqual({ done: false, pending: 0, failed: true }); + warn.mockRestore(); + }); +}); + +describe('sign-out', () => { + it('lets an account without device sync go at once', async () => { + const confirm = vi.fn(async () => true); + expect(await releaseDeviceSyncBeforeSignOut([BOB], confirm)).toBe(true); + expect(teardown).not.toHaveBeenCalled(); + expect(confirm).not.toHaveBeenCalled(); + }); + + it('uploads, deletes and removes the Android account before the credentials go', async () => { + await enableDeviceSync(ALICE, CONTACTS_AUTHORITY); + h.state.log = []; + const confirm = vi.fn(async () => true); + expect(await releaseDeviceSyncBeforeSignOut([ALICE], confirm)).toBe(true); + expect(confirm).not.toHaveBeenCalled(); + expect(teardown).toHaveBeenCalledWith(ALICE, 'alice', CONTACTS_AUTHORITY, { uploadFirst: true, timeoutMs: TEARDOWN_TIMEOUT_MS }); + expect(teardown).toHaveBeenCalledWith(ALICE, 'alice', CALENDAR_AUTHORITY, { uploadFirst: true, timeoutMs: TEARDOWN_TIMEOUT_MS }); + expect(h.state.accounts).toEqual([]); + expect(useDeviceSyncStore.getState().accounts[ALICE]).toBeUndefined(); + // Automatic sync went off before the first teardown. + expect(h.state.log.indexOf(`auto alice ${CONTACTS_AUTHORITY} false`)) + .toBeLessThan(h.state.log.findIndex((l) => l.startsWith('teardown'))); + }); + + it('asks before losing changes, and keeps syncing when the user stays', async () => { + await enableDeviceSync(ALICE, CONTACTS_AUTHORITY); + teardown.mockImplementation(async (_r: string, _n: string, authority: string) => ({ pending: authority === CONTACTS_AUTHORITY ? 4 : 0 })); + const confirm = vi.fn(async () => false); + expect(await releaseDeviceSyncBeforeSignOut([ALICE], confirm)).toBe(false); + expect(confirm).toHaveBeenCalledWith(4); + expect(h.find('alice')?.sync[CONTACTS_AUTHORITY].automatic).toBe(true); + expect(accountDeviceSync(ALICE).androidAccountName).toBe('alice'); + }); + + it('deletes what is left once the user agrees', async () => { + await enableDeviceSync(ALICE, CONTACTS_AUTHORITY); + teardown.mockImplementation(async (_r: string, _n: string, authority: string, options?: { force?: boolean }) => + ({ pending: authority === CONTACTS_AUTHORITY && !options?.force ? 4 : 0 })); + const confirm = vi.fn(async () => true); + expect(await releaseDeviceSyncBeforeSignOut([ALICE], confirm)).toBe(true); + expect(teardown).toHaveBeenCalledWith(ALICE, 'alice', CONTACTS_AUTHORITY, { force: true, timeoutMs: 5_000 }); + expect(h.state.accounts).toEqual([]); + }); + + it('asks once for several accounts', async () => { + await enableDeviceSync(ALICE, CONTACTS_AUTHORITY); + await enableDeviceSync(BOB, CALENDAR_AUTHORITY); + teardown.mockImplementation(async (_r: string, _n: string, _a: string, options?: { force?: boolean }) => ({ pending: options?.force ? 0 : 1 })); + const confirm = vi.fn(async () => true); + expect(await releaseDeviceSyncBeforeSignOut([ALICE, BOB], confirm)).toBe(true); + expect(confirm).toHaveBeenCalledTimes(1); + expect(confirm).toHaveBeenCalledWith(4); + expect(h.state.accounts).toEqual([]); + }); +}); + +describe('reconcile', () => { + it('suspends an Android account whose app account the registry dropped, once', async () => { + h.state.accounts.push({ + name: 'carol', + registryId: 'carol@mail.example.org', + sync: { [CONTACTS_AUTHORITY]: { automatic: true, periodicSeconds: 3600 } }, + }); + await reconcileDeviceAccounts(); + expect(h.find('carol')?.sync[CONTACTS_AUTHORITY].automatic).toBe(false); + expect(h.find('carol')).toBeDefined(); + expect(native.showSyncProblem).toHaveBeenCalledTimes(1); + expect(vi.mocked(native.showSyncProblem).mock.calls[0][3]).toBe('bulwarkmobile://settings/account'); + const carol = accountDeviceSync('carol@mail.example.org'); + expect(carol.suspended).toBe(true); + expect(carol.enabled).toEqual({ [CONTACTS_AUTHORITY]: true }); + + await reconcileDeviceAccounts(); + expect(native.showSyncProblem).toHaveBeenCalledTimes(1); + }); + + it('drops a suspended account and its data when the user removes it', async () => { + h.state.accounts.push({ name: 'carol', registryId: 'carol@mail.example.org', sync: {} }); + await reconcileDeviceAccounts(); + h.state.log = []; + await removeSuspendedAccount('carol@mail.example.org'); + expect(h.state.log).toEqual(['remove carol', 'clear carol']); + expect(useDeviceSyncStore.getState().accounts['carol@mail.example.org']).toBeUndefined(); + }); + + it('never acts on an empty registry', async () => { + await ready([]); + h.state.accounts.push({ name: 'carol', registryId: 'carol@mail.example.org', sync: {} }); + await reconcileDeviceAccounts(); + expect(native.showSyncProblem).not.toHaveBeenCalled(); + expect(setSyncEnabled).not.toHaveBeenCalled(); + }); + + it('shows sync off for an account removed in Android Settings and never recreates it', async () => { + await enableDeviceSync(ALICE, CONTACTS_AUTHORITY); + h.state.accounts = []; + vi.mocked(native.ensureAndroidAccount).mockClear(); + await reconcileDeviceAccounts(); + const alice = accountDeviceSync(ALICE); + expect(alice.removedInAndroidSettings).toBe(true); + expect(alice.androidAccountName).toBeUndefined(); + expect(alice.enabled).toEqual({}); + await reconcileDeviceAccounts(); + expect(native.ensureAndroidAccount).not.toHaveBeenCalled(); + expect(h.state.accounts).toEqual([]); + }); + + it('forgets an account that is neither signed in nor on the device', async () => { + useDeviceSyncStore.getState().update('gone@mail.example.org', { androidAccountName: 'gone', enabled: { [CONTACTS_AUTHORITY]: true } }); + await reconcileDeviceAccounts(); + expect(useDeviceSyncStore.getState().accounts['gone@mail.example.org']).toBeUndefined(); + }); + + it('takes an authority turned on in Android Settings as on', async () => { + h.state.accounts.push({ + name: 'alice', + registryId: ALICE, + sync: { [CALENDAR_AUTHORITY]: { automatic: true, periodicSeconds: 3600 } }, + }); + await reconcileDeviceAccounts(); + const alice = accountDeviceSync(ALICE); + expect(alice.androidAccountName).toBe('alice'); + expect(alice.enabled).toEqual({ [CALENDAR_AUTHORITY]: true }); + }); + + it('hands the push router the accounts that feed each Android account', async () => { + await enableDeviceSync(ALICE, CONTACTS_AUTHORITY); + useDeviceSyncStore.getState().rememberPrimaryJmapAccount(ALICE, CONTACTS_AUTHORITY, 'c'); + await reconcileDeviceAccounts(); + expect(native.setPushRoutes).toHaveBeenLastCalledWith({ + c: [{ accountName: 'alice', authorities: [CONTACTS_AUTHORITY] }], + }); + }); +}); + +describe('sign-in', () => { + it('clears the sign-in notice and resumes a suspended account where it synced', async () => { + h.state.accounts.push({ + name: 'alice', + registryId: ALICE, + sync: { [CONTACTS_AUTHORITY]: { automatic: false, periodicSeconds: 3600 } }, + }); + useDeviceSyncStore.getState().update(ALICE, { + androidAccountName: 'alice', + suspended: true, + enabled: { [CONTACTS_AUTHORITY]: true }, + }); + await deviceSyncSignedIn(ALICE); + expect(h.state.log).toEqual([ + 'clear alice', + `auto alice ${CONTACTS_AUTHORITY} true`, + `sync alice ${CONTACTS_AUTHORITY} {"manual":true}`, + ]); + expect(accountDeviceSync(ALICE).suspended).toBeUndefined(); + }); + + it('does nothing for an account without an Android account', async () => { + await deviceSyncSignedIn(BOB); + expect(h.state.log).toEqual([]); + }); +}); + +describe('settings actions', () => { + it('applies a new interval to Android while the authority syncs', async () => { + await enableDeviceSync(ALICE, CONTACTS_AUTHORITY); + h.state.log = []; + await changeSyncInterval(ALICE, CONTACTS_AUTHORITY, 0); + expect(h.state.log).toEqual([`periodic alice ${CONTACTS_AUTHORITY} 0`]); + await changeSyncInterval(ALICE, CALENDAR_AUTHORITY, 900); + expect(h.state.log).toEqual([`periodic alice ${CONTACTS_AUTHORITY} 0`]); + expect(accountDeviceSync(ALICE).intervalSeconds[CALENDAR_AUTHORITY as Authority]).toBe(900); + }); + + it('syncs the calendar when the reminder owner changes', async () => { + await enableDeviceSync(ALICE, CALENDAR_AUTHORITY); + h.state.log = []; + await changeReminderOwner(ALICE, 'device'); + expect(h.state.log).toEqual([`sync alice ${CALENDAR_AUTHORITY} {"manual":true}`]); + h.state.log = []; + await changeReminderOwner(ALICE, 'device'); + expect(h.state.log).toEqual([]); + }); +}); diff --git a/src/device-sync/__tests__/app/triggers.test.ts b/src/device-sync/__tests__/app/triggers.test.ts new file mode 100644 index 00000000..ab057388 --- /dev/null +++ b/src/device-sync/__tests__/app/triggers.test.ts @@ -0,0 +1,285 @@ +import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest'; +import AsyncStorage from '@react-native-async-storage/async-storage'; + +const h = vi.hoisted(() => ({ + available: true, + appState: [] as Array<(state: string) => void>, + accountsChanged: [] as Array<() => void>, +})); + +vi.mock('react-native', () => ({ + Platform: { OS: 'android', Version: 34, select: (s: { android?: T; default?: T }) => s.android ?? s.default }, + NativeModules: {}, + NativeEventEmitter: class { + addListener() { + return { remove: () => undefined }; + } + }, + AppState: { + currentState: 'active', + addEventListener: (_type: string, listener: (state: string) => void) => { + h.appState.push(listener); + return { remove: () => { h.appState = h.appState.filter((l) => l !== listener); } }; + }, + }, +})); + +vi.mock('../../native', () => ({ + isDeviceSyncAvailable: () => h.available, + requestSync: vi.fn(async () => undefined), + onAccountsChanged: vi.fn((listener: () => void) => { + h.accountsChanged.push(listener); + return () => { h.accountsChanged = h.accountsChanged.filter((l) => l !== listener); }; + }), +})); + +vi.mock('../../app/lifecycle', () => ({ + reconcileDeviceAccounts: vi.fn(async () => undefined), + refreshPushRoutes: vi.fn(async () => undefined), +})); + +import * as native from '../../native'; +import * as lifecycle from '../../app/lifecycle'; +import { CALENDAR_AUTHORITY, CONTACTS_AUTHORITY } from '../../types'; +import { dispatchStateChange } from '../../../lib/state-change-bus'; +import { useAccountStore, type AccountEntry } from '../../../stores/account-store'; +import { useDeviceSyncStore, waitForDeviceSyncHydration } from '../../../stores/device-sync-store'; +import { cancelDeviceSyncRequests, requestDeviceSync, TRIGGER_DELAY_MS } from '../../app/request'; +import { + FOREGROUND_MIN_INTERVAL_MS, + handleForeground, + handleStateChange, + startDeviceSyncTriggers, +} from '../../app/triggers'; + +const ALICE = 'alice@mail.example.org'; +const BOB = 'bob@mail.example.org'; +const requestSync = vi.mocked(native.requestSync); + +function account(id: string): AccountEntry { + return { + id, + serverUrl: 'https://mail.example.org', + username: id.split('@')[0], + displayName: id, + email: id, + avatarColor: '#000', + lastLoginAt: 0, + isConnected: true, + hasError: false, + isDefault: false, + }; +} + +// Alice syncs contacts and calendars; Bob only calendars. +function syncing(): void { + useDeviceSyncStore.setState({ + accounts: { + [ALICE]: { + contactsSelection: {}, + calendarSelection: { 'team/cal': true }, + reminderOwner: null, + intervalSeconds: {}, + lastRun: {}, + androidAccountName: 'alice', + enabled: { [CONTACTS_AUTHORITY]: true, [CALENDAR_AUTHORITY]: true }, + }, + [BOB]: { + contactsSelection: {}, + calendarSelection: {}, + reminderOwner: null, + intervalSeconds: {}, + lastRun: {}, + androidAccountName: 'bob', + enabled: { [CALENDAR_AUTHORITY]: true }, + }, + }, + }); +} + +let stop: (() => void) | null = null; + +beforeEach(async () => { + await AsyncStorage.clear(); + await waitForDeviceSyncHydration(); + vi.clearAllMocks(); + vi.useFakeTimers(); + h.available = true; + cancelDeviceSyncRequests(); + useAccountStore.setState({ accounts: [account(ALICE), account(BOB)], activeAccountId: ALICE }); + syncing(); +}); + +afterEach(() => { + stop?.(); + stop = null; + cancelDeviceSyncRequests(); + vi.useRealTimers(); +}); + +describe('requestDeviceSync', () => { + it('makes one sync of the edits and echoes of 5 seconds', async () => { + requestDeviceSync(CONTACTS_AUTHORITY); + await vi.advanceTimersByTimeAsync(2000); + requestDeviceSync(CONTACTS_AUTHORITY, ALICE); + requestDeviceSync(CONTACTS_AUTHORITY, ALICE, { jmapAccountId: 'c', type: 'ContactCard', state: 's2' }); + expect(requestSync).not.toHaveBeenCalled(); + await vi.advanceTimersByTimeAsync(TRIGGER_DELAY_MS - 2000); + expect(requestSync).toHaveBeenCalledTimes(1); + expect(requestSync).toHaveBeenCalledWith('alice', CONTACTS_AUTHORITY, {}); + + // The next change arms a new request. + requestDeviceSync(CONTACTS_AUTHORITY); + await vi.advanceTimersByTimeAsync(TRIGGER_DELAY_MS); + expect(requestSync).toHaveBeenCalledTimes(2); + }); + + it('keeps accounts and authorities apart', async () => { + requestDeviceSync(CONTACTS_AUTHORITY, ALICE); + requestDeviceSync(CALENDAR_AUTHORITY, ALICE); + requestDeviceSync(CALENDAR_AUTHORITY, BOB); + await vi.advanceTimersByTimeAsync(TRIGGER_DELAY_MS); + expect(requestSync.mock.calls.map(([name, authority]) => `${name} ${authority}`).sort()).toEqual([ + `alice ${CALENDAR_AUTHORITY}`, + `alice ${CONTACTS_AUTHORITY}`, + `bob ${CALENDAR_AUTHORITY}`, + ]); + }); + + it('asks nothing for an authority the account does not sync, or without device sync', async () => { + requestDeviceSync(CONTACTS_AUTHORITY, BOB); + h.available = false; + requestDeviceSync(CALENDAR_AUTHORITY, BOB); + await vi.advanceTimersByTimeAsync(TRIGGER_DELAY_MS); + expect(requestSync).not.toHaveBeenCalled(); + }); + + it('drops a request when sync was turned off meanwhile', async () => { + requestDeviceSync(CONTACTS_AUTHORITY, ALICE); + useDeviceSyncStore.getState().setEnabled(ALICE, CONTACTS_AUTHORITY, false); + await vi.advanceTimersByTimeAsync(TRIGGER_DELAY_MS); + expect(requestSync).not.toHaveBeenCalled(); + }); + + it('can be cancelled', async () => { + requestDeviceSync(CONTACTS_AUTHORITY, ALICE); + requestDeviceSync(CALENDAR_AUTHORITY, ALICE); + cancelDeviceSyncRequests(ALICE, CONTACTS_AUTHORITY); + await vi.advanceTimersByTimeAsync(TRIGGER_DELAY_MS); + expect(requestSync).toHaveBeenCalledTimes(1); + expect(requestSync).toHaveBeenCalledWith('alice', CALENDAR_AUTHORITY, {}); + }); +}); + +describe('StateChanges of the live stream', () => { + it('syncs the active account on a contact or calendar change', async () => { + handleStateChange(CALENDAR_AUTHORITY, 'CalendarEvent', 'c', 'e7'); + await vi.advanceTimersByTimeAsync(TRIGGER_DELAY_MS); + expect(requestSync).toHaveBeenCalledWith('alice', CALENDAR_AUTHORITY, {}); + }); + + it('skips the echo of what the engine synced itself', async () => { + useDeviceSyncStore.getState().recordKnownState(ALICE, 'c', 'ContactCard', 's9'); + handleStateChange(CONTACTS_AUTHORITY, 'ContactCard', 'c', 's9'); + await vi.advanceTimersByTimeAsync(TRIGGER_DELAY_MS); + expect(requestSync).not.toHaveBeenCalled(); + }); + + it('skips an echo that arrived before the engine recorded its state', async () => { + handleStateChange(CONTACTS_AUTHORITY, 'ContactCard', 'c', 's10'); + // The engine's upload returns and records the state it moved the account to. + useDeviceSyncStore.getState().recordKnownState(ALICE, 'c', 'ContactCard', 's10'); + await vi.advanceTimersByTimeAsync(TRIGGER_DELAY_MS); + expect(requestSync).not.toHaveBeenCalled(); + }); + + it('still syncs when an edit in the app joined the echo', async () => { + handleStateChange(CONTACTS_AUTHORITY, 'ContactCard', 'c', 's11'); + requestDeviceSync(CONTACTS_AUTHORITY); + useDeviceSyncStore.getState().recordKnownState(ALICE, 'c', 'ContactCard', 's11'); + await vi.advanceTimersByTimeAsync(TRIGGER_DELAY_MS); + expect(requestSync).toHaveBeenCalledTimes(1); + }); + + it('ignores shared accounts nothing is synced from, once the personal account is known', async () => { + handleStateChange(CALENDAR_AUTHORITY, 'CalendarEvent', 'other', 'x1'); + await vi.advanceTimersByTimeAsync(TRIGGER_DELAY_MS); + // Unknown yet: anything may feed the device. + expect(requestSync).toHaveBeenCalledTimes(1); + + useDeviceSyncStore.getState().rememberPrimaryJmapAccount(ALICE, CALENDAR_AUTHORITY, 'c'); + handleStateChange(CALENDAR_AUTHORITY, 'CalendarEvent', 'other', 'x2'); + await vi.advanceTimersByTimeAsync(TRIGGER_DELAY_MS); + expect(requestSync).toHaveBeenCalledTimes(1); + + // The shared account a calendar was picked in does feed it. + handleStateChange(CALENDAR_AUTHORITY, 'Calendar', 'team', 'x3'); + await vi.advanceTimersByTimeAsync(TRIGGER_DELAY_MS); + expect(requestSync).toHaveBeenCalledTimes(2); + }); + + it('does nothing for an authority the active account does not sync', async () => { + useAccountStore.setState({ activeAccountId: BOB }); + handleStateChange(CONTACTS_AUTHORITY, 'AddressBook', 'b', 'x'); + await vi.advanceTimersByTimeAsync(TRIGGER_DELAY_MS); + expect(requestSync).not.toHaveBeenCalled(); + }); +}); + +describe('foreground', () => { + it('reconciles the accounts, then syncs everything that syncs, at most once a minute', async () => { + await handleForeground(1_000_000); + expect(lifecycle.reconcileDeviceAccounts).toHaveBeenCalledTimes(1); + await vi.advanceTimersByTimeAsync(TRIGGER_DELAY_MS); + expect(requestSync.mock.calls.map(([name, authority]) => `${name} ${authority}`).sort()).toEqual([ + `alice ${CALENDAR_AUTHORITY}`, + `alice ${CONTACTS_AUTHORITY}`, + `bob ${CALENDAR_AUTHORITY}`, + ]); + + await handleForeground(1_000_000 + FOREGROUND_MIN_INTERVAL_MS - 1); + await vi.advanceTimersByTimeAsync(TRIGGER_DELAY_MS); + expect(requestSync).toHaveBeenCalledTimes(3); + + await handleForeground(1_000_000 + FOREGROUND_MIN_INTERVAL_MS); + await vi.advanceTimersByTimeAsync(TRIGGER_DELAY_MS); + expect(requestSync).toHaveBeenCalledTimes(6); + }); +}); + +describe('startDeviceSyncTriggers', () => { + it('listens to the stream, the foreground, account changes and the preferences', async () => { + stop = startDeviceSyncTriggers(); + // Launch counts as coming to the foreground. + await vi.advanceTimersByTimeAsync(TRIGGER_DELAY_MS); + expect(lifecycle.reconcileDeviceAccounts).toHaveBeenCalledTimes(1); + requestSync.mockClear(); + + dispatchStateChange({ '@type': 'StateChange', changed: { c: { AddressBook: 'a2', Email: 'm1' } } }); + await vi.advanceTimersByTimeAsync(TRIGGER_DELAY_MS); + expect(requestSync).toHaveBeenCalledWith('alice', CONTACTS_AUTHORITY, {}); + + for (const listener of h.appState) listener('background'); + for (const listener of h.appState) listener('active'); + await vi.advanceTimersByTimeAsync(0); + expect(lifecycle.reconcileDeviceAccounts).toHaveBeenCalledTimes(2); + + for (const listener of h.accountsChanged) listener(); + expect(lifecycle.reconcileDeviceAccounts).toHaveBeenCalledTimes(3); + + useDeviceSyncStore.getState().rememberPrimaryJmapAccount(ALICE, CONTACTS_AUTHORITY, 'c'); + await vi.advanceTimersByTimeAsync(1000); + expect(lifecycle.refreshPushRoutes).toHaveBeenCalled(); + + // Started once only. + expect(startDeviceSyncTriggers()).toBe(stop); + }); + + it('starts nothing without device sync', async () => { + h.available = false; + startDeviceSyncTriggers()(); + await vi.advanceTimersByTimeAsync(TRIGGER_DELAY_MS); + expect(lifecycle.reconcileDeviceAccounts).not.toHaveBeenCalled(); + expect(native.onAccountsChanged).not.toHaveBeenCalled(); + }); +}); diff --git a/src/device-sync/app/available.ts b/src/device-sync/app/available.ts new file mode 100644 index 00000000..dde03815 --- /dev/null +++ b/src/device-sync/app/available.ts @@ -0,0 +1,17 @@ +/** + * The gate of every app-side device sync feature (docs/device-sync.md, "App + * integration"): Android, and a build that carries the native module. iOS, + * and an Android build without the module, behave as if device sync did not + * exist. + */ +import { supportsDeviceSync } from '../../lib/platform-capabilities'; +import { isDeviceSyncAvailable } from '../native'; + +export function deviceSyncAvailable(): boolean { + if (!supportsDeviceSync) return false; + try { + return isDeviceSyncAvailable(); + } catch { + return false; + } +} diff --git a/src/device-sync/app/confirm.ts b/src/device-sync/app/confirm.ts new file mode 100644 index 00000000..ddd357cc --- /dev/null +++ b/src/device-sync/app/confirm.ts @@ -0,0 +1,40 @@ +/** + * The questions device sync asks outside the settings screen, as system + * alerts: signing out while changes made on the device have not reached the + * server (docs/device-sync.md, "Lifecycle"). + */ +import { Alert } from 'react-native'; +import { t } from '../../stores/locale-store'; + +/** + * Asks whether to sign out although `pending` device changes (unknown when + * the upload could not even be tried) would be lost. Resolves true to go on. + */ +export function confirmSignOutLosingChanges(pending: number): Promise { + return new Promise((resolve) => { + let settled = false; + const answer = (value: boolean) => { + if (settled) return; + settled = true; + resolve(value); + }; + Alert.alert( + t('device_sync.sign_out_pending_title', 'Sign out anyway?'), + pending > 0 + ? t( + 'device_sync.sign_out_pending_message', + "{count, plural, one {# change made on this device hasn't reached the server. It is lost if you sign out now.} other {# changes made on this device haven't reached the server. They are lost if you sign out now.}}", + { count: pending }, + ) + : t( + 'device_sync.sign_out_unknown_message', + "Changes made on this device may not have reached the server. They are lost if you sign out now.", + ), + [ + { text: t('common.cancel', 'Cancel'), style: 'cancel', onPress: () => answer(false) }, + { text: t('device_sync.sign_out_anyway', 'Sign out'), style: 'destructive', onPress: () => answer(true) }, + ], + { cancelable: true, onDismiss: () => answer(false) }, + ); + }); +} diff --git a/src/device-sync/app/lifecycle.ts b/src/device-sync/app/lifecycle.ts new file mode 100644 index 00000000..f87e9397 --- /dev/null +++ b/src/device-sync/app/lifecycle.ts @@ -0,0 +1,622 @@ +/** + * Turning device sync on and off, signing out, and keeping Android's accounts + * in step with the app's account registry (docs/device-sync.md, "Lifecycle"). + * + * - Enable: permissions (asked by the settings first) → the Android account + * → automatic sync on → the periodic sync → a manual sync → push routes and + * push types. + * - Disable an authority: automatic sync off, then the engine's teardown + * uploads what the device changed and deletes the authority's rows. When + * changes could not be uploaded nothing is deleted and the caller asks the + * user; only a confirmed turn-off deletes them. With both authorities off + * the Android account goes. + * - Sign-out: the same for every authority, before the credentials go. + * - Reconcile (launch, foreground, accounts changed): an Android account whose + * app account the registry dropped is suspended (sync off, one notification, + * data kept); an app account whose Android account was removed in Android + * Settings shows sync off and is never recreated on its own. + * + * The engine (`../task`) is imported on first use only, so starting the app + * does not load it. + */ +import { + clearSyncProblem, + ensureAndroidAccount, + getSyncSettings, + listAndroidAccounts, + openAccountSettings, + removeAndroidAccount, + requestSync, + setPeriodicSync, + setPushRoutes, + setSyncEnabled, + showSyncProblem, +} from '../native'; +import { + AUTHORITIES, + CALENDAR_AUTHORITY, + type AndroidAccount, + type Authority, + type ReminderOwner, +} from '../types'; +import { useAccountStore } from '../../stores/account-store'; +import { + accountDeviceSync, + anySyncOnInApp, + computePushRoutes, + intervalFor, + syncOnInApp, + useDeviceSyncStore, + waitForDeviceSyncHydration, + type AccountDeviceSync, +} from '../../stores/device-sync-store'; +import { t } from '../../stores/locale-store'; +import { toast } from '../../stores/toast-store'; +import { refreshPushSubscriptionTypes } from '../../lib/push-notifications'; +import { deviceSyncAvailable } from './available'; +import { confirmSignOutLosingChanges } from './confirm'; +import { hasSyncPermissions } from './permissions'; +import { STATE_TYPES_BY_AUTHORITY } from './prefs'; +import { cancelDeviceSyncRequests } from './request'; + +/** How long the teardown may spend uploading before it gives up (and asks). */ +export const TEARDOWN_TIMEOUT_MS = 30_000; + +/** After the user agreed to lose what is waiting: one short last try to upload it. */ +export const FORCED_TEARDOWN_TIMEOUT_MS = 5_000; + +/** Where the "sign in again" notification of a suspended account leads: the accounts pane. */ +export const SIGN_IN_AGAIN_URI = 'bulwarkmobile://settings/account'; + +const REGISTRY_HYDRATION_TIMEOUT_MS = 5000; + +const noop = () => undefined; + +function errorMessage(err: unknown): string { + return err instanceof Error && err.message ? err.message : String(err); +} + +/** The registry, once read; null when it could not be read in time. */ +async function readRegistry(): Promise['accounts'] | null> { + const persist = useAccountStore.persist; + if (!persist.hasHydrated()) { + const hydrated = await new Promise((resolve) => { + const timer = setTimeout(() => { + unsubscribe(); + resolve(false); + }, REGISTRY_HYDRATION_TIMEOUT_MS); + const unsubscribe = persist.onFinishHydration(() => { + clearTimeout(timer); + unsubscribe(); + resolve(true); + }); + }); + if (!hydrated) return null; + } + return useAccountStore.getState().accounts; +} + +// ─── Android account names ───────────────────────────── + +/** + * The Android account name for an app account: its login, or its registry id + * when another app account's Android account already has that name (the same + * login on two servers). + */ +export function preferredAndroidAccountName(registryId: string, androidAccounts: readonly AndroidAccount[]): string { + const login = useAccountStore.getState().getAccountById(registryId)?.username?.trim(); + if (!login) return registryId; + const takenOnDevice = androidAccounts.some((a) => a.name === login && a.registryId !== registryId); + const takenInApp = Object.entries(useDeviceSyncStore.getState().accounts) + .some(([id, entry]) => id !== registryId && entry.androidAccountName === login); + return takenOnDevice || takenInApp ? registryId : login; +} + +/** Creates the app account's Android account if needed; resolves its name. */ +async function ensureAccountFor(registryId: string): Promise { + const androidAccounts = await listAndroidAccounts(); + const own = androidAccounts.find((a) => a.registryId === registryId); + if (own) return own.name; + const name = preferredAndroidAccountName(registryId, androidAccounts); + try { + await ensureAndroidAccount(name, registryId); + return name; + } catch (err) { + // The name belongs to another app account after all (the list was stale). + if (name === registryId) throw err; + await ensureAndroidAccount(registryId, registryId); + return registryId; + } +} + +async function androidAccountOf(registryId: string): Promise { + try { + return (await listAndroidAccounts()).find((a) => a.registryId === registryId) ?? null; + } catch { + return null; + } +} + +// ─── Push ────────────────────────────────────────────── + +let lastRoutesJson: string | null = null; + +/** Hands the native push router the JMAP accounts that feed each Android account. */ +export async function refreshPushRoutes(): Promise { + if (!deviceSyncAvailable()) return; + await waitForDeviceSyncHydration(); + const routes = computePushRoutes(useDeviceSyncStore.getState().accounts); + const json = JSON.stringify(routes); + if (json === lastRoutesJson) return; + try { + await setPushRoutes(routes); + lastRoutesJson = json; + } catch { + // Kept for the next refresh. + } +} + +/** Push routes plus the account's push subscription types, after a toggle changed. */ +async function refreshPushState(registryId: string): Promise { + await refreshPushRoutes(); + await refreshPushSubscriptionTypes(registryId).catch(noop); +} + +// ─── Enable ──────────────────────────────────────────── + +export type EnableOutcome = + | { kind: 'enabled'; accountName: string } + /** The runtime permissions are missing: the caller asks for them first. */ + | { kind: 'permission' } + | { kind: 'failed'; message: string }; + +/** + * Turns device sync on for one authority of an app account. The caller has + * asked for the permissions (and, for calendars, who reminds the user). + */ +export async function enableDeviceSync(registryId: string, authority: Authority): Promise { + if (!deviceSyncAvailable()) return { kind: 'failed', message: 'Device sync is not available' }; + if (!(await hasSyncPermissions(authority))) return { kind: 'permission' }; + await waitForDeviceSyncHydration(); + let accountName: string | null = null; + try { + accountName = await ensureAccountFor(registryId); + useDeviceSyncStore.getState().update(registryId, { + androidAccountName: accountName, + removedInAndroidSettings: undefined, + suspended: undefined, + }); + await setSyncEnabled(accountName, authority, true); + await setPeriodicSync(accountName, authority, intervalFor(accountDeviceSync(registryId), authority)); + const entry = accountDeviceSync(registryId); + useDeviceSyncStore.getState().update(registryId, { enabled: { ...entry.enabled, [authority]: true } }); + await requestSync(accountName, authority, { manual: true }).catch(noop); + void refreshPushState(registryId); + return { kind: 'enabled', accountName }; + } catch (err) { + if (accountName) { + await setSyncEnabled(accountName, authority, false).catch(noop); + // Don't leave an account behind that syncs nothing. + if (!anySyncOnInApp(accountDeviceSync(registryId))) { + useDeviceSyncStore.getState().update(registryId, { androidAccountName: undefined }); + await removeAndroidAccount(accountName).catch(noop); + } + } + return { kind: 'failed', message: errorMessage(err) }; + } +} + +/** Automatic sync back on after the user kept syncing instead of losing changes. */ +export async function resumeDeviceSync(registryId: string, authority: Authority): Promise { + if (!deviceSyncAvailable()) return; + await waitForDeviceSyncHydration(); + const accountName = accountDeviceSync(registryId).androidAccountName ?? (await androidAccountOf(registryId))?.name; + if (!accountName) return; + await setSyncEnabled(accountName, authority, true).catch(noop); + await requestSync(accountName, authority, { manual: true }).catch(noop); +} + +// ─── Disable ─────────────────────────────────────────── + +export interface TurnOffResult { + /** Rows deleted and the authority off in the app. */ + done: boolean; + /** Device changes that could not be uploaded (nothing was deleted). */ + pending: number; + /** The teardown itself failed: whether changes are waiting is unknown. */ + failed?: boolean; +} + +interface TeardownResult { + pending: number; + failed: boolean; +} + +// The engine, loaded once on first use (sign-outs tear several accounts down +// at the same time). +let engine: Promise | null = null; + +function loadEngine(): Promise { + engine ??= import('../task').catch((err: unknown) => { + engine = null; + throw err; + }); + return engine; +} + +async function teardown( + registryId: string, + accountName: string, + authority: Authority, + force: boolean, +): Promise { + try { + const { teardownAuthority } = await loadEngine(); + const result = await teardownAuthority( + registryId, + accountName, + authority, + force + ? { force: true, timeoutMs: FORCED_TEARDOWN_TIMEOUT_MS } + : { uploadFirst: true, timeoutMs: TEARDOWN_TIMEOUT_MS }, + ); + return { pending: Math.max(0, result?.pending ?? 0), failed: false }; + } catch (err) { + console.warn('[device-sync] teardown failed', errorMessage(err)); + return { pending: 0, failed: true }; + } +} + +/** The authority's JMAP states, dropped with its rows so no old state passes for an echo. */ +function withoutAuthorityStates(entry: AccountDeviceSync, authority: Authority): AccountDeviceSync['knownStates'] { + if (!entry.knownStates) return undefined; + const types = STATE_TYPES_BY_AUTHORITY[authority]; + const out: Record> = {}; + for (const [jmapAccountId, states] of Object.entries(entry.knownStates)) { + const kept = Object.fromEntries(Object.entries(states ?? {}).filter(([type]) => !types.includes(type))); + if (Object.keys(kept).length > 0) out[jmapAccountId] = kept; + } + return Object.keys(out).length > 0 ? out : undefined; +} + +/** + * Turns device sync off for one authority. Uploads what the device changed + * first; when that is not possible, nothing is deleted and the result says + * how many changes are waiting: call again with `force` once the user agreed + * to lose them, or `resumeDeviceSync` to keep syncing. + */ +export async function disableDeviceSync( + registryId: string, + authority: Authority, + options: { force?: boolean } = {}, +): Promise { + if (!deviceSyncAvailable()) return { done: true, pending: 0 }; + await waitForDeviceSyncHydration(); + const store = useDeviceSyncStore.getState(); + const accountName = accountDeviceSync(registryId).androidAccountName ?? (await androidAccountOf(registryId))?.name; + cancelDeviceSyncRequests(registryId, authority); + if (!accountName) { + store.setEnabled(registryId, authority, false); + return { done: true, pending: 0 }; + } + await setSyncEnabled(accountName, authority, false).catch(noop); + const result = await teardown(registryId, accountName, authority, !!options.force); + if (result.failed || result.pending > 0) { + return { done: false, pending: result.pending, failed: result.failed || undefined }; + } + + const before = accountDeviceSync(registryId); + const { [authority]: _lastRun, ...lastRun } = before.lastRun; + useDeviceSyncStore.getState().update(registryId, { + enabled: { ...before.enabled, [authority]: false }, + lastRun, + knownStates: withoutAuthorityStates(before, authority), + // Asked again the next time calendar sync is turned on. + ...(authority === CALENDAR_AUTHORITY ? { reminderOwner: null } : {}), + }); + if (!anySyncOnInApp(accountDeviceSync(registryId))) { + // Forget the name first: the removal fires `accountsChanged`, and the + // reconcile must not take it for a removal in Android Settings. + useDeviceSyncStore.getState().update(registryId, { androidAccountName: undefined, knownStates: undefined }); + await removeAndroidAccount(accountName).catch(noop); + await clearSyncProblem(accountName).catch(noop); + } + void refreshPushState(registryId); + return { done: true, pending: 0 }; +} + +// ─── Sign-out ────────────────────────────────────────── + +/** + * Before app accounts sign out (`auth-store` logout, logoutAll, + * removeAccount): uploads what their devices changed, deletes the synced + * rows and removes their Android accounts, while the credentials are still + * there. When changes could not be uploaded, `confirm` asks whether to sign + * out anyway. Resolves false when the user chose to stay signed in; their + * accounts then keep syncing. + */ +export async function releaseDeviceSyncBeforeSignOut( + registryIds: readonly string[], + confirm: (pending: number) => Promise = confirmSignOutLosingChanges, +): Promise { + if (!deviceSyncAvailable() || registryIds.length === 0) return true; + try { + return await release(registryIds, confirm); + } catch (err) { + // Never keep the user from signing out over a bug here; an Android + // account left behind is suspended by the next reconcile. + console.warn('[device-sync] release before sign-out failed', errorMessage(err)); + return true; + } +} + +async function release( + registryIds: readonly string[], + confirm: (pending: number) => Promise, +): Promise { + await waitForDeviceSyncHydration(); + let androidAccounts: AndroidAccount[] = []; + try { + androidAccounts = await listAndroidAccounts(); + } catch { + androidAccounts = []; + } + const targets = registryIds.flatMap((registryId) => { + const accountName = androidAccounts.find((a) => a.registryId === registryId)?.name + ?? accountDeviceSync(registryId).androidAccountName; + return accountName ? [{ registryId, accountName }] : []; + }); + if (targets.length > 0) { + toast.info(t('device_sync.signing_out', 'Removing synced contacts and calendars from this device…')); + } + + // Accounts in parallel (each has its own server and run lock), the + // authorities of one account one after the other. + const leftovers: Array<{ registryId: string; accountName: string; authority: Authority }> = []; + let pending = 0; + await Promise.all(targets.map(async ({ registryId, accountName }) => { + cancelDeviceSyncRequests(registryId); + for (const authority of AUTHORITIES) { + await setSyncEnabled(accountName, authority, false).catch(noop); + const result = await teardown(registryId, accountName, authority, false); + if (result.failed || result.pending > 0) { + leftovers.push({ registryId, accountName, authority }); + pending += result.pending; + } + } + })); + + if (leftovers.length > 0) { + if (!(await confirm(pending))) { + // Staying signed in: sync goes on where it was on. + for (const { registryId, accountName } of targets) { + const entry = accountDeviceSync(registryId); + for (const authority of AUTHORITIES) { + if (syncOnInApp(entry, authority)) await setSyncEnabled(accountName, authority, true).catch(noop); + } + } + return false; + } + for (const { registryId, accountName, authority } of leftovers) { + await teardown(registryId, accountName, authority, true); + } + } + + // Forgotten before the removal: its `accountsChanged` must not read as a + // removal in Android Settings. + for (const registryId of registryIds) { + cancelDeviceSyncRequests(registryId); + useDeviceSyncStore.getState().forget(registryId); + } + for (const { accountName } of targets) { + await removeAndroidAccount(accountName).catch(noop); + await clearSyncProblem(accountName).catch(noop); + } + void refreshPushRoutes(); + return true; +} + +// ─── Sign-in and reconcile ───────────────────────────── + +/** + * After an app account signed in successfully: clears its "sign in again" + * notification, resumes a suspended account where it synced, and syncs what + * a failed sign-in held back. + */ +export async function deviceSyncSignedIn(registryId: string): Promise { + if (!deviceSyncAvailable()) return; + try { + await waitForDeviceSyncHydration(); + const account = await androidAccountOf(registryId); + if (!account) return; + await clearSyncProblem(account.name).catch(noop); + const entry = accountDeviceSync(registryId); + useDeviceSyncStore.getState().update(registryId, { androidAccountName: account.name, suspended: undefined }); + const resumed = accountDeviceSync(registryId); + for (const authority of AUTHORITIES) { + if (!syncOnInApp(resumed, authority)) continue; + if (entry.suspended) await setSyncEnabled(account.name, authority, true).catch(noop); + await requestSync(account.name, authority, { manual: true }).catch(noop); + } + await refreshPushRoutes(); + } catch { + // Best effort: the next reconcile catches up. + } +} + +/** An Android account whose app account the registry dropped: sync off, one notification, data kept. */ +async function suspendAccount(account: AndroidAccount & { registryId: string }): Promise { + const entry = useDeviceSyncStore.getState().accounts[account.registryId]; + if (entry?.suspended) return; + // Remember what synced, so signing in again resumes exactly that. + const enabled: Partial> = { ...entry?.enabled }; + try { + const settings = await getSyncSettings(account.name); + for (const authority of AUTHORITIES) { + if (settings.authorities?.[authority]?.automatic) enabled[authority] = true; + } + } catch { + // What the app knows will do. + } + for (const authority of AUTHORITIES) await setSyncEnabled(account.name, authority, false).catch(noop); + cancelDeviceSyncRequests(account.registryId); + useDeviceSyncStore.getState().update(account.registryId, { + androidAccountName: account.name, + enabled, + suspended: true, + }); + await showSyncProblem( + account.name, + t('device_sync.signed_out_title', 'Sign in to keep syncing'), + t( + 'device_sync.signed_out_text', + '{account} was signed out, so its contacts and calendars on this device no longer sync. Sign in again to resume. Changes made on this device are kept.', + { account: account.name }, + ), + SIGN_IN_AGAIN_URI, + t('device_sync.problem_channel', 'Sync problems'), + ).catch(noop); +} + +let reconciling: Promise | null = null; + +/** + * Brings the app's view in step with Android's accounts: at launch, on + * foreground and when accounts of our type change. Concurrent calls share + * one pass. + */ +export function reconcileDeviceAccounts(): Promise { + if (!deviceSyncAvailable()) return Promise.resolve(); + if (!reconciling) { + reconciling = reconcile() + .catch((err) => console.warn('[device-sync] reconcile failed', errorMessage(err))) + .finally(() => { + reconciling = null; + }); + } + return reconciling; +} + +async function reconcile(): Promise { + await waitForDeviceSyncHydration(); + const registry = await readRegistry(); + // Never act on an empty or unreadable registry: a failed read must not + // suspend or forget anything. + if (!registry || registry.length === 0) return; + const androidAccounts = await listAndroidAccounts(); + const registryIds = new Set(registry.map((a) => a.id)); + const store = () => useDeviceSyncStore.getState(); + + // Android accounts that are gone: removed in Android Settings (or by the + // system). Sync shows off and the app does not recreate them. + for (const [registryId, entry] of Object.entries(store().accounts)) { + const name = entry.androidAccountName; + if (!name || androidAccounts.some((a) => a.name === name && (a.registryId ?? registryId) === registryId)) continue; + cancelDeviceSyncRequests(registryId); + if (!registryIds.has(registryId)) { + // Neither signed in nor on the device any more: nothing left to keep. + store().forget(registryId); + continue; + } + store().update(registryId, { + androidAccountName: undefined, + enabled: {}, + suspended: undefined, + knownStates: undefined, + removedInAndroidSettings: anySyncOnInApp(entry) || entry.removedInAndroidSettings || undefined, + }); + void refreshPushSubscriptionTypes(registryId).catch(noop); + } + + for (const account of androidAccounts) { + const registryId = account.registryId; + if (!registryId) continue; + if (!registryIds.has(registryId)) { + await suspendAccount({ ...account, registryId }); + continue; + } + const entry = accountDeviceSync(registryId); + if (entry.suspended) { + // Signed in again through a path that did not report it. + await deviceSyncSignedIn(registryId); + continue; + } + // Android is the source of truth: an authority turned on in Android + // Settings counts as on in the app too. + const enabled = { ...entry.enabled }; + let changed = entry.androidAccountName !== account.name || !!entry.removedInAndroidSettings; + try { + const settings = await getSyncSettings(account.name); + for (const authority of AUTHORITIES) { + if (settings.authorities?.[authority]?.automatic && !enabled[authority]) { + enabled[authority] = true; + changed = true; + } + } + } catch { + // Keep what the app knows. + } + if (changed) { + store().update(registryId, { androidAccountName: account.name, enabled, removedInAndroidSettings: undefined }); + } + } + await refreshPushRoutes(); +} + +// ─── Settings actions ────────────────────────────────── + +/** The periodic sync interval (0 = manual only). */ +export async function changeSyncInterval(registryId: string, authority: Authority, seconds: number): Promise { + await waitForDeviceSyncHydration(); + useDeviceSyncStore.getState().setIntervalSeconds(registryId, authority, seconds); + const entry = accountDeviceSync(registryId); + if (!deviceSyncAvailable() || !entry.androidAccountName || !syncOnInApp(entry, authority)) return; + await setPeriodicSync(entry.androidAccountName, authority, intervalFor(entry, authority)).catch(noop); +} + +/** Who reminds the user of synced events; a change rewrites the device's reminders with a calendar sync. */ +export async function changeReminderOwner(registryId: string, owner: ReminderOwner): Promise { + await waitForDeviceSyncHydration(); + const before = accountDeviceSync(registryId).reminderOwner; + useDeviceSyncStore.getState().setReminderOwner(registryId, owner); + const entry = accountDeviceSync(registryId); + if (before === owner || !deviceSyncAvailable() || !entry.androidAccountName) return; + if (!syncOnInApp(entry, CALENDAR_AUTHORITY)) return; + await requestSync(entry.androidAccountName, CALENDAR_AUTHORITY, { manual: true }).catch(noop); +} + +/** A sync right away: manual and expedited. Resolves false when the account has no Android account. */ +export async function syncNow(registryId: string, authority: Authority): Promise { + const accountName = accountDeviceSync(registryId).androidAccountName; + if (!deviceSyncAvailable() || !accountName) return false; + await requestSync(accountName, authority, { manual: true, expedited: true }); + return true; +} + +/** Android's sync screen of the account ("Android account settings", "Review deletions"). */ +export async function openAndroidAccountSettings(registryId: string): Promise { + const accountName = accountDeviceSync(registryId).androidAccountName; + if (!deviceSyncAvailable() || !accountName) return false; + await openAccountSettings(accountName); + return true; +} + +/** + * Drops a suspended account (one the app signed out while it synced) from + * this device: its Android account and, with it, every synced row. Changes + * made on the device that never reached the server are lost; the caller + * asks first. + */ +export async function removeSuspendedAccount(registryId: string): Promise { + if (!deviceSyncAvailable()) return; + await waitForDeviceSyncHydration(); + const accountName = accountDeviceSync(registryId).androidAccountName ?? (await androidAccountOf(registryId))?.name; + cancelDeviceSyncRequests(registryId); + // Forgotten first, so the removal's `accountsChanged` finds nothing to mark. + useDeviceSyncStore.getState().forget(registryId); + if (accountName) { + await removeAndroidAccount(accountName).catch(noop); + await clearSyncProblem(accountName).catch(noop); + } + void refreshPushRoutes(); +} diff --git a/src/device-sync/app/permissions.ts b/src/device-sync/app/permissions.ts new file mode 100644 index 00000000..ff54d7ff --- /dev/null +++ b/src/device-sync/app/permissions.ts @@ -0,0 +1,54 @@ +/** + * Runtime permissions device sync needs (docs/device-sync.md, "Settings + * sections"): read and write access to contacts or calendars. Without them + * Android fails every sync before our code runs, so the settings check them + * themselves. + */ +import { Linking, PermissionsAndroid } from 'react-native'; +import { CALENDAR_AUTHORITY, CONTACTS_AUTHORITY, type Authority } from '../types'; + +type Permission = Parameters[0]; + +export const SYNC_PERMISSIONS: Record = { + [CONTACTS_AUTHORITY]: ['android.permission.READ_CONTACTS', 'android.permission.WRITE_CONTACTS'], + [CALENDAR_AUTHORITY]: ['android.permission.READ_CALENDAR', 'android.permission.WRITE_CALENDAR'], +}; + +/** + * `granted`: both permissions held. `denied`: the user said no; asking again + * shows the dialog again. `blocked`: Android no longer asks ("don't ask + * again", or denied twice), only the app's system settings can grant them. + */ +export type PermissionOutcome = 'granted' | 'denied' | 'blocked'; + +export async function hasSyncPermissions(authority: Authority): Promise { + try { + const held = await Promise.all(SYNC_PERMISSIONS[authority].map((p) => PermissionsAndroid.check(p))); + return held.every(Boolean); + } catch { + return false; + } +} + +export async function requestSyncPermissions(authority: Authority): Promise { + if (await hasSyncPermissions(authority)) return 'granted'; + const wanted = SYNC_PERMISSIONS[authority]; + try { + const results = (await PermissionsAndroid.requestMultiple([...wanted])) as Record; + const answers = wanted.map((p) => results[p]); + if (answers.every((a) => a === 'granted')) return 'granted'; + if (answers.some((a) => a === 'never_ask_again')) return 'blocked'; + return 'denied'; + } catch { + return 'denied'; + } +} + +/** The app's page in Android's settings, where blocked permissions are granted. */ +export async function openAppSystemSettings(): Promise { + try { + await Linking.openSettings(); + } catch { + // Nothing else to offer. + } +} diff --git a/src/device-sync/app/prefs.ts b/src/device-sync/app/prefs.ts new file mode 100644 index 00000000..12190c3d --- /dev/null +++ b/src/device-sync/app/prefs.ts @@ -0,0 +1,80 @@ +/** + * Pure helpers over the persisted device sync preferences, shared by the + * store (src/stores/device-sync-store.ts) and by code that must not create + * the store: the push setup reads the persisted JSON straight from + * AsyncStorage (docs/device-sync.md, "Triggers"). + * + * Types only; nothing here may import react-native or a store. + */ +import { + AUTHORITIES, + CALENDAR_AUTHORITY, + CONTACTS_AUTHORITY, + type Authority, +} from '../types'; + +/** AsyncStorage key of the device sync store (zustand `persist` name). */ +export const DEVICE_SYNC_STORAGE_KEY = 'device-sync:v1'; + +/** The fields that say whether an account syncs an authority. */ +export interface SyncSwitches { + /** Authorities the user turned on in the app. */ + enabled?: Partial>; + /** The Android account was removed in Android Settings. */ + removedInAndroidSettings?: boolean; + /** The app dropped the account from its registry; automatic sync is off until the user signs in again. */ + suspended?: boolean; +} + +/** + * Whether the app has device sync on for this authority: turned on in the + * app and not removed in Android Settings since. Android's own toggle can + * still pause it (see `getSyncAutomatically`). + */ +export function syncOnInApp(entry: SyncSwitches | undefined | null, authority: Authority): boolean { + return !!entry && !!entry.enabled?.[authority] && !entry.removedInAndroidSettings && !entry.suspended; +} + +/** JMAP data types whose changes a synced authority cares about. */ +export const STATE_TYPES_BY_AUTHORITY: Record = { + [CONTACTS_AUTHORITY]: ['ContactCard', 'AddressBook'], + [CALENDAR_AUTHORITY]: ['CalendarEvent', 'Calendar'], +}; + +/** The authority a JMAP data type belongs to, if any. */ +export function authorityOfType(type: string): Authority | null { + for (const authority of AUTHORITIES) { + if (STATE_TYPES_BY_AUTHORITY[authority].includes(type)) return authority; + } + return null; +} + +/** + * Push types device sync adds to an account's push subscription: + * `ContactCard`, `AddressBook` while contacts sync, `CalendarEvent`, + * `Calendar` while calendars sync. + */ +export function deviceSyncPushTypes(entry: SyncSwitches | undefined | null): string[] { + const types: string[] = []; + for (const authority of AUTHORITIES) { + if (syncOnInApp(entry, authority)) types.push(...STATE_TYPES_BY_AUTHORITY[authority]); + } + return types; +} + +/** + * The persisted accounts map from the store's JSON (`{ state: { accounts } }`), + * or an empty map when it is missing or unreadable. + */ +export function persistedAccounts(raw: string | null): Record { + if (!raw) return {}; + try { + const parsed = JSON.parse(raw) as { state?: { accounts?: unknown } } | null; + const accounts = parsed?.state?.accounts; + return accounts && typeof accounts === 'object' && !Array.isArray(accounts) + ? (accounts as Record) + : {}; + } catch { + return {}; + } +} diff --git a/src/device-sync/app/request.ts b/src/device-sync/app/request.ts new file mode 100644 index 00000000..4d24e891 --- /dev/null +++ b/src/device-sync/app/request.ts @@ -0,0 +1,92 @@ +/** + * Asks Android to sync one account's contacts or calendars because something + * changed while the app runs (docs/device-sync.md, "Triggers"): a StateChange + * on the live stream, the app's own contact or calendar edit, the app coming + * back to the foreground. + * + * Requests are coalesced per account and authority: the first one arms a + * 5-second timer and the rest ride along, so an edit and the StateChange + * echoing it make one sync. A request caused only by StateChanges is dropped + * when the timer fires if the engine has synced exactly those states in the + * meantime (its own uploads echo back). + * + * The request is an ordinary (not manual) one: Android drops it while the + * user has paused the account or the device's auto-sync is off. + */ +import { requestSync } from '../native'; +import type { Authority } from '../types'; +import { useAccountStore } from '../../stores/account-store'; +import { isKnownState, syncOnInApp, useDeviceSyncStore } from '../../stores/device-sync-store'; +import { deviceSyncAvailable } from './available'; + +export const TRIGGER_DELAY_MS = 5000; + +/** The StateChange that asked for a sync. */ +export interface StateReason { + jmapAccountId: string; + type: string; + state: string; +} + +interface Pending { + timer: ReturnType; + reasons: StateReason[]; + /** Asked for by something other than a StateChange: never skipped as an echo. */ + unconditional: boolean; +} + +const pending = new Map(); + +const slotKey = (registryId: string, authority: Authority) => `${registryId}\n${authority}`; + +/** + * Request a sync of `authority` for an app account (the active one when + * `registryId` is omitted), coalesced for 5 seconds. A no-op unless the + * account has device sync on for the authority. + */ +export function requestDeviceSync(authority: Authority, registryId?: string | null, reason?: StateReason): void { + if (!deviceSyncAvailable()) return; + const id = registryId ?? useAccountStore.getState().activeAccountId; + if (!id) return; + if (!syncOnInApp(useDeviceSyncStore.getState().accounts[id], authority)) return; + const key = slotKey(id, authority); + let slot = pending.get(key); + if (!slot) { + slot = { + timer: setTimeout(() => fire(key, id, authority), TRIGGER_DELAY_MS), + reasons: [], + unconditional: false, + }; + pending.set(key, slot); + } + if (reason) slot.reasons.push(reason); + else slot.unconditional = true; +} + +function fire(key: string, registryId: string, authority: Authority): void { + const slot = pending.get(key); + pending.delete(key); + if (!slot) return; + const entry = useDeviceSyncStore.getState().accounts[registryId]; + const accountName = entry?.androidAccountName; + if (!accountName || !syncOnInApp(entry, authority)) return; + const echoesOnly = !slot.unconditional + && slot.reasons.every((r) => isKnownState(registryId, r.jmapAccountId, r.type, r.state)); + if (echoesOnly) return; + void Promise.resolve() + .then(() => requestSync(accountName, authority, {})) + .catch(() => undefined); +} + +/** + * Drops waiting requests: of one authority of an account, of every authority + * of an account, or all of them. Used before sync is turned off. + */ +export function cancelDeviceSyncRequests(registryId?: string, authority?: Authority): void { + for (const [key, slot] of pending) { + if (registryId && authority && key !== slotKey(registryId, authority)) continue; + if (registryId && !key.startsWith(`${registryId}\n`)) continue; + clearTimeout(slot.timer); + pending.delete(key); + } +} diff --git a/src/device-sync/app/triggers.ts b/src/device-sync/app/triggers.ts new file mode 100644 index 00000000..73a29c6a --- /dev/null +++ b/src/device-sync/app/triggers.ts @@ -0,0 +1,119 @@ +/** + * What makes the running app ask Android for device syncs + * (docs/device-sync.md, "Triggers"), started once from the app's launch + * effect: + * + * - StateChanges on the live stream for `AddressBook`/`ContactCard` and + * `Calendar`/`CalendarEvent` (the stream serves the active account), except + * the states the engine synced itself; + * - the app returning to the foreground, and the launch itself: every account + * with sync on, after the accounts were reconciled with Android's; + * - accounts of our type changing (removed in Android Settings); + * - the preferences changing: the native push router's routes follow them. + * + * The app's own contact and calendar edits call `requestDeviceSync` from the + * stores. Every request is coalesced per account and authority (./request). + */ +import { AppState, type AppStateStatus } from 'react-native'; +import { onAccountsChanged } from '../native'; +import { AUTHORITIES, type Authority } from '../types'; +import { onStateChangeType } from '../../lib/state-change-bus'; +import { useAccountStore } from '../../stores/account-store'; +import { + feedingJmapAccounts, + isKnownState, + syncOnInApp, + useDeviceSyncStore, +} from '../../stores/device-sync-store'; +import { deviceSyncAvailable } from './available'; +import { reconcileDeviceAccounts, refreshPushRoutes } from './lifecycle'; +import { STATE_TYPES_BY_AUTHORITY } from './prefs'; +import { requestDeviceSync } from './request'; + +/** Foreground syncs of one account and authority at most this often. */ +export const FOREGROUND_MIN_INTERVAL_MS = 60_000; +const ROUTES_DELAY_MS = 1000; + +let stopTriggers: (() => void) | null = null; +const lastForegroundRequest = new Map(); + +/** A StateChange of the active account's stream. */ +export function handleStateChange(authority: Authority, type: string, jmapAccountId: string, state: string): void { + const registryId = useAccountStore.getState().activeAccountId; + if (!registryId) return; + const entry = useDeviceSyncStore.getState().accounts[registryId]; + if (!entry || !syncOnInApp(entry, authority)) return; + // The engine's own upload echoing back. + if (isKnownState(registryId, jmapAccountId, type, state)) return; + // A shared account nothing is synced from, once the app knows which + // accounts feed the device. + if (entry.primaryJmapAccounts?.[authority] && !feedingJmapAccounts(entry, authority).includes(jmapAccountId)) return; + requestDeviceSync(authority, registryId, { jmapAccountId, type, state }); +} + +/** Launch and every return to the foreground. */ +export async function handleForeground(now: number = Date.now()): Promise { + await reconcileDeviceAccounts(); + const { accounts } = useDeviceSyncStore.getState(); + for (const account of useAccountStore.getState().accounts) { + for (const authority of AUTHORITIES) { + if (!syncOnInApp(accounts[account.id], authority)) continue; + const key = `${account.id}\n${authority}`; + const last = lastForegroundRequest.get(key); + if (last !== undefined && now - last < FOREGROUND_MIN_INTERVAL_MS) continue; + lastForegroundRequest.set(key, now); + requestDeviceSync(authority, account.id); + } + } +} + +/** Starts the triggers; idempotent. Returns the stop function (for tests). */ +export function startDeviceSyncTriggers(): () => void { + if (stopTriggers) return stopTriggers; + if (!deviceSyncAvailable()) return () => undefined; + const cleanups: Array<() => void> = []; + + for (const authority of AUTHORITIES) { + for (const type of STATE_TYPES_BY_AUTHORITY[authority]) { + cleanups.push(onStateChangeType(type, (jmapAccountId, state) => { + handleStateChange(authority, type, jmapAccountId, state); + })); + } + } + + let appState: AppStateStatus = AppState.currentState; + const appStateSubscription = AppState.addEventListener('change', (next) => { + const cameBack = next === 'active' && appState !== 'active'; + appState = next; + if (cameBack) void handleForeground(); + }); + cleanups.push(() => appStateSubscription.remove()); + + try { + cleanups.push(onAccountsChanged(() => { void reconcileDeviceAccounts(); })); + } catch { + // Without the event, launch and foreground still reconcile. + } + + let routesTimer: ReturnType | null = null; + cleanups.push(useDeviceSyncStore.subscribe((state, prev) => { + if (state.accounts === prev.accounts) return; + if (routesTimer) clearTimeout(routesTimer); + routesTimer = setTimeout(() => { + routesTimer = null; + void refreshPushRoutes(); + }, ROUTES_DELAY_MS); + })); + cleanups.push(() => { + if (routesTimer) clearTimeout(routesTimer); + }); + + void handleForeground(); + + stopTriggers = () => { + for (const cleanup of cleanups) cleanup(); + stopTriggers = null; + lastForegroundRequest.clear(); + }; + return stopTriggers; +} diff --git a/src/lib/__tests__/push-background-task.test.ts b/src/lib/__tests__/push-background-task.test.ts index ee1c02ae..c8984ad9 100644 --- a/src/lib/__tests__/push-background-task.test.ts +++ b/src/lib/__tests__/push-background-task.test.ts @@ -15,6 +15,7 @@ import { NativeModules } from 'react-native'; import { jmapClient } from '../../api/jmap-client'; import { secureFetch } from '../client-cert'; import { + carriesNoMail, matchAccountsForPush, parseRelayPushData, pushBackgroundTask, @@ -186,6 +187,58 @@ describe('pushBackgroundTask notifications', () => { }); }); +describe('pushes for device sync (#34)', () => { + const LOCAL = 'alice@mail.example.com'; + const showNotification = vi.fn(async () => undefined); + const stateChange = (changed: Record>) => ({ + kind: 'jmap-state-change', + accountLabel: 'alice', + accountId: Object.keys(changed)[0], + emailIds: '[]', + changed: JSON.stringify(changed), + }); + + beforeEach(async () => { + vi.clearAllMocks(); + await AsyncStorage.clear(); + await AsyncStorage.setItem('push:accountIds:v1', JSON.stringify([LOCAL])); + await AsyncStorage.setItem('push:jmapAccountIds:v1', JSON.stringify({ [LOCAL]: 'jmap-primary' })); + (NativeModules as Record).BulwarkFcm = { showNotification }; + (jmapClient.getStoredCredentials as ReturnType).mockResolvedValue({ + serverUrl: 'https://mail.example.com', + username: 'alice', + password: 'secret', + }); + (secureFetch as ReturnType).mockResolvedValue({ ok: false, status: 500 }); + }); + + afterEach(() => { + delete (NativeModules as Record).BulwarkFcm; + }); + + it('tells contact and calendar changes from new mail', () => { + expect(carriesNoMail(parseRelayPushData(stateChange({ a: { ContactCard: 's1', AddressBook: 's2' } })))).toBe(true); + expect(carriesNoMail(parseRelayPushData(stateChange({ a: { CalendarEvent: 'e1' }, b: { Calendar: 'c1' } })))).toBe(true); + // Mail in any account of the map, an EmailPush, or a map it can't read: as before. + expect(carriesNoMail(parseRelayPushData(stateChange({ a: { ContactCard: 's1' }, b: { EmailDelivery: 'm1' } })))).toBe(false); + expect(carriesNoMail(parseRelayPushData({ kind: 'jmap-email-push', accountId: 'a', emailIds: '["m1"]' }))).toBe(false); + expect(carriesNoMail(parseRelayPushData({ kind: 'jmap-state-change', emailIds: '[]', changed: '{}' }))).toBe(false); + expect(carriesNoMail(parseRelayPushData({ kind: 'jmap-state-change', emailIds: '[]' }))).toBe(false); + }); + + it('neither looks for mail nor notifies for a change without new mail', async () => { + await pushBackgroundTask(stateChange({ 'jmap-primary': { ContactCard: 's1' }, team: { CalendarEvent: 'e1' } })); + expect(jmapClient.getStoredCredentials).not.toHaveBeenCalled(); + expect(secureFetch).not.toHaveBeenCalled(); + expect(showNotification).not.toHaveBeenCalled(); + }); + + it('still checks for mail when the map carries new mail', async () => { + await pushBackgroundTask(stateChange({ 'jmap-primary': { EmailDelivery: 'm1', ContactCard: 's1' } })); + expect(jmapClient.getStoredCredentials).toHaveBeenCalledWith(LOCAL); + }); +}); + describe('selectNotifiableEmails', () => { const email = (id: string, keywords: Record = {}): Email => ({ id, threadId: 't', keywords, mailboxIds: {}, size: 0, receivedAt: '', hasAttachment: false } as Email); diff --git a/src/lib/__tests__/push-notifications.test.ts b/src/lib/__tests__/push-notifications.test.ts index a324a717..519b47ce 100644 --- a/src/lib/__tests__/push-notifications.test.ts +++ b/src/lib/__tests__/push-notifications.test.ts @@ -42,6 +42,12 @@ const { CREATED } = vi.hoisted(() => ({ CREATED: { id: 'new-server-id', expires: null as string | null }, })); +// A client of another signed-in account (device sync patches its push types +// without the singleton). +const { DETACHED } = vi.hoisted(() => ({ + DETACHED: { loaded: true, types: ['EmailDelivery'] as string[], log: [] as unknown[] }, +})); + vi.mock('../../api/jmap-client', () => ({ jmapClient: { username: 'user@example.com', @@ -49,6 +55,20 @@ vi.mock('../../api/jmap-client', () => ({ accountId: 'jmap-primary', currentSession: { capabilities: { 'urn:ietf:params:jmap:core': {} } }, }, + JMAPClient: class { + async loadAccount(id: string) { + DETACHED.log.push(['load', id]); + return DETACHED.loaded; + } + async request(calls: Array<[string, Record, string]>) { + const [name, args, id] = calls[0]; + DETACHED.log.push([name, args]); + if (name === 'PushSubscription/get') { + return { methodResponses: [[name, { list: [{ id: args.ids[0], types: DETACHED.types }] }, id]] }; + } + return { methodResponses: [[name, { updated: { [Object.keys(args.update)[0]]: null } }, id]] }; + } + }, })); vi.mock('../../api/email', () => ({ @@ -77,6 +97,8 @@ import { readPushAccountIds, readPushJmapAccountIds, PushSetupError, + pushTypesFor, + refreshPushSubscriptionTypes, resyncPushNotifications, revokePushDevice, teardownPushNotificationsForAccount, @@ -354,6 +376,117 @@ describe('setupPushNotifications subscription shape', () => { }); }); +describe('push types for device sync (#34)', () => { + const DEVICE_SYNC_KEY = 'device-sync:v1'; + const CONTACTS = 'com.android.contacts'; + const CALENDAR = 'com.android.calendar'; + const healthy = (types: string[]) => ({ + id: 'existing', + deviceClientId: OUR_DCID, + expires: new Date(Date.now() + 80 * 86400000).toISOString(), + types, + }); + + async function syncing(enabled: Record, extra: Record = {}): Promise { + await AsyncStorage.setItem(DEVICE_SYNC_KEY, JSON.stringify({ + state: { accounts: { [ACCOUNT_ID]: { enabled, ...extra } } }, + version: 0, + })); + } + + beforeEach(async () => { + vi.clearAllMocks(); + await AsyncStorage.clear(); + await AsyncStorage.setItem(deviceClientIdKey(ACCOUNT_ID), OUR_DCID); + (jmapClient as { currentSession: unknown }).currentSession = { + capabilities: { 'urn:ietf:params:jmap:core': {} }, + }; + listMock.mockResolvedValue([]); + updateMock.mockResolvedValue(true); + installFetch({}); + }); + + it('adds the contact and calendar types of what the account syncs to the device', async () => { + expect(await pushTypesFor(ACCOUNT_ID)).toEqual(['EmailDelivery']); + await syncing({ [CONTACTS]: true, [CALENDAR]: false }); + expect(await pushTypesFor(ACCOUNT_ID)).toEqual(['EmailDelivery', 'ContactCard', 'AddressBook']); + expect(await pushTypesFor('someone@else.example')).toEqual(['EmailDelivery']); + // Removed in Android Settings: nothing to route any more. + await syncing({ [CONTACTS]: true }, { removedInAndroidSettings: true }); + expect(await pushTypesFor(ACCOUNT_ID)).toEqual(['EmailDelivery']); + }); + + it('subscribes with them', async () => { + await syncing({ [CALENDAR]: true }); + await setupPushNotifications({ relayBaseUrl: RELAY }); + expect(createMock.mock.calls[0][0].types).toEqual(['EmailDelivery', 'CalendarEvent', 'Calendar']); + }); + + it('patches a subscription made before sync was turned on', async () => { + await AsyncStorage.setItem(SUB_KEY, 'existing'); + listMock.mockResolvedValue([healthy(['EmailDelivery'])]); + await syncing({ [CONTACTS]: true, [CALENDAR]: true }); + await setupPushNotifications({ relayBaseUrl: RELAY }); + expect(createMock).not.toHaveBeenCalled(); + expect(updateMock.mock.calls[0][1].types) + .toEqual(['EmailDelivery', 'ContactCard', 'AddressBook', 'CalendarEvent', 'Calendar']); + }); + + it('re-applies the types after a toggle', async () => { + await AsyncStorage.setItem(SUB_KEY, 'existing'); + listMock.mockResolvedValue([healthy(['EmailDelivery'])]); + await syncing({ [CONTACTS]: true }); + await refreshPushSubscriptionTypes(ACCOUNT_ID); + expect(updateMock).toHaveBeenCalledWith('existing', { types: ['EmailDelivery', 'ContactCard', 'AddressBook'] }); + + // An account without a push subscription has nothing to patch. + updateMock.mockClear(); + DETACHED.log = []; + await refreshPushSubscriptionTypes('someone@else.example'); + expect(updateMock).not.toHaveBeenCalled(); + expect(DETACHED.log).toEqual([]); + + // Already what it should be, in another order: nothing to send. + listMock.mockResolvedValue([healthy(['AddressBook', 'EmailDelivery', 'ContactCard'])]); + await refreshPushSubscriptionTypes(ACCOUNT_ID); + expect(updateMock).not.toHaveBeenCalled(); + }); + + it("patches another account's subscription through a client of its own", async () => { + const OTHER = 'bob@other.example.net'; + await AsyncStorage.setItem(`push:subscriptionId:v2:${OTHER}`, 'bob-sub'); + await AsyncStorage.setItem(DEVICE_SYNC_KEY, JSON.stringify({ + state: { accounts: { [OTHER]: { enabled: { [CALENDAR]: true } } } }, + version: 0, + })); + DETACHED.log = []; + DETACHED.types = ['EmailDelivery']; + await refreshPushSubscriptionTypes(OTHER); + expect(DETACHED.log).toEqual([ + ['load', OTHER], + ['PushSubscription/get', { ids: ['bob-sub'], properties: ['id', 'types'] }], + ['PushSubscription/set', { update: { 'bob-sub': { types: ['EmailDelivery', 'CalendarEvent', 'Calendar'] } } }], + ]); + // The singleton's subscriptions were not touched. + expect(updateMock).not.toHaveBeenCalled(); + + // Already right: read, nothing written. + DETACHED.log = []; + DETACHED.types = ['Calendar', 'CalendarEvent', 'EmailDelivery']; + await refreshPushSubscriptionTypes(OTHER); + expect(DETACHED.log.map((entry) => (entry as unknown[])[0])).toEqual(['load', 'PushSubscription/get']); + }); + + it('leaves the subscription of an account that turned push off alone', async () => { + await AsyncStorage.setItem(SUB_KEY, 'existing'); + await AsyncStorage.setItem(`push:optedOut:v1:${ACCOUNT_ID}`, '1'); + listMock.mockResolvedValue([healthy(['EmailDelivery'])]); + await syncing({ [CONTACTS]: true }); + await refreshPushSubscriptionTypes(ACCOUNT_ID); + expect(updateMock).not.toHaveBeenCalled(); + }); +}); + describe('setupPushNotifications with ACL-shared accounts (B18)', () => { const REFUSED_KEY = 'push:emailPushRefused:v1:' + ACCOUNT_ID; const sharedMock = getSharedMailboxes as ReturnType; diff --git a/src/lib/platform-capabilities.ts b/src/lib/platform-capabilities.ts index 785786c6..60489183 100644 --- a/src/lib/platform-capabilities.ts +++ b/src/lib/platform-capabilities.ts @@ -14,3 +14,15 @@ import { Platform } from 'react-native'; * check) is hidden when this is false. */ export const supportsSideloadUpdates = Platform.OS === 'android'; + +/** + * Device sync: address books and calendars in Android's Contacts and Calendar + * providers, under an account of the app's own type (#34, + * docs/device-sync.md). + * + * Android only: iOS has no third-party sync-provider API for either. The UI + * also needs the native module (`isDeviceSyncAvailable()` in + * `device-sync/native`), so a build without it hides the feature instead of + * failing. + */ +export const supportsDeviceSync = Platform.OS === 'android'; diff --git a/src/lib/push-background-task.ts b/src/lib/push-background-task.ts index dacd2778..776b39c4 100644 --- a/src/lib/push-background-task.ts +++ b/src/lib/push-background-task.ts @@ -378,12 +378,29 @@ async function detachedNewestUnreadInboxIds( return (qBody.ids as string[]) ?? []; } +/** + * A StateChange push in which no account got new mail: only contact or + * calendar changes, which the native push router already turned into device + * syncs (#34, docs/device-sync.md "Triggers"). Without `EmailDelivery` there + * is nothing to notify, and the legacy path (the newest unread mail) must + * not run for it. An empty or missing `changed` map keeps the old behaviour. + */ +export function carriesNoMail(payload: RelayPushData): boolean { + if (payload.kind !== 'jmap-state-change' || !payload.changed) return false; + const accounts = Object.values(payload.changed); + if (accounts.length === 0) return false; + return !accounts.some((types) => !!types && typeof types === 'object' && 'EmailDelivery' in types); +} + // Fired by BulwarkPushTaskService when a data FCM message arrives. Runs in a // short-lived headless JS runtime - keep it fast, catch all errors, and always // resolve so the native service can release its wake lock. Never touches the // singleton jmapClient's session (see "Detached JMAP access" above). export async function pushBackgroundTask(data: unknown): Promise { try { + const payload = parseRelayPushData(data); + if (carriesNoMail(payload)) return; + if (!(await emailNotificationsAllowed())) return; await migrateLegacyPushKeys(); @@ -392,7 +409,6 @@ export async function pushBackgroundTask(data: unknown): Promise { if (accountIds.length === 0) return; const registry = await readAccountRegistry(); - const payload = parseRelayPushData(data); const jmapAccountIds = await readPushJmapAccountIds(); const accountsToCheck = matchAccountsForPush(payload, accountIds, jmapAccountIds, registry); diff --git a/src/lib/push-notifications.ts b/src/lib/push-notifications.ts index e25a79b8..4151665b 100644 --- a/src/lib/push-notifications.ts +++ b/src/lib/push-notifications.ts @@ -9,12 +9,13 @@ import { } from '../api/push'; import { getMailboxes, getSharedMailboxes } from '../api/email'; import { loadedMailboxes } from './mailbox-source'; -import { jmapClient } from '../api/jmap-client'; -import { JMAPMethodError } from '../api/jmap-result'; +import { jmapClient, JMAPClient } from '../api/jmap-client'; +import { JMAPMethodError, requireMethodResult } from '../api/jmap-result'; import { CAPABILITIES } from '../api/types'; import type { EmailPushConfig, JMAPAccountInfo, Mailbox } from '../api/types'; import { generateAccountId } from './account-utils'; import { t } from '../stores/locale-store'; +import { DEVICE_SYNC_STORAGE_KEY, deviceSyncPushTypes, persistedAccounts } from '../device-sync/app/prefs'; import { getUnifiedPushDistributors, isUnifiedPushSupported, @@ -206,6 +207,25 @@ export function isValidRelayUrl(value: string): boolean { // In-app sync uses the separate SSE channel and is unaffected. export const PUSH_TYPES = ['EmailDelivery'] as const; +/** + * The push types of one account's subscription: `EmailDelivery`, plus the + * contact and calendar types while the account syncs them to the device + * (#34, docs/device-sync.md "Triggers"): the native push router turns those + * into device syncs, and a push carrying only them never notifies. + * + * Read from the device sync store's persisted JSON rather than the store, so + * code that runs without it (the push task) never creates it. + */ +export async function pushTypesFor(accountId: string): Promise { + let extra: string[] = []; + try { + extra = deviceSyncPushTypes(persistedAccounts(await AsyncStorage.getItem(DEVICE_SYNC_STORAGE_KEY))[accountId]); + } catch { + extra = []; + } + return [...PUSH_TYPES, ...extra]; +} + // draft-ietf-jmap-emailpush (Stalwart >= 0.16.16). `EmailDelivery` alone // still fires for every ingested message - including spam the server files // straight into Junk - because the server can't know which folders a client @@ -927,6 +947,7 @@ async function setupPushNotificationsInner( : null; const subKey = subscriptionIdKey(accountId); const storedServerId = await AsyncStorage.getItem(subKey); + const types = await pushTypesFor(accountId); let jmapAccountId: string | null = null; try { jmapAccountId = jmapClient.accountId; @@ -939,7 +960,7 @@ async function setupPushNotificationsInner( if (match) { if (match.expires) await AsyncStorage.setItem(subscriptionExpiresKey(accountId), match.expires); if (!params.forceRecreate) { - const refreshed = await refreshSubscriptionExpires(match, emailPush); + const refreshed = await refreshSubscriptionExpires(match, emailPush, types); if (refreshed) { await rememberRefusedEmailPushAccounts(accountId, refusedBefore, emailPush, refreshed.emailPush); await addPushAccountId(accountId); @@ -989,7 +1010,7 @@ async function setupPushNotificationsInner( createPushSubscription({ deviceClientId, url: buildRelayUrl(relayBaseUrl, `/api/push/jmap/${encodeURIComponent(deviceClientId)}`), - types: [...PUSH_TYPES], + types: [...types], expires: expiresFromNow(SUBSCRIPTION_EXPIRES_DAYS), ...(filter ? { emailPush: filter } : {}), }), @@ -1040,10 +1061,10 @@ async function addPushAccountId(accountId: string): Promise { // Push the subscription's expires forward when it's getting close to the // server's ceiling, and re-sync `types` / the delivery filter when they drift // from what this client wants (a subscription created by an older build still -// listens to `Email`/`Mailbox`; a Junk mailbox id can change under us). -// Returns false if the server rejects the update, which the caller treats as -// "replace"; otherwise the emailPush map the update installed (null when it -// left the filter alone). +// listens to `Email`/`Mailbox`; device sync was turned on or off; a Junk +// mailbox id can change under us). Returns false if the server rejects the +// update, which the caller treats as "replace"; otherwise the emailPush map +// the update installed (null when it left the filter alone). async function refreshSubscriptionExpires( sub: { id: string; @@ -1053,8 +1074,9 @@ async function refreshSubscriptionExpires( }, // null when the server has no emailPush support - leave the property alone. desiredEmailPush: Record | null, + types: readonly string[], ): Promise | null }> { - const typesNeedUpdate = !sameTypes(sub.types, PUSH_TYPES); + const typesNeedUpdate = !sameTypes(sub.types, types); const emailPushNeedsUpdate = desiredEmailPush !== null && !sameEmailPush(sub.emailPush, desiredEmailPush); if (!typesNeedUpdate && !emailPushNeedsUpdate && sub.expires) { @@ -1069,7 +1091,7 @@ async function refreshSubscriptionExpires( const patch: { expires?: string; types?: string[] } = { expires: expiresFromNow(SUBSCRIPTION_EXPIRES_DAYS), }; - if (typesNeedUpdate) patch.types = [...PUSH_TYPES]; + if (typesNeedUpdate) patch.types = [...types]; const { emailPush } = await writeWithEmailPush( emailPushNeedsUpdate ? desiredEmailPush : null, (filter) => updatePushSubscription(sub.id, filter ? { ...patch, emailPush: filter } : patch), @@ -1198,6 +1220,47 @@ export async function resyncPushNotifications( return setupPushNotifications(params); } +/** + * Re-apply an account's push types to its subscription after device sync was + * turned on or off for it (#34). The account the client serves goes through + * the api/push helpers; another signed-in account through a client of its + * own, never the singleton. Best effort: the next resync of the account + * applies them anyway. + */ +export async function refreshPushSubscriptionTypes(accountId: string): Promise { + if (await AsyncStorage.getItem(optedOutKey(accountId))) return; + const subscriptionId = await AsyncStorage.getItem(subscriptionIdKey(accountId)); + if (!subscriptionId) return; + const types = await pushTypesFor(accountId); + const username = jmapClient.username; + const serverUrl = jmapClient.serverUrl; + const served = !!username && !!serverUrl && generateAccountId(username, serverUrl) === accountId; + try { + if (served) { + const current = (await listPushSubscriptions()).find((s) => s.id === subscriptionId); + if (!current || sameTypes(current.types, types)) return; + await updatePushSubscription(subscriptionId, { types: [...types] }); + return; + } + const client = new JMAPClient(); + if (!(await client.loadAccount(accountId))) return; + const using = [CAPABILITIES.CORE]; + const res = await client.request( + [['PushSubscription/get', { ids: [subscriptionId], properties: ['id', 'types'] }, '0']], + using, + ); + const current = (requireMethodResult(res, '0', 'PushSubscription/get').list as Array<{ id: string; types?: string[] | null }> | undefined) + ?.find((s) => s.id === subscriptionId); + if (!current || sameTypes(current.types, types)) return; + await client.request( + [['PushSubscription/set', { update: { [subscriptionId]: { types: [...types] } } }, '0']], + using, + ); + } catch { + // The next resync re-applies them. + } +} + /** * Tear down push for ALL accounts on this device. Used by the logout-all * flow; best-effort because we typically aren't authenticated to every diff --git a/src/stores/__tests__/device-sync-store.test.ts b/src/stores/__tests__/device-sync-store.test.ts new file mode 100644 index 00000000..9484c300 --- /dev/null +++ b/src/stores/__tests__/device-sync-store.test.ts @@ -0,0 +1,268 @@ +import { describe, it, expect, beforeEach, vi } from 'vitest'; +import AsyncStorage from '@react-native-async-storage/async-storage'; +import { CALENDAR_AUTHORITY, CONTACTS_AUTHORITY, type RunStatus } from '../../device-sync/types'; +import { + DEVICE_SYNC_STORAGE_KEY, + deviceSyncPushTypes, + persistedAccounts, + syncOnInApp, +} from '../../device-sync/app/prefs'; + +type StoreModule = typeof import('../device-sync-store'); + +// A fresh store module, hydrated from whatever AsyncStorage holds now. +async function freshStore(): Promise { + vi.resetModules(); + const mod = await import('../device-sync-store'); + await mod.waitForDeviceSyncHydration(); + return mod; +} + +async function stored(): Promise> { + const raw = await AsyncStorage.getItem(DEVICE_SYNC_STORAGE_KEY); + return raw ? JSON.parse(raw).state.accounts : {}; +} + +const ALICE = 'alice@mail.example.org'; +const BOB = 'bob@mail.example.org'; + +const run = (outcome: RunStatus['outcome']): RunStatus => ({ + at: 1, + outcome, + durationMs: 10, + conflicts: 0, + itemErrors: 0, + stats: { + downloaded: { created: 0, updated: 0, deleted: 0 }, + uploaded: { created: 0, updated: 0, deleted: 0 }, + entries: 0, + skipped: 0, + }, +}); + +beforeEach(async () => { + await AsyncStorage.clear(); +}); + +describe('selection', () => { + it("syncs the personal account's collections unless turned off and shared ones only when turned on", async () => { + const { isCollectionSelected } = await freshStore(); + expect(isCollectionSelected({}, CONTACTS_AUTHORITY, { key: 'c/b1', isPersonal: true })).toBe(true); + expect(isCollectionSelected({}, CONTACTS_AUTHORITY, { key: 'd/b9', isPersonal: false })).toBe(false); + expect(isCollectionSelected({ 'c/b1': false }, CONTACTS_AUTHORITY, { key: 'c/b1', isPersonal: true })).toBe(false); + expect(isCollectionSelected({ 'd/b9': true }, CONTACTS_AUTHORITY, { key: 'd/b9', isPersonal: false })).toBe(true); + // The webmail's allow-list book stays off unless the user picks it. + const trusted = { key: 'c/b2', isPersonal: true, name: 'Trusted Senders' }; + expect(isCollectionSelected({}, CONTACTS_AUTHORITY, trusted)).toBe(false); + expect(isCollectionSelected({ 'c/b2': true }, CONTACTS_AUTHORITY, trusted)).toBe(true); + expect(isCollectionSelected({}, CALENDAR_AUTHORITY, trusted)).toBe(true); + }); + + it('lists the collections that sync, per authority, from explicit choices and the default', async () => { + const m = await freshStore(); + const store = m.useDeviceSyncStore.getState(); + store.setCollectionSelected(ALICE, CONTACTS_AUTHORITY, 'c/b2', false); + store.setCollectionSelected(ALICE, CONTACTS_AUTHORITY, 'd/b9', true); + store.setCollectionSelected(ALICE, CALENDAR_AUTHORITY, 'c/cal1', false); + const entry = m.accountDeviceSync(ALICE); + const books = [ + { key: 'c/b1', isPersonal: true }, + { key: 'c/b2', isPersonal: true }, + { key: 'd/b9', isPersonal: false }, + { key: 'd/b8', isPersonal: false }, + ]; + expect(m.effectiveSelection(entry, CONTACTS_AUTHORITY, books)).toEqual(['c/b1', 'd/b9']); + // The calendar choice does not leak into the contacts selection. + expect(m.selectionFor(entry, CONTACTS_AUTHORITY)).toEqual({ 'c/b2': false, 'd/b9': true }); + expect(m.selectionFor(entry, CALENDAR_AUTHORITY)).toEqual({ 'c/cal1': false }); + }); + + it('gives the engine its preferences, with the calendar app reminding until the user chose', async () => { + const m = await freshStore(); + expect(m.accountSyncPrefs(ALICE)).toEqual({ + contactsSelection: {}, + calendarSelection: {}, + newContactsAddressBook: undefined, + reminderOwner: 'device', + }); + m.useDeviceSyncStore.getState().setReminderOwner(ALICE, 'bulwark'); + m.useDeviceSyncStore.getState().setNewContactsAddressBook(ALICE, 'c/b2'); + expect(m.accountSyncPrefs(ALICE)).toMatchObject({ reminderOwner: 'bulwark', newContactsAddressBook: 'c/b2' }); + m.useDeviceSyncStore.getState().setNewContactsAddressBook(ALICE, null); + expect(m.accountSyncPrefs(ALICE).newContactsAddressBook).toBeUndefined(); + }); +}); + +describe('actions', () => { + it('records what the user turned on in the app, apart from removals and suspensions', async () => { + const m = await freshStore(); + const store = m.useDeviceSyncStore.getState(); + store.setEnabled(ALICE, CONTACTS_AUTHORITY, true); + expect(syncOnInApp(m.accountDeviceSync(ALICE), CONTACTS_AUTHORITY)).toBe(true); + expect(syncOnInApp(m.accountDeviceSync(ALICE), CALENDAR_AUTHORITY)).toBe(false); + store.setRemovedInAndroidSettings(ALICE, true); + expect(syncOnInApp(m.accountDeviceSync(ALICE), CONTACTS_AUTHORITY)).toBe(false); + store.setRemovedInAndroidSettings(ALICE, false); + expect(m.accountDeviceSync(ALICE).removedInAndroidSettings).toBeUndefined(); + store.update(ALICE, { suspended: true }); + expect(m.anySyncOnInApp(m.accountDeviceSync(ALICE))).toBe(false); + }); + + it('keeps an interval per authority and defaults to an hour', async () => { + const m = await freshStore(); + const store = m.useDeviceSyncStore.getState(); + expect(m.intervalFor(m.accountDeviceSync(ALICE), CONTACTS_AUTHORITY)).toBe(m.DEFAULT_INTERVAL_SECONDS); + store.setIntervalSeconds(ALICE, CONTACTS_AUTHORITY, 900); + store.setIntervalSeconds(ALICE, CALENDAR_AUTHORITY, 0); + expect(m.intervalFor(m.accountDeviceSync(ALICE), CONTACTS_AUTHORITY)).toBe(900); + expect(m.intervalFor(m.accountDeviceSync(ALICE), CALENDAR_AUTHORITY)).toBe(0); + store.setIntervalSeconds(ALICE, CALENDAR_AUTHORITY, -5); + expect(m.intervalFor(m.accountDeviceSync(ALICE), CALENDAR_AUTHORITY)).toBe(0); + }); + + it('records run statuses and the states the engine synced, and knows its own echo', async () => { + const m = await freshStore(); + const store = m.useDeviceSyncStore.getState(); + store.recordRunStatus(ALICE, CONTACTS_AUTHORITY, run('ok')); + store.recordRunStatus(ALICE, CALENDAR_AUTHORITY, run('io')); + store.recordKnownState(ALICE, 'c', 'ContactCard', 's1'); + store.recordKnownState(ALICE, 'c', 'CalendarEvent', 'e1'); + const entry = m.accountDeviceSync(ALICE); + expect(entry.lastRun[CONTACTS_AUTHORITY]?.outcome).toBe('ok'); + expect(entry.lastRun[CALENDAR_AUTHORITY]?.outcome).toBe('io'); + expect(m.isKnownState(ALICE, 'c', 'ContactCard', 's1')).toBe(true); + expect(m.isKnownState(ALICE, 'c', 'ContactCard', 's2')).toBe(false); + expect(m.isKnownState(ALICE, 'd', 'ContactCard', 's1')).toBe(false); + expect(m.isKnownState(BOB, 'c', 'ContactCard', 's1')).toBe(false); + }); + + it('forgets an account entirely', async () => { + const m = await freshStore(); + m.useDeviceSyncStore.getState().setEnabled(ALICE, CONTACTS_AUTHORITY, true); + m.useDeviceSyncStore.getState().setEnabled(BOB, CONTACTS_AUTHORITY, true); + m.useDeviceSyncStore.getState().forget(ALICE); + expect(Object.keys(m.useDeviceSyncStore.getState().accounts)).toEqual([BOB]); + }); +}); + +describe('persistence', () => { + it('writes every change at once under device-sync:v1', async () => { + const m = await freshStore(); + m.useDeviceSyncStore.getState().setEnabled(ALICE, CALENDAR_AUTHORITY, true); + // No write delay: the change is in AsyncStorage before any timer runs. + await Promise.resolve(); + expect((await stored())[ALICE].enabled).toEqual({ [CALENDAR_AUTHORITY]: true }); + }); + + it('reads the stored preferences back on the next start', async () => { + let m = await freshStore(); + m.useDeviceSyncStore.getState().setCollectionSelected(ALICE, CONTACTS_AUTHORITY, 'd/b9', true); + m.useDeviceSyncStore.getState().update(ALICE, { androidAccountName: 'alice' }); + await Promise.resolve(); + m = await freshStore(); + const entry = m.accountDeviceSync(ALICE); + expect(entry.contactsSelection).toEqual({ 'd/b9': true }); + expect(entry.androidAccountName).toBe('alice'); + }); + + it('waits for the stored state before a headless writer may set anything', async () => { + await AsyncStorage.setItem(DEVICE_SYNC_STORAGE_KEY, JSON.stringify({ + state: { accounts: { [ALICE]: { contactsSelection: { 'c/b1': false }, calendarSelection: {}, reminderOwner: 'bulwark', intervalSeconds: {}, lastRun: {} } } }, + version: 0, + })); + vi.resetModules(); + const m = await import('../device-sync-store'); + await m.waitForDeviceSyncHydration(); + expect(m.useDeviceSyncStore.persist.hasHydrated()).toBe(true); + // A run status recorded after hydration keeps the stored preferences. + m.useDeviceSyncStore.getState().recordRunStatus(ALICE, CONTACTS_AUTHORITY, run('ok')); + await Promise.resolve(); + const entry = (await stored())[ALICE]; + expect(entry.contactsSelection).toEqual({ 'c/b1': false }); + expect(entry.reminderOwner).toBe('bulwark'); + expect(entry.lastRun[CONTACTS_AUTHORITY].outcome).toBe('ok'); + }); + + it('drops malformed entries and fields instead of failing to start', async () => { + await AsyncStorage.setItem(DEVICE_SYNC_STORAGE_KEY, JSON.stringify({ + state: { + accounts: { + [ALICE]: { + contactsSelection: 'nope', + reminderOwner: 'someone', + lastRun: [], + enabled: { [CONTACTS_AUTHORITY]: true }, + knownStates: { c: { ContactCard: 's1' }, d: 'broken' }, + primaryJmapAccounts: 7, + androidAccountName: 12, + }, + [BOB]: 42, + }, + }, + version: 0, + })); + const m = await freshStore(); + expect(Object.keys(m.useDeviceSyncStore.getState().accounts)).toEqual([ALICE]); + const entry = m.accountDeviceSync(ALICE); + expect(entry.contactsSelection).toEqual({}); + expect(entry.calendarSelection).toEqual({}); + expect(entry.reminderOwner).toBeNull(); + expect(entry.lastRun).toEqual({}); + expect(entry.enabled).toEqual({ [CONTACTS_AUTHORITY]: true }); + expect(entry.knownStates).toEqual({ c: { ContactCard: 's1' } }); + expect(entry.primaryJmapAccounts).toBeUndefined(); + expect(entry.androidAccountName).toBeUndefined(); + // Readers built on the entry keep working. + expect(m.computePushRoutes(m.useDeviceSyncStore.getState().accounts)).toEqual({}); + }); +}); + +describe('routes and push types', () => { + it('knows which JMAP accounts feed an authority', async () => { + const m = await freshStore(); + const store = m.useDeviceSyncStore.getState(); + store.rememberPrimaryJmapAccount(ALICE, CONTACTS_AUTHORITY, 'c'); + store.recordKnownState(ALICE, 'g', 'AddressBook', 'x'); + store.recordKnownState(ALICE, 'h', 'CalendarEvent', 'y'); + store.setCollectionSelected(ALICE, CONTACTS_AUTHORITY, 'd/b9', true); + store.setCollectionSelected(ALICE, CONTACTS_AUTHORITY, 'e/b1', false); + const entry = m.accountDeviceSync(ALICE); + expect(m.feedingJmapAccounts(entry, CONTACTS_AUTHORITY)).toEqual(['c', 'd', 'g']); + expect(m.feedingJmapAccounts(entry, CALENDAR_AUTHORITY)).toEqual(['h']); + }); + + it('routes every feeding JMAP account to the Android accounts that sync it', async () => { + const m = await freshStore(); + const store = m.useDeviceSyncStore.getState(); + store.update(ALICE, { + androidAccountName: 'alice', + enabled: { [CONTACTS_AUTHORITY]: true, [CALENDAR_AUTHORITY]: true }, + primaryJmapAccounts: { [CONTACTS_AUTHORITY]: 'c', [CALENDAR_AUTHORITY]: 'c' }, + }); + store.setCollectionSelected(ALICE, CALENDAR_AUTHORITY, 'team/cal', true); + // Bob syncs nothing yet, a removed account and a suspended one route nowhere. + store.update(BOB, { androidAccountName: 'bob', enabled: {} }); + store.update('carol@x', { androidAccountName: 'carol', enabled: { [CONTACTS_AUTHORITY]: true }, removedInAndroidSettings: true, primaryJmapAccounts: { [CONTACTS_AUTHORITY]: 'k' } }); + store.update('dave@x', { androidAccountName: 'dave', enabled: { [CONTACTS_AUTHORITY]: true }, suspended: true, primaryJmapAccounts: { [CONTACTS_AUTHORITY]: 'k' } }); + expect(m.computePushRoutes(m.useDeviceSyncStore.getState().accounts)).toEqual({ + c: [{ accountName: 'alice', authorities: [CONTACTS_AUTHORITY, CALENDAR_AUTHORITY] }], + team: [{ accountName: 'alice', authorities: [CALENDAR_AUTHORITY] }], + }); + }); + + it('adds the contact and calendar push types while they sync', () => { + expect(deviceSyncPushTypes(undefined)).toEqual([]); + expect(deviceSyncPushTypes({ enabled: { [CONTACTS_AUTHORITY]: true } })).toEqual(['ContactCard', 'AddressBook']); + expect(deviceSyncPushTypes({ enabled: { [CONTACTS_AUTHORITY]: true, [CALENDAR_AUTHORITY]: true } })) + .toEqual(['ContactCard', 'AddressBook', 'CalendarEvent', 'Calendar']); + expect(deviceSyncPushTypes({ enabled: { [CALENDAR_AUTHORITY]: true }, removedInAndroidSettings: true })).toEqual([]); + }); + + it('reads the persisted accounts without the store, tolerating garbage', () => { + expect(persistedAccounts(null)).toEqual({}); + expect(persistedAccounts('{not json')).toEqual({}); + expect(persistedAccounts(JSON.stringify({ state: { accounts: [] } }))).toEqual({}); + expect(persistedAccounts(JSON.stringify({ state: { accounts: { [ALICE]: { enabled: {} } } } }))) + .toEqual({ [ALICE]: { enabled: {} } }); + }); +}); diff --git a/src/stores/calendar-store.ts b/src/stores/calendar-store.ts index bfac097a..3bdcbbe4 100644 --- a/src/stores/calendar-store.ts +++ b/src/stores/calendar-store.ts @@ -47,6 +47,8 @@ import { } from '../lib/recurrence-instances'; import { findTasksOnlyCalendarIds, isTaskLikeObject } from '../lib/calendar-component-detection'; import { mergeRangeEvents, mergedSpanLimit, planRangeLoad, sameRange } from '../lib/calendar-range-cache'; +import { requestDeviceSync } from '../device-sync/app/request'; +import { CALENDAR_AUTHORITY } from '../device-sync/types'; // extendRange merges up to twice the requested range (at least half a year, // at most about three years); past that it starts over with the request. @@ -777,6 +779,7 @@ export const useCalendarStore = create()( ? expandRecurringEvents([mapped], loadedRange.after, loadedRange.before) : [mapped]; set({ events: [...get().events, ...inserted] }); + requestDeviceSync(CALENDAR_AUTHORITY); return full; }, @@ -818,6 +821,7 @@ export const useCalendarStore = create()( set({ events: get().events.map((e) => (e.id === id ? { ...e, ...(changes as Partial) } : e)), }); + requestDeviceSync(CALENDAR_AUTHORITY); // A series mutation (an occurrence override, a truncated/changed rule, a // master edit) touches every expanded sibling, and the optimistic merge // above only updated the tapped one; server-expanded occurrences also @@ -853,6 +857,7 @@ export const useCalendarStore = create()( && !(seriesIdOf(e) === realId && (e.accountId ?? undefined) === (accountId ?? undefined))), }); } + requestDeviceSync(CALENDAR_AUTHORITY); if (touchesSeries) { await get().refresh(); } @@ -915,6 +920,7 @@ export const useCalendarStore = create()( : undefined; await apiRsvpEvent(realId, participantId, status, repair, accountId); } + requestDeviceSync(CALENDAR_AUTHORITY); set({ events: get().events.map((e) => { if (e.id !== eventId || !e.participants?.[participantId]) return e; @@ -996,7 +1002,10 @@ export const useCalendarStore = create()( } } finally { // Also when a chunk was refused: show what did get in. - if (count > 0 || linked > 0) await get().refresh(); + if (count > 0 || linked > 0) { + requestDeviceSync(CALENDAR_AUTHORITY); + await get().refresh(); + } } if (count + linked === 0 && refused.length > 0) throw new ImportRefusedError(refused); return { imported: count + linked, refused }; @@ -1005,6 +1014,7 @@ export const useCalendarStore = create()( createCalendar: async (name, color, description) => { const created = await apiCreateCalendar(name, color, description); set({ calendars: [...get().calendars, created] }); + requestDeviceSync(CALENDAR_AUTHORITY); return created; }, @@ -1014,6 +1024,7 @@ export const useCalendarStore = create()( set({ calendars: get().calendars.map((c) => (c.id === id ? { ...c, ...updates } as Calendar : c)), }); + requestDeviceSync(CALENDAR_AUTHORITY); }, removeCalendar: async (id) => { @@ -1025,12 +1036,15 @@ export const useCalendarStore = create()( tasks: get().tasks.filter((t) => !t.calendarIds?.[id]), hiddenCalendarIds: get().hiddenCalendarIds.filter((x) => x !== id), }); + requestDeviceSync(CALENDAR_AUTHORITY); }, clearCalendarEvents: async (id) => { const cal = get().calendars.find((c) => c.id === id); try { - return await apiClearCalendarEvents(cal?.originalId || id, cal?.accountId); + const cleared = await apiClearCalendarEvents(cal?.originalId || id, cal?.accountId); + requestDeviceSync(CALENDAR_AUTHORITY); + return cleared; } finally { // Also after a refused batch: earlier batches may have gone through. await get().refresh(); diff --git a/src/stores/contacts-store.ts b/src/stores/contacts-store.ts index b4a588fb..366f399f 100644 --- a/src/stores/contacts-store.ts +++ b/src/stores/contacts-store.ts @@ -33,6 +33,8 @@ import { matchesContactSearch, } from '../lib/contact-utils'; import { sanitizeDisplayName, splitMailbox } from '../lib/rfc5322-mailbox'; +import { requestDeviceSync } from '../device-sync/app/request'; +import { CONTACTS_AUTHORITY } from '../device-sync/types'; const SELECTED_CATEGORY_STORAGE_KEY = 'webmail:contacts:category:v1'; @@ -403,6 +405,7 @@ export const useContactsStore = create()( const { originalId, accountId, book } = bookTarget(addressBookId); const created = tagCreated(await apiCreateContact(contact, originalId, accountId), book); set({ contacts: [...get().contacts, created] }); + requestDeviceSync(CONTACTS_AUTHORITY); return created; }, @@ -422,12 +425,14 @@ export const useContactsStore = create()( return merged; }), }); + requestDeviceSync(CONTACTS_AUTHORITY); }, deleteContact: async (id) => { const { originalId, accountId } = contactTarget(id); await apiDeleteContacts([originalId], accountId); set({ contacts: cleanGroupMembers(get().contacts, new Set([id])).filter((c) => c.id !== id) }); + requestDeviceSync(CONTACTS_AUTHORITY); }, bulkDelete: async (ids) => { @@ -460,6 +465,7 @@ export const useContactsStore = create()( contacts: cleanGroupMembers(get().contacts, removed).filter((c) => !removed.has(c.id)), ...(failed > 0 ? { error: `Failed to delete ${failed} contact${failed === 1 ? '' : 's'}` } : {}), }); + if (removed.size > 0) requestDeviceSync(CONTACTS_AUTHORITY); }, importContacts: async (incoming, addressBookId) => { @@ -478,6 +484,7 @@ export const useContactsStore = create()( console.warn('[contacts-store] import failed for one contact', err); } } + if (imported > 0) requestDeviceSync(CONTACTS_AUTHORITY); return { imported, failed }; }, @@ -544,6 +551,7 @@ export const useContactsStore = create()( set({ contacts: get().contacts.map((c) => (c.id === id ? created : c)) }); } } + requestDeviceSync(CONTACTS_AUTHORITY); }, addContactsToGroup: async (groupId, contactIds) => { @@ -602,6 +610,7 @@ export const useContactsStore = create()( if (!trimmed) throw new Error('Address book name is required'); const book = await apiCreateAddressBook(trimmed); set({ addressBooks: [...get().addressBooks, book] }); + requestDeviceSync(CONTACTS_AUTHORITY); return book; }, @@ -613,6 +622,7 @@ export const useContactsStore = create()( set({ addressBooks: get().addressBooks.map((b) => (b.id === id ? { ...b, name: trimmed } : b)), }); + requestDeviceSync(CONTACTS_AUTHORITY); }, deleteAddressBook: async (id) => { @@ -624,6 +634,7 @@ export const useContactsStore = create()( addressBooks: get().addressBooks.filter((b) => b.id !== id), contacts: get().contacts.filter((c) => !c.addressBookIds?.[id]), }); + requestDeviceSync(CONTACTS_AUTHORITY); const category = get().selectedCategory; if (category.type === 'addressBook' && category.addressBookId === id) { get().setSelectedCategory({ type: 'all' }); @@ -799,6 +810,7 @@ export const useContactsStore = create()( bookId, ); set({ trustedSenderEmails: [...get().trustedSenderEmails, email] }); + requestDeviceSync(CONTACTS_AUTHORITY); }, removeFromTrustedSendersBook: async (input) => { @@ -810,6 +822,7 @@ export const useContactsStore = create()( c.emails && Object.values(c.emails).some((e) => e.address?.toLowerCase().trim() === email)); if (matches.length > 0) await apiDeleteContacts(matches.map((m) => m.id)); set({ trustedSenderEmails: get().trustedSenderEmails.filter((e) => e !== email) }); + if (matches.length > 0) requestDeviceSync(CONTACTS_AUTHORITY); }, isTrustedAddressBookSender: (email) => diff --git a/src/stores/device-sync-store.ts b/src/stores/device-sync-store.ts index 2350fb4d..bfa98ce4 100644 --- a/src/stores/device-sync-store.ts +++ b/src/stores/device-sync-store.ts @@ -2,9 +2,11 @@ * Device sync preferences and status per app account (Android only, #34; * docs/device-sync.md, "App integration"). Keyed by `AccountEntry.id`. * - * Whether sync is on is not stored here: Android's sync settings + * Whether sync runs is not stored here: Android's sync settings * (`getSyncAutomatically` per authority) are the source of truth, so the - * toggles in Settings → Accounts and the app always agree. + * toggles in Settings → Accounts and the app always agree. `enabled` only + * records what the user turned on in the app, which tells "paused in Android + * settings" (on here, off there) from "off". * * Persisted under `device-sync:v1` without the write delay the caches use: a * headless sync run may be the only thing alive, and its status must survive @@ -15,13 +17,23 @@ import { create } from 'zustand'; import { persist } from 'zustand/middleware'; import { createPersistStorage } from './persist-storage'; -import type { - AccountSyncPrefs, - Authority, - CollectionKey, - ReminderOwner, - RunStatus, +import { + AUTHORITIES, + CONTACTS_AUTHORITY, + type AccountSyncPrefs, + type Authority, + type CollectionKey, + type ReminderOwner, + type RunStatus, } from '../device-sync/types'; +import { + DEVICE_SYNC_STORAGE_KEY, + STATE_TYPES_BY_AUTHORITY, + syncOnInApp, +} from '../device-sync/app/prefs'; +import { collectionDefaultOn } from '../device-sync/engine/selection'; + +export { syncOnInApp }; /** Periodic sync interval when the user has not picked one: push covers freshness. */ export const DEFAULT_INTERVAL_SECONDS = 3600; @@ -47,6 +59,16 @@ export interface AccountDeviceSync { * our own echo and triggers no sync. */ knownStates?: Record>; + /** Authorities the user turned on in the app (Android may still pause them). */ + enabled?: Partial>; + /** + * The app dropped the account from its registry (its sign-in failed) while + * it synced: automatic sync was turned off and the device data kept until + * the user signs in again. + */ + suspended?: boolean; + /** The JMAP account holding the personal collections, per authority, as last seen. */ + primaryJmapAccounts?: Partial>; } export function emptyAccountDeviceSync(): AccountDeviceSync { @@ -65,53 +87,114 @@ interface DeviceSyncState { recordRunStatus: (registryId: string, authority: Authority, status: RunStatus) => void; recordKnownState: (registryId: string, jmapAccountId: string, type: string, state: string) => void; forget: (registryId: string) => void; + + /** Turns an authority on or off in the app (not in Android). */ + setEnabled: (registryId: string, authority: Authority, on: boolean) => void; + /** An explicit choice for one address book or calendar. */ + setCollectionSelected: (registryId: string, authority: Authority, key: CollectionKey, on: boolean) => void; + /** Where contacts created on the device go; null = the server's default book. */ + setNewContactsAddressBook: (registryId: string, key: CollectionKey | null) => void; + setIntervalSeconds: (registryId: string, authority: Authority, seconds: number) => void; + setReminderOwner: (registryId: string, owner: ReminderOwner | null) => void; + setRemovedInAndroidSettings: (registryId: string, removed: boolean) => void; + setAndroidAccountName: (registryId: string, name: string | undefined) => void; + rememberPrimaryJmapAccount: (registryId: string, authority: Authority, jmapAccountId: string) => void; +} + +function selectionField(authority: Authority): 'contactsSelection' | 'calendarSelection' { + return authority === CONTACTS_AUTHORITY ? 'contactsSelection' : 'calendarSelection'; +} + +// Persisted entries are trusted only as far as their shape: a field of the +// wrong type falls back to the empty default instead of crashing a reader. +function sanitizeEntry(value: unknown): AccountDeviceSync | null { + if (!value || typeof value !== 'object' || Array.isArray(value)) return null; + const v = value as Record; + const isRecord = (x: unknown): x is Record => !!x && typeof x === 'object' && !Array.isArray(x); + const record = (x: unknown) => (isRecord(x) ? x : {}); + const optionalRecord = (x: unknown) => (isRecord(x) ? x : undefined); + const knownStates = isRecord(v.knownStates) + ? Object.fromEntries(Object.entries(v.knownStates).filter(([, states]) => isRecord(states))) + : undefined; + return { + ...(v as Partial), + contactsSelection: record(v.contactsSelection) as Record, + calendarSelection: record(v.calendarSelection) as Record, + reminderOwner: v.reminderOwner === 'device' || v.reminderOwner === 'bulwark' ? v.reminderOwner : null, + intervalSeconds: record(v.intervalSeconds) as Partial>, + lastRun: record(v.lastRun) as Partial>, + androidAccountName: typeof v.androidAccountName === 'string' && v.androidAccountName ? v.androidAccountName : undefined, + enabled: optionalRecord(v.enabled) as AccountDeviceSync['enabled'], + knownStates: knownStates as AccountDeviceSync['knownStates'], + primaryJmapAccounts: optionalRecord(v.primaryJmapAccounts) as AccountDeviceSync['primaryJmapAccounts'], + }; } export const useDeviceSyncStore = create()( persist( - (set) => ({ - accounts: {}, - update: (registryId, patch) => - set((s) => ({ - accounts: { - ...s.accounts, - [registryId]: { ...(s.accounts[registryId] ?? emptyAccountDeviceSync()), ...patch }, - }, - })), - recordRunStatus: (registryId, authority, status) => + (set) => { + // Applies `change` to one account's entry, creating it when missing. + const edit = (registryId: string, change: (current: AccountDeviceSync) => Partial) => set((s) => { const current = s.accounts[registryId] ?? emptyAccountDeviceSync(); - return { - accounts: { - ...s.accounts, - [registryId]: { ...current, lastRun: { ...current.lastRun, [authority]: status } }, - }, - }; - }), - recordKnownState: (registryId, jmapAccountId, type, state) => - set((s) => { - const current = s.accounts[registryId] ?? emptyAccountDeviceSync(); - const known = current.knownStates ?? {}; - return { - accounts: { - ...s.accounts, - [registryId]: { - ...current, - knownStates: { ...known, [jmapAccountId]: { ...(known[jmapAccountId] ?? {}), [type]: state } }, - }, - }, - }; - }), - forget: (registryId) => - set((s) => { - const { [registryId]: _gone, ...rest } = s.accounts; - return { accounts: rest }; - }), - }), + return { accounts: { ...s.accounts, [registryId]: { ...current, ...change(current) } } }; + }); + + return { + accounts: {}, + update: (registryId, patch) => edit(registryId, () => patch), + recordRunStatus: (registryId, authority, status) => + edit(registryId, (current) => ({ lastRun: { ...current.lastRun, [authority]: status } })), + recordKnownState: (registryId, jmapAccountId, type, state) => + edit(registryId, (current) => { + const known = current.knownStates ?? {}; + return { + knownStates: { ...known, [jmapAccountId]: { ...(known[jmapAccountId] ?? {}), [type]: state } }, + }; + }), + forget: (registryId) => + set((s) => { + const { [registryId]: _gone, ...rest } = s.accounts; + return { accounts: rest }; + }), + + setEnabled: (registryId, authority, on) => + edit(registryId, (current) => ({ enabled: { ...current.enabled, [authority]: on } })), + setCollectionSelected: (registryId, authority, key, on) => + edit(registryId, (current) => { + const field = selectionField(authority); + return { [field]: { ...current[field], [key]: on } }; + }), + setNewContactsAddressBook: (registryId, key) => + edit(registryId, () => ({ newContactsAddressBook: key ?? undefined })), + setIntervalSeconds: (registryId, authority, seconds) => + edit(registryId, (current) => ({ + intervalSeconds: { ...current.intervalSeconds, [authority]: Math.max(0, Math.round(seconds)) }, + })), + setReminderOwner: (registryId, owner) => edit(registryId, () => ({ reminderOwner: owner })), + setRemovedInAndroidSettings: (registryId, removed) => + edit(registryId, () => ({ removedInAndroidSettings: removed || undefined })), + setAndroidAccountName: (registryId, name) => edit(registryId, () => ({ androidAccountName: name })), + rememberPrimaryJmapAccount: (registryId, authority, jmapAccountId) => + edit(registryId, (current) => ({ + primaryJmapAccounts: { ...current.primaryJmapAccounts, [authority]: jmapAccountId }, + })), + }; + }, { - name: 'device-sync:v1', + name: DEVICE_SYNC_STORAGE_KEY, storage: createPersistStorage({ writeDelayMs: 0 }), partialize: (state) => ({ accounts: state.accounts }), + merge: (persisted, current) => { + const stored = (persisted as { accounts?: unknown } | undefined)?.accounts; + if (!stored || typeof stored !== 'object' || Array.isArray(stored)) return current; + const accounts: Record = {}; + for (const [id, value] of Object.entries(stored as Record)) { + const entry = sanitizeEntry(value); + if (entry) accounts[id] = entry; + } + return { ...current, accounts }; + }, }, ), ); @@ -127,9 +210,14 @@ export function waitForDeviceSyncHydration(): Promise { }); } +/** One account's entry, or the empty default. */ +export function accountDeviceSync(registryId: string): AccountDeviceSync { + return useDeviceSyncStore.getState().accounts[registryId] ?? emptyAccountDeviceSync(); +} + /** The engine's view of one account's preferences. */ export function accountSyncPrefs(registryId: string): AccountSyncPrefs { - const a = useDeviceSyncStore.getState().accounts[registryId] ?? emptyAccountDeviceSync(); + const a = accountDeviceSync(registryId); return { contactsSelection: a.contactsSelection, calendarSelection: a.calendarSelection, @@ -137,3 +225,111 @@ export function accountSyncPrefs(registryId: string): AccountSyncPrefs { reminderOwner: a.reminderOwner ?? 'device', }; } + +// ─── Selection ───────────────────────────────────────── + +/** The explicit choices for an authority's collections. */ +export function selectionFor(entry: AccountDeviceSync, authority: Authority): Record { + return entry[selectionField(authority)] ?? {}; +} + +/** What the default rule needs to know about a collection. */ +export interface SelectableCollection { + key: CollectionKey; + isPersonal: boolean; + name?: string | null; +} + +/** + * Whether a collection syncs: the explicit choice when there is one, else the + * engine's default (`collectionDefaultOn`): the personal account's + * collections on, shared ones and the Trusted Senders address book off. + */ +export function isCollectionSelected( + selection: Record, + authority: Authority, + collection: SelectableCollection, +): boolean { + const explicit = selection[collection.key]; + return typeof explicit === 'boolean' ? explicit : collectionDefaultOn(collection.isPersonal, authority, collection.name); +} + +/** The keys of the collections that sync, out of those the server lists. */ +export function effectiveSelection( + entry: AccountDeviceSync, + authority: Authority, + collections: ReadonlyArray, +): CollectionKey[] { + const selection = selectionFor(entry, authority); + return collections.filter((c) => isCollectionSelected(selection, authority, c)).map((c) => c.key); +} + +/** The periodic sync interval of an authority in seconds (0 = manual only). */ +export function intervalFor(entry: AccountDeviceSync, authority: Authority): number { + const seconds = entry.intervalSeconds?.[authority]; + return typeof seconds === 'number' && Number.isFinite(seconds) && seconds >= 0 ? seconds : DEFAULT_INTERVAL_SECONDS; +} + +// ─── Echoes and routes ───────────────────────────────── + +/** True when the engine already synced exactly this state (a StateChange that is our own echo). */ +export function isKnownState(registryId: string, jmapAccountId: string, type: string, state: string): boolean { + return useDeviceSyncStore.getState().accounts[registryId]?.knownStates?.[jmapAccountId]?.[type] === state; +} + +/** + * The JMAP accounts whose data feeds this account's Android account for an + * authority, as far as the app knows: the personal one, every account the + * engine synced a state of, and every account a collection was explicitly + * picked in. + */ +export function feedingJmapAccounts(entry: AccountDeviceSync, authority: Authority): string[] { + const out = new Set(); + const primary = entry.primaryJmapAccounts?.[authority]; + if (primary) out.add(primary); + const types = STATE_TYPES_BY_AUTHORITY[authority]; + for (const [jmapAccountId, states] of Object.entries(entry.knownStates ?? {})) { + if (states && types.some((type) => type in states)) out.add(jmapAccountId); + } + for (const [key, on] of Object.entries(selectionFor(entry, authority))) { + const slash = key.indexOf('/'); + if (on && slash > 0) out.add(key.slice(0, slash)); + } + return [...out].sort(); +} + +export interface PushRoute { + accountName: string; + authorities: Authority[]; +} + +/** + * Routes for the native push router: every JMAP account id that feeds an + * Android account, with the account and the authorities to sync. + */ +export function computePushRoutes(accounts: Record): Record { + const routes: Record = {}; + for (const registryId of Object.keys(accounts).sort()) { + const entry = accounts[registryId]; + const accountName = entry.androidAccountName; + if (!accountName) continue; + for (const authority of AUTHORITIES) { + if (!syncOnInApp(entry, authority)) continue; + for (const jmapAccountId of feedingJmapAccounts(entry, authority)) { + const list = (routes[jmapAccountId] ??= []); + let route = list.find((r) => r.accountName === accountName); + if (!route) { + route = { accountName, authorities: [] }; + list.push(route); + } + if (!route.authorities.includes(authority)) route.authorities.push(authority); + } + } + } + return routes; +} + +/** Whether any authority of the account is on in the app. */ +export function anySyncOnInApp(entry: AccountDeviceSync | undefined): boolean { + return AUTHORITIES.some((authority) => syncOnInApp(entry, authority)); +} From eb8b33160c82b54b9995024e932e956e4cd525b6 Mon Sep 17 00:00:00 2001 From: Linus Rath <139418639+rathlinus@users.noreply.github.com> Date: Sun, 27 Sep 2026 04:38:34 +0200 Subject: [PATCH 11/87] feat: remove synced contacts and calendars from the device on sign-out #34 Signing out of an account (or of all of them, or removing one) now first uploads what the device changed in its synced contacts and calendars, then deletes them and removes its Android account, while the credentials still work. When changes could not be uploaded (offline, server errors) the app asks before signing out and losing them; staying signed in keeps the account syncing. Several accounts are released in parallel, and an internal error never keeps anyone from signing out: the next check against Android's accounts stops what was left from syncing. A successful sign-in clears the account's "Sign in again" notification and resumes an account that stopped syncing when it was signed out. Checked: `npm run typecheck` and `npm test` on this commit (the release and resume paths are covered by the lifecycle tests of e67e240). --- src/stores/auth-store.ts | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/src/stores/auth-store.ts b/src/stores/auth-store.ts index 67bf018a..b4e28c56 100644 --- a/src/stores/auth-store.ts +++ b/src/stores/auth-store.ts @@ -19,6 +19,7 @@ import { teardownPushNotifications, teardownPushNotificationsForAccount, } from '../lib/push-notifications'; +import { deviceSyncSignedIn, releaseDeviceSyncBeforeSignOut } from '../device-sync/app/lifecycle'; // Persist middleware hydrates asynchronously on cold start. Without this // guard, restoreSession() can read the account-store before AsyncStorage has @@ -245,6 +246,8 @@ async function completeOAuthHandoff( // mounted; the screen joins this load. void useEmailStore.getState().fetchMailboxes(); void syncAccountDisplayName(accountId); + // Device sync (#34): drop a "sign in again" notice, resume a suspended account. + void deviceSyncSignedIn(accountId); } function applyConnectedState( @@ -323,6 +326,8 @@ export const useAuthStore = create((set, get) => ({ // mounted; the screen joins this load. void useEmailStore.getState().fetchMailboxes(); void syncAccountDisplayName(accountId); + // Device sync (#34): drop a "sign in again" notice, resume a suspended account. + void deviceSyncSignedIn(accountId); } catch (err) { if (err instanceof Error && err.name === 'TotpRequiredError') { // Keep what the user (or the webmail hand-off) supplied so the code @@ -463,6 +468,11 @@ export const useAuthStore = create((set, get) => ({ const accountStore = useAccountStore.getState(); const currentId = get().activeAccountId; + // Device sync (#34): upload what this device changed and remove the + // Android account while the credentials are still here. When changes + // could not be uploaded the user is asked, and may stay signed in. + if (currentId && !(await releaseDeviceSyncBeforeSignOut([currentId]))) return; + // Best-effort: revoke this account's JMAP PushSubscription and drop its // relay mapping before we lose credentials. Other logged-in accounts' // push setups remain untouched. Do not abort logout on failure. @@ -513,6 +523,8 @@ export const useAuthStore = create((set, get) => ({ logoutAll: async () => { const accountStore = useAccountStore.getState(); const ids = accountStore.accounts.map((a) => a.id); + // Device sync (#34): as in logout, for every account. + if (!(await releaseDeviceSyncBeforeSignOut(ids))) return; await teardownPushNotifications().catch(() => undefined); for (const id of ids) await revokeStoredRefreshToken(id); await jmapClient.clearAllCredentials(ids); @@ -629,6 +641,8 @@ export const useAuthStore = create((set, get) => ({ } const accountStore = useAccountStore.getState(); if (!accountStore.getAccountById(accountId)) return; + // Device sync (#34): as in logout. + if (!(await releaseDeviceSyncBeforeSignOut([accountId]))) return; await teardownPushNotificationsForAccount(accountId).catch(() => undefined); await revokeStoredRefreshToken(accountId); await jmapClient.clearAccountCredentials(accountId).catch(() => undefined); From 1c574a5a934bbb31a33a9419f4fd432d579a9f31 Mon Sep 17 00:00:00 2001 From: Linus Rath <139418639+rathlinus@users.noreply.github.com> Date: Sun, 27 Sep 2026 04:39:53 +0200 Subject: [PATCH 12/87] feat: leave reminders of synced events to the device's calendar app when asked #34 When an account's calendars sync to the device and the user chose the device's calendar app to remind them, that app gets the events' reminders. Bulwark's own reminder scheduler now skips the events of those calendars, which would otherwise remind the user twice. Tasks never sync and keep Bulwark's reminders, and a change of who reminds, or of which calendars sync, reschedules right away. Checked: the reminder filter and scheduler tests, `npm run typecheck` and `npm test` on this commit. --- .../__tests__/app/reminders.test.ts | 104 ++++++++++++++++++ src/device-sync/app/reminders.ts | 79 +++++++++++++ .../__tests__/calendar-notifications.test.ts | 55 +++++++++ src/lib/calendar-notifications.ts | 20 +++- 4 files changed, 257 insertions(+), 1 deletion(-) create mode 100644 src/device-sync/__tests__/app/reminders.test.ts create mode 100644 src/device-sync/app/reminders.ts diff --git a/src/device-sync/__tests__/app/reminders.test.ts b/src/device-sync/__tests__/app/reminders.test.ts new file mode 100644 index 00000000..744081c4 --- /dev/null +++ b/src/device-sync/__tests__/app/reminders.test.ts @@ -0,0 +1,104 @@ +import { describe, it, expect, vi, beforeEach } from 'vitest'; + +const h = vi.hoisted(() => ({ available: true })); + +vi.mock('../../native', () => ({ + isDeviceSyncAvailable: () => h.available, +})); + +import { CALENDAR_AUTHORITY, CONTACTS_AUTHORITY } from '../../types'; +import type { CalendarEvent } from '../../../api/types'; +import { useDeviceSyncStore, waitForDeviceSyncHydration } from '../../../stores/device-sync-store'; +import { onDeviceRemindersChange, remindedOnDevice } from '../../app/reminders'; + +const ALICE = 'alice@mail.example.org'; + +function event(id: string, calendarIds: Record): CalendarEvent { + return { id, calendarIds, title: id, start: '2026-09-24T12:00:00', duration: 'PT1H' } as CalendarEvent; +} + +function calendarSync(extra: Record = {}): void { + useDeviceSyncStore.setState({ + accounts: { + [ALICE]: { + contactsSelection: {}, + calendarSelection: { 'c/work': false, 'team/shared-on': true }, + reminderOwner: 'device', + intervalSeconds: {}, + lastRun: {}, + androidAccountName: 'alice', + enabled: { [CALENDAR_AUTHORITY]: true }, + ...extra, + }, + }, + }); +} + +beforeEach(async () => { + await waitForDeviceSyncHydration(); + h.available = true; + calendarSync(); +}); + +describe('remindedOnDevice', () => { + it("picks the events of the account's synced calendars", () => { + const onDevice = remindedOnDevice(ALICE, 'c')!; + // Own calendars sync unless turned off. + expect(onDevice(event('a', { personal: true }))).toBe(true); + expect(onDevice(event('b', { work: true }))).toBe(false); + expect(onDevice(event('c', { work: true, personal: true }))).toBe(true); + // Shared calendars: namespaced ids in calendarIds, raw ones in originalCalendarIds. + expect(onDevice({ + ...event('d', { 'team:shared-on': true }), + accountId: 'team', + originalCalendarIds: { 'shared-on': true }, + })).toBe(true); + expect(onDevice({ + ...event('e', { 'team:other': true }), + accountId: 'team', + originalCalendarIds: { other: true }, + })).toBe(false); + }); + + it('leaves everything to Bulwark when Bulwark reminds, or calendars do not sync', () => { + calendarSync({ reminderOwner: 'bulwark' }); + expect(remindedOnDevice(ALICE, 'c')).toBeNull(); + calendarSync({ enabled: { [CONTACTS_AUTHORITY]: true } }); + expect(remindedOnDevice(ALICE, 'c')).toBeNull(); + calendarSync({ removedInAndroidSettings: true }); + expect(remindedOnDevice(ALICE, 'c')).toBeNull(); + expect(remindedOnDevice('someone@else', 'c')).toBeNull(); + expect(remindedOnDevice(null, 'c')).toBeNull(); + }); + + it('counts the calendar app as the owner until the user chose', () => { + calendarSync({ reminderOwner: null }); + expect(remindedOnDevice(ALICE, 'c')?.(event('a', { personal: true }))).toBe(true); + }); + + it('does nothing where device sync is not available', () => { + h.available = false; + expect(remindedOnDevice(ALICE, 'c')).toBeNull(); + }); +}); + +describe('onDeviceRemindersChange', () => { + it('fires for the owner and the calendar selection, not for sync bookkeeping', () => { + const listener = vi.fn(); + const off = onDeviceRemindersChange(listener); + const store = useDeviceSyncStore.getState(); + + store.recordKnownState(ALICE, 'c', 'CalendarEvent', 's1'); + store.setIntervalSeconds(ALICE, CALENDAR_AUTHORITY, 900); + expect(listener).not.toHaveBeenCalled(); + + store.setReminderOwner(ALICE, 'bulwark'); + expect(listener).toHaveBeenCalledTimes(1); + store.setReminderOwner(ALICE, 'device'); + store.setCollectionSelected(ALICE, CALENDAR_AUTHORITY, 'c/work', true); + expect(listener).toHaveBeenCalledTimes(3); + off(); + store.setEnabled(ALICE, CALENDAR_AUTHORITY, false); + expect(listener).toHaveBeenCalledTimes(3); + }); +}); diff --git a/src/device-sync/app/reminders.ts b/src/device-sync/app/reminders.ts new file mode 100644 index 00000000..023bac16 --- /dev/null +++ b/src/device-sync/app/reminders.ts @@ -0,0 +1,79 @@ +/** + * Who reminds the user of synced events (docs/device-sync.md, "Reminder + * owner"). When an account's calendars sync to the device and the calendar + * app owns the reminders, the device fires them from its Reminders rows, so + * Bulwark's own scheduler skips the events of those calendars. Tasks are not + * synced and keep Bulwark's reminders. + */ +import type { CalendarEvent } from '../../api/types'; +import { + isCollectionSelected, + selectionFor, + syncOnInApp, + useDeviceSyncStore, + type AccountDeviceSync, +} from '../../stores/device-sync-store'; +import { collectionKey } from '../common/ids'; +import { CALENDAR_AUTHORITY } from '../types'; +import { deviceSyncAvailable } from './available'; + +/** Whether the device's calendar app fires the reminders of an account's synced events. */ +function deviceOwnsReminders(entry: AccountDeviceSync | undefined): entry is AccountDeviceSync { + // Unasked means the calendar app, as the engine writes Reminders rows then. + return !!entry && syncOnInApp(entry, CALENDAR_AUTHORITY) && (entry.reminderOwner ?? 'device') === 'device'; +} + +/** + * A test for the events whose reminders the device's calendar app fires for + * this app account, so Bulwark schedules none; null when there are none. + * `primaryJmapAccountId`: the JMAP account the app's calendar store loads + * events from when they carry no `accountId` (the user's own). + */ +export function remindedOnDevice( + registryId: string | null | undefined, + primaryJmapAccountId: string | null | undefined, +): ((event: CalendarEvent) => boolean) | null { + if (!registryId || !deviceSyncAvailable()) return null; + const entry = useDeviceSyncStore.getState().accounts[registryId]; + if (!deviceOwnsReminders(entry)) return null; + const selection = selectionFor(entry, CALENDAR_AUTHORITY); + const personal = entry.primaryJmapAccounts?.[CALENDAR_AUTHORITY] ?? primaryJmapAccountId ?? null; + return (event) => { + // Events of shared calendars carry their account and a namespaced copy + // of calendarIds; the raw ids are in originalCalendarIds. + const jmapAccountId = event.accountId ?? primaryJmapAccountId; + if (!jmapAccountId) return false; + const calendarIds = event.originalCalendarIds ?? event.calendarIds ?? {}; + return Object.entries(calendarIds).some(([calendarId, member]) => + !!member && isCollectionSelected(selection, CALENDAR_AUTHORITY, { + key: collectionKey(jmapAccountId, calendarId), + isPersonal: jmapAccountId === personal, + })); + }; +} + +// What decides which events the device reminds of, as one comparable string. +function reminderSignature(accounts: Record): string { + const parts: unknown[] = []; + for (const registryId of Object.keys(accounts).sort()) { + const entry = accounts[registryId]; + if (!deviceOwnsReminders(entry)) continue; + parts.push([registryId, entry.calendarSelection, entry.primaryJmapAccounts?.[CALENDAR_AUTHORITY] ?? null]); + } + return JSON.stringify(parts); +} + +/** + * Calls `listener` whenever the set of events the device reminds of may have + * changed (reminder owner, calendar sync on or off, the calendar selection). + * Returns the unsubscribe. + */ +export function onDeviceRemindersChange(listener: () => void): () => void { + let last = reminderSignature(useDeviceSyncStore.getState().accounts); + return useDeviceSyncStore.subscribe((state) => { + const next = reminderSignature(state.accounts); + if (next === last) return; + last = next; + listener(); + }); +} diff --git a/src/lib/__tests__/calendar-notifications.test.ts b/src/lib/__tests__/calendar-notifications.test.ts index e678dbbc..4b995773 100644 --- a/src/lib/__tests__/calendar-notifications.test.ts +++ b/src/lib/__tests__/calendar-notifications.test.ts @@ -21,6 +21,9 @@ const h = vi.hoisted(() => ({ client: { isConnected: true, accountId: 'acc-1' }, loadEventsInRange: vi.fn(), calendarListener: null as Listener | null, + // Device sync (#34): which events the device's calendar app reminds of. + onDevice: null as ((event: { id: string }) => boolean) | null, + reminderListener: null as (() => void) | null, })); vi.mock('react-native', () => ({ @@ -63,6 +66,13 @@ vi.mock('../../stores/calendar-store', () => ({ loadEventsInRange: h.loadEventsInRange, })); vi.mock('../capabilities', () => ({ hasCalendarCapability: () => true })); +vi.mock('../../device-sync/app/reminders', () => ({ + remindedOnDevice: vi.fn(() => h.onDevice), + onDeviceRemindersChange: vi.fn((listener: () => void) => { + h.reminderListener = listener; + return () => undefined; + }), +})); const NOW = new Date('2026-09-24T08:00:00Z').getTime(); const TAG = 'bulwark-calendar-alert'; @@ -103,6 +113,8 @@ beforeEach(async () => { h.calendar.loadedRange = null; h.client.isConnected = true; h.calendarListener = null; + h.onDevice = null; + h.reminderListener = null; h.loadEventsInRange.mockResolvedValue([upcoming]); mod = await import('../calendar-notifications'); N = await import('expo-notifications'); @@ -249,6 +261,49 @@ describe('calendar reminders (B17)', () => { }); }); +describe('reminders of events synced to the device (#34)', () => { + const task = { + id: 'task-1', + '@type': 'Task', + title: 'Pay rent', + calendarIds: { 'cal-1': true }, + alerts: { a1: { trigger: { '@type': 'AbsoluteTrigger', when: '2026-09-24T09:30:00Z' }, action: 'display' } }, + }; + + it("leaves synced events to the device's calendar app; tasks keep their reminders", async () => { + const reminders = await import('../../device-sync/app/reminders'); + h.calendar.tasks = [task]; + h.onDevice = (event) => event.id === 'ev1'; + + await mod.rescheduleCalendarNotifications(); + + expect(reminders.remindedOnDevice).toHaveBeenCalledWith('app-1', 'acc-1'); + expect(N.scheduleNotificationAsync).toHaveBeenCalledTimes(1); + const request = vi.mocked(N.scheduleNotificationAsync).mock.calls[0][0]; + expect(request.content.data).toMatchObject({ kind: 'task', eventId: 'task-1' }); + }); + + it('cancels the reminders the calendar app took over', async () => { + h.pending = [reminder('ours', upcomingKey)]; + h.onDevice = () => true; + + await mod.rescheduleCalendarNotifications(); + + expect(N.cancelScheduledNotificationAsync).toHaveBeenCalledWith('ours'); + expect(N.scheduleNotificationAsync).not.toHaveBeenCalled(); + }); + + it('reschedules when who reminds, or what syncs, changes', async () => { + mod.startCalendarNotificationSync(); + await vi.advanceTimersByTimeAsync(1500); + expect(h.loadEventsInRange).toHaveBeenCalledTimes(1); + + h.reminderListener!(); + await vi.advanceTimersByTimeAsync(1500); + expect(h.loadEventsInRange).toHaveBeenCalledTimes(2); + }); +}); + describe('tapping a calendar reminder', () => { const DEFAULT = 'expo.modules.notifications.actions.DEFAULT'; function response(identifier: string, data: Record, action = DEFAULT) { diff --git a/src/lib/calendar-notifications.ts b/src/lib/calendar-notifications.ts index 19cb47de..b7f4312b 100644 --- a/src/lib/calendar-notifications.ts +++ b/src/lib/calendar-notifications.ts @@ -10,6 +10,7 @@ import { hasCalendarCapability } from './capabilities'; import { onStateChangeType } from './state-change-bus'; import { getUpcomingAlerts, type ScheduledAlert } from './calendar-alert-scheduler'; import type { CalendarReminderTarget } from '../navigation/pending-calendar-open'; +import { onDeviceRemindersChange, remindedOnDevice } from '../device-sync/app/reminders'; // Local reminders for calendar events and tasks. The webmail polls // `getPendingAlerts` every minute while a tab is open; a phone is mostly @@ -147,6 +148,16 @@ function isSignedOut(): boolean { return hasRestoredSession && !isAuthenticated; } +// The JMAP account the upcoming events are loaded from when they carry no +// account of their own. +function primaryJmapAccountId(): string | null { + try { + return jmapClient.accountId; + } catch { + return null; + } +} + /** * The events whose alerts can fire within the horizon, straight from the * server. `null` when that can't be known (offline, no session yet, the @@ -219,10 +230,15 @@ export async function rescheduleCalendarNotifications(): Promise { if (!events) return; await ensureTasksLoaded(); const { tasks, calendars } = useCalendarStore.getState(); + // Events of calendars synced to this device get their reminders from + // the device's calendar app when the user chose it to remind them + // (#34); tasks are not synced and keep theirs. + const onDevice = remindedOnDevice(useAuthStore.getState().activeAccountId, primaryJmapAccountId()); + const ownEvents = onDevice ? events.filter((event) => !onDevice(event)) : events; // Reminders fire while the app is closed, so they are worded now, in // the app language. const { locale, t } = useLocaleStore.getState(); - const wanted = getUpcomingAlerts(events, tasks, calendars, { + const wanted = getUpcomingAlerts(ownEvents, tasks, calendars, { now, horizonMs: HORIZON_MS, limit: MAX_SCHEDULED, @@ -382,6 +398,8 @@ export function startCalendarNotificationSync(options?: { askPermission?: boolea }); onStateChangeType('CalendarEvent', scheduleSoon); onStateChangeType('Calendar', scheduleSoon); + // Who reminds of synced events changed, or which calendars sync (#34). + onDeviceRemindersChange(scheduleSoon); AppState.addEventListener('change', (state) => { if (state === 'active') scheduleSoon(); }); From 434c55403837542d339dd002742885b4e6534451 Mon Sep 17 00:00:00 2001 From: Linus Rath <139418639+rathlinus@users.noreply.github.com> Date: Sun, 27 Sep 2026 05:06:36 +0200 Subject: [PATCH 13/87] fix: keep syncing when a shared account refuses access to its address books or calendars #34 Stalwart lists an account shared with the user for mail only with every capability, and answers `forbidden` when the user asks for its address books or calendars. The engine treated that like any failed request, so on the local test server every sync of usera ended with an I/O error and never reached the device, only because userb shares a mailbox. An account that refuses to list its collections (`forbidden`, `accountNotFound`, `accountNotSupportedByMethod`) is now left out of the run and reported as an item error; the other accounts sync as before. The rows written for such an account stay on the device: a refusal can pass, and dropping the account's contacts and events for one refused request would throw away device edits that are still waiting. Checked: three new engine tests (a shared account that never listed, one that stops listing its address books, one that stops listing its calendars), which fail without the fix, `npm run typecheck` and `npm test` on this commit, and a contacts sync of usera on the API 32 emulator, which now ends `ok` with one item error. --- .../__tests__/engine/failures.test.ts | 62 +++++++++++++++++++ src/device-sync/engine/calendar-sync.ts | 4 +- src/device-sync/engine/contacts-sync.ts | 4 +- src/device-sync/engine/item-sync.ts | 34 +++++++++- 4 files changed, 100 insertions(+), 4 deletions(-) diff --git a/src/device-sync/__tests__/engine/failures.test.ts b/src/device-sync/__tests__/engine/failures.test.ts index e27ec3f4..efa05ce6 100644 --- a/src/device-sync/__tests__/engine/failures.test.ts +++ b/src/device-sync/__tests__/engine/failures.test.ts @@ -9,6 +9,7 @@ import type { JmapInvocation, JmapResponse } from '../../types'; import { addDeviceContact, addServerCards, + CALENDAR_AUTHORITY, CONTACTS_AUTHORITY, createHarness, CrashError, @@ -608,3 +609,64 @@ describe('device sync engine: deletion guards', () => { expect(new CrashError('x').name).toBe('CrashError'); }); }); + +describe('device sync engine: accounts that refuse access', () => { + // Stalwart lists an account shared for mail only with every capability, + // and its /get calls answer `forbidden`. + it('syncs the personal account when a shared one refuses to list its address books', async () => { + const h = createHarness({ accounts: 'withShared' }); + addServerCards(h, ['Ada']); + h.server.failNextMethod('AddressBook/get', { type: 'forbidden', description: 'You do not have access to account team' }, { accountId: 'team' }); + + const report = await h.run(); + + expect(report.outcome).toBe('ok'); + expect(report.itemErrors).toEqual([expect.objectContaining({ ref: 'team', side: 'download', type: 'forbidden' })]); + expect(h.contactNamed('Ada')).toBeDefined(); + }); + + it('keeps the contacts of an account that stops listing its address books, and syncs it again once it does', async () => { + const h = createHarness({ accounts: 'withShared' }); + const teamBook = h.server.all('AddressBook', 'team')[0].id as string; + h.prefs.contactsSelection[`team/${teamBook}`] = true; + const [mate] = addServerCards(h, ['Teammate'], 'team', teamBook); + await h.run(); + expect(h.contactNamed('Teammate')).toBeDefined(); + + addServerCards(h, ['Ada']); + h.server.serverUpdate('ContactCard', 'team', mate, { 'name/full': 'Team mate' }); + h.server.failNextMethod('AddressBook/get', { type: 'forbidden' }, { accountId: 'team' }); + const report = await h.run(); + + expect(report.outcome).toBe('ok'); + expect(h.contactNamed('Ada')).toBeDefined(); + expect(h.contactNamed('Teammate')).toBeDefined(); + + expect((await h.run()).itemErrors).toEqual([]); + expect(h.contactNamed('Team mate')).toBeDefined(); + }); + + it('keeps the calendar and events of an account that refuses to list its calendars', async () => { + const h = createHarness({ accounts: 'withShared' }); + const teamCalendar = h.server.all('Calendar', 'team')[0].id as string; + h.prefs.calendarSelection[`team/${teamCalendar}`] = true; + h.server.addEvent('team', { + uid: 'uid-team', + title: 'Team meeting', + start: '2026-09-28T09:00:00', + duration: 'PT1H', + timeZone: 'Etc/UTC', + calendarIds: { [teamCalendar]: true }, + }); + await h.run(CALENDAR_AUTHORITY); + expect(h.events()).toHaveLength(1); + + h.server.failNextMethod('Calendar/get', { type: 'forbidden' }, { accountId: 'team' }); + const report = await h.run(CALENDAR_AUTHORITY); + + expect(report.outcome).toBe('ok'); + expect(report.itemErrors).toEqual([expect.objectContaining({ ref: 'team', type: 'forbidden' })]); + expect(h.events()).toHaveLength(1); + expect(h.device.rows('calendars').some((c) => c._sync_id === `team/${teamCalendar}`)).toBe(true); + }); +}); diff --git a/src/device-sync/engine/calendar-sync.ts b/src/device-sync/engine/calendar-sync.ts index 38997b03..1f78fecf 100644 --- a/src/device-sync/engine/calendar-sync.ts +++ b/src/device-sync/engine/calendar-sync.ts @@ -219,7 +219,9 @@ export class CalendarSync extends ItemSync { this.readOnlyKeys.clear(); const selected: Array<{ jmapAccountId: string; calendar: CalendarLike; readOnly: boolean; accountName: string }> = []; for (const account of this.env.accounts) { - const { list, state } = await this.env.jmap.getContainers('Calendar', account.id, CALENDAR_PROPERTIES); + const containers = await this.containersOf('Calendar', account.id, CALENDAR_PROPERTIES); + if (!containers) continue; + const { list, state } = containers; this.calendars.set(account.id, list); if (state) { this.collectionStates.set(account.id, state); diff --git a/src/device-sync/engine/contacts-sync.ts b/src/device-sync/engine/contacts-sync.ts index 2064a4b4..3d5c55ee 100644 --- a/src/device-sync/engine/contacts-sync.ts +++ b/src/device-sync/engine/contacts-sync.ts @@ -286,7 +286,9 @@ export class ContactsSync extends ItemSync { protected async collections(write: boolean): Promise { this.syncedKeys.clear(); for (const account of this.env.accounts) { - const { list, state } = await this.env.jmap.getContainers('AddressBook', account.id, ADDRESS_BOOK_PROPERTIES); + const containers = await this.containersOf('AddressBook', account.id, ADDRESS_BOOK_PROPERTIES); + if (!containers) continue; + const { list, state } = containers; this.books.set(account.id, list); for (const book of list) { if (isCollectionSelected(this.env.prefs.contactsSelection, account, CONTACTS_AUTHORITY, book)) { diff --git a/src/device-sync/engine/item-sync.ts b/src/device-sync/engine/item-sync.ts index 7f0f731c..942e1204 100644 --- a/src/device-sync/engine/item-sync.ts +++ b/src/device-sync/engine/item-sync.ts @@ -66,6 +66,9 @@ type Removal = 'destroyed' | 'outside'; /** Method-level refusals of a whole `/set` that concern the account, not the objects. */ const ACCOUNT_REFUSALS = new Set(['accountReadOnly', 'forbidden', 'accountNotFound', 'accountNotSupportedByMethod']); +/** Refusals to list an account's collections: the account is not readable for this user. */ +const ACCOUNT_UNREADABLE = new Set(['forbidden', 'accountNotFound', 'accountNotSupportedByMethod']); + export abstract class ItemSync { /** Per account: object ids put on `stale` (true) or fetched fine again (false), for the next state op. */ private readonly staleMarks = new Map>(); @@ -75,6 +78,8 @@ export abstract class ItemSync { private readonly extraCreated = new Map>(); /** Per account: `ifInState` of the next `/set` (the download's state, then our last `newState`). */ private readonly setStates = new Map(); + /** Accounts that refused to list their collections this run: left out of it (see `containersOf`). */ + private readonly unreadable = new Set(); constructor(protected readonly env: RunEnv) {} @@ -171,12 +176,37 @@ export abstract class ItemSync { } protected accountIds(): string[] { - return this.env.accounts.map((a) => a.id); + return this.env.accounts.map((a) => a.id).filter((id) => !this.unreadable.has(id)); } /** The session's accounts plus accounts the rows were written for that are gone from it. */ protected knownAccountIds(): string[] { - return [...new Set([...this.accountIds(), ...Object.keys(this.env.store.committed.accounts)])]; + return [...new Set([...this.accountIds(), ...Object.keys(this.env.store.committed.accounts)])] + .filter((id) => !this.unreadable.has(id)); + } + + /** + * One account's collections, or null when the account refuses to list them + * (`forbidden`, `accountNotFound`, `accountNotSupportedByMethod`): Stalwart + * lists an account shared for mail only with every capability, and access + * can go away. Such an account is left out of the run and reported; the + * rows written for it stay as they are, since a refusal can pass and must + * not drop anything from the device. + */ + protected async containersOf( + type: 'AddressBook' | 'Calendar', + accountId: string, + properties: readonly string[], + ): Promise<{ list: T[]; state: string | null } | null> { + try { + return await this.env.jmap.getContainers(type, accountId, properties); + } catch (error) { + if (!isMethodError(error) || !ACCOUNT_UNREADABLE.has(error.type)) throw error; + this.unreadable.add(accountId); + this.env.report.itemError({ ref: accountId, side: 'download', type: error.type, description: error.message }); + this.env.log(`account ${accountId} left out: ${error.type}`); + return null; + } } // ── State ── From ad0f83c72fc31c34c75a35af284c003af0ff7503 Mon Sep 17 00:00:00 2001 From: Linus Rath <139418639+rathlinus@users.noreply.github.com> Date: Sun, 27 Sep 2026 05:07:58 +0200 Subject: [PATCH 14/87] fix: stop reading the read-only flag the contacts provider never returns #34 The contacts planner read RAW_CONTACT_IS_READ_ONLY with every raw contact, but ContactsProvider takes that column only on writes: every projection and selection that names it fails with "Invalid column", on API 32 and API 35 alike. Every contacts sync on a device therefore failed before it wrote a row. The fake provider accepted the column, so no test noticed. The flag is now only written, along with any raw contact write that happens anyway, so echoes stay write-free. Whether a contact's device edits are put back is decided from the server's current rights on the card, which is also right when only the rights changed. The fake now refuses the column in queries and selections like the real providers, and every other column the planners query was checked against the real providers on both emulators. Checked: the contacts, engine and fake provider tests, `npm run typecheck` and `npm test` on this commit, and a contacts sync of usera on the API 32 emulator, which now completes. --- src/device-sync/__tests__/contacts/units.test.ts | 8 +++++--- src/device-sync/__tests__/engine/toy-planners.ts | 7 +++---- src/device-sync/__tests__/fake-provider.test.ts | 13 +++++++++++++ src/device-sync/__tests__/fakes/fake-provider.ts | 6 ++++++ src/device-sync/contacts/columns.ts | 7 +++++-- src/device-sync/contacts/download.ts | 7 ++++++- src/device-sync/contacts/upload.ts | 13 +++++++++++-- src/device-sync/planner.ts | 1 - 8 files changed, 49 insertions(+), 13 deletions(-) diff --git a/src/device-sync/__tests__/contacts/units.test.ts b/src/device-sync/__tests__/contacts/units.test.ts index 497dc2c1..7738244e 100644 --- a/src/device-sync/__tests__/contacts/units.test.ts +++ b/src/device-sync/__tests__/contacts/units.test.ts @@ -12,8 +12,10 @@ import { JPEG } from './fixtures'; describe('contacts decode', () => { it('queries the columns the planner reads', () => { expect(contactsPlanner.rawContactColumns).toEqual(expect.arrayContaining([ - '_id', 'sourceid', 'version', 'dirty', 'deleted', 'sync1', 'sync2', 'sync3', 'sync4', 'raw_contact_is_read_only', + '_id', 'sourceid', 'version', 'dirty', 'deleted', 'sync1', 'sync2', 'sync3', 'sync4', ])); + // The providers refuse it in every projection ("Invalid column"): it is only written. + expect(contactsPlanner.rawContactColumns).not.toContain('raw_contact_is_read_only'); expect(contactsPlanner.dataColumns).toEqual(expect.arrayContaining([ '_id', 'raw_contact_id', 'mimetype', 'is_primary', 'data1', 'data14', 'data_sync1', 'data_sync2', 'data_sync3', 'group_sourceid', ])); @@ -23,14 +25,14 @@ describe('contacts decode', () => { it('reads numbers given as text (data columns are TEXT) and tolerates what other apps left in sync columns', () => { const local = contactsPlanner.decodeContact( - { _id: '5', sourceid: 'c/e1', version: '3', dirty: '1', deleted: null, sync1: 'c/b,garbage,,c/x', sync2: '{"not":"a card"}', sync3: '{"uid":1}', sync4: 'nope', raw_contact_is_read_only: 0 }, + { _id: '5', sourceid: 'c/e1', version: '3', dirty: '1', deleted: null, sync1: 'c/b,garbage,,c/x', sync2: '{"not":"a card"}', sync3: '{"uid":1}', sync4: 'nope' }, [ { _id: '9', raw_contact_id: '5', mimetype: MimeType.EMAIL, data1: 'a@b', data2: '2', data_sync1: 'emails:e1', data_sync2: '', data_sync3: '[1]' }, { _id: 10, raw_contact_id: 6, mimetype: MimeType.NOTE, data1: 'another contact' }, ], ); expect(local).toMatchObject({ - rawContactId: 5, sourceId: 'c/e1', version: 3, dirty: true, deleted: false, readOnly: false, + rawContactId: 5, sourceId: 'c/e1', version: 3, dirty: true, deleted: false, collections: ['c/b', 'c/x'], shadow: null, pending: null, poison: null, }); expect(local.rows).toEqual([{ id: 9, mimetype: MimeType.EMAIL, cells: { data1: 'a@b', data2: '2' }, key: 'emails:e1', photoHash: null, baseline: null }]); diff --git a/src/device-sync/__tests__/engine/toy-planners.ts b/src/device-sync/__tests__/engine/toy-planners.ts index 8bb7136a..10a01ad7 100644 --- a/src/device-sync/__tests__/engine/toy-planners.ts +++ b/src/device-sync/__tests__/engine/toy-planners.ts @@ -97,7 +97,7 @@ function membershipOps(local: LocalContact | null, card: ContactCardWire, ctx: T } export const toyContactsPlanner: ContactsPlanner = { - rawContactColumns: ['_id', 'sourceid', 'version', 'dirty', 'deleted', 'sync1', 'sync2', 'sync3', 'sync4', 'raw_contact_is_read_only'], + rawContactColumns: ['_id', 'sourceid', 'version', 'dirty', 'deleted', 'sync1', 'sync2', 'sync3', 'sync4'], dataColumns: ['_id', 'raw_contact_id', 'mimetype', 'data1', 'data_sync1', 'data_sync3', 'group_sourceid'], groupColumns: ['_id', 'sourceid', 'version', 'dirty', 'deleted', 'title', 'sync2', 'sync3', 'sync4'], @@ -108,7 +108,6 @@ export const toyContactsPlanner: ContactsPlanner = { version: num(rc.version), dirty: flag(rc.dirty), deleted: flag(rc.deleted), - readOnly: flag(rc.raw_contact_is_read_only), collections: (str(rc.sync1) ?? '').split(',').filter(Boolean), shadow: parseJsonColumn(rc.sync2), pending: parseJsonColumn(rc.sync3), @@ -207,7 +206,7 @@ export const toyContactsPlanner: ContactsPlanner = { stillDirty = true; } ops.push(...membershipOps(local, card, ctx, local.rawContactId)); - const shadowSame = deepEqual(local.shadow, card) && local.collections.join(',') === raw.sync1 && local.readOnly === !!raw.raw_contact_is_read_only; + const shadowSame = deepEqual(local.shadow, card) && local.collections.join(',') === raw.sync1; if (!shadowSame) ops.push({ op: 'update', table: 'raw_contacts', id: local.rawContactId, values: raw }); const writes = ops.length - 1; return { ops: { ref, ops }, conflicts, stillDirty, effect: writes > 0 ? 'update' : 'none', writes }; @@ -263,7 +262,7 @@ export const toyContactsPlanner: ContactsPlanner = { actions: [{ kind: 'create', uid: local.pending.uid, collectionId: target.id, object: { uid: local.pending.uid, name: { full: value } } }], }; } - if (local.readOnly) { + if (local.shadow && ctx.isReadOnly(local.shadow)) { const shadowName = nameOf(local.shadow); return { kind: 'revert', diff --git a/src/device-sync/__tests__/fake-provider.test.ts b/src/device-sync/__tests__/fake-provider.test.ts index 335c8047..98803ef9 100644 --- a/src/device-sync/__tests__/fake-provider.test.ts +++ b/src/device-sync/__tests__/fake-provider.test.ts @@ -441,6 +441,19 @@ describe('FakeDeviceProviders: the native bridge rules', () => { expect(fake.readSyncState(ME, CONTACTS_AUTHORITY)).toBeNull(); }); + it('takes raw_contact_is_read_only on writes but refuses it in projections and selections', async () => { + const fake = new FakeDeviceProviders(); + const port = fake.port(ME, CONTACTS_AUTHORITY); + const res = await port.applyBatch([{ op: 'insert', table: 'raw_contacts', values: { [RawContacts.RAW_CONTACT_IS_READ_ONLY]: 1 } }]); + expect(res.ok).toBe(true); + await expect(port.query({ table: 'raw_contacts', columns: [RawContacts.RAW_CONTACT_IS_READ_ONLY] })).rejects.toMatchObject({ + code: 'provider', + }); + await expect( + port.query({ table: 'raw_contacts', columns: [RawContacts._ID], where: `${RawContacts.RAW_CONTACT_IS_READ_ONLY} = 1` }), + ).rejects.toMatchObject({ code: 'provider' }); + }); + it('returns no id for a settings insert, which cannot be addressed by id either', async () => { const fake = new FakeDeviceProviders(); const port = fake.port(ME, CONTACTS_AUTHORITY); diff --git a/src/device-sync/__tests__/fakes/fake-provider.ts b/src/device-sync/__tests__/fakes/fake-provider.ts index 1acb2f6a..8fbad6e2 100644 --- a/src/device-sync/__tests__/fakes/fake-provider.ts +++ b/src/device-sync/__tests__/fakes/fake-provider.ts @@ -106,6 +106,11 @@ const COLUMNS: Record> = { colors: new Set(Object.values(Colors)), }; +/** Columns the providers take on writes but refuse in every projection and selection ("Invalid column"). */ +const WRITE_ONLY_COLUMNS: Partial>> = { + raw_contacts: new Set([RawContacts.RAW_CONTACT_IS_READ_ONLY]), +}; + /** Columns only the provider computes; writes to them are refused like SQLite would for a view. */ const READ_ONLY_COLUMNS: Partial>> = { data: new Set([GroupMembership.GROUP_SOURCE_ID]), @@ -557,6 +562,7 @@ function tablesFor(authority: Authority): readonly ProviderTable[] { function checkColumns(table: ProviderTable, columns: Iterable, write: boolean): void { for (const c of columns) { if (!COLUMNS[table].has(c)) throw new FakeProviderError('provider', `no such column: ${c} in ${table}`); + if (!write && WRITE_ONLY_COLUMNS[table]?.has(c)) throw new FakeProviderError('provider', `Invalid column ${c}`); if (write && READ_ONLY_COLUMNS[table]?.has(c)) throw new FakeProviderError('provider', `${c} is read-only in ${table}`); } } diff --git a/src/device-sync/contacts/columns.ts b/src/device-sync/contacts/columns.ts index 87701327..d7bb176a 100644 --- a/src/device-sync/contacts/columns.ts +++ b/src/device-sync/contacts/columns.ts @@ -17,9 +17,13 @@ const DATA_CELLS = [ GroupMembership.GROUP_SOURCE_ID, ]; +/** + * RAW_CONTACT_IS_READ_ONLY is not among them: the providers take it on + * writes but refuse it in every projection ("Invalid column"). + */ export const RAW_CONTACT_COLUMNS: readonly string[] = [ RawContacts._ID, RawContacts.SOURCE_ID, RawContacts.VERSION, RawContacts.DIRTY, RawContacts.DELETED, - RawContacts.SYNC1, RawContacts.SYNC2, RawContacts.SYNC3, RawContacts.SYNC4, RawContacts.RAW_CONTACT_IS_READ_ONLY, + RawContacts.SYNC1, RawContacts.SYNC2, RawContacts.SYNC3, RawContacts.SYNC4, ]; export const DATA_COLUMNS: readonly string[] = [ @@ -89,7 +93,6 @@ export function decodeContact(rawContact: Row, data: Row[]): LocalContact { version: num(rawContact[RawContacts.VERSION]) ?? 0, dirty: flag(rawContact[RawContacts.DIRTY]), deleted: flag(rawContact[RawContacts.DELETED]), - readOnly: flag(rawContact[RawContacts.RAW_CONTACT_IS_READ_ONLY]), collections, shadow: decodeShadow(rawContact[RawContacts.SYNC2]), pending: decodePending(rawContact[RawContacts.SYNC3]), diff --git a/src/device-sync/contacts/download.ts b/src/device-sync/contacts/download.ts index 4cf57042..e6c0eafb 100644 --- a/src/device-sync/contacts/download.ts +++ b/src/device-sync/contacts/download.ts @@ -50,11 +50,11 @@ function changedRawColumns(local: LocalContact, values: WriteRow): WriteRow { [RawContacts.SYNC2]: local.shadow ? JSON.stringify(local.shadow) : null, [RawContacts.SYNC3]: local.pending ? JSON.stringify(local.pending) : null, [RawContacts.SYNC4]: local.poison ? JSON.stringify(local.poison) : null, - [RawContacts.RAW_CONTACT_IS_READ_ONLY]: local.readOnly ? 1 : 0, [RawContacts.DIRTY]: local.dirty ? 1 : 0, }; const out: WriteRow = {}; for (const [column, value] of Object.entries(values)) { + if (column === RawContacts.RAW_CONTACT_IS_READ_ONLY) continue; if (column === RawContacts.SYNC2) { const before = local.shadow ? canonicalJson(local.shadow) : null; const after = typeof value === 'string' ? canonicalJson(JSON.parse(value)) : null; @@ -63,6 +63,11 @@ function changedRawColumns(local: LocalContact, values: WriteRow): WriteRow { out[column] = value; } } + // The providers never return the read-only flag, so it can't be compared: + // it goes along whenever the raw contact is written anyway. + if (Object.keys(out).length && RawContacts.RAW_CONTACT_IS_READ_ONLY in values) { + out[RawContacts.RAW_CONTACT_IS_READ_ONLY] = values[RawContacts.RAW_CONTACT_IS_READ_ONLY]; + } return out; } diff --git a/src/device-sync/contacts/upload.ts b/src/device-sync/contacts/upload.ts index b9d221dc..debff260 100644 --- a/src/device-sync/contacts/upload.ts +++ b/src/device-sync/contacts/upload.ts @@ -205,11 +205,20 @@ function updatePatch(local: LocalContact, ctx: Ctx): PatchObject { const group = (local: LocalContact, ops: ProviderOp[]): OpGroup => ({ ref: local.sourceId ?? `row:${local.rawContactId}`, ops }); +/** + * Whether the card's address books take no writes now. Asked of the server's + * rights, not of RAW_CONTACT_IS_READ_ONLY on the device, which the providers + * never return (and which lags when only the rights changed). + */ +function readOnlyOnServer(local: LocalContact, ctx: Ctx): boolean { + return !!local.shadow && ctx.isReadOnly(withoutMemberOf(local.shadow)); +} + function deletedPlan(local: LocalContact, ctx: Ctx): UploadPlan { const purge = (): OpGroup => group(local, [deleteWhereId('raw_contacts', local.rawContactId)]); const ref = parseObjectRef(local.sourceId); if (ref) { - if (local.readOnly) return { kind: 'revert', ops: purge(), refetch: true }; + if (readOnlyOnServer(local, ctx)) return { kind: 'revert', ops: purge(), refetch: true }; const books = Object.entries(local.shadow?.addressBookIds ?? {}).filter(([, on]) => on).map(([id]) => id); const selected = books.filter((id) => ctx.isSelected(collectionKey(ref.accountId, id))); // A card that is also in books this device does not sync only leaves the synced ones. @@ -245,7 +254,7 @@ export function planContactUpload(local: LocalContact, ctx: Ctx): UploadPlan Date: Sun, 27 Sep 2026 05:14:04 +0200 Subject: [PATCH 15/87] fix: stop warning about shared accounts that share only mail #34 434c554 left an account that refuses to list its address books or calendars out of the run, but reported it as an item error. The sync settings then said "1 item could not be synced" after every run for an account that merely shares a mailbox: nothing the user can fix, and no item behind it. Such an account is now only logged. Checked: the engine tests, `npm run typecheck` and `npm test` on this commit, and the sync settings of usera on the API 32 emulator after a new run. --- src/device-sync/__tests__/engine/failures.test.ts | 4 ++-- src/device-sync/engine/item-sync.ts | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/device-sync/__tests__/engine/failures.test.ts b/src/device-sync/__tests__/engine/failures.test.ts index efa05ce6..af4903cc 100644 --- a/src/device-sync/__tests__/engine/failures.test.ts +++ b/src/device-sync/__tests__/engine/failures.test.ts @@ -621,7 +621,7 @@ describe('device sync engine: accounts that refuse access', () => { const report = await h.run(); expect(report.outcome).toBe('ok'); - expect(report.itemErrors).toEqual([expect.objectContaining({ ref: 'team', side: 'download', type: 'forbidden' })]); + expect(report.itemErrors).toEqual([]); expect(h.contactNamed('Ada')).toBeDefined(); }); @@ -665,7 +665,7 @@ describe('device sync engine: accounts that refuse access', () => { const report = await h.run(CALENDAR_AUTHORITY); expect(report.outcome).toBe('ok'); - expect(report.itemErrors).toEqual([expect.objectContaining({ ref: 'team', type: 'forbidden' })]); + expect(report.itemErrors).toEqual([]); expect(h.events()).toHaveLength(1); expect(h.device.rows('calendars').some((c) => c._sync_id === `team/${teamCalendar}`)).toBe(true); }); diff --git a/src/device-sync/engine/item-sync.ts b/src/device-sync/engine/item-sync.ts index 942e1204..e4c96911 100644 --- a/src/device-sync/engine/item-sync.ts +++ b/src/device-sync/engine/item-sync.ts @@ -189,9 +189,10 @@ export abstract class ItemSync { * One account's collections, or null when the account refuses to list them * (`forbidden`, `accountNotFound`, `accountNotSupportedByMethod`): Stalwart * lists an account shared for mail only with every capability, and access - * can go away. Such an account is left out of the run and reported; the - * rows written for it stay as they are, since a refusal can pass and must - * not drop anything from the device. + * can go away. Such an account is left out of the run, and not reported + * as a problem: for a mail-only share there is nothing to fix, and the + * warning would never go away. The rows written for it stay as they are, + * since a refusal can pass and must not drop anything from the device. */ protected async containersOf( type: 'AddressBook' | 'Calendar', @@ -203,7 +204,6 @@ export abstract class ItemSync { } catch (error) { if (!isMethodError(error) || !ACCOUNT_UNREADABLE.has(error.type)) throw error; this.unreadable.add(accountId); - this.env.report.itemError({ ref: accountId, side: 'download', type: error.type, description: error.message }); this.env.log(`account ${accountId} left out: ${error.type}`); return null; } From 77fd494633c5624f4c9eb97b67206da34225dad0 Mon Sep 17 00:00:00 2001 From: Linus Rath <139418639+rathlinus@users.noreply.github.com> Date: Sun, 27 Sep 2026 05:15:32 +0200 Subject: [PATCH 16/87] feat: turn on contacts and calendar sync to this device in settings #34 On Android, Settings > Contacts and Settings > Calendar get a "Sync to this device" section with a row per signed-in account. This makes device sync reachable: - The switch asks for the contacts or calendar permission after a short explanation (denied: sync stays off and says why, with a link to the app's system settings once Android stops asking), creates the account's Bulwark account in Android and syncs right away. Turning it off uploads what the device changed first and asks before anything that could not go up is lost. The switch shows Android's own state, so sync turned off in Android Settings shows as paused. - Which address books or calendars sync: the personal account's are on by default, shared ones and the Trusted Senders book off; read-only ones are marked, and a note says when another signed-in account already syncs the same shared one. - The address book new contacts from other apps go to, who reminds of synced events (the calendar app or Bulwark, asked when calendar sync is turned on while Bulwark's reminders are on), and how often Android syncs on its own (every 15 minutes to every 6 hours, or not at all). - "Sync now", the last sync with its result, or the error with what to do about it ("Sign in again", "Grant access", "Review deletions"), and a link to the account in Android Settings. - Accounts the app signed out while they synced are listed with "Sign in again" and "Remove from this device". The settings search finds both sections. iOS, and Android builds without the native module, show nothing new. Checked: the collection listing and status tests, the settings search and i18n tests, `npm run typecheck`, `npm test` and `npm run i18n:check` on this commit, and on the API 32 emulator with usera on the local server: the section, turning contacts sync on (Android account created, 152 contacts on the device, one per card) and the status after a run. --- .../__tests__/settings-i18n.test.ts | 5 + src/components/settings/CalendarSettings.tsx | 247 +++++---- src/components/settings/ContactsSettings.tsx | 16 + .../device-sync/DeviceSyncAccountRow.tsx | 502 ++++++++++++++++++ .../DeviceSyncCollectionsSheet.tsx | 310 +++++++++++ .../device-sync/DeviceSyncSection.tsx | 118 ++++ .../device-sync/__tests__/status.test.ts | 81 +++ src/components/settings/device-sync/status.ts | 133 +++++ .../settings/device-sync/use-android-sync.ts | 101 ++++ .../device-sync/use-sync-collections.ts | 61 +++ .../__tests__/app/collections.test.ts | 230 ++++++++ src/device-sync/app/collections.ts | 334 ++++++++++++ src/lib/__tests__/settings-search.test.ts | 15 + src/lib/settings-search.ts | 10 +- 14 files changed, 2047 insertions(+), 116 deletions(-) create mode 100644 src/components/settings/device-sync/DeviceSyncAccountRow.tsx create mode 100644 src/components/settings/device-sync/DeviceSyncCollectionsSheet.tsx create mode 100644 src/components/settings/device-sync/DeviceSyncSection.tsx create mode 100644 src/components/settings/device-sync/__tests__/status.test.ts create mode 100644 src/components/settings/device-sync/status.ts create mode 100644 src/components/settings/device-sync/use-android-sync.ts create mode 100644 src/components/settings/device-sync/use-sync-collections.ts create mode 100644 src/device-sync/__tests__/app/collections.test.ts create mode 100644 src/device-sync/app/collections.ts diff --git a/src/components/__tests__/settings-i18n.test.ts b/src/components/__tests__/settings-i18n.test.ts index 5e1fadb3..cfd9ce8b 100644 --- a/src/components/__tests__/settings-i18n.test.ts +++ b/src/components/__tests__/settings-i18n.test.ts @@ -40,6 +40,11 @@ const TRANSLATED_PANES: Record = { 'settings/UpdatesSettings.tsx': [], 'settings/VacationSettings.tsx': ['YYYY-MM-DD HH:MM'], 'settings/settings-section.tsx': [], + // Device sync (Android, #34). + 'settings/device-sync/DeviceSyncSection.tsx': [], + 'settings/device-sync/DeviceSyncAccountRow.tsx': [], + 'settings/device-sync/DeviceSyncCollectionsSheet.tsx': [], + 'settings/device-sync/status.ts': [], 'filters/FilterRuleModal.tsx': [], 'filters/SieveEditorSheet.tsx': [], }; diff --git a/src/components/settings/CalendarSettings.tsx b/src/components/settings/CalendarSettings.tsx index fa50d6b8..dab6f860 100644 --- a/src/components/settings/CalendarSettings.tsx +++ b/src/components/settings/CalendarSettings.tsx @@ -8,6 +8,9 @@ import { } from '../../stores/settings-store'; import { useLocaleStore } from '../../stores/locale-store'; import { AUTO_TIME_ZONE, getDeviceTimeZone, isValidTimeZone } from '../../lib/calendar-timezone'; +import { deviceSyncAvailable } from '../../device-sync/app/available'; +import { CALENDAR_AUTHORITY } from '../../device-sync/types'; +import { DeviceSyncSection } from './device-sync/DeviceSyncSection'; // A compact list of IANA zones for the picker (#755). The device zone and // any previously stored value are always offered too. @@ -46,6 +49,9 @@ export function CalendarSettings() { const tasksEnabled = useSettingsStore((s) => s.enableCalendarTasks); const showTasksOnCal = useSettingsStore((s) => s.showTasksOnCalendar); + // Android with the native module only (#34). + const deviceSync = React.useMemo(() => deviceSyncAvailable(), []); + useEffect(() => { if (!hydrated) void hydrate(); }, [hydrated, hydrate]); @@ -64,130 +70,143 @@ export function CalendarSettings() { }, [deviceZone, timeZone, t]); return ( - - - update('calendarDefaultView', v as CalendarView)} + options={[ + { value: 'month', label: t('calendar.views.month', 'Month') }, + { value: 'week', label: t('calendar.views.week', 'Week') }, + { value: 'day', label: t('calendar.views.day', 'Day') }, + { value: 'agenda', label: t('calendar.views.agenda', 'Agenda') }, + ]} + /> + - - update('calendarFirstDayOfWeek', Number(v) as FirstDayOfWeek)} + options={[ + { value: '1', label: t('calendar.days.monday', 'Monday') }, + { value: '6', label: t('calendar.days.saturday', 'Saturday') }, + { value: '0', label: t('calendar.days.sunday', 'Sunday') }, + ]} + /> + - - update('calendarTimeFormat', v as TimeFormat)} - options={[ - { value: '12h', label: t('calendar.settings.time_format_12h', '12-hour') }, - { value: '24h', label: t('calendar.settings.time_format_24h', '24-hour') }, - ]} - /> - + + update('calendarTimeFormat', v as TimeFormat)} + options={[ + { value: '12h', label: t('calendar.settings.time_format_12h', '12-hour') }, + { value: '24h', label: t('calendar.settings.time_format_24h', '24-hour') }, + ]} + /> + - - update('calendarTimeZone', v)} + options={timeZoneOptions} + /> + + + update('showTasksOnCalendar', v)} + checked={showTimeInMonth} + onChange={(v) => update('calendarShowTimeInMonth', v)} /> + + + update('calendarShowWeekNumbers', v)} + /> + + + + update('calendarFreeScroll', v)} + /> + + + + update('showBirthdayCalendar', v)} + /> + + + + update('enableCalendarTasks', v)} + /> + + + {tasksEnabled && ( + + update('showTasksOnCalendar', v)} + /> + + )} + + + {deviceSync && ( + )} - + ); } diff --git a/src/components/settings/ContactsSettings.tsx b/src/components/settings/ContactsSettings.tsx index da8cb4bc..04b468c4 100644 --- a/src/components/settings/ContactsSettings.tsx +++ b/src/components/settings/ContactsSettings.tsx @@ -9,11 +9,14 @@ import { SettingsSection, SettingItem, ToggleSwitch } from './settings-section'; import Button from '../Button'; import Dialog from '../Dialog'; import { ContactImportSheet, AddressBookPickerSheet } from '../contacts'; +import { DeviceSyncSection } from './device-sync/DeviceSyncSection'; import { useSettingsStore } from '../../stores/settings-store'; import { useLocaleStore } from '../../stores/locale-store'; import { useContactsStore, selectAddressBooksWithCount } from '../../stores/contacts-store'; import { contactsToVCard } from '../../lib/vcard'; import { isGroup } from '../../lib/contact-utils'; +import { deviceSyncAvailable } from '../../device-sync/app/available'; +import { CONTACTS_AUTHORITY } from '../../device-sync/types'; import { spacing, radius, typography, componentSizes, type ThemePalette } from '../../theme/tokens'; import { useColors } from '../../theme/colors'; @@ -53,6 +56,8 @@ export function ContactsSettings() { const [newName, setNewName] = React.useState(''); const [busy, setBusy] = React.useState(false); const [deleteTarget, setDeleteTarget] = React.useState<{ id: string; name: string; count: number } | null>(null); + // Android with the native module only (#34). + const deviceSync = React.useMemo(() => deviceSyncAvailable(), []); const exportable = React.useMemo(() => contacts.filter((cc) => !isGroup(cc)), [contacts]); const exportLabel = @@ -356,6 +361,17 @@ export function ContactsSettings() { + {deviceSync && ( + + )} + setImportOpen(false)} diff --git a/src/components/settings/device-sync/DeviceSyncAccountRow.tsx b/src/components/settings/device-sync/DeviceSyncAccountRow.tsx new file mode 100644 index 00000000..dc3d68c3 --- /dev/null +++ b/src/components/settings/device-sync/DeviceSyncAccountRow.tsx @@ -0,0 +1,502 @@ +// One signed-in account in "Sync to this device" (docs/device-sync.md, +// "Settings sections"): the switch (Android's automatic sync for the +// authority), the status or the error with what to do about it, and, while +// sync is on, which collections sync, where new contacts go or who reminds of +// events, the interval, "Sync now" and Android's account settings. +import React from 'react'; +import { Alert, StyleSheet, Text, View, ActivityIndicator } from 'react-native'; +import { useNavigation } from '@react-navigation/native'; +import type { NativeStackNavigationProp } from '@react-navigation/native-stack'; +import { AlertTriangle, RefreshCw, Settings as SettingsIcon, XCircle } from 'lucide-react-native'; +import { SettingItem, Select, RadioGroup, ToggleSwitch } from '../settings-section'; +import Button from '../../Button'; +import Dialog from '../../Dialog'; +import { spacing, typography, type ThemePalette } from '../../../theme/tokens'; +import { useColors } from '../../../theme/colors'; +import { useLocaleStore, type TranslateFn } from '../../../stores/locale-store'; +import { useSettingsStore } from '../../../stores/settings-store'; +import type { AccountEntry } from '../../../stores/account-store'; +import { + effectiveSelection, + emptyAccountDeviceSync, + intervalFor, + syncOnInApp, + useDeviceSyncStore, +} from '../../../stores/device-sync-store'; +import { + changeReminderOwner, + changeSyncInterval, + disableDeviceSync, + enableDeviceSync, + openAndroidAccountSettings, + resumeDeviceSync, + syncNow, +} from '../../../device-sync/app/lifecycle'; +import { hasSyncPermissions, openAppSystemSettings, requestSyncPermissions } from '../../../device-sync/app/permissions'; +import { CALENDAR_AUTHORITY, CONTACTS_AUTHORITY, type Authority, type ReminderOwner } from '../../../device-sync/types'; +import type { RootStackParamList } from '../../../navigation/types'; +import { DeviceSyncCollectionsSheet } from './DeviceSyncCollectionsSheet'; +import { describeSyncStatus, type StatusAction } from './status'; +import { useAndroidSync } from './use-android-sync'; +import { useSyncCollections } from './use-sync-collections'; + +interface Props { + account: AccountEntry; + authority: Authority; +} + +type Busy = 'on' | 'off' | 'sync' | null; + +type Prompt = + | { kind: 'rationale' } + | { kind: 'blocked' } + | { kind: 'pending'; pending: number }; + +/** Asks who reminds of synced events; null when the user backed out. */ +function askReminderOwner(t: TranslateFn): Promise { + return new Promise((resolve) => { + let settled = false; + const answer = (owner: ReminderOwner | null) => { + if (settled) return; + settled = true; + resolve(owner); + }; + Alert.alert( + t('settings.device_sync.reminder_question_title', 'Who should remind you?'), + t( + 'settings.device_sync.reminder_question_message', + 'Synced events can remind you from the calendar app on this device or from Bulwark. Pick one so you are not reminded twice. You can change it later.', + ), + [ + { text: t('common.cancel', 'Cancel'), style: 'cancel', onPress: () => answer(null) }, + { text: t('settings.device_sync.reminder_owner_bulwark', 'Bulwark'), onPress: () => answer('bulwark') }, + { text: t('settings.device_sync.reminder_owner_device', 'Calendar app'), onPress: () => answer('device') }, + ], + { cancelable: true, onDismiss: () => answer(null) }, + ); + }); +} + +export function DeviceSyncAccountRow({ account, authority }: Props) { + const c = useColors(); + const styles = React.useMemo(() => makeStyles(c), [c]); + const t = useLocaleStore((s) => s.t); + const navigation = useNavigation>(); + const storedEntry = useDeviceSyncStore((s) => s.accounts[account.id]); + const entry = React.useMemo(() => storedEntry ?? emptyAccountDeviceSync(), [storedEntry]); + const calendarReminders = useSettingsStore((s) => s.calendarNotificationsEnabled); + const isContacts = authority === CONTACTS_AUTHORITY; + + const onInApp = syncOnInApp(entry, authority); + const { view, permitted, refresh } = useAndroidSync(entry.androidAccountName, authority); + const [busy, setBusy] = React.useState(null); + const [prompt, setPrompt] = React.useState(null); + const [chooserOpen, setChooserOpen] = React.useState(false); + + // The switch follows Android; until Android answered, what the app knows. + const androidOn = view ? view.exists && view.automatic : onInApp; + const checked = busy === 'on' ? true : busy === 'off' ? false : androidOn; + const paused = onInApp && !!view?.exists && !view.automatic; + const showDetails = (checked || paused) && busy !== 'off'; + + const { state: collectionsState } = useSyncCollections(account.id, authority, showDetails); + const collections = collectionsState.kind === 'loaded' ? collectionsState.collections : null; + const selectedKeys = React.useMemo( + () => (collections ? effectiveSelection(entry, authority, collections) : null), + [collections, entry, authority], + ); + + const accountLabel = account.email || account.username; + const status = describeSyncStatus( + { + authority, + on: checked, + paused, + removed: !!entry.removedInAndroidSettings, + permitted, + syncing: !!view?.active, + lastRun: entry.lastRun[authority], + now: Date.now(), + }, + t, + ); + + const alertFailed = (message: string) => { + Alert.alert( + t('settings.device_sync.failed_title', 'Sync could not be turned on'), + message, + [{ text: t('settings.device_sync.ok', 'OK') }], + ); + }; + + const alertDenied = () => { + Alert.alert( + t('settings.device_sync.denied_title', 'Sync stays off'), + isContacts + ? t('settings.device_sync.denied_contacts', 'Without access to contacts, Bulwark cannot put your address books on this device.') + : t('settings.device_sync.denied_calendar', 'Without access to calendars, Bulwark cannot put your calendars on this device.'), + [{ text: t('settings.device_sync.ok', 'OK') }], + ); + }; + + const turnOn = async () => { + setPrompt(null); + setBusy('on'); + try { + const permission = await requestSyncPermissions(authority); + if (permission === 'blocked') { + setPrompt({ kind: 'blocked' }); + return; + } + if (permission === 'denied') { + alertDenied(); + return; + } + if (authority === CALENDAR_AUTHORITY && entry.reminderOwner == null) { + // Bulwark reminds nobody while its event notifications are off: the + // calendar app does, without asking. + const owner = calendarReminders ? await askReminderOwner(t) : 'device'; + if (!owner) return; + await changeReminderOwner(account.id, owner); + } + const outcome = await enableDeviceSync(account.id, authority); + if (outcome.kind === 'permission') alertDenied(); + else if (outcome.kind === 'failed') alertFailed(outcome.message); + } finally { + setBusy(null); + void refresh(); + } + }; + + const startTurnOn = async () => { + // Explain first, unless access was granted before (another account). + if (await hasSyncPermissions(authority)) void turnOn(); + else setPrompt({ kind: 'rationale' }); + }; + + const turnOff = async (force: boolean) => { + setPrompt(null); + setBusy('off'); + try { + const result = await disableDeviceSync(account.id, authority, { force }); + if (result.done) return; + if (!force) { + setPrompt({ kind: 'pending', pending: result.pending }); + return; + } + Alert.alert( + t('settings.device_sync.turn_off_failed_title', 'Sync could not be turned off'), + t( + 'settings.device_sync.turn_off_failed', + 'The synced data could not be removed from this device. Sync is paused; try again later.', + ), + [{ text: t('settings.device_sync.ok', 'OK') }], + ); + } finally { + setBusy(null); + void refresh(); + } + }; + + const keepSyncing = async () => { + setPrompt(null); + await resumeDeviceSync(account.id, authority); + void refresh(); + }; + + const handleSyncNow = async () => { + setBusy('sync'); + try { + await syncNow(account.id, authority); + } catch (err) { + alertFailed(err instanceof Error ? err.message : String(err)); + } finally { + setBusy(null); + void refresh(); + } + }; + + const grantAccess = async () => { + const permission = await requestSyncPermissions(authority); + if (permission === 'blocked') setPrompt({ kind: 'blocked' }); + else if (permission === 'denied') alertDenied(); + else if (checked) await syncNow(account.id, authority).catch(() => false); + void refresh(); + }; + + const runAction = (action: StatusAction) => { + if (action === 'signIn') navigation.navigate('AddAccount'); + else if (action === 'grantAccess') void grantAccess(); + else void openAndroidAccountSettings(account.id); + }; + + const closeChooser = (changed: boolean) => { + setChooserOpen(false); + // A new selection is loaded (or dropped) by the next sync: run it now. + // (A sync Android paused would only report that it is off.) + if (changed && checked) void syncNow(account.id, authority).catch(() => false); + }; + + const intervalOptions = [ + { value: '900', label: t('settings.device_sync.interval_15m', 'Every 15 minutes') }, + { value: '1800', label: t('settings.device_sync.interval_30m', 'Every 30 minutes') }, + { value: '3600', label: t('settings.device_sync.interval_1h', 'Every hour') }, + { value: '21600', label: t('settings.device_sync.interval_6h', 'Every 6 hours') }, + { value: '0', label: t('settings.device_sync.interval_manual', 'No regular sync') }, + ]; + + const writableSelected = (collections ?? []).filter((col) => !col.readOnly && selectedKeys?.includes(col.key)); + const newContactsOptions = [ + { value: '', label: t('settings.device_sync.new_contacts_default', 'Default address book') }, + ...writableSelected.map((col) => ({ + value: col.key, + label: col.isPersonal ? col.name : `${col.name} (${col.accountName})`, + })), + ]; + const newContactsValue = entry.newContactsAddressBook && newContactsOptions.some((o) => o.value === entry.newContactsAddressBook) + ? entry.newContactsAddressBook + : ''; + + const collectionsDescription = collections && selectedKeys + ? t('settings.device_sync.collections_count', '{selected} of {total} sync', { + selected: selectedKeys.length, + total: collections.length, + }) + : isContacts + ? t('settings.device_sync.books_default', 'Your own address books sync; shared ones only when you pick them.') + : t('settings.device_sync.calendars_default', 'Your own calendars sync; shared ones only when you pick them.'); + + const statusStyle = status.tone === 'error' ? styles.statusError : status.tone === 'warning' ? styles.statusWarning : styles.statusMuted; + const busyText = busy === 'on' + ? t('settings.device_sync.turning_on', 'Setting up…') + : busy === 'off' + ? t('settings.device_sync.turning_off', 'Uploading changes, then removing the data from this device…') + : null; + + return ( + + + + {(busy === 'on' || busy === 'off') && } + { + if (next) void startTurnOn(); + else void turnOff(false); + }} + disabled={busy !== null} + accessibilityLabel={t('settings.device_sync.switch_label', 'Sync {account} to this device', { account: accountLabel })} + /> + + + + {!busyText && ( + + {status.tone === 'error' && } + {status.tone === 'warning' && } + {status.text} + + )} + {!busyText && status.action && ( + + + + )} + {showDetails && view && !view.masterAutomatic && ( + + {t( + 'settings.device_sync.master_off', + 'Auto-sync is off on this device, so syncs only run when you tap Sync now.', + )} + + )} + + {showDetails && ( + + + + + + {isContacts ? ( + + { void changeSyncInterval(account.id, authority, Number(v)); }} + options={intervalOptions} + /> + + + + + + + + )} + + + + { void turnOn(); }} + onCancel={() => setPrompt(null)} + /> + + { + setPrompt(null); + void openAppSystemSettings(); + }} + onCancel={() => setPrompt(null)} + /> + + 0 + ? t( + 'settings.device_sync.turn_off_pending', + "{count, plural, one {# change made on this device hasn't reached the server yet. It is lost if you turn sync off now.} other {# changes made on this device haven't reached the server yet. They are lost if you turn sync off now.}}", + { count: prompt.pending }, + ) + : t( + 'settings.device_sync.turn_off_unknown', + 'Bulwark could not check whether changes made on this device reached the server. They are lost if you turn sync off now.', + )} + confirmText={t('settings.device_sync.turn_off_anyway', 'Turn off')} + cancelText={t('settings.device_sync.keep_syncing', 'Keep syncing')} + onConfirm={() => { void turnOff(true); }} + onCancel={() => { void keepSyncing(); }} + /> + + ); +} + +function makeStyles(c: ThemePalette) { + return StyleSheet.create({ + block: { + borderBottomWidth: 1, + borderBottomColor: c.border, + paddingBottom: spacing.md, + }, + row: { flexDirection: 'row', alignItems: 'center', gap: spacing.sm }, + statusRow: { flexDirection: 'row', alignItems: 'flex-start', gap: 6 }, + statusText: { ...typography.caption, flexShrink: 1 }, + statusMuted: { color: c.mutedForeground }, + statusWarning: { color: c.warning }, + statusError: { color: c.error }, + details: { + marginTop: spacing.sm, + marginStart: spacing.sm, + paddingStart: spacing.md, + borderStartWidth: 2, + borderStartColor: c.muted, + }, + actions: { flexDirection: 'row', flexWrap: 'wrap', gap: spacing.sm, marginTop: spacing.sm }, + }); +} diff --git a/src/components/settings/device-sync/DeviceSyncCollectionsSheet.tsx b/src/components/settings/device-sync/DeviceSyncCollectionsSheet.tsx new file mode 100644 index 00000000..7f440ed1 --- /dev/null +++ b/src/components/settings/device-sync/DeviceSyncCollectionsSheet.tsx @@ -0,0 +1,310 @@ +// Which address books or calendars of one account sync to this device +// (docs/device-sync.md, "Settings sections"). The account's own collections +// are on unless turned off, shared ones off unless turned on; read-only ones +// are marked, and a collection another signed-in account already syncs gets +// a warning (it would show twice on the phone). +import React from 'react'; +import { + ActivityIndicator, Animated, Easing, Modal, Pressable, ScrollView, StyleSheet, Text, View, +} from 'react-native'; +import { useSafeAreaInsets } from 'react-native-safe-area-context'; +import { AlertTriangle, Check, X } from 'lucide-react-native'; +import Button from '../../Button'; +import { spacing, radius, typography, type ThemePalette } from '../../../theme/tokens'; +import { useColors } from '../../../theme/colors'; +import { useSheetDrag } from '../../../lib/use-sheet-drag'; +import { useLocaleStore } from '../../../stores/locale-store'; +import { useAccountStore } from '../../../stores/account-store'; +import { + isCollectionSelected, + selectionFor, + useDeviceSyncStore, + emptyAccountDeviceSync, +} from '../../../stores/device-sync-store'; +import { otherAccountsSyncing, type SyncCollection } from '../../../device-sync/app/collections'; +import { CONTACTS_AUTHORITY, type Authority } from '../../../device-sync/types'; +import { useSyncCollections } from './use-sync-collections'; + +interface Props { + visible: boolean; + /** `changed`: the selection was edited while the sheet was open. */ + onClose: (changed: boolean) => void; + registryId: string; + authority: Authority; + /** The account's address, under the title. */ + accountLabel: string; +} + +export function DeviceSyncCollectionsSheet({ visible, onClose, registryId, authority, accountLabel }: Props) { + const c = useColors(); + const styles = React.useMemo(() => makeStyles(c), [c]); + const t = useLocaleStore((s) => s.t); + const insets = useSafeAreaInsets(); + const { state, reload } = useSyncCollections(registryId, authority, visible); + const entry = useDeviceSyncStore((s) => s.accounts[registryId]); + const allEntries = useDeviceSyncStore((s) => s.accounts); + const setCollectionSelected = useDeviceSyncStore((s) => s.setCollectionSelected); + const registry = useAccountStore((s) => s.accounts); + const changed = React.useRef(false); + + const slideY = React.useRef(new Animated.Value(600)).current; + const overlayOpacity = React.useRef(new Animated.Value(0)).current; + const close = React.useCallback(() => { + const wasChanged = changed.current; + changed.current = false; + onClose(wasChanged); + }, [onClose]); + const dragHandlers = useSheetDrag({ slideY, closedY: 600, onClose: close }); + + React.useEffect(() => { + if (visible) { + changed.current = false; + Animated.parallel([ + Animated.timing(slideY, { toValue: 0, duration: 220, easing: Easing.out(Easing.cubic), useNativeDriver: true }), + Animated.timing(overlayOpacity, { toValue: 1, duration: 220, useNativeDriver: true }), + ]).start(); + } else { + Animated.parallel([ + Animated.timing(slideY, { toValue: 600, duration: 180, easing: Easing.in(Easing.cubic), useNativeDriver: true }), + Animated.timing(overlayOpacity, { toValue: 0, duration: 180, useNativeDriver: true }), + ]).start(); + } + }, [visible, slideY, overlayOpacity]); + + const selection = selectionFor(entry ?? emptyAccountDeviceSync(), authority); + const collections = state.kind === 'loaded' || state.kind === 'loading' ? state.collections ?? [] : []; + const groups = React.useMemo(() => { + const out: Array<{ jmapAccountId: string; isPersonal: boolean; accountName: string; items: SyncCollection[] }> = []; + for (const collection of collections) { + let group = out.find((g) => g.jmapAccountId === collection.jmapAccountId); + if (!group) { + group = { + jmapAccountId: collection.jmapAccountId, + isPersonal: collection.isPersonal, + accountName: collection.accountName, + items: [], + }; + out.push(group); + } + group.items.push(collection); + } + return out; + }, [collections]); + + const toggle = (collection: SyncCollection) => { + const on = isCollectionSelected(selection, authority, collection); + setCollectionSelected(registryId, authority, collection.key, !on); + changed.current = true; + }; + + const errorText = state.kind !== 'error' + ? '' + : state.error === 'signedOut' + ? t('settings.device_sync.list_error_signed_out', 'This account is not signed in on this device.') + : state.error === 'auth' + ? t('settings.device_sync.list_error_auth', 'The server did not accept the sign-in.') + : state.error === 'network' + ? t('settings.device_sync.list_error_network', 'Could not reach the server.') + : t('settings.device_sync.list_error_server', 'The server could not list them.'); + + return ( + + + + + + + + + + + + + {authority === CONTACTS_AUTHORITY + ? t('settings.device_sync.choose_books_title', 'Address books to sync') + : t('settings.device_sync.choose_calendars_title', 'Calendars to sync')} + + {accountLabel} + + + + + + + + + {state.kind === 'error' && ( + + {errorText} + + + )} + {(state.kind === 'loading' || state.kind === 'idle') && collections.length === 0 && ( + + + + )} + {state.kind === 'loaded' && collections.length === 0 && ( + + + {authority === CONTACTS_AUTHORITY + ? t('settings.device_sync.no_books', 'This account has no address books.') + : t('settings.device_sync.no_calendars', 'This account has no calendars to sync.')} + + + )} + + {groups.map((group) => ( + + + {group.isPersonal + ? t('settings.device_sync.own_collections', 'Your own') + : t('settings.device_sync.shared_by', 'Shared by {name}', { name: group.accountName })} + + {group.items.map((collection) => { + const on = isCollectionSelected(selection, authority, collection); + const others = on ? otherAccountsSyncing(registryId, authority, collection, registry, allEntries) : []; + const tint = collection.color ?? c.primary; + return ( + toggle(collection)} + style={({ pressed }) => [styles.row, pressed && styles.rowPressed]} + accessibilityRole="checkbox" + accessibilityState={{ checked: on }} + accessibilityLabel={collection.name} + > + + {on && } + + + + {collection.name} + + {(collection.readOnly || collection.isDefault) && ( + + {[ + collection.isDefault ? t('contacts.address_books.default', 'Default') : null, + collection.readOnly ? t('settings.device_sync.read_only', 'Read-only') : null, + ].filter(Boolean).join(' · ')} + + )} + {others.length > 0 && ( + + + + {t( + 'settings.device_sync.also_synced_by', + 'Also synced for {accounts}, so it shows twice on this device.', + { accounts: others.join(', ') }, + )} + + + )} + + + ); + })} + + ))} + + + {authority === CONTACTS_AUTHORITY + ? t( + 'settings.device_sync.books_hint', + 'Your own address books sync unless you turn them off; shared ones only when you turn them on.', + ) + : t( + 'settings.device_sync.calendars_hint', + 'Your own calendars sync unless you turn them off; shared ones only when you turn them on. Task lists stay in Bulwark.', + )} + + + + + ); +} + +function makeStyles(c: ThemePalette) { + return StyleSheet.create({ + overlay: { ...StyleSheet.absoluteFillObject, backgroundColor: 'rgba(0,0,0,0.5)' }, + sheet: { + position: 'absolute', + left: 0, + right: 0, + bottom: 0, + maxHeight: '80%', + backgroundColor: c.popover, + borderTopLeftRadius: radius.lg, + borderTopRightRadius: radius.lg, + borderTopWidth: 1, + borderColor: c.border, + paddingTop: spacing.sm, + }, + handleHit: { alignItems: 'center', paddingTop: spacing.xs, paddingBottom: spacing.sm }, + handle: { width: 36, height: 4, borderRadius: 2, backgroundColor: c.border }, + header: { + flexDirection: 'row', + alignItems: 'center', + gap: spacing.sm, + paddingHorizontal: spacing.lg, + paddingBottom: spacing.sm, + borderBottomWidth: 1, + borderBottomColor: c.border, + }, + title: { ...typography.bodySemibold, color: c.text }, + subtitle: { ...typography.caption, color: c.mutedForeground }, + close: { width: 28, height: 28, alignItems: 'center', justifyContent: 'center', borderRadius: radius.xs }, + body: { paddingBottom: spacing.md }, + message: { alignItems: 'center', gap: spacing.sm, padding: spacing.lg }, + errorText: { ...typography.caption, color: c.error, textAlign: 'center' }, + muted: { ...typography.caption, color: c.mutedForeground }, + group: { paddingTop: spacing.sm }, + groupTitle: { + ...typography.captionMedium, + color: c.mutedForeground, + paddingHorizontal: spacing.lg, + paddingVertical: spacing.xs, + }, + row: { + flexDirection: 'row', + alignItems: 'center', + gap: spacing.md, + paddingHorizontal: spacing.lg, + paddingVertical: spacing.sm + 2, + minHeight: 44, + }, + rowPressed: { backgroundColor: c.surfaceHover }, + box: { + width: 18, + height: 18, + borderRadius: radius.xs, + borderWidth: 2, + alignItems: 'center', + justifyContent: 'center', + }, + rowName: { ...typography.body, color: c.text }, + rowNameOff: { color: c.textSecondary }, + warningRow: { flexDirection: 'row', alignItems: 'flex-start', gap: 4, marginTop: 2 }, + warningText: { ...typography.caption, color: c.warning, flex: 1 }, + hint: { + ...typography.caption, + color: c.mutedForeground, + paddingHorizontal: spacing.lg, + paddingTop: spacing.md, + }, + }); +} diff --git a/src/components/settings/device-sync/DeviceSyncSection.tsx b/src/components/settings/device-sync/DeviceSyncSection.tsx new file mode 100644 index 00000000..46382501 --- /dev/null +++ b/src/components/settings/device-sync/DeviceSyncSection.tsx @@ -0,0 +1,118 @@ +// "Sync to this device" in Settings → Contacts and Settings → Calendar +// (Android only, #34; docs/device-sync.md "Settings sections"): one block per +// signed-in account, then the accounts the app signed out while they synced +// (their data stays until the user signs in again or removes it). The panes +// pass the title and description, so the settings search indexes them where +// they are rendered. +import React from 'react'; +import { StyleSheet, View } from 'react-native'; +import { useNavigation } from '@react-navigation/native'; +import type { NativeStackNavigationProp } from '@react-navigation/native-stack'; +import { SettingsSection, SettingItem } from '../settings-section'; +import Button from '../../Button'; +import Dialog from '../../Dialog'; +import { spacing, type ThemePalette } from '../../../theme/tokens'; +import { useColors } from '../../../theme/colors'; +import { useAccountStore } from '../../../stores/account-store'; +import { useDeviceSyncStore } from '../../../stores/device-sync-store'; +import { useLocaleStore } from '../../../stores/locale-store'; +import { removeSuspendedAccount } from '../../../device-sync/app/lifecycle'; +import type { Authority } from '../../../device-sync/types'; +import type { RootStackParamList } from '../../../navigation/types'; +import { DeviceSyncAccountRow } from './DeviceSyncAccountRow'; + +interface Props { + authority: Authority; + title: string; + description: string; +} + +export function DeviceSyncSection({ authority, title, description }: Props) { + const c = useColors(); + const styles = React.useMemo(() => makeStyles(c), [c]); + const accounts = useAccountStore((s) => s.accounts); + const syncEntries = useDeviceSyncStore((s) => s.accounts); + const suspended = React.useMemo( + () => Object.keys(syncEntries) + .filter((id) => syncEntries[id].suspended && !accounts.some((a) => a.id === id)) + .sort(), + [syncEntries, accounts], + ); + if (accounts.length === 0 && suspended.length === 0) return null; + return ( + + + {accounts.map((account) => ( + + ))} + {suspended.map((registryId) => ( + + ))} + + + ); +} + +// An account the app signed out (its sign-in failed) while it synced: sync is +// off, the data stays. Signing in again resumes it; removing drops it. +function SuspendedAccountRow({ registryId, styles }: { registryId: string; styles: ReturnType }) { + const t = useLocaleStore((s) => s.t); + const navigation = useNavigation>(); + const [confirming, setConfirming] = React.useState(false); + const [removing, setRemoving] = React.useState(false); + + const remove = async () => { + setConfirming(false); + setRemoving(true); + try { + await removeSuspendedAccount(registryId); + } finally { + setRemoving(false); + } + }; + + return ( + + + + + + + { void remove(); }} + onCancel={() => setConfirming(false)} + /> + + ); +} + +function makeStyles(c: ThemePalette) { + return StyleSheet.create({ + section: { marginTop: spacing.xxxl }, + block: { + borderBottomWidth: 1, + borderBottomColor: c.border, + paddingBottom: spacing.md, + }, + actions: { flexDirection: 'row', flexWrap: 'wrap', gap: spacing.sm }, + }); +} diff --git a/src/components/settings/device-sync/__tests__/status.test.ts b/src/components/settings/device-sync/__tests__/status.test.ts new file mode 100644 index 00000000..9de1f544 --- /dev/null +++ b/src/components/settings/device-sync/__tests__/status.test.ts @@ -0,0 +1,81 @@ +import { describe, it, expect } from 'vitest'; +import { translate, type MessageParams } from '../../../../i18n'; +import { CALENDAR_AUTHORITY, CONTACTS_AUTHORITY, type RunStatus } from '../../../../device-sync/types'; +import { describeSyncStatus, relativeTime, type StatusInput } from '../status'; + +const t = (key: string, fallback?: string, params?: MessageParams) => translate('en', key, fallback, params); + +const NOW = Date.UTC(2026, 8, 27, 12, 0, 0); + +const run = (outcome: RunStatus['outcome'], extra: Partial = {}): RunStatus => ({ + at: NOW - 5 * 60_000, + outcome, + durationMs: 100, + conflicts: 0, + itemErrors: 0, + stats: { + downloaded: { created: 0, updated: 0, deleted: 0 }, + uploaded: { created: 0, updated: 0, deleted: 0 }, + entries: 0, + skipped: 0, + }, + ...extra, +}); + +const base: StatusInput = { + authority: CONTACTS_AUTHORITY, + on: true, + paused: false, + removed: false, + permitted: true, + syncing: false, + now: NOW, +}; + +describe('describeSyncStatus', () => { + it('says off, or that the account was removed in Android settings', () => { + expect(describeSyncStatus({ ...base, on: false }, t)).toEqual({ text: 'Off', tone: 'muted' }); + expect(describeSyncStatus({ ...base, on: false, removed: true }, t).tone).toBe('warning'); + }); + + it('puts a revoked permission first, with a way to grant it again', () => { + const line = describeSyncStatus({ ...base, permitted: false, lastRun: run('ok') }, t); + expect(line).toMatchObject({ tone: 'error', action: 'grantAccess' }); + expect(line.text).toContain('contacts'); + expect(describeSyncStatus({ ...base, authority: CALENDAR_AUTHORITY, permitted: false }, t).text).toContain('calendars'); + }); + + it('says when Android paused the account', () => { + expect(describeSyncStatus({ ...base, on: false, paused: true }, t)) + .toEqual({ text: 'Paused in Android settings', tone: 'warning' }); + }); + + it('shows a running sync, then when it last synced', () => { + expect(describeSyncStatus({ ...base, syncing: true, lastRun: run('ok') }, t).text).toBe('Syncing…'); + expect(describeSyncStatus({ ...base, lastRun: run('ok') }, t)).toEqual({ text: 'Synced 5 min ago', tone: 'muted' }); + expect(describeSyncStatus({ ...base, lastRun: run('ok', { itemErrors: 2 }) }, t)).toEqual({ + text: 'Synced 5 min ago · 2 items could not be synced.', + tone: 'warning', + }); + expect(describeSyncStatus(base, t).text).toBe('Waiting for the first sync'); + }); + + it('offers what fixes a failed run', () => { + expect(describeSyncStatus({ ...base, lastRun: run('auth') }, t)).toMatchObject({ tone: 'error', action: 'signIn' }); + expect(describeSyncStatus({ ...base, lastRun: run('tooManyDeletions') }, t)).toMatchObject({ action: 'reviewDeletions' }); + expect(describeSyncStatus({ ...base, lastRun: run('permission') }, t)).toMatchObject({ action: 'grantAccess' }); + expect(describeSyncStatus({ ...base, lastRun: run('io') }, t).text).toBe('Could not reach the server 5 min ago. Android tries again.'); + expect(describeSyncStatus({ ...base, lastRun: run('internal', { message: 'Boom' }) }, t).text) + .toBe('The last sync failed: Boom'); + // A run that found sync off says nothing new. + expect(describeSyncStatus({ ...base, lastRun: run('disabled') }, t).text).toBe('Waiting for the next sync'); + }); +}); + +describe('relativeTime', () => { + it('counts minutes, hours and days', () => { + expect(relativeTime(NOW - 10_000, NOW, t)).toBe('just now'); + expect(relativeTime(NOW - 3 * 3600_000, NOW, t)).toBe('3 h ago'); + expect(relativeTime(NOW - 2 * 86_400_000, NOW, t)).toBe('2 d ago'); + }); +}); diff --git a/src/components/settings/device-sync/status.ts b/src/components/settings/device-sync/status.ts new file mode 100644 index 00000000..57d6a940 --- /dev/null +++ b/src/components/settings/device-sync/status.ts @@ -0,0 +1,133 @@ +// The status line of one account's device sync in Settings: off, paused in +// Android settings, syncing, when it last synced, or what went wrong and what +// the user can do about it (docs/device-sync.md, "Settings sections", +// "Failure matrix"). +import { CONTACTS_AUTHORITY, type Authority, type RunStatus } from '../../../device-sync/types'; +import type { TranslateFn } from '../../../stores/locale-store'; + +/** What the status line offers to do. */ +export type StatusAction = 'signIn' | 'grantAccess' | 'reviewDeletions'; + +export interface StatusLine { + text: string; + tone: 'muted' | 'warning' | 'error'; + action?: StatusAction; +} + +export interface StatusInput { + authority: Authority; + /** The switch: Android syncs this authority automatically. */ + on: boolean; + /** On in the app but off in Android's settings for the account. */ + paused: boolean; + /** The Android account was removed in Android Settings. */ + removed: boolean; + /** The runtime permissions; null while unknown. */ + permitted: boolean | null; + syncing: boolean; + lastRun?: RunStatus; + now: number; +} + +/** "just now", "5 min ago", … */ +export function relativeTime(at: number, now: number, t: TranslateFn): string { + const diff = Math.max(0, now - at); + if (diff < 60_000) return t('settings.offline.just_now', 'just now'); + if (diff < 60 * 60_000) return t('settings.offline.minutes_ago', '{count} min ago', { count: Math.round(diff / 60_000) }); + if (diff < 24 * 60 * 60_000) { + return t('settings.offline.hours_ago', '{count} h ago', { count: Math.round(diff / (60 * 60_000)) }); + } + return t('settings.offline.days_ago', '{count} d ago', { count: Math.round(diff / (24 * 60 * 60_000)) }); +} + +function permissionLine(authority: Authority, t: TranslateFn): StatusLine { + return { + text: authority === CONTACTS_AUTHORITY + ? t('settings.device_sync.status_permission_contacts', 'Access to contacts was turned off, so nothing syncs.') + : t('settings.device_sync.status_permission_calendar', 'Access to calendars was turned off, so nothing syncs.'), + tone: 'error', + action: 'grantAccess', + }; +} + +function runLine(run: RunStatus, input: StatusInput, t: TranslateFn): StatusLine | null { + const when = relativeTime(run.at, input.now, t); + switch (run.outcome) { + case 'ok': { + const synced = t('settings.device_sync.status_synced', 'Synced {when}', { when }); + if (run.itemErrors <= 0) return { text: synced, tone: 'muted' }; + const skipped = t( + 'settings.device_sync.status_item_errors', + '{count, plural, one {# item could not be synced.} other {# items could not be synced.}}', + { count: run.itemErrors }, + ); + return { text: `${synced} · ${skipped}`, tone: 'warning' }; + } + case 'auth': + return { + text: t('settings.device_sync.status_auth', 'The server no longer accepts the sign-in. Sign in again to keep syncing.'), + tone: 'error', + action: 'signIn', + }; + case 'permission': + return permissionLine(input.authority, t); + case 'tooManyDeletions': + return { + text: t( + 'settings.device_sync.status_too_many_deletions', + 'Many items were deleted on this device. They are not deleted on the server until you confirm.', + ), + tone: 'warning', + action: 'reviewDeletions', + }; + case 'io': + return { + text: t('settings.device_sync.status_io', 'Could not reach the server {when}. Android tries again.', { when }), + tone: 'warning', + }; + case 'unsupported': + return { + text: input.authority === CONTACTS_AUTHORITY + ? t('settings.device_sync.status_unsupported_contacts', 'The server offers no address books for this account.') + : t('settings.device_sync.status_unsupported_calendar', 'The server offers no calendars for this account.'), + tone: 'error', + }; + case 'safetyAbort': + return { + text: t( + 'settings.device_sync.status_safety_abort', + 'Stopped: the server returned nothing while this device still holds items. Check the server, then sync again.', + ), + tone: 'error', + }; + case 'cancelled': + return { text: t('settings.device_sync.status_cancelled', 'The last sync stopped early. It continues next time.'), tone: 'muted' }; + case 'internal': + return { + text: run.message + ? t('settings.device_sync.status_failed', 'The last sync failed: {message}', { message: run.message }) + : t('settings.device_sync.status_failed_plain', 'The last sync failed.'), + tone: 'error', + }; + case 'disabled': + default: + return null; + } +} + +export function describeSyncStatus(input: StatusInput, t: TranslateFn): StatusLine { + if (!input.on && !input.paused) { + return input.removed + ? { + text: t('settings.device_sync.status_removed', 'Removed in Android settings. Turn on to add the account again.'), + tone: 'warning', + } + : { text: t('settings.device_sync.status_off', 'Off'), tone: 'muted' }; + } + if (input.permitted === false) return permissionLine(input.authority, t); + if (input.paused) return { text: t('settings.device_sync.status_paused', 'Paused in Android settings'), tone: 'warning' }; + if (input.syncing) return { text: t('settings.device_sync.status_syncing', 'Syncing…'), tone: 'muted' }; + if (!input.lastRun) return { text: t('settings.device_sync.status_waiting', 'Waiting for the first sync'), tone: 'muted' }; + return runLine(input.lastRun, input, t) + ?? { text: t('settings.device_sync.status_waiting_next', 'Waiting for the next sync'), tone: 'muted' }; +} diff --git a/src/components/settings/device-sync/use-android-sync.ts b/src/components/settings/device-sync/use-android-sync.ts new file mode 100644 index 00000000..22f5fc75 --- /dev/null +++ b/src/components/settings/device-sync/use-android-sync.ts @@ -0,0 +1,101 @@ +// Android's side of one account's device sync for the settings: whether the +// Android account exists, its automatic sync for the authority (the switch), +// the device's master auto-sync, whether a sync runs or waits, and whether +// the runtime permissions are still granted. Re-read on mount, when the app +// returns to the foreground, when accounts of our type change, and every few +// seconds while a sync runs or waits. +import React from 'react'; +import { AppState } from 'react-native'; +import { getSyncSettings, listAndroidAccounts, onAccountsChanged } from '../../../device-sync/native'; +import { hasSyncPermissions } from '../../../device-sync/app/permissions'; +import type { Authority } from '../../../device-sync/types'; + +export interface AndroidSyncView { + /** The Android account exists. */ + exists: boolean; + /** Automatic sync for the authority: the toggle in Settings → Accounts. */ + automatic: boolean; + /** The device's "Auto-sync data". */ + masterAutomatic: boolean; + active: boolean; + pending: boolean; +} + +const POLL_MS = 3000; + +export function useAndroidSync(accountName: string | undefined, authority: Authority): { + view: AndroidSyncView | null; + permitted: boolean | null; + refresh: () => Promise; +} { + const [view, setView] = React.useState(null); + const [permitted, setPermitted] = React.useState(null); + const alive = React.useRef(true); + React.useEffect(() => { + alive.current = true; + return () => { alive.current = false; }; + }, []); + // Read at call time: turning sync on creates the account while the + // caller still holds the refresh of the render before. + const nameRef = React.useRef(accountName); + nameRef.current = accountName; + // Only the latest read may land; an older one resolving late is stale. + const reads = React.useRef(0); + + const refresh = React.useCallback(async () => { + const read = ++reads.current; + const accountName = nameRef.current; + const granted = await hasSyncPermissions(authority); + let next: AndroidSyncView = { exists: false, automatic: false, masterAutomatic: true, active: false, pending: false }; + if (accountName) { + try { + const accounts = await listAndroidAccounts(); + if (accounts.some((a) => a.name === accountName)) { + const settings = await getSyncSettings(accountName); + const own = settings.authorities?.[authority]; + next = { + exists: true, + automatic: !!own?.automatic, + masterAutomatic: settings.masterAutomatic !== false, + active: !!own?.active, + pending: !!own?.pending, + }; + } + } catch { + // Shown as off until the next read. + } + } + if (!alive.current || read !== reads.current) return; + setPermitted(granted); + setView(next); + }, [authority]); + + React.useEffect(() => { + void refresh(); + }, [refresh, accountName]); + + React.useEffect(() => { + const subscription = AppState.addEventListener('change', (state) => { + if (state === 'active') void refresh(); + }); + let unsubscribe: () => void = () => undefined; + try { + unsubscribe = onAccountsChanged(() => { void refresh(); }); + } catch { + // Foreground reads still catch up. + } + return () => { + subscription.remove(); + unsubscribe(); + }; + }, [refresh]); + + const busy = !!view?.active || !!view?.pending; + React.useEffect(() => { + if (!busy) return undefined; + const timer = setInterval(() => { void refresh(); }, POLL_MS); + return () => clearInterval(timer); + }, [busy, refresh]); + + return { view, permitted, refresh }; +} diff --git a/src/components/settings/device-sync/use-sync-collections.ts b/src/components/settings/device-sync/use-sync-collections.ts new file mode 100644 index 00000000..3f5baf6b --- /dev/null +++ b/src/components/settings/device-sync/use-sync-collections.ts @@ -0,0 +1,61 @@ +// The address books or calendars of one account for the device sync settings, +// listed through the account's own client (device-sync/app/collections) and +// cached for a little while, so the chooser, the counts and the "new contacts +// go to" picker share one request. +import React from 'react'; +import { + cachedSyncCollections, + CollectionsError, + listSyncCollections, + type CollectionsErrorKind, + type SyncCollection, +} from '../../../device-sync/app/collections'; +import { CALENDAR_AUTHORITY, type Authority } from '../../../device-sync/types'; +import { useAuthStore } from '../../../stores/auth-store'; +import { useCalendarSubscriptionsStore } from '../../../stores/calendar-subscriptions-store'; + +export type CollectionsState = + | { kind: 'idle' } + | { kind: 'loading'; collections: SyncCollection[] | null } + | { kind: 'loaded'; collections: SyncCollection[] } + | { kind: 'error'; error: CollectionsErrorKind }; + +export function useSyncCollections(registryId: string, authority: Authority, active: boolean): { + state: CollectionsState; + reload: () => void; +} { + const [state, setState] = React.useState(() => { + const cached = cachedSyncCollections(registryId, authority); + return cached ? { kind: 'loaded', collections: cached } : { kind: 'idle' }; + }); + const [attempt, setAttempt] = React.useState(0); + // Calendars mirroring subscribed feeds are read-only on the device. Only + // the active account's subscriptions are known here. + const isActive = useAuthStore((s) => s.activeAccountId === registryId); + const subscriptions = useCalendarSubscriptionsStore((s) => s.subscriptions); + const feeds = React.useMemo( + () => (authority === CALENDAR_AUTHORITY && isActive + ? subscriptions.map((sub) => ({ calendarId: sub.calendarId, accountId: sub.accountId })) + : []), + [authority, isActive, subscriptions], + ); + + React.useEffect(() => { + if (!active) return undefined; + let cancelled = false; + const force = attempt > 0; + setState((prev) => ({ kind: 'loading', collections: prev.kind === 'loaded' ? prev.collections : null })); + listSyncCollections(registryId, authority, { force, feeds }) + .then((collections) => { + if (!cancelled) setState({ kind: 'loaded', collections }); + }) + .catch((err: unknown) => { + if (cancelled) return; + setState({ kind: 'error', error: err instanceof CollectionsError ? err.kind : 'server' }); + }); + return () => { cancelled = true; }; + }, [registryId, authority, active, attempt, feeds]); + + const reload = React.useCallback(() => setAttempt((n) => n + 1), []); + return { state, reload }; +} diff --git a/src/device-sync/__tests__/app/collections.test.ts b/src/device-sync/__tests__/app/collections.test.ts new file mode 100644 index 00000000..023268f9 --- /dev/null +++ b/src/device-sync/__tests__/app/collections.test.ts @@ -0,0 +1,230 @@ +import { describe, it, expect, beforeEach } from 'vitest'; +import { + CollectionsError, + forgetSyncCollections, + listSyncCollections, + otherAccountsSyncing, + type CollectionsClient, +} from '../../app/collections'; +import { CALENDAR_AUTHORITY, CONTACTS_AUTHORITY, JMAP_CALENDARS, JMAP_CONTACTS } from '../../types'; +import { useDeviceSyncStore, waitForDeviceSyncHydration } from '../../../stores/device-sync-store'; +import type { JMAPMethodCall } from '../../../api/types'; + +type Handler = (args: Record, calls: JMAPMethodCall[]) => any; + +// A client for one account with the personal JMAP account `c` and a shared +// account `team`, answering the calls the listing makes. +function fakeClient(handlers: Record, options: { loaded?: boolean | Error; maxCalls?: number } = {}) { + const requests: JMAPMethodCall[][] = []; + const client: CollectionsClient = { + async loadAccount() { + if (options.loaded instanceof Error) throw options.loaded; + return options.loaded ?? true; + }, + get currentSession() { + return { + primaryAccounts: { [JMAP_CONTACTS]: 'c', [JMAP_CALENDARS]: 'c' }, + accounts: { + c: { name: 'alice@example.org', isPersonal: true, accountCapabilities: { [JMAP_CONTACTS]: {}, [JMAP_CALENDARS]: {} } }, + team: { name: 'Team', isPersonal: false, accountCapabilities: { [JMAP_CONTACTS]: {}, [JMAP_CALENDARS]: {} } }, + mailonly: { name: 'Archive', isPersonal: false, accountCapabilities: {} }, + }, + }; + }, + async request(calls) { + requests.push(calls); + const responses: Array<[string, any, string]> = []; + for (const [name, args, id] of calls) { + const handler = handlers[`${name} ${args.accountId}`] ?? handlers[name]; + if (!handler) { + responses.push(['error', { type: 'unknownMethod' }, id]); + continue; + } + const body = handler(args, calls); + responses.push(body === null ? ['error', { type: 'forbidden' }, id] : [name, body, id]); + } + return { methodResponses: responses }; + }, + getMaxCallsInRequest: () => options.maxCalls ?? 16, + }; + return { client, requests }; +} + +beforeEach(async () => { + await waitForDeviceSyncHydration(); + useDeviceSyncStore.setState({ accounts: {} }); + forgetSyncCollections(); +}); + +describe('listSyncCollections', () => { + it('lists the personal address books first, then the shared ones, and marks read-only ones', async () => { + const { client, requests } = fakeClient({ + 'AddressBook/get c': () => ({ list: [ + { id: 'b2', name: 'Work', sortOrder: 2, myRights: { mayWrite: true } }, + { id: 'b1', name: 'Personal', isDefault: true, sortOrder: 1, myRights: { mayWrite: true } }, + ] }), + 'AddressBook/get team': () => ({ list: [{ id: 'b1', name: 'Team contacts', myRights: { mayWrite: false } }] }), + }); + const list = await listSyncCollections('alice@x', CONTACTS_AUTHORITY, { client }); + expect(list.map((c) => [c.key, c.name, c.isPersonal, c.readOnly, c.isDefault])).toEqual([ + ['c/b1', 'Personal', true, false, true], + ['c/b2', 'Work', true, false, false], + ['team/b1', 'Team contacts', false, true, false], + ]); + expect(list[2].accountName).toBe('Team'); + // Explicit properties, the contacts capability, and never the mail-only account. + const [calls] = requests; + expect(calls.map(([name, args]) => `${name} ${args.accountId}`)).toEqual(['AddressBook/get c', 'AddressBook/get team']); + expect(calls[0][1].properties).toEqual(expect.arrayContaining(['id', 'name', 'myRights'])); + }); + + it('remembers the personal JMAP account for the push routes', async () => { + const { client } = fakeClient({ 'AddressBook/get': () => ({ list: [] }) }); + await listSyncCollections('alice@x', CONTACTS_AUTHORITY, { client }); + expect(useDeviceSyncStore.getState().accounts['alice@x']?.primaryJmapAccounts).toEqual({ [CONTACTS_AUTHORITY]: 'c' }); + }); + + it('keeps the list when a shared account refuses', async () => { + const { client } = fakeClient({ + 'AddressBook/get c': () => ({ list: [{ id: 'b1', name: 'Personal' }] }), + 'AddressBook/get team': () => null, + }); + const list = await listSyncCollections('alice@x', CONTACTS_AUTHORITY, { client }); + expect(list.map((c) => c.key)).toEqual(['c/b1']); + }); + + it('leaves out calendars it may not read and calendars holding only tasks', async () => { + // What each calendar's scan finds: `todo` holds only tasks, `empty` nothing. + const scans: Record = { + cal1: [{ id: 'e', '@type': 'Event' }], + todo: [{ id: 't', '@type': 'Task' }, { id: 'u', due: '2026-10-01T00:00:00' }], + empty: [], + }; + const { client } = fakeClient({ + 'Calendar/get c': () => ({ list: [ + { id: 'cal1', name: 'Personal', color: '#ff0000', myRights: { mayReadItems: true, mayWriteAll: true } }, + { id: 'todo', name: 'Tasks', myRights: { mayReadItems: true, mayWriteAll: true } }, + { id: 'empty', name: 'New', myRights: { mayReadItems: true, mayWriteAll: true } }, + { id: 'hidden', name: 'Free/busy only', myRights: { mayReadItems: false } }, + ] }), + 'Calendar/get team': () => ({ list: [ + { id: 'cal1', name: 'Team', myRights: { mayReadItems: true, mayWriteAll: false, mayWriteOwn: false } }, + ] }), + 'CalendarEvent/query': (args) => ({ ids: (scans[args.filter.inCalendar] ?? []).map((o) => o.id) }), + 'CalendarEvent/get': (args, calls) => { + // The get reads the ids of the query it references. + const query = calls.find(([, , id]) => id === args['#ids'].resultOf)!; + return { list: scans[(query[1].filter as { inCalendar: string }).inCalendar] ?? [] }; + }, + }); + const list = await listSyncCollections('alice@x', CALENDAR_AUTHORITY, { client }); + // By name within an account (no sort order given). + expect(list.map((c) => [c.key, c.readOnly])).toEqual([ + ['c/empty', false], + ['c/cal1', false], + ['team/cal1', true], + ]); + expect(list.find((c) => c.key === 'c/cal1')?.color).toBe('#ff0000'); + }); + + it('marks calendars mirroring a subscribed feed read-only', async () => { + const { client } = fakeClient({ + 'Calendar/get c': () => ({ list: [ + { id: 'feed', name: 'Holidays', myRights: { mayReadItems: true, mayWriteAll: true } }, + ] }), + 'Calendar/get team': () => ({ list: [] }), + 'CalendarEvent/query': () => ({ ids: [] }), + 'CalendarEvent/get': () => ({ list: [] }), + }); + const list = await listSyncCollections('alice@x', CALENDAR_AUTHORITY, { client, feeds: [{ calendarId: 'feed' }] }); + expect(list.map((c) => [c.key, c.readOnly])).toEqual([['c/feed', true]]); + }); + + it('keeps a query and the get that reads it in one request', async () => { + const { client, requests } = fakeClient({ + 'Calendar/get': () => ({ list: [ + { id: 'a', name: 'A' }, { id: 'b', name: 'B' }, { id: 'c', name: 'C' }, + ] }), + 'CalendarEvent/query': () => ({ ids: [] }), + 'CalendarEvent/get': () => ({ list: [] }), + }, { maxCalls: 5 }); + await listSyncCollections('alice@x', CALENDAR_AUTHORITY, { client }); + const scanRequests = requests.filter((calls) => calls[0][0] === 'CalendarEvent/query'); + for (const calls of scanRequests) { + expect(calls.length % 2).toBe(0); + expect(calls.length).toBeLessThanOrEqual(5); + } + }); + + it('says why the list could not be loaded', async () => { + const signedOut = fakeClient({}, { loaded: false }).client; + await expect(listSyncCollections('alice@x', CONTACTS_AUTHORITY, { client: signedOut })) + .rejects.toMatchObject({ kind: 'signedOut' }); + const auth = Object.assign(new Error('Invalid credentials'), { name: 'AuthenticationError' }); + await expect(listSyncCollections('alice@x', CONTACTS_AUTHORITY, { client: fakeClient({}, { loaded: auth }).client })) + .rejects.toBeInstanceOf(CollectionsError); + await expect(listSyncCollections('alice@x', CONTACTS_AUTHORITY, { client: fakeClient({}, { loaded: auth }).client })) + .rejects.toMatchObject({ kind: 'auth' }); + const offline = Object.assign(new Error('Server unreachable'), { name: 'NetworkError' }); + await expect(listSyncCollections('alice@x', CONTACTS_AUTHORITY, { client: fakeClient({}, { loaded: offline }).client })) + .rejects.toMatchObject({ kind: 'network' }); + }); +}); + +describe('otherAccountsSyncing', () => { + const registry = [ + { id: 'alice@mail.example.org', serverUrl: 'https://mail.example.org', email: 'alice@example.org', username: 'alice' }, + { id: 'bob@mail.example.org', serverUrl: 'https://mail.example.org/', email: 'bob@example.org', username: 'bob' }, + { id: 'bob@other.example.net', serverUrl: 'https://other.example.net', email: 'bob@example.net', username: 'bob' }, + ]; + + it('names the other accounts on the same server that sync the same collection', () => { + const accounts = { + 'bob@mail.example.org': { + contactsSelection: { 'team/b1': true }, + calendarSelection: {}, + reminderOwner: null, + intervalSeconds: {}, + lastRun: {}, + enabled: { [CONTACTS_AUTHORITY]: true }, + }, + // Another server: the same JMAP ids mean other data. + 'bob@other.example.net': { + contactsSelection: { 'team/b1': true }, + calendarSelection: {}, + reminderOwner: null, + intervalSeconds: {}, + lastRun: {}, + enabled: { [CONTACTS_AUTHORITY]: true }, + }, + }; + const collection = { key: 'team/b1', jmapAccountId: 'team' }; + expect(otherAccountsSyncing('alice@mail.example.org', CONTACTS_AUTHORITY, collection, registry, accounts)) + .toEqual(['bob@example.org']); + // Not for another authority, nor once Bob's contacts sync is off. + expect(otherAccountsSyncing('alice@mail.example.org', CALENDAR_AUTHORITY, collection, registry, accounts)).toEqual([]); + accounts['bob@mail.example.org'].enabled = { [CONTACTS_AUTHORITY]: false }; + expect(otherAccountsSyncing('alice@mail.example.org', CONTACTS_AUTHORITY, collection, registry, accounts)).toEqual([]); + }); + + it("counts the other account's own collections, which sync by default", () => { + const accounts = { + 'bob@mail.example.org': { + contactsSelection: {}, + calendarSelection: {}, + reminderOwner: null, + intervalSeconds: {}, + lastRun: {}, + enabled: { [CALENDAR_AUTHORITY]: true }, + primaryJmapAccounts: { [CALENDAR_AUTHORITY]: 'bobacct' }, + }, + }; + expect(otherAccountsSyncing( + 'alice@mail.example.org', + CALENDAR_AUTHORITY, + { key: 'bobacct/cal1', jmapAccountId: 'bobacct' }, + registry, + accounts, + )).toEqual(['bob@example.org']); + }); +}); diff --git a/src/device-sync/app/collections.ts b/src/device-sync/app/collections.ts new file mode 100644 index 00000000..923faf4b --- /dev/null +++ b/src/device-sync/app/collections.ts @@ -0,0 +1,334 @@ +/** + * The address books and calendars an app account can sync, for the chooser in + * Settings (docs/device-sync.md, "Settings sections"). + * + * Listed through a JMAPClient of the account's own, never the UI's + * singleton: the chooser covers every signed-in account, and re-binding the + * singleton would send the UI's requests with another account's credentials. + * Collections of the capability's primary account are "personal" (on by + * default); every other session account that advertises the capability holds + * shared ones (off by default). Calendars without `mayReadItems` and + * calendars that hold only tasks are not offered (the engine does not sync + * them); the birthday calendar is the app's own and never on the server. + */ +import { AuthenticationError, JMAPClient, NetworkError } from '../../api/jmap-client'; +import type { JMAPMethodCall } from '../../api/types'; +import { + findTasksOnlyCalendarIds, + SCAN_PROPERTIES, + type ScannedCalendarObject, +} from '../../lib/calendar-component-detection'; +import { + isCollectionSelected, + selectionFor, + syncOnInApp, + useDeviceSyncStore, + type AccountDeviceSync, +} from '../../stores/device-sync-store'; +import { collectionKey } from '../common/ids'; +import { + CALENDAR_AUTHORITY, + CONTACTS_AUTHORITY, + JMAP_CALENDARS, + JMAP_CONTACTS, + JMAP_CORE, + type Authority, + type CollectionKey, +} from '../types'; + +export interface SyncCollection { + /** `/`, as in the selections. */ + key: CollectionKey; + jmapAccountId: string; + id: string; + name: string; + /** The JMAP account's name (who shares it, for a shared collection). */ + accountName: string; + /** In the capability's primary account: synced unless the user turned it off. */ + isPersonal: boolean; + /** No write rights, or (calendars) a subscribed feed. */ + readOnly: boolean; + isDefault: boolean; + color?: string; +} + +/** Why the list could not be loaded. */ +export type CollectionsErrorKind = 'signedOut' | 'auth' | 'network' | 'server'; + +export class CollectionsError extends Error { + constructor(readonly kind: CollectionsErrorKind, message: string) { + super(message); + this.name = 'CollectionsError'; + } +} + +/** The part of JMAPClient the listing uses. */ +export interface CollectionsClient { + loadAccount(registryId: string): Promise; + readonly currentSession: { + primaryAccounts: Record; + accounts: Record }>; + } | null; + request(calls: JMAPMethodCall[], using?: string[]): Promise<{ methodResponses: Array<[string, any, string]> }>; + getMaxCallsInRequest(): number; +} + +export interface ListOptions { + /** Skip the cache. */ + force?: boolean; + /** For tests: the client to list with. */ + client?: CollectionsClient; + /** Calendars mirroring subscribed iCal feeds (read-only on the device); `accountId` absent = the primary account. */ + feeds?: ReadonlyArray<{ calendarId: string; accountId?: string }>; +} + +const CACHE_TTL_MS = 2 * 60_000; +/** How many objects of a calendar tell whether it holds only tasks (the engine checks the same). */ +const TASK_SCAN_LIMIT = 50; + +const ADDRESS_BOOK_PROPERTIES = ['id', 'name', 'isDefault', 'sortOrder', 'myRights']; +const CALENDAR_PROPERTIES = ['id', 'name', 'color', 'isDefault', 'sortOrder', 'isSubscribed', 'myRights']; + +const cache = new Map(); +const inFlight = new Map>(); + +const cacheKey = (registryId: string, authority: Authority) => `${registryId}\n${authority}`; + +/** The last list loaded for an account, if it is still fresh. */ +export function cachedSyncCollections(registryId: string, authority: Authority): SyncCollection[] | null { + const hit = cache.get(cacheKey(registryId, authority)); + return hit && Date.now() - hit.at < CACHE_TTL_MS ? hit.collections : null; +} + +/** Drops the cached lists of one account, or of all (tests). */ +export function forgetSyncCollections(registryId?: string): void { + for (const key of [...cache.keys()]) { + if (!registryId || key.startsWith(`${registryId}\n`)) cache.delete(key); + } +} + +/** The collections of one app account for an authority, personal ones first. */ +export function listSyncCollections( + registryId: string, + authority: Authority, + options: ListOptions = {}, +): Promise { + const key = cacheKey(registryId, authority); + if (!options.force && !options.client) { + const hit = cachedSyncCollections(registryId, authority); + if (hit) return Promise.resolve(hit); + const running = inFlight.get(key); + if (running) return running; + } + const run = load(registryId, authority, options).then((collections) => { + cache.set(key, { at: Date.now(), collections }); + return collections; + }); + if (!options.client) { + inFlight.set(key, run); + void run.finally(() => { + if (inFlight.get(key) === run) inFlight.delete(key); + }).catch(() => undefined); + } + return run; +} + +async function openClient(registryId: string, client: CollectionsClient): Promise { + let ok: boolean; + try { + ok = await client.loadAccount(registryId); + } catch (err) { + if (err instanceof AuthenticationError || (err instanceof Error && err.name === 'AuthenticationError')) { + throw new CollectionsError('auth', err.message); + } + if (err instanceof NetworkError || (err instanceof Error && err.name === 'NetworkError')) { + throw new CollectionsError('network', err.message); + } + throw new CollectionsError('server', err instanceof Error ? err.message : String(err)); + } + if (!ok) throw new CollectionsError('signedOut', 'No stored credentials'); + return client; +} + +async function send( + client: CollectionsClient, + calls: JMAPMethodCall[], + using: string[], +): Promise> { + const out: Array<[string, any, string]> = []; + // Result references only point inside one request, so a query and the + // get that reads it always go together. + const perRequest = Math.max(2, client.getMaxCallsInRequest() - (client.getMaxCallsInRequest() % 2)); + for (let i = 0; i < calls.length; i += perRequest) { + try { + const res = await client.request(calls.slice(i, i + perRequest), using); + out.push(...(res.methodResponses ?? [])); + } catch (err) { + if (err instanceof Error && err.name === 'AuthenticationError') throw new CollectionsError('auth', err.message); + throw new CollectionsError('network', err instanceof Error ? err.message : String(err)); + } + } + return out; +} + +function responseOf(responses: Array<[string, any, string]>, callId: string): any { + const hit = responses.find((r) => r[2] === callId); + return hit && hit[0] !== 'error' ? hit[1] : null; +} + +interface RawCollection { + id: string; + name?: string; + color?: string; + isDefault?: boolean; + sortOrder?: number; + myRights?: Record; +} + +async function load(registryId: string, authority: Authority, options: ListOptions): Promise { + const client = await openClient(registryId, options.client ?? new JMAPClient()); + const session = client.currentSession; + if (!session) throw new CollectionsError('server', 'No session'); + const capability = authority === CONTACTS_AUTHORITY ? JMAP_CONTACTS : JMAP_CALENDARS; + const using = [JMAP_CORE, capability]; + const capable = Object.entries(session.accounts ?? {}) + .filter(([, info]) => !!info?.accountCapabilities?.[capability]) + .map(([id]) => id); + const primary = session.primaryAccounts?.[capability] ?? capable[0]; + if (!primary) return []; + const accountIds = [primary, ...capable.filter((id) => id !== primary)]; + // For the push routes and the trigger filter; only into a store that has + // been read, which would otherwise be overwritten. + if (useDeviceSyncStore.persist.hasHydrated() + && useDeviceSyncStore.getState().accounts[registryId]?.primaryJmapAccounts?.[authority] !== primary) { + useDeviceSyncStore.getState().rememberPrimaryJmapAccount(registryId, authority, primary); + } + + const method = authority === CONTACTS_AUTHORITY ? 'AddressBook/get' : 'Calendar/get'; + const properties = authority === CONTACTS_AUTHORITY ? ADDRESS_BOOK_PROPERTIES : CALENDAR_PROPERTIES; + const responses = await send( + client, + accountIds.map((accountId, i) => [method, { accountId, properties }, `g${i}`]), + using, + ); + + const listed: Array<{ jmapAccountId: string; raw: RawCollection }> = []; + accountIds.forEach((jmapAccountId, i) => { + const body = responseOf(responses, `g${i}`); + // An account that refuses (a revoked share) must not hide the others. + if (!body) { + if (jmapAccountId === primary) throw new CollectionsError('server', `${method} failed`); + return; + } + for (const raw of (body.list ?? []) as RawCollection[]) { + if (raw && typeof raw.id === 'string') listed.push({ jmapAccountId, raw }); + } + }); + + let offered = listed; + if (authority === CALENDAR_AUTHORITY) { + offered = listed.filter(({ raw }) => raw.myRights?.mayReadItems !== false); + const taskOnly = await taskOnlyCalendars(client, offered, using); + offered = offered.filter(({ jmapAccountId, raw }) => !taskOnly.has(collectionKey(jmapAccountId, raw.id))); + } + + const feeds = new Set((options.feeds ?? []).map((f) => collectionKey(f.accountId ?? primary, f.calendarId))); + const collections = offered.map(({ jmapAccountId, raw }): SyncCollection => { + const key = collectionKey(jmapAccountId, raw.id); + const rights = raw.myRights; + const readOnly = authority === CONTACTS_AUTHORITY + ? rights?.mayWrite === false + : feeds.has(key) || (!!rights && !rights.mayWriteAll && !rights.mayWriteOwn); + return { + key, + jmapAccountId, + id: raw.id, + name: raw.name?.trim() || raw.id, + accountName: session.accounts?.[jmapAccountId]?.name ?? jmapAccountId, + isPersonal: jmapAccountId === primary, + readOnly, + isDefault: !!raw.isDefault, + ...(typeof raw.color === 'string' && raw.color ? { color: raw.color } : {}), + }; + }); + const order = new Map(accountIds.map((id, i) => [id, i])); + const sortOf = new Map(offered.map(({ jmapAccountId, raw }) => [collectionKey(jmapAccountId, raw.id), raw.sortOrder ?? 0])); + return collections.sort((a, b) => + (order.get(a.jmapAccountId)! - order.get(b.jmapAccountId)!) + || (sortOf.get(a.key)! - sortOf.get(b.key)!) + || a.name.localeCompare(b.name)); +} + +/** + * Calendars whose first objects are all tasks. One query and one get per + * calendar; a calendar whose scan fails is kept (offered). + */ +async function taskOnlyCalendars( + client: CollectionsClient, + calendars: Array<{ jmapAccountId: string; raw: RawCollection }>, + using: string[], +): Promise> { + const calls: JMAPMethodCall[] = []; + calendars.forEach(({ jmapAccountId, raw }, i) => { + calls.push(['CalendarEvent/query', { accountId: jmapAccountId, filter: { inCalendar: raw.id }, limit: TASK_SCAN_LIMIT }, `q${i}`]); + calls.push([ + 'CalendarEvent/get', + { + accountId: jmapAccountId, + '#ids': { resultOf: `q${i}`, name: 'CalendarEvent/query', path: '/ids' }, + properties: SCAN_PROPERTIES, + }, + `o${i}`, + ]); + }); + const out = new Set(); + if (calls.length === 0) return out; + let responses: Array<[string, any, string]>; + try { + responses = await send(client, calls, using); + } catch { + return out; + } + calendars.forEach(({ jmapAccountId, raw }, i) => { + const body = responseOf(responses, `o${i}`); + if (!body) return; + const objects = ((body.list ?? []) as ScannedCalendarObject[]) + .map((o) => ({ ...o, calendarIds: { [raw.id]: true } })); + if (findTasksOnlyCalendarIds(objects, [raw.id]).has(raw.id)) out.add(collectionKey(jmapAccountId, raw.id)); + }); + return out; +} + +function serverOf(serverUrl: string): string { + try { + return new URL(serverUrl).host.toLowerCase(); + } catch { + return serverUrl.replace(/^https?:\/\//i, '').replace(/\/.*$/, '').toLowerCase(); + } +} + +/** + * The other signed-in accounts on the same server that already sync this + * collection (the same JMAP account and id): the chooser warns about the + * second copy on the device, it does not refuse it. + */ +export function otherAccountsSyncing( + registryId: string, + authority: Authority, + collection: Pick & Partial>, + registry: ReadonlyArray<{ id: string; serverUrl: string; email?: string; username: string }>, + accounts: Record, +): string[] { + const self = registry.find((a) => a.id === registryId); + if (!self) return []; + const server = serverOf(self.serverUrl); + return registry + .filter((other) => other.id !== registryId && serverOf(other.serverUrl) === server) + .filter((other) => { + const entry = accounts[other.id]; + if (!entry || !syncOnInApp(entry, authority)) return false; + const isPersonal = entry.primaryJmapAccounts?.[authority] === collection.jmapAccountId; + return isCollectionSelected(selectionFor(entry, authority), authority, { ...collection, isPersonal }); + }) + .map((other) => other.email || other.username); +} diff --git a/src/lib/__tests__/settings-search.test.ts b/src/lib/__tests__/settings-search.test.ts index d2ceb492..0b08cb57 100644 --- a/src/lib/__tests__/settings-search.test.ts +++ b/src/lib/__tests__/settings-search.test.ts @@ -93,6 +93,7 @@ describe('settings search index (English catalog)', () => { downloads: 'DownloadsSettings.tsx', security: 'AccountSecuritySettings.tsx', calendar: 'CalendarSettings.tsx', + contacts: 'ContactsSettings.tsx', files: 'FilesSettings.tsx', sidebar_apps: 'SidebarAppsSettings.tsx', about_data: 'AboutDataSettings.tsx', @@ -120,6 +121,20 @@ describe('settings search index (English catalog)', () => { expect(labelsFor('calendar')).not.toContain('Free scrolling'); expect(labelsFor('notifications')).not.toContain('Notification sound'); }); + + it('finds device sync in Contacts and Calendar on Android (#34)', () => { + // The tests run as Android (src/test-setup.ts). + expect(labelsFor('contacts')).toContain('Sync to this device'); + expect(labelsFor('calendar')).toContain('Sync to this device'); + const index = buildSettingsSearchIndex(en, tEn); + expect(subResultsForQuery(index, 'contacts', 'contacts app')).toEqual([ + { + label: 'Sync to this device', + description: 'Show your address books in the Contacts app and other apps on this device. Changes sync both ways.', + }, + ]); + expect(tabMatchesQuery(index, 'calendar', 'Calendar', 'android')).toBe(true); + }); }); describe('buildSettingsSearchIndex', () => { diff --git a/src/lib/settings-search.ts b/src/lib/settings-search.ts index c8bdb69a..86c0f776 100644 --- a/src/lib/settings-search.ts +++ b/src/lib/settings-search.ts @@ -11,6 +11,8 @@ // through t() for the current locale, with the English fallback. // Kept free of React so it can be unit-tested. +import { supportsDeviceSync } from './platform-capabilities'; + export type SettingsTabId = | 'account' | 'language' | 'notifications' | 'appearance' | 'layout' @@ -166,6 +168,8 @@ export const SETTINGS_SEARCH_PATHS: Record = { 'calendar.settings.show_birthday_calendar', 'calendar.settings.enable_tasks', 'calendar.settings.show_tasks_on_calendar', + // Android only: calendars in the phone's Calendar app (#34). + ...(supportsDeviceSync ? ['calendar.settings.device_sync_title'] : []), ], contacts: [ 'settings.contacts.group_by_letter_label', @@ -173,6 +177,8 @@ export const SETTINGS_SEARCH_PATHS: Record = { 'settings.contacts.import_label', 'settings.contacts.export_label', 'settings.contacts.manage_title', + // Android only: address books in the phone's Contacts app (#34). + ...(supportsDeviceSync ? ['settings.contacts.device_sync_title'] : []), ], files: [ 'files.settings_display', @@ -238,8 +244,8 @@ export const SETTINGS_SEARCH_KEYWORDS: Record = { security: 'password 2fa two-factor totp app password mfa api key certificate encryption', encryption: 's/mime smime certificate sign encrypt', content_senders: 'block sender remote images privacy tracking', - calendar: 'event schedule appointment meeting timezone', - contacts: 'address book contact', + calendar: `event schedule appointment meeting timezone${supportsDeviceSync ? ' android phone device sync' : ''}`, + contacts: `address book contact${supportsDeviceSync ? ' android phone device sync' : ''}`, files: 'attachments cloud drive storage upload', sidebar_apps: 'apps webview iframe', about_data: 'export import backup offline cache debug logs reset version', From 417ed5a326dbef66ee49314bc8e31db200f0e4ee Mon Sep 17 00:00:00 2001 From: Linus Rath <139418639+rathlinus@users.noreply.github.com> Date: Sun, 27 Sep 2026 05:27:13 +0200 Subject: [PATCH 17/87] fix: log why a device sync run failed #34 The sync adapter logged only the SyncResult counters of a run, so a failed sync showed up in logcat as `ioErrors=1` and nothing else. The reason was in the run's report, and JS warnings from a headless run don't reach logcat; finding out that a run failed on "Invalid column raw_contact_is_read_only" took reading the app's storage. The log line now carries the report's outcome, its message (up to 300 characters) and the number of failed items. Checked: 55 JVM tests (`gradlew :app:testDebugUnitTest`, one new) and the debug build. --- .../bulwarkmobile/sync/DeviceSyncAdapter.kt | 2 +- .../bulwarkmobile/sync/SyncReports.kt | 22 +++++++++++++++++++ .../bulwarkmobile/sync/SyncReportsTest.kt | 15 +++++++++++++ 3 files changed, 38 insertions(+), 1 deletion(-) diff --git a/android/app/src/main/java/com/anonymous/bulwarkmobile/sync/DeviceSyncAdapter.kt b/android/app/src/main/java/com/anonymous/bulwarkmobile/sync/DeviceSyncAdapter.kt index aa4b851d..8df852ae 100644 --- a/android/app/src/main/java/com/anonymous/bulwarkmobile/sync/DeviceSyncAdapter.kt +++ b/android/app/src/main/java/com/anonymous/bulwarkmobile/sync/DeviceSyncAdapter.kt @@ -91,7 +91,7 @@ class DeviceSyncAdapter(context: Context) : val outcome = SyncReports.outcome(run.report, run.endReason) SyncReports.apply(outcome, syncResult) Log.i(TAG, "Sync ${run.runId} of $authority ended (${run.endReason}) after " + - "${SystemClock.elapsedRealtime() - startedAt} ms: $outcome") + "${SystemClock.elapsedRealtime() - startedAt} ms: ${SyncReports.summary(run.report)}; $outcome") } catch (e: InterruptedException) { // onSyncCanceled interrupts this thread. JS sees the run as // cancelled at its next checkpoint (isRunCancelled). diff --git a/android/app/src/main/java/com/anonymous/bulwarkmobile/sync/SyncReports.kt b/android/app/src/main/java/com/anonymous/bulwarkmobile/sync/SyncReports.kt index 280523ef..37e6dc38 100644 --- a/android/app/src/main/java/com/anonymous/bulwarkmobile/sync/SyncReports.kt +++ b/android/app/src/main/java/com/anonymous/bulwarkmobile/sync/SyncReports.kt @@ -74,6 +74,28 @@ object SyncReports { } } + /** + * One line for logcat: the report's outcome, its message and how many + * items failed. The reason a run failed is otherwise only in the report, + * and JS warnings from a headless run don't reach logcat. + */ + fun summary(reportJson: String?): String { + if (reportJson == null) return "no report" + val report = try { + JSONObject(reportJson) + } catch (e: JSONException) { + return "unreadable report" + } + val parts = mutableListOf("outcome=" + report.optString("outcome", "?")) + val items = report.optJSONArray("itemErrors")?.length() ?: 0 + if (items > 0) parts += "itemErrors=$items" + val message = report.optString("message", "") + if (message.isNotEmpty()) parts += "message=" + message.take(MAX_LOGGED_MESSAGE) + return parts.joinToString(" ") + } + + private const val MAX_LOGGED_MESSAGE = 300 + fun apply(outcome: SyncOutcome, result: SyncResult) { result.stats.numIoExceptions += outcome.ioErrors result.stats.numAuthExceptions += outcome.authErrors diff --git a/android/app/src/test/java/com/anonymous/bulwarkmobile/sync/SyncReportsTest.kt b/android/app/src/test/java/com/anonymous/bulwarkmobile/sync/SyncReportsTest.kt index 273e53b8..3594a338 100644 --- a/android/app/src/test/java/com/anonymous/bulwarkmobile/sync/SyncReportsTest.kt +++ b/android/app/src/test/java/com/anonymous/bulwarkmobile/sync/SyncReportsTest.kt @@ -81,4 +81,19 @@ class SyncReportsTest { fun `an outcome this build does not know is retried`() { assertEquals(1L, SyncReports.outcome(report("somethingNew"), "reported").ioErrors) } + + @Test + fun `the log summary names the outcome, failed items and the message`() { + assertEquals("outcome=ok", SyncReports.summary(report("ok"))) + assertEquals( + "outcome=internal message=Invalid column x", + SyncReports.summary(report("internal", ",\"message\":\"Invalid column x\"")), + ) + val withItems = report("ok").replace("\"itemErrors\":[]", "\"itemErrors\":[{\"ref\":\"a\"},{\"ref\":\"b\"}]") + assertEquals("outcome=ok itemErrors=2", SyncReports.summary(withItems)) + assertEquals("no report", SyncReports.summary(null)) + assertEquals("unreadable report", SyncReports.summary("{")) + val long = SyncReports.summary(report("io", ",\"message\":\"" + "m".repeat(400) + "\"")) + assertEquals("outcome=io message=".length + 300, long.length) + } } From d135bb24492f2e93894f9a02aaf07f68b7f53960 Mon Sep 17 00:00:00 2001 From: Linus Rath <139418639+rathlinus@users.noreply.github.com> Date: Sun, 27 Sep 2026 05:41:26 +0200 Subject: [PATCH 18/87] docs: describe device sync as it was built #34 The design in f666699 was written before the code, and building it changed a number of details, several of them after runs against the real providers on the emulators. The doc now says what the code does: - a sync run's steps in the order they run, the stale list, the reconcile cursor, the owner check, the time margins, and accounts that are left out when they refuse to list their collections; - uploads in phases across accounts, the `ifInState` catch-up, account refusals, series splits and copied events, creates adopting only in their target collection, and poison markers; - the contacts and calendar mappings as built (membership rows, text limits, postal and relation rules, read-only rows and calendars, exceptions, reminders under the old owner), and the columns the providers refuse to return; - the native module's scoping, the app's store, lifecycle and triggers, and the failure matrix; - decisions and limitations, with the questions still open. Checked: every statement against the code of this branch. --- docs/device-sync.md | 636 ++++++++++++++++++++++++++++++-------------- 1 file changed, 437 insertions(+), 199 deletions(-) diff --git a/docs/device-sync.md b/docs/device-sync.md index 16a0e106..ef5ad0dc 100644 --- a/docs/device-sync.md +++ b/docs/device-sync.md @@ -60,8 +60,9 @@ Why this split: second owner outside JS's de-duplication could evict the account. - Mappers and merge rules are pure TypeScript with no React Native import, so vitest covers them with golden fixtures and property tests. A test walks the - import graph of `src/device-sync/{contacts,calendar,engine}` and fails on - `react-native`, expo, stores or `jmap-client`: the global mocks in + import graph of every module in `src/device-sync` except the React Native + boundary (`native.ts`, `task.ts`, `jmap/client-port.ts` and `app/`) and + fails on `react-native`, expo, stores or `jmap-client`: the global mocks in `src/test-setup.ts` would otherwise hide such an import. - Kotlin only translates: URIs, `ContentProviderOperation`s, `SyncResult`. Its logic has JVM unit tests (`android/app/src/test`). @@ -87,15 +88,17 @@ mutex coordinates runs with app-side operations. | `android/app/src/main/res/xml/{authenticator,sync_contacts,sync_calendar,contacts}.xml` | framework metadata | | `android/app/src/debug/java/…/sync/DeviceSyncDebugReceiver.kt` | adb hooks for device tests (debug builds only) | | `src/device-sync/types.ts`, `android-columns.ts` | contracts: bridge API, row model, ops, payload, report; provider column names | +| `src/device-sync/planner.ts`, `wire.ts` | the contract between the engine and the two planners; JMAP objects as Stalwart sends them | | `src/device-sync/native.ts` | typed wrapper around `NativeModules.BulwarkDeviceSync` | -| `src/device-sync/common/` | pure helpers: canonical JSON, hashing, JSON Pointer patches, zoned time, durations | +| `src/device-sync/common/` | pure helpers: identities and minted keys, canonical JSON, hashing, JSON Pointer patches | | `src/device-sync/contacts/` | JSContact ↔ contacts rows, row matching, contact merge rules (pure) | -| `src/device-sync/calendar/` | JSCalendar ↔ calendar rows, RRULE, exceptions, merge rules (pure) | -| `src/device-sync/jmap/` | the JMAP side on a detached client: `/changes`, full reconcile, `/set`, uid lookup | -| `src/device-sync/engine/` | orchestration: run entry, mutex, checkpoints, batches, SyncState, report | -| `src/device-sync/task.ts` | the headless task entry (registered in `index.ts`) | +| `src/device-sync/calendar/` | JSCalendar ↔ calendar rows, RRULE, exceptions, zoned time, durations, merge rules (pure) | +| `src/device-sync/jmap/` | the JMAP side on a detached client: session accounts, `/get`, `/changes`, `/query` paging, `/set`, uid lookup, the auth rebuild | +| `src/device-sync/engine/` | orchestration: run entry, mutex, checkpoints, batches, SyncState, poison markers, default selection, teardown, report | +| `src/device-sync/task.ts` | the headless task entry (registered in `index.ts`) and the teardown the app calls | +| `src/device-sync/app/` | the app side: turning sync on and off, sign-out, reconciling Android's accounts, triggers, the chooser's collection list, the reminder filter | | `src/stores/device-sync-store.ts` | per-account preferences and the last run status | -| `src/components/settings/ContactsSettings.tsx`, `CalendarSettings.tsx` | the "Sync to this device" sections | +| `src/components/settings/device-sync/` | the "Sync to this device" sections, rendered by `ContactsSettings.tsx` and `CalendarSettings.tsx` | ## The Android shell @@ -236,24 +239,26 @@ marker: raw contacts of a sync account are always soft-deleted. A **new** local item is a raw contact without `SOURCE_ID`, or a master event (`ORIGINAL_ID` and `ORIGINAL_SYNC_ID` both null) whose `_SYNC_ID` is null or -pending. DIRTY is not the test: a raw contact inserted without data rows is not -dirty. A row with `ORIGINAL_*` set is always an exception of its master and -never uploads as an event of its own. +pending, or the newer of two masters sharing a `_SYNC_ID` (a +`CONTENT_EXCEPTION_URI` split, see [Recurrence](#recurrence)). DIRTY is not +the test: a raw contact inserted without data rows is not dirty. A row with +`ORIGINAL_*` set is always an exception of its master and never uploads as an +event of its own. ### Contacts columns | Column | Holds | |---|---| | `RawContacts.SOURCE_ID` | identity (above) | -| `RawContacts.SYNC1` | the card's collections: `/` joined by `,` | -| `RawContacts.SYNC2` | the shadow: the last server card, wire format, with each `data:` photo URI replaced by `sha256:` of its bytes | +| `RawContacts.SYNC1` | the card's selected address books: `/` joined by `,` | +| `RawContacts.SYNC2` | the shadow: the last server card, wire format, with each `data:` photo URI replaced by `sha256:` of its bytes, plus a `~memberOf` member: the group cards whose memberships the rows held after our last write, so a removal on the device and an addition on the server can be told apart (stripped before any projection or patch) | | `RawContacts.SYNC3` | the pending create: `{"uid":…,"target":"/"}`, written before the `/set` and kept until the identity is | | `RawContacts.SYNC4` | poison marker `{"fp":…,"type":…,"n":…,"until":…}` | -| `RawContacts.RAW_CONTACT_IS_READ_ONLY` | 1 when none of the card's address books grants `mayWrite` | +| `RawContacts.RAW_CONTACT_IS_READ_ONLY` | 1 when none of the card's address books grants `mayWrite`. Only written, and only along with other raw contact writes, so an echo stays write-free: ContactsProvider refuses the column in every projection and selection ("Invalid column", API 32 and 35). Whether device edits are put back is decided from the server's current rights on the stored card | | `Data.DATA_SYNC1` | the entry key(s), e.g. `emails:work1`; see [units](#units) | | `Data.DATA_SYNC2` | photo rows: the hash of the server photo applied to the row | | `Data.DATA_SYNC3` | the baseline: the row's mapped columns as the provider stored them after our last write (JSON; text over 1 KB as its hash) | -| `Groups.SOURCE_ID`, `SYNC2`, `SYNC3`, `SYNC4` | as for raw contacts, for group cards (`SHOULD_SYNC` is never written: an update carrying it makes ContactsProvider request a sync) | +| `Groups.SOURCE_ID`, `SYNC2`, `SYNC3`, `SYNC4` | as for raw contacts, for group cards; `GROUP_IS_READ_ONLY` is written like `RAW_CONTACT_IS_READ_ONLY`, for groups in read-only books (`SHOULD_SYNC` is never written: an update carrying it makes ContactsProvider request a sync) | | `Settings` | one row per account: `UNGROUPED_VISIBLE=1`, `SHOULD_SYNC=1` (the default 0 hides every contact without a visible group) | Writing `DIRTY=0`, `SOURCE_ID` or `SYNC1-4` does not bump @@ -269,7 +274,7 @@ VERSION without DIRTY; the assert then fails once and the row is re-read.) | `Calendars._SYNC_ID` | identity | | `Calendars.CAL_SYNC1` | unused: CalendarProvider sends it as the `feed` extra of the sync it requests whenever a calendar with `SYNC_EVENTS=1` is inserted or changed, so anything there splits those requests apart | | `Calendars.CAL_SYNC2` | the shadow: the last server `Calendar` object | -| `Calendars.CAL_SYNC3` | flags JSON: `readOnly` reason (`rights`, `subscription`), `taskOnly` | +| `Calendars.CAL_SYNC3` | flags JSON: the `readOnly` reason (`rights`, `subscription`) | | `Events._SYNC_ID` | identity | | `Events.SYNC_DATA1` | masters and single events: the shadow, the full server event with its `recurrenceOverrides` | | `Events.SYNC_DATA2` | exceptions: the recurrence id, exactly as the server's override key | @@ -282,9 +287,9 @@ Attendees and reminders have no sync columns, which is why their baseline sits on the event row. `Events.MUTATORS` (the packages that changed the event since our last write) tells which app made a device edit. -A description longer than 64 KB is written truncated, marked in the baseline, -and never uploaded: an event row carries the description, the shadow and the -baseline, and one Binder transaction holds 1 MB. +A description longer than 64 KB of UTF-8 is written truncated, marked in the +baseline, and never uploaded: an event row carries the description, the +shadow and the baseline, and one Binder transaction holds 1 MB. ### SyncState @@ -308,26 +313,34 @@ JSON, versioned. } }, "deviceZone": "Europe/Berlin", - "deviceZonePending": null + "deviceZonePending": null, + "reminderOwner": "device" } ``` - `owner`: the registry account and the server origin the rows were written - for. A run for another owner stops with `internal` and the app offers to - start over (after uploading what it can). + for. A run for another owner stops with `internal`, and a teardown uploads + nothing for it; turning sync off and on again starts over. - `itemsState` is the `ContactCard` or `CalendarEvent` state the rows describe. It is written in the same `applyBatch` as the last item group of a `/changes` page: the `syncState` op is the last op of that batch, is never `yieldAllowed`, and the native side refuses a batch that breaks either rule, so a provider yield can never commit the state ahead of its rows. - No stored `itemsState` is newer than the shadow of an object changed before - it: an object whose rows could not be written (an item error) goes on - `stale` in the same batch, is fetched again at the start of every run, and - uploads nothing until it has been. + it: an object whose rows could not be written (a planner error, a provider + refusal that re-planning did not fix) goes on `stale` in the same batch, is + fetched again first in every run, and uploads nothing until it has been. + The state still advances, so one bad object never blocks the pages after + it. - `reconcile` marks a full reconcile in progress: `{ "from": , "phase": "ids" | "objects", "position": n }`, written with every - chunk. The next run continues it; its deletions are applied only when the - complete id list is known, and `itemsState` becomes `from` when it ends. + first>, "phase": "ids" | "objects", "position": n, "after": }`, + written with every chunk of objects. `after` is the last id processed in + the reconcile's sorted order: the next run lists the ids again and + continues after it (`phase` and `position` are informative only). `from: + null` asks for a reconcile that has not started (after + `cannotCalculateChanges`, or when local deletions were discarded). Its + deletions are applied only when the complete id list is known, and + `itemsState` becomes `from` when it ends. - `selected` is the selection the rows were written for; a collection is added in the batch with its last loaded chunk, and removed with its last deleted row. @@ -337,44 +350,52 @@ JSON, versioned. - `deviceZone` (calendar) is the zone floating events were written in; `deviceZonePending` is set while a zone-change pass runs and cleared, with `deviceZone` updated, in the batch of its last chunk. -- The providers delete the row with the account. +- `reminderOwner` (calendar) is who the Reminders rows were written for + (`device` or `bulwark`), so a run notices a changed choice; see + [Reminder owner](#reminder-owner). +- The providers delete the row with the account. A teardown clears it by + writing `''`. ## A sync run One run handles one Android account and one authority. A module-level mutex per (registry account, authority) serialises it with other runs and with app-side -operations such as turning sync off. +operations such as turning sync off; a run that cannot get it before its +deadline reports `cancelled` with `moreRecordsToGet`. -1. **Preflight.** - - Permissions for the authority (a revoked one → `permission`). - - The Android account and that the authority is still enabled +1. **Preflight**, in this order: + - The authority is still enabled for the Android account (`getSyncAutomatically`) → otherwise `disabled`. - - The SyncState's `owner` matches this registry account and server origin - (see [Identity](#identity)). + - The SyncState is read; without the runtime permission this first + provider read fails with `permission`. A blob that cannot be understood + (another version, garbage) counts as none: every JMAP account starts over + with a full reconcile. Its `owner` must be this registry account (see + [Identity](#identity)) → otherwise `internal`. + - An upload-only sync (`extras.upload`) with nothing dirty, deleted or new + for our account ends here with `ok`, before JMAP is opened: + ContactsProvider and CalendarProvider schedule an upload sync for every + account 30 s after any app write, other accounts' included. A pending + reconcile or objects on `stale` make it a full run. - `new JMAPClient().loadAccount(registryId)`: `false` or `AuthenticationError` → rebuild once (another client may have rotated the refresh token); still failing → `auth`, plus one notification that - deep-links to sign-in. Nothing local is touched. + deep-links to sign-in. Nothing local is touched. The SyncState's `owner` + must also name this server's origin → otherwise `internal`. - The capability on at least one JMAP account → else `unsupported`. - - An upload-only sync (`extras.upload`) with nothing dirty, deleted or new - for our account ends here with `ok`: ContactsProvider and CalendarProvider - schedule an upload sync for every account 30 s after any app write, other - accounts' included. -2. **Local checks.** Calendar: two masters sharing a `_SYNC_ID` (a - `CONTENT_EXCEPTION_URI` split, which also caps the old master's rule - without DIRTY) are resolved before any merge or baseline heal: the newer row - becomes a new event and the older master's rule is uploaded as it stands. - Objects on `stale` are fetched again. -3. **Collections.** `AddressBook/get` or `Calendar/get` (explicit properties) +2. **Collections.** `AddressBook/get` or `Calendar/get` (explicit properties) for the primary JMAP account of the capability (`primaryAccounts[capability]`) and every shared account that advertises it; calendar rows are inserted and - updated to match. Newly selected collections are loaded after step 4 has + updated to match. An account that refuses to list its collections + (`forbidden`, `accountNotFound`, `accountNotSupportedByMethod`) is left out + of the run: nothing downloads or uploads for it, its rows stay on the + device untouched, and it is only logged, without an item error (Stalwart + lists an account shared for mail only with every capability, and a + refusal can pass). Newly selected collections are loaded after step 3 has brought the account up to date, restricted to them, so the stored state never skips changes of the other collections. Deselected collections are - dropped after step 5, and only their clean rows go: dirty items there are - uploaded first (they are still valid objects on the server), and an item - whose upload failed keeps its rows and is reported. -4. **Download.** + dropped in step 5. +3. **Download.** + - Objects on `stale` are fetched again first. - With a stored `itemsState`: `/changes` with `maxChanges: 256` and a `hasMoreChanges` loop; each page's changed ids are fetched with explicit `properties` in batches of `maxObjectsInGet`. @@ -382,37 +403,63 @@ operations such as turning sync off. checkpointed in `reconcile` (see [SyncState](#syncstate)). The state is taken first (`/get` with `ids: []`), then every selected collection's ids (`/query`, `inAddressBook`/`inCalendar`, `limit` 5000 with `position` - paging), then the objects in batches. Rows without an identity or with a - pending one (and exceptions of pending masters) are never reconcile - deletions. A reconcile that finds zero remote objects while the device - holds more than 10 rows for that JMAP account stops with `safetyAbort` - instead of wiping the device. + paging; pages overlap by up to 100 ids, so a deletion between two pages + skips nothing), then the objects in batches. Rows without an identity or + with a pending one (and exceptions of pending masters) are never + reconcile deletions. A reconcile that finds zero remote objects while the + device holds more than 10 rows for that JMAP account stops with + `safetyAbort` instead of wiping the device. + - Contacts: a page's group cards are written before its contacts. When a + group card's `members` changed, its old and new member contacts are + fetched in the same page, before the page's state op: their membership + rows change although their own cards did not. - An object without rows first looks for a local item it may already be: - a new row of the same JMAP account whose pending uid is the object's - `uid` (our own create whose identity never got written) is adopted: it - takes the identity and shadow and is merged as a dirty item. + a new row whose pending create targets a collection the object is in and + whose pending uid is the object's `uid` (our own create whose identity + never got written) is adopted: it takes the identity and shadow and is + merged as a dirty item. - Each object is [merged](#merge-rules) into its rows. An object that left every selected collection loses its rows only when they are clean; dirty units are uploaded first. + - An object whose rows could not be written (a planner error, a provider + refusal that re-planning did not fix) goes on `stale`, and the page's + state still advances. - Rows are written in chunks; the new state goes into the last chunk of each page, so every page is a checkpoint. -5. **Upload**, always after download so merges see the latest server truth; see +4. **Upload**, always after download so merges see the latest server truth; see [Uploads](#uploads). +5. **After the upload.** Deselected collections are dropped, and only their + clean rows go: dirty items there were uploaded first (they are still valid + objects on the server), and an item whose upload failed keeps its rows, + and its collection stays in `selected`. Rows of objects that left every + selected collection go the same way. Calendar: then the device-zone pass + and the reminder-owner pass rewrite clean events (see [Timing](#timing) + and [Reminder owner](#reminder-owner)). 6. **Report** counts, conflicts, per-item errors and duration; persist the status for the settings UI; call `finishRun` before the task's promise settles (JS timers stop once the task is over). `/get` always lists `properties` explicitly: with `properties: null` Stalwart leaves out JMAP-only fields such as `useDefaultAlerts`, and `ids: null` returns -at most 500 objects without saying so. +at most 500 objects without saying so (only collections, of which Stalwart +allows 250 per account, are read with `ids: null`). Checkpoints sit between chunks and phases. At each one the run checks -`isRunCancelled`, its deadline and the time since the last JMAP request. When -the deadline is near it stops cleanly and reports `moreRecordsToGet`, which -the adapter turns into `fullSyncRequested` (SyncManager ignores -`moreRecordsToGet` itself). The step from a create's `/set` to its identity -write is not interruptible, and a create batch starts only while at least two -request timeouts (60 s) of budget remain. +`isRunCancelled` (and whether a teardown asked it to stop), its deadline and +the time since the last JMAP request. Less than 30 s before the deadline it +stops cleanly and reports `moreRecordsToGet`, which the adapter turns into +`fullSyncRequested` (SyncManager ignores `moreRecordsToGet` itself). The step +from a create's `/set` to its identity write is not interruptible, and a +create batch starts only while at least two request timeouts (60 s) of budget +remain before the deadline (`deadlineMarginMs` and `createBudgetMs` in +`DEFAULT_TUNING`, `engine/deps.ts`). An expedited sync gets 2.5 minutes, of +which the adapter keeps 50 s in reserve; when it first has to start React +Native, little of the rest is left in which a create batch may start. + +Failures stay contained. A planner that throws on one item affects only that +item: a download puts the object on `stale`, an upload skips the item and +reports it as `plannerError`. An account-level refusal of a whole `/set` holds +back only that account (see [Uploads](#uploads)). ### Invariants @@ -441,8 +488,8 @@ Every reviewer checks against these. written in the same `applyBatch` as the last chunk of rows it describes, as that batch's last op. Creates carry a client-minted `uid`, written to the row with the create's target before the `/set`; a download adopts a pending - row by uid, and an upload looks the uid up and adopts what it finds, so a - retry never duplicates on either side. + row by uid, and an upload looks the uid up and adopts what it finds in the + create's target collection, so a retry never duplicates on either side. 4. **No feedback loops.** Adapter writes use `CALLER_IS_SYNCADAPTER`, which neither sets DIRTY nor schedules an upload sync, and never touch the columns that make a provider request a sync (`CAL_SYNC1`, `Groups.SHOULD_SYNC`). @@ -488,9 +535,9 @@ exceptions: - the read-back also reads DIRTY (and VERSION): a row that turned dirty in the meantime keeps its old baseline, so the user's edit still counts; -- `timing` and `rule` of a recurring master are never healed: a - `CONTENT_EXCEPTION_URI` split changes them without DIRTY (see the - [local checks](#a-sync-run)). +- `timing` and `rule` of a recurring master are never healed, and the rows of + a split are not healed at all: a `CONTENT_EXCEPTION_URI` split changes them + without DIRTY (see [Recurrence](#recurrence)). ### Units @@ -525,9 +572,14 @@ per data kind: base36 characters (Stalwart parses numeric keys as array indexes and generates `k` itself, so neither is used). 5. Shadow entries still without a row count as deleted on the device **only - when the kind still has rows that carry keys** (an in-place editor removed - one). When every row of the kind is keyless, an editor rewrote them all, and - a missing entry may just be one it could not show: it stays on the server. + when the kind still has rows matched by their key** (an in-place editor + removed one), or when the kind has no row left at all and some other kind + that re-inserting editors rewrite still has rows matched by their key (the + editor worked in place: AOSP deleting a contact's only email, note or + birthday). When every row of the kind is keyless, an editor rewrote them + all, and a missing entry may just be one it could not show: it stays on the + server. A missing name row is never a deletion; a missing photo row always + is (no editor re-inserts photo rows). A row matched by its key is compared column by column with its baseline; an emptied column is a real deletion. A row matched by value has no baseline: its @@ -536,6 +588,15 @@ and LABEL only for phones, emails, addresses and events (editors that re-insert rows write a constant type for websites), and never `IS_PRIMARY`; an empty column never clears anything. +A merge into a dirty contact never gives keys to a kind whose rows are all +keyless: a keyed row there would later make the entries the editor could not +show look deleted. Every write that leaves a contact clean (a download into a +clean contact, an accepted upload, an upload with nothing to send, a revert) +makes its rows match the card: rows are rewritten, re-inserted or deleted, not +just given new baselines. Otherwise an entry Fossify could not show (a second +nickname) would read as deleted at the next in-place edit; it also puts back a +department Fossify dropped. + The name: `""` and NULL are the same. `name/full` uploads when DISPLAY_NAME changed against the baseline and is not simply the provider's join of the current components (a typed display name); when only components changed, @@ -600,17 +661,36 @@ times); after that the item goes on `stale`. ## Uploads -Per JMAP account, in this order: pairs (below), creates, updates, deletions, -then group memberships; in `/set` calls of at most `maxObjectsInSet` objects -(creates: 50) and about 1 MB of JSON (Stalwart splits bigger calls into -several commits, which can apply partially), one request in flight, and one -`sendSchedulingMessages` value per call (it is a request-level flag). +The upload runs phase by phase across the JMAP accounts: new rows are claimed +(below), each account resolves its series splits and pairs, then every +account's creates go up, then every account's updates, then every account's +deletions, then (contacts) groups deleted by absence and group memberships. So +a move between accounts never destroys before it created. Changes go in `/set` +calls of at most `maxObjectsInSet` objects (creates: 50) and about 1 MB of +JSON (Stalwart splits bigger calls into several commits, which can apply +partially), one request in flight, and one `sendSchedulingMessages` value per +call (it is a request-level flag). - **ifInState.** Every `/set` carries `ifInState` = the item state the merge was based on (then the `newState` of our previous `/set`). A `stateMismatch` - means the server changed since the download: the run downloads again and - retries the upload, at most three times, then reports `io` without progress - (SyncManager would otherwise retry at once, without back-off). + (or Stalwart's `serverUnavailable`, a concurrent write that failed the whole + call) means the server changed since the download: the run downloads that + account again and repeats the phase. When `/changes` from the stored state + then shows nothing new, the next `/set` uses the type's current state + instead: Stalwart can name one point of its history two ways (a `/changes` + state `s…` against the type's state `n` before the first item change), and + `ifInState` compares them as text. After three retries the run reports `io` + without progress (SyncManager would otherwise retry at once, without + back-off). +- **Account refusals.** A method error on a whole `/set` that concerns the + account (`accountReadOnly`, `forbidden`, `accountNotFound`, + `accountNotSupportedByMethod`) holds back only that account: its items wait, + the report lists the account, and the other accounts go on. +- **Splits.** A `CONTENT_EXCEPTION_URI` split (two masters sharing a + `_SYNC_ID`, see [Recurrence](#recurrence)) is resolved first: the older + master's capped rule is planned while the split is still visible, the newer + row is claimed as a new event, then the rule is sent. The new event is + created with the other creates. - **Pairs.** Some apps turn an edit into a delete plus an insert: Etar moves an event to another calendar that way, and turns a recurring series into a single event (or edits "this and following" from its first instance) the @@ -618,25 +698,32 @@ several commits, which can apply partially), one request in flight, and one match apart from the calendar (or the rule) are uploaded as a patch of the existing object (`calendarIds` swapped; `recurrenceRule: null` with its overrides removed); the new row takes the identity and exceptions of the old - one, which is purged. Across JMAP accounts a move is a create in the target - account first, identity written, then a destroy in the source (the engine - records its own destroys so the next download treats them as echoes). - Unpaired rows are ordinary creates and deletions. + one, which is purged. Rows of different JMAP accounts never pair: that move + is a create in the target account, identity written, and later in the run a + destroy in the source, whose echo finds no rows left to change. Unpaired + rows are ordinary creates and deletions. - **New rows** (see [Identity](#identity) for what counts as new): 1. Claim: write the pending create (`SYNC3` / `SYNC_DATA3`: the uid and the target collection; for events also `UID_2445` and `_SYNC_ID = ~pending/`). A row keeps a uid it already carries unless another row, synced or pending, uses the same uid (a - `CONTENT_EXCEPTION_URI` split clones `UID_2445`); a server object with - that uid does not count, because it may be this row's own earlier create. - The target is fixed at the claim: contacts go to the address book chosen - in the settings if it is selected and writable, else the first selected - writable book of the personal account (none → the item is poisoned with - `noWritableAddressBook`); events go to the calendar of their row. + `CONTENT_EXCEPTION_URI` split clones `UID_2445`, an app's "duplicate" + copies it): then it gets a fresh one, and the engine loads every master + that shares a new master's `UID_2445` so the planner sees the collision. + A server object with that uid does not count, because it may be this + row's own earlier create. The target is fixed at the claim: contacts go + to the address book chosen in the settings if it is selected and + writable, else to the personal account's default book (or its first by + sort order) among the selected writable ones; with none, the item is + skipped and reported as `noWritableAddressBook` in every run. Events go + to the calendar of their row. 2. Look the uid up: contacts in the target address book (card uids are unique per book), events in the whole account (event uids are unique per - account). A hit is adopted: the row takes its identity and is merged as a - dirty item. + account). A hit in the create's target collection is adopted: the row + takes its identity and is merged as a dirty item. A hit elsewhere (an + event with the same uid in another calendar) is not this row's create, + and adopting it would overwrite an unrelated event: the item is poisoned + as `uidConflict`. 3. Otherwise `create`. 4. `invalidProperties` on `uid` means the object exists: Stalwart's duplicate check is synchronous but the `uid` query reads an asynchronous @@ -648,8 +735,8 @@ several commits, which can apply partially), one request in flight, and one create batch starts only with at least two request timeouts (60 s) of budget left. - **Exception rows** (`ORIGINAL_*` set) upload only as - `recurrenceOverrides/` of their master, after the master's create when - the master is new. + `recurrenceOverrides/` of their master; those of a new master go up + inside its create. - **Dirty rows**: the patch from the changed units. An empty patch clears DIRTY without a request. - **Deleted rows** (`DELETED=1`): an object that belongs to several @@ -662,25 +749,39 @@ several commits, which can apply partially), one request in flight, and one - **Afterwards**: a `/get` of every touched id (a separate request for creates: Stalwart does not resolve result references to `/set`). The result becomes the new shadow, and the rows' baselines are refreshed from it, in one group - per item that first asserts the VERSION/projection read before the upload - and clears DIRTY. If the assert fails (edited again meanwhile), the shadow and - identity are still written, the baselines of the uploaded units are set to - the values that were uploaded, and DIRTY stays: the newer edit uploads next - time, and so does a revert to the old value. + per item that first asserts the VERSION/projection read before the upload, + clears DIRTY and clears a poison marker the item carried. If the assert + fails (edited again meanwhile), the shadow and identity are still written, + the baselines of the uploaded units are set to the values that were + uploaded, and DIRTY stays: the newer edit uploads next time, and so does a + revert to the old value. For contacts these baseline writes address their + data rows with `where _id = ?`, so a row deleted meanwhile does not fail the + group. - **SetErrors**: - - `notFound` on update → treated as a remote delete (the rows go); + - `notFound` on update → treated as a remote delete (the rows go); on a + destroy it counts as done; on a create the item is poisoned; - `forbidden` in a call that asked for scheduling messages → the item is retried once without them and the report notes "invitations not sent" (Stalwart refuses scheduling when it is disabled, when the account has no - calendar address, or without the scheduling permission); `forbidden` - otherwise → the rows are rewritten from the shadow (read-only on the server - after all); + calendar address, or without the scheduling permission). `forbidden` + otherwise: on a create the item is poisoned (a revert would delete the + user's new item); on an update or a deletion the object is read-only on + the server after all, and its rows are rewritten from the server's + current version, fetched for it (the shadow holds photos only as hashes), + or removed when the object is gone or outside the selection; - `stateMismatch` → as above; - `invalidProperties` on `uid` → as above; - `invalidProperties`, `invalidPatch`, `tooLarge`, `overQuota` and the rest → - the item is poisoned: its marker records a fingerprint of the patch and a - back-off (1 h, doubling to 1 day). It is not retried until the back-off - ends or the row changes, and the report lists it. + the item is poisoned. +- **Poison markers.** The engine writes the marker (`SYNC4` / `SYNC_DATA5`, + `{"fp":…,"type":…,"n":…,"until":…}`): `fp` is a fingerprint of what the + item uploads from (its flags, shadow and mapped rows; for events also the + attendees, reminders and exception rows), `until` the end of a back-off of + 1 h that doubles up to 24 h while the item stays the same. The engine checks + the marker before `planUpload`: until the back-off ends or the item + changes, the item is skipped and the report lists it. The marker is cleared + once an upload of the item is accepted. The planners check the same marker + with the same fingerprint content, a duplicate of the engine's check. ### Deletion threshold @@ -688,16 +789,22 @@ More than 50 local deletions of an authority in one run, and more than 20 % of that authority's synced objects, set `tooManyDeletions` unless the run carries `overrideTooManyDeletions`. Deletions inferred from absence (groups) count. No deletion of that run is uploaded; everything else proceeds. With -`discardLocalDeletions`, the deleted rows are purged and, in the same batch, the -affected JMAP accounts get a `reconcile` marker, so the next runs download the -objects again whatever their extras. +`discardLocalDeletions` (the user chose "undo" in Android's too-many-deletions +notification), the affected JMAP accounts first get a `reconcile` marker in a +batch of its own, then the deleted rows are purged, so the next runs download +the objects again whatever their extras; a crash in between only asks the +question again. A teardown applies the same threshold: deletions above it stay +on the device and count as waiting changes, so the app asks before turning +sync off (see [Lifecycle](#lifecycle)). ## Contacts mapping JSContact (RFC 9553) as Stalwart stores it, to ContactsContract. Anything not listed is preserved through patching. Data rows are written with both the display name and the components (the provider recomputes whichever is missing), -and every entry row stores its key(s) in `DATA_SYNC1`. +and every entry row stores its key(s) in `DATA_SYNC1`. Text is cut to what +ContactsProvider keeps (10 KiB; phone numbers 1000 characters), and a column +that was cut never uploads. ### Name (StructuredName, one row, key `name`) @@ -717,7 +824,9 @@ component (separators, repeats, unknown kinds) in place, and sends the whole `name/components` array (JSON Pointer can't address array elements). When `name/full` goes along is in [row matching](#row-matching-contacts): a typed display name uploads; a recomputed one re-derives `full` only when the server's -was derived too. +was derived too. A name the server holds without components uploads whole: +the components from the row and `full` = the display name, so retyping "Ada" +as "Ada King" on the device sends both. ### Entries @@ -726,15 +835,15 @@ was derived too. | `nicknames/` | Nickname | `NAME`, `TYPE=DEFAULT` | | `emails/` | Email | `ADDRESS`; `TYPE` from contexts: `private`→HOME, `work`→WORK, none→OTHER, `label`→CUSTOM+`LABEL`; `pref=1`→`IS_PRIMARY` | | `phones/` | Phone | `NUMBER`; `TYPE` from features + contexts (table below); `label`→CUSTOM+`LABEL`; `pref=1`→`IS_PRIMARY` | -| `addresses/` | StructuredPostal | components → `STREET`, `POBOX`, `NEIGHBORHOOD`, `CITY`, `REGION`, `POSTCODE`, `COUNTRY` (table below), `full`→`FORMATTED_ADDRESS`; contexts → TYPE HOME/WORK/OTHER, `label`→CUSTOM | +| `addresses/` | StructuredPostal | components → `STREET`, `POBOX`, `NEIGHBORHOOD`, `CITY`, `REGION`, `POSTCODE`, `COUNTRY` (table below), `full`→`FORMATTED_ADDRESS` (a card without `full` gets a one-line address joined from the components, so the provider leaves the row as written); contexts → TYPE HOME/WORK/OTHER, `label`→CUSTOM | | `organizations/` + `titles` pointing at it | Organization | `name`→`COMPANY`, `units` joined with `, `→`DEPARTMENT`, title with `kind: title`→`TITLE`, title with `kind: role`→`JOB_DESCRIPTION`; key `organizations:|titles:,`; titles without an organization get their own row | | `links/` | Website | `URL` ← `uri`; contexts `private`→HOME, `work`→WORK, else OTHER; `label`→CUSTOM | -| `anniversaries/` | Event | `START_DATE`: `YYYY-MM-DD`, `--MM-DD` without year (`partialDateToString`); kind `birth`→BIRTHDAY, `wedding`→ANNIVERSARY, `other`→OTHER, anything else → CUSTOM + its kind as `LABEL` | +| `anniversaries/` | Event | `START_DATE`: `YYYY-MM-DD`, `--MM-DD` without year (`partialDateToString`); kind `birth`→BIRTHDAY, `wedding`→ANNIVERSARY, `other`→OTHER, anything else → CUSTOM + its kind as `LABEL`; a custom type uploads its label, lowercased, as the kind | | `relatedTo/` | Relation | `NAME` ← the key (a `urn:uuid:` of a synced card shows that card's name), `TYPE` from the relation set (table below) | | first entry of `notes` | Note | `NOTE` ← `note` | | first `media` entry with `kind: "photo"` | Photo | full-size bytes into `PHOTO`; the provider scales them, stores the display photo and sets `PHOTO_FILE_ID` | | group cards (`kind: "group"`) | Groups | `TITLE` ← `name.full`; `GROUP_VISIBLE=1`; `members` keys are uids of the member cards | -| membership | GroupMembership | one row per group card whose `members` holds this card's `uid` (`GROUP_SOURCE_ID`) | +| membership | GroupMembership | one row per group card whose `members` holds this card's `uid`, written with `GROUP_ROW_ID` (`data1`, the group's row id); `GROUP_SOURCE_ID` is only read | Phone types (first match wins; reverse: the table read right to left, and `TYPE_OTHER` → no features): @@ -768,7 +877,8 @@ built from with one component of the first kind, keeps the rest): | `COUNTRY` | `country` | Legacy flat address fields (`street`, `locality`, …) are read when there are no -components and never written. +components and never written. When an editor wrote only the formatted address +and the provider copied it into `STREET`, only `full` uploads. Relations: `friend`↔FRIEND, `spouse`↔SPOUSE, `child`↔CHILD, `parent`↔PARENT, `sibling`→RELATIVE, `kin`→RELATIVE, `co-worker`/`colleague`→CUSTOM with the @@ -776,7 +886,14 @@ type as label, `agent`→ASSISTANT, `emergency`→CUSTOM. Device types without a RFC 9553 equivalent (BROTHER, SISTER, MOTHER, FATHER, MANAGER, ASSISTANT, DOMESTIC_PARTNER, PARTNER, REFERRED_BY) upload as the nearest RFC type (`sibling`, `parent`, `colleague`, `agent`, `spouse`, `spouse`, `contact`) and are -only sent when the user changed the type. +only sent when the user changed the type. RELATIVE uploads as `kin`; a custom +type uploads its label, lowercased, as the relation type. Only a row matched by +its key can rename a relation (`relatedTo` is keyed by its value, so another +name is another entry): a re-inserted row showing another name may just show a +related card that was renamed since. + +For emails, phones, addresses and websites, changing only the label of a custom +type uploads only `label`; the contexts the entry had stay. Deleting an entry resends its whole map (Stalwart turns `map/key: null` into a valueless vCard line or, for `relatedTo`, ignores it). Only `addresses`, @@ -796,13 +913,26 @@ most 512 px, JPEG, and sent as a `data:image/jpeg;base64,…` URI in `media//uri`, matching the app's picker. `DATA_SYNC2` holds the hash of the server photo last applied, the baseline holds `PHOTO_FILE_ID`; a new file id means the user changed the photo. Photos travel in batches of their own -(Binder budget). +(Binder budget). A photo whose bytes could not be fetched keeps its row and +counts as unchanged. -Groups: created before memberships (a membership naming an unknown -`GROUP_SOURCE_ID` makes the provider create an empty group). A group deleted on -the device is found by `DELETED`, or by its absence from the rows while -`SyncState.groups` still lists it (Fossify hard-deletes groups through a -sync-adapter URI); both count toward the deletion threshold. +Groups: group cards are written before the contacts of the same download, and +membership rows name their group by row id (`GROUP_ROW_ID`, looked up by the +group's `SOURCE_ID`), never by `GROUP_SOURCE_ID`: a membership naming an +unknown `GROUP_SOURCE_ID` makes the provider create an empty group. A +membership whose group has no row is not written. A contact's memberships are +merged from three sets: what the rows held after our last write (`~memberOf` +in the shadow), the rows now, and the group cards that list the card's uid +now; both sides can only flip a membership the same way, so memberships never +conflict. Membership edits upload after the deletions, as patches of the +group cards' `members` (`members/: true` or `null`; the whole map for a +group without members). A contact stays dirty until the server's group cards +show its membership edits (the clean and accepted writes keep DIRTY); the +next run's clean write clears it. + +A group deleted on the device is found by `DELETED`, or by its absence from the +rows while `SyncState.groups` still lists it (Fossify hard-deletes groups +through a sync-adapter URI); both count toward the deletion threshold. ## Calendar mapping @@ -811,12 +941,12 @@ sync-adapter URI); both count toward the deletion threshold. | JMAP `Calendar` | Android `Calendars` | |---|---| | `id` | `_SYNC_ID` = `/` | -| `name` | `NAME`, `CALENDAR_DISPLAY_NAME` (shared accounts append the account's name) | +| `name` | `NAME`, `CALENDAR_DISPLAY_NAME`; a shared account's calendars show ` ()` (the JMAP account's name, unless it is one of the user's addresses), the personal account's get no suffix | | `color` | `CALENDAR_COLOR` (CSS color → ARGB; absent → the app's default palette colour) | | `myRights` | `CALENDAR_ACCESS_LEVEL`: `mayWriteAll` → OWNER (700); `mayWriteOwn` → CONTRIBUTOR (500); `mayRSVP` → RESPOND (300); else READ (200); subscribed iCal feed → READ | | the user's calendar address | `OWNER_ACCOUNT` (never null: Etar crashes on it) | | `timeZone` | `CALENDAR_TIME_ZONE` (device zone when null) | -| — | `SYNC_EVENTS=1` (the default 0 hides every instance), `VISIBLE=1` on insert only (local afterwards), `ALLOWED_REMINDERS="1,2"`, `ALLOWED_AVAILABILITY="0,1"`, `ALLOWED_ATTENDEE_TYPES="0,1,2,3"`, `MAX_REMINDERS=5` (0 when Bulwark owns reminders), `CAN_ORGANIZER_RESPOND=0`, `CAN_PARTIALLY_UPDATE=0` | +| — | `SYNC_EVENTS=1` (the default 0 hides every instance; written again when an app turns it off), `VISIBLE=1` on insert only (local afterwards), `ALLOWED_REMINDERS="1,2"`, `ALLOWED_AVAILABILITY="0,1"`, `ALLOWED_ATTENDEE_TYPES="0,1,2,3"`, `MAX_REMINDERS=5` (0 when Bulwark owns reminders), `CAN_ORGANIZER_RESPOND=0`, `CAN_PARTIALLY_UPDATE=0` | Not synced: the virtual birthday calendar (client-side only), calendars without `mayReadItems`, and task-only calendars: a calendar whose first 50 @@ -826,6 +956,14 @@ reconcile. Deleting a calendar row deletes its events (provider trigger); the engine first uploads the calendar's dirty events. +A read-only calendar (without `mayWriteAll` and `mayWriteOwn`, or a subscribed +feed) takes no device changes: a dirty event is rewritten from its shadow, a +deleted one is purged and fetched again, a new one is removed. The one +exception is the user's own RSVP on the master +(`participants//participationStatus`), which uploads when the calendar +grants `mayRSVP`. Edits of an event Stalwart holds only as an instance (no +series) are reverted too, in any calendar. + ### Timing | JSCalendar | Android | @@ -840,8 +978,9 @@ first uploads the calendar's dirty events. Android's Duration parser takes `[+-]P` then `` (with `M` always minutes, no fractions, no `Y`); all-day durations must be `PD` or the provider throws. Server durations are parsed in full (weeks included, `P1W` -comes from CalDAV DTEND). A duration with a time part on an all-day event is -rounded up to whole days on the device and never uploaded unless changed. +comes from CalDAV DTEND). All-day events are at least one day long on the +device: a duration with a time part is rounded up to whole days, a zero one +becomes one day, and neither is uploaded unless the timing changes. Upload of timing: only when a timing column differs from the baseline. The local start is recomputed in the event's zone (floating events stay floating, @@ -863,15 +1002,16 @@ an upload: the offset is not a whole hour): same instant, same wall time, only the zone name changes; - a recurring event whose start or an override key falls in an overlap is not - uploaded: the item is poisoned with `dstAmbiguous`; -- an `until` in a gap or overlap moves one hour later. + uploaded: the item stays dirty and is reported as `dstAmbiguous`; +- an `until` in a gap or overlap moves one hour later (better than Stalwart + dropping it and the series running forever). ### Recurrence | JSCalendar | Android | |---|---| | `recurrenceRule` | `RRULE`: `FREQ`, `INTERVAL` (>1), `COUNT`, `UNTIL` (UTC `…Z` for timed events, `YYYYMMDD` for all-day), every `BY*` part, `BYSETPOS`, `WKST` (when not Monday) | -| `rscale` other than gregorian, `skip`, leap months (`byMonth` with `L`) | not representable: the rule is written without them, the event's timing and rule are never uploaded, and the item is marked `ruleNotRepresentable` | +| `rscale` other than gregorian, `skip`, leap months (`byMonth` with `L`) | not representable: the rule is written without them, the event's timing and rule are never uploaded, and an item whose only changes are to them is reported as `ruleNotRepresentable` | | `excludedRecurrenceRule` | not supported by Stalwart; an `EXRULE` written by a device app is ignored | | override `{excluded: true}` | `EXDATE`: comma list of UTC `YYYYMMDDTHHMMSSZ`, or `YYYYMMDD` for all-day (no prefix means UTC) | | any other override, including `{}` | an exception row: master ⊕ override (applied as a PatchObject); `ORIGINAL_SYNC_ID`, `ORIGINAL_INSTANCE_TIME` = the key's instant in the master's zone, `ORIGINAL_ALL_DAY`; `SYNC_DATA2` = the key | @@ -893,11 +1033,18 @@ Device-side changes map back as follows: | exception row inserted (Etar "this event": `ORIGINAL_SYNC_ID` + `ORIGINAL_INSTANCE_TIME`, `DIRTY=1`) or edited | `recurrenceOverrides/`; the key is `ORIGINAL_INSTANCE_TIME` in the master's zone (a date at midnight for all-day). A new override carries the master's title, locations, participants and sequence (Stalwart stores a bare override as a separate event otherwise); an existing one is patched one level deep (`recurrenceOverrides//`), never deeper: Stalwart turns a deeper pointer into a partial override (other attendees vanish for that instance) | | exception row with `STATUS_CANCELED` (Etar and Fossify "delete this event"), a deleted exception row, or an EXDATE added to the master | `recurrenceOverrides/: {excluded: true}` | | EXDATE removed | `recurrenceOverrides/: null` | -| master `RRULE` capped with `UNTIL`/`COUNT` + a new event without identity (Etar "this and following") | `recurrenceRule` (whole object) with the new `until`/`count`, overrides after it removed (`recurrenceOverrides/: null`), and the new event created with a fresh uid | +| master `RRULE` capped with `UNTIL`/`COUNT` + a new event without identity (Etar "this and following") | the master uploads only `recurrenceRule` (whole object) with the new `until`/`count` and removes the overrides after the new end (`recurrenceOverrides/: null`; for a `COUNT` the end is the start of the last instance, CalendarProvider's `LAST_DATE` − `DURATION`); the new series is a row of its own, claimed and created as a new event with a fresh uid | | master start moved (Etar "all events") | `start`/`duration`/`timeZone`, and every override key moved by the same local offset (overrides whose `start` equalled their old key move too), sent as the whole `recurrenceOverrides` map | -| two masters with the same `_SYNC_ID` (a `CONTENT_EXCEPTION_URI` split clones `_SYNC_ID`, `SYNC_DATA*` and `UID_2445`, and caps the old master's rule without `DIRTY`) | found before any merge or baseline heal: the newer row becomes a new event (fresh uid), the older master's current RRULE is uploaded | -| event moved to another calendar, or a series turned into a single event (Etar deletes and re-inserts both) | paired with the deleted row and uploaded as a patch of the same object (see [Uploads](#uploads)) | -| an RSVP for one instance (Etar goes through `CONTENT_EXCEPTION_URI`) | the override's whole `participants` map with the user's new status | +| two masters with the same `_SYNC_ID` (a `CONTENT_EXCEPTION_URI` split clones `_SYNC_ID`, `SYNC_DATA*` and `UID_2445`, and caps the old master's rule without `DIRTY`) | both masters are always loaded together. The download merges into the older master as a dirty item, leaves the newer one alone and heals no baseline of either. The upload resolves the split in its first step: the older master's current RRULE is uploaded, and the newer row is claimed as a new event with a fresh uid; that claim points the source's exception rows' `ORIGINAL_SYNC_ID` back at the source and sets the source `DIRTY=1`, so its capped rule uploads | +| a new master carrying another master's `UID_2445` (an app's "duplicate" copies it) | a clone without a source: claimed with a fresh uid and created as a new event, never adopting the other master's object (the engine loads every master that shares a new master's `UID_2445`) | +| event moved to another calendar, or a series turned into a single event (Etar deletes and re-inserts both) | paired with the deleted row and uploaded as a patch of the same object (see [Uploads](#uploads)); never across JMAP accounts, where the move is a create in the target account and a destroy in the source | +| `CALENDAR_ID` changed in place to a calendar of another JMAP account | not uploaded; the next download of the object puts the row back | +| an RSVP for one instance (Etar goes through `CONTENT_EXCEPTION_URI`) | the override's whole `participants` map with the user's new status. An exception row's status uploads only when the user organizes the event: Etar also puts `STATUS_CONFIRMED` on the instance, and Stalwart refuses an attendee's change to an event's status | + +On an exception row an app created, an empty title, description, location or +colour, or no attendee or reminder rows, means "inherit from the series", never +a deletion. An override never carries `privacy`: Stalwart drops it from +overrides. The first override of an event with none sends the whole `recurrenceOverrides` map (a patch below a missing property fails). @@ -943,8 +1090,10 @@ calendar app would notify about: - the user is the organizer and the event has other participants: `true` when title, description, timing, rule, overrides, location, status or attendees - changed, and on destroy (CANCEL); reminders, colour, availability and - privacy alone send nothing; + changed, on a create, and on destroy (CANCEL); reminders, colour, + availability and privacy alone send nothing. An event without an organizer + counts as the user's: once it has attendees, Stalwart makes the account's + first address its organizer; - the user is an attendee: `true` for their own RSVP change and on destroy (Stalwart sends REPLY DECLINED); other edits are stored for the user only; - events in the past: Stalwart sends nothing anyway. @@ -961,10 +1110,13 @@ app asks who should remind the user, per account: so apps offer no reminder editor; reminder rows an app adds anyway are neither uploaded nor deleted. -Changing the choice first uploads pending reminder edits (a normal run), then -rewrites the Reminders rows of that account's events together with their -baselines, one group per event that asserts the projection it read; no server -writes. +Changing the choice requests a calendar sync. The calendar SyncState records +who the rows were written for (`reminderOwner`), so that run notices the +change: its uploads still work for the old owner, so pending reminder edits +go up first. After the upload it rewrites the Reminders rows of that account's +clean events together with their baselines, one group per event that asserts +the projection it read, and sets the calendars' `MAX_REMINDERS`; no server +writes. A reminder edit that has not reached the server is never overwritten. ## Headless payload, report and SyncResult @@ -991,13 +1143,16 @@ tooManyDeletions?: {count, threshold}, delayUntil?, moreRecordsToGet? }`. | `cancelled` | framework cancel or out of time | stats only | | `disabled` | sync is off | nothing | -`delayUntil` (a 429's `Retry-After`, epoch seconds) and `moreRecordsToGet` -(→ `fullSyncRequested`) pass through. A run without a report is a soft error. -The stats count only rows actually written and objects actually sent, so a run -that repeated a page without writing anything is not "progress" to SyncManager. -A change pushed while a run is active (SyncManager drops a `requestSync` that -matches a running sync) flags the run; the adapter reads the flag after the run -is closed and then sets `fullSyncRequested`. +A 429 ends the run with `io` and `delayUntil` from its `Retry-After` (epoch +seconds; 60 s when the server sends none). `delayUntil` and +`moreRecordsToGet` (→ `fullSyncRequested`) pass through. A run without a +report is a soft error. The stats count only rows actually written and objects +actually sent, so a run that repeated a page without writing anything is not +"progress" to SyncManager. A change pushed while a run is active (SyncManager +drops a `requestSync` that matches a running sync) flags the run; the adapter +reads the flag after the run is closed and then sets `fullSyncRequested`. The +push of our own upload arrives while the run is still open, so every run that +uploads gets one follow-up sync (see [Triggers](#triggers)). ## Native module API @@ -1009,7 +1164,7 @@ nested maps. Every provider call is scoped to one account of our type. |---|---| | `getInfo()` | `{ accountType, sdkInt }` | | `listAccounts()` | our accounts with their `registryId` | -| `ensureAccount(name, registryId)` | adds the account (both authorities syncable, automatic off); true when created, false when it exists for the same registry id; rejects (`conflict`) when an account of that name belongs to another registry id | +| `ensureAccount(name, registryId)` | adds the account (both authorities syncable, automatic off); true when created, false when it exists for the same registry id or without one (it is adopted); rejects (`conflict`) when an account of that name belongs to another registry id | | `removeAccount(name)` | removes it; the providers drop its rows | | `getSyncSettings(name)` | master auto-sync, and per authority: syncable, automatic, periodic seconds, active, pending | | `setSyncEnabled(name, authority, on)` | syncable = 1 and `setSyncAutomatically` | @@ -1064,7 +1219,10 @@ Scoping, enforced in Kotlin for every op (`data1` of a membership row) checks it: gone fails with `assert` (re-read and re-plan), another account's with `scope`. A back-reference must point to an earlier insert of the right table; settings inserts return no id. -- `account_name`/`account_type`/`data_set` may only be written to restate the +- Inserts into the tables with account columns (raw contacts, groups, + settings, calendars, colors) get `account_name` and `account_type` added, + so calendar inserts carry them too. Otherwise + `account_name`/`account_type`/`data_set` may only be written to restate the account, on the tables that have them (`scope` otherwise). - Tables are an enumerated set per authority; anything else is refused (`scope`). @@ -1098,12 +1256,17 @@ What the providers do with values, which the planners account for: 1-second delayed storage) and not part of the settings export. Per account: `contactsSelection` / `calendarSelection` (explicit choices only; unlisted collections follow the default: the personal account's on, shared accounts' -off), `newContactsAddressBook`, `reminderOwner`, `intervalSeconds` per -authority, the last `RunStatus` per authority, and `removedInAndroidSettings`. -Whether sync is on is not stored: Android's `getSyncAutomatically` is the -source of truth. Headless runs await hydration before any `set()` (zustand 5 -writes on every `set()`, even before hydration, and would overwrite the stored -preferences). +off, and the webmail's "Trusted Senders" address book off too: it is the +allow-list of senders whose images load, not an address book people expect +in their phone's contacts), `newContactsAddressBook`, `reminderOwner`, +`intervalSeconds` per authority, the last `RunStatus` per authority, +`androidAccountName` (the Android account it uses), `knownStates` (the latest +states the engine synced, per JMAP account and type, for the echo skip), +`enabled` (what the user turned on in the app), `removedInAndroidSettings`, +`suspended` and `primaryJmapAccounts`. Android's `getSyncAutomatically` stays +the source of truth for whether an authority syncs (see Lifecycle). Headless +runs await hydration before any `set()` (zustand 5 writes on every `set()`, +even before hydration, and would overwrite the stored preferences). ### Settings sections @@ -1115,8 +1278,9 @@ native module present). Settings → Contacts and Settings → Calendar each get settings" when the account exists but the authority is off there; a hint when the device's master auto-sync is off); - a chooser (sheet) for address books or calendars, with personal ones on and - shared ones off by default, read-only ones marked, and a warning when another - signed-in account already syncs the same shared collection; + shared ones (and Trusted Senders) off by default, read-only ones marked, and + a warning when another signed-in account already syncs the same shared + collection; - contacts: the address book for new contacts; - calendar: who reminds you (calendar app or Bulwark); - the interval: 15 min, 30 min, 1 h (default), 6 h, manual; @@ -1134,27 +1298,54 @@ app's system settings when Android no longer asks. - **Enable:** permissions → `ensureAccount` → `setSyncEnabled(true)` → `setPeriodicSync(interval)` → `requestSync(manual)`; push routes and push types are refreshed. -- **Disable an authority:** the engine's teardown runs under the run mutex: - it uploads the authority's dirty, deleted and new items (bounded to 30 s; - it is not a sync run, so the "disabled" preflight does not stop it), then - deletes the authority's rows and SyncState. When items could not be uploaded - (offline, errors), nothing is deleted and the app asks: "N changes made on - this device haven't reached the server. Turn off anyway?" Only then - `setSyncEnabled(false)`; when both authorities are off the Android account - is removed. +- **Disable an authority:** automatic sync goes off first + (`setSyncEnabled(false)`), so no queued sync downloads everything again + right after the rows are gone. Then the engine's teardown runs under the + run mutex; a sync of that account and authority that holds it is asked to + stop at its next checkpoint. The teardown uploads the authority's dirty, + deleted and new items (bounded to 30 s; it is not a sync run, so the + "disabled" preflight does not stop it), with the [deletion + threshold](#deletion-threshold) applied, and never uploads rows written for + another registry account or server (SyncState `owner`). Then it deletes the + authority's rows and clears the SyncState by writing `''`. When items could + not be uploaded (offline, errors, deletions above the threshold), nothing is + deleted and the app asks: "N changes made on this device haven't reached + the server. Turn off anyway?" Turning off anyway still makes one short + upload attempt (5 s) before the rows go; "Keep syncing" turns automatic sync + back on. With nothing waiting the rows go without a question: the server + keeps everything. When both authorities are off the Android account is + removed. Turning calendar sync off also forgets who reminds of synced + events, so the question comes again the next time. - **Logout** (`auth-store.logout`, `logoutAll`, `removeAccount`): the same as disabling both, before the credentials go, with the same question when - changes would be lost. + changes would be lost; "stay signed in" keeps the accounts syncing. Several + accounts are released in parallel, a toast says it is working, and an + internal error never blocks signing out (the next reconcile suspends what + was left). - **Registry evictions** (the app drops an account on its own, e.g. after an `AuthenticationError`): reconciling Android accounts against the registry at - every engine start and on foreground finds an Android account whose registry - id is gone. It is not removed: automatic sync is turned off and one - notification says how many device changes are waiting ("Sign in again"). - Signing in again resumes it; removing it in the app or in Android Settings - drops it. An empty or unreadable registry is never acted on. + launch, on foreground and whenever accounts of our type change finds an + Android account whose registry id is gone. It is not removed: automatic + sync is turned off and one notification says it stopped syncing ("Sign in + again"; it gives no count of waiting changes, which would need a provider + read). Signing in again resumes exactly the authorities that synced; the + section lists the account with "Sign in again" and "Remove from this + device", and removing it there or in Android Settings drops it. An empty or + unreadable registry is never acted on. - **Account removed in Android Settings:** detected by the accounts listener and on foreground; the store marks it, the switches show off, and the app does not recreate the account. +- The store keeps what the user turned on in the app (`enabled`) next to + Android's switches, which stay the source of truth: an authority switched + on in Android Settings is adopted as on, one switched off there shows as + "Paused in Android settings" (not "Off"), and "Sync now" is disabled while + paused, since a manual run would only report `disabled`. `suspended` marks + an evicted account, `primaryJmapAccounts` the personal JMAP account per + authority (push routes and the trigger filter). +- An Android account that exists without a registry id (created, then the + app died before writing it) is adopted by the next `ensureAccount`; one + with another registry id is refused (`conflict`) and the app uses the + registry id as the account name instead. - **Uninstall:** Android removes the accounts of an uninstalled authenticator, and the providers delete their rows. - **Permission revoked:** Android fails the sync before our code runs @@ -1171,20 +1362,36 @@ app's system settings when Android no longer asks. `Calendar`/`CalendarEvent` on a synced account (`onStateChangeType` in `state-change-bus.ts`; the stream serves the active account), the app's own successful contact/calendar mutations (store actions), and returning to the - foreground → `requestSync` for that authority, debounced 5 s. A StateChange - whose state equals the one the engine last recorded is our own echo and is - skipped. + foreground → `requestSync` for that authority. The first request of an + account and authority arms a 5 s timer and later ones ride along (the timer + is not restarted), so an edit and the StateChange echoing it make one sync. + A StateChange whose state equals the one the engine last recorded is our + own echo and is skipped, also when the engine recorded it while the timer + ran. Foreground requests go out at most once a minute per account and + authority, and the launch counts as one. Edits that change no device row + (tasks, calendar sharing, the default collections) request nothing. These + are ordinary requests, which Android drops while the account is paused. +- A push with no readable `changed` map (older relays) starts the mail task + as before and requests no syncs; periodic syncs cover those setups. +- Every run that uploads gets one follow-up sync: the server's push of our + own change arrives while the run is open, and SyncManager drops a request + matching the running sync, so the run asks for another one when it ends. + The follow-up finds nothing to do, so it stays one. - **App closed:** while an account has device sync on, its push subscription adds `ContactCard`, `AddressBook` (contacts) and `CalendarEvent`, `Calendar` - (calendar) to `EmailDelivery`, per account. The relay forwards the - StateChange's `changed` map verbatim (FCM data key `changed`; only its first - account key is repeated as `accountId`). A native router in the FCM and - UnifiedPush services walks every account key of `changed`, maps contact and - calendar types to `requestSync` through the routes JS stored, and starts the - mail push task only when some account's map has `EmailDelivery`. A guard at - the top of `pushBackgroundTask` returns for StateChanges without - `EmailDelivery`, so a contact change can never take the legacy "notify the - newest unread mail" path. + (calendar) to `EmailDelivery`, per account. Turning sync on or off updates + the subscription right away; for a signed-in account other than the one the + UI serves, through a client of its own, never the UI's singleton. The relay + forwards the StateChange's `changed` map verbatim (FCM data key `changed`; + only its first account key is repeated as `accountId`). A native router in + the FCM and UnifiedPush services walks every account key of `changed`, maps + contact and calendar types to `requestSync` through the routes JS stored + (`setPushRoutes`; the device-sync layer refreshes them whenever the device + sync preferences change), and starts the mail push task only when some + account's map has `EmailDelivery`. A guard at the top of + `pushBackgroundTask` returns for StateChanges without `EmailDelivery`, so a + contact change can never take the legacy "notify the newest unread mail" + path. ## Failure matrix @@ -1197,20 +1404,23 @@ app's system settings when Android no longer asks. | Password revoked | `auth` (hard, no retry) + "Sign in again" | user signs in; the next trigger syncs | | Permission revoked | Android fails the sync (`databaseError`) | "Grant access" in settings | | `cannotCalculateChanges` | full reconcile of that JMAP account | automatic | -| Server `SetError` on an item | notFound → local delete; forbidden with scheduling → retried without it ("invitations not sent"); forbidden → revert; others → poisoned with back-off, listed in the report | automatic / user fixes the item | -| `stateMismatch` | re-download, retry up to 3 times, then `io` without progress | automatic (back-off) | -| 429 | `delayUntil` = Retry-After | automatic | +| Server `SetError` on an item | notFound on an update → local delete; forbidden with scheduling → retried without it ("invitations not sent"); forbidden on an update or deletion → rows rewritten from the server's current version; forbidden on a create, and the rest → poisoned with back-off, listed in the report | automatic / user fixes the item | +| A whole `/set` refused for an account (`accountReadOnly`, `forbidden`, `accountNotFound`, `accountNotSupportedByMethod`) | that account's items wait, the report lists the account; the other accounts go on | automatic once the account takes writes | +| A shared account refuses to list its address books or calendars (`forbidden`, `accountNotFound`, `accountNotSupportedByMethod`) | the account is left out of the run and only logged; its rows stay untouched | automatic once it lists them again | +| A planner throws on an item | download: the object goes on `stale`; upload: the item is skipped (`plannerError`); the rest of the run goes on | fixed in code; listed in the report | +| `stateMismatch` or `serverUnavailable` on a `/set` | re-download, retry up to 3 times, then `io` without progress | automatic (back-off) | +| 429 | `io` with `delayUntil` = Retry-After (60 s without one) | automatic | | Too many local deletions | `tooManyDeletions` + system notification | user picks delete or undo | | Empty server on a full reconcile | `safetyAbort`, device untouched | user checks the server; turning sync off and on starts over | | Edit during an upload | clearing assert fails → DIRTY stays, shadow updated | next run uploads the newer edit | | JS crash or reload mid-run | task ends without report → soft error | automatic | | React fails to start (debug without Metro) | boot fault → soft error in ~2 s | automatic | | Account removed in Settings | providers drop rows; app turns sync off | user turns it on again | -| Logout / turning sync off | pending changes uploaded first; if that fails the user is asked before anything is deleted | — | -| Provider refuses a batch (`assert`/`tooLarge`/`provider`) | per-item retry / split; an item that still fails goes on `stale` and blocks its uploads | automatic, or listed in the report | -| App drops the account from its registry (e.g. after an auth error) | Android account kept, sync off, notification with the number of waiting changes | user signs in again, or removes it | -| Rows written for another server or registry account (SyncState `owner`) | the run stops with `internal` | the app offers to start over after uploading what it can | -| DST-ambiguous start on a recurring event, unrepresentable rule | item poisoned with a reason; device shows it, never uploads its timing | limitation | +| Logout / turning sync off | pending changes uploaded first; if that fails (or deletions exceed the threshold) the user is asked before anything is deleted | — | +| Provider refuses a batch (`assert`/`tooLarge`/`provider`) | per-item retry / split; an item that still fails goes on `stale` (the state still advances) and uploads nothing until it was fetched again | automatic, or listed in the report | +| App drops the account from its registry (e.g. after an auth error) | Android account kept, sync off, one "Sign in again" notification without a count of waiting changes; the section lists the account | user signs in again, or removes it from the device | +| Rows written for another server or registry account (SyncState `owner`) | the run stops with `internal`; the settings show its message | turning sync off and on starts over; the teardown uploads nothing for the other owner and asks before its changes are lost | +| DST-ambiguous start on a recurring event, unrepresentable rule | item skipped and reported with a reason (`dstAmbiguous`, `ruleNotRepresentable`); device shows it, never uploads its timing | limitation | ## Limits and performance @@ -1219,12 +1429,19 @@ app's system settings when Android no longer asks. in flight), `maxSizeRequest` (10 MB; the engine targets 1 MB), queries of at most 5000 ids per call, `/changes` pages of 256 (Stalwart caps at 5000), cards and events of at most 512 KiB. -- Provider: ≤ 499 ops between yield points, ~300 KB per `applyBatch`, photos in - their own batch; 50 raw contacts or 50 events per chunk. +- Provider: ≤ 499 ops between yield points (the engine packs at most 400 per + batch), ~300 KB per `applyBatch`, photos in their own batch; 50 raw contacts + or 50 events per chunk. - JS thread: a `setTimeout(0)` yield between chunks, so the UI keeps rendering when the app is open; JSON crosses the bridge as strings. - Time: 9.5 min per run (2.5 min expedited), checkpoints between chunks, - continuation through `fullSyncRequested`. + continuation through `fullSyncRequested`. Checkpoints stop 30 s before the + payload's deadline and a create batch needs 60 s left (see [A sync + run](#a-sync-run)), so an expedited sync that cold-starts React Native has + little room for creates. +- Contacts: the first card with a relation to name, or the first change of a + group's members, reads every synced contact's shadow (`SYNC2`) once per run + to index the cards by uid. The 2,000 + 2,000 scale run is recorded in the parity notes when #34 closes. @@ -1276,6 +1493,10 @@ Decisions made where the brief left a choice (safe defaults): - An app account dropped from the registry keeps its Android account (sync off) until the user signs in again or removes it: its device changes would otherwise be lost. +- A shared account that refuses to list its address books or calendars is + left out of the run and only logged, not reported: Stalwart lists an account + shared for mail only with every capability, so the warning would never go + away. Its rows stay on the device, since a refusal can pass. - The user's own address for "me" and `OWNER_ACCOUNT`: the session login when it is an email address, else the first identity's address; aliases from identities also count as "me". @@ -1287,8 +1508,9 @@ Known limitations: - Fossify Contacts re-inserts rows on save: a field or entry it does not model (a second nickname or organization, a department, a label) is kept on the server, but an entry deleted or a field cleared in Fossify is not deleted or - cleared on the server. Fossify Calendar's rule edits upload only in the parts - its editor models. + cleared on the server, and comes back on the device with the contact's next + clean write. Fossify Calendar's rule edits upload only in the parts its + editor models. - Android shows extra instances for BYSETPOS rules other than monthly-BYDAY, and none for a daily series older than about 5.5 years (AOSP's 2000-period expansion cap). @@ -1296,11 +1518,18 @@ Known limitations: uploaded. `EXRULE` is not supported by Stalwart. - Recurring events whose start or override falls in a DST overlap cannot be uploaded (Stalwart drops ambiguous local times). +- An `UNTIL` in a DST gap or overlap moves one hour later, so an Etar "this + and following" split exactly at an instance inside the repeated hour can + include that instance in the old series again (better than Stalwart + dropping the `UNTIL`). - A single occurrence that Stalwart holds only as an instance (no master) is shown but read-only. +- In a calendar that grants RSVP but no writes, only the user's answer to the + whole series uploads; an answer for one instance is reverted. - Moving an event to a calendar of another JMAP account uploads as a create in the target account and a destroy in the source (a new uid; participants are - notified accordingly). + notified accordingly). An app that moves it by changing `CALENDAR_ID` in + place gets the row put back by the next download of the event. - Descriptions over 64 KB are shown truncated and are not uploaded when edited. - A shared address book or calendar that two signed-in accounts can both see appears once per account when both sync it (each Android account holds its @@ -1310,6 +1539,15 @@ Known limitations: - Photos travel as 512 px JPEG `data:` URIs; a larger original is replaced by its scaled copy only when the user changes the photo on the device. +Open: + +- Contacts: a custom `name/full` is kept even when a component edit makes it + stale; a group title cleared on the device is not uploaded; clearing company + and department leaves an organization entry without a name on the server. +- Calendar: a recurrence id in a repeated hour uses the first instant (RFC + 5545) for `ORIGINAL_INSTANCE_TIME`; to be checked against CalendarProvider's + instance expansion on a device. + ## Out of scope Possible follow-ups: From 49c5d05c3077c79fe4b1beaeb43d20b997a7ae8c Mon Sep 17 00:00:00 2001 From: Linus Rath <139418639+rathlinus@users.noreply.github.com> Date: Sun, 27 Sep 2026 06:12:08 +0200 Subject: [PATCH 19/87] fix: keep every provider write of device sync inside its own account #34 The native side ANDs the account's scope onto each selection the engine sends, as `(where) AND (scope)`. A selection such as `1) OR (1` closed the scope's parenthesis itself: on the calendar tables, where CalendarProvider adds no account filter of its own, an update or delete built that way would have reached the events, attendees and calendars of other accounts, e.g. the user's Google calendar. The engine only sends constant selections, so this guards against bugs rather than anything an outside app could send, which is what the scope is for. - Selections and sort orders must balance their parentheses outside quoted literals, and may not end the statement, carry comments or query other tables (`SELECT`, `UNION`, `ATTACH`, `PRAGMA`). - An update of a data row by id that writes `data1` without restating the mimetype has the row's mimetype looked up: on a membership row `data1` names a group, which must be the account's, as on insert. Such writes by selection are refused. - `data14`, a photo row's file id, is never written: another account's file id would have let `readPhoto` read that account's photo. Checked: 58 JVM tests (`gradlew :app:testDebugUnitTest`, three new), the debug build, and every selection the engine and planners send against the new rules. --- .../bulwarkmobile/sync/ProviderIo.kt | 8 +++ .../bulwarkmobile/sync/ProviderOps.kt | 60 +++++++++++++++- .../bulwarkmobile/sync/ProviderScope.kt | 68 +++++++++++++++++-- .../bulwarkmobile/sync/ProviderOpsTest.kt | 47 +++++++++++++ 4 files changed, 177 insertions(+), 6 deletions(-) diff --git a/android/app/src/main/java/com/anonymous/bulwarkmobile/sync/ProviderIo.kt b/android/app/src/main/java/com/anonymous/bulwarkmobile/sync/ProviderIo.kt index cb0e250d..37f23290 100644 --- a/android/app/src/main/java/com/anonymous/bulwarkmobile/sync/ProviderIo.kt +++ b/android/app/src/main/java/com/anonymous/bulwarkmobile/sync/ProviderIo.kt @@ -199,6 +199,14 @@ class ProviderIo(context: Context, accountName: String, private val authority: S override fun dataRawContacts(ids: Set): Map = idPairs(ProviderTable.DATA, "raw_contact_id", ids) + override fun dataMimetypes(ids: Set): Map { + val mimetypes = HashMap() + forEachRow(ProviderTable.DATA, arrayOf(ID, "mimetype"), ID, ids) { c -> + if (!c.isNull(1)) mimetypes[c.getLong(0)] = c.getString(1) + } + return mimetypes + } + override fun eventCalendars(ids: Set): Map = idPairs(ProviderTable.EVENTS, "calendar_id", ids) diff --git a/android/app/src/main/java/com/anonymous/bulwarkmobile/sync/ProviderOps.kt b/android/app/src/main/java/com/anonymous/bulwarkmobile/sync/ProviderOps.kt index 167bbd4d..b5d9555b 100644 --- a/android/app/src/main/java/com/anonymous/bulwarkmobile/sync/ProviderOps.kt +++ b/android/app/src/main/java/com/anonymous/bulwarkmobile/sync/ProviderOps.kt @@ -158,6 +158,61 @@ sealed interface ProviderOp { const val GROUP_MEMBERSHIP_MIMETYPE = "vnd.android.cursor.item/group_membership" +/** + * Checks a selection or sort order before the account's scope is ANDed onto + * it as `(text) AND (scope)`: its parentheses must balance outside quoted + * literals, or `1) OR (1` would close the scope's own parenthesis. It may + * not end or comment out the statement, nor query other tables. The engine's + * selections are constants with `?` arguments; this guards against bugs. + */ +object SqlFragments { + private val FORBIDDEN_WORDS = Regex("\\b(SELECT|UNION|ATTACH|DETACH|PRAGMA)\\b", RegexOption.IGNORE_CASE) + + fun check(text: String, what: String) { + var depth = 0 + var quote: Char? = null + val outside = StringBuilder(text.length) + var i = 0 + while (i < text.length) { + val c = text[i] + if (quote != null) { + if (c == quote) { + // A doubled quote is an escaped one inside the literal. + if (i + 1 < text.length && text[i + 1] == quote) i++ else quote = null + } + outside.append(' ') + } else { + when (c) { + '\'', '"' -> { + quote = c + outside.append(' ') + } + '(' -> { + depth++ + outside.append(c) + } + ')' -> { + if (--depth < 0) refuse(what, "closes a parenthesis it did not open") + outside.append(c) + } + ';' -> refuse(what, "ends the statement") + '\u0000' -> refuse(what, "contains a NUL character") + else -> outside.append(c) + } + } + i++ + } + if (quote != null) refuse(what, "leaves a quote open") + if (depth != 0) refuse(what, "leaves a parenthesis open") + val bare = outside.toString() + if ("--" in bare || "/*" in bare) refuse(what, "contains a comment") + FORBIDDEN_WORDS.find(bare)?.let { refuse(what, "uses ${it.value.uppercase()}") } + } + + private fun refuse(what: String, why: String): Nothing = + throw ProviderRefusal(BatchFailure.SCOPE, "$what $why") +} + /** * The table a written column points into when the row it names must belong to * the account as well: the parent of a row, the master of an exception event, @@ -191,7 +246,8 @@ object ProviderOpParser { } if (columns.isEmpty()) throw invalid("query needs at least one column") val (where, args) = o.whereArgs("query") - return ProviderQuery(table, columns, where, args, o.optionalString("orderBy")) + val orderBy = o.optionalString("orderBy")?.also { SqlFragments.check(it, "query: orderBy") } + return ProviderQuery(table, columns, where, args, orderBy) } fun parseBatch(authority: String, json: String): List { @@ -323,7 +379,7 @@ object ProviderOpParser { } private fun JSONObject.whereArgs(where: String): Pair> { - val sql = optionalString("where") + val sql = optionalString("where")?.also { SqlFragments.check(it, "$where: where") } val args = when (val raw = opt("args")) { null, JSONObject.NULL -> emptyList() is JSONArray -> (0 until raw.length()).map { i -> diff --git a/android/app/src/main/java/com/anonymous/bulwarkmobile/sync/ProviderScope.kt b/android/app/src/main/java/com/anonymous/bulwarkmobile/sync/ProviderScope.kt index 01729241..7706b0a1 100644 --- a/android/app/src/main/java/com/anonymous/bulwarkmobile/sync/ProviderScope.kt +++ b/android/app/src/main/java/com/anonymous/bulwarkmobile/sync/ProviderScope.kt @@ -45,21 +45,45 @@ class ScopePlanner(private val account: AccountRef, private val lookup: ScopeLoo } fun planBatch(ops: List): List { - val owners = Owners(ownershipRequests(ops)) - return ops.mapIndexed { i, op -> plan(i, op, owners) } + val memberships = membershipUpdates(ops) + val owners = Owners(ownershipRequests(ops, memberships)) + return ops.mapIndexed { i, op -> plan(i, op, owners, memberships) } } - private fun plan(i: Int, op: ProviderOp, owners: Owners): PlannedOp = when (op) { + /** + * Membership rows among the data rows that updates by id write `data1` + * of without restating the mimetype: their `data1` names a group, which + * must be the account's like on insert. + */ + private fun membershipUpdates(ops: List): Set { + val ids = ops.asSequence() + .filterIsInstance() + .filter { it.table == ProviderTable.DATA && writesUntypedData1(it.values) } + .mapNotNull { it.target.id } + .toSet() + if (ids.isEmpty()) return emptySet() + return lookup.dataMimetypes(ids).filterValues { it == GROUP_MEMBERSHIP_MIMETYPE }.keys + } + + private fun writesUntypedData1(values: Map) = + MIMETYPE !in values && (values[DATA1] ?: Cell.Null) != Cell.Null + + private fun plan(i: Int, op: ProviderOp, owners: Owners, memberships: Set): PlannedOp = when (op) { is ProviderOp.SetSyncState -> PlannedOp.SetSyncState(op.value) is ProviderOp.Insert -> { checkAccountColumns(i, op.table, op.values) + checkPhotoFile(i, op.table, op.values) checkReferences(i, op.table, op.values - op.refs.keys, owners) val values = if (op.table.scope == TableScope.ACCOUNT_COLUMNS) op.values + accountCells() else op.values PlannedOp.Insert(op.table, values, op.refs, op.yieldAllowed) } is ProviderOp.Update -> { checkAccountColumns(i, op.table, op.values) + checkPhotoFile(i, op.table, op.values) checkReferences(i, op.table, op.values, owners) + if (op.table == ProviderTable.DATA && writesUntypedData1(op.values)) { + checkMembershipUpdate(i, op, owners, memberships) + } checkTarget(i, op.table, op.target, owners) if (op.table == ProviderTable.EXTENDED_PROPERTIES) { // CalendarProvider updates extended properties only through @@ -146,6 +170,35 @@ class ScopePlanner(private val account: AccountRef, private val lookup: ScopeLoo } } + /** + * `data1` written by an update that doesn't restate the mimetype: on a + * membership row it names a group. Rows chosen by a selection could be + * memberships the lookup didn't see, so only updates by id may do it. + */ + private fun checkMembershipUpdate(i: Int, op: ProviderOp.Update, owners: Owners, memberships: Set) { + val id = op.target.id + ?: throw ProviderRefusal(BatchFailure.SCOPE, "op $i: data1 may only be written to a data row by id") + if (id !in memberships) return + val group = op.values[DATA1]?.rowId() + ?: throw ProviderRefusal(BatchFailure.PROVIDER, "op $i: a membership's data1 must be a group id") + when (owners.of(ProviderTable.GROUPS, group)) { + Owner.OURS -> Unit + Owner.FOREIGN -> throw ProviderRefusal(BatchFailure.SCOPE, "op $i: data.data1 names group $group of another account") + Owner.MISSING -> throw ProviderRefusal(BatchFailure.ASSERT, "op $i: data.data1 names group $group, which no longer exists") + } + } + + /** + * A photo row's file id (`data14`) is the provider's: it points into the + * photo store, where another account's file id would let `readPhoto` + * read that account's photo. Photos are written as bytes (`data15`). + */ + private fun checkPhotoFile(i: Int, table: ProviderTable, values: Map) { + if (table == ProviderTable.DATA && (values[DATA14] ?: Cell.Null) != Cell.Null) { + throw ProviderRefusal(BatchFailure.SCOPE, "op $i: data.data14 (a photo file id) is written by the provider only") + } + } + /** A row named by id must not belong to another account; a missing one fails on the expected count instead. */ private fun checkTarget(i: Int, table: ProviderTable, target: RowTarget, owners: Owners) { val id = target.id ?: return @@ -192,7 +245,7 @@ class ScopePlanner(private val account: AccountRef, private val lookup: ScopeLoo } /** Every row id the batch names, by table: op targets and referenced rows. */ - private fun ownershipRequests(ops: List): Map> { + private fun ownershipRequests(ops: List, memberships: Set): Map> { val requests = HashMap>() fun request(table: ProviderTable, id: Long) { requests.getOrPut(table) { HashSet() } += id @@ -209,6 +262,7 @@ class ScopePlanner(private val account: AccountRef, private val lookup: ScopeLoo is ProviderOp.Update -> { requestReferences(op.table, op.values) op.target.id?.let { request(op.table, it) } + if (op.target.id in memberships) op.values[DATA1]?.rowId()?.let { request(ProviderTable.GROUPS, it) } } is ProviderOp.Delete -> op.target.id?.let { request(op.table, it) } is ProviderOp.Assert -> op.target.id?.let { request(op.table, it) } @@ -275,6 +329,9 @@ class ScopePlanner(private val account: AccountRef, private val lookup: ScopeLoo const val ACCOUNT_NAME = "account_name" const val ACCOUNT_TYPE = "account_type" const val DATA_SET = "data_set" + private const val MIMETYPE = "mimetype" + private const val DATA1 = "data1" + private const val DATA14 = "data14" /** Ids per `IN (…)` list: a few KB of selection, far from SQLite's and Binder's limits. */ const val ID_LIST_CHUNK = 500 @@ -307,6 +364,9 @@ interface ScopeLookup { /** The raw contact of each data row that exists. */ fun dataRawContacts(ids: Set): Map + /** The mimetype of each data row that exists. */ + fun dataMimetypes(ids: Set): Map + /** The calendar of each event that exists. */ fun eventCalendars(ids: Set): Map diff --git a/android/app/src/test/java/com/anonymous/bulwarkmobile/sync/ProviderOpsTest.kt b/android/app/src/test/java/com/anonymous/bulwarkmobile/sync/ProviderOpsTest.kt index 751f0ad5..2d1f6930 100644 --- a/android/app/src/test/java/com/anonymous/bulwarkmobile/sync/ProviderOpsTest.kt +++ b/android/app/src/test/java/com/anonymous/bulwarkmobile/sync/ProviderOpsTest.kt @@ -23,6 +23,8 @@ private class FakeLookup(private val me: AccountRef = ME) : ScopeLookup { val colors = HashMap() /** data id → raw contact */ val data = HashMap() + /** data id → mimetype */ + val mimetypes = HashMap() /** event id → calendar */ val events = HashMap() /** child table → child id → event */ @@ -42,6 +44,8 @@ private class FakeLookup(private val me: AccountRef = ME) : ScopeLookup { override fun dataRawContacts(ids: Set) = data.filterKeys { it in ids } + override fun dataMimetypes(ids: Set) = mimetypes.filterKeys { it in ids } + override fun eventCalendars(ids: Set) = events.filterKeys { it in ids } override fun childEvents(table: ProviderTable, ids: Set): Map = @@ -306,6 +310,49 @@ class ProviderOpsTest { plan(CONTACTS, """{"op":"insert","table":"data","values":{"raw_contact_id":1,"mimetype":"x","data1":11}}""") } + @Test + fun `a membership row's group is checked on updates by id too`() { + lookup.mimetypes[100] = GROUP_MEMBERSHIP_MIMETYPE + plan(CONTACTS, """{"op":"update","table":"data","id":100,"values":{"data1":10}}""") + refused(BatchFailure.SCOPE) { plan(CONTACTS, """{"op":"update","table":"data","id":100,"values":{"data1":11}}""") } + refused(BatchFailure.ASSERT) { plan(CONTACTS, """{"op":"update","table":"data","id":100,"values":{"data1":12}}""") } + // data1 by a selection could reach memberships the lookup never saw. + refused(BatchFailure.SCOPE) { + plan(CONTACTS, """{"op":"update","table":"data","where":"raw_contact_id = ?","args":[1],"values":{"data1":11}}""") + } + // data1 of other kinds is plain data. + lookup.mimetypes[100] = "vnd.android.cursor.item/phone_v2" + plan(CONTACTS, """{"op":"update","table":"data","id":100,"values":{"data1":11}}""") + } + + @Test + fun `photo file ids are never written`() { + val photo = """"raw_contact_id":1,"mimetype":"vnd.android.cursor.item/photo"""" + plan(CONTACTS, """{"op":"insert","table":"data","values":{$photo,"data14":null,"data15":{"b64":"AAAA"}}}""") + refused(BatchFailure.SCOPE) { plan(CONTACTS, """{"op":"insert","table":"data","values":{$photo,"data14":5}}""") } + refused(BatchFailure.SCOPE) { plan(CONTACTS, """{"op":"update","table":"data","id":100,"values":{"data14":5}}""") } + } + + @Test + fun `a selection can't close the scope's parentheses or query elsewhere`() { + for (where in listOf("1) OR (1", "dirty = 1) OR (1 = 1", "(1", "1; DELETE FROM events", "1 -- x", "1 /* x */", + "_id IN (SELECT _id FROM events)", "1 UNION ALL x", "title = 'open")) { + val quoted = JSONObject.quote(where) + refused(BatchFailure.SCOPE) { plan(CALENDAR, """{"op":"update","table":"events","where":$quoted,"values":{"dirty":1}}""") } + refused(BatchFailure.SCOPE) { plan(CALENDAR, """{"op":"delete","table":"attendees","where":$quoted}""") } + refused(BatchFailure.SCOPE) { + ProviderOpParser.parseQuery(CALENDAR, """{"table":"attendees","columns":["_id"],"where":$quoted}""") + } + } + refused(BatchFailure.SCOPE) { + ProviderOpParser.parseQuery(CONTACTS, """{"table":"raw_contacts","columns":["_id"],"orderBy":"_id); DROP TABLE x; --"}""") + } + // Parentheses and keywords inside quoted literals are data. + plan(CALENDAR, """{"op":"update","table":"events","where":"title = '(x' OR title = 'select'","values":{"dirty":1}}""") + plan(CALENDAR, """{"op":"assert","table":"events","where":"(dirty = 1 OR deleted = 1) AND _sync_id LIKE '~pending/%'","expectCount":0}""") + ProviderOpParser.parseQuery(CONTACTS, """{"table":"raw_contacts","columns":["_id"],"orderBy":"_id DESC"}""") + } + @Test fun `moving a row to another parent is checked like an insert`() { refused(BatchFailure.SCOPE) { plan(CALENDAR, """{"op":"update","table":"events","id":30,"values":{"calendar_id":22}}""") } From 232e892328bbc13c18562ca5eee1a9d6b1630093 Mon Sep 17 00:00:00 2001 From: Linus Rath <139418639+rathlinus@users.noreply.github.com> Date: Sun, 27 Sep 2026 06:57:28 +0200 Subject: [PATCH 20/87] fix: keep an occurrence's description or location empty once cleared #34 Clearing the description or location of one occurrence of a series in a calendar app removed the value from the occurrence's override. An override without a value of its own shows the series' one, so the write after the upload put "Daily agenda" or "Room 1" back into the occurrence and made it clean: the user's edit was gone without a trace. An existing override now gets an explicit empty value while the series has one to fall back to: the empty text for the description, and a location without a name, the only empty forms Stalwart keeps (calcard writes no LOCATION line for an empty map, which then reads back as the series' locations). Without a series value the override's own is removed as before. New exception rows keep treating an empty value as "inherit". Checked with a planner test and a run through the engine and the fake server: the row stays cleared and clean, and the echo writes nothing. --- .../__tests__/calendar/engine.test.ts | 47 +++++++++++++++++++ .../__tests__/calendar/exceptions.test.ts | 43 +++++++++++++++++ src/device-sync/calendar/upload.ts | 20 +++++++- 3 files changed, 108 insertions(+), 2 deletions(-) create mode 100644 src/device-sync/__tests__/calendar/engine.test.ts diff --git a/src/device-sync/__tests__/calendar/engine.test.ts b/src/device-sync/__tests__/calendar/engine.test.ts new file mode 100644 index 00000000..d7d30337 --- /dev/null +++ b/src/device-sync/__tests__/calendar/engine.test.ts @@ -0,0 +1,47 @@ +// The calendar planner through the engine and the fake JMAP server: device +// edits reach the server as the user made them and the rows stay as the user +// left them, end to end. Mapping details are the planner's own tests. + +import { describe, expect, it } from 'vitest'; +import { Events } from '../../android-columns'; +import { calendarPlanner } from '../../calendar/planner'; +import { contactsPlanner } from '../../contacts/planner'; +import { CALENDAR_AUTHORITY, createHarness, type Harness } from '../engine/harness'; + +function real(options: Parameters[0] = {}): Harness { + const h = createHarness(options); + h.deps.planners = { contacts: contactsPlanner, calendar: calendarPlanner }; + return h; +} + +type Overrides = Record>; + +describe('calendar device edits through the engine', () => { + it('keeps a description and location cleared on one occurrence cleared', async () => { + const h = real(); + const id = h.server.addEvent('a', { + uid: 'standup-uid', + title: 'Standup', + description: 'Daily agenda', + start: '2026-09-28T09:00:00', + duration: 'PT15M', + timeZone: 'Europe/Berlin', + recurrenceRule: { '@type': 'RecurrenceRule', frequency: 'daily', count: 5 }, + locations: { locA: { '@type': 'Location', name: 'Room 1' } }, + recurrenceOverrides: { + '2026-09-29T09:00:00': { title: 'Standup', description: 'Special agenda', locations: { locB: { '@type': 'Location', name: 'Room 2' } } }, + }, + calendarIds: { [h.calendar]: true }, + }); + await h.run(CALENDAR_AUTHORITY); + const exception = () => h.events().find((e) => e[Events.ORIGINAL_SYNC_ID] === `a/${id}`)!; + expect(exception()).toMatchObject({ [Events.DESCRIPTION]: 'Special agenda', [Events.EVENT_LOCATION]: 'Room 2' }); + + h.device.user.updateEvent(Number(exception()._id), { [Events.DESCRIPTION]: null, [Events.EVENT_LOCATION]: null }); + expect(await h.run(CALENDAR_AUTHORITY)).toMatchObject({ outcome: 'ok', itemErrors: [], stats: { uploaded: { updated: 1 } } }); + + expect(exception()).toMatchObject({ [Events.DESCRIPTION]: null, [Events.EVENT_LOCATION]: null, [Events.DIRTY]: 0 }); + const overrides = h.server.get('CalendarEvent', 'a', id)!.recurrenceOverrides as Overrides; + expect(overrides['2026-09-29T09:00:00']).toMatchObject({ description: '', locations: { locB: { name: '' } } }); + }); +}); diff --git a/src/device-sync/__tests__/calendar/exceptions.test.ts b/src/device-sync/__tests__/calendar/exceptions.test.ts index 540f73a6..82007b54 100644 --- a/src/device-sync/__tests__/calendar/exceptions.test.ts +++ b/src/device-sync/__tests__/calendar/exceptions.test.ts @@ -103,6 +103,49 @@ describe('calendar planner: recurrence edits made on the device', () => { expect(h.row(x.eventId)).toBeUndefined(); }); + it('clears an occurrence\'s own description and location instead of letting the series\' come back', async () => { + const event = weekly(); + event.description = 'Daily agenda'; + event.recurrenceOverrides!['2026-10-07T09:00:00'] = { + ...event.recurrenceOverrides!['2026-10-07T09:00:00'], + description: 'Special agenda', + locations: { locB: { '@type': 'Location', name: 'Room 2' } }, + }; + const { h } = await synced(event); + const x = (await h.local('bl'))!.exceptions[0]; + expect(h.row(x.eventId)).toMatchObject({ [Events.DESCRIPTION]: 'Special agenda', [Events.EVENT_LOCATION]: 'Room 2' }); + h.fake.user.updateEvent(x.eventId, { [Events.DESCRIPTION]: null, [Events.EVENT_LOCATION]: null }); + const local = (await h.local('bl'))!; + const plan = calendarPlanner.planUpload(local, h.ctx); + const server = await accept(h, event, local, plan); + expect(h.row(x.eventId)).toMatchObject({ [Events.DESCRIPTION]: null, [Events.EVENT_LOCATION]: null, [Events.DIRTY]: 0 }); + expect(calendarPlanner.planDownload(server, (await h.local('bl'))!, h.ctx).effect).toBe('none'); + // Without a value of its own the override would show the series' "Daily agenda" and "Room 1" again, so it + // gets the empty ones Stalwart keeps (an empty map would read back as no locations, i.e. the series' ones). + expect(patchOf(plan).patch).toEqual({ + 'recurrenceOverrides/2026-10-07T09:00:00/description': '', + 'recurrenceOverrides/2026-10-07T09:00:00/locations': { locB: { '@type': 'Location', name: '' } }, + }); + }); + + it('removes an occurrence\'s own description and location when the series has none to show instead', async () => { + const event = weekly(); + delete event.description; + delete event.locations; + event.recurrenceOverrides!['2026-10-07T09:00:00'] = { + ...event.recurrenceOverrides!['2026-10-07T09:00:00'], + description: 'Special agenda', + locations: { locB: { '@type': 'Location', name: 'Room 2' } }, + }; + const { h } = await synced(event); + const x = (await h.local('bl'))!.exceptions[0]; + h.fake.user.updateEvent(x.eventId, { [Events.DESCRIPTION]: null, [Events.EVENT_LOCATION]: null }); + expect(patchOf(calendarPlanner.planUpload((await h.local('bl'))!, h.ctx)).patch).toEqual({ + 'recurrenceOverrides/2026-10-07T09:00:00/description': null, + 'recurrenceOverrides/2026-10-07T09:00:00/locations': null, + }); + }); + it('patches an existing override one level deep, never deeper', async () => { const { h } = await synced(weekly()); const x = (await h.local('bl'))!.exceptions[0]; diff --git a/src/device-sync/calendar/upload.ts b/src/device-sync/calendar/upload.ts index d312f3b8..fb877a51 100644 --- a/src/device-sync/calendar/upload.ts +++ b/src/device-sync/calendar/upload.ts @@ -454,10 +454,10 @@ function exceptionChanges( break; case 'description': if (x.baseline && isTruncatedBaseline(x.baseline.cells[Events.DESCRIPTION])) continue; - values.description = descriptionValue(cells); + values.description = descriptionValue(cells) ?? clearedInOverride('description', instance, shadow); break; case 'location': - values.locations = locationChanges(instance, cells[Events.EVENT_LOCATION], ctx.mintKey).locations; + values.locations = locationChanges(instance, cells[Events.EVENT_LOCATION], ctx.mintKey).locations ?? clearedInOverride('location', instance, shadow); break; case 'status': values.status = statusFromDevice(cells[Events.STATUS]); @@ -530,6 +530,22 @@ function isEmptyUnit(unit: string, cells: Row): boolean { return v === null || v === undefined || v === ''; } +/** + * A description or location cleared on an existing override. An override + * without one of its own shows the series' again, so while the series has + * one the override gets the empty value Stalwart keeps: the empty text, or a + * location without a name (an empty map writes no LOCATION line and reads + * back as none of its own). Null, a clean removal, when nothing would come + * back. + */ +function clearedInOverride(unit: 'description' | 'location', instance: CalendarEventWire, master: CalendarEventWire): unknown { + if (unit === 'description') return typeof master.description === 'string' && master.description ? '' : null; + const series = Object.keys(master.locations ?? {}); + if (!series.length) return null; + const key = Object.keys(instance.locations ?? {})[0] ?? series[0]; + return { [key]: { '@type': 'Location', name: '' } }; +} + /** * A new override: the changes plus the details Stalwart needs on it (title, * locations, participants, sequence, …; `withNewOverrideDetails`), minus what From 2870a5997165f0147c2f99b8a6cec26c52bf9e36 Mon Sep 17 00:00:00 2001 From: Linus Rath <139418639+rathlinus@users.noreply.github.com> Date: Sun, 27 Sep 2026 06:58:16 +0200 Subject: [PATCH 21/87] fix: stop events from Google Calendar arriving as tentative #34 Google Calendar leaves STATUS empty on the events and exceptions it creates, and CalendarProvider has no default for it. Device sync read the empty value as 0, STATUS_TENTATIVE, so every event and every "this event" edit made there uploaded as "tentative" and came back to the phone that way; a copy made with "Copy to" also never paired with the deleted original. An empty STATUS now sets no status: a create leaves `status` out, so the server keeps its default of confirmed, an override gets none of its own, an edited event never uploads one, a merge takes the server's without counting a conflict, and a pair ignores it. After the upload the rows hold the server's status. A new series from Etar's "this and following" also no longer uploads the old series' exclusions from before its start, which Etar copies into the new row's EXDATE. Checked with planner tests for each path and a run through the engine and the fake server. --- .../__tests__/calendar/engine.test.ts | 41 ++++++++++++++ .../__tests__/calendar/exceptions.test.ts | 15 +++++ .../__tests__/calendar/lifecycle.test.ts | 56 +++++++++++++++++++ src/device-sync/calendar/create.ts | 10 +++- src/device-sync/calendar/merge.ts | 3 +- src/device-sync/calendar/pairs.ts | 3 + src/device-sync/calendar/units.ts | 11 ++++ src/device-sync/calendar/upload.ts | 5 +- src/device-sync/calendar/values.ts | 13 ++++- 9 files changed, 150 insertions(+), 7 deletions(-) diff --git a/src/device-sync/__tests__/calendar/engine.test.ts b/src/device-sync/__tests__/calendar/engine.test.ts index d7d30337..9729c4cf 100644 --- a/src/device-sync/__tests__/calendar/engine.test.ts +++ b/src/device-sync/__tests__/calendar/engine.test.ts @@ -17,6 +17,47 @@ function real(options: Parameters[0] = {}): Harness { type Overrides = Record>; describe('calendar device edits through the engine', () => { + it('creates an event and an occurrence edit an app left without a status as confirmed ones', async () => { + const h = real(); + const series = h.server.addEvent('a', { + uid: 'weekly-uid', + title: 'Weekly', + start: '2026-10-05T08:00:00', + duration: 'PT1H', + timeZone: 'Europe/Berlin', + recurrenceRule: { '@type': 'RecurrenceRule', frequency: 'weekly', count: 8 }, + calendarIds: { [h.calendar]: true }, + }); + await h.run(CALENDAR_AUTHORITY); + const master = h.events().find((e) => e[Events._SYNC_ID] === `a/${series}`)!; + const rowId = Number(master[Events.CALENDAR_ID]); + + // Google Calendar leaves STATUS NULL on the events and exceptions it inserts. + const created = h.device.user.insertEvent(rowId, { + [Events.TITLE]: 'New from Google Calendar', + [Events.DTSTART]: Date.UTC(2026, 9, 14, 13), + [Events.DTEND]: Date.UTC(2026, 9, 14, 14), + [Events.EVENT_TIMEZONE]: 'Europe/Berlin', + [Events.STATUS]: null, + }); + h.device.user.insertException(Number(master._id), Date.UTC(2026, 10, 9, 7), { + [Events.TITLE]: 'Only Nov 9', + [Events.DTSTART]: Date.UTC(2026, 10, 9, 7), + [Events.DTEND]: Date.UTC(2026, 10, 9, 8), + [Events.STATUS]: null, + }); + expect(await h.run(CALENDAR_AUTHORITY)).toMatchObject({ outcome: 'ok', itemErrors: [], stats: { uploaded: { created: 1, updated: 1 } } }); + + const event = h.server.all('CalendarEvent', 'a').find((e) => e.title === 'New from Google Calendar')!; + expect(event.status).toBeUndefined(); + const overrides = h.server.get('CalendarEvent', 'a', series)!.recurrenceOverrides as Overrides; + expect(overrides['2026-11-09T08:00:00']).toMatchObject({ title: 'Only Nov 9' }); + expect(overrides['2026-11-09T08:00:00'].status).toBeUndefined(); + // The rows hold the default, confirmed, and are clean. + expect(h.events().find((e) => Number(e._id) === created)).toMatchObject({ [Events.STATUS]: 1, [Events.DIRTY]: 0 }); + expect(h.events().filter((e) => e[Events.ORIGINAL_SYNC_ID] === `a/${series}`).map((e) => [e[Events.STATUS], e[Events.DIRTY]])).toEqual([[1, 0]]); + }); + it('keeps a description and location cleared on one occurrence cleared', async () => { const h = real(); const id = h.server.addEvent('a', { diff --git a/src/device-sync/__tests__/calendar/exceptions.test.ts b/src/device-sync/__tests__/calendar/exceptions.test.ts index 82007b54..0b35614d 100644 --- a/src/device-sync/__tests__/calendar/exceptions.test.ts +++ b/src/device-sync/__tests__/calendar/exceptions.test.ts @@ -103,6 +103,21 @@ describe('calendar planner: recurrence edits made on the device', () => { expect(h.row(x.eventId)).toBeUndefined(); }); + it('does not make an occurrence an app left without a status (Google Calendar\'s "this event") tentative', async () => { + const event = single({ recurrenceRule: { frequency: 'weekly', count: 5 } }); + const { h, id } = await synced(event); + h.fake.user.insertException(id, utc('2026-10-13T10:00:00Z'), { + [Events.TITLE]: 'Only this Tuesday', [Events.DTSTART]: utc('2026-10-13T10:00:00Z'), [Events.DTEND]: utc('2026-10-13T11:00:00Z'), [Events.STATUS]: null, + }); + const local = (await h.local('e1'))!; + const plan = calendarPlanner.planUpload(local, h.ctx); + const overrides = patchOf(plan).patch.recurrenceOverrides as Record>; + expect(overrides['2026-10-13T12:00:00']).toEqual({ title: 'Only this Tuesday', start: '2026-10-13T12:00:00', duration: 'PT1H' }); + await accept(h, event, local, plan); + const [x] = (await h.local('e1'))!.exceptions; + expect(x.cells).toMatchObject({ [Events.STATUS]: 1, [Events.DIRTY]: 0 }); + }); + it('clears an occurrence\'s own description and location instead of letting the series\' come back', async () => { const event = weekly(); event.description = 'Daily agenda'; diff --git a/src/device-sync/__tests__/calendar/lifecycle.test.ts b/src/device-sync/__tests__/calendar/lifecycle.test.ts index 994f2972..5acc036e 100644 --- a/src/device-sync/__tests__/calendar/lifecycle.test.ts +++ b/src/device-sync/__tests__/calendar/lifecycle.test.ts @@ -100,6 +100,37 @@ describe('calendar planner: new events', () => { expect(calendarPlanner.planDownload(server, after, h.ctx).effect).toBe('none'); }); + it('creates an event an app left without a status (Google Calendar) without making it tentative', async () => { + const h = await new Harness().setup(); + // CalendarProvider has no default for STATUS: an app that does not write it leaves it NULL. + const id = h.fake.user.insertEvent(h.rowIds.get('b')!, { + [Events.TITLE]: 'From Google Calendar', [Events.DTSTART]: utc('2026-10-14T15:00:00Z'), [Events.DTEND]: utc('2026-10-14T16:00:00Z'), [Events.EVENT_TIMEZONE]: 'Europe/Berlin', [Events.STATUS]: null, + }); + const { local, create } = await claimAndCreate(h, id); + expect(create.object).not.toHaveProperty('status'); + const server = { ...(create.object as CalendarEventWire), id: 'g1' } as CalendarEventWire; + await h.apply(calendarPlanner.planAccepted(local, server, h.ctx).ops); + // Written as JSCalendar's default, confirmed. + expect(h.row(id)).toMatchObject({ [Events.STATUS]: 1, [Events.DIRTY]: 0 }); + expect(calendarPlanner.planDownload(server, (await h.local('g1'))!, h.ctx).effect).toBe('none'); + }); + + it('leaves out exclusions a new series inherited from before its start (Etar\'s split copies the EXDATE)', async () => { + const h = await new Harness().setup(); + const id = h.fake.user.insertEvent(h.rowIds.get('b')!, { + [Events.TITLE]: 'Weekly (new time)', + [Events.DTSTART]: utc('2026-11-16T09:00:00Z'), + [Events.DURATION]: 'P3600S', + [Events.RRULE]: 'FREQ=WEEKLY;COUNT=3;BYDAY=MO', + [Events.EVENT_TIMEZONE]: 'Europe/Berlin', + // The old series' exclusions (Oct 12, Nov 2) and one of this series (Nov 23). + [Events.EXDATE]: '20261012T060000Z,20261102T070000Z,20261123T090000Z', + [Events.STATUS]: 1, + }); + const { create } = await claimAndCreate(h, id); + expect(create.object.recurrenceOverrides).toEqual({ '2026-11-23T10:00:00': { excluded: true } }); + }); + it('invites the attendees of a new event as its organizer', async () => { const h = await new Harness().setup(); const id = h.fake.user.insertEvent( @@ -232,6 +263,20 @@ describe('calendar planner: accepted uploads, merges and deletions', () => { expect(h.row(id)).toMatchObject({ [Events.TITLE]: 'Server title', [Events.DIRTY]: 0 }); }); + it('never reads a status an app left NULL as an edit, in uploads or merges', async () => { + const { h, id } = await synced(weekly()); + expect(h.row(id)[Events.STATUS]).toBe(0); + // A row an app wrote without STATUS (only an insert can leave it NULL), then edited. + h.fake.table('events').get(id)![Events.STATUS] = null; + h.fake.user.updateEvent(id, { [Events.TITLE]: 'Renamed' }); + expect(actionsOf(calendarPlanner.planUpload((await h.local('bl'))!, h.ctx))[0]).toMatchObject({ kind: 'update', patch: { title: 'Renamed' } }); + expect(Object.keys((actionsOf(calendarPlanner.planUpload((await h.local('bl'))!, h.ctx))[0] as { patch: object }).patch)).toEqual(['title']); + // The server's status change meanwhile is no conflict: the row takes it, the title stays for the upload. + const plan = await h.download({ ...weekly(), status: 'cancelled' }); + expect(plan).toMatchObject({ conflicts: 0, stillDirty: true }); + expect(h.row(id)).toMatchObject({ [Events.STATUS]: 2, [Events.TITLE]: 'Renamed', [Events.DIRTY]: 1 }); + }); + it('treats the same change on both sides as converged', async () => { const { h, id } = await synced(weekly()); h.fake.user.updateEvent(id, { [Events.TITLE]: 'Same' }); @@ -362,6 +407,17 @@ describe('calendar planner: pairs', () => { expect(pairs.map((p) => p.actions)).toEqual([[{ kind: 'update', id: 'e1', patch: { recurrenceRule: null } }]]); }); + it('pairs a copy an app inserted without a status (Google Calendar\'s "Copy to" + delete) with the original', async () => { + const { h, id } = await synced(single()); + h.fake.user.deleteEvent(id); + h.fake.user.insertEvent(h.rowIds.get('w')!, { + [Events.TITLE]: 'Lunch', [Events.DTSTART]: utc('2026-10-06T10:00:00Z'), [Events.DTEND]: utc('2026-10-06T11:00:00Z'), [Events.EVENT_TIMEZONE]: 'Europe/Berlin', [Events.STATUS]: null, [Events.AVAILABILITY]: 0, [Events.ACCESS_LEVEL]: 0, + }); + const events = await h.events(); + const pairs = calendarPlanner.planPairs(events.filter((e) => e.deleted), events.filter((e) => !e.syncId), h.ctx); + expect(pairs.map((p) => p.actions)).toEqual([[{ kind: 'update', id: 'e1', patch: { 'calendarIds/b': null, 'calendarIds/w': true } }]]); + }); + it('does not pair unrelated rows', async () => { const { h, id } = await synced(single()); h.fake.user.deleteEvent(id); diff --git a/src/device-sync/calendar/create.ts b/src/device-sync/calendar/create.ts index bad4ec4f..e1ed3fa3 100644 --- a/src/device-sync/calendar/create.ts +++ b/src/device-sync/calendar/create.ts @@ -54,9 +54,11 @@ export function computeCreate(local: LocalEvent, uid: string, calendarId: string title: titleValue(cells), start: t.start, duration: t.duration, - status: statusFromDevice(cells[Events.STATUS]), freeBusyStatus: availabilityFromDevice(cells[Events.AVAILABILITY]), }; + // A row without STATUS gets JSCalendar's default, confirmed. + const status = statusFromDevice(cells[Events.STATUS]); + if (status) object.status = status; if (t.showWithoutTime) object.showWithoutTime = true; else object.timeZone = t.timeZone; const description = descriptionValue(cells); @@ -88,8 +90,10 @@ export function computeCreate(local: LocalEvent, uid: string, calendarId: string return key; }; for (const entry of exdateEntries(sideOfRow(local))) { - const key = checked(exdateEntryKey(entry, zone)); - if (key) overrides[key] = { excluded: true }; + const key = exdateEntryKey(entry, zone); + // Etar's "this and following" copies the old series' EXDATE: dates before the start are none of this series. + if (!key || key < t.start) continue; + overrides[checked(key)!] = { excluded: true }; } for (const x of local.exceptions) { const key = checked(x.recurrenceId); diff --git a/src/device-sync/calendar/merge.ts b/src/device-sync/calendar/merge.ts index 53715e5a..dc65b48e 100644 --- a/src/device-sync/calendar/merge.ts +++ b/src/device-sync/calendar/merge.ts @@ -42,6 +42,7 @@ import { attendeesByKey, baselineOfImage, columnUnitDiffers, + deviceChangedUnit, exdateEntries, isLossyEditor, remindersDiffer, @@ -135,7 +136,7 @@ export function mergeRow(input: RowMergeInput): RowMerge { }; for (const unit of input.isException ? EXCEPTION_COLUMN_UNITS : MASTER_COLUMN_UNITS) { - const local = (dirty || forced.has(unit)) && columnUnitDiffers(unit, cur, bl); + const local = (dirty || forced.has(unit)) && deviceChangedUnit(unit, cur, bl); const choice = decide(unit, local, columnUnitDiffers(unit, bs, rm), !columnUnitDiffers(unit, cur, rm)); if (choice === 'local') { for (const c of COLUMN_UNITS[unit]) { diff --git a/src/device-sync/calendar/pairs.ts b/src/device-sync/calendar/pairs.ts index 8f4eeb81..80513bd2 100644 --- a/src/device-sync/calendar/pairs.ts +++ b/src/device-sync/calendar/pairs.ts @@ -21,6 +21,7 @@ import { GroupBuilder, assertRow, deleteRow, updateRow } from './rows'; import { rruleParts } from './rrule'; import { sendsSchedulingMessages } from './scheduling'; import { put } from './upload'; +import { isUnsetStatus } from './values'; /** What must be equal for two rows to be one event (the calendar and the rule aside). */ const PAIR_COLUMNS = [ @@ -58,6 +59,8 @@ export interface PairKind { /** How a deleted and a new row are one edit, or null. */ export function pairKind(deleted: LocalEvent, fresh: LocalEvent): PairKind | null { for (const c of PAIR_COLUMNS) { + // A copy inserted without STATUS (Google Calendar) says nothing about it. + if (c === Events.STATUS && (isUnsetStatus(deleted.cells[c]) || isUnsetStatus(fresh.cells[c]))) continue; if (String(deleted.cells[c] ?? '') !== String(fresh.cells[c] ?? '')) return null; } const allDay = Number(fresh.cells[Events.ALL_DAY] ?? 0) === 1; diff --git a/src/device-sync/calendar/units.ts b/src/device-sync/calendar/units.ts index b397857e..2046f8fa 100644 --- a/src/device-sync/calendar/units.ts +++ b/src/device-sync/calendar/units.ts @@ -12,6 +12,7 @@ import { MASTER_CELLS, EXCEPTION_CELLS, makeBaseline, sameCell } from './columns import { exdateSet } from './exdate'; import { sameReminders } from './reminders'; import type { RowImage } from './image'; +import { isUnsetStatus } from './values'; export const COLUMN_UNITS: Record = { title: [Events.TITLE], @@ -73,6 +74,16 @@ export function columnUnitDiffers(unit: string, a: Side, b: Side): boolean { return (COLUMN_UNITS[unit] ?? []).some((c) => !sameCell(c, a.cells[c], b.cells[c])); } +/** + * Whether the device changed a column unit against what it is measured by + * (the baseline, or the plain instance of a new exception row). A STATUS an + * app left NULL is no change: it sets no status (`statusFromDevice`). + */ +export function deviceChangedUnit(unit: string, current: Side, reference: Side): boolean { + if (unit === 'status' && isUnsetStatus(current.cells[Events.STATUS])) return false; + return columnUnitDiffers(unit, current, reference); +} + /** Attendee rows by address key (the user's aliases are one key). */ export function attendeesByKey(rows: Row[], ctx: SelfContext): Map { const out = new Map(); diff --git a/src/device-sync/calendar/upload.ts b/src/device-sync/calendar/upload.ts index fb877a51..012f611b 100644 --- a/src/device-sync/calendar/upload.ts +++ b/src/device-sync/calendar/upload.ts @@ -58,6 +58,7 @@ import { MASTER_COLUMN_UNITS, baselineOfImage, columnUnitDiffers, + deviceChangedUnit, exdateEntries, isLossyEditor, remindersDiffer, @@ -219,7 +220,7 @@ export function computeUpdate(local: LocalEvent, ctx: CalendarContext): UploadCo const changed = (unit: string) => { if (!bl) return false; const forced = local.split === 'source' && unit === 'rule'; - return (local.dirty || forced) && columnUnitDiffers(unit, cur, bl); + return (local.dirty || forced) && deviceChangedUnit(unit, cur, bl); }; for (const unit of MASTER_COLUMN_UNITS) { @@ -444,7 +445,7 @@ function exceptionChanges( if (unit === 'privacy') continue; // Stalwart drops privacy from overrides. // Etar answers one instance with STATUS_CONFIRMED on it; an attendee may not change an event's status. if (unit === 'status' && !organizer) continue; - if (unit !== 'timing' && !columnUnitDiffers(unit, cur, reference)) continue; + if (unit !== 'timing' && !deviceChangedUnit(unit, cur, reference)) continue; // A new exception row holds what its app copied: an empty value there is one it did not model, not a deletion. if (!existing && INHERITED_WHEN_EMPTY.has(unit) && isEmptyUnit(unit, x.cells)) continue; const cells = x.cells; diff --git a/src/device-sync/calendar/values.ts b/src/device-sync/calendar/values.ts index 402d0639..49449ac9 100644 --- a/src/device-sync/calendar/values.ts +++ b/src/device-sync/calendar/values.ts @@ -27,13 +27,24 @@ export function statusToDevice(status: unknown): number { } } -export function statusFromDevice(value: unknown): 'confirmed' | 'tentative' | 'cancelled' { +/** + * STATUS → `status`; null when the row has none. CalendarProvider has no + * default for STATUS, and apps that do not model it (Google Calendar) leave + * it NULL on the events and exceptions they insert: that sets no status. + */ +export function statusFromDevice(value: unknown): 'confirmed' | 'tentative' | 'cancelled' | null { + if (isUnsetStatus(value)) return null; const n = Number(value); if (n === EventStatus.TENTATIVE) return 'tentative'; if (n === EventStatus.CANCELED) return 'cancelled'; return 'confirmed'; } +/** A STATUS cell an app left NULL (see `statusFromDevice`). */ +export function isUnsetStatus(value: unknown): boolean { + return value === null || value === undefined || value === ''; +} + /** `freeBusyStatus` → AVAILABILITY; JSCalendar's default is busy. */ export function availabilityToDevice(freeBusy: unknown): number { return String(freeBusy ?? '').toLowerCase() === 'free' ? EventAvailability.FREE : EventAvailability.BUSY; From 7736b3b55176fa18e9626957b3c5e1bd88d1ad20 Mon Sep 17 00:00:00 2001 From: Linus Rath <139418639+rathlinus@users.noreply.github.com> Date: Sun, 27 Sep 2026 06:59:02 +0200 Subject: [PATCH 22/87] fix: keep a reminder changed on one occurrence of a series #34 In a series that uses the calendar's default reminders, a reminder set on one occurrence went into the occurrence's override, but the occurrence still used the defaults it inherits from the series, which ignore an event's own alerts. The phone showed the default reminder again after the sync while the server held the new one. The override now also leaves the defaults (`useDefaultAlerts: false`), so only that occurrence uses its own alerts and the series keeps the calendar's defaults. Stalwart stores the flag of an override with the override (as a JSPROP) and serves it back. Checked with a planner test (an existing override and a new "this event" edit) and a run through the engine and the fake server. --- .../__tests__/calendar/engine.test.ts | 38 ++++++++++++++++++- .../__tests__/calendar/exceptions.test.ts | 35 ++++++++++++++++- src/device-sync/calendar/upload.ts | 3 ++ 3 files changed, 74 insertions(+), 2 deletions(-) diff --git a/src/device-sync/__tests__/calendar/engine.test.ts b/src/device-sync/__tests__/calendar/engine.test.ts index 9729c4cf..e24bef26 100644 --- a/src/device-sync/__tests__/calendar/engine.test.ts +++ b/src/device-sync/__tests__/calendar/engine.test.ts @@ -3,7 +3,7 @@ // left them, end to end. Mapping details are the planner's own tests. import { describe, expect, it } from 'vitest'; -import { Events } from '../../android-columns'; +import { Events, Reminders } from '../../android-columns'; import { calendarPlanner } from '../../calendar/planner'; import { contactsPlanner } from '../../contacts/planner'; import { CALENDAR_AUTHORITY, createHarness, type Harness } from '../engine/harness'; @@ -58,6 +58,42 @@ describe('calendar device edits through the engine', () => { expect(h.events().filter((e) => e[Events.ORIGINAL_SYNC_ID] === `a/${series}`).map((e) => [e[Events.STATUS], e[Events.DIRTY]])).toEqual([[1, 0]]); }); + it('keeps a reminder changed on one occurrence of a series on the calendar\'s default alerts', async () => { + const h = real(); + h.server.serverUpdate('Calendar', 'a', h.calendar, { + defaultAlertsWithTime: { d1: { '@type': 'Alert', trigger: { '@type': 'OffsetTrigger', offset: '-PT15M' }, action: 'display' } }, + }); + const id = h.server.addEvent('a', { + uid: 'standup-uid', + title: 'Standup', + start: '2026-09-28T09:00:00', + duration: 'PT15M', + timeZone: 'Europe/Berlin', + recurrenceRule: { '@type': 'RecurrenceRule', frequency: 'daily', count: 5 }, + recurrenceOverrides: { '2026-09-29T09:00:00': { title: 'Standup (late)', start: '2026-09-29T10:00:00' } }, + useDefaultAlerts: true, + calendarIds: { [h.calendar]: true }, + }); + await h.run(CALENDAR_AUTHORITY); + const minutes = (eventId: unknown) => h.device.rows('reminders').filter((r) => r[Reminders.EVENT_ID] === Number(eventId)).map((r) => Number(r[Reminders.MINUTES])); + const master = h.events().find((e) => e[Events._SYNC_ID] === `a/${id}`)!; + const exception = () => h.events().find((e) => e[Events.ORIGINAL_SYNC_ID] === `a/${id}`)!; + expect([minutes(master._id), minutes(exception()._id)]).toEqual([[15], [15]]); + + h.device.user.setReminders(Number(exception()._id), [{ minutes: 60 }]); + expect(await h.run(CALENDAR_AUTHORITY)).toMatchObject({ outcome: 'ok', itemErrors: [], stats: { uploaded: { updated: 1 } } }); + + expect(exception()[Events.DIRTY]).toBe(0); + expect([minutes(master._id), minutes(exception()._id)]).toEqual([[15], [60]]); + const event = h.server.get('CalendarEvent', 'a', id)!; + expect(event.useDefaultAlerts).toBe(true); + expect((event.recurrenceOverrides as Overrides)['2026-09-29T09:00:00']).toMatchObject({ useDefaultAlerts: false }); + // Its echo writes nothing. + const before = h.batches.log.length; + await h.run(CALENDAR_AUTHORITY); + expect(h.batches.log.slice(before).filter((ops) => ops.some((op) => op.op !== 'syncState' && op.op !== 'assert'))).toEqual([]); + }); + it('keeps a description and location cleared on one occurrence cleared', async () => { const h = real(); const id = h.server.addEvent('a', { diff --git a/src/device-sync/__tests__/calendar/exceptions.test.ts b/src/device-sync/__tests__/calendar/exceptions.test.ts index 0b35614d..7e371a89 100644 --- a/src/device-sync/__tests__/calendar/exceptions.test.ts +++ b/src/device-sync/__tests__/calendar/exceptions.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it } from 'vitest'; -import { Attendees, Events } from '../../android-columns'; +import { Attendees, Events, Reminders } from '../../android-columns'; import { calendarPlanner } from '../../calendar/planner'; import type { LocalEvent, UploadPlan } from '../../planner'; import type { CalendarEventWire } from '../../wire'; @@ -161,6 +161,39 @@ describe('calendar planner: recurrence edits made on the device', () => { }); }); + it('keeps a reminder changed on one occurrence of a series that uses the calendar\'s default alerts', async () => { + const event = weekly(); + delete event.alerts; + event.useDefaultAlerts = true; + const { h, id } = await synced(event); + const minutes = (eventId: number) => h.fake.rows('reminders', `${Reminders.EVENT_ID} = ?`, [eventId]).map((r) => Number(r[Reminders.MINUTES])); + const x = (await h.local('bl'))!.exceptions[0]; + // The calendar's default (10 minutes) on the series and on its occurrences. + expect([minutes(id), minutes(x.eventId)]).toEqual([[10], [10]]); + h.fake.user.setReminders(x.eventId, [{ minutes: 60 }]); + // A new occurrence edit with its own reminder, too (Etar "this event"). + const added = h.fake.user.insertException(id, utc('2026-10-14T13:00:00Z'), { + [Events.TITLE]: 'Probe weekly', [Events.DTSTART]: utc('2026-10-14T13:00:00Z'), [Events.DTEND]: utc('2026-10-14T14:30:00Z'), + }); + h.fake.user.setReminders(added, [{ minutes: 5 }]); + const local = (await h.local('bl'))!; + const plan = calendarPlanner.planUpload(local, h.ctx); + const server = await accept(h, event, local, plan); + const after = (await h.local('bl'))!; + expect(after.exceptions.map((e) => [e.recurrenceId, minutes(e.eventId), e.dirty])).toEqual([ + ['2026-10-07T09:00:00', [60], false], + ['2026-10-14T09:00:00', [5], false], + ]); + // The series and its other occurrences keep the calendar's defaults. + expect(server.useDefaultAlerts).toBe(true); + expect(minutes(id)).toEqual([10]); + expect(calendarPlanner.planDownload(server, after, h.ctx).effect).toBe('none'); + // Only the occurrences leave the defaults: `useDefaultAlerts` is a property an override can patch. + const patch = patchOf(plan).patch; + expect(patch['recurrenceOverrides/2026-10-07T09:00:00/useDefaultAlerts']).toBe(false); + expect(patch['recurrenceOverrides/2026-10-14T09:00:00']).toMatchObject({ useDefaultAlerts: false }); + }); + it('patches an existing override one level deep, never deeper', async () => { const { h } = await synced(weekly()); const x = (await h.local('bl'))!.exceptions[0]; diff --git a/src/device-sync/calendar/upload.ts b/src/device-sync/calendar/upload.ts index 012f611b..4476719e 100644 --- a/src/device-sync/calendar/upload.ts +++ b/src/device-sync/calendar/upload.ts @@ -509,6 +509,9 @@ function exceptionChanges( } if (ctx.reminderOwner === 'device' && (existing || cur.reminders!.length) && remindersDiffer(cur, reference)) { values.alerts = reminderChanges(instance, ctx.calendar(calendarId), cur.reminders!, ctx.mintKey).alerts; + // An occurrence on the calendar's defaults ignores its own alerts: it leaves them, as a series does + // (Stalwart keeps the override's `useDefaultAlerts` as a JSPROP; the series stays on the defaults). + if (instance.useDefaultAlerts) values.useDefaultAlerts = false; units.add('reminders'); } From d3262ff387fd38a322575ce07974e6e615a039f6 Mon Sep 17 00:00:00 2001 From: Linus Rath <139418639+rathlinus@users.noreply.github.com> Date: Sun, 27 Sep 2026 06:59:48 +0200 Subject: [PATCH 23/87] fix: send an answer to one occurrence in an RSVP-only calendar #34 In a calendar the user may only answer (mayRSVP without write rights), an answer to the whole series was uploaded, but an answer to a single occurrence goes into the occurrence's override and counted as an edit: it was undone on the next sync without a word. A change that holds nothing but the user's own answer, to the series or to occurrences, now counts as an RSVP and is uploaded. Stalwart 0.16.23 refuses every update in a shared calendar without write rights, answers included; the engine then puts the rows back from the server and reports the refusal, as it already did for an answer to the series, so the answer is no longer lost silently. Checked with a planner test (a new and an existing override, and an extra edit that is still reverted) and a run through the engine and the fake server. --- .../__tests__/calendar/engine.test.ts | 37 +++++++++++++++++++ .../__tests__/calendar/exceptions.test.ts | 27 ++++++++++++++ src/device-sync/calendar/planner.ts | 5 +-- src/device-sync/calendar/upload.ts | 15 ++++++-- 4 files changed, 77 insertions(+), 7 deletions(-) diff --git a/src/device-sync/__tests__/calendar/engine.test.ts b/src/device-sync/__tests__/calendar/engine.test.ts index e24bef26..836abbb3 100644 --- a/src/device-sync/__tests__/calendar/engine.test.ts +++ b/src/device-sync/__tests__/calendar/engine.test.ts @@ -94,6 +94,43 @@ describe('calendar device edits through the engine', () => { expect(h.batches.log.slice(before).filter((ops) => ops.some((op) => op.op !== 'syncState' && op.op !== 'assert'))).toEqual([]); }); + it('sends an answer to one occurrence in a calendar that grants only RSVP, and reports a refusal', async () => { + const h = real({ accounts: 'withShared' }); + const team = h.server.addCalendar('team', { + name: 'Team events', + myRights: { mayReadFreeBusy: true, mayReadItems: true, mayWriteAll: false, mayWriteOwn: false, mayUpdatePrivate: false, mayRSVP: true, mayShare: false, mayDelete: false }, + }); + h.prefs.calendarSelection[`team/${team}`] = true; + const id = h.server.addEvent('team', { + uid: 'team-weekly', + title: 'Team weekly', + start: '2026-10-05T09:00:00', + duration: 'PT1H', + timeZone: 'Europe/Berlin', + recurrenceRule: { '@type': 'RecurrenceRule', frequency: 'weekly', count: 5 }, + organizerCalendarAddress: 'mailto:boss@example.net', + participants: { + boss: { '@type': 'Participant', calendarAddress: 'mailto:boss@example.net', roles: { owner: true }, participationStatus: 'accepted' }, + me: { '@type': 'Participant', calendarAddress: 'mailto:alice@example.com', roles: { attendee: true }, participationStatus: 'needs-action' }, + }, + calendarIds: { [team]: true }, + }); + await h.run(CALENDAR_AUTHORITY); + const master = h.events().find((e) => e[Events._SYNC_ID] === `team/${id}`)!; + + // Etar answers the Oct 12 occurrence through CONTENT_EXCEPTION_URI. + h.device.user.exceptionViaUri(Number(master._id), Date.UTC(2026, 9, 12, 7), { [Events.SELF_ATTENDEE_STATUS]: 2, [Events.STATUS]: 1 }); + const sets = () => h.server.requests.filter((r) => r.methods.includes('CalendarEvent/set')).length; + const before = sets(); + const report = await h.run(CALENDAR_AUTHORITY); + + // The answer was sent (this server takes no RSVP without write rights), the refusal is reported and the row put back. + expect(sets()).toBeGreaterThan(before); + expect(report.itemErrors).toEqual([expect.objectContaining({ ref: `team/${id}`, side: 'upload', type: 'forbidden' })]); + expect(h.events().filter((e) => e[Events.ORIGINAL_SYNC_ID] === `team/${id}`)).toEqual([]); + expect(h.events().every((e) => Number(e[Events.DIRTY] ?? 0) === 0)).toBe(true); + }); + it('keeps a description and location cleared on one occurrence cleared', async () => { const h = real(); const id = h.server.addEvent('a', { diff --git a/src/device-sync/__tests__/calendar/exceptions.test.ts b/src/device-sync/__tests__/calendar/exceptions.test.ts index 7e371a89..965bbf38 100644 --- a/src/device-sync/__tests__/calendar/exceptions.test.ts +++ b/src/device-sync/__tests__/calendar/exceptions.test.ts @@ -225,6 +225,33 @@ describe('calendar planner: recurrence edits made on the device', () => { expect(action.sendSchedulingMessages).toBe(true); }); + it('lets an answer to one occurrence through in a calendar that grants only RSVP, and nothing more', async () => { + const invite = weekly(); + invite.calendarIds = { r: true }; + invite.organizerCalendarAddress = 'mailto:boss@example.net'; + invite.participants = { + boss: { calendarAddress: 'mailto:boss@example.net', roles: { owner: true }, participationStatus: 'accepted' }, + me: { calendarAddress: 'mailto:usera@example.org', roles: { attendee: true }, participationStatus: 'needs-action' }, + }; + const { h, id } = await synced(invite); + // Etar answers Oct 19 through CONTENT_EXCEPTION_URI (a new override)... + const added = h.fake.user.exceptionViaUri(id, utc('2026-10-19T13:00:00Z'), { [Events.SELF_ATTENDEE_STATUS]: 2, [Events.STATUS]: 1 }); + // ... and another app answers the existing override of Oct 7 on its row. + const existing = (await h.local('bl'))!.exceptions.find((x) => x.recurrenceId === '2026-10-07T09:00:00')!; + h.fake.table('attendees').forEach((a) => { + if (a.event_id === existing.eventId && a[Attendees.ATTENDEE_EMAIL] === ME) a[Attendees.ATTENDEE_STATUS] = 1; + }); + h.fake.user.updateEvent(existing.eventId, {}); + const action = patchOf(calendarPlanner.planUpload((await h.local('bl'))!, h.ctx)); + type Answer = { participants: Record }; + expect((action.patch['recurrenceOverrides/2026-10-19T09:00:00'] as Answer).participants.me.participationStatus).toBe('declined'); + expect((action.patch['recurrenceOverrides/2026-10-07T09:00:00/participants'] as Answer['participants']).me.participationStatus).toBe('accepted'); + expect(action.sendSchedulingMessages).toBe(true); + // Anything more than the answer is still the calendar owner's to change. + h.fake.user.updateEvent(added, { [Events.TITLE]: 'Mine now' }); + expect(calendarPlanner.planUpload((await h.local('bl'))!, h.ctx).kind).toBe('revert'); + }); + it('turns EXDATE entries an app adds or removes into exclusions and re-inclusions', async () => { const { h, id } = await synced(weekly()); h.fake.user.updateEvent(id, { [Events.EXDATE]: '20261012T130000Z\nAmerica/New_York;20261021T090000' }); diff --git a/src/device-sync/calendar/planner.ts b/src/device-sync/calendar/planner.ts index 31e11f68..4d6157ed 100644 --- a/src/device-sync/calendar/planner.ts +++ b/src/device-sync/calendar/planner.ts @@ -308,9 +308,8 @@ function planUpload(local: LocalEvent, ctx: CalendarContext): Plan { return m && computation.patch[k] === true && ctx.isReadOnly(m[1]); }); if ((readOnly && keys.length) || movedToReadOnly) { - // RSVP is the one change a calendar without write rights may take. - const rsvpOnly = keys.length > 0 && keys.every((k) => /^participants\/[^/]+\/participationStatus$/.test(k)); - if (!(rsvpOnly && ctx.calendar(calendar.calendarId)?.myRights?.mayRSVP && !movedToReadOnly)) { + // RSVP, to the series or to occurrences, is the one change a calendar without write rights may take. + if (!(computation.rsvpOnly && ctx.calendar(calendar.calendarId)?.myRights?.mayRSVP && !movedToReadOnly)) { const ops = revertGroup(local, ctx); return ops ? { kind: 'revert', ops } : { kind: 'skip', reason: 'readOnly' }; } diff --git a/src/device-sync/calendar/upload.ts b/src/device-sync/calendar/upload.ts index 4476719e..269924f9 100644 --- a/src/device-sync/calendar/upload.ts +++ b/src/device-sync/calendar/upload.ts @@ -169,6 +169,8 @@ export interface UploadComputation { /** Exception rows whose instance is excluded by the patch, or already was. */ removedExceptions: Set; sendSchedulingMessages: boolean; + /** The patch holds nothing but the user's own answer, to the series or to occurrences (what `mayRSVP` allows). */ + rsvpOnly: boolean; /** Units changed on the device that can't be uploaded (reported when nothing else is). */ dropped: string[]; } @@ -362,6 +364,7 @@ export function computeUpdate(local: LocalEvent, ctx: CalendarContext): UploadCo } // Exception rows. + let answersOnly = true; if (!ruleRemoved && shadow.recurrenceRule) { for (const x of local.exceptions) { const units = exceptionChanges(x, local, shadow, calendarId, zone, overrides, removedExceptions, ctx, lossy); @@ -369,6 +372,7 @@ export function computeUpdate(local: LocalEvent, ctx: CalendarContext): UploadCo exceptionUnits.set(x.eventId, units.units); overridesChanged = true; rsvp = rsvp || units.rsvp; + answersOnly &&= units.rsvp && !units.others && units.units.size === 1 && units.units.has('attendees'); } } } @@ -377,8 +381,9 @@ export function computeUpdate(local: LocalEvent, ctx: CalendarContext): UploadCo const notifying = NOTIFYING_UNITS.some((u) => masterUnits.has(u)) || overridesChanged || attendeeOthers; const sendSchedulingMessages = sendsSchedulingMessages(shadow, { kind: 'update', notifying, rsvp, addsAttendees: attendeeOthers }, ctx); + const rsvpOnly = rsvp && answersOnly && !attendeeOthers && [...masterUnits].every((u) => u.startsWith('attendee:')); - return { patch, masterUnits, exceptionUnits, removedExceptions, sendSchedulingMessages, dropped }; + return { patch, masterUnits, exceptionUnits, removedExceptions, sendSchedulingMessages, dropped, rsvpOnly }; } /** Timing properties that differ from the server's. */ @@ -413,7 +418,7 @@ function exceptionChanges( removed: Set, ctx: CalendarContext, lossy: boolean, -): { units: Set; rsvp: boolean } { +): { units: Set; rsvp: boolean; others?: boolean } { const units = new Set(); const key = x.recurrenceId; if (!key) return { units, rsvp: false }; @@ -439,6 +444,7 @@ function exceptionChanges( const cur = sideOfRow(x); const values: Record = {}; let rsvp = false; + let others = false; const organizer = userIsOrganizer(shadow, ctx); for (const unit of EXCEPTION_COLUMN_UNITS) { @@ -504,6 +510,7 @@ function exceptionChanges( if (result.changes.length) { values.participants = result.participants; rsvp = result.rsvp; + others = result.others; units.add('attendees'); } } @@ -517,12 +524,12 @@ function exceptionChanges( if (existing) { if (units.size) work.props.set(key, { ...(work.props.get(key) ?? {}), ...values }); - return { units, rsvp }; + return { units, rsvp, others }; } // A new override: nothing but its pinned time is still an override (an app saving the instance unchanged). if (!units.size) units.add('new'); work.whole.set(key, newOverride(shadow, key, values)); - return { units, rsvp }; + return { units, rsvp, others }; } /** Units a new exception row inherits from its series when it leaves them empty. */ From 6785276377ab0771a3864f09700cf3cb2d5d2cca Mon Sep 17 00:00:00 2001 From: Linus Rath <139418639+rathlinus@users.noreply.github.com> Date: Sun, 27 Sep 2026 07:00:45 +0200 Subject: [PATCH 24/87] fix: put back an event moved in place to another account's calendar #34 An app that moves an event by changing its calendar in place can move it into a calendar of another account on the device. A patch cannot move an event between accounts, so the move was dropped; with nothing else changed, the row was then marked clean where it stood. The phone showed the event in the other account's calendar while the server kept it in the old one, until the next server change moved it back. The row now goes back to its calendar at once, rewritten from the last server version. Changes made together with the move still upload, and their accepted write puts the row back as before. Checked with a run through the engine and the fake server with a shared account: the event is back in its calendar and clean, and the server and the other account are untouched. --- .../__tests__/calendar/engine.test.ts | 35 +++++++++++++++++++ src/device-sync/calendar/planner.ts | 7 ++++ 2 files changed, 42 insertions(+) diff --git a/src/device-sync/__tests__/calendar/engine.test.ts b/src/device-sync/__tests__/calendar/engine.test.ts index 836abbb3..76f4c661 100644 --- a/src/device-sync/__tests__/calendar/engine.test.ts +++ b/src/device-sync/__tests__/calendar/engine.test.ts @@ -131,6 +131,41 @@ describe('calendar device edits through the engine', () => { expect(h.events().every((e) => Number(e[Events.DIRTY] ?? 0) === 0)).toBe(true); }); + it('puts an event an app moved in place to a calendar of another account back into its calendar', async () => { + const h = real({ accounts: 'withShared' }); + const team = String(h.server.all('Calendar', 'team')[0].id); + h.prefs.calendarSelection[`team/${team}`] = true; + const id = h.server.addEvent('a', { + uid: 'lunch-uid', + title: 'Lunch', + start: '2026-10-06T12:00:00', + duration: 'PT1H', + timeZone: 'Europe/Berlin', + calendarIds: { [h.calendar]: true }, + }); + await h.run(CALENDAR_AUTHORITY); + const calendarRow = (key: string) => Number(h.device.rows('calendars').find((c) => c._sync_id === key)!._id); + const row = () => h.events().find((e) => e[Events._SYNC_ID] === `a/${id}`)!; + expect(row()[Events.CALENDAR_ID]).toBe(calendarRow(`a/${h.calendar}`)); + + // An app moves it by changing CALENDAR_ID in place (a move across JMAP accounts can't be a patch). + h.device.user.updateEvent(Number(row()._id), { [Events.CALENDAR_ID]: calendarRow(`team/${team}`) }); + const report = await h.run(CALENDAR_AUTHORITY); + + expect(row()).toMatchObject({ [Events.CALENDAR_ID]: calendarRow(`a/${h.calendar}`), [Events.DIRTY]: 0 }); + expect(h.server.get('CalendarEvent', 'a', id)!.calendarIds).toEqual({ [h.calendar]: true }); + expect(h.server.all('CalendarEvent', 'team')).toEqual([]); + // Nothing was sent; the change counts as skipped (a report entry needs the engine to report reverts). + expect(report).toMatchObject({ outcome: 'ok', stats: { uploaded: { created: 0, updated: 0, deleted: 0 }, skipped: 1 } }); + + // Together with another edit: that one uploads, and the row goes back all the same. + h.device.user.updateEvent(Number(row()._id), { [Events.CALENDAR_ID]: calendarRow(`team/${team}`), [Events.TITLE]: 'Team lunch' }); + expect(await h.run(CALENDAR_AUTHORITY)).toMatchObject({ outcome: 'ok', stats: { uploaded: { updated: 1 } } }); + expect(h.server.get('CalendarEvent', 'a', id)).toMatchObject({ title: 'Team lunch', calendarIds: { [h.calendar]: true } }); + expect(row()).toMatchObject({ [Events.CALENDAR_ID]: calendarRow(`a/${h.calendar}`), [Events.TITLE]: 'Team lunch', [Events.DIRTY]: 0 }); + expect(h.server.all('CalendarEvent', 'team')).toEqual([]); + }); + it('keeps a description and location cleared on one occurrence cleared', async () => { const h = real(); const id = h.server.addEvent('a', { diff --git a/src/device-sync/calendar/planner.ts b/src/device-sync/calendar/planner.ts index 4d6157ed..abda5068 100644 --- a/src/device-sync/calendar/planner.ts +++ b/src/device-sync/calendar/planner.ts @@ -316,6 +316,13 @@ function planUpload(local: LocalEvent, ctx: CalendarContext): Plan { } if (!keys.length) { if (computation.dropped.some((u) => u === 'timing' || u === 'rule')) return { kind: 'skip', reason: SKIP.ruleNotRepresentable }; + // A move into a calendar of another JMAP account is no patch (only new rows are created across accounts): + // the row goes back to its calendar instead of staying there unsynced. With other changes, their accepted + // write puts it back. + if (computation.dropped.includes('calendar')) { + const ops = revertGroup(local, ctx); + if (ops) return { kind: 'revert', ops }; + } return { kind: 'clean', ops: cleanGroup(local, computation) }; } const action: UploadAction = { kind: 'update', id: ref.id, patch: computation.patch }; From c777dc4089c13911ddcec473ff4ce6310db607ef Mon Sep 17 00:00:00 2001 From: Linus Rath <139418639+rathlinus@users.noreply.github.com> Date: Sun, 27 Sep 2026 07:01:24 +0200 Subject: [PATCH 25/87] fix: keep syncing a big meeting after it was edited on the phone #34 After an edit on the phone, every write into a recurring event asserted each attendee and reminder row of every occurrence. For a meeting with 25 attendees and 52 occurrences with details of their own that alone made the write 1.6 MB, over Android's 1 MB transaction limit: the event could no longer be written, went stale in every run and kept its edit. Only rows that are dirty themselves now assert their attendees and reminders. A clean row's DIRTY flag already catches an app edit of them, since CalendarProvider sets it on every app write to an event, its attendees or reminders, so such a write still fails and is planned again. The same meeting's writes now take about 0.65 MB. Checked with the meeting above through the planner, and with an app edit of a clean occurrence's attendees between planning and writing. --- .../__tests__/calendar/big-events.test.ts | 90 +++++++++++++++++++ src/device-sync/calendar/merge.ts | 11 +-- src/device-sync/calendar/planner.ts | 5 +- 3 files changed, 99 insertions(+), 7 deletions(-) create mode 100644 src/device-sync/__tests__/calendar/big-events.test.ts diff --git a/src/device-sync/__tests__/calendar/big-events.test.ts b/src/device-sync/__tests__/calendar/big-events.test.ts new file mode 100644 index 00000000..d367ab0f --- /dev/null +++ b/src/device-sync/__tests__/calendar/big-events.test.ts @@ -0,0 +1,90 @@ +// Big meetings (many attendees on many overrides): a write into such an event +// has to fit one provider transaction (Binder: about 1 MB; a Parcel carries +// the JSON as UTF-16, estimated at 2 bytes per character plus ~300 per op). +import { describe, expect, it } from 'vitest'; +import { Attendees, Events } from '../../android-columns'; +import { calendarPlanner } from '../../calendar/planner'; +import type { OpGroup } from '../../planner'; +import type { CalendarEventWire } from '../../wire'; +import { Harness, ME, serverApply } from './harness'; + +const TRANSACTION = 1_000_000; +const estimate = (group: OpGroup) => 2 * JSON.stringify(group.ops).length + 300 * group.ops.length; + +/** A daily meeting with `attendees` guests whose `overrides` occurrences each carry the whole participants map. */ +function meeting(attendees: number, overrides: number): CalendarEventWire { + const participants: Record = { + me: { '@type': 'Participant', calendarAddress: `mailto:${ME}`, roles: { owner: true, attendee: true }, participationStatus: 'accepted' }, + }; + for (let i = 0; i < attendees; i++) { + participants[`p${i}`] = { '@type': 'Participant', calendarAddress: `mailto:person${i}@example.com`, name: `Person ${i}`, roles: { attendee: true }, participationStatus: 'needs-action', expectReply: true }; + } + const recurrenceOverrides: Record = {}; + for (let i = 0; i < overrides; i++) { + const day = new Date(Date.UTC(2026, 9, 6) + i * 86_400_000).toISOString().slice(0, 10); + recurrenceOverrides[`${day}T09:00:00`] = { title: `Standup #${i}`, participants }; + } + return { + id: 'big', + '@type': 'Event', + uid: 'big@example.org', + calendarIds: { b: true }, + title: 'Standup', + description: 'Daily standup', + start: '2026-10-05T09:00:00', + timeZone: 'Europe/Berlin', + duration: 'PT15M', + recurrenceRule: { frequency: 'daily', count: 300 }, + organizerCalendarAddress: `mailto:${ME}`, + participants, + recurrenceOverrides, + } as unknown as CalendarEventWire; +} + +async function synced(event: CalendarEventWire) { + const h = await new Harness().setup(); + await h.download(event); + return { h, id: Number(h.masterRow(event.id)._id) }; +} + +describe('calendar planner: big meetings', () => { + it('fits the writes after an edit of a big meeting into one transaction', async () => { + const event = meeting(25, 52); + const { h, id } = await synced(event); + // Only the series row is edited; its 52 occurrences with 26 attendees each stay clean. + h.fake.user.updateEvent(id, { [Events.TITLE]: 'Daily standup' }); + const local = (await h.local('big'))!; + + const plan = calendarPlanner.planUpload(local, h.ctx); + if (plan.kind !== 'upload' || plan.actions[0].kind !== 'update') throw new Error(plan.kind); + const server = serverApply(event, plan.actions[0].patch); + const accepted = calendarPlanner.planAccepted(local, server, h.ctx); + expect(estimate(accepted.ops)).toBeLessThan(TRANSACTION); + expect(estimate(accepted.keepDirtyOps)).toBeLessThan(TRANSACTION); + + // A download into the edited event (the server changed one occurrence meanwhile). + const remote = { ...event, recurrenceOverrides: { ...event.recurrenceOverrides, '2026-10-09T09:00:00': { title: 'Moved to Friday' } } } as CalendarEventWire; + expect(estimate(calendarPlanner.planDownload(remote, local, h.ctx).ops)).toBeLessThan(TRANSACTION); + + // A save that changed nothing mapped is cleared the same way. + h.fake.user.updateEvent(id, { [Events.TITLE]: 'Standup' }); + const clean = calendarPlanner.planUpload((await h.local('big'))!, h.ctx); + expect(clean.kind).toBe('clean'); + if (clean.kind === 'clean') expect(estimate(clean.ops)).toBeLessThan(TRANSACTION); + }); + + it('still fails a write when an app changed a clean occurrence\'s attendees meanwhile', async () => { + const event = meeting(3, 4); + const { h, id } = await synced(event); + h.fake.user.updateEvent(id, { [Events.TITLE]: 'Daily standup' }); + const local = (await h.local('big'))!; + const x = local.exceptions[0]; + const plan = calendarPlanner.planUpload(local, h.ctx); + if (plan.kind !== 'upload' || plan.actions[0].kind !== 'update') throw new Error(plan.kind); + const accepted = calendarPlanner.planAccepted(local, serverApply(event, plan.actions[0].patch), h.ctx); + // The occurrence's attendees are not asserted one by one: CalendarProvider marks the event DIRTY on any app write to them. + h.fake.beforeNextBatch(() => h.fake.user.setAttendeeStatus(x.eventId, 'person1@example.com', 1)); + expect(await h.tryApply(accepted.ops)).toMatchObject({ ok: false, reason: 'assert' }); + expect(h.fake.rows('attendees', `${Attendees.EVENT_ID} = ? AND ${Attendees.ATTENDEE_EMAIL} = ?`, [x.eventId, 'person1@example.com'])[0][Attendees.ATTENDEE_STATUS]).toBe(1); + }); +}); diff --git a/src/device-sync/calendar/merge.ts b/src/device-sync/calendar/merge.ts index dc65b48e..eadd6b71 100644 --- a/src/device-sync/calendar/merge.ts +++ b/src/device-sync/calendar/merge.ts @@ -15,7 +15,8 @@ * * Every write into an existing item is guarded: the group first asserts * DIRTY and the projection the plan was made from (with attendees and - * reminders when the item is dirty) and the number of exception rows. + * reminders for the rows that are dirty: a clean row's DIRTY catches any app + * edit of them) and the number of exception rows. */ import { Events } from '../android-columns'; import type { CalendarContext, EventBaseline, LocalEvent, LocalEventRow, LocalException } from '../planner'; @@ -284,11 +285,11 @@ export function writeEvent( const baseTarget = options.base ? { calendarRowId: local.calendarRowId, calendarId: ctx.calendarIdOfRow(local.calendarRowId)?.calendarId ?? target.calendarId } : target; const baseImage = options.base ? eventImage(baseEvent, baseTarget.calendarRowId, baseTarget.calendarId, ctx) ?? image : image; const lossy = isLossyEditor(local.mutators); - const itemDirty = local.dirty || local.exceptions.some((x) => x.dirty || x.deleted); - // Asserts first: the rows as read. - assertRow(group, local, false, itemDirty); - for (const x of local.exceptions) assertRow(group, x, true, itemDirty); + // Asserts first: the rows as read. Attendees and reminders only for dirty rows (see `assertRow`): a big + // meeting's occurrences would otherwise assert every attendee and outgrow one provider transaction. + assertRow(group, local, false, local.dirty || local.deleted); + for (const x of local.exceptions) assertRow(group, x, true, x.dirty || x.deleted); assertExceptionCount(group, local.eventId, linkedExceptionCount(local)); let conflicts = 0; diff --git a/src/device-sync/calendar/planner.ts b/src/device-sync/calendar/planner.ts index abda5068..08a2ac15 100644 --- a/src/device-sync/calendar/planner.ts +++ b/src/device-sync/calendar/planner.ts @@ -223,8 +223,9 @@ function revertGroup(local: LocalEvent, ctx: CalendarContext): OpGroup | null { /** Clears DIRTY where nothing uploadable changed, taking the rows as they are as the new baselines. */ function cleanGroup(local: LocalEvent, computation: UploadComputation | null): OpGroup { const group = new GroupBuilder(refOf(local)); - assertRow(group, local, false, true); - for (const x of local.exceptions) assertRow(group, x, true, true); + // Attendees and reminders of the dirty rows only (see `assertRow`), so a big meeting fits one transaction. + assertRow(group, local, false, local.dirty || local.deleted); + for (const x of local.exceptions) assertRow(group, x, true, x.dirty || x.deleted); const settle = (row: LocalEventRow, isException: boolean) => { if (!row.dirty) return; const truncated = isTruncatedBaseline(row.baseline?.cells[Events.DESCRIPTION]); From 2b9d19521fcd85f2e6c83c869f443c43eff8b3b0 Mon Sep 17 00:00:00 2001 From: Linus Rath <139418639+rathlinus@users.noreply.github.com> Date: Sun, 27 Sep 2026 07:12:12 +0200 Subject: [PATCH 26/87] fix: keep sync on when it is turned back on during a turn-off #34 Turning device sync off uploads what the device changed and then deletes --- .../__tests__/app/lifecycle.test.ts | 20 +++++++++++ src/device-sync/app/lifecycle.ts | 36 +++++++++++++++++-- 2 files changed, 53 insertions(+), 3 deletions(-) diff --git a/src/device-sync/__tests__/app/lifecycle.test.ts b/src/device-sync/__tests__/app/lifecycle.test.ts index d328abe6..019dd5c9 100644 --- a/src/device-sync/__tests__/app/lifecycle.test.ts +++ b/src/device-sync/__tests__/app/lifecycle.test.ts @@ -260,6 +260,26 @@ describe('disable', () => { expect(stored.reminderOwner).toBeNull(); }); + it('lets a turn-on wait for a turn-off that is still tearing down, instead of being undone by it', async () => { + await disableDeviceSync(ALICE, CALENDAR_AUTHORITY); + let finishTeardown: () => void = () => undefined; + teardown.mockImplementationOnce(async (_registryId, name, authority, options) => { + h.state.log.push(`teardown ${name} ${authority} ${JSON.stringify(options ?? {})}`); + await new Promise((resolve) => { finishTeardown = resolve; }); + return { pending: 0 }; + }); + const turningOff = disableDeviceSync(ALICE, CONTACTS_AUTHORITY); + await vi.waitFor(() => expect(h.state.log).toContain(`teardown alice ${CONTACTS_AUTHORITY} {"uploadFirst":true,"timeoutMs":${TEARDOWN_TIMEOUT_MS}}`)); + // The user opens the settings again and turns contacts sync back on. + const turningOn = enableDeviceSync(ALICE, CONTACTS_AUTHORITY); + finishTeardown(); + expect(await turningOff).toEqual({ done: true, pending: 0 }); + expect(await turningOn).toMatchObject({ kind: 'enabled' }); + expect(h.find('alice')?.sync[CONTACTS_AUTHORITY]?.automatic).toBe(true); + expect(accountDeviceSync(ALICE).enabled?.[CONTACTS_AUTHORITY]).toBe(true); + expect(accountDeviceSync(ALICE).androidAccountName).toBe('alice'); + }); + it('removes the Android account once both authorities are off', async () => { await disableDeviceSync(ALICE, CALENDAR_AUTHORITY); await disableDeviceSync(ALICE, CONTACTS_AUTHORITY); diff --git a/src/device-sync/app/lifecycle.ts b/src/device-sync/app/lifecycle.ts index f87e9397..91fd5364 100644 --- a/src/device-sync/app/lifecycle.ts +++ b/src/device-sync/app/lifecycle.ts @@ -162,6 +162,24 @@ async function refreshPushState(registryId: string): Promise { await refreshPushSubscriptionTypes(registryId).catch(noop); } +// ─── One step at a time ──────────────────────────────── + +// Turning sync on and off for one app account runs one step after the other: +// a turn-on that starts while a turn-off still tears down (the user left the +// settings and came back) would otherwise be undone by it, down to the +// Android account being removed right after it was turned on again. +const accountSteps = new Map>(); + +function oneAtATime(registryId: string, step: () => Promise): Promise { + const next = (accountSteps.get(registryId) ?? Promise.resolve()).catch(noop).then(step); + const settled = next.catch(noop); + accountSteps.set(registryId, settled); + void settled.then(() => { + if (accountSteps.get(registryId) === settled) accountSteps.delete(registryId); + }); + return next; +} + // ─── Enable ──────────────────────────────────────────── export type EnableOutcome = @@ -174,7 +192,11 @@ export type EnableOutcome = * Turns device sync on for one authority of an app account. The caller has * asked for the permissions (and, for calendars, who reminds the user). */ -export async function enableDeviceSync(registryId: string, authority: Authority): Promise { +export function enableDeviceSync(registryId: string, authority: Authority): Promise { + return oneAtATime(registryId, () => enable(registryId, authority)); +} + +async function enable(registryId: string, authority: Authority): Promise { if (!deviceSyncAvailable()) return { kind: 'failed', message: 'Device sync is not available' }; if (!(await hasSyncPermissions(authority))) return { kind: 'permission' }; await waitForDeviceSyncHydration(); @@ -207,7 +229,11 @@ export async function enableDeviceSync(registryId: string, authority: Authority) } /** Automatic sync back on after the user kept syncing instead of losing changes. */ -export async function resumeDeviceSync(registryId: string, authority: Authority): Promise { +export function resumeDeviceSync(registryId: string, authority: Authority): Promise { + return oneAtATime(registryId, () => resume(registryId, authority)); +} + +async function resume(registryId: string, authority: Authority): Promise { if (!deviceSyncAvailable()) return; await waitForDeviceSyncHydration(); const accountName = accountDeviceSync(registryId).androidAccountName ?? (await androidAccountOf(registryId))?.name; @@ -285,11 +311,15 @@ function withoutAuthorityStates(entry: AccountDeviceSync, authority: Authority): * how many changes are waiting: call again with `force` once the user agreed * to lose them, or `resumeDeviceSync` to keep syncing. */ -export async function disableDeviceSync( +export function disableDeviceSync( registryId: string, authority: Authority, options: { force?: boolean } = {}, ): Promise { + return oneAtATime(registryId, () => disable(registryId, authority, options)); +} + +async function disable(registryId: string, authority: Authority, options: { force?: boolean }): Promise { if (!deviceSyncAvailable()) return { done: true, pending: 0 }; await waitForDeviceSyncHydration(); const store = useDeviceSyncStore.getState(); From 7a53cd5d84ba90ccf0a608cabf00486d94f76d7b Mon Sep 17 00:00:00 2001 From: Linus Rath <139418639+rathlinus@users.noreply.github.com> Date: Sun, 27 Sep 2026 07:13:08 +0200 Subject: [PATCH 27/87] fix: let the user delete or restore held-back deletions from the sync settings #34 When a sync held back many deletions made on the device (Android's --- locales/rn/en.json | 4 ++++ .../device-sync/DeviceSyncAccountRow.tsx | 23 ++++++++++++++++++- .../__tests__/app/lifecycle.test.ts | 14 +++++++++++ src/device-sync/app/lifecycle.ts | 18 ++++++++++++++- 4 files changed, 57 insertions(+), 2 deletions(-) diff --git a/locales/rn/en.json b/locales/rn/en.json index ff5d09ef..dff3fc6b 100644 --- a/locales/rn/en.json +++ b/locales/rn/en.json @@ -646,6 +646,10 @@ "choose_calendars_title": "Calendars to sync", "collections_count": "{selected} of {total} sync", "continue": "Continue", + "deletions_delete": "Delete on server", + "deletions_message": "Many items were deleted on this device at once, so they were not deleted on the server. Delete them there as well, or bring them back on this device.", + "deletions_restore": "Bring them back", + "deletions_title": "Delete them on the server too?", "denied_calendar": "Without access to calendars, Bulwark cannot put your calendars on this device.", "denied_contacts": "Without access to contacts, Bulwark cannot put your address books on this device.", "denied_title": "Sync stays off", diff --git a/src/components/settings/device-sync/DeviceSyncAccountRow.tsx b/src/components/settings/device-sync/DeviceSyncAccountRow.tsx index dc3d68c3..a2b35c30 100644 --- a/src/components/settings/device-sync/DeviceSyncAccountRow.tsx +++ b/src/components/settings/device-sync/DeviceSyncAccountRow.tsx @@ -29,6 +29,7 @@ import { disableDeviceSync, enableDeviceSync, openAndroidAccountSettings, + resolveDeletions, resumeDeviceSync, syncNow, } from '../../../device-sync/app/lifecycle'; @@ -224,10 +225,30 @@ export function DeviceSyncAccountRow({ account, authority }: Props) { void refresh(); }; + const reviewDeletions = () => { + const resolve = (choice: 'delete' | 'restore') => { + void resolveDeletions(account.id, authority, choice) + .catch((err: unknown) => alertFailed(err instanceof Error ? err.message : String(err))) + .finally(() => { void refresh(); }); + }; + Alert.alert( + t('settings.device_sync.deletions_title', 'Delete them on the server too?'), + t( + 'settings.device_sync.deletions_message', + 'Many items were deleted on this device at once, so they were not deleted on the server. Delete them there as well, or bring them back on this device.', + ), + [ + { text: t('common.cancel', 'Cancel'), style: 'cancel' }, + { text: t('settings.device_sync.deletions_restore', 'Bring them back'), onPress: () => resolve('restore') }, + { text: t('settings.device_sync.deletions_delete', 'Delete on server'), style: 'destructive', onPress: () => resolve('delete') }, + ], + ); + }; + const runAction = (action: StatusAction) => { if (action === 'signIn') navigation.navigate('AddAccount'); else if (action === 'grantAccess') void grantAccess(); - else void openAndroidAccountSettings(account.id); + else reviewDeletions(); }; const closeChooser = (changed: boolean) => { diff --git a/src/device-sync/__tests__/app/lifecycle.test.ts b/src/device-sync/__tests__/app/lifecycle.test.ts index 019dd5c9..c61d4a85 100644 --- a/src/device-sync/__tests__/app/lifecycle.test.ts +++ b/src/device-sync/__tests__/app/lifecycle.test.ts @@ -108,6 +108,7 @@ import { reconcileDeviceAccounts, releaseDeviceSyncBeforeSignOut, removeSuspendedAccount, + resolveDeletions, resumeDeviceSync, TEARDOWN_TIMEOUT_MS, } from '../../app/lifecycle'; @@ -483,6 +484,19 @@ describe('sign-in', () => { }); describe('settings actions', () => { + it('deletes held-back deletions on the server, or brings the items back, with a sync', async () => { + await enableDeviceSync(ALICE, CONTACTS_AUTHORITY); + h.state.log = []; + expect(await resolveDeletions(ALICE, CONTACTS_AUTHORITY, 'delete')).toBe(true); + expect(await resolveDeletions(ALICE, CONTACTS_AUTHORITY, 'restore')).toBe(true); + expect(h.state.log).toEqual([ + `sync alice ${CONTACTS_AUTHORITY} {"manual":true,"overrideTooManyDeletions":true}`, + `sync alice ${CONTACTS_AUTHORITY} {"manual":true,"discardLocalDeletions":true}`, + ]); + // Without an Android account there is nothing to resolve. + expect(await resolveDeletions(BOB, CONTACTS_AUTHORITY, 'delete')).toBe(false); + }); + it('applies a new interval to Android while the authority syncs', async () => { await enableDeviceSync(ALICE, CONTACTS_AUTHORITY); h.state.log = []; diff --git a/src/device-sync/app/lifecycle.ts b/src/device-sync/app/lifecycle.ts index 91fd5364..e873595e 100644 --- a/src/device-sync/app/lifecycle.ts +++ b/src/device-sync/app/lifecycle.ts @@ -623,7 +623,23 @@ export async function syncNow(registryId: string, authority: Authority): Promise return true; } -/** Android's sync screen of the account ("Android account settings", "Review deletions"). */ +/** + * After a run held back many deletions made on the device + * (`tooManyDeletions`): delete those items on the server too, or bring them + * back on the device from the server. Android's own screens offer neither. + */ +export async function resolveDeletions(registryId: string, authority: Authority, choice: 'delete' | 'restore'): Promise { + const accountName = accountDeviceSync(registryId).androidAccountName; + if (!deviceSyncAvailable() || !accountName) return false; + await requestSync( + accountName, + authority, + choice === 'delete' ? { manual: true, overrideTooManyDeletions: true } : { manual: true, discardLocalDeletions: true }, + ); + return true; +} + +/** Android's sync screen of the account ("Android account settings"). */ export async function openAndroidAccountSettings(registryId: string): Promise { const accountName = accountDeviceSync(registryId).androidAccountName; if (!deviceSyncAvailable() || !accountName) return false; From 13264d64791a95375744f4effe42724314f8fb76 Mon Sep 17 00:00:00 2001 From: Linus Rath <139418639+rathlinus@users.noreply.github.com> Date: Sun, 27 Sep 2026 07:14:50 +0200 Subject: [PATCH 28/87] fix: keep contacts the server also filed in unsynced books when deleted on the device #34 A contact or group deleted on the device was uploaded against the shadow of its last sync: a download into a deleted item wrote nothing. When the server had meanwhile also filed the card in an address book this device does not sync, the deletion still saw only the synced book and destroyed the card, its copy in the other book included. A download into a deleted contact or group now writes the new server card into its shadow, and takes over an identity that was never written, so deleting our own lost create goes by id rather than by a uid lookup. The rows and the group title stay as they are: the device delete still wins. The upload then removes only the synced memberships of a card that is also in other books, as the calendar planner already does for events. Checked with planner tests for contacts and groups and end to end through the engine with the real planner: the card stays in the unsynced book, the device rows are purged, and an echo writes nothing. --- .../__tests__/contacts/groups.test.ts | 15 ++++++ .../__tests__/contacts/merge.test.ts | 13 +++-- .../__tests__/contacts/real-engine.test.ts | 54 +++++++++++++++++++ .../__tests__/contacts/upload-editors.test.ts | 30 +++++++++++ src/device-sync/contacts/download.ts | 15 +++++- src/device-sync/contacts/groups.ts | 8 ++- 6 files changed, 129 insertions(+), 6 deletions(-) create mode 100644 src/device-sync/__tests__/contacts/real-engine.test.ts diff --git a/src/device-sync/__tests__/contacts/groups.test.ts b/src/device-sync/__tests__/contacts/groups.test.ts index 74ffb7dd..f29efe9b 100644 --- a/src/device-sync/__tests__/contacts/groups.test.ts +++ b/src/device-sync/__tests__/contacts/groups.test.ts @@ -183,6 +183,21 @@ describe('contacts groups: device edits of groups', () => { }); }); + it('only leaves the synced book when the server filed the group in an unsynced one after the device deleted it', async () => { + const { h, gid } = await withGroup(); + const archive = h.server.addAddressBook(JMAP, { name: 'Archive' }); + h.unselected.add(`${JMAP}/${archive}`); + h.device.user.deleteGroup((await localGroup(h, gid)).groupId); + h.server.serverUpdate('ContactCard', JMAP, gid, { [`addressBookIds/${archive}`]: true, 'name/full': 'Old friends' }); + const plan = h.planner.planGroupDownload(h.card(gid), await localGroup(h, gid), h.ctx); + await h.applyOk(plan.ops); + // The delete wins over the rename; only the shadow follows the card. + expect(await localGroup(h, gid)).toMatchObject({ deleted: true, title: 'Friends' }); + expect(h.planner.planGroupUpload(await localGroup(h, gid), h.ctx)).toEqual({ + kind: 'upload', actions: [{ kind: 'update', id: gid, patch: { [`addressBookIds/${h.book}`]: null } }], + }); + }); + it('reverts a rename of a read-only group', async () => { const { h, gid } = await withGroup(); h.readOnlyBooks.add(h.book); diff --git a/src/device-sync/__tests__/contacts/merge.test.ts b/src/device-sync/__tests__/contacts/merge.test.ts index 709f3d92..2d07db5a 100644 --- a/src/device-sync/__tests__/contacts/merge.test.ts +++ b/src/device-sync/__tests__/contacts/merge.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it } from 'vitest'; -import { Data, MimeType } from '../../android-columns'; +import { Data, MimeType, RawContacts } from '../../android-columns'; import type { ContactCardWire } from '../../planner'; import { appleCard, googleCard } from './fixtures'; import { Harness, JMAP } from './harness'; @@ -139,13 +139,20 @@ describe('contacts merge: a dirty contact meets a new server version', () => { expect(typeof patch === 'object' && Object.keys(patch).some((k) => k.startsWith('nicknames'))).toBe(false); }); - it('leaves a contact deleted on the device alone: the delete wins and uploads', async () => { + it('leaves the rows of a contact deleted on the device alone: the delete wins and uploads', async () => { const h = new Harness(); const { id, rawId } = await h.seed(appleCard()); h.device.user.deleteContact(rawId); h.server.serverUpdate('ContactCard', JMAP, id, { 'emails/k1/address': 'x@example.com' }); const plan = h.planner.planDownload(h.card(id), await h.contact(rawId), h.ctx); - expect(plan).toMatchObject({ effect: 'none', stillDirty: true, ops: { ops: [] } }); + // Only the shadow follows the card, for the deletion's upload. + expect(plan).toMatchObject({ effect: 'update', stillDirty: true, conflicts: 0 }); + expect(plan.ops.ops.filter((o) => o.op !== 'assert')).toEqual([{ + op: 'update', table: 'raw_contacts', id: rawId, expectCount: 1, + values: { [RawContacts.SYNC2]: expect.any(String) }, + }]); + await h.applyOk(plan.ops); + expect(h.planner.planDownload(h.card(id), await h.contact(rawId), h.ctx)).toMatchObject({ effect: 'none', ops: { ops: [] } }); expect(h.planner.planUpload(await h.contact(rawId), h.ctx)).toMatchObject({ kind: 'upload', actions: [{ kind: 'destroy', id }] }); }); diff --git a/src/device-sync/__tests__/contacts/real-engine.test.ts b/src/device-sync/__tests__/contacts/real-engine.test.ts new file mode 100644 index 00000000..f42ba6ae --- /dev/null +++ b/src/device-sync/__tests__/contacts/real-engine.test.ts @@ -0,0 +1,54 @@ +// The contacts planner driven by the engine (fake providers, fake Stalwart): +// cases that only show end to end, over several runs. The planner's rules +// themselves are tested in the other files of this folder. + +import { describe, expect, it } from 'vitest'; +import { calendarPlanner } from '../../calendar/planner'; +import { contactsPlanner } from '../../contacts/planner'; +import { addServerCards, createHarness, type Harness } from '../engine/harness'; + +function real(): Harness { + const h = createHarness(); + h.deps.planners = { contacts: contactsPlanner, calendar: calendarPlanner }; + return h; +} + +/** An address book of the personal account that this device does not sync. */ +function unsyncedBook(h: Harness, name = 'Archive'): string { + const id = h.server.addAddressBook('a', { name }); + h.prefs.contactsSelection[`a/${id}`] = false; + return id; +} + +describe('contacts through the engine', () => { + it('keeps a card deleted on the device that the server meanwhile also filed in an unsynced book', async () => { + const h = real(); + const archive = unsyncedBook(h); + const [ada] = addServerCards(h, ['Ada Lovelace']); + expect((await h.run()).outcome).toBe('ok'); + + h.device.user.deleteContact(h.contactNamed('Ada Lovelace')!.id); + h.server.serverUpdate('ContactCard', 'a', ada, { [`addressBookIds/${archive}`]: true }); + const report = await h.run(); + + expect(report).toMatchObject({ outcome: 'ok', itemErrors: [], stats: { uploaded: { deleted: 1 } } }); + expect(h.server.get('ContactCard', 'a', ada)).toMatchObject({ addressBookIds: { [archive]: true } }); + expect(h.contacts()).toEqual([]); + }); + + it('keeps a group deleted on the device that the server meanwhile also filed in an unsynced book', async () => { + const h = real(); + const archive = unsyncedBook(h); + const group = h.server.addCard('a', { uid: 'g1', kind: 'group', name: { full: 'Friends' }, addressBookIds: { [h.book]: true } }); + expect((await h.run()).outcome).toBe('ok'); + + const row = h.device.rows('groups').find((g) => g.sourceid === `a/${group}`)!; + h.device.user.deleteGroup(Number(row._id)); + h.server.serverUpdate('ContactCard', 'a', group, { [`addressBookIds/${archive}`]: true }); + const report = await h.run(); + + expect(report).toMatchObject({ outcome: 'ok', itemErrors: [], stats: { uploaded: { deleted: 1 } } }); + expect(h.server.get('ContactCard', 'a', group)).toMatchObject({ addressBookIds: { [archive]: true } }); + expect(h.device.rows('groups')).toEqual([]); + }); +}); diff --git a/src/device-sync/__tests__/contacts/upload-editors.test.ts b/src/device-sync/__tests__/contacts/upload-editors.test.ts index 9bf3fe16..0fcc6a4c 100644 --- a/src/device-sync/__tests__/contacts/upload-editors.test.ts +++ b/src/device-sync/__tests__/contacts/upload-editors.test.ts @@ -309,6 +309,36 @@ describe('contacts upload: deleted contacts', () => { expect(await patchOf(h, rawId)).toEqual({ [`addressBookIds/${h.book}`]: null }); }); + it('only leaves the synced book when the server filed the card in an unsynced one after the device deleted it', async () => { + const h = new Harness(); + const archive = h.server.addAddressBook(JMAP, { name: 'Archive' }); + h.unselected.add(`${JMAP}/${archive}`); + const { id, rawId } = await h.seed(appleCard()); + h.device.user.deleteContact(rawId); + h.server.serverUpdate('ContactCard', JMAP, id, { [`addressBookIds/${archive}`]: true, 'emails/k1/address': 'new@example.com' }); + const rows = JSON.stringify(h.device.rows('data', 'raw_contact_id = ?', [rawId])); + await h.applyOk(h.planner.planDownload(h.card(id), await h.contact(rawId), h.ctx).ops); + // The delete wins over the server's edit: the rows stay as they were, only the shadow follows. + expect(JSON.stringify(h.device.rows('data', 'raw_contact_id = ?', [rawId]))).toBe(rows); + expect(await uploadPlan(h, rawId)).toEqual({ kind: 'upload', actions: [{ kind: 'update', id, patch: { [`addressBookIds/${h.book}`]: null } }] }); + await h.upload(rawId); + expect(h.server.get('ContactCard', JMAP, id)).toMatchObject({ addressBookIds: { [archive]: true } }); + expect(h.device.row('raw_contacts', rawId)).toBeUndefined(); + }); + + it('deletes its own create by id once the download found it, not by a uid lookup', async () => { + const h = new Harness(); + const rawId = h.device.user.insertContact(ACCOUNT, [{ [Data.MIMETYPE]: MimeType.NOTE, data1: 'y' }]); + const plan = await h.planUpload(rawId); + if (plan.kind !== 'upload') throw new Error(plan.kind); + // The create went through but its response was lost; then the user deleted the contact. + const id = h.send(plan.actions)!; + h.device.user.deleteContact(rawId); + await h.applyOk(h.planner.planDownload(h.card(id), await h.contact(rawId), h.ctx).ops); + expect(await h.contact(rawId)).toMatchObject({ sourceId: `${JMAP}/${id}`, pending: null, deleted: true }); + expect(await uploadPlan(h, rawId)).toEqual({ kind: 'upload', actions: [{ kind: 'destroy', id, uid: h.card(id).uid }] }); + }); + it('purges a new contact that never reached the server, and destroys a claimed one by uid', async () => { const h = new Harness(); const fresh = h.device.user.insertContact(ACCOUNT, [{ [Data.MIMETYPE]: MimeType.NOTE, data1: 'x' }]); diff --git a/src/device-sync/contacts/download.ts b/src/device-sync/contacts/download.ts index e6c0eafb..066605f6 100644 --- a/src/device-sync/contacts/download.ts +++ b/src/device-sync/contacts/download.ts @@ -288,8 +288,19 @@ export function planContactDownload(card: ContactCardWire, local: LocalContact | const identity = objectRef(ctx.jmapAccountId, server.id); if (!local) return insertPlan(server, ctx); const none = (stillDirty: boolean): DownloadPlan => ({ ops: { ref: identity, ops: [] }, conflicts: 0, stillDirty, effect: 'none', writes: 0 }); - // Deleted on the device: the delete wins and uploads as a destroy. - if (local.deleted) return none(true); + if (local.deleted) { + // Deleted on the device: the delete wins over a server edit, so the rows stay as they are. + // Only the shadow (and an identity never written) follows the card: the deletion uploads + // against the card as it is now, which may also sit in books this device does not sync. + const shadow = withMemberOf(shadowOf(server), shadowMemberOf(local.shadow)); + const raw = changedRawColumns(local, { + ...(local.sourceId ? {} : { [RawContacts.SOURCE_ID]: identity, [RawContacts.SYNC3]: null }), + [RawContacts.SYNC2]: JSON.stringify(shadow), + }); + if (!Object.keys(raw).length) return none(true); + const ops = [assertContact(local, local.dirty), updateRawContact(local.rawContactId, raw)]; + return { ops: { ref: identity, ops }, conflicts: 0, stillDirty: true, effect: 'update', writes: 1 }; + } if (!local.sourceId) { // Our own create whose identity was never written: adopt it. The rows stay diff --git a/src/device-sync/contacts/groups.ts b/src/device-sync/contacts/groups.ts index 6e9d8dae..c9d0b3f7 100644 --- a/src/device-sync/contacts/groups.ts +++ b/src/device-sync/contacts/groups.ts @@ -94,7 +94,13 @@ export function planGroupDownload(card: ContactCardWire, local: LocalGroup | nul values: { [Groups.SOURCE_ID]: identity, [Groups.TITLE]: remoteTitle ?? '', [Groups.GROUP_VISIBLE]: 1, [Groups.DIRTY]: 0, ...cardColumns(card, ctx) }, }], 'insert'); } - if (local.deleted) return plan([], 'none', 0, true); + if (local.deleted) { + // Deleted on the device: the delete wins over a server edit. Only the shadow (and an identity + // never written) follows the card, so the deletion uploads against the card as it is now. + const diff = changed(local, { ...(local.sourceId ? {} : { [Groups.SOURCE_ID]: identity, [Groups.SYNC3]: null }), ...cardColumns(card, ctx) }); + if (!Object.keys(diff).length) return plan([], 'none', 0, true); + return plan([assertGroup(local, local.dirty), updateGroup(local, diff)], 'update', 0, true); + } if (!local.sourceId) { // Our own create whose identity was never written. return plan([ From c30b4a23d21cba6e659fe81babd1622863baa93d Mon Sep 17 00:00:00 2001 From: Linus Rath <139418639+rathlinus@users.noreply.github.com> Date: Sun, 27 Sep 2026 07:15:25 +0200 Subject: [PATCH 29/87] fix: stop contacts waiting forever on group memberships the server can't take #34 A contact added to or removed from a group this device can't change stayed dirty on every run: the membership upload skips such groups, while the contact's clean write kept the change pending until the server showed it. Turning sync off and signing out then always warned about changes waiting for the server. The clean write now puts such a membership back to what the server has, as it does for any edit of a read-only contact, and clears DIRTY; when that was the contact's only change the upload counts it as skipped. Memberships of writable groups in the contact's own account still wait until the group card shows them. This covers groups of another JMAP account. For groups in read-only address books the planner is ready, but the engine does not tell it a group's rights yet; that follows separately. Checked with planner tests for additions, removals, both kinds of group and a change next to an edit that uploads, and end to end through the engine for a group of a shared account, where turning sync off afterwards finds nothing waiting. --- .../__tests__/contacts/groups.test.ts | 74 +++++++++++++++++++ src/device-sync/__tests__/contacts/harness.ts | 13 +++- .../__tests__/contacts/real-engine.test.ts | 31 +++++++- src/device-sync/contacts/download.ts | 14 +++- src/device-sync/contacts/members.ts | 42 ++++++++++- src/device-sync/contacts/upload.ts | 11 +-- 6 files changed, 166 insertions(+), 19 deletions(-) diff --git a/src/device-sync/__tests__/contacts/groups.test.ts b/src/device-sync/__tests__/contacts/groups.test.ts index f29efe9b..c83c2637 100644 --- a/src/device-sync/__tests__/contacts/groups.test.ts +++ b/src/device-sync/__tests__/contacts/groups.test.ts @@ -20,6 +20,15 @@ async function localGroup(h: Harness, groupId: string) { return g; } +/** A group card in an address book this user may only read, on the device. */ +async function readOnlyGroup(h: Harness, members: Record = {}) { + const shared = h.server.addAddressBook(JMAP, { name: 'Shared' }); + h.readOnlyBooks.add(shared); + const gid = h.addCard({ uid: 'urn:uuid:group-shared', kind: 'group', name: { full: 'Shared' }, members, addressBookIds: { [shared]: true } }); + await h.download(gid); + return gid; +} + /** A group card holding Anna (apple card), both on the device. */ async function withGroup(members: Record = { [appleCard().uid as string]: true }) { const h = new Harness(); @@ -127,6 +136,71 @@ describe('contacts groups: membership edits', () => { expect(h.planner.planMembershipUploads([await h.contact(rawId)], await h.groups(), h.ctx)).toEqual([]); }); + it('puts back an addition to a group in read-only books instead of keeping the contact dirty', async () => { + const h = new Harness(); + const gid = await readOnlyGroup(h, { 'urn:uuid:someone': true }); + const { rawId } = await h.seed(appleCard()); + h.device.user.addToGroup(rawId, (await localGroup(h, gid)).groupId); + expect((await h.upload(rawId)).kind).toBe('revert'); + expect(h.membershipGroups(rawId)).toEqual([]); + expect(h.dirty(rawId)).toBe(false); + expect(h.planner.planMembershipUploads([await h.contact(rawId)], await h.groups(), h.ctx)).toEqual([]); + }); + + it('puts back a removal from a group in read-only books', async () => { + const h = new Harness(); + const gid = await readOnlyGroup(h, { [appleCard().uid as string]: true }); + const { rawId } = await h.seed(appleCard()); + const membership = (await h.contact(rawId)).rows.find((r) => r.mimetype === MimeType.GROUP_MEMBERSHIP)!; + h.device.user.deleteData(membership.id); + expect((await h.upload(rawId)).kind).toBe('revert'); + expect(h.membershipGroups(rawId)).toEqual([`${JMAP}/${gid}`]); + expect(h.dirty(rawId)).toBe(false); + }); + + it('puts back an addition to a group of another JMAP account', async () => { + const h = new Harness(); + h.server.addAccount('t', { name: 'Team' }); + const book = h.server.addAddressBook('t', { name: 'Team contacts' }); + const team = h.server.addCard('t', { uid: 'urn:uuid:team', kind: 'group', name: { full: 'Team' }, addressBookIds: { [book]: true } }); + const card = h.server.get('ContactCard', 't', team) as unknown as ContactCardWire; + await h.applyOk(h.planner.planGroupDownload(card, null, { ...h.ctx, jmapAccountId: 't' }).ops); + const { rawId } = await h.seed(appleCard()); + h.device.user.addToGroup(rawId, (await h.groups()).find((g) => g.sourceId === `t/${team}`)!.groupId); + expect(h.membershipGroups(rawId)).toEqual([`t/${team}`]); + expect((await h.upload(rawId)).kind).toBe('revert'); + expect(h.membershipGroups(rawId)).toEqual([]); + expect(h.dirty(rawId)).toBe(false); + }); + + it('still uploads a read-only contact joining a writable group (the group card is what changes)', async () => { + const h = new Harness(); + const other = h.server.addAddressBook(JMAP, { name: 'Shared' }); + h.readOnlyBooks.add(other); + const gid = h.addCard({ uid: 'urn:uuid:g5', kind: 'group', name: { full: 'Mine' }, members: { 'urn:uuid:someone': true } }); + await h.download(gid); + const { rawId } = await h.seed({ ...appleCard(), addressBookIds: { [other]: true } }); + h.device.user.addToGroup(rawId, (await localGroup(h, gid)).groupId); + expect((await h.upload(rawId)).kind).toBe('revert'); + expect(h.membershipGroups(rawId)).toEqual([`${JMAP}/${gid}`]); + expect(h.dirty(rawId)).toBe(true); + expect(h.planner.planMembershipUploads([await h.contact(rawId)], await h.groups(), h.ctx)).toEqual([ + { kind: 'update', id: gid, patch: { [`members/${appleCard().uid as string}`]: true } }, + ]); + }); + + it('uploads the edits it can next to a membership it puts back, and settles', async () => { + const h = new Harness(); + const gid = await readOnlyGroup(h); + const { id, rawId } = await h.seed(appleCard()); + h.device.user.addToGroup(rawId, (await localGroup(h, gid)).groupId); + h.device.user.updateData(h.rowsByKey(rawId)['emails:k1']._id as number, { data1: 'anna@new.example' }); + const plan = await h.upload(rawId); + expect(actionsOf(plan)).toEqual([{ kind: 'update', id, patch: { 'emails/k1/address': 'anna@new.example' } }]); + expect(h.membershipGroups(rawId)).toEqual([]); + expect(h.dirty(rawId)).toBe(false); + }); + it('does not take a server-side addition for a local removal while the contact is dirty', async () => { const { h, gid } = await withGroup({}); const { rawId } = await h.seed(googleCard()); diff --git a/src/device-sync/__tests__/contacts/harness.ts b/src/device-sync/__tests__/contacts/harness.ts index 095a1c9b..663dfbd0 100644 --- a/src/device-sync/__tests__/contacts/harness.ts +++ b/src/device-sync/__tests__/contacts/harness.ts @@ -6,6 +6,7 @@ */ import { GroupMembership, Groups, RawContacts } from '../../android-columns'; import { makeKeyMinter, parseObjectRef, uuidFrom } from '../../common/ids'; +import type { GroupRights } from '../../contacts/members'; import { contactsPlanner } from '../../contacts/planner'; import type { ContactCardWire, @@ -48,7 +49,7 @@ export class Harness { readonly unselected = new Set(); /** When false the context has no `groupsOf` (memberships not synced). */ groupIndex = true; - readonly ctx: ContactsContext; + readonly ctx: ContactsContext & GroupRights; constructor() { this.server.addAccount(JMAP, { name: ACCOUNT }); @@ -79,14 +80,20 @@ export class Harness { .filter((c) => c.kind === 'group' && (c.members as Record | undefined)?.[uid] === true) .map((c) => `${JMAP}/${c.id as string}`); }, + groupReadOnly(sourceId: string) { + const ref = parseObjectRef(sourceId); + const card = ref ? self.server.get('ContactCard', ref.accountId, ref.id) : undefined; + const books = Object.keys((card?.addressBookIds as Record | undefined) ?? {}); + return books.length > 0 && books.every((id) => self.readOnlyBooks.has(id)); + }, }; } /** The context as the engine may build it without a group index. */ - context(): ContactsContext { + context(): ContactsContext & GroupRights { if (this.groupIndex) return this.ctx; const { groupsOf: _drop, ...rest } = this.ctx; - return rest as ContactsContext; + return rest as ContactsContext & GroupRights; } async apply(group: OpGroup): Promise { diff --git a/src/device-sync/__tests__/contacts/real-engine.test.ts b/src/device-sync/__tests__/contacts/real-engine.test.ts index f42ba6ae..79207b23 100644 --- a/src/device-sync/__tests__/contacts/real-engine.test.ts +++ b/src/device-sync/__tests__/contacts/real-engine.test.ts @@ -3,16 +3,21 @@ // themselves are tested in the other files of this folder. import { describe, expect, it } from 'vitest'; +import { MimeType } from '../../android-columns'; import { calendarPlanner } from '../../calendar/planner'; import { contactsPlanner } from '../../contacts/planner'; -import { addServerCards, createHarness, type Harness } from '../engine/harness'; +import { addServerCards, CONTACTS_AUTHORITY, createHarness, type Harness } from '../engine/harness'; -function real(): Harness { - const h = createHarness(); +function real(options: Parameters[0] = {}): Harness { + const h = createHarness(options); h.deps.planners = { contacts: contactsPlanner, calendar: calendarPlanner }; return h; } +function memberships(h: Harness, rawContactId: number) { + return h.device.rows('data').filter((d) => Number(d.raw_contact_id) === rawContactId && d.mimetype === MimeType.GROUP_MEMBERSHIP); +} + /** An address book of the personal account that this device does not sync. */ function unsyncedBook(h: Harness, name = 'Archive'): string { const id = h.server.addAddressBook('a', { name }); @@ -51,4 +56,24 @@ describe('contacts through the engine', () => { expect(h.server.get('ContactCard', 'a', group)).toMatchObject({ addressBookIds: { [archive]: true } }); expect(h.device.rows('groups')).toEqual([]); }); + + it('puts back a membership in a group of another account, so no change waits for ever', async () => { + const h = real({ accounts: 'withShared' }); + const book = h.server.all('AddressBook', 'team')[0].id as string; + h.prefs.contactsSelection[`team/${book}`] = true; + const team = h.server.addCard('team', { uid: 'g-team', kind: 'group', name: { full: 'Team' }, addressBookIds: { [book]: true } }); + addServerCards(h, ['Ada Lovelace']); + expect((await h.run()).outcome).toBe('ok'); + + const ada = h.contactNamed('Ada Lovelace')!.id; + h.device.user.addToGroup(ada, Number(h.device.rows('groups').find((g) => g.sourceid === `team/${team}`)!._id)); + const report = await h.run(); + + expect(report).toMatchObject({ outcome: 'ok', stats: { skipped: 1 } }); + expect(h.contactNamed('Ada Lovelace')).toMatchObject({ dirty: false }); + expect(memberships(h, ada)).toEqual([]); + expect(h.server.get('ContactCard', 'team', team)?.members).toBeUndefined(); + // Turning sync off finds nothing waiting. + expect(await h.teardown(CONTACTS_AUTHORITY)).toEqual({ pending: 0 }); + }); }); diff --git a/src/device-sync/contacts/download.ts b/src/device-sync/contacts/download.ts index 066605f6..feacece2 100644 --- a/src/device-sync/contacts/download.ts +++ b/src/device-sync/contacts/download.ts @@ -14,13 +14,13 @@ */ import { Data, MimeType, RawContacts } from '../android-columns'; import { canonicalJson } from '../common/json'; -import { objectRef } from '../common/ids'; +import { objectRef, parseCollectionKey, parseObjectRef } from '../common/ids'; import type { ContactCardWire, ContactsContext, DownloadPlan, LocalContact, LocalDataRow } from '../planner'; import type { ProviderOp, WriteRow } from '../types'; import { changedFromBaseline, encodeBaseline, sameCell } from './cells'; import { localChanges, sameUnit } from './diff'; import { deletionsInferred, inPlaceEvidence, matchKind, reconcileKeys, type KindMatch } from './matching'; -import { planMemberships, remoteMemberships, shadowMemberOf, withMemberOf, withoutMemberOf } from './members'; +import { editableGroups, planMemberships, remoteMemberships, shadowMemberOf, withMemberOf, withoutMemberOf } from './members'; import { assertContact, deleteData, insertUnit, updateData, updateRawContact, updateUnit } from './ops'; import { shadowOf } from './photo'; import { KIND_ORDER, projectCard, rowMatchesUnit, SPECS, type Unit } from './project'; @@ -224,10 +224,16 @@ export interface CleanWriteOptions { accepted?: boolean; } +/** The JMAP account a local contact belongs to. */ +export function accountOf(local: LocalContact, ctx: Ctx): string { + return parseObjectRef(local.sourceId)?.accountId ?? parseCollectionKey(local.pending?.target)?.accountId ?? ctx.jmapAccountId; +} + /** * The ops that make a local contact's rows, shadow and raw columns the * server card's, behind the VERSION assert. Membership changes that the - * server does not have yet stay (and keep the contact dirty). + * server does not have yet stay (and keep the contact dirty), unless it can + * never take them: those are put back (`revertedMembers`). */ export function cleanWrite(local: LocalContact, card: ContactCardWire, ctx: Ctx, options: CleanWriteOptions) { const server = withoutMemberOf(card); @@ -243,6 +249,7 @@ export function cleanWrite(local: LocalContact, card: ContactCardWire, ctx: Ctx, restore: true, parent: local.rawContactId, groupRowIdBySourceId: (g) => ctx.groupRowIdBySourceId(g), + editable: editableGroups(ctx, accountOf(local, ctx)), }); const shadow = withMemberOf(shadowOf(server), members.memberOf); const raw = changedRawColumns(local, { @@ -258,6 +265,7 @@ export function cleanWrite(local: LocalContact, card: ContactCardWire, ctx: Ctx, ops: ops.length ? [assertContact(local, options.assertDirty), ...ops] : [], writes: ops.length, pendingMembers: members.pending, + revertedMembers: members.reverted, }; } diff --git a/src/device-sync/contacts/members.ts b/src/device-sync/contacts/members.ts index 0a07983b..118f3cb7 100644 --- a/src/device-sync/contacts/members.ts +++ b/src/device-sync/contacts/members.ts @@ -13,9 +13,12 @@ * Per group, a local change wins where the server did not change, the server * wins elsewhere; both sides can only flip a membership the same way, so * memberships never conflict. A local change stays in the base until the - * server has it, which keeps it visible to `planMembershipUploads`. + * server has it, which keeps it visible to `planMembershipUploads`; one the + * server can't take (a group in read-only books, or in another JMAP account) + * is put back by the next clean write instead. */ import { Data, GroupMembership, MimeType } from '../android-columns'; +import { parseObjectRef } from '../common/ids'; import type { ContactCardWire, ContactsContext, LocalDataRow } from '../planner'; import type { ProviderOp } from '../types'; import { text } from './cells'; @@ -23,6 +26,24 @@ import { entryKey } from './keys'; import { deleteData, insertUnit, updateData } from './ops'; import { MEMBER_OF, type Unit } from './project'; +/** + * Group rights the engine can add to the context (not part of + * ContactsContext yet): whether a group card, by SOURCE_ID, is in read-only + * address books only. Without it every synced group counts as writable. + */ +export interface GroupRights { + groupReadOnly?(sourceId: string): boolean; +} + +/** + * The groups whose members an upload for a contact of `account` can change: + * that account's groups outside read-only books. A membership change of any + * other group can never reach the server (`planMembershipUploads` skips it). + */ +export function editableGroups(ctx: ContactsContext & GroupRights, account: string): (sourceId: string) => boolean { + return (sourceId) => parseObjectRef(sourceId)?.accountId === account && !(ctx.groupReadOnly?.(sourceId) ?? false); +} + /** Remote memberships of a card, or null when the engine can't tell (memberships are then left alone). */ export function remoteMemberships(card: ContactCardWire, ctx: ContactsContext): string[] | null { if (typeof ctx.groupsOf !== 'function' || typeof card.uid !== 'string' || !card.uid) return null; @@ -71,6 +92,8 @@ export interface MemberPlan { /** Groups the device added the contact to, or removed it from, that the server does not show yet. */ added: string[]; removed: string[]; + /** Local changes of groups that are not `editable` were put back to what the server has. */ + reverted: boolean; } export interface MemberInput { @@ -83,13 +106,19 @@ export interface MemberInput { restore: boolean; parent: number | { ref: number }; groupRowIdBySourceId(sourceId: string): number | null; + /** + * Groups whose members the device may change (see `editableGroups`); a local + * change of another group is put back to what the server has rather than + * waiting for an upload that never comes. Without it every group counts. + */ + editable?(sourceId: string): boolean; } export function planMemberships(input: MemberInput): MemberPlan { const live = input.rows.filter((r) => r.mimetype === MimeType.GROUP_MEMBERSHIP); if (input.remote === null) { // Without the engine's group index memberships are not synced: nothing is written or kept pending. - return { ops: [], writes: 0, memberOf: input.base, pending: false, added: [], removed: [] }; + return { ops: [], writes: 0, memberOf: input.base, pending: false, added: [], removed: [], reverted: false }; } const ops: ProviderOp[] = []; const rowOf = new Map(); @@ -119,11 +148,17 @@ export function planMemberships(input: MemberInput): MemberPlan { after.add(g); }; + let reverted = false; for (const g of [...new Set([...base, ...remote, ...rowOf.keys()])].sort()) { const inBase = base.has(g); const inRemote = remote.has(g); const row = rowOf.get(g); - const change = inBase ? (!row && deletions ? 'removed' : null) : row ? 'added' : null; + let change = inBase ? (!row && deletions ? 'removed' : null) : row ? 'added' : null; + if (change && input.editable && !input.editable(g)) { + // A change the server can't take (a group in read-only books, or another account's): its state comes back. + if (change === 'added' ? !inRemote : inRemote) reverted = true; + change = null; + } if (change === 'added') { keep(g, row!); if (!inRemote) pendingAdded.add(g); @@ -145,5 +180,6 @@ export function planMemberships(input: MemberInput): MemberPlan { pending: unsynced || pendingAdded.size > 0 || pendingRemoved.size > 0, added: [...pendingAdded], removed: [...pendingRemoved], + reverted, }; } diff --git a/src/device-sync/contacts/upload.ts b/src/device-sync/contacts/upload.ts index debff260..34430e1b 100644 --- a/src/device-sync/contacts/upload.ts +++ b/src/device-sync/contacts/upload.ts @@ -23,7 +23,7 @@ import type { import type { ProviderOp } from '../types'; import { encodeBaseline } from './cells'; import { localChanges } from './diff'; -import { cardUnits, cleanWrite } from './download'; +import { accountOf, cardUnits, cleanWrite } from './download'; import { newEntries, unitEdits, unitEntries, type Edit, type NewEntry } from './entries'; import { parseEntryKey } from './keys'; import { deletionsInferred, inPlaceEvidence, matchKind, type KindMatch } from './matching'; @@ -263,18 +263,15 @@ export function planContactUpload(local: LocalContact, ctx: Ctx): UploadPlan[] = [{ kind: 'update', id: parseObjectRef(local.sourceId)!.id, patch }]; if (backedOff(local, ctx, actions)) return { kind: 'skip', reason: `poisoned:${local.poison!.type}` }; return { kind: 'upload', actions }; } -/** The JMAP account a local contact belongs to. */ -function accountOf(local: LocalContact, ctx: Ctx): string { - return parseObjectRef(local.sourceId)?.accountId ?? parseCollectionKey(local.pending?.target)?.accountId ?? ctx.jmapAccountId; -} - export function planContactAccepted(local: LocalContact, server: ContactCardWire, ctx: Ctx): AcceptedPlan { if (local.deleted) { const purge = group(local, [deleteWhereId('raw_contacts', local.rawContactId)]); From 20effab062d3f36cf3b853a598a0f5e0fb5e13a1 Mon Sep 17 00:00:00 2001 From: Linus Rath <139418639+rathlinus@users.noreply.github.com> Date: Sun, 27 Sep 2026 07:16:01 +0200 Subject: [PATCH 30/87] fix: keep an address's parts in step with a one-line edit on the device #34 AOSP Contacts edits a postal address as one formatted line; the provider then copies it into the street and empties the other parts. The upload sent only the new `full`, so the components on the server kept the old values: CardDAV clients and the app's editor, which show the parts, still showed the old city. A changed line now uploads with the parts it implies. They are read back along the old line: each of its segments must be some of the old parts, and a segment whose change is confined to one part rewrites that part like an in-place edit of its field. When the layout changed or a change spans two parts, the address keeps the whole line as its street, which is what the device holds, rather than an address without parts that the app's editor would show empty and drop on its next save. An unchanged line still uploads nothing, and editors that write the parts themselves are handled as before. Checked with planner tests for the edit from the device test, a street-only edit, the fallback, a re-inserted line and an editor that keeps the parts, and with unit tests of the read-back. --- .../__tests__/contacts/units.test.ts | 20 +++- .../__tests__/contacts/upload-fields.test.ts | 107 ++++++++++++++++++ src/device-sync/contacts/diff.ts | 50 +++++--- src/device-sync/contacts/postal.ts | 74 ++++++++++++ src/device-sync/contacts/upload.ts | 4 +- 5 files changed, 237 insertions(+), 18 deletions(-) diff --git a/src/device-sync/__tests__/contacts/units.test.ts b/src/device-sync/__tests__/contacts/units.test.ts index 7738244e..e9bb609c 100644 --- a/src/device-sync/__tests__/contacts/units.test.ts +++ b/src/device-sync/__tests__/contacts/units.test.ts @@ -2,7 +2,7 @@ import { describe, expect, it } from 'vitest'; import { Data, MimeType, PhoneType, RelationType } from '../../android-columns'; import { contactsPlanner } from '../../contacts/planner'; import { entryKey, orgKey, parseEntryKey, parseOrgKey } from '../../contacts/keys'; -import { editPostalComponents, postalCells } from '../../contacts/postal'; +import { editPostalComponents, postalCells, resplitPostal } from '../../contacts/postal'; import { base64ToBytes, dataUriBase64, shadowOf } from '../../contacts/photo'; import { fullIsDerived, nameCells } from '../../contacts/name'; import { phoneFlags, phoneType, relationName, relationType } from '../../contacts/types-map'; @@ -118,6 +118,24 @@ describe('contacts name and postal helpers', () => { expect(fullIsDerived({ components, full: 'B, A' })).toBe(false); }); + it('reads the parts of a rewritten one-line address back along the old line', () => { + // Apple's layout: street / city region postcode / country. + const old = { data1: '1 Infinite Loop\nCupertino CA 95014\nUSA', data4: '1 Infinite Loop', data7: 'Cupertino', data8: 'CA', data9: '95014', data10: 'USA' }; + const parts = (line: string) => { + const out = resplitPostal(old, line); + return out && { street: out.data4, city: out.data7, region: out.data8, postcode: out.data9, country: out.data10 }; + }; + expect(parts('1 Apple Park Way\nCupertino CA 95014\nUSA')).toEqual({ street: '1 Apple Park Way', city: 'Cupertino', region: 'CA', postcode: '95014', country: 'USA' }); + expect(parts('1 Infinite Loop\nSan Jose CA 95014\nUSA')).toMatchObject({ city: 'San Jose', region: 'CA', postcode: '95014' }); + expect(parts('1 Infinite Loop\nCupertino 95014\nUSA')).toMatchObject({ city: 'Cupertino', region: null, postcode: '95014' }); + expect(parts('1 Infinite Loop, Cupertino CA 95014, USA')).toMatchObject({ street: '1 Infinite Loop', city: 'Cupertino' }); + // Two parts of one segment changed, or the layout changed: nothing to pin the words to. + expect(parts('1 Infinite Loop\nSan Jose NV 95014\nUSA')).toBeNull(); + expect(parts('1 Infinite Loop\nCupertino CA 95014\nCalifornia\nUSA')).toBeNull(); + // An old line that isn't made of the parts (a `full` written apart from its components). + expect(resplitPostal({ ...old, data1: 'Apple HQ, Cupertino' }, 'Apple Park, Cupertino')).toBeNull(); + }); + it('keeps separators between street parts and falls back to the legacy flat fields', () => { expect(postalCells({ street: 'Old Rd 1', locality: 'Town' })).toMatchObject({ data4: 'Old Rd 1', data7: 'Town' }); const address = { components: [{ kind: 'number', value: '12' }, { kind: 'separator', value: '-' }, { kind: 'name', value: 'Rue X' }] }; diff --git a/src/device-sync/__tests__/contacts/upload-fields.test.ts b/src/device-sync/__tests__/contacts/upload-fields.test.ts index 62fb106d..e1582aa1 100644 --- a/src/device-sync/__tests__/contacts/upload-fields.test.ts +++ b/src/device-sync/__tests__/contacts/upload-fields.test.ts @@ -108,6 +108,113 @@ describe('contacts upload: one edited field, one pointer', () => { }); }); +describe('contacts upload: an address edited as one line (AOSP Contacts)', () => { + const munich = () => ({ + name: { full: 'Maria Muster' }, + addresses: { + home: { + contexts: { private: true }, + coordinates: 'geo:48.137,11.575', + components: [ + { kind: 'name', value: 'Marienplatz 1' }, + { kind: 'postcode', value: '80331' }, + { kind: 'locality', value: 'Munich' }, + { kind: 'region', value: 'Bavaria' }, + { kind: 'country', value: 'Germany' }, + ], + }, + }, + }); + const LINE = 'Marienplatz 1, 80331 Munich, Bavaria, Germany'; + + /** AOSP writes the formatted address alone; the provider keeps it in STREET and empties the other parts. */ + async function editLine(line: string) { + const h = new Harness(); + const { id, rawId } = await h.seed(munich()); + const row = h.rowsByKey(rawId)['addresses:home']; + expect(row.data1).toBe(LINE); + h.device.user.updateData(row._id as number, { data1: line, data4: null, data5: null, data6: null, data7: null, data8: null, data9: null, data10: null }); + expect(h.rowsByKey(rawId)['addresses:home']).toMatchObject({ data1: line, data4: line, data7: null }); + return { h, id, rawId }; + } + + async function patchOf(h: Harness, rawId: number): Promise { + const plan = h.planner.planUpload(await h.contact(rawId), h.ctx); + if (plan.kind !== 'upload' || plan.actions[0].kind !== 'update') throw new Error(`expected an update, got ${plan.kind}`); + return plan.actions[0].patch; + } + + const components = (street: string, locality = 'Munich', postcode = '80331') => [ + { kind: 'name', value: street }, + { kind: 'postcode', value: postcode }, + { kind: 'locality', value: locality }, + { kind: 'region', value: 'Bavaria' }, + { kind: 'country', value: 'Germany' }, + ]; + + it('reads the parts back from the new line, so `full` and the components agree', async () => { + const line = 'Industrieweg 42, 80331 Augsburg, Bavaria, Germany'; + const { h, id, rawId } = await editLine(line); + expect(await patchOf(h, rawId)).toEqual({ + 'addresses/home/components': components('Industrieweg 42', 'Augsburg'), + 'addresses/home/full': line, + }); + await h.upload(rawId); + expect(h.card(id).addresses!.home).toEqual({ + contexts: { private: true }, coordinates: 'geo:48.137,11.575', isOrdered: true, full: line, + components: components('Industrieweg 42', 'Augsburg'), + }); + // The device gets its parts back and settles. + expect(h.rowsByKey(rawId)['addresses:home']).toMatchObject({ data1: line, data4: 'Industrieweg 42', data7: 'Augsburg', data9: '80331' }); + const local = await h.contact(rawId); + expect(local.dirty).toBe(false); + expect(h.planner.planDownload(h.card(id), local, h.ctx).effect).toBe('none'); + }); + + it('changes only the part whose words changed', async () => { + const { h, rawId } = await editLine('Marienplatz 2, 80331 Munich, Bavaria, Germany'); + expect(await patchOf(h, rawId)).toEqual({ + 'addresses/home/components': components('Marienplatz 2'), + 'addresses/home/full': 'Marienplatz 2, 80331 Munich, Bavaria, Germany', + }); + }); + + it('keeps the whole line as the street when the change can not be pinned to one part', async () => { + const line = 'Industrieweg 42, 86150 Augsburg, Bavaria, Germany'; + const { h, id, rawId } = await editLine(line); + expect(await patchOf(h, rawId)).toEqual({ 'addresses/home/components': [{ kind: 'name', value: line }], 'addresses/home/full': line }); + await h.upload(rawId); + expect(h.card(id).addresses!.home).toMatchObject({ contexts: { private: true }, coordinates: 'geo:48.137,11.575', full: line, components: [{ kind: 'name', value: line }] }); + const local = await h.contact(rawId); + expect(local.dirty).toBe(false); + expect(h.planner.planDownload(h.card(id), local, h.ctx).effect).toBe('none'); + }); + + it('uploads nothing for the same line saved again', async () => { + const { h, rawId } = await editLine(LINE); + expect(h.planner.planUpload(await h.contact(rawId), h.ctx).kind).toBe('clean'); + }); + + it('reads the parts back from a line an editor re-inserted without its key', async () => { + const line = 'Marienplatz 1, 80331 Munich, Upper Bavaria, Germany'; + const h = new Harness(); + const { rawId } = await h.seed(munich()); + h.device.user.fossifySave(rawId, null, [{ [Data.MIMETYPE]: MimeType.STRUCTURED_POSTAL, data1: line, data2: 1 }]); + expect(await patchOf(h, rawId)).toEqual({ + 'addresses/home/components': components('Marienplatz 1').map((c) => (c.kind === 'region' ? { ...c, value: 'Upper Bavaria' } : c)), + 'addresses/home/full': line, + }); + }); + + it('edits the parts an editor kept as parts, whatever its line says', async () => { + const line = 'Marienplatz 1, 80331 Augsburg, Bavaria, Germany'; + const h = new Harness(); + const { rawId } = await h.seed(munich()); + h.device.user.updateData(h.rowsByKey(rawId)['addresses:home']._id as number, { data7: 'Augsburg', data1: line }); + expect(await patchOf(h, rawId)).toEqual({ 'addresses/home/components': components('Marienplatz 1', 'Augsburg'), 'addresses/home/full': line }); + }); +}); + describe('contacts upload: the name', () => { const NAME_KEY = 'name'; diff --git a/src/device-sync/contacts/diff.ts b/src/device-sync/contacts/diff.ts index e0dac5d5..7e9bd855 100644 --- a/src/device-sync/contacts/diff.ts +++ b/src/device-sync/contacts/diff.ts @@ -10,29 +10,49 @@ import { CommonKind, Data, MimeType, StructuredPostal as SP } from '../android-c import type { Row } from '../types'; import { changedFromBaseline, sameCell, text } from './cells'; import type { Match } from './matching'; -import { POSTAL_PART_COLUMNS } from './postal'; +import { POSTAL_PART_COLUMNS, providerSplitPostal, resplitPostal } from './postal'; import { SPECS, type Unit } from './project'; +const isPart = (column: string) => (POSTAL_PART_COLUMNS as readonly string[]).includes(column); + +/** A postal row holding the provider's split of a formatted address an editor wrote alone. */ +function isProviderSplit(m: Match): boolean { + const { row } = m; + if (row.mimetype !== MimeType.STRUCTURED_POSTAL || !providerSplitPostal(row.cells)) return false; + // A multi-part formatted address in STREET was the provider's split, not a typed street. + return m.how !== 'key' || !row.baseline || /[,\n]/.test(text(row.cells[SP.FORMATTED_ADDRESS]) ?? ''); +} + +/** The address as the device showed it before the edit: a keyed row's baseline, else the shadow's. */ +function postalBefore(m: Match): Row { + return m.how === 'key' && m.row.baseline ? m.row.baseline : m.unit.cells; +} + /** - * A postal row whose parts are only its formatted address copied into STREET: - * the provider's split of an editor that wrote FORMATTED_ADDRESS alone. + * The cells an upload takes a row's values from: its own, except for a + * provider split whose line changed. Its parts are read back from the new + * line along the old one (`resplitPostal`), else they stay what the device + * holds, the whole line as the street: `full` and the components never + * contradict each other on the server. */ -function providerSplitPostal(cells: Row): boolean { - const street = text(cells[SP.STREET]); - return street !== null - && street === text(cells[SP.FORMATTED_ADDRESS]) - && POSTAL_PART_COLUMNS.every((c) => c === SP.STREET || text(cells[c]) === null); +export function uploadCells(m: Match): Row { + if (!isProviderSplit(m)) return m.row.cells; + const before = postalBefore(m); + const line = text(m.row.cells[SP.FORMATTED_ADDRESS]); + if (!line || sameCell(line, before[SP.FORMATTED_ADDRESS])) return m.row.cells; + return { ...m.row.cells, ...(resplitPostal(before, line) ?? {}) }; } export function localChanges(m: Match): string[] { - const { row, unit } = m; + const { unit } = m; const changed = rawChanges(m).filter((c) => !unit.truncated?.includes(c)); - if (row.mimetype === MimeType.STRUCTURED_POSTAL && providerSplitPostal(row.cells)) { - // A multi-part formatted address in STREET was the provider's split, not a typed street. - const split = m.how !== 'key' || !row.baseline || /[,\n]/.test(text(row.cells[SP.FORMATTED_ADDRESS]) ?? ''); - if (split) return changed.filter((c) => !(POSTAL_PART_COLUMNS as readonly string[]).includes(c)); - } - return changed; + if (!isProviderSplit(m)) return changed; + // The provider's split is no typed street: only a new line changes the parts, to what it says. + const others = changed.filter((c) => !isPart(c)); + if (!others.includes(SP.FORMATTED_ADDRESS)) return others; + const before = postalBefore(m); + const after = uploadCells(m); + return [...others, ...POSTAL_PART_COLUMNS.filter((c) => !sameCell(after[c], before[c]))]; } function rawChanges(m: Match): string[] { diff --git a/src/device-sync/contacts/postal.ts b/src/device-sync/contacts/postal.ts index fd7d590e..c3795f7d 100644 --- a/src/device-sync/contacts/postal.ts +++ b/src/device-sync/contacts/postal.ts @@ -90,6 +90,80 @@ export function predictStoredPostal(cells: Row): Row { return out; } +/** + * A postal row whose parts are only its formatted address copied into STREET: + * the provider's split of an editor that wrote FORMATTED_ADDRESS alone. + */ +export function providerSplitPostal(cells: Row): boolean { + const street = text(cells[SP.STREET]); + return street !== null + && street === text(cells[SP.FORMATTED_ADDRESS]) + && POSTAL_PART_COLUMNS.every((c) => c === SP.STREET || text(cells[c]) === null); +} + +/** Lines and comma-separated segments of a formatted address. */ +const SEGMENT = /\s*(?:,|\r?\n)\s*/; + +const wordsOf = (value: string) => value.split(/\s+/).filter(Boolean); + +function wordsAt(words: string[], part: string[], at: number): boolean { + return at >= 0 && at + part.length <= words.length && part.every((w, i) => words[at + i] === w); +} + +/** The parts one segment of the old line is made of, in their order there (each part used once). */ +function segmentParts(segment: string[], parts: Map, used: Set): string[] | null { + if (!segment.length) return []; + for (const [column, words] of parts) { + if (used.has(column) || !wordsAt(segment, words, 0)) continue; + used.add(column); + const rest = segmentParts(segment.slice(words.length), parts, used); + if (rest) return [column, ...rest]; + used.delete(column); + } + return null; +} + +/** + * The parts of an address whose formatted line an editor rewrote alone (the + * provider then empties them on the device), read back along the old line: + * each segment of the old line must be the words of some old parts in a row, + * and the new line must have as many segments. A changed segment gives its + * words to its one part, or to the one part left once the parts at its start + * and end are found unchanged there. Null when the layout changed, or a change + * can't be pinned to one part. + */ +export function resplitPostal(old: Row, line: string): Row | null { + const before = text(old[SP.FORMATTED_ADDRESS]); + if (!before) return null; + const parts = new Map(); + for (const c of POSTAL_PART_COLUMNS) { + const value = text(old[c]); + if (value) parts.set(c, wordsOf(value)); + } + const oldSegments = before.split(SEGMENT).map(wordsOf).filter((w) => w.length); + const newSegments = line.split(SEGMENT).map(wordsOf).filter((w) => w.length); + if (oldSegments.length !== newSegments.length) return null; + const out: Row = { [SP.FORMATTED_ADDRESS]: line }; + for (const c of POSTAL_PART_COLUMNS) out[c] = text(old[c]); + const used = new Set(); + for (let i = 0; i < oldSegments.length; i++) { + const columns = segmentParts(oldSegments[i], parts, used); + if (!columns) return null; + const words = newSegments[i]; + if (words.join(' ') === oldSegments[i].join(' ')) continue; + const size = (k: number) => parts.get(columns[k])!.length; + let first = 0; + let start = 0; + while (first < columns.length - 1 && wordsAt(words, parts.get(columns[first])!, start)) start += size(first++); + let last = columns.length; + let end = words.length; + while (last - 1 > first && end - size(last - 1) >= start && wordsAt(words, parts.get(columns[last - 1])!, end - size(last - 1))) end -= size(--last); + if (last - first !== 1) return null; + out[columns[first]] = words.slice(start, end).join(' ') || null; + } + return out; +} + /** The address's components with the edited columns applied (a list for `addresses//components`). */ export function editPostalComponents(address: ContactAddress, cells: Row, changed: ReadonlySet): Component[] { const comps = clone(components(address)); diff --git a/src/device-sync/contacts/upload.ts b/src/device-sync/contacts/upload.ts index 34430e1b..dbd8b9a1 100644 --- a/src/device-sync/contacts/upload.ts +++ b/src/device-sync/contacts/upload.ts @@ -22,7 +22,7 @@ import type { } from '../planner'; import type { ProviderOp } from '../types'; import { encodeBaseline } from './cells'; -import { localChanges } from './diff'; +import { localChanges, uploadCells } from './diff'; import { accountOf, cardUnits, cleanWrite } from './download'; import { newEntries, unitEdits, unitEntries, type Edit, type NewEntry } from './entries'; import { parseEntryKey } from './keys'; @@ -88,7 +88,7 @@ function collectChanges(local: LocalContact, shadow: ContactCardWire, ctx: Ctx): const changed = localChanges(m); if (!changed.length) continue; const shown = kind.mimetype === MimeType.RELATION ? (m.unit.cells[Data.DATA1] as string | null) : null; - const e = unitEdits(kind.mimetype, m.unit.key, shadow, m.row.cells, m.unit.cells, new Set(changed), shown, m.how === 'key'); + const e = unitEdits(kind.mimetype, m.unit.key, shadow, uploadCells(m), m.unit.cells, new Set(changed), shown, m.how === 'key'); out.edits.push(...e.edits); out.added.push(...e.added); out.removed.push(...e.removed); From 05e3605ec7f54db3a974d251ca6749dfc3aa2bc0 Mon Sep 17 00:00:00 2001 From: Linus Rath <139418639+rathlinus@users.noreply.github.com> Date: Sun, 27 Sep 2026 07:16:43 +0200 Subject: [PATCH 31/87] fix: keep a group rename when a membership change of the group uploads first #34 Accepting a membership patch of a group card wrote the server's title over the device's and cleared DIRTY. A rename that had not reached the server in the same run, because the server refused it or the user made it after the group's own upload was planned, was silently undone. The accepted write now keeps a pending rename that the accepted card does not show: it stores the new shadow but leaves the title and DIRTY, so the rename uploads against it. A card that shows the device's title, or a different one from a concurrent server rename, is taken as before. Checked with a planner test and end to end through the engine with a refused rename and a membership change in the same run. --- .../__tests__/contacts/groups.test.ts | 19 +++++++++++++ .../__tests__/contacts/real-engine.test.ts | 27 ++++++++++++++++++- src/device-sync/contacts/groups.ts | 13 ++++++--- 3 files changed, 54 insertions(+), 5 deletions(-) diff --git a/src/device-sync/__tests__/contacts/groups.test.ts b/src/device-sync/__tests__/contacts/groups.test.ts index c83c2637..d9ac7869 100644 --- a/src/device-sync/__tests__/contacts/groups.test.ts +++ b/src/device-sync/__tests__/contacts/groups.test.ts @@ -111,6 +111,25 @@ describe('contacts groups: membership edits', () => { expect(h.planner.planMembershipUploads([await h.contact(rawId)], await h.groups(), h.ctx)).toEqual([]); }); + it('keeps a rename the server does not have when a membership patch of the group is accepted', async () => { + const h = new Harness(); + const gid = h.addCard({ uid: 'urn:uuid:g2', kind: 'group', name: { full: 'Work' }, members: { 'urn:uuid:someone': true } }); + await h.download(gid); + const { rawId } = await h.seed(appleCard()); + const g = await localGroup(h, gid); + h.device.user.addToGroup(rawId, g.groupId); + await h.upload(rawId); + // Renamed after the group's own upload was planned, before the membership phase read it. + h.device.user.updateGroup(g.groupId, { [Groups.TITLE]: 'Colleagues' }); + const read = await localGroup(h, gid); + h.send(h.planner.planMembershipUploads([await h.contact(rawId)], await h.groups(), h.ctx)); + await h.applyOk(h.planner.planGroupAccepted(read, h.card(gid), h.ctx).ops); + expect(await localGroup(h, gid)).toMatchObject({ title: 'Colleagues', dirty: true, shadow: { members: { [appleCard().uid as string]: true } } }); + expect(h.planner.planGroupUpload(await localGroup(h, gid), h.ctx)).toEqual({ + kind: 'upload', actions: [{ kind: 'update', id: gid, patch: { 'name/full': 'Colleagues' } }], + }); + }); + it('sends the whole map for the first member of an empty group', async () => { const h = new Harness(); const empty = h.addCard({ uid: 'urn:uuid:g3', kind: 'group', name: { full: 'Empty' } }); diff --git a/src/device-sync/__tests__/contacts/real-engine.test.ts b/src/device-sync/__tests__/contacts/real-engine.test.ts index 79207b23..758b5e3c 100644 --- a/src/device-sync/__tests__/contacts/real-engine.test.ts +++ b/src/device-sync/__tests__/contacts/real-engine.test.ts @@ -3,10 +3,11 @@ // themselves are tested in the other files of this folder. import { describe, expect, it } from 'vitest'; -import { MimeType } from '../../android-columns'; +import { Groups, MimeType } from '../../android-columns'; import { calendarPlanner } from '../../calendar/planner'; import { contactsPlanner } from '../../contacts/planner'; import { addServerCards, CONTACTS_AUTHORITY, createHarness, type Harness } from '../engine/harness'; +import type { SetTarget } from '../fakes/fake-jmap-server'; function real(options: Parameters[0] = {}): Harness { const h = createHarness(options); @@ -76,4 +77,28 @@ describe('contacts through the engine', () => { // Turning sync off finds nothing waiting. expect(await h.teardown(CONTACTS_AUTHORITY)).toEqual({ pending: 0 }); }); + + it('keeps a group rename the server refused when a membership patch of that group goes through', async () => { + const h = real(); + const [ada] = addServerCards(h, ['Ada Lovelace']); + const group = h.server.addCard('a', { uid: 'g1', kind: 'group', name: { full: 'Friends' }, members: { other: true }, addressBookIds: { [h.book]: true } }); + expect((await h.run()).outcome).toBe('ok'); + const row = () => h.device.rows('groups').find((g) => g.sourceid === `a/${group}`)!; + const groupId = Number(row()._id); + h.device.user.updateGroup(groupId, { [Groups.TITLE]: 'Best friends' }); + h.device.user.addToGroup(h.contactNamed('Ada Lovelace')!.id, groupId); + const rename = (t: SetTarget) => t.op === 'update' && t.id === group && !!t.object && 'name/full' in t.object; + h.server.setErrorFor('ContactCard', 'a', rename, { type: 'invalidProperties', properties: ['name'] }); + + const report = await h.run(); + expect(report.itemErrors).toMatchObject([{ ref: `a/${group}`, type: 'invalidProperties' }]); + const uid = h.server.get('ContactCard', 'a', ada)!.uid as string; + expect(h.server.get('ContactCard', 'a', group)).toMatchObject({ name: { full: 'Friends' }, members: { other: true, [uid]: true } }); + expect(row()).toMatchObject({ [Groups.TITLE]: 'Best friends', [Groups.DIRTY]: 1 }); + + // The rename is still the device's and goes up once the server takes it. + expect((await h.run()).outcome).toBe('ok'); + expect(h.server.get('ContactCard', 'a', group)).toMatchObject({ name: { full: 'Best friends' } }); + expect(row()).toMatchObject({ [Groups.TITLE]: 'Best friends', [Groups.DIRTY]: 0 }); + }); }); diff --git a/src/device-sync/contacts/groups.ts b/src/device-sync/contacts/groups.ts index c9d0b3f7..45829807 100644 --- a/src/device-sync/contacts/groups.ts +++ b/src/device-sync/contacts/groups.ts @@ -193,12 +193,17 @@ export function planGroupAccepted(local: LocalGroup, server: ContactCardWire, ct const identity = objectRef(account, server.id); const raw: WriteRow = { [Groups.SOURCE_ID]: identity, [Groups.SYNC3]: null, [Groups.SYNC4]: null, ...cardColumns(server, ctx) }; const title = groupTitle(server); + const keepDirtyOps = { ref: identity, ops: [updateGroup(local, raw)] }; + // A rename the server still lacks (the upload it accepted was another one, a membership + // patch) stays the device's: title and DIRTY are kept, and it uploads against the new shadow. + const shadowTitle = groupTitle(local.shadow); + const renamed = local.dirty && !!local.title?.trim() && !sameCell(local.title, shadowTitle); + if (renamed && sameCell(title, shadowTitle)) { + return { ops: { ref: identity, ops: [assertGroup(local, null), updateGroup(local, raw)] }, keepDirtyOps }; + } const ops: WriteRow = { ...raw, [Groups.DIRTY]: 0 }; if (!sameCell(local.title, title)) ops[Groups.TITLE] = title ?? ''; - return { - ops: { ref: identity, ops: [assertGroup(local, null), updateGroup(local, ops)] }, - keepDirtyOps: { ref: identity, ops: [updateGroup(local, raw)] }, - }; + return { ops: { ref: identity, ops: [assertGroup(local, null), updateGroup(local, ops)] }, keepDirtyOps }; } /** From 00f1a66095e428ccef881152548f124458418c19 Mon Sep 17 00:00:00 2001 From: Linus Rath <139418639+rathlinus@users.noreply.github.com> Date: Sun, 27 Sep 2026 07:17:21 +0200 Subject: [PATCH 32/87] fix: keep phone, email and website types as picked on the device #34 Types Android has but RFC 9553 lacks uploaded as a label naming them, yet every label read back as a custom type, and MMS and company main read back as Other and Main. A number typed Car showed up as "Custom: car" right after the sync, an MMS number as Other, a mobile email as "Custom: mobile" and a blog link as "Custom: blog". Such a label now names its type again, for phones only next to the feature the type uploads with, so a custom label stays custom. MMS uploads its `mms` label next to `text`, which keeps a text-capable number from reading as MMS, and `main-number` with `work` reads as company main. Checked with a round trip of every phone, email and website type, both for a contact created on the device and for a type changed on a synced entry. --- .../__tests__/contacts/download.test.ts | 5 ++- .../__tests__/contacts/units.test.ts | 15 +++++++- .../__tests__/contacts/upload-fields.test.ts | 38 ++++++++++++++++++- src/device-sync/contacts/types-map.ts | 29 ++++++++++++-- 4 files changed, 79 insertions(+), 8 deletions(-) diff --git a/src/device-sync/__tests__/contacts/download.test.ts b/src/device-sync/__tests__/contacts/download.test.ts index 3bdf68f2..4fd6e639 100644 --- a/src/device-sync/__tests__/contacts/download.test.ts +++ b/src/device-sync/__tests__/contacts/download.test.ts @@ -99,7 +99,7 @@ describe('contacts download: golden rows', () => { expect(Object.keys(rows).some((k) => k.startsWith('onlineServices'))).toBe(false); }); - it('maps a Google card: custom display name, legacy cell phone, labelled link, department list, relation by name', async () => { + it('maps a Google card: custom display name, legacy cell phone, profile link, department list, relation by name', async () => { const h = new Harness(); const { rawId } = await h.seed(googleCard()); const rows = h.rowsByKey(rawId); @@ -110,7 +110,8 @@ describe('contacts download: golden rows', () => { expect(cells(rows['nicknames:k1'], ['data1'])).toEqual({ data1: 'Gü' }); expect(cells(rows['nicknames:k2'], ['data1'])).toEqual({ data1: 'GKG' }); expect(cells(rows['phones:k1'], TYPED)).toEqual({ data1: '0171 2345678', data2: '2', data3: null }); - expect(cells(rows['links:k1'], TYPED)).toEqual({ data1: 'https://example.org/~guenther', data2: '0', data3: 'profile' }); + // The label names a type Android has. + expect(cells(rows['links:k1'], TYPED)).toEqual({ data1: 'https://example.org/~guenther', data2: '3', data3: null }); expect(cells(rows['organizations:k1|titles:k1,'], ORG)).toEqual({ data1: 'Uni Beispiel', data4: 'Professor', data5: 'Informatik, AG Sync', data6: null }); expect(cells(rows['relatedTo:Maria Groß'], TYPED)).toEqual({ data1: 'Maria Groß', data2: '14', data3: null }); expect(cells(rows['anniversaries:k1'], TYPED)).toEqual({ data1: '--08-15', data2: '1', data3: null }); diff --git a/src/device-sync/__tests__/contacts/units.test.ts b/src/device-sync/__tests__/contacts/units.test.ts index e9bb609c..87ed4457 100644 --- a/src/device-sync/__tests__/contacts/units.test.ts +++ b/src/device-sync/__tests__/contacts/units.test.ts @@ -1,11 +1,11 @@ import { describe, expect, it } from 'vitest'; -import { Data, MimeType, PhoneType, RelationType } from '../../android-columns'; +import { Data, EmailType, MimeType, PhoneType, RelationType, WebsiteType } from '../../android-columns'; import { contactsPlanner } from '../../contacts/planner'; import { entryKey, orgKey, parseEntryKey, parseOrgKey } from '../../contacts/keys'; import { editPostalComponents, postalCells, resplitPostal } from '../../contacts/postal'; import { base64ToBytes, dataUriBase64, shadowOf } from '../../contacts/photo'; import { fullIsDerived, nameCells } from '../../contacts/name'; -import { phoneFlags, phoneType, relationName, relationType } from '../../contacts/types-map'; +import { emailType, phoneFlags, phoneType, relationName, relationType, websiteType } from '../../contacts/types-map'; import type { ContactCardWire } from '../../planner'; import { JPEG } from './fixtures'; @@ -94,6 +94,17 @@ describe('contacts type tables', () => { expect(phoneFlags(PhoneType.CAR, null)).toEqual({ contexts: [], features: ['voice'], label: 'car' }); }); + it('reads back the types RFC 9553 lacks from the label they upload with', () => { + expect(phoneType({ features: { 'main-number': true }, contexts: { work: true } }).type).toBe(PhoneType.COMPANY_MAIN); + expect(phoneType({ features: { voice: true }, label: 'car' })).toEqual({ type: PhoneType.CAR, label: null }); + expect(phoneType({ features: { text: true }, label: 'mms' })).toEqual({ type: PhoneType.MMS, label: null }); + // A custom label uploads without the feature, so it stays custom; a lone `text` is no MMS. + expect(phoneType({ label: 'car' })).toEqual({ type: PhoneType.CUSTOM, label: 'car' }); + expect(phoneType({ features: { text: true } }).type).toBe(PhoneType.OTHER); + expect(emailType({ label: 'mobile' })).toEqual({ type: EmailType.MOBILE, label: null }); + expect(websiteType({ label: 'blog', contexts: { work: true } })).toEqual({ type: WebsiteType.BLOG, label: null }); + }); + it('maps relations both ways, device-only types to the nearest RFC type', () => { expect(relationType({ friend: true })).toEqual({ type: RelationType.FRIEND, label: null }); expect(relationType({ 'co-worker': true })).toEqual({ type: RelationType.CUSTOM, label: 'co-worker' }); diff --git a/src/device-sync/__tests__/contacts/upload-fields.test.ts b/src/device-sync/__tests__/contacts/upload-fields.test.ts index e1582aa1..cca03bb9 100644 --- a/src/device-sync/__tests__/contacts/upload-fields.test.ts +++ b/src/device-sync/__tests__/contacts/upload-fields.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it } from 'vitest'; -import { Data, MimeType } from '../../android-columns'; +import { Data, EmailType, MimeType, PhoneType, WebsiteType } from '../../android-columns'; import { deepEqual } from '../../common/json'; import { applyPatch, assertPatchWellFormed, type PatchObject } from '../../common/patch'; import type { ContactCardWire } from '../../planner'; @@ -108,6 +108,42 @@ describe('contacts upload: one edited field, one pointer', () => { }); }); +describe('contacts upload: every device type comes back as the device wrote it', () => { + const cases = (types: Record, custom: number) => + Object.entries(types).map(([name, type]) => [name, type, type === custom ? 'Boat' : null] as const); + const kinds = [ + { mimetype: MimeType.PHONE, value: '+49 30 1', map: 'phones', entry: { number: '+49 30 1', features: { voice: true } }, types: cases(PhoneType, PhoneType.CUSTOM) }, + { mimetype: MimeType.EMAIL, value: 'typed@example.org', map: 'emails', entry: { address: 'typed@example.org' }, types: cases(EmailType, EmailType.CUSTOM) }, + { mimetype: MimeType.WEBSITE, value: 'https://typed.example', map: 'links', entry: { uri: 'https://typed.example' }, types: cases(WebsiteType, WebsiteType.CUSTOM) }, + ]; + const typeOf = (row: Row) => [Number(row.data2), row.data3 ?? null]; + + describe.each(kinds)('$map', ({ mimetype, value, map, entry, types }) => { + it.each(types)('%s, on a contact created on the device', async (_name, type, label) => { + const h = new Harness(); + const rawId = h.device.user.insertContact('usera@example.org', [ + { [Data.MIMETYPE]: MimeType.STRUCTURED_NAME, data1: 'Typed Person', data2: 'Typed', data3: 'Person' }, + { [Data.MIMETYPE]: mimetype, data1: value, data2: type, data3: label }, + ]); + await h.upload(rawId); + const [row] = h.device.rows('data', 'raw_contact_id = ? AND mimetype = ?', [rawId, mimetype]); + expect(typeOf(row)).toEqual([type, label]); + const local = await h.contact(rawId); + expect(local.dirty).toBe(false); + expect(h.planner.planDownload(h.card(h.cardIdOf(local)), local, h.ctx).effect).toBe('none'); + }); + + it.each(types)('%s, chosen on the device for a synced entry', async (_name, type, label) => { + const h = new Harness(); + const { id, rawId } = await h.seed({ name: { full: 'Typed Person' }, [map]: { e1: entry } }); + h.device.user.updateData(h.rowsByKey(rawId)[`${map}:e1`]._id as number, { data2: type, data3: label }); + await h.upload(rawId); + expect(typeOf(h.rowsByKey(rawId)[`${map}:e1`])).toEqual([type, label]); + expect(h.planner.planDownload(h.card(id), await h.contact(rawId), h.ctx).effect).toBe('none'); + }); + }); +}); + describe('contacts upload: an address edited as one line (AOSP Contacts)', () => { const munich = () => ({ name: { full: 'Maria Muster' }, diff --git a/src/device-sync/contacts/types-map.ts b/src/device-sync/contacts/types-map.ts index bd39d86c..d8d9c290 100644 --- a/src/device-sync/contacts/types-map.ts +++ b/src/device-sync/contacts/types-map.ts @@ -34,9 +34,22 @@ function labelled(label: unknown, type: number, custom: number): TypeCells { return typeof label === 'string' && label.trim() ? { type: custom, label } : { type, label: null }; } +/** + * Android types RFC 9553 has no word for upload as a label naming them; that + * label (exactly as written) names the type again on the way back. + */ +function namedType(labels: Record, label: unknown): number | null { + const hit = Object.entries(labels).find(([, name]) => name === label); + return hit ? Number(hit[0]) : null; +} + // ─── Emails ──────────────────────────────────────────── +const EMAIL_LABELS: Record = { [EmailType.MOBILE]: 'mobile' }; + export function emailType(entry: { contexts?: unknown; label?: unknown }): TypeCells { + const named = namedType(EMAIL_LABELS, entry.label); + if (named !== null) return { type: named, label: null }; const type = on(entry.contexts, 'private') ? EmailType.HOME : on(entry.contexts, 'work') ? EmailType.WORK : EmailType.OTHER; return labelled(entry.label, type, EmailType.CUSTOM); } @@ -45,7 +58,7 @@ export function emailFlags(type: number | null, label: string | null): EntryFlag switch (type) { case EmailType.HOME: return { contexts: ['private'], label: null }; case EmailType.WORK: return { contexts: ['work'], label: null }; - case EmailType.MOBILE: return { contexts: [], label: 'mobile' }; + case EmailType.MOBILE: return { contexts: [], label: EMAIL_LABELS[EmailType.MOBILE] }; case EmailType.CUSTOM: return { contexts: [], label }; default: return { contexts: [], label: null }; } @@ -53,10 +66,16 @@ export function emailFlags(type: number | null, label: string | null): EntryFlag // ─── Phones ──────────────────────────────────────────── -/** First match wins, as in the design's table; `cell` is read as `mobile`. */ +/** + * First match wins, as in the design's table; `cell` is read as `mobile`. A + * label naming a type RFC 9553 lacks, next to the feature that type uploads + * with, is that type (a custom "car" uploads without the feature). + */ export function phoneType(entry: { features?: unknown; contexts?: unknown; label?: unknown }): TypeCells { const f = (n: string) => on(entry.features, n); const c = (n: string) => on(entry.contexts, n); + const named = namedType(PHONE_LABELS, entry.label); + if (named !== null && (phoneFlags(named, null).features ?? []).every(f)) return { type: named, label: null }; const mobile = f('mobile') || f('cell'); let type: number = PhoneType.OTHER; if (mobile && c('work')) type = PhoneType.WORK_MOBILE; @@ -67,6 +86,7 @@ export function phoneType(entry: { features?: unknown; contexts?: unknown; label else if (f('pager') && c('work')) type = PhoneType.WORK_PAGER; else if (f('pager')) type = PhoneType.PAGER; else if (f('textphone')) type = PhoneType.TTY_TDD; + else if (f('main-number') && c('work')) type = PhoneType.COMPANY_MAIN; else if (f('main-number')) type = PhoneType.MAIN; else if (c('work')) type = PhoneType.WORK; else if (c('private')) type = PhoneType.HOME; @@ -83,6 +103,7 @@ const PHONE_LABELS: Record = { [PhoneType.RADIO]: 'radio', [PhoneType.TELEX]: 'telex', [PhoneType.ASSISTANT]: 'assistant', + [PhoneType.MMS]: 'mms', }; export function phoneFlags(type: number | null, label: string | null): EntryFlags { @@ -99,7 +120,7 @@ export function phoneFlags(type: number | null, label: string | null): EntryFlag case PhoneType.COMPANY_MAIN: return { contexts: ['work'], features: ['main-number'], label: null }; case PhoneType.WORK: return { contexts: ['work'], features: ['voice'], label: null }; case PhoneType.HOME: return { contexts: ['private'], features: ['voice'], label: null }; - case PhoneType.MMS: return { contexts: [], features: ['text'], label: null }; + case PhoneType.MMS: return { contexts: [], features: ['text'], label: PHONE_LABELS[PhoneType.MMS] }; case PhoneType.CUSTOM: return { contexts: [], features: [], label }; case null: case PhoneType.OTHER: return { contexts: [], features: [], label: null }; @@ -124,6 +145,8 @@ export function postalFlags(type: number | null, label: string | null): EntryFla } export function websiteType(entry: { contexts?: unknown; label?: unknown }): TypeCells { + const named = namedType(WEBSITE_LABELS, entry.label); + if (named !== null) return { type: named, label: null }; const type = on(entry.contexts, 'private') ? WebsiteType.HOME : on(entry.contexts, 'work') ? WebsiteType.WORK : WebsiteType.OTHER; return labelled(entry.label, type, WebsiteType.CUSTOM); } From d34641d1f9a2495a877bb1d2812d106f6377ce82 Mon Sep 17 00:00:00 2001 From: Linus Rath <139418639+rathlinus@users.noreply.github.com> Date: Sun, 27 Sep 2026 07:17:56 +0200 Subject: [PATCH 33/87] fix: sync contacts whose photo is too large for the device #34 A contact's rows are written in one provider batch, and one batch is one Binder transaction of at most 1 MB. A contact with a larger photo failed with `tooLarge` on every run, stayed on `stale`, never showed on the device, and its device edits never uploaded. Photos above about 512 KiB are now kept on the server only: the contact is written without the photo. The shadow records that the device holds no row for the card's photo, so the missing row is not read as a deletion. The same applies to a photo whose bytes could not be fetched, which an unrelated edit of the contact used to delete from the server. A photo set on the device still replaces the server's, and one deleted there is still deleted on the server. Checked with planner tests for the insert, the size limit, a photo that could not be fetched and a merge, and end to end through the engine with the probe's 900 KB photo. --- .../__tests__/contacts/lifecycle.test.ts | 83 ++++++++++++++++++- .../__tests__/contacts/merge.test.ts | 12 ++- .../__tests__/contacts/real-engine.test.ts | 24 +++++- src/device-sync/contacts/download.ts | 34 ++++++-- src/device-sync/contacts/members.ts | 7 +- src/device-sync/contacts/photo.ts | 32 ++++++- src/device-sync/contacts/upload.ts | 14 +++- 7 files changed, 186 insertions(+), 20 deletions(-) diff --git a/src/device-sync/__tests__/contacts/lifecycle.test.ts b/src/device-sync/__tests__/contacts/lifecycle.test.ts index 94c414ff..9e69cd74 100644 --- a/src/device-sync/__tests__/contacts/lifecycle.test.ts +++ b/src/device-sync/__tests__/contacts/lifecycle.test.ts @@ -1,7 +1,8 @@ import { describe, expect, it } from 'vitest'; import { Data, MimeType } from '../../android-columns'; +import { MAX_PHOTO_BASE64 } from '../../contacts/photo'; import type { LocalContact } from '../../planner'; -import { JPEG, probeCard } from './fixtures'; +import { JPEG, JPEG2, probeCard } from './fixtures'; import { Harness, JMAP } from './harness'; /** The engine's view after every write: a clean contact whose rows the server card describes. */ @@ -100,6 +101,86 @@ describe('contacts lifecycle', () => { expect(plan.kind === 'upload' && Object.keys((plan.actions[0] as { patch: object }).patch)).toEqual(['emails/e1/address']); }); + it('keeps a photo too large for a provider batch on the server only, and never reads it as deleted', async () => { + const h = new Harness(); + // About 900 KB of JPEG: the fake provider refuses a batch over 1 MB, as Binder does. + const big = 'A'.repeat(1_200_000); + const { id, rawId } = await h.seed({ + name: { full: 'Big Photo' }, + emails: { e1: { address: 'a@example.org' } }, + media: { p: { kind: 'photo', uri: `data:image/jpeg;base64,${big}`, mediaType: 'image/jpeg' } }, + }); + // The contact is there, without the photo; its echo writes nothing. + expect(Object.keys(h.rowsByKey(rawId)).sort()).toEqual(['emails:e1', 'name']); + await expectSettled(h, id, rawId); + + h.device.user.updateData(h.rowsByKey(rawId)['emails:e1']._id as number, { data1: 'b@example.org' }); + expect(h.planner.planUpload(await h.contact(rawId), h.ctx)).toEqual({ + kind: 'upload', actions: [{ kind: 'update', id, patch: { 'emails/e1/address': 'b@example.org' } }], + }); + await h.upload(rawId); + expect(h.card(id).media).toEqual({ p: { kind: 'photo', uri: `data:image/jpeg;base64,${big}`, mediaType: 'image/jpeg' } }); + await expectSettled(h, id, rawId); + + // A photo set on the device replaces it. + h.device.user.setPhoto(rawId, JPEG2); + h.devicePhotos.set(rawId, JPEG2); + expect(h.planner.planUpload(await h.contact(rawId), h.ctx)).toEqual({ + kind: 'upload', actions: [{ kind: 'update', id, patch: { 'media/p/uri': `data:image/jpeg;base64,${JPEG2}`, 'media/p/mediaType': 'image/jpeg' } }], + }); + await h.upload(rawId); + expect(h.device.photo(Number(h.rowsByKey(rawId)['media:p'].data14))).toBe(JPEG2); + await expectSettled(h, id, rawId); + }); + + it('writes a photo up to the budget of a provider batch, and none beyond it', async () => { + const h = new Harness(); + const seedPhoto = (b64: string) => h.seed({ name: { full: 'Sized' }, media: { p: { kind: 'photo', uri: `data:image/jpeg;base64,${b64}` } } }); + const fits = 'A'.repeat(MAX_PHOTO_BASE64); + const within = await seedPhoto(fits); + expect(h.device.photo(Number(h.rowsByKey(within.rawId)['media:p'].data14))).toBe(fits); + await expectSettled(h, within.id, within.rawId); + const beyond = await seedPhoto('A'.repeat(MAX_PHOTO_BASE64 + 4)); + expect(h.rowsByKey(beyond.rawId)['media:p']).toBeUndefined(); + await expectSettled(h, beyond.id, beyond.rawId); + }); + + it('never reads a photo whose bytes could not be fetched as deleted, and writes it once they can be', async () => { + const h = new Harness(); + const { id, rawId } = await h.seed({ + name: { full: 'Blob Photo' }, + emails: { e1: { address: 'a@example.org' } }, + media: { p: { kind: 'photo', blobId: 'B1', mediaType: 'image/jpeg' } }, + }); + expect(h.rowsByKey(rawId)['media:p']).toBeUndefined(); + h.device.user.updateData(h.rowsByKey(rawId)['emails:e1']._id as number, { data1: 'b@example.org' }); + expect(h.planner.planUpload(await h.contact(rawId), h.ctx)).toEqual({ + kind: 'upload', actions: [{ kind: 'update', id, patch: { 'emails/e1/address': 'b@example.org' } }], + }); + await h.upload(rawId); + expect(h.card(id).media).toEqual({ p: { kind: 'photo', blobId: 'B1', mediaType: 'image/jpeg' } }); + + // With the bytes at hand, the next write of the card brings the photo. + h.blobs.set('B1', JPEG); + h.server.serverUpdate('ContactCard', JMAP, id, { 'emails/e1/address': 'c@example.org' }); + await h.download(id); + expect(h.device.photo(Number(h.rowsByKey(rawId)['media:p'].data14))).toBe(JPEG); + await expectSettled(h, id, rawId); + }); + + it('does not read a server photo it could not write as deleted after merging it into an edited contact', async () => { + const h = new Harness(); + const { id, rawId } = await h.seed({ name: { full: 'Merge Photo' }, emails: { e1: { address: 'a@example.org' } }, phones: { p1: { number: '1' } } }); + // The server adds a photo too large to write while the device edits the phone. + h.device.user.updateData(h.rowsByKey(rawId)['phones:p1']._id as number, { data1: '2' }); + const big = 'A'.repeat(1_200_000); + h.server.serverUpdate('ContactCard', JMAP, id, { media: { p: { kind: 'photo', uri: `data:image/jpeg;base64,${big}` } } }); + await h.applyOk(h.planner.planDownload(h.card(id), await h.contact(rawId), h.ctx).ops); + expect(h.planner.planUpload(await h.contact(rawId), h.ctx)).toEqual({ + kind: 'upload', actions: [{ kind: 'update', id, patch: { 'phones/p1/number': '2' } }], + }); + }); + it('heals a baseline the provider stored differently from the prediction', async () => { const h = new Harness(); const { id, rawId } = await h.seed({ name: { full: 'Dr. Jane Q. Public' } }); diff --git a/src/device-sync/__tests__/contacts/merge.test.ts b/src/device-sync/__tests__/contacts/merge.test.ts index 2d07db5a..2879454b 100644 --- a/src/device-sync/__tests__/contacts/merge.test.ts +++ b/src/device-sync/__tests__/contacts/merge.test.ts @@ -1,7 +1,7 @@ import { describe, expect, it } from 'vitest'; import { Data, MimeType, RawContacts } from '../../android-columns'; import type { ContactCardWire } from '../../planner'; -import { appleCard, googleCard } from './fixtures'; +import { appleCard, googleCard, probeCard } from './fixtures'; import { Harness, JMAP } from './harness'; /** A seeded card, edited on the device (in place, like AOSP) and on the server. */ @@ -85,6 +85,16 @@ describe('contacts merge: a dirty contact meets a new server version', () => { expect(Object.keys(patch.phones as object)).toEqual(['k1', 'k2']); }); + it('still uploads a photo deleted on the device after merging a server edit of another entry', async () => { + const { h, rawId, plan } = await bothEdited( + probeCard, + (hh, raw) => hh.device.user.deleteData(hh.rowsByKey(raw)['media:ph']._id as number), + { 'emails/zz9/address': 'new@example.org' }, + ); + expect(plan).toMatchObject({ conflicts: 0, stillDirty: true }); + expect(await patchOf(h, rawId)).toEqual({ media: null }); + }); + it('matches an entry both sides added instead of duplicating it', async () => { const { h, rawId, plan } = await bothEdited( appleCard, diff --git a/src/device-sync/__tests__/contacts/real-engine.test.ts b/src/device-sync/__tests__/contacts/real-engine.test.ts index 758b5e3c..f6a160a8 100644 --- a/src/device-sync/__tests__/contacts/real-engine.test.ts +++ b/src/device-sync/__tests__/contacts/real-engine.test.ts @@ -6,7 +6,7 @@ import { describe, expect, it } from 'vitest'; import { Groups, MimeType } from '../../android-columns'; import { calendarPlanner } from '../../calendar/planner'; import { contactsPlanner } from '../../contacts/planner'; -import { addServerCards, CONTACTS_AUTHORITY, createHarness, type Harness } from '../engine/harness'; +import { addServerCards, CONTACTS_AUTHORITY, createHarness, rowWrites, type Harness } from '../engine/harness'; import type { SetTarget } from '../fakes/fake-jmap-server'; function real(options: Parameters[0] = {}): Harness { @@ -78,6 +78,28 @@ describe('contacts through the engine', () => { expect(await h.teardown(CONTACTS_AUTHORITY)).toEqual({ pending: 0 }); }); + it('syncs a contact whose photo is too large for a provider batch, without the photo', async () => { + const h = real(); + // About 900 KB of JPEG as a data: URI (a camera photo set over CardDAV). + const media = { p1: { '@type': 'Media', kind: 'photo', uri: `data:image/jpeg;base64,${'A'.repeat(1_200_000)}`, mediaType: 'image/jpeg' } }; + const card = h.server.addCard('a', { uid: 'u-photo', name: { full: 'Photo Person' }, emails: { e1: { address: 'photo@example.org' } }, addressBookIds: { [h.book]: true }, media }); + addServerCards(h, ['Plain Person']); + + expect(await h.run()).toMatchObject({ outcome: 'ok', itemErrors: [] }); + expect(h.contacts().map((c) => c.name).sort()).toEqual(['Photo Person', 'Plain Person']); + expect(h.state()?.accounts.a.stale).toEqual([]); + const before = h.batches.log.length; + expect((await h.run()).outcome).toBe('ok'); + expect(rowWrites(h.batches.log.slice(before))).toEqual([]); + + // Its device edits upload, and the photo stays on the server. + const id = h.contactNamed('Photo Person')!.id; + const email = h.device.rows('data').find((d) => Number(d.raw_contact_id) === id && d.mimetype === MimeType.EMAIL)!; + h.device.user.updateData(Number(email._id), { data1: 'photo2@example.org' }); + expect(await h.run()).toMatchObject({ outcome: 'ok', itemErrors: [], stats: { uploaded: { updated: 1 } } }); + expect(h.server.get('ContactCard', 'a', card)).toMatchObject({ emails: { e1: { address: 'photo2@example.org' } }, media }); + }); + it('keeps a group rename the server refused when a membership patch of that group goes through', async () => { const h = real(); const [ada] = addServerCards(h, ['Ada Lovelace']); diff --git a/src/device-sync/contacts/download.ts b/src/device-sync/contacts/download.ts index feacece2..0d8e41b3 100644 --- a/src/device-sync/contacts/download.ts +++ b/src/device-sync/contacts/download.ts @@ -22,7 +22,7 @@ import { localChanges, sameUnit } from './diff'; import { deletionsInferred, inPlaceEvidence, matchKind, reconcileKeys, type KindMatch } from './matching'; import { editableGroups, planMemberships, remoteMemberships, shadowMemberOf, withMemberOf, withoutMemberOf } from './members'; import { assertContact, deleteData, insertUnit, updateData, updateRawContact, updateUnit } from './ops'; -import { shadowOf } from './photo'; +import { isPhotoAbsent, shadowOf, withPhotoAbsent } from './photo'; import { KIND_ORDER, projectCard, rowMatchesUnit, SPECS, type Unit } from './project'; const ENTRY_KINDS = KIND_ORDER.filter((k) => k !== MimeType.GROUP_MEMBERSHIP); @@ -90,6 +90,8 @@ interface RowPlan { conflicts: number; stillDirty: boolean; evidence: boolean; + /** The card's photo is left without a row because it could not be written (the shadow's `~noPhoto`). */ + photoAbsent: boolean; } /** @@ -99,6 +101,7 @@ interface RowPlan { */ function cleanRows(local: LocalContact, remote: Unit[], keyMap: Map, base: Unit[], accepted: boolean): RowPlan { const ops: ProviderOp[] = []; + let photoAbsent = false; const rows = renameRows(local.rows, base, keyMap); const kinds = ENTRY_KINDS.map((k) => matchKind(k, rows, remote)); for (const kind of kinds) { @@ -130,11 +133,14 @@ function cleanRows(local: LocalContact, remote: Unit[], keyMap: Map matchKind(k, rows, renamedBase)); @@ -149,11 +156,14 @@ function mergeRows(local: LocalContact, base: Unit[], remote: Unit[], keyMap: Ma const mapped = new Set(keyMap.values()); for (const kind of kinds) { + const start = ops.length; const remoteOf = new Map(remote.filter((u) => u.mimetype === kind.mimetype).map((u) => [u.key, u])); const hasKeyed = kind.matched.some((m) => m.how === 'key'); const hasKeyless = kind.fresh.length > 0 || kind.matched.some((m) => m.how !== 'key'); const keyed = hasKeyed || !hasKeyless; - const deletions = kind.mimetype !== MimeType.STRUCTURED_NAME && deletionsInferred(kind, evidence); + // A photo the device never held (`~noPhoto`) was not deleted there. + const deletions = kind.mimetype !== MimeType.STRUCTURED_NAME && deletionsInferred(kind, evidence) + && !(kind.mimetype === MimeType.PHOTO && isPhotoAbsent(local.shadow)); const write = (row: LocalDataRow, unit: Unit) => { if (unit.mimetype === MimeType.PHOTO && !unit.extra) return; ops.push(updateUnit(row.id, unit, keyed)); @@ -209,8 +219,15 @@ function mergeRows(local: LocalContact, base: Unit[], remote: Unit[], keyMap: Ma } } if (fresh.length) stillDirty = true; + + if (kind.mimetype === MimeType.PHOTO) { + // The server's photo is left without a row it could not get, unless a local deletion of it waits for its upload. + const photo = [...remoteOf.values()][0]; + const deletionWaits = !!photo && deletions && kind.missing.some((u) => sameUnit(u, photo)); + photoAbsent = !!photo && !kind.rows.length && !ops.slice(start).some((o) => o.op === 'insert') && !deletionWaits; + } } - return { ops, conflicts, stillDirty, evidence }; + return { ops, conflicts, stillDirty, evidence, photoAbsent }; } export interface CleanWriteOptions { @@ -251,7 +268,7 @@ export function cleanWrite(local: LocalContact, card: ContactCardWire, ctx: Ctx, groupRowIdBySourceId: (g) => ctx.groupRowIdBySourceId(g), editable: editableGroups(ctx, accountOf(local, ctx)), }); - const shadow = withMemberOf(shadowOf(server), members.memberOf); + const shadow = withPhotoAbsent(withMemberOf(shadowOf(server), members.memberOf), rows.photoAbsent); const raw = changedRawColumns(local, { ...cardColumns(server, shadow, ctx), ...(options.raw ?? {}), @@ -282,7 +299,8 @@ function insertPlan(card: ContactCardWire, ctx: Ctx): DownloadPlan { parent: { ref: 0 }, groupRowIdBySourceId: (g) => ctx.groupRowIdBySourceId(g), }); - const shadow = withMemberOf(shadowOf(card), members.memberOf); + // A photo without bytes to write (too large, or not fetched) stays on the server only. + const shadow = withPhotoAbsent(withMemberOf(shadowOf(card), members.memberOf), units.some((u) => u.mimetype === MimeType.PHOTO && !u.extra)); const ops: ProviderOp[] = [ { op: 'insert', table: 'raw_contacts', values: { [RawContacts.SOURCE_ID]: identity, [RawContacts.DIRTY]: 0, ...cardColumns(card, shadow, ctx) } }, ...units.filter((u) => u.mimetype !== MimeType.PHOTO || u.extra).map((u) => insertUnit({ ref: 0 }, u, true)), @@ -349,7 +367,7 @@ export function planContactDownload(card: ContactCardWire, local: LocalContact | parent: local.rawContactId, groupRowIdBySourceId: (g) => ctx.groupRowIdBySourceId(g), }); - const shadow = withMemberOf(shadowOf(server), members.memberOf); + const shadow = withPhotoAbsent(withMemberOf(shadowOf(server), members.memberOf), rows.photoAbsent); const raw = changedRawColumns(local, cardColumns(server, shadow, ctx)); const ops: ProviderOp[] = []; if (Object.keys(raw).length) ops.push(updateRawContact(local.rawContactId, raw)); diff --git a/src/device-sync/contacts/members.ts b/src/device-sync/contacts/members.ts index 118f3cb7..3a331d1a 100644 --- a/src/device-sync/contacts/members.ts +++ b/src/device-sync/contacts/members.ts @@ -24,6 +24,7 @@ import type { ProviderOp } from '../types'; import { text } from './cells'; import { entryKey } from './keys'; import { deleteData, insertUnit, updateData } from './ops'; +import { NO_PHOTO } from './photo'; import { MEMBER_OF, type Unit } from './project'; /** @@ -55,10 +56,10 @@ export function shadowMemberOf(shadow: ContactCardWire | null): string[] | null return Array.isArray(v) && v.every((g) => typeof g === 'string') ? [...v] : null; } -/** A card without the `~memberOf` a shadow carries. */ +/** A card without the device-only members a shadow carries: `~memberOf`, and `~noPhoto` (photo.ts). */ export function withoutMemberOf(card: ContactCardWire): ContactCardWire { - if (!(MEMBER_OF in card)) return card; - const { [MEMBER_OF]: _drop, ...rest } = card; + if (!(MEMBER_OF in card) && !(NO_PHOTO in card)) return card; + const { [MEMBER_OF]: _drop, [NO_PHOTO]: _photo, ...rest } = card; return rest as ContactCardWire; } diff --git a/src/device-sync/contacts/photo.ts b/src/device-sync/contacts/photo.ts index 05eb171e..30ccac2e 100644 --- a/src/device-sync/contacts/photo.ts +++ b/src/device-sync/contacts/photo.ts @@ -11,6 +11,14 @@ import { clone } from '../common/json'; import type { ContactCardWire, ContactsContext } from '../planner'; const HASH_URI = 'sha256:'; + +/** + * The largest photo (base64 characters, about 512 KiB of JPEG) written to the + * device: one applyBatch is one Binder transaction of at most 1 MB, and a + * contact's rows travel in one op group, which is never split. A larger photo + * stays on the server only (docs/device-sync.md, "Limits and performance"). + */ +export const MAX_PHOTO_BASE64 = 700_000; const B64 = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/'; const B64_INDEX = new Map([...B64].map((c, i) => [c, i])); @@ -58,13 +66,33 @@ export function photoEntry(card: ContactCardWire): { key: string; entry: Contact return null; } -/** Bytes (base64) and hash of a photo entry; bytes are null for a shadow's hash-only entry. */ +/** + * Bytes (base64) and hash of a photo entry; bytes are null for a shadow's + * hash-only entry and for a photo too large to write (MAX_PHOTO_BASE64). + */ export function photoPayload(entry: ContactMedia, ctx: Pick): { b64: string | null; hash: string } | null { const uri = typeof entry.uri === 'string' ? entry.uri : ''; if (uri.startsWith(HASH_URI)) return { b64: null, hash: uri }; const b64 = dataUriBase64(uri) ?? ctx.photoBytes(entry as { uri?: string; blobId?: string; mediaType?: string }); const hash = b64 ? hashBase64(b64) : null; - return b64 && hash ? { b64, hash } : null; + if (!b64 || !hash) return null; + return { b64: b64.length > MAX_PHOTO_BASE64 ? null : b64, hash }; +} + +/** + * Set in a shadow whose card's photo has no row on the device because it + * could not be written (too large, or its bytes could not be fetched), so the + * missing row is no deletion. Device-only, like `~memberOf`: stripped before + * any projection or patch. + */ +export const NO_PHOTO = '~noPhoto'; + +export function isPhotoAbsent(shadow: ContactCardWire | null): boolean { + return shadow?.[NO_PHOTO] === true; +} + +export function withPhotoAbsent(shadow: ContactCardWire, absent: boolean): ContactCardWire { + return absent ? { ...shadow, [NO_PHOTO]: true } : shadow; } /** The shadow of a card: every `data:` photo URI replaced by the hash of its bytes. */ diff --git a/src/device-sync/contacts/upload.ts b/src/device-sync/contacts/upload.ts index dbd8b9a1..67542358 100644 --- a/src/device-sync/contacts/upload.ts +++ b/src/device-sync/contacts/upload.ts @@ -30,8 +30,8 @@ import { deletionsInferred, inPlaceEvidence, matchKind, type KindMatch } from '. import { shadowMemberOf, withMemberOf, withoutMemberOf } from './members'; import { NAME_COLUMNS, namePatch } from './name'; import { assertContact, deleteWhereId, updateRawContact, updateWhereId } from './ops'; -import { isHashUri, jpegDataUri, shadowOf } from './photo'; -import { KIND_ORDER, SPECS } from './project'; +import { isHashUri, isPhotoAbsent, jpegDataUri, photoEntry, shadowOf, withPhotoAbsent } from './photo'; +import { isEmptyRow, KIND_ORDER, SPECS } from './project'; type Ctx = ContactsContext; @@ -116,7 +116,8 @@ function photoChanges(local: LocalContact, kind: KindMatch, evidence: boolean, c } else { out.added.push({ map: 'media', value: { kind: 'photo', uri, mediaType: 'image/jpeg' } }); } - } else if (!row && deletionsInferred(kind, evidence)) { + } else if (!row && deletionsInferred(kind, evidence) && !isPhotoAbsent(local.shadow)) { + // (A photo the device could not be given was never there to delete.) for (const unit of kind.missing) out.removed.push(...unitEntries(kind.mimetype, unit.key)); } } @@ -286,7 +287,12 @@ export function planContactAccepted(local: LocalContact, server: ContactCardWire // Edited again during the upload: identity and shadow, and the baselines of // the uploaded units set to what was uploaded; rows and DIRTY stay. const changes = local.shadow ? collectChanges(local, withoutMemberOf(local.shadow), ctx) : null; - const shadow = withMemberOf(shadowOf(withoutMemberOf(server)), shadowMemberOf(local.shadow) ?? (local.shadow ? null : [])); + // The rows stay, so a photo the device could not be given is still not there. + const photoRow = local.rows.some((r) => r.mimetype === MimeType.PHOTO && !isEmptyRow(MimeType.PHOTO, r.cells)); + const shadow = withPhotoAbsent( + withMemberOf(shadowOf(withoutMemberOf(server)), shadowMemberOf(local.shadow) ?? (local.shadow ? null : [])), + isPhotoAbsent(local.shadow) && !photoRow && photoEntry(server) !== null, + ); const keep: ProviderOp[] = [ updateRawContact(local.rawContactId, { ...raw, [RawContacts.SYNC2]: JSON.stringify(shadow) }), ]; From 6f574de0b9625e67df79b2113e2126ea6e243250 Mon Sep 17 00:00:00 2001 From: Linus Rath <139418639+rathlinus@users.noreply.github.com> Date: Sun, 27 Sep 2026 07:18:33 +0200 Subject: [PATCH 34/87] fix: keep small contact photos on the device and on the server #34 ContactsProvider keeps a photo within its 96 px thumbnail as the thumbnail alone, without a PHOTO_FILE_ID, and the thumbnail does not read back. Such a Photo row looked empty to the planner. Every download of the card inserted the photo again (two, then three rows), and any edit of the contact on the device deleted the photo on the server. A Photo row now always counts as the contact's photo. Photo rows that earlier downloads added again are deleted by the contact's next clean write, one row is kept, and nothing reaches the server for them. A photo set on the device is still uploaded. Checked with planner tests and end to end through the engine, with a provider that stores small photos as thumbnails as the device does. --- src/device-sync/__tests__/contacts/harness.ts | 44 ++++++++++++- .../__tests__/contacts/lifecycle.test.ts | 61 +++++++++++++++++++ .../__tests__/contacts/real-engine.test.ts | 26 +++++++- src/device-sync/contacts/project.ts | 9 ++- 4 files changed, 134 insertions(+), 6 deletions(-) diff --git a/src/device-sync/__tests__/contacts/harness.ts b/src/device-sync/__tests__/contacts/harness.ts index 663dfbd0..27f1a243 100644 --- a/src/device-sync/__tests__/contacts/harness.ts +++ b/src/device-sync/__tests__/contacts/harness.ts @@ -4,7 +4,7 @@ * way the engine applies them (one op group per batch); uploads go through * the fake server's patch handling and come back through its normalisation. */ -import { GroupMembership, Groups, RawContacts } from '../../android-columns'; +import { Data, GroupMembership, Groups, RawContacts } from '../../android-columns'; import { makeKeyMinter, parseObjectRef, uuidFrom } from '../../common/ids'; import type { GroupRights } from '../../contacts/members'; import { contactsPlanner } from '../../contacts/planner'; @@ -17,7 +17,7 @@ import type { UploadAction, UploadPlan, } from '../../planner'; -import { CONTACTS_AUTHORITY, type BatchResult, type ProviderPort, type Row } from '../../types'; +import { CONTACTS_AUTHORITY, type BatchResult, type OpResult, type ProviderOp, type ProviderPort, type Row } from '../../types'; import { FakeJmapServer } from '../fakes/fake-jmap-server'; import { FakeDeviceProviders } from '../fakes/fake-provider'; @@ -35,6 +35,40 @@ function rowsOf(res: { columns: string[]; rows: unknown[][] }): Row[] { return res.rows.map((cells) => Object.fromEntries(res.columns.map((c, i) => [c, cells[i]])) as Row); } +/** + * ContactsProvider keeps a photo no larger than its 96 px thumbnail as the + * thumbnail only: PHOTO_FILE_ID (DATA14) stays NULL, and the thumbnail + * (DATA15) does not read back. The fake stores every photo with a display + * photo; this makes the photo rows a batch wrote thumbnail-only, as a device + * does with the test fixtures' 1×1 JPEGs. + */ +export function keepThumbnailsOnly(device: FakeDeviceProviders, ops: ProviderOp[], results: OpResult[]): void { + ops.forEach((op, i) => { + if ((op.op !== 'insert' && op.op !== 'update') || op.table !== 'data') return; + const bytes = op.values[Data.DATA15]; + if (!bytes || typeof bytes !== 'object') return; + const id = op.op === 'insert' ? results[i]?.id : op.id; + const row = id === undefined ? undefined : device.table('data').get(id); + if (row) row[Data.DATA14] = null; + }); +} + +/** A provider port whose photo writes come out thumbnail-only (see `keepThumbnailsOnly`). */ +export function thumbnailOnlyPort(device: FakeDeviceProviders, port: ProviderPort): ProviderPort { + return { + accountName: port.accountName, + authority: port.authority, + query: (q) => port.query(q), + readSyncState: () => port.readSyncState(), + readPhoto: (id, px) => port.readPhoto(id, px), + applyBatch: async (ops) => { + const res = await port.applyBatch(ops); + if (res.ok) keepThumbnailsOnly(device, ops, res.results); + return res; + }, + }; +} + export class Harness { readonly device = new FakeDeviceProviders(); readonly port: ProviderPort = this.device.port(ACCOUNT, CONTACTS_AUTHORITY); @@ -49,6 +83,8 @@ export class Harness { readonly unselected = new Set(); /** When false the context has no `groupsOf` (memberships not synced). */ groupIndex = true; + /** When true the photos the planner writes come out thumbnail-only (see `keepThumbnailsOnly`). */ + thumbnailPhotos = false; readonly ctx: ContactsContext & GroupRights; constructor() { @@ -98,7 +134,9 @@ export class Harness { async apply(group: OpGroup): Promise { if (!group.ops.length) return { ok: true, results: [] }; - return this.port.applyBatch(group.ops); + const res = await this.port.applyBatch(group.ops); + if (res.ok && this.thumbnailPhotos) keepThumbnailsOnly(this.device, group.ops, res.results); + return res; } async applyOk(group: OpGroup): Promise { diff --git a/src/device-sync/__tests__/contacts/lifecycle.test.ts b/src/device-sync/__tests__/contacts/lifecycle.test.ts index 9e69cd74..8d66eff4 100644 --- a/src/device-sync/__tests__/contacts/lifecycle.test.ts +++ b/src/device-sync/__tests__/contacts/lifecycle.test.ts @@ -181,6 +181,67 @@ describe('contacts lifecycle', () => { }); }); + describe('a small photo the provider keeps as its thumbnail only (no PHOTO_FILE_ID)', () => { + const photo = { ph: { kind: 'photo', uri: `data:image/jpeg;base64,${JPEG}`, mediaType: 'image/jpeg' } }; + const small = () => ({ name: { full: 'Photo Smalltest' }, emails: { e1: { address: 'small@example.org' } }, notes: { n1: { note: 'v1' } }, media: photo }); + const photos = (h: Harness, rawId: number) => h.device.rows('data', 'raw_contact_id = ? AND mimetype = ?', [rawId, MimeType.PHOTO]); + + it('keeps one row through downloads and merges, and never reads it as deleted', async () => { + const h = new Harness(); + h.thumbnailPhotos = true; + const { id, rawId } = await h.seed(small()); + expect(photos(h, rawId)).toMatchObject([{ data14: null, data_sync1: 'media:ph' }]); + await expectSettled(h, id, rawId); + for (const note of ['v2', 'v3']) { + h.server.serverUpdate('ContactCard', JMAP, id, { 'notes/n1/note': note }); + await h.download(id); + expect(photos(h, rawId)).toHaveLength(1); + } + // A merge into an edited contact. + h.device.user.updateData(h.rowsByKey(rawId)['emails:e1']._id as number, { data1: 'small2@example.org' }); + h.server.serverUpdate('ContactCard', JMAP, id, { 'notes/n1/note': 'v4' }); + await h.applyOk(h.planner.planDownload(h.card(id), await h.contact(rawId), h.ctx).ops); + expect(photos(h, rawId)).toHaveLength(1); + // An edit of another field uploads alone. + expect(h.planner.planUpload(await h.contact(rawId), h.ctx)).toEqual({ + kind: 'upload', actions: [{ kind: 'update', id, patch: { 'emails/e1/address': 'small2@example.org' } }], + }); + await h.upload(rawId); + expect(h.card(id).media).toEqual(photo); + expect(photos(h, rawId)).toHaveLength(1); + await expectSettled(h, id, rawId); + // A photo set on the device (it gets a display photo) replaces it. + h.device.user.setPhoto(rawId, JPEG2); + h.devicePhotos.set(rawId, JPEG2); + expect(h.planner.planUpload(await h.contact(rawId), h.ctx)).toEqual({ + kind: 'upload', actions: [{ kind: 'update', id, patch: { 'media/ph/uri': `data:image/jpeg;base64,${JPEG2}`, 'media/ph/mediaType': 'image/jpeg' } }], + }); + }); + + it('heals the photo rows the same photo was inserted as again, keeping one and the server photo', async () => { + const h = new Harness(); + h.thumbnailPhotos = true; + const { id, rawId } = await h.seed(small()); + const [row] = photos(h, rawId); + const again = { [Data.MIMETYPE]: MimeType.PHOTO, [Data.DATA15]: { b64: JPEG }, data_sync1: row.data_sync1, data_sync2: row.data_sync2, data_sync3: row.data_sync3 }; + for (let i = 0; i < 2; i++) await h.applyOk({ ref: 'again', ops: [{ op: 'insert', table: 'data', values: { ...again, [Data.RAW_CONTACT_ID]: rawId } }] }); + expect(photos(h, rawId)).toHaveLength(3); + // An edit on the device uploads without touching the photo, and its accepted write keeps one row. + h.device.user.updateData(h.rowsByKey(rawId)['notes:n1']._id as number, { data1: 'v2 edited on device' }); + expect(h.planner.planUpload(await h.contact(rawId), h.ctx)).toEqual({ + kind: 'upload', actions: [{ kind: 'update', id, patch: { 'notes/n1/note': 'v2 edited on device' } }], + }); + await h.upload(rawId); + expect(photos(h, rawId)).toHaveLength(1); + expect(h.card(id).media).toEqual(photo); + await expectSettled(h, id, rawId); + // An unchanged card downloaded again (a full reconcile) heals too. + for (let i = 0; i < 2; i++) await h.applyOk({ ref: 'again', ops: [{ op: 'insert', table: 'data', values: { ...again, [Data.RAW_CONTACT_ID]: rawId } }] }); + await h.applyOk(h.planner.planDownload(h.card(id), await h.contact(rawId), h.ctx).ops); + expect(photos(h, rawId)).toHaveLength(1); + }); + }); + it('heals a baseline the provider stored differently from the prediction', async () => { const h = new Harness(); const { id, rawId } = await h.seed({ name: { full: 'Dr. Jane Q. Public' } }); diff --git a/src/device-sync/__tests__/contacts/real-engine.test.ts b/src/device-sync/__tests__/contacts/real-engine.test.ts index f6a160a8..df779301 100644 --- a/src/device-sync/__tests__/contacts/real-engine.test.ts +++ b/src/device-sync/__tests__/contacts/real-engine.test.ts @@ -3,11 +3,13 @@ // themselves are tested in the other files of this folder. import { describe, expect, it } from 'vitest'; -import { Groups, MimeType } from '../../android-columns'; +import { Data, Groups, MimeType } from '../../android-columns'; import { calendarPlanner } from '../../calendar/planner'; import { contactsPlanner } from '../../contacts/planner'; import { addServerCards, CONTACTS_AUTHORITY, createHarness, rowWrites, type Harness } from '../engine/harness'; import type { SetTarget } from '../fakes/fake-jmap-server'; +import { JPEG } from './fixtures'; +import { thumbnailOnlyPort } from './harness'; function real(options: Parameters[0] = {}): Harness { const h = createHarness(options); @@ -78,6 +80,28 @@ describe('contacts through the engine', () => { expect(await h.teardown(CONTACTS_AUTHORITY)).toEqual({ pending: 0 }); }); + it('keeps one row for a small photo the provider stores as its thumbnail, and never deletes it on the server', async () => { + const h = real(); + const base = h.deps.provider; + h.deps.provider = (name, authority) => thumbnailOnlyPort(h.device, base(name, authority)); + const media = { ph: { kind: 'photo', uri: `data:image/jpeg;base64,${JPEG}`, mediaType: 'image/jpeg' } }; + const card = h.server.addCard('a', { uid: 'u-small', name: { full: 'Photo Smalltest' }, notes: { n1: { note: 'v1' } }, addressBookIds: { [h.book]: true }, media }); + expect((await h.run()).outcome).toBe('ok'); + const id = h.contactNamed('Photo Smalltest')!.id; + const rowsOf = (mimetype: string) => h.device.rows('data').filter((d) => Number(d.raw_contact_id) === id && d.mimetype === mimetype); + expect(rowsOf(MimeType.PHOTO)).toMatchObject([{ data14: null }]); + + for (const note of ['v2', 'v3']) { + h.server.serverUpdate('ContactCard', 'a', card, { 'notes/n1/note': note }); + expect((await h.run()).outcome).toBe('ok'); + expect(rowsOf(MimeType.PHOTO)).toHaveLength(1); + } + h.device.user.updateData(Number(rowsOf(MimeType.NOTE)[0]._id), { [Data.DATA1]: 'v4 edited on device' }); + expect(await h.run()).toMatchObject({ outcome: 'ok', itemErrors: [], stats: { uploaded: { updated: 1 } } }); + expect(h.server.get('ContactCard', 'a', card)).toMatchObject({ notes: { n1: { note: 'v4 edited on device' } }, media }); + expect(rowsOf(MimeType.PHOTO)).toHaveLength(1); + }); + it('syncs a contact whose photo is too large for a provider batch, without the photo', async () => { const h = real(); // About 900 KB of JPEG as a data: URI (a camera photo set over CardDAV). diff --git a/src/device-sync/contacts/project.ts b/src/device-sync/contacts/project.ts index b548fd13..1ff0a655 100644 --- a/src/device-sync/contacts/project.ts +++ b/src/device-sync/contacts/project.ts @@ -318,11 +318,16 @@ export function rowMatchesUnit(u: Pick, stored: Row, return spec.columns.every((c) => same(u.cells[c], stored[c], c)); } -/** Whether a row has nothing in it (an editor's blank nickname or note). */ +/** + * Whether a row has nothing in it (an editor's blank nickname or note). A + * photo row always holds a photo: ContactsProvider keeps one within its 96 px + * thumbnail as the thumbnail alone, without PHOTO_FILE_ID, and the thumbnail + * (DATA15, a blob) does not read back. + */ export function isEmptyRow(mimetype: string, cells: Row): boolean { const spec = SPECS[mimetype]; if (!spec) return true; - if (mimetype === MimeType.PHOTO) return cellText(cells[Data.DATA14]) === null; + if (mimetype === MimeType.PHOTO) return false; if (mimetype === MimeType.GROUP_MEMBERSHIP) return cellText(cells[GroupMembership.GROUP_SOURCE_ID]) === null && cellText(cells[Data.DATA1]) === null; return spec.valueColumns.every((c) => text(cells[c]) === null); } From 534cb8239eb33804ba88218d75c82eb8be7c787e Mon Sep 17 00:00:00 2001 From: Linus Rath <139418639+rathlinus@users.noreply.github.com> Date: Sun, 27 Sep 2026 07:19:16 +0200 Subject: [PATCH 35/87] fix: keep the groups of a contact created on the device #34 A contact created on the device and put into a group before its first sync lost that group: the write after its create took the rows' own memberships as what the server already had, found them missing there and deleted the rows, so the group card was never patched. The create's write now starts from no memberships, as its fallback already did: the device's memberships count as additions, stay on the device, keep the contact dirty until the group cards show them, and upload as patches of the group cards. Checked with a planner test and end to end through the engine. --- .../__tests__/contacts/groups.test.ts | 24 ++++++++++++++++++- .../__tests__/contacts/real-engine.test.ts | 17 ++++++++++++- src/device-sync/contacts/download.ts | 4 +++- src/device-sync/contacts/upload.ts | 4 +++- 4 files changed, 45 insertions(+), 4 deletions(-) diff --git a/src/device-sync/__tests__/contacts/groups.test.ts b/src/device-sync/__tests__/contacts/groups.test.ts index d9ac7869..786271b9 100644 --- a/src/device-sync/__tests__/contacts/groups.test.ts +++ b/src/device-sync/__tests__/contacts/groups.test.ts @@ -2,7 +2,7 @@ import { describe, expect, it } from 'vitest'; import { Data, Groups, MimeType } from '../../android-columns'; import type { ContactCardWire, OpGroup, UploadPlan } from '../../planner'; import { appleCard, googleCard } from './fixtures'; -import { Harness, JMAP } from './harness'; +import { ACCOUNT, Harness, JMAP } from './harness'; function actionsOf(plan: UploadPlan) { if (plan.kind !== 'upload') throw new Error(`expected an upload, got ${plan.kind}`); @@ -130,6 +130,28 @@ describe('contacts groups: membership edits', () => { }); }); + it('keeps the groups of a contact created on the device and adds it to their cards', async () => { + const h = new Harness(); + const gid = h.addCard({ uid: 'urn:uuid:g7', kind: 'group', name: { full: 'Team' }, members: { 'urn:uuid:someone': true } }); + await h.download(gid); + const g = await localGroup(h, gid); + const rawId = h.device.user.insertContact(ACCOUNT, [ + { [Data.MIMETYPE]: MimeType.STRUCTURED_NAME, data1: 'New Person', data2: 'New', data3: 'Person' }, + { [Data.MIMETYPE]: MimeType.GROUP_MEMBERSHIP, [Data.DATA1]: g.groupId }, + ]); + await h.upload(rawId); + expect(h.membershipGroups(rawId)).toEqual([`${JMAP}/${gid}`]); + expect(h.dirty(rawId)).toBe(true); + const uid = (await h.contact(rawId)).shadow!.uid as string; + const actions = h.planner.planMembershipUploads([await h.contact(rawId)], await h.groups(), h.ctx); + expect(actions).toEqual([{ kind: 'update', id: gid, patch: { [`members/${uid}`]: true } }]); + h.send(actions); + await h.applyOk(h.planner.planGroupAccepted(g, h.card(gid), h.ctx).ops); + await h.upload(rawId); + expect(h.dirty(rawId)).toBe(false); + expect(h.membershipGroups(rawId)).toEqual([`${JMAP}/${gid}`]); + }); + it('sends the whole map for the first member of an empty group', async () => { const h = new Harness(); const empty = h.addCard({ uid: 'urn:uuid:g3', kind: 'group', name: { full: 'Empty' } }); diff --git a/src/device-sync/__tests__/contacts/real-engine.test.ts b/src/device-sync/__tests__/contacts/real-engine.test.ts index df779301..2edfebe5 100644 --- a/src/device-sync/__tests__/contacts/real-engine.test.ts +++ b/src/device-sync/__tests__/contacts/real-engine.test.ts @@ -6,7 +6,7 @@ import { describe, expect, it } from 'vitest'; import { Data, Groups, MimeType } from '../../android-columns'; import { calendarPlanner } from '../../calendar/planner'; import { contactsPlanner } from '../../contacts/planner'; -import { addServerCards, CONTACTS_AUTHORITY, createHarness, rowWrites, type Harness } from '../engine/harness'; +import { addServerCards, ANDROID_ACCOUNT, CONTACTS_AUTHORITY, createHarness, rowWrites, type Harness } from '../engine/harness'; import type { SetTarget } from '../fakes/fake-jmap-server'; import { JPEG } from './fixtures'; import { thumbnailOnlyPort } from './harness'; @@ -80,6 +80,21 @@ describe('contacts through the engine', () => { expect(await h.teardown(CONTACTS_AUTHORITY)).toEqual({ pending: 0 }); }); + it('puts a contact created on the device into the group it was added to', async () => { + const h = real(); + const group = h.server.addCard('a', { uid: 'g1', kind: 'group', name: { full: 'Friends' }, members: { other: true }, addressBookIds: { [h.book]: true } }); + expect((await h.run()).outcome).toBe('ok'); + const id = h.device.user.insertContact(ANDROID_ACCOUNT, [ + { [Data.MIMETYPE]: MimeType.STRUCTURED_NAME, [Data.DATA1]: 'New Person' }, + { [Data.MIMETYPE]: MimeType.GROUP_MEMBERSHIP, [Data.DATA1]: Number(h.device.rows('groups')[0]._id) }, + ]); + + expect(await h.run()).toMatchObject({ outcome: 'ok', itemErrors: [], stats: { uploaded: { created: 1, updated: 1 } } }); + const uid = h.server.all('ContactCard', 'a').find((c) => c.kind !== 'group')!.uid as string; + expect(h.server.get('ContactCard', 'a', group)?.members).toEqual({ other: true, [uid]: true }); + expect(memberships(h, id)).toHaveLength(1); + }); + it('keeps one row for a small photo the provider stores as its thumbnail, and never deletes it on the server', async () => { const h = real(); const base = h.deps.provider; diff --git a/src/device-sync/contacts/download.ts b/src/device-sync/contacts/download.ts index 0d8e41b3..1e1575e7 100644 --- a/src/device-sync/contacts/download.ts +++ b/src/device-sync/contacts/download.ts @@ -239,6 +239,8 @@ export interface CleanWriteOptions { raw?: WriteRow; /** The rows' changes were just uploaded (planAccepted): a changed photo is the server's now. */ accepted?: boolean; + /** The memberships the rows held at the last write, when there is no shadow to say: a new contact's are none. */ + memberBase?: string[]; } /** The JMAP account a local contact belongs to. */ @@ -260,7 +262,7 @@ export function cleanWrite(local: LocalContact, card: ContactCardWire, ctx: Ctx, const rows = cleanRows(local, remote, keyMap, base, options.accepted ?? false); const members = planMemberships({ rows: local.rows, - base: shadowMemberOf(local.shadow), + base: shadowMemberOf(local.shadow) ?? options.memberBase ?? null, remote: remoteMemberships(server, ctx), evidence: rows.evidence, restore: true, diff --git a/src/device-sync/contacts/upload.ts b/src/device-sync/contacts/upload.ts index 67542358..3c1e7c1c 100644 --- a/src/device-sync/contacts/upload.ts +++ b/src/device-sync/contacts/upload.ts @@ -280,7 +280,9 @@ export function planContactAccepted(local: LocalContact, server: ContactCardWire } const identity = objectRef(accountOf(local, ctx), server.id); const raw = { [RawContacts.SOURCE_ID]: identity, [RawContacts.SYNC3]: null, [RawContacts.SYNC4]: null }; - const write = cleanWrite(local, server, ctx, { assertDirty: null, clearDirty: true, accepted: true, raw }); + // Just created: the groups it was put in on the device are all still to upload. + const memberBase = local.shadow ? undefined : []; + const write = cleanWrite(local, server, ctx, { assertDirty: null, clearDirty: true, accepted: true, raw, memberBase }); const ops = group(local, write.ops.length ? write.ops : [assertContact(local, null)]); ops.ref = identity; From c635ccd586913dc9055aab535526ab4b19b2bb8b Mon Sep 17 00:00:00 2001 From: Linus Rath <139418639+rathlinus@users.noreply.github.com> Date: Sun, 27 Sep 2026 07:49:53 +0200 Subject: [PATCH 36/87] fix: open the account's sync page from the sync settings #34 "Android account settings" opened Android's page for the Bulwark account through `ACCOUNT_SYNC_SETTINGS` with the account as the `account` extra. The Settings app of Android 12L and later passes only `:settings:show_fragment_args` on to that page, which then found no account and closed at once ("Account provided does not exist: null"). Since starting it succeeded, the fallback to the list of synced accounts never ran, and the button seemed to do nothing. The account now also goes in the fragment arguments, where those Settings apps read it; older ones still read the plain extra. Checked on the API 32 and API 34 emulators: the button opens the account's "Account sync" page with its Contacts and Calendar switches. --- .../bulwarkmobile/sync/BulwarkDeviceSyncModule.kt | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/android/app/src/main/java/com/anonymous/bulwarkmobile/sync/BulwarkDeviceSyncModule.kt b/android/app/src/main/java/com/anonymous/bulwarkmobile/sync/BulwarkDeviceSyncModule.kt index 34b95d3f..230141db 100644 --- a/android/app/src/main/java/com/anonymous/bulwarkmobile/sync/BulwarkDeviceSyncModule.kt +++ b/android/app/src/main/java/com/anonymous/bulwarkmobile/sync/BulwarkDeviceSyncModule.kt @@ -7,6 +7,7 @@ import android.content.ActivityNotFoundException import android.content.ContentResolver import android.content.Intent import android.os.Build +import android.os.Bundle import android.os.Handler import android.os.Looper import android.provider.Settings @@ -162,7 +163,12 @@ class BulwarkDeviceSyncModule(reactContext: ReactApplicationContext) : @ReactMethod fun openAccountSettings(name: String, promise: Promise) = settle(promise) { val account = DeviceSyncAccounts.account(reactApplicationContext, name) - val perAccount = Intent(ACTION_ACCOUNT_SYNC_SETTINGS).putExtra(EXTRA_ACCOUNT, account) + val perAccount = Intent(ACTION_ACCOUNT_SYNC_SETTINGS) + .putExtra(EXTRA_ACCOUNT, account) + // Newer Settings apps pass only these arguments on to the account's + // screen; without them it closes at once ("Account provided does + // not exist: null", Android 12L), and no exception tells us so. + .putExtra(EXTRA_SHOW_FRAGMENT_ARGUMENTS, Bundle().apply { putParcelable(EXTRA_ACCOUNT, account) }) val overview = Intent(Settings.ACTION_SYNC_SETTINGS) .putExtra(Settings.EXTRA_ACCOUNT_TYPES, arrayOf(account.type)) val activity = reactApplicationContext.currentActivity @@ -309,6 +315,8 @@ class BulwarkDeviceSyncModule(reactContext: ReactApplicationContext) : /** Settings' per-account sync screen; public on AOSP-based settings apps, but not in the SDK. */ private const val ACTION_ACCOUNT_SYNC_SETTINGS = "android.settings.ACCOUNT_SYNC_SETTINGS" private const val EXTRA_ACCOUNT = "account" + /** SettingsActivity.EXTRA_SHOW_FRAGMENT_ARGUMENTS: the arguments of the screen it shows. */ + private const val EXTRA_SHOW_FRAGMENT_ARGUMENTS = ":settings:show_fragment_args" @Volatile private var currentInstance: BulwarkDeviceSyncModule? = null From 78684e2492162c71d51378db7923c8595509712c Mon Sep 17 00:00:00 2001 From: Linus Rath <139418639+rathlinus@users.noreply.github.com> Date: Sun, 27 Sep 2026 07:52:12 +0200 Subject: [PATCH 37/87] fix: keep the last sync time current while the sync settings stay open #34 --- .../settings/device-sync/DeviceSyncAccountRow.tsx | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/src/components/settings/device-sync/DeviceSyncAccountRow.tsx b/src/components/settings/device-sync/DeviceSyncAccountRow.tsx index a2b35c30..e83b9910 100644 --- a/src/components/settings/device-sync/DeviceSyncAccountRow.tsx +++ b/src/components/settings/device-sync/DeviceSyncAccountRow.tsx @@ -78,6 +78,9 @@ function askReminderOwner(t: TranslateFn): Promise { }); } +/** How often the status line's "… ago" is brought up to date. */ +const RELATIVE_TIME_REFRESH_MS = 30_000; + export function DeviceSyncAccountRow({ account, authority }: Props) { const c = useColors(); const styles = React.useMemo(() => makeStyles(c), [c]); @@ -91,6 +94,12 @@ export function DeviceSyncAccountRow({ account, authority }: Props) { const onInApp = syncOnInApp(entry, authority); const { view, permitted, refresh } = useAndroidSync(entry.androidAccountName, authority); const [busy, setBusy] = React.useState(null); + // "Synced 2 min ago" keeps counting while the settings stay open. + const [now, setNow] = React.useState(() => Date.now()); + React.useEffect(() => { + const timer = setInterval(() => setNow(Date.now()), RELATIVE_TIME_REFRESH_MS); + return () => clearInterval(timer); + }, []); const [prompt, setPrompt] = React.useState(null); const [chooserOpen, setChooserOpen] = React.useState(false); @@ -117,7 +126,7 @@ export function DeviceSyncAccountRow({ account, authority }: Props) { permitted, syncing: !!view?.active, lastRun: entry.lastRun[authority], - now: Date.now(), + now, }, t, ); From 7dbbd1ce25d36ad1cc5266785d8208c8b6026b9b Mon Sep 17 00:00:00 2001 From: Linus Rath <139418639+rathlinus@users.noreply.github.com> Date: Sun, 27 Sep 2026 07:40:50 +0200 Subject: [PATCH 38/87] fix: move events between calendars without recreating them on the server #34 Etar moves an event to another calendar, and turns a series into a single event, by deleting the row and inserting a copy. The engine was meant to pair the two rows and upload a patch of the existing event, but it claimed every new row (a uid and a pending create) before it asked the planner for pairs, and the planner pairs only unclaimed rows. No pair ever formed: the event was destroyed and created again under a new uid, which lost what the device cannot show (keywords, virtual locations, links, the other attendees' answers, alerts relative to the end) and sent every attendee a cancellation and a new invitation. Pairs now upload before new rows are claimed, and a paired deletion no longer counts toward the deletion threshold, since it is a move. When an app changes the rows while the patch is on its way, they are read and paired again; rows that no longer pair are left without the identity instead of having the server's version written into the new row, which overwrote the edit and left two rows with one identity. Checked with the real planners: a move and a series turned single keep their uid, keywords and attendee answers and send nothing; an edit made during the upload is kept; an unchanged rewrite still pairs. --- .../__tests__/engine/real-planners.test.ts | 180 +++++++++++++++++- src/device-sync/engine/calendar-sync.ts | 96 +++++++--- src/device-sync/engine/item-sync.ts | 20 +- 3 files changed, 262 insertions(+), 34 deletions(-) diff --git a/src/device-sync/__tests__/engine/real-planners.test.ts b/src/device-sync/__tests__/engine/real-planners.test.ts index 7c3c00c6..d44e3e06 100644 --- a/src/device-sync/__tests__/engine/real-planners.test.ts +++ b/src/device-sync/__tests__/engine/real-planners.test.ts @@ -3,9 +3,10 @@ // the planners' own tests; these check what the engine guarantees. import { describe, expect, it } from 'vitest'; -import { Data, MimeType } from '../../android-columns'; +import { Attendees, Data, Events, MimeType } from '../../android-columns'; import { calendarPlanner } from '../../calendar/planner'; import { contactsPlanner } from '../../contacts/planner'; +import type { Row } from '../../types'; import { addDeviceContact, addServerCards, @@ -21,6 +22,36 @@ function real(options: Parameters[0] = {}): Harness { return h; } +/** The event columns Etar copies into the row it inserts when it moves an event or ends its recurrence. */ +const ETAR_COPY = [ + Events.TITLE, Events.DESCRIPTION, Events.EVENT_LOCATION, Events.STATUS, Events.AVAILABILITY, Events.ACCESS_LEVEL, + Events.EVENT_COLOR, Events.DTSTART, Events.DTEND, Events.DURATION, Events.EVENT_TIMEZONE, Events.ALL_DAY, Events.RRULE, +]; + +/** Etar's move or "series to single event": the row is deleted and a copy (with its attendees) inserted. */ +function etarReinsert(h: Harness, old: Row, calendarRowId: number, changes: Row = {}): number { + const values: Row = {}; + for (const c of ETAR_COPY) if (old[c] !== undefined) values[c] = old[c]; + const attendees = h.device.rows('attendees') + .filter((a) => Number(a.event_id) === Number(old._id)) + .map((a) => ({ + [Attendees.ATTENDEE_EMAIL]: a[Attendees.ATTENDEE_EMAIL], + [Attendees.ATTENDEE_NAME]: a[Attendees.ATTENDEE_NAME], + [Attendees.ATTENDEE_RELATIONSHIP]: a[Attendees.ATTENDEE_RELATIONSHIP], + [Attendees.ATTENDEE_TYPE]: a[Attendees.ATTENDEE_TYPE], + [Attendees.ATTENDEE_STATUS]: a[Attendees.ATTENDEE_STATUS], + })); + h.device.user.deleteEvent(Number(old._id)); + return h.device.user.insertEvent(calendarRowId, { ...values, ...changes }, { attendees }); +} + +/** Masters (no ORIGINAL_*) that share a `_SYNC_ID`: a split the next run would resolve by destroy + create. */ +function sharedIdentities(h: Harness): string[] { + const masters = h.events().filter((e) => e[Events.ORIGINAL_ID] === null && e[Events.ORIGINAL_SYNC_ID] === null && e._sync_id); + const ids = masters.map((e) => String(e._sync_id)); + return ids.filter((id, i) => ids.indexOf(id) !== i); +} + /** A device edit of the display name only, as AOSP Contacts makes it (the provider fills the components). */ function retypeName(h: Harness, rawContactId: number, name: string): void { const row = h.nameDataRow(rawContactId)!; @@ -192,6 +223,153 @@ describe('device sync engine with the real planners', () => { expect(h.events().every((e) => Number(e.dirty ?? 0) === 0)).toBe(true); }); + it("uploads Etar's move to another calendar as a patch of the same event, keeping what the device can't show", async () => { + const h = real(); + const work = h.server.addCalendar('a', { name: 'Work' }); + const lunch = h.server.addEvent('a', { + uid: 'lunch-uid', + title: 'Lunch', + start: '2026-10-06T12:00:00', + duration: 'PT1H', + timeZone: 'Europe/Berlin', + keywords: { important: true }, + virtualLocations: { v1: { '@type': 'VirtualLocation', uri: 'https://meet.example.com/lunch' } }, + organizerCalendarAddress: 'mailto:alice@example.com', + participants: { + me: { '@type': 'Participant', calendarAddress: 'mailto:alice@example.com', roles: { owner: true, attendee: true }, participationStatus: 'accepted' }, + bob: { '@type': 'Participant', calendarAddress: 'mailto:bob@example.com', name: 'Bob', roles: { attendee: true }, participationStatus: 'accepted' }, + }, + calendarIds: { [h.calendar]: true }, + }); + await h.run(CALENDAR_AUTHORITY); + const old = h.events().find((e) => e._sync_id === `a/${lunch}`)!; + const workRow = Number(h.device.rows('calendars').find((c) => c._sync_id === `a/${work}`)!._id); + + const moved = etarReinsert(h, old, workRow); + const report = await h.run(CALENDAR_AUTHORITY); + + expect(report).toMatchObject({ outcome: 'ok', itemErrors: [], stats: { uploaded: { created: 0, updated: 1, deleted: 0 } } }); + expect(h.server.all('CalendarEvent', 'a')).toHaveLength(1); + const event = h.server.get('CalendarEvent', 'a', lunch)!; + expect(event.calendarIds).toEqual({ [work]: true }); + expect(event).toMatchObject({ + uid: 'lunch-uid', + keywords: { important: true }, + virtualLocations: { v1: { uri: 'https://meet.example.com/lunch' } }, + participants: { bob: { participationStatus: 'accepted' } }, + }); + // A move tells nobody. + expect(h.server.scheduling).toEqual([]); + expect(h.events()).toEqual([expect.objectContaining({ _id: moved, _sync_id: `a/${lunch}`, uid2445: 'lunch-uid', calendar_id: workRow, dirty: 0 })]); + + const before = h.batches.log.length; + expect((await h.run(CALENDAR_AUTHORITY)).outcome).toBe('ok'); + expect(rowWrites(h.batches.log.slice(before))).toEqual([]); + }); + + it('uploads a series Etar turned into a single event as the removal of its rule', async () => { + const h = real(); + const series = h.server.addEvent('a', { + uid: 'series-uid', + title: 'Standup', + start: '2026-09-28T09:00:00', + duration: 'PT15M', + timeZone: 'Europe/Berlin', + keywords: { team: true }, + recurrenceRule: { '@type': 'RecurrenceRule', frequency: 'daily' }, + calendarIds: { [h.calendar]: true }, + }); + await h.run(CALENDAR_AUTHORITY); + const old = h.events().find((e) => e._sync_id === `a/${series}`)!; + + const single = etarReinsert(h, old, Number(old.calendar_id), { + [Events.RRULE]: null, + [Events.DURATION]: null, + [Events.DTEND]: Number(old[Events.DTSTART]) + 15 * 60_000, + }); + const report = await h.run(CALENDAR_AUTHORITY); + + expect(report).toMatchObject({ outcome: 'ok', itemErrors: [], stats: { uploaded: { created: 0, updated: 1, deleted: 0 } } }); + expect(h.server.all('CalendarEvent', 'a')).toHaveLength(1); + const event = h.server.get('CalendarEvent', 'a', series)!; + expect(event).toMatchObject({ uid: 'series-uid', title: 'Standup', keywords: { team: true } }); + expect(event.recurrenceRule).toBeUndefined(); + expect(h.events()).toEqual([expect.objectContaining({ _id: single, _sync_id: `a/${series}`, rrule: null, dirty: 0 })]); + }); + + it('never leaves two rows with one identity when the moved event is edited while its move uploads', async () => { + const h = real(); + const work = h.server.addCalendar('a', { name: 'Work' }); + const lunch = h.server.addEvent('a', { + uid: 'lunch-uid', + title: 'Lunch', + start: '2026-10-06T12:00:00', + duration: 'PT1H', + timeZone: 'Europe/Berlin', + calendarIds: { [h.calendar]: true }, + }); + await h.run(CALENDAR_AUTHORITY); + const old = h.events().find((e) => e._sync_id === `a/${lunch}`)!; + const workRow = Number(h.device.rows('calendars').find((c) => c._sync_id === `a/${work}`)!._id); + const moved = etarReinsert(h, old, workRow); + let edited = false; + const stop = h.server.onAfterRequest((request) => { + if (edited || !request.methods.includes('CalendarEvent/set')) return; + edited = true; + h.device.user.updateEvent(moved, { [Events.TITLE]: 'Team lunch' }); + }); + + await h.run(CALENDAR_AUTHORITY); + stop(); + + expect(edited).toBe(true); + expect(sharedIdentities(h)).toEqual([]); + expect(h.events().find((e) => Number(e._id) === moved)).toMatchObject({ title: 'Team lunch' }); + + expect((await h.run(CALENDAR_AUTHORITY)).outcome).toBe('ok'); + expect(sharedIdentities(h)).toEqual([]); + const server = h.server.all('CalendarEvent', 'a'); + expect(server.map((e) => [e.title, e.calendarIds])).toEqual([['Team lunch', { [work]: true }]]); + expect(h.events()).toEqual([expect.objectContaining({ _id: moved, _sync_id: `a/${server[0].id}`, dirty: 0 })]); + }); + + it('pairs the rows again when an app rewrote the moved event unchanged while its move uploaded', async () => { + const h = real(); + const work = h.server.addCalendar('a', { name: 'Work' }); + const lunch = h.server.addEvent('a', { + uid: 'lunch-uid', + title: 'Lunch', + start: '2026-10-06T12:00:00', + duration: 'PT1H', + timeZone: 'Europe/Berlin', + keywords: { important: true }, + organizerCalendarAddress: 'mailto:alice@example.com', + participants: { + me: { '@type': 'Participant', calendarAddress: 'mailto:alice@example.com', roles: { owner: true, attendee: true }, participationStatus: 'accepted' }, + bob: { '@type': 'Participant', calendarAddress: 'mailto:bob@example.com', name: 'Bob', roles: { attendee: true }, participationStatus: 'accepted' }, + }, + calendarIds: { [h.calendar]: true }, + }); + await h.run(CALENDAR_AUTHORITY); + const old = h.events().find((e) => e._sync_id === `a/${lunch}`)!; + const workRow = Number(h.device.rows('calendars').find((c) => c._sync_id === `a/${work}`)!._id); + const moved = etarReinsert(h, old, workRow); + const stop = h.server.onAfterRequest((request) => { + if (!request.methods.includes('CalendarEvent/set')) return; + stop(); + // Fossify's save: the same attendees, deleted and inserted again. + const attendees = h.device.rows('attendees').filter((a) => Number(a.event_id) === moved).map(({ _id, event_id, ...a }) => a); + h.device.user.fossifySaveEvent(moved, {}, attendees, []); + }); + + const report = await h.run(CALENDAR_AUTHORITY); + + expect(report).toMatchObject({ outcome: 'ok', itemErrors: [], stats: { uploaded: { created: 0, updated: 1, deleted: 0 } } }); + expect(h.server.all('CalendarEvent', 'a')).toHaveLength(1); + expect(h.server.get('CalendarEvent', 'a', lunch)).toMatchObject({ uid: 'lunch-uid', calendarIds: { [work]: true }, keywords: { important: true } }); + expect(h.events()).toEqual([expect.objectContaining({ _id: moved, _sync_id: `a/${lunch}`, dirty: 0 })]); + }); + it('syncs events both ways and writes nothing for its own echo', async () => { const h = real(); const standup = h.server.addEvent('a', { diff --git a/src/device-sync/engine/calendar-sync.ts b/src/device-sync/engine/calendar-sync.ts index 1f78fecf..e14d8264 100644 --- a/src/device-sync/engine/calendar-sync.ts +++ b/src/device-sync/engine/calendar-sync.ts @@ -5,6 +5,7 @@ */ import { findTasksOnlyCalendarIds, isTaskLikeObject, SCAN_PROPERTIES, type ScannedCalendarObject } from '../../lib/calendar-component-detection'; import { collectionKey, parseCollectionKey, parseObjectRef, pendingUidOf } from '../common/ids'; +import { deepEqual } from '../common/json'; import type { CalendarContext, CalendarEventWire, CalendarLike, CalendarPlanner, LocalCalendar, LocalEvent, OpGroup } from '../planner'; import { CALENDAR_AUTHORITY, type ReminderOwner, type Row } from '../types'; import { CALENDAR_EVENT_PROPERTIES, CALENDAR_PROPERTIES } from '../wire'; @@ -15,7 +16,7 @@ import type { SubscriptionCalendar } from './deps'; import { ItemSync, type Pending } from './item-sync'; import { accountOfRef, idInAccount, refOf, type Held, type Kind, type ServerObject } from './kinds'; import { poisonFingerprint } from './poison'; -import { flag, num, str } from './provider'; +import { flag, hasWrites, num, str } from './provider'; import { isCollectionSelected } from './selection'; import { accountOf, type SyncState } from './sync-state'; @@ -24,6 +25,8 @@ const WITH_IDENTITY = "_sync_id IS NOT NULL AND _sync_id NOT LIKE '~pending/%'"; /** CalendarProvider leaves DIRTY (and may leave DELETED) NULL on sync-adapter inserts: NULL is clean. */ const CLEAN = '(dirty IS NULL OR dirty = 0) AND (deleted IS NULL OR deleted = 0)'; +type PairPlan = ReturnType[number]; + function onIds(map: Record | undefined | null): string[] { return Object.entries(map ?? {}) .filter(([, on]) => on === true) @@ -411,41 +414,47 @@ export class CalendarSync extends ItemSync { /** * Before the creates: a split's source uploads its rule while the split is - * still visible, then its clone is claimed as a new event; and deleted + - * new masters of the run that are one edit (a move, a series turned - * single) upload as patches of the existing object. + * still visible, then its clone is claimed as a new event. */ protected async prePhase(acct: string): Promise { - let items = (await this.loadUploadItems()).filter((h) => this.accountOfItem(h) === acct); + const items = (await this.loadUploadItems()).filter((h) => this.accountOfItem(h) === acct); const sources = items.filter((h) => (h.local as LocalEvent).split === 'source'); const clones = items.filter((h) => (h.local as LocalEvent).split === 'clone'); - if (sources.length || clones.length) { - const works: Work[] = []; - const ready: Pending[] = []; - for (const held of sources) { - const pending = await this.planItem(acct, held, works); - if (pending) ready.push(pending); - } - // A clone (of a split, or a new master whose uid another row carries) needs no source: - // it claims a fresh uid here and is created with the other new events. - for (const held of clones) { - const pending = await this.planItem(acct, held, works); - if (pending) ready.push(pending); - } - await this.env.writer.write(works); - if (ready.length) await this.send(acct, ready); - items = (await this.loadUploadItems()).filter((h) => this.accountOfItem(h) === acct); + if (!sources.length && !clones.length) return; + const works: Work[] = []; + const ready: Pending[] = []; + for (const held of sources) { + const pending = await this.planItem(acct, held, works); + if (pending) ready.push(pending); } + // A clone (of a split, or a new master whose uid another row carries) needs no source: + // it claims a fresh uid here and is created with the other new events. + for (const held of clones) { + const pending = await this.planItem(acct, held, works); + if (pending) ready.push(pending); + } + await this.env.writer.write(works); + if (ready.length) await this.send(acct, ready); + } + + /** + * Before new rows are claimed: deleted and new masters of the run that are + * one edit (a move, a series turned single) upload as patches of the + * existing object, and the new row takes the old one's identity. The + * planner pairs only a new row that has not claimed a uid of its own. + */ + protected async pairPhase(acct: string): Promise { + const items = (await this.loadUploadItems()).filter((h) => this.accountOfItem(h) === acct); const deleted = items.map((h) => h.local as LocalEvent).filter((e) => { const m = this.eventKind.meta(e); return m.deleted && !!m.sourceId; }); const fresh = items.map((h) => h.local as LocalEvent).filter((e) => { const m = this.eventKind.meta(e); - return m.isNew && !m.deleted && !!m.pending; + return m.isNew && !m.deleted && !m.pending; }); if (!deleted.length || !fresh.length) return; - let pairs: ReturnType; + let pairs: PairPlan[]; try { pairs = this.planner.planPairs(deleted, fresh, this.ctx(acct)); } catch (error) { @@ -465,16 +474,25 @@ export class CalendarSync extends ItemSync { actions: pair.actions as Pending['actions'], accept: async (objects) => { // The new row takes the identity and exceptions of the old one, which is purged. - await this.env.writer.write([{ group: pair.ops }]); - const [moved] = await this.eventKind.loadByRowIds([pair.fresh.eventId]); + let paired = false; + await this.env.writer.write([this.pairWork(acct, pair, () => { + paired = true; + })]); + const [moved] = paired ? await this.eventKind.loadByRowIds([pair.fresh.eventId]) : []; const server = ownId ? objects.get(ownId) : undefined; if (!moved || !server) { + // Fetched again first next run. Rows that no longer pair (the new one was edited + // meanwhile) upload as a create and a deletion; neither takes the identity here. if (ownId) this.markStale(acct, ownId); return []; } const held: Held = { kind: this.eventKind, local: moved }; try { - return [this.acceptWork(acct, { held, meta: this.eventKind.meta(moved), fingerprint: '', actions: [] }, server, false)]; + const work = this.acceptWork(acct, { held, meta: this.eventKind.meta(moved), fingerprint: '', actions: [] }, server, false); + if (hasWrites(work.group.ops)) return [work]; + // The pair's ops already wrote what the server holds. + this.env.report.stats.uploaded.updated++; + return []; } catch (error) { this.plannerFailed(acct, refOf(acct, server.id), 'upload', error, server.id); return []; @@ -485,6 +503,32 @@ export class CalendarSync extends ItemSync { await this.send(acct, pending); } + /** + * A pair's rows after the server took its patch. When their assert fails + * (an app changed them during the upload), both rows are read and paired + * again; only a pair that uploads the same patch is written. + */ + private pairWork(acct: string, pair: PairPlan, applied: () => void): Work { + return { + group: pair.ops, + applied, + replan: async () => { + const rows = await this.eventKind.loadByRowIds([pair.deleted.eventId, pair.fresh.eventId]); + const deleted = rows.find((e) => e.eventId === pair.deleted.eventId); + const fresh = rows.find((e) => e.eventId === pair.fresh.eventId); + if (!deleted || !fresh) return null; + let again: PairPlan | undefined; + try { + [again] = this.planner.planPairs([deleted], [fresh], this.ctx(acct)); + } catch (error) { + this.plannerFailed(acct, pair.deleted.syncId ?? `pairs:${acct}`, 'upload', error); + return null; + } + return again && deepEqual(again.actions, pair.actions) ? this.pairWork(acct, again, applied) : null; + }, + }; + } + // ── After the upload ── /** A master with nothing waiting for an upload. */ diff --git a/src/device-sync/engine/item-sync.ts b/src/device-sync/engine/item-sync.ts index e4c96911..f390a57f 100644 --- a/src/device-sync/engine/item-sync.ts +++ b/src/device-sync/engine/item-sync.ts @@ -142,7 +142,9 @@ export abstract class ItemSync { const meta = held.kind.meta(held.local); return accountOfRef(meta.sourceId) ?? accountOfRef(meta.pending?.target ?? null); } - /** Before the creates (calendar: split series and pairs). */ + /** Before new rows are claimed (calendar: deleted and new rows that are one edit, uploaded as patches). */ + protected async pairPhase(_acct: string): Promise {} + /** Before the creates (calendar: split series). */ protected async prePhase(_acct: string): Promise {} /** After the deletions (contacts: groups deleted by absence, memberships). */ protected async extraPhase(_acct: string, _deletionsAllowed: boolean): Promise {} @@ -691,16 +693,20 @@ export abstract class ItemSync { // ── Uploads ── /** - * Local changes to the server, per phase across the accounts: pairs and - * splits, creates, updates, deletions, then memberships. Creates of every - * account come before deletions of any, so a move between accounts never - * destroys first. A `stateMismatch` downloads the account again and - * repeats the phase, at most three times. + * Local changes to the server, per phase across the accounts: pairs, the + * claims of new rows, splits, creates, updates, deletions, then + * memberships. Pairs go first: the new row of a pair takes the deleted + * row's identity, so it must not have claimed a uid of its own, and a + * paired deletion is a move, not a deletion. Creates of every account come + * before deletions of any, so a move between accounts never destroys + * first. A `stateMismatch` downloads the account again and repeats the + * phase, at most three times. */ protected async uploadAll(): Promise { + const accounts = this.accountIds(); + for (const acct of accounts) await this.withRetries(acct, () => this.pairPhase(acct)); await this.claimNewItems(); const deletionsAllowed = await this.deletionPolicy(await this.loadUploadItems()); - const accounts = this.accountIds(); for (const acct of accounts) await this.withRetries(acct, () => this.prePhase(acct)); for (const phase of ['create', 'update', 'delete'] as const) { for (const acct of accounts) await this.withRetries(acct, () => this.phase(acct, phase, deletionsAllowed)); From 7187cab904bbbfc84058ffa77f87a303e2d88934 Mon Sep 17 00:00:00 2001 From: Linus Rath <139418639+rathlinus@users.noreply.github.com> Date: Sun, 27 Sep 2026 07:41:27 +0200 Subject: [PATCH 39/87] fix: never undo server changes to a card moved out of synced books #34 When another client moved a contact or an event out of every address book or calendar the device syncs while the device had an unsent edit of it, the download only noted it for dropping after the upload. Its shadow stayed at the old version while the stored state moved past the change, so the upload passed `ifInState` and resent whole maps built from the old shadow: a number or an occurrence added on the server meanwhile was lost. A full reconcile that found such an object did the same. The download now merges the new version into the dirty rows like any other change, so the upload builds on it, and the rows still go once it is accepted. A contact's collections are read from its shadow: SYNC1 lists only the selected books and is empty for a card that left them all, so such a contact was never dropped after its upload. The calendar planner cannot merge an event that is in no synced calendar yet; until it can, such an event waits on the device and is reported instead of overwriting the server. Checked with the real planners through `/changes` and through a full reconcile. --- .../__tests__/engine/real-planners.test.ts | 84 +++++++++++++++++++ src/device-sync/engine/contacts-sync.ts | 3 +- src/device-sync/engine/item-sync.ts | 19 ++++- 3 files changed, 101 insertions(+), 5 deletions(-) diff --git a/src/device-sync/__tests__/engine/real-planners.test.ts b/src/device-sync/__tests__/engine/real-planners.test.ts index d44e3e06..2a20e232 100644 --- a/src/device-sync/__tests__/engine/real-planners.test.ts +++ b/src/device-sync/__tests__/engine/real-planners.test.ts @@ -155,6 +155,90 @@ describe('device sync engine with the real planners', () => { expect(h.contacts().every((c) => !c.dirty)).toBe(true); }); + describe('a device edit of an object the server moved out of every synced collection', () => { + /** Ada with two numbers in the synced book; the device deletes the second in place (AOSP). */ + async function adaWithDeletedNumber() { + const h = real(); + const archive = h.server.addAddressBook('a', { name: 'Archive' }); + h.prefs.contactsSelection[`a/${archive}`] = false; + const ada = h.server.addCard('a', { + uid: 'ada-uid', + name: { full: 'Ada Lovelace' }, + phones: { p1: { '@type': 'Phone', number: '+1 111' }, p2: { '@type': 'Phone', number: '+1 222' } }, + addressBookIds: { [h.book]: true }, + }); + await h.run(); + const contact = h.contactNamed('Ada Lovelace')!; + const p2 = h.device.rows('data').find((d) => Number(d.raw_contact_id) === contact.id && d[Data.DATA1] === '+1 222')!; + h.device.user.deleteData(Number(p2._id)); + // Meanwhile another client files Ada in Archive only and adds a number. + h.server.serverUpdate('ContactCard', 'a', ada, { addressBookIds: { [archive]: true }, 'phones/p3': { '@type': 'Phone', number: '+1 333' } }); + return { h, ada, archive }; + } + + const numbers = (card: Record) => + Object.values((card.phones ?? {}) as Record).map((p) => p.number).sort(); + + it('uploads it against the server version, then drops the rows', async () => { + const { h, ada, archive } = await adaWithDeletedNumber(); + + const report = await h.run(); + + expect(report).toMatchObject({ outcome: 'ok', itemErrors: [], stats: { uploaded: { updated: 1 } } }); + const card = h.server.get('ContactCard', 'a', ada)!; + expect(numbers(card)).toEqual(['+1 111', '+1 333']); + expect(card.addressBookIds).toEqual({ [archive]: true }); + expect(h.contacts()).toEqual([]); + }); + + it('does the same when a full reconcile finds the object outside', async () => { + const { h, ada, archive } = await adaWithDeletedNumber(); + h.server.truncateChangeLog('a', 'contacts'); + + const report = await h.run(); + + expect(report).toMatchObject({ outcome: 'ok', itemErrors: [] }); + const card = h.server.get('ContactCard', 'a', ada)!; + expect(numbers(card)).toEqual(['+1 111', '+1 333']); + expect(card.addressBookIds).toEqual({ [archive]: true }); + expect(h.contacts()).toEqual([]); + }); + + it('never uploads an event edit against an outdated shadow', async () => { + const h = real(); + const archive = h.server.addCalendar('a', { name: 'Archive' }); + h.prefs.calendarSelection[`a/${archive}`] = false; + const series = h.server.addEvent('a', { + uid: 'series-uid', + title: 'Standup', + start: '2026-09-28T09:00:00', + duration: 'PT15M', + timeZone: 'Europe/Berlin', + recurrenceRule: { '@type': 'RecurrenceRule', frequency: 'daily' }, + recurrenceOverrides: { '2026-09-29T09:00:00': { title: 'Planning' } }, + calendarIds: { [h.calendar]: true }, + }); + await h.run(CALENDAR_AUTHORITY); + const master = h.events().find((e) => e._sync_id === `a/${series}`)!; + const later = Number(master[Events.DTSTART]) + 3_600_000; + // The device moves the series by an hour; another client files it in Archive and changes another instance. + h.device.user.updateEvent(Number(master._id), { [Events.DTSTART]: later }); + h.server.serverUpdate('CalendarEvent', 'a', series, { + calendarIds: { [archive]: true }, + 'recurrenceOverrides/2026-10-01T09:00:00': { title: 'Retro' }, + }); + + expect((await h.run(CALENDAR_AUTHORITY)).outcome).toBe('ok'); + + const event = h.server.get('CalendarEvent', 'a', series)!; + const titles = Object.values((event.recurrenceOverrides ?? {}) as Record).map((o) => o.title).sort(); + expect(titles).toEqual(['Planning', 'Retro']); + // The move went up, or it still waits on the device: it is not lost. + const row = h.events().find((e) => Number(e._id) === Number(master._id)); + expect(event.start === '2026-09-28T10:00:00' || (Number(row?.dirty) === 1 && Number(row?.[Events.DTSTART]) === later)).toBe(true); + }); + }); + it('creates a device event that carries the uid of a synced event (an app copied it) under a fresh uid', async () => { const h = real(); const standup = h.server.addEvent('a', { diff --git a/src/device-sync/engine/contacts-sync.ts b/src/device-sync/engine/contacts-sync.ts index 3d5c55ee..40257ee4 100644 --- a/src/device-sync/engine/contacts-sync.ts +++ b/src/device-sync/engine/contacts-sync.ts @@ -151,7 +151,8 @@ export class ContactsSync extends ItemSync { deleted: c.deleted, isNew: !c.sourceId, poison: c.poison, - collections: c.collections, + // The card's books as last seen: SYNC1 lists only the selected ones, none for a card that left them all. + collections: c.shadow ? this.cardKeys(accountOfRef(c.sourceId), c.shadow) : c.collections, }), fingerprint: (c) => poisonFingerprint({ diff --git a/src/device-sync/engine/item-sync.ts b/src/device-sync/engine/item-sync.ts index f390a57f..e93ec84a 100644 --- a/src/device-sync/engine/item-sync.ts +++ b/src/device-sync/engine/item-sync.ts @@ -408,14 +408,16 @@ export abstract class ItemSync { if (flags.dirty || flags.deleted) verify.push(id); else gone.add(id); } + const outside: string[] = []; if (verify.length) { - // Destroyed, or moved out of the synced collections? Only a destroy takes dirty rows. + // Destroyed, or moved out of the synced collections? Only a destroy takes dirty rows; the + // others are downloaded like any change, so a dirty item merges the server's version. const { list, notFound } = await this.env.jmap.get(this.itemType, acct, verify, ['id']); for (const id of notFound) gone.add(id); - for (const object of list) this.outsideOf(acct).add(refOf(acct, object.id)); + for (const object of list) outside.push(object.id); } const selected = collections.map((c) => collectionKey(acct, c)); - await this.processIds(acct, [], [...gone], (next) => { + await this.processIds(acct, outside, [...gone], (next) => { const account = accountOf(next, acct); account.itemsState = from; account.reconcile = null; @@ -555,7 +557,14 @@ export abstract class ItemSync { written: Map> | null, ): Work | null { if (!kind || !this.inSelection(acct, object)) { - return found ? this.removeWork(acct, object.id, found, 'outside') : null; + if (!found) return null; + const meta = found.kind.meta(found.local); + if (found.kind === kind && meta.dirty && !meta.deleted && !meta.isNew) { + // Uploaded first, dropped afterwards: merged like any dirty item, so the upload builds on the server's version. + this.outsideOf(acct).add(refOf(acct, object.id)); + return this.downloadWork(kind, acct, object, found.local, written); + } + return this.removeWork(acct, object.id, found, 'outside'); } if (found && found.kind !== kind) { const meta = found.kind.meta(found.local); @@ -1263,6 +1272,8 @@ export abstract class ItemSync { const counted = () => { if (created) this.env.report.stats.uploaded.created++; else this.env.report.stats.uploaded.updated++; + // In no synced collection any more (moved away on the server): the rows go after the upload. + if (!this.inSelection(acct, server)) this.outsideOf(acct).add(refOf(acct, server.id)); this.onAccepted(kind, acct, server); }; const failed = (reason: string, message: string) => { From 58b74527b221cf7286e1acabe58f43dd1a5e1f59 Mon Sep 17 00:00:00 2001 From: Linus Rath <139418639+rathlinus@users.noreply.github.com> Date: Sun, 27 Sep 2026 07:42:08 +0200 Subject: [PATCH 40/87] fix: keep group cards on the server when a sync stops while dropping them #34 The contacts SyncState lists the group rows on the device, so a group an app hard-deleted (Fossify does) is found by its absence and deleted on the server. The list changed only with a later state op, not with the batch that inserted or deleted the rows. When a run stopped in between (cancelled, out of time, a crash, or turning sync off while a deselected book was being dropped), a group the engine had removed was still listed, taken for a hard delete by the next run or by the teardown's upload, and its card destroyed on the server; a group it had inserted was never listed, so a later hard delete never went up. A batch that inserts or deletes a group row now ends with a state op holding that change, retries included, so the list and the rows never disagree. A group a download finds unchanged is listed too, for rows whose batch lost its state op at a provider yield point. And an absent group whose card is no longer a group or is in no synced book is only forgotten, never destroyed. Checked with a crash at every checkpoint of a deselection, a teardown and a deadline stop during one, a server-side move in a page of several chunks, and a group written before an interruption and then deleted in Fossify. --- .../__tests__/engine/group-listing.test.ts | 195 ++++++++++++++++++ .../__tests__/engine/internals.test.ts | 32 ++- src/device-sync/engine/batch.ts | 44 +++- src/device-sync/engine/contacts-sync.ts | 30 ++- src/device-sync/engine/item-sync.ts | 41 +++- src/device-sync/engine/run.ts | 10 +- src/device-sync/engine/sync-state.ts | 20 +- 7 files changed, 350 insertions(+), 22 deletions(-) create mode 100644 src/device-sync/__tests__/engine/group-listing.test.ts diff --git a/src/device-sync/__tests__/engine/group-listing.test.ts b/src/device-sync/__tests__/engine/group-listing.test.ts new file mode 100644 index 00000000..16e47c49 --- /dev/null +++ b/src/device-sync/__tests__/engine/group-listing.test.ts @@ -0,0 +1,195 @@ +// SyncState.groups lists the group cards present on the device, so a group an +// app hard-deleted (Fossify) is found by its absence and deleted on the server. +// The list must change in the same batch as the rows: a group row the engine +// removed but still listed reads as a hard delete and destroys the card, and a +// row it inserted but never listed hides a later hard delete. + +import { describe, expect, it } from 'vitest'; +import { calendarPlanner } from '../../calendar/planner'; +import { contactsPlanner } from '../../contacts/planner'; +import { serializeSyncState } from '../../engine/sync-state'; +import { addServerCards, ANDROID_ACCOUNT, CONTACTS_AUTHORITY, createHarness, RUN_BUDGET, type Harness } from './harness'; + +function real(): Harness { + const h = createHarness(); + h.deps.planners = { contacts: contactsPlanner, calendar: calendarPlanner }; + return h; +} + +/** The personal book with Ada, and a Work book with Boss, Colleague and the group "Team" (Boss). */ +async function withWorkGroup() { + const h = real(); + const work = h.server.addAddressBook('a', { name: 'Work' }); + addServerCards(h, ['Ada']); + const [boss] = addServerCards(h, ['Boss', 'Colleague'], 'a', work); + const bossUid = h.server.get('ContactCard', 'a', boss)!.uid as string; + const group = h.server.addCard('a', { + uid: 'g-team', + kind: 'group', + name: { full: 'Team' }, + addressBookIds: { [work]: true }, + members: { [bossUid]: true }, + }); + expect((await h.run()).outcome).toBe('ok'); + expect(h.state()?.accounts.a.groups).toEqual([`a/${group}`]); + h.checkpoints.count = 0; + return { h, group, work }; +} + +/** Replaces the stored SyncState's `groups` of account `a` (a list that drifted from the rows). */ +async function storeGroups(h: Harness, groups: string[]): Promise { + const state = h.state()!; + state.accounts.a.groups = groups; + await h.device.port(ANDROID_ACCOUNT, CONTACTS_AUTHORITY).applyBatch([{ op: 'syncState', value: serializeSyncState(state) }]); +} + +describe('device sync engine: the groups a SyncState lists', () => { + it('keeps a group card on the server after a deselection stopped at any checkpoint', async () => { + const probe = await withWorkGroup(); + probe.h.prefs.contactsSelection[`a/${probe.work}`] = false; + await probe.h.run(); + const total = probe.h.checkpoints.count; + expect(probe.h.server.get('ContactCard', 'a', probe.group)).toBeDefined(); + + for (let n = 1; n <= total; n++) { + const { h, group, work } = await withWorkGroup(); + h.prefs.contactsSelection[`a/${work}`] = false; + h.checkpoints.crashAt = n; + await h.run(); + h.checkpoints.crashAt = null; + expect((await h.run()).outcome, `run after a crash at checkpoint ${n}`).toBe('ok'); + expect(h.server.get('ContactCard', 'a', group), `group card after a crash at checkpoint ${n}`).toBeDefined(); + expect(h.device.rows('groups')).toEqual([]); + expect(h.state()?.accounts.a.groups).toEqual([]); + } + }); + + it('keeps a group card on the server when its book is selected again after a stopped deselection', async () => { + const { h, group, work } = await withWorkGroup(); + h.prefs.contactsSelection[`a/${work}`] = false; + // Checkpoint 3 comes right after the deselected book's group rows went. + h.checkpoints.crashAt = 3; + await h.run(); + h.checkpoints.crashAt = null; + expect(h.device.rows('groups')).toEqual([]); + + h.prefs.contactsSelection[`a/${work}`] = true; + const report = await h.run(); + + expect(report).toMatchObject({ outcome: 'ok', stats: { uploaded: { deleted: 0 } } }); + expect(h.server.get('ContactCard', 'a', group)).toBeDefined(); + }); + + it('keeps a group card on the server when sync is turned off while a deselection drops it', async () => { + const { h, group, work } = await withWorkGroup(); + h.prefs.contactsSelection[`a/${work}`] = false; + let teardown: Promise<{ pending: number }> | null = null; + h.checkpoints.onCheckpoint = (n) => { + // Right before the run drops the deselected book's rows. + if (n === 2 && !teardown) teardown = h.teardown(CONTACTS_AUTHORITY); + }; + + const run = await h.run(); + h.checkpoints.onCheckpoint = undefined; + + expect(run.outcome).toBe('cancelled'); + expect(await teardown!).toEqual({ pending: 0 }); + expect(h.contacts()).toHaveLength(0); + expect(h.server.get('ContactCard', 'a', group)).toBeDefined(); + }); + + it('keeps a group card on the server when the deselecting run runs out of time after dropping the group', async () => { + const { h, group, work } = await withWorkGroup(); + h.prefs.contactsSelection[`a/${work}`] = false; + h.checkpoints.onCheckpoint = (n) => { + if (n === 2) h.clock.now += RUN_BUDGET; + }; + + expect((await h.run()).outcome).toBe('cancelled'); + h.checkpoints.onCheckpoint = undefined; + expect((await h.run()).outcome).toBe('ok'); + + expect(h.server.get('ContactCard', 'a', group)).toBeDefined(); + }); + + it('keeps a group card the server moved to a book that does not sync when the run stops within the page', async () => { + const h = real(); + const archive = h.server.addAddressBook('a', { name: 'Archive' }); + h.prefs.contactsSelection[`a/${archive}`] = false; + const ids = addServerCards(h, Array.from({ length: 60 }, (_, i) => `P${String(i).padStart(2, '0')}`)); + const group = h.server.addCard('a', { uid: 'g-team', kind: 'group', name: { full: 'Team' }, addressBookIds: { [h.book]: true }, members: {} }); + await h.run(); + expect(h.device.rows('groups')).toHaveLength(1); + // Another client moves the group to Archive and renames 60 contacts: one /changes page, several chunks. + h.server.serverUpdate('ContactCard', 'a', group, { addressBookIds: { [archive]: true } }); + ids.forEach((id, i) => h.server.serverUpdate('ContactCard', 'a', id, { 'name/full': `Q${i}` })); + // Checkpoints: 'changes', 'download' (the group's chunk), 'download' (the first contacts chunk) … + h.checkpoints.count = 0; + h.checkpoints.crashAt = 3; + + await h.run(); + h.checkpoints.crashAt = null; + expect(h.device.rows('groups')).toHaveLength(0); + expect((await h.run()).outcome).toBe('ok'); + + expect(h.server.get('ContactCard', 'a', group)).toBeDefined(); + expect(h.state()?.accounts.a.groups).toEqual([]); + }); + + it('lists a group written before an interruption, so a hard delete by an app still reaches the server', async () => { + const h = real(); + addServerCards(h, Array.from({ length: 60 }, (_, i) => `P${String(i).padStart(2, '0')}`)); + const group = h.server.addCard('a', { uid: 'g1', kind: 'group', name: { full: 'Friends' }, members: {} }); + // First sync: 'reconcile', 'reconcile:ids', 'reconcile:objects' (the group's chunk), then out of time. + h.checkpoints.onCheckpoint = (n) => { + if (n === 3) h.clock.now += RUN_BUDGET; + }; + expect((await h.run()).outcome).toBe('cancelled'); + h.checkpoints.onCheckpoint = undefined; + expect(h.device.rows('groups')).toHaveLength(1); + expect(h.state()?.accounts.a.groups).toEqual([`a/${group}`]); + expect((await h.run()).outcome).toBe('ok'); + expect(h.contacts()).toHaveLength(60); + + // Fossify hard-deletes the group through a sync-adapter URI. + h.device.user.fossifyDeleteGroup(Number(h.device.rows('groups')[0]._id)); + expect((await h.run()).outcome).toBe('ok'); + + expect(h.server.get('ContactCard', 'a', group)).toBeUndefined(); + }); + + it('lists a group again when it comes back as an echo', async () => { + const h = real(); + const group = h.server.addCard('a', { uid: 'g1', kind: 'group', name: { full: 'Friends' }, members: {} }); + await h.run(); + // A list that lost the group (a write committed without its state op), then a download of the unchanged card. + await storeGroups(h, []); + addServerCards(h, ['Ada']); + h.server.truncateChangeLog('a', 'contacts'); + expect((await h.run()).outcome).toBe('ok'); + expect(h.state()?.accounts.a.groups).toEqual([`a/${group}`]); + + h.device.user.fossifyDeleteGroup(Number(h.device.rows('groups')[0]._id)); + await h.run(); + + expect(h.server.get('ContactCard', 'a', group)).toBeUndefined(); + }); + + it('never destroys a listed group card that is in no synced book or no longer a group, and forgets it', async () => { + const h = real(); + const archive = h.server.addAddressBook('a', { name: 'Archive' }); + h.prefs.contactsSelection[`a/${archive}`] = false; + const filed = h.server.addCard('a', { uid: 'g1', kind: 'group', name: { full: 'Filed' }, addressBookIds: { [archive]: true }, members: {} }); + const person = h.server.addCard('a', { uid: 'p1', name: { full: 'Was a group' }, addressBookIds: { [h.book]: true } }); + await h.run(); + // The list names both as groups whose rows are gone (the engine removed them, and the batch storing that was lost). + await storeGroups(h, [`a/${filed}`, `a/${person}`]); + + const report = await h.run(); + + expect(report).toMatchObject({ outcome: 'ok', stats: { uploaded: { deleted: 0 } } }); + expect(h.server.get('ContactCard', 'a', filed)).toBeDefined(); + expect(h.server.get('ContactCard', 'a', person)).toBeDefined(); + expect(h.state()?.accounts.a.groups).toEqual([]); + }); +}); diff --git a/src/device-sync/__tests__/engine/internals.test.ts b/src/device-sync/__tests__/engine/internals.test.ts index 8b58b263..9609e1b4 100644 --- a/src/device-sync/__tests__/engine/internals.test.ts +++ b/src/device-sync/__tests__/engine/internals.test.ts @@ -8,7 +8,7 @@ import { classifyFailure, RunAbort, StateMismatch, StopRun } from '../../engine/ import { acquireLock } from '../../engine/mutex'; import { isBackedOff, nextMarker } from '../../engine/poison'; import { collectionDefaultOn, isCollectionSelected } from '../../engine/selection'; -import { emptySyncState, parseSyncState, serializeSyncState, StateStore } from '../../engine/sync-state'; +import { accountOf, emptySyncState, parseSyncState, serializeSyncState, StateStore, type SyncState } from '../../engine/sync-state'; import { classifySetError, existingIdFromUidError, JmapCaller } from '../../jmap/caller'; import { openWithAuthRebuild } from '../../jmap/connection'; import { JMAPMethodError } from '../../jmap/errors'; @@ -127,6 +127,36 @@ describe('provider batches', () => { await expect(writer(scope).write([{ group: { ref: 'g', ops: [insert('raw_contacts')] } }])).rejects.toThrow(/not our account/); }); + it("stores a group's state change in the batch that applies it, its retry included, and never without it", async () => { + const store = new StateStore(emptySyncState(), { registryId: 'r', origin: 'o' }); + let failures = 1; + const p = port((ops) => (ops.some((op) => op.op === 'assert') && failures-- > 0 ? { ok: false, reason: 'assert', message: 'stale' } : ok(ops))); + const listed = (ref: string) => (next: SyncState) => { + accountOf(next, 'a').groups = [...(accountOf(next, 'a').groups ?? []), ref]; + }; + const groupOps = (id: number): ProviderOp[] => [ + { op: 'assert', table: 'groups', id, values: { version: 1 } }, + { op: 'update', table: 'groups', id, values: { sourceid: `a/g${id}` } }, + ]; + const plain: Work = { group: { ref: 'c', ops: [insert('raw_contacts'), insert('data')] } }; + const retried: Work = { group: { ref: 'a/g1', ops: groupOps(1) }, state: listed('a/g1') }; + retried.replan = async () => ({ group: retried.group, state: retried.state }); + const w = new BatchWriter(p, { maxOps: 2, maxBytes: 300_000, maxReplans: 3 }, store); + + await w.write([plain, retried]); + + // The plain group alone, without a state op; the group with its state op, refused; its retry, with the state op. + expect(p.sent.map((b) => b.map((op) => op.op))).toEqual([['insert', 'insert'], ['assert', 'update', 'syncState'], ['assert', 'update', 'syncState']]); + expect(store.committed.accounts.a.groups).toEqual(['a/g1']); + + failures = Infinity; + const given: Work = { group: { ref: 'a/g2', ops: groupOps(2) }, state: listed('a/g2'), replan: async () => null }; + await w.write([given], store.tail(() => undefined)); + + expect(p.sent.at(-1)!.map((op) => op.op)).toEqual(['syncState']); + expect(store.committed.accounts.a.groups).toEqual(['a/g1']); + }); + it('skips groups that only assert', async () => { const p = port(ok); await writer(p).write([{ group: { ref: 'g', ops: [{ op: 'assert', table: 'raw_contacts', id: 1, values: { dirty: 0 } }] } }]); diff --git a/src/device-sync/engine/batch.ts b/src/device-sync/engine/batch.ts index 2b4095ee..2375b86f 100644 --- a/src/device-sync/engine/batch.ts +++ b/src/device-sync/engine/batch.ts @@ -15,12 +15,16 @@ * a committed group fail harmlessly); a group that fails an assert is * re-read and re-planned up to `maxReplans` times, then given up. `tooLarge` * splits the batch; `permission` ends the run. + * + * A group whose rows the SyncState describes (a group row: `groups`) carries + * that `state` change: every batch that applies such a group, a retry + * included, ends with a state op holding it. */ import type { OpGroup } from '../planner'; import type { BatchFailure, OpResult, ProviderOp, ProviderPort } from '../types'; import { RunAbort } from './errors'; import { hasWrites } from './provider'; -import type { Tail } from './sync-state'; +import type { StateChange, StateStore, Tail } from './sync-state'; export interface Work { group: OpGroup; @@ -30,6 +34,8 @@ export interface Work { replan?(): Promise; /** Given up: the item's rows stay as they are. */ failed?(reason: BatchFailure, message: string): void | Promise; + /** What the SyncState lists changes with these rows: stored by the batch that applies them. */ + state?: StateChange; } export interface BatchWriterOptions { @@ -100,6 +106,8 @@ export class BatchWriter { constructor( private readonly port: ProviderPort, private readonly options: BatchWriterOptions, + /** Where the `state` of works goes; without it, it is ignored. */ + private readonly store?: StateStore, ) {} /** @@ -162,9 +170,26 @@ export class BatchWriter { return result; } + /** The works' state changes go into the state op of the batch being sent: `tail`, or one of its own. */ + private stage(works: readonly Work[], tail?: Tail): { staged: Work[]; tail?: Tail } { + const store = this.store; + const staged = store ? works.filter((w) => w.state) : []; + for (const work of staged) store!.stage(work, work.state!); + return { staged, tail: tail ?? (staged.length ? store!.tail(() => undefined) : undefined) }; + } + + private unstage(staged: readonly Work[]): void { + for (const work of staged) this.store!.unstage(work); + } + private async applyBatch(works: Work[], tail?: Tail): Promise { - const tailOp = tail?.op(); - const result = await this.send(buildBatch(works.map((w) => w.group), tailOp)); + const stated = this.stage(works, tail); + let result; + try { + result = await this.send(buildBatch(works.map((w) => w.group), stated.tail?.op())); + } finally { + this.unstage(stated.staged); + } if (result.ok) { let offset = 0; for (const work of works) { @@ -172,7 +197,7 @@ export class BatchWriter { await work.applied?.(result.results.slice(offset, offset + n)); offset += n; } - tail?.applied(); + stated.tail?.applied(); return; } if (result.reason === 'tooLarge' && works.length > 1) { @@ -191,9 +216,16 @@ export class BatchWriter { private async applyAlone(work: Work, attempt: number): Promise { if (!hasWrites(work.group.ops)) return; - const result = await this.send(buildBatch([work.group])); + const stated = this.stage([work]); + let result; + try { + result = await this.send(buildBatch([work.group], stated.tail?.op())); + } finally { + this.unstage(stated.staged); + } if (result.ok) { - await work.applied?.(result.results); + await work.applied?.(result.results.slice(0, work.group.ops.length)); + stated.tail?.applied(); return; } await this.retryAlone(work, result.reason, result.message, attempt); diff --git a/src/device-sync/engine/contacts-sync.ts b/src/device-sync/engine/contacts-sync.ts index 40257ee4..9a35eb32 100644 --- a/src/device-sync/engine/contacts-sync.ts +++ b/src/device-sync/engine/contacts-sync.ts @@ -24,7 +24,7 @@ import { accountOfRef, idInAccount, refOf, type Held, type Kind, type ServerObje import { poisonFingerprint } from './poison'; import { flag, num, str } from './provider'; import { isCollectionSelected } from './selection'; -import { accountOf, type SyncState } from './sync-state'; +import { accountOf, type StateChange, type SyncState } from './sync-state'; /** * The context the engine hands the contacts planner. `groupsOf` is @@ -496,10 +496,29 @@ export class ContactsSync extends ItemSync { else this.indexCard(acct, object as ContactCardWire); } + /** A group whose row is there already is listed too: the batch that listed it may never have been stored. */ + protected onEcho(kind: Kind, acct: string, object: ServerObject): void { + const ref = refOf(acct, object.id); + if (kind === this.groupKind && !(this.env.store.account(acct).groups ?? []).includes(ref)) this.groupChange(acct, ref, true); + } + protected onRemoved(kind: Kind, acct: string, id: string): void { if (kind === this.groupKind) this.groupChange(acct, refOf(acct, id), false); } + /** `groups` names a group row from the batch that inserts it until the batch that deletes it. */ + protected listing(kind: Kind, acct: string, id: string, present: boolean): StateChange | undefined { + if (kind !== this.groupKind) return undefined; + const ref = refOf(acct, id); + return (next) => { + const account = accountOf(next, acct); + const groups = new Set(account.groups ?? []); + if (present) groups.add(ref); + else groups.delete(ref); + account.groups = [...groups]; + }; + } + protected onAccepted(kind: Kind, acct: string, server: ServerObject): void { if (kind === this.groupKind) this.groupChange(acct, refOf(acct, server.id), true); else this.indexCard(acct, server as ContactCardWire); @@ -628,21 +647,24 @@ export class ContactsSync extends ItemSync { /** * A group hard-deleted on the device: destroyed on the server, or, when * it is also in address books that do not sync, removed from the synced - * ones only. + * ones only. A card that is no longer a group, or in no book that syncs, + * is not what the device showed: it is only forgotten. */ private async deleteAbsentGroups(acct: string): Promise { const refs = (this.absent.get(acct) ?? []).filter((ref) => this.groupChanges.get(acct)?.get(ref) !== false); if (!refs.length) return; await this.env.checkpoints.check('upload:groups'); const ids = refs.map((ref) => idInAccount(ref, acct)).filter((id): id is string => !!id); - const { list, notFound } = await this.env.jmap.get('ContactCard', acct, ids, ['id', 'addressBookIds']); + const { list, notFound } = await this.env.jmap.get('ContactCard', acct, ids, ['id', 'addressBookIds', 'kind']); for (const id of notFound) this.groupChange(acct, refOf(acct, id), false); const update: Record> = {}; const destroy: string[] = []; for (const card of list) { const books = onIds(card.addressBookIds); const synced = books.filter((id) => this.syncedKeys.has(collectionKey(acct, id))); - if (synced.length && synced.length < books.length) { + if (card.kind !== 'group' || !synced.length) { + this.groupChange(acct, refOf(acct, card.id), false); + } else if (synced.length < books.length) { update[card.id] = Object.fromEntries(synced.map((id) => [`addressBookIds/${id}`, null])); } else { destroy.push(card.id); diff --git a/src/device-sync/engine/item-sync.ts b/src/device-sync/engine/item-sync.ts index e93ec84a..84e9c254 100644 --- a/src/device-sync/engine/item-sync.ts +++ b/src/device-sync/engine/item-sync.ts @@ -32,10 +32,18 @@ import { } from './kinds'; import { isBackedOff, nextMarker } from './poison'; import { hasWrites } from './provider'; -import { accountOf, type SyncState, type Tail } from './sync-state'; +import { accountOf, type StateChange, type SyncState, type Tail } from './sync-state'; type Action = UploadAction; +function both(first: StateChange | undefined, second: StateChange | undefined): StateChange | undefined { + if (!first || !second) return first ?? second; + return (next) => { + first(next); + second(next); + }; +} + /** An item's upload, on its way to the server. */ export interface Pending { held: Held; @@ -122,8 +130,18 @@ export abstract class ItemSync { /** Authority changes folded into a state op of the account; the callback runs once that op is stored. */ protected decorateState(_acct: string, _next: SyncState): (() => void) | void {} protected onDownloaded(_kind: Kind, _acct: string, _object: ServerObject): void {} + /** A download found the object's rows as they should be already. */ + protected onEcho(_kind: Kind, _acct: string, _object: ServerObject): void {} protected onRemoved(_kind: Kind, _acct: string, _id: string): void {} protected onAccepted(_kind: Kind, _acct: string, _server: ServerObject): void {} + /** + * What the SyncState lists changes when a row of `kind` is inserted + * (`present`) or deleted (contacts: `groups`): stored by the batch that + * writes the row, so the list never disagrees with the rows. + */ + protected listing(_kind: Kind, _acct: string, _id: string, _present: boolean): StateChange | undefined { + return undefined; + } protected onDiscard(_next: SyncState, _accounts: Set): void {} /** Whether a pending create of `target` may adopt this object (contacts: the target book holds it). */ protected adoptionTargetMatches(acct: string, _object: ServerObject, target: string): boolean { @@ -588,6 +606,7 @@ export abstract class ItemSync { const ref = refOf(acct, object.id); const plan = kind.planDownload(object, local ?? null, acct); if (plan.effect === 'none' || !hasWrites(plan.ops.ops)) { + if (local) this.onEcho(kind, acct, object); if (!local || kind.meta(local).dirty) return null; const heal = kind.planBaselineHeal(local); return heal && hasWrites(heal.ops) ? { group: heal } : null; @@ -595,6 +614,7 @@ export abstract class ItemSync { const group = local ? plan.ops : prependOps(plan.ops, [insertGuard(kind, ref)]); return { group, + state: this.listing(kind, acct, object.id, true), applied: () => { if (plan.effect === 'insert') this.env.report.stats.downloaded.created++; else this.env.report.stats.downloaded.updated++; @@ -629,6 +649,7 @@ export abstract class ItemSync { const plan = kind.planDownload(object, null, acct); return { group: concatGroups(ref, found.kind.planLocalDelete(found.local), plan.ops), + state: both(this.listing(found.kind, acct, object.id, false), this.listing(kind, acct, object.id, true)), applied: () => { this.env.report.stats.downloaded.updated++; this.markResolved(acct, object.id); @@ -657,6 +678,7 @@ export abstract class ItemSync { const conflict = reason === 'destroyed' && meta.dirty && !meta.deleted; return { group: found.kind.planLocalDelete(found.local), + state: this.listing(found.kind, acct, id, false), applied: () => { this.env.report.stats.downloaded.deleted++; if (conflict) this.env.report.conflicts++; @@ -952,6 +974,7 @@ export abstract class ItemSync { const id = idInAccount(meta.sourceId, acct); return { group: held.kind.planLocalDelete(held.local), + state: id ? this.listing(held.kind, acct, id, false) : undefined, applied: () => { if (uploaded) this.env.report.stats.uploaded.deleted++; if (id) this.onRemoved(held.kind, acct, id); @@ -1280,10 +1303,12 @@ export abstract class ItemSync { this.markStale(acct, server.id); this.env.report.itemError({ ref: refOf(acct, server.id), side: 'upload', type: reason, description: message }); }; + const state = this.listing(kind, acct, server.id, true); return { group: withExtra(plan.ops), + state, applied: counted, - replan: async () => ({ group: withExtra(plan.keepDirtyOps), applied: counted, failed }), + replan: async () => ({ group: withExtra(plan.keepDirtyOps), state, applied: counted, failed }), failed, }; } @@ -1310,11 +1335,19 @@ export abstract class ItemSync { const object = id ? (await this.env.jmap.get(this.itemType, acct, [id], this.itemProperties)).list[0] : undefined; const objectKind = object ? this.kindOf(object) : null; if (!object || !objectKind || !this.inSelection(acct, object)) { - return [{ group: remove, applied: () => (id ? this.onRemoved(kind, acct, id) : undefined) }]; + return [{ + group: remove, + state: id ? this.listing(kind, acct, id, false) : undefined, + applied: () => (id ? this.onRemoved(kind, acct, id) : undefined), + }]; } await this.beforePlanning(acct, [object]); const insert = objectKind.planDownload(object, null, acct); - return [{ group: concatGroups(ref, remove, insert.ops), applied: () => this.onDownloaded(objectKind, acct, object) }]; + return [{ + group: concatGroups(ref, remove, insert.ops), + state: both(this.listing(kind, acct, object.id, false), this.listing(objectKind, acct, object.id, true)), + applied: () => this.onDownloaded(objectKind, acct, object), + }]; } /** diff --git a/src/device-sync/engine/run.ts b/src/device-sync/engine/run.ts index d037c332..39c06178 100644 --- a/src/device-sync/engine/run.ts +++ b/src/device-sync/engine/run.ts @@ -124,11 +124,11 @@ export async function createEnv(options: EnvOptions): Promise<{ env: RunEnv; syn authority, extras: options.extras, reader: options.reader, - writer: new BatchWriter(options.reader.port, { - maxOps: tuning.maxBatchOps, - maxBytes: tuning.maxBatchBytes, - maxReplans: tuning.maxReplans, - }), + writer: new BatchWriter( + options.reader.port, + { maxOps: tuning.maxBatchOps, maxBytes: tuning.maxBatchBytes, maxReplans: tuning.maxReplans }, + store, + ), report: options.report, checkpoints, store, diff --git a/src/device-sync/engine/sync-state.ts b/src/device-sync/engine/sync-state.ts index f9df3106..46e1e73a 100644 --- a/src/device-sync/engine/sync-state.ts +++ b/src/device-sync/engine/sync-state.ts @@ -132,14 +132,20 @@ export interface Tail { applied(): void; } +/** A change of the SyncState that has to be stored together with the rows it describes. */ +export type StateChange = (next: SyncState) => void; + /** * The committed SyncState of a run. Changes are proposed as a tail: the * batch writer calls `op()` right before sending the last batch (so the * change sees what earlier batches decided, e.g. items put on `stale`), and - * the state is only adopted after that batch applied. + * the state is only adopted after that batch applied. Changes that describe + * the rows of one batch (contacts: the groups present) are staged by the + * batch writer while it sends that batch, so its state op carries them. */ export class StateStore { private state: SyncState; + private readonly staged = new Map(); constructor( initial: SyncState, @@ -157,7 +163,16 @@ export class StateStore { return this.state.accounts[jmapAccountId] ?? emptyAccountState(); } - /** A tail applying `mutate` to a copy of the committed state (plus the owner). */ + /** Every state op built until `unstage(key)` applies `change` too, after its own changes. */ + stage(key: object, change: StateChange): void { + this.staged.set(key, change); + } + + unstage(key: object): void { + this.staged.delete(key); + } + + /** A tail applying `mutate` (then the staged changes) to a copy of the committed state (plus the owner). */ tail(mutate: (next: SyncState) => void): Tail { let next: SyncState | null = null; return { @@ -165,6 +180,7 @@ export class StateStore { next = clone(this.state); if (this.owner) next.owner = { ...this.owner }; mutate(next); + for (const change of this.staged.values()) change(next); return { op: 'syncState', value: serializeSyncState(next) }; }, applied: () => { From d088dd14e9e77fd829cb2143cf5535aa483abd3e Mon Sep 17 00:00:00 2001 From: Linus Rath <139418639+rathlinus@users.noreply.github.com> Date: Sun, 27 Sep 2026 07:42:50 +0200 Subject: [PATCH 41/87] fix: stop asking about waiting changes after a group edit reached the server #34 A contact stays marked as changed until the group cards on the server show its membership edits. The engine uploaded those edits as patches of the group cards after the contacts, but never looked at the contact again in that run, so it stayed marked until the next sync. Turning contacts sync off or signing out right after adding a contact to a group therefore asked whether to drop one waiting change, although nothing was waiting. Once the membership patches are accepted, the contacts that have nothing else to upload are now cleared in the same run, behind the same version check as any other clean write. --- .../__tests__/engine/real-planners.test.ts | 32 +++++++++++++++++++ src/device-sync/engine/contacts-sync.ts | 26 ++++++++++++++- 2 files changed, 57 insertions(+), 1 deletion(-) diff --git a/src/device-sync/__tests__/engine/real-planners.test.ts b/src/device-sync/__tests__/engine/real-planners.test.ts index 2a20e232..fc4e91d5 100644 --- a/src/device-sync/__tests__/engine/real-planners.test.ts +++ b/src/device-sync/__tests__/engine/real-planners.test.ts @@ -11,6 +11,7 @@ import { addDeviceContact, addServerCards, CALENDAR_AUTHORITY, + CONTACTS_AUTHORITY, createHarness, rowWrites, type Harness, @@ -155,6 +156,37 @@ describe('device sync engine with the real planners', () => { expect(h.contacts().every((c) => !c.dirty)).toBe(true); }); + describe('a membership edit on the device', () => { + async function adaJoinsFriends() { + const h = real(); + const [ada] = addServerCards(h, ['Ada Lovelace']); + const group = h.server.addCard('a', { uid: 'g1', kind: 'group', name: { full: 'Friends' }, members: {} }); + await h.run(); + const contact = h.contactNamed('Ada Lovelace')!; + h.device.user.addToGroup(contact.id, Number(h.device.rows('groups')[0]._id)); + return { h, ada, group, contact }; + } + + it('leaves the contact clean once the group card shows it', async () => { + const { h, ada, group, contact } = await adaJoinsFriends(); + + expect((await h.run()).outcome).toBe('ok'); + + const uid = h.server.get('ContactCard', 'a', ada)!.uid as string; + expect(h.server.get('ContactCard', 'a', group)!.members).toEqual({ [uid]: true }); + expect(h.contacts().find((c) => c.id === contact.id)).toMatchObject({ dirty: false }); + }); + + it('leaves nothing waiting for the teardown once it is uploaded', async () => { + const { h, ada, group } = await adaJoinsFriends(); + + expect(await h.teardown(CONTACTS_AUTHORITY)).toEqual({ pending: 0 }); + + const uid = h.server.get('ContactCard', 'a', ada)!.uid as string; + expect(h.server.get('ContactCard', 'a', group)!.members).toEqual({ [uid]: true }); + }); + }); + describe('a device edit of an object the server moved out of every synced collection', () => { /** Ada with two numbers in the synced book; the device deletes the second in place (AOSP). */ async function adaWithDeletedNumber() { diff --git a/src/device-sync/engine/contacts-sync.ts b/src/device-sync/engine/contacts-sync.ts index 9a35eb32..fe917a84 100644 --- a/src/device-sync/engine/contacts-sync.ts +++ b/src/device-sync/engine/contacts-sync.ts @@ -710,7 +710,31 @@ export class ContactsSync extends ItemSync { } pending.actions.push(action as Pending['actions'][number]); } - if (byGroup.size) await this.send(acct, [...byGroup.values()]); + if (!byGroup.size) return; + await this.send(acct, [...byGroup.values()]); + await this.cleanAfterMemberships(acct, rows); + } + + /** + * A contact stays dirty until the group cards show its membership edits. + * Those that now have nothing else to upload are cleaned right away, so + * nothing looks like it still waits (turning sync off would ask about it). + */ + private async cleanAfterMemberships(acct: string, rowIds: number[]): Promise { + await this.refreshGroups(); + const works: Work[] = []; + for (const contact of await this.contactKind.loadByRowIds(rowIds)) { + if (!contact.dirty || contact.deleted || this.isStale(acct, contact.sourceId)) continue; + let plan; + try { + plan = this.contactKind.planUpload(contact, acct); + } catch (error) { + this.plannerFailed(acct, contact.sourceId ?? `row:${contact.rawContactId}`, 'upload', error); + continue; + } + if (plan.kind === 'clean') works.push({ group: plan.ops }); + } + await this.env.writer.write(works); } // ── Deselection ── From 09c2ff1f67a0fe981989a8ec891d85d66eb65a21 Mon Sep 17 00:00:00 2001 From: Linus Rath <139418639+rathlinus@users.noreply.github.com> Date: Sun, 27 Sep 2026 07:43:31 +0200 Subject: [PATCH 42/87] fix: upload a contact's removal from a group made on the phone #34 Whether a membership edit still has to go up is decided against the group cards on the server, which the engine indexes from their shadows. The index was only read before planning a download that held contacts, so in a run that downloaded no contact, such as the upload sync after an edit, it was empty: removing a contact from a group looked already done on the server, the contact was marked clean and the removal was never sent. The engine now reads the index before planning uploads too. --- .../__tests__/engine/real-planners.test.ts | 18 ++++++++++++++++++ src/device-sync/engine/contacts-sync.ts | 8 +++++++- 2 files changed, 25 insertions(+), 1 deletion(-) diff --git a/src/device-sync/__tests__/engine/real-planners.test.ts b/src/device-sync/__tests__/engine/real-planners.test.ts index fc4e91d5..097c1f2b 100644 --- a/src/device-sync/__tests__/engine/real-planners.test.ts +++ b/src/device-sync/__tests__/engine/real-planners.test.ts @@ -185,6 +185,24 @@ describe('device sync engine with the real planners', () => { const uid = h.server.get('ContactCard', 'a', ada)!.uid as string; expect(h.server.get('ContactCard', 'a', group)!.members).toEqual({ [uid]: true }); }); + + it('uploads a removal from a group in a run that downloads no contact', async () => { + const h = real(); + // A keyed phone row shows the editor works in place, so a missing membership row is a removal. + h.server.addCard('a', { uid: 'ada-uid', name: { full: 'Ada Lovelace' }, phones: { p1: { '@type': 'Phone', number: '+1 111' } } }); + const group = h.server.addCard('a', { uid: 'g1', kind: 'group', name: { full: 'Friends' }, members: { 'ada-uid': true } }); + await h.run(); + const contact = h.contactNamed('Ada Lovelace')!; + const membership = h.device.rows('data').find((d) => Number(d.raw_contact_id) === contact.id && d.mimetype === MimeType.GROUP_MEMBERSHIP)!; + h.device.user.deleteData(Number(membership._id)); + + // The upload sync Android requests after the edit: the server has nothing new. + expect((await h.run(CONTACTS_AUTHORITY, { upload: true })).outcome).toBe('ok'); + + expect(h.server.get('ContactCard', 'a', group)!.members ?? {}).toEqual({}); + expect(h.contacts().find((c) => c.id === contact.id)).toMatchObject({ dirty: false }); + expect(h.device.rows('data').filter((d) => d.mimetype === MimeType.GROUP_MEMBERSHIP)).toEqual([]); + }); }); describe('a device edit of an object the server moved out of every synced collection', () => { diff --git a/src/device-sync/engine/contacts-sync.ts b/src/device-sync/engine/contacts-sync.ts index fe917a84..87854e03 100644 --- a/src/device-sync/engine/contacts-sync.ts +++ b/src/device-sync/engine/contacts-sync.ts @@ -576,8 +576,14 @@ export class ContactsSync extends ItemSync { for (const held of items) if (held.kind === this.contactKind) rows.add(held.kind.meta(held.local).rowId); } - /** The device photo of every contact about to be planned that has a photo row (JPEG, ≤ 512 px). */ + /** + * The group index `groupsOf` answers from (a run that downloaded no + * contact never read it: every removal from a group would look done), and + * the device photo of every contact about to be planned that has a photo + * row (JPEG, ≤ 512 px). + */ protected async beforeUploadPlanning(items: Held[]): Promise { + if (this.groupsStale && items.some((held) => held.kind === this.contactKind)) await this.refreshGroups(); for (const held of items) { if (held.kind !== this.contactKind) continue; const contact = held.local as LocalContact; From 34236f677b3ff06beb6e5220ef42802f43816859 Mon Sep 17 00:00:00 2001 From: Linus Rath <139418639+rathlinus@users.noreply.github.com> Date: Sun, 27 Sep 2026 07:44:10 +0200 Subject: [PATCH 43/87] fix: never resend a group's members from an outdated copy #34 A group whose latest server version the device could not store waits on the stale list, and nothing uploads for it until it has been fetched again. Membership edits bypassed that check: adding a contact to such a group sent the whole member list as the device last knew it, which removed members another client had added meanwhile. Membership edits for a stale group now wait for that fetch, and the report lists them. --- .../__tests__/engine/real-planners.test.ts | 30 +++++++++++++++++++ src/device-sync/engine/contacts-sync.ts | 5 ++++ 2 files changed, 35 insertions(+) diff --git a/src/device-sync/__tests__/engine/real-planners.test.ts b/src/device-sync/__tests__/engine/real-planners.test.ts index 097c1f2b..a493d065 100644 --- a/src/device-sync/__tests__/engine/real-planners.test.ts +++ b/src/device-sync/__tests__/engine/real-planners.test.ts @@ -186,6 +186,36 @@ describe('device sync engine with the real planners', () => { expect(h.server.get('ContactCard', 'a', group)!.members).toEqual({ [uid]: true }); }); + it('waits with a membership edit while the group card is stale, instead of resending its members', async () => { + const { h, ada, group } = await adaJoinsFriends(); + const [bob] = addServerCards(h, ['Bob']); + const bobUid = h.server.get('ContactCard', 'a', bob)!.uid as string; + // Another client adds Bob; this device cannot store the new version of the group this run. + h.server.serverUpdate('ContactCard', 'a', group, { members: { [bobUid]: true } }); + h.deps.planners = { + ...h.deps.planners, + contacts: { + ...contactsPlanner, + planGroupDownload: (card, local, ctx) => { + if (card.id === group) throw new Error('cannot store this group now'); + return contactsPlanner.planGroupDownload(card, local, ctx); + }, + }, + }; + + await h.run(); + + const adaUid = h.server.get('ContactCard', 'a', ada)!.uid as string; + expect(h.server.get('ContactCard', 'a', group)!.members).toEqual({ [bobUid]: true }); + expect(h.contacts().find((c) => c.name === 'Ada Lovelace')).toMatchObject({ dirty: true }); + + h.deps.planners = { contacts: contactsPlanner, calendar: calendarPlanner }; + expect((await h.run()).outcome).toBe('ok'); + + expect(h.server.get('ContactCard', 'a', group)!.members).toEqual({ [bobUid]: true, [adaUid]: true }); + expect(h.contacts().find((c) => c.name === 'Ada Lovelace')).toMatchObject({ dirty: false }); + }); + it('uploads a removal from a group in a run that downloads no contact', async () => { const h = real(); // A keyed phone row shows the editor works in place, so a missing membership row is a removal. diff --git a/src/device-sync/engine/contacts-sync.ts b/src/device-sync/engine/contacts-sync.ts index 87854e03..3c8660ba 100644 --- a/src/device-sync/engine/contacts-sync.ts +++ b/src/device-sync/engine/contacts-sync.ts @@ -708,6 +708,11 @@ export class ContactsSync extends ItemSync { this.env.report.itemError({ ref: id ? refOf(acct, id) : 'membership', side: 'upload', type: 'unknownGroup' }); continue; } + if (this.isStale(acct, group.sourceId)) { + // Its shadow is behind the server: members built from it could undo server changes. The contact stays dirty. + this.env.report.itemError({ ref: group.sourceId as string, side: 'upload', type: 'stale', description: 'Waits until the server version could be stored' }); + continue; + } let pending = byGroup.get(group.sourceId as string); if (!pending) { const held: Held = { kind: this.groupKind, local: group }; From fad7b30dbf67de28dff85ea815a1558d3147fe5a Mon Sep 17 00:00:00 2001 From: Linus Rath <139418639+rathlinus@users.noreply.github.com> Date: Sun, 27 Sep 2026 07:44:47 +0200 Subject: [PATCH 44/87] fix: upload new contacts whose address book was deleted meanwhile #34 A new contact is claimed for an address book before its create is sent, so a retry after a lost response finds it again. When that book was deleted on the server before the create went up, every attempt failed and was retried after up to a day, forever. A book that is gone can hold nothing of an earlier attempt, so the contact is now claimed again for the book the settings choose. An event cannot move on its own to another calendar: a create into a calendar that is gone is no longer sent, and the report says why, until the user moves the event in the calendar app. --- .../__tests__/engine/real-planners.test.ts | 49 +++++++++++++++++++ src/device-sync/engine/calendar-sync.ts | 6 +++ src/device-sync/engine/contacts-sync.ts | 8 +++ src/device-sync/engine/item-sync.ts | 22 +++++++-- src/device-sync/engine/kinds.ts | 2 + 5 files changed, 84 insertions(+), 3 deletions(-) diff --git a/src/device-sync/__tests__/engine/real-planners.test.ts b/src/device-sync/__tests__/engine/real-planners.test.ts index a493d065..9a040a0d 100644 --- a/src/device-sync/__tests__/engine/real-planners.test.ts +++ b/src/device-sync/__tests__/engine/real-planners.test.ts @@ -235,6 +235,55 @@ describe('device sync engine with the real planners', () => { }); }); + describe('a create whose target the server deleted after the claim', () => { + it('claims a new contact again for a book that exists', async () => { + const h = real(); + const work = h.server.addAddressBook('a', { name: 'Work' }); + h.prefs.newContactsAddressBook = `a/${work}`; + await h.run(); + const id = addDeviceContact(h, 'Hedy Lamarr'); + // The claim names Work, the create is lost; then another client deletes Work. + h.server.failNextRequest('network', { match: 'ContactCard/set' }); + expect((await h.run()).outcome).toBe('io'); + h.server.serverDestroy('AddressBook', 'a', work); + + const report = await h.run(); + + expect(report).toMatchObject({ outcome: 'ok', itemErrors: [], stats: { uploaded: { created: 1 } } }); + const hedy = h.server.all('ContactCard', 'a').find((c) => (c.name as { full?: string }).full === 'Hedy Lamarr')!; + expect(hedy.addressBookIds).toEqual({ [h.book]: true }); + expect(h.contacts().find((c) => c.id === id)).toMatchObject({ sourceId: `a/${hedy.id}`, dirty: false }); + expect(h.contacts().find((c) => c.id === id)!.row.sync4).toBeNull(); + }); + + it('keeps a new event whose calendar is gone, without poisoning it, until the user moves it', async () => { + const h = real(); + const work = h.server.addCalendar('a', { name: 'Work' }); + await h.run(CALENDAR_AUTHORITY); + const rowOf = (calendarId: string) => Number(h.device.rows('calendars').find((c) => c._sync_id === `a/${calendarId}`)!._id); + const workRow = rowOf(work); + const dentist = { title: 'Dentist', dtstart: Date.UTC(2026, 9, 1, 8), dtend: Date.UTC(2026, 9, 1, 9), eventTimezone: 'Europe/Berlin' }; + const id = h.device.user.insertEvent(workRow, dentist); + h.server.failNextRequest('network', { match: 'CalendarEvent/set' }); + expect((await h.run(CALENDAR_AUTHORITY)).outcome).toBe('io'); + h.server.serverDestroy('Calendar', 'a', work); + + const report = await h.run(CALENDAR_AUTHORITY); + + expect(report.itemErrors).toEqual([expect.objectContaining({ ref: `row:${id}`, type: 'collectionGone' })]); + expect(h.server.all('CalendarEvent', 'a')).toEqual([]); + expect(h.events().find((e) => Number(e._id) === id)).toMatchObject({ title: 'Dentist', sync_data5: null }); + + // The user moves it to the personal calendar (Etar: delete + insert). + h.device.user.deleteEvent(id); + h.device.user.insertEvent(rowOf(h.calendar), dentist); + expect(await h.run(CALENDAR_AUTHORITY)).toMatchObject({ outcome: 'ok', itemErrors: [] }); + + expect(h.server.all('CalendarEvent', 'a').map((e) => [e.title, e.calendarIds])).toEqual([['Dentist', { [h.calendar]: true }]]); + expect(h.device.rows('calendars').map((c) => c._sync_id)).toEqual([`a/${h.calendar}`]); + }); + }); + describe('a device edit of an object the server moved out of every synced collection', () => { /** Ada with two numbers in the synced book; the device deletes the second in place (AOSP). */ async function adaWithDeletedNumber() { diff --git a/src/device-sync/engine/calendar-sync.ts b/src/device-sync/engine/calendar-sync.ts index e14d8264..fab6327c 100644 --- a/src/device-sync/engine/calendar-sync.ts +++ b/src/device-sync/engine/calendar-sync.ts @@ -257,6 +257,12 @@ export class CalendarSync extends ItemSync { await this.refreshCalendarRows(); } + protected collectionGone(key: string): boolean { + const parsed = parseCollectionKey(key); + const list = parsed ? this.calendars.get(parsed.accountId) : undefined; + return !!list && !list.some((c) => c.id === parsed!.id); + } + protected syncedCollections(acct: string): string[] { return (this.calendars.get(acct) ?? []) .filter((c) => this.syncedKeys.has(collectionKey(acct, c.id))) diff --git a/src/device-sync/engine/contacts-sync.ts b/src/device-sync/engine/contacts-sync.ts index 3c8660ba..228443e8 100644 --- a/src/device-sync/engine/contacts-sync.ts +++ b/src/device-sync/engine/contacts-sync.ts @@ -172,6 +172,7 @@ export class ContactsSync extends ItemSync { planBaselineHeal: (c) => planner.planBaselineHeal(c), planUpload: (c, acct) => planner.planUpload(c, this.ctx(acct)), planAccepted: (c, card, acct) => planner.planAccepted(c, card, this.ctx(acct)), + unclaimed: (c) => ({ ...c, pending: null }), }; } @@ -205,6 +206,7 @@ export class ContactsSync extends ItemSync { planBaselineHeal: () => null, planUpload: (g, acct) => planner.planGroupUpload(g, this.ctx(acct)), planAccepted: (g, card, acct) => planner.planGroupAccepted(g, card, this.ctx(acct)), + unclaimed: (g) => ({ ...g, pending: null }), }; } @@ -284,6 +286,12 @@ export class ContactsSync extends ItemSync { return !!book && book.myRights?.mayWrite !== false; } + protected collectionGone(key: string): boolean { + const parsed = parseCollectionKey(key); + const books = parsed ? this.books.get(parsed.accountId) : undefined; + return !!books && !books.some((b) => b.id === parsed!.id); + } + protected async collections(write: boolean): Promise { this.syncedKeys.clear(); for (const account of this.env.accounts) { diff --git a/src/device-sync/engine/item-sync.ts b/src/device-sync/engine/item-sync.ts index 84e9c254..4bda226a 100644 --- a/src/device-sync/engine/item-sync.ts +++ b/src/device-sync/engine/item-sync.ts @@ -155,6 +155,10 @@ export abstract class ItemSync { protected claimable(_held: Held): boolean { return true; } + /** Whether a collection key names a collection its account no longer has (listed this run). */ + protected collectionGone(_key: string): boolean { + return false; + } /** The JMAP account an item uploads to. */ protected accountOfItem(held: Held): string | null { const meta = held.kind.meta(held.local); @@ -792,12 +796,18 @@ export abstract class ItemSync { return this.setStates.get(acct) ?? null; } - /** New items get their uid and target written before anything is sent. */ + /** + * New items get their uid and target written before anything is sent. A + * claim whose collection is gone from the server is made again where the + * kind allows it (contacts: nothing of it can exist in a deleted book). + */ private async claimNewItems(): Promise { const works: Work[] = []; for (const held of await this.loadUploadItems()) { const meta = held.kind.meta(held.local); - if (!meta.isNew || meta.pending || meta.deleted || !this.claimable(held)) continue; + if (!meta.isNew || meta.deleted || !this.claimable(held)) continue; + const reclaim = !!meta.pending && !!held.kind.unclaimed && this.collectionGone(meta.pending.target); + if (meta.pending && !reclaim) continue; const ref = `row:${meta.rowId}`; const acct = this.claimAccount(held); if (!acct) { @@ -807,7 +817,7 @@ export abstract class ItemSync { } let plan; try { - plan = held.kind.planUpload(held.local, acct); + plan = held.kind.planUpload(reclaim ? held.kind.unclaimed!(held.local) : held.local, acct); } catch (error) { this.plannerFailed(acct, ref, 'upload', error); continue; @@ -915,6 +925,12 @@ export abstract class ItemSync { report.itemError({ ref, side: 'upload', type: 'stale', description: 'Waits until the server version could be stored' }); return null; } + if (meta.isNew && !meta.deleted && meta.pending && this.collectionGone(meta.pending.target)) { + // Nothing can be created there (an event stays in its calendar until the user moves it). + report.stats.skipped++; + report.itemError({ ref, side: 'upload', type: 'collectionGone', description: `${meta.pending.target} no longer exists on the server` }); + return null; + } const fingerprint = kind.fingerprint(local); if (meta.poison && isBackedOff(meta.poison, fingerprint, this.env.now())) { report.stats.skipped++; diff --git a/src/device-sync/engine/kinds.ts b/src/device-sync/engine/kinds.ts index 7ef6538f..10881a05 100644 --- a/src/device-sync/engine/kinds.ts +++ b/src/device-sync/engine/kinds.ts @@ -57,6 +57,8 @@ export interface Kind { planBaselineHeal(local: L): OpGroup | null; planUpload(local: L, jmapAccountId: string): UploadPlan; planAccepted(local: L, server: O, jmapAccountId: string): AcceptedPlan; + /** The item without its pending create, so `planUpload` claims it again; absent when the target can't change. */ + unclaimed?(local: L): L; } export function refOf(jmapAccountId: string, id: string): string { From 2e9e5bb7855ad96eaab75b0fd609de1fc66a5397 Mon Sep 17 00:00:00 2001 From: Linus Rath <139418639+rathlinus@users.noreply.github.com> Date: Sun, 27 Sep 2026 07:45:24 +0200 Subject: [PATCH 45/87] fix: remove items the server deleted right after this device created them #34 After an upload the stored state stays where the download left it, and the engine relied on /changes to report its own creates back. /changes leaves out an object that was created and destroyed after the state it is asked from (Stalwart does, RFC 8620 allows it), so an event or contact created on the device and deleted on the server before the next download stayed on the device for good. The SyncState now lists the objects our uploads created, written with their identity, and the next run fetches them first: one that is gone loses its rows, an unchanged one writes nothing. An upload-only sync runs in full while the list is not empty. --- .../__tests__/engine/own-creates.test.ts | 77 +++++++++++++++++++ src/device-sync/engine/item-sync.ts | 42 ++++++++-- src/device-sync/engine/run.ts | 2 +- src/device-sync/engine/sync-state.ts | 6 ++ 4 files changed, 121 insertions(+), 6 deletions(-) create mode 100644 src/device-sync/__tests__/engine/own-creates.test.ts diff --git a/src/device-sync/__tests__/engine/own-creates.test.ts b/src/device-sync/__tests__/engine/own-creates.test.ts new file mode 100644 index 00000000..76eb8c94 --- /dev/null +++ b/src/device-sync/__tests__/engine/own-creates.test.ts @@ -0,0 +1,77 @@ +// Objects our uploads created. The stored state stays where the download left +// it, and `/changes` omits an object created and destroyed after its +// `sinceState` (Stalwart does, RFC 8620 allows it): such an object must still +// leave the device when another client destroys it before the next download. + +import { describe, expect, it } from 'vitest'; +import { calendarPlanner } from '../../calendar/planner'; +import { contactsPlanner } from '../../contacts/planner'; +import { addDeviceContact, addServerCards, CALENDAR_AUTHORITY, createHarness, type Harness } from './harness'; + +function real(): Harness { + const h = createHarness(); + h.deps.planners = { contacts: contactsPlanner, calendar: calendarPlanner }; + return h; +} + +describe('device sync engine: objects our uploads created', () => { + it('removes a device-created contact that another client destroyed before the next download', async () => { + const h = real(); + addServerCards(h, ['Ada Lovelace']); + await h.run(); + const id = addDeviceContact(h, 'Hedy Lamarr'); + expect((await h.run()).stats.uploaded.created).toBe(1); + const hedy = h.contacts().find((c) => c.id === id)!.sourceId!.split('/')[1]; + h.server.serverDestroy('ContactCard', 'a', hedy); + + const report = await h.run(); + + expect(report).toMatchObject({ outcome: 'ok', stats: { downloaded: { deleted: 1 } } }); + expect(h.contacts().map((c) => c.name)).toEqual(['Ada Lovelace']); + expect(h.state()?.accounts.a.created ?? []).toEqual([]); + }); + + it('removes a device-created event that another client destroyed before the next download', async () => { + const h = real(); + await h.run(CALENDAR_AUTHORITY); + const rowId = Number(h.device.rows('calendars')[0]._id); + const id = h.device.user.insertEvent(rowId, { title: 'Dentist', dtstart: Date.UTC(2026, 9, 1, 8), dtend: Date.UTC(2026, 9, 1, 9), eventTimezone: 'Europe/Berlin' }); + expect((await h.run(CALENDAR_AUTHORITY)).stats.uploaded.created).toBe(1); + const created = String(h.events().find((e) => Number(e._id) === id)!._sync_id).split('/')[1]; + h.server.serverDestroy('CalendarEvent', 'a', created); + + const report = await h.run(CALENDAR_AUTHORITY); + + expect(report).toMatchObject({ outcome: 'ok', stats: { downloaded: { deleted: 1 } } }); + expect(h.events()).toEqual([]); + }); + + it('fetches a created object once, and writes nothing when it is unchanged', async () => { + const h = real(); + await h.run(); + addDeviceContact(h, 'Hedy Lamarr'); + await h.run(); + expect(h.state()?.accounts.a.created).toHaveLength(1); + + const before = h.batches.log.length; + expect((await h.run()).outcome).toBe('ok'); + + expect(h.state()?.accounts.a.created).toEqual([]); + expect(h.batches.log.slice(before).flat().filter((op) => op.op !== 'syncState')).toEqual([]); + expect(h.contacts().map((c) => c.name)).toEqual(['Hedy Lamarr']); + }); + + it('turns an upload-only sync into a full one while a created object waits to be checked', async () => { + const h = real(); + await h.run(); + const id = addDeviceContact(h, 'Hedy Lamarr'); + await h.run(); + const hedy = h.contacts().find((c) => c.id === id)!.sourceId!.split('/')[1]; + h.server.serverDestroy('ContactCard', 'a', hedy); + + // Android's upload sync 30 s after an app wrote anything, with nothing dirty in this account. + expect((await h.run(undefined, { upload: true })).outcome).toBe('ok'); + + expect(h.contacts()).toEqual([]); + }); +}); diff --git a/src/device-sync/engine/item-sync.ts b/src/device-sync/engine/item-sync.ts index 4bda226a..b0eae42d 100644 --- a/src/device-sync/engine/item-sync.ts +++ b/src/device-sync/engine/item-sync.ts @@ -44,6 +44,19 @@ function both(first: StateChange | undefined, second: StateChange | undefined): }; } +/** The work with one more state change, also after a re-plan. */ +function withState(work: Work, change: StateChange): Work { + const replan = work.replan; + return { + ...work, + state: both(work.state, change), + replan: replan && (async () => { + const next = await replan(); + return next && withState(next, change); + }), + }; +} + /** An item's upload, on its way to the server. */ export interface Pending { held: Held; @@ -324,10 +337,20 @@ export abstract class ItemSync { // ── Downloads ── - /** Objects put on `stale` by an earlier run: fetched again first (invariant 10). */ + /** + * Objects put on `stale` by an earlier run (invariant 10), and objects our + * uploads created since the stored state (`/changes` from it omits one that + * was created and destroyed since): fetched again first. + */ protected async refetchStale(acct: string): Promise { - const ids = [...this.env.store.account(acct).stale]; - if (ids.length) await this.processIds(acct, ids, [], () => undefined); + const account = this.env.store.account(acct); + const created = account.created ?? []; + const ids = [...new Set([...account.stale, ...created])]; + if (!ids.length) return; + await this.processIds(acct, ids, [], (next) => { + const a = accountOf(next, acct); + a.created = (a.created ?? []).filter((id) => !created.includes(id)); + }); } /** `/changes` pages from the stored state; a full reconcile without one or after `cannotCalculateChanges`. */ @@ -1070,7 +1093,8 @@ export abstract class ItemSync { } await this.beforePlanning(acct, [object]); const work = this.downloadWork(p.held.kind, acct, object, p.held.local, null); - return work ? [work] : []; + // Our own earlier create: remembered like one of this run. + return work ? [withState(work, this.rememberCreated(acct, id))] : []; } /** Sends the items' actions in `/set` calls and settles each item once all its actions have an answer. */ @@ -1319,7 +1343,7 @@ export abstract class ItemSync { this.markStale(acct, server.id); this.env.report.itemError({ ref: refOf(acct, server.id), side: 'upload', type: reason, description: message }); }; - const state = this.listing(kind, acct, server.id, true); + const state = both(this.listing(kind, acct, server.id, true), created ? this.rememberCreated(acct, server.id) : undefined); return { group: withExtra(plan.ops), state, @@ -1329,6 +1353,14 @@ export abstract class ItemSync { }; } + /** An object our upload created is remembered with its identity until the next run fetched it (see `refetchStale`). */ + private rememberCreated(acct: string, id: string): StateChange { + return (next) => { + const account = accountOf(next, acct); + account.created = [...new Set([...(account.created ?? []), id])]; + }; + } + private poisonWork(p: Pending, error: SetErrorShape): Work { const marker = nextMarker(p.meta.poison, p.fingerprint, error, this.env.now(), { firstMs: this.env.tuning.poisonBackoffMs, diff --git a/src/device-sync/engine/run.ts b/src/device-sync/engine/run.ts index 39c06178..883be9c5 100644 --- a/src/device-sync/engine/run.ts +++ b/src/device-sync/engine/run.ts @@ -48,7 +48,7 @@ function itemTypeOf(authority: Authority): string { /** Rows waiting for an upload (dirty, deleted, new), or state work a run must do whatever its extras. */ export async function hasLocalWork(reader: ProviderReader, authority: Authority, state: SyncState): Promise { for (const account of Object.values(state.accounts)) { - if (account.reconcile || account.stale.length) return true; + if (account.reconcile || account.stale.length || account.created?.length) return true; } if (authority === CALENDAR_AUTHORITY) { const rows = await reader.rows('events', ['_id'], "dirty = 1 OR deleted = 1 OR _sync_id IS NULL OR _sync_id LIKE '~pending/%'"); diff --git a/src/device-sync/engine/sync-state.ts b/src/device-sync/engine/sync-state.ts index 46e1e73a..2d07f9ab 100644 --- a/src/device-sync/engine/sync-state.ts +++ b/src/device-sync/engine/sync-state.ts @@ -29,6 +29,11 @@ export interface AccountState { selected: string[]; /** Object ids whose rows could not be written; fetched again at the start of every run, nothing uploads for them meanwhile. */ stale: string[]; + /** + * Objects our uploads created after `itemsState`: fetched again at the start of the next run, since `/changes` + * omits an object created and destroyed after its `sinceState`. + */ + created?: string[]; reconcile: ReconcileMarker | null; /** Contacts: refs of the group cards present on the device (a hard-deleted group is found by its absence). */ groups?: string[]; @@ -81,6 +86,7 @@ function parseAccount(value: unknown): AccountState { stale: strings(a.stale), reconcile: parseReconcile(a.reconcile), }; + if (a.created !== undefined) out.created = strings(a.created); if (a.groups !== undefined) out.groups = strings(a.groups); if (a.taskOnly !== undefined) out.taskOnly = strings(a.taskOnly); return out; From 7701a67dff89b50bd855087560d03d97ff09af65 Mon Sep 17 00:00:00 2001 From: Linus Rath <139418639+rathlinus@users.noreply.github.com> Date: Sun, 27 Sep 2026 07:46:01 +0200 Subject: [PATCH 46/87] fix: keep favourites and joins when the server refuses an edit #34 When the server refused an edit made on the device (a book or calendar that turned read-only, an event in a calendar where only one's own events may be changed), the engine put the item back by deleting its rows and inserting a fresh copy. A contact lost its star, ringtone and the link to the contacts it was joined with; an event got a new row id. The rows are now rewritten in place from the server's version, as after an accepted upload. A refused deletion still brings the item back as a new copy. --- .../__tests__/engine/real-planners.test.ts | 57 +++++++++++++++++++ src/device-sync/engine/item-sync.ts | 19 +++++++ 2 files changed, 76 insertions(+) diff --git a/src/device-sync/__tests__/engine/real-planners.test.ts b/src/device-sync/__tests__/engine/real-planners.test.ts index 9a040a0d..03291e0b 100644 --- a/src/device-sync/__tests__/engine/real-planners.test.ts +++ b/src/device-sync/__tests__/engine/real-planners.test.ts @@ -13,6 +13,7 @@ import { CALENDAR_AUTHORITY, CONTACTS_AUTHORITY, createHarness, + renameDeviceContact, rowWrites, type Harness, } from './harness'; @@ -235,6 +236,62 @@ describe('device sync engine with the real planners', () => { }); }); + describe('a device edit the server refuses (forbidden)', () => { + it('puts a contact back in place, keeping what only the device knows', async () => { + const h = real(); + const [ada] = addServerCards(h, ['Ada Lovelace']); + await h.run(); + const before = h.contactNamed('Ada Lovelace')!; + h.device.user.star(before.id); + renameDeviceContact(h, before.id, 'Ada King'); + h.server.setErrorFor('ContactCard', 'a', ada, { type: 'forbidden' }); + + const report = await h.run(); + + expect(report.itemErrors).toEqual([expect.objectContaining({ ref: `a/${ada}`, type: 'forbidden' })]); + expect(h.server.get('ContactCard', 'a', ada)).toMatchObject({ name: { full: 'Ada Lovelace' } }); + expect(h.contacts()).toEqual([expect.objectContaining({ id: before.id, name: 'Ada Lovelace', dirty: false })]); + expect(h.contacts()[0].row.starred).toBe(1); + }); + + it('brings a contact back whose deletion the server refuses', async () => { + const h = real(); + const [ada] = addServerCards(h, ['Ada Lovelace']); + await h.run(); + h.device.user.deleteContact(h.contactNamed('Ada Lovelace')!.id); + h.server.setErrorFor('ContactCard', 'a', ada, { type: 'forbidden' }); + + const report = await h.run(); + + expect(report.itemErrors).toEqual([expect.objectContaining({ ref: `a/${ada}`, type: 'forbidden' })]); + expect(h.server.get('ContactCard', 'a', ada)).toBeDefined(); + expect(h.contacts()).toEqual([expect.objectContaining({ sourceId: `a/${ada}`, name: 'Ada Lovelace', dirty: false, deleted: false })]); + }); + + it('puts an event back in place', async () => { + const h = real(); + const standup = h.server.addEvent('a', { + uid: 'standup-uid', + title: 'Standup', + start: '2026-09-28T09:00:00', + duration: 'PT15M', + timeZone: 'Europe/Berlin', + calendarIds: { [h.calendar]: true }, + }); + await h.run(CALENDAR_AUTHORITY); + const row = h.events()[0]; + h.device.user.updateEvent(Number(row._id), { [Events.TITLE]: 'Mine', [Events.CUSTOM_APP_PACKAGE]: 'org.example.app' }); + h.server.setErrorFor('CalendarEvent', 'a', standup, { type: 'forbidden' }); + + const report = await h.run(CALENDAR_AUTHORITY); + + expect(report.itemErrors).toEqual([expect.objectContaining({ ref: `a/${standup}`, type: 'forbidden' })]); + expect(h.events()).toEqual([ + expect.objectContaining({ _id: row._id, title: 'Standup', dirty: 0, [Events.CUSTOM_APP_PACKAGE]: 'org.example.app' }), + ]); + }); + }); + describe('a create whose target the server deleted after the claim', () => { it('claims a new contact again for a book that exists', async () => { const h = real(); diff --git a/src/device-sync/engine/item-sync.ts b/src/device-sync/engine/item-sync.ts index b0eae42d..ee169339 100644 --- a/src/device-sync/engine/item-sync.ts +++ b/src/device-sync/engine/item-sync.ts @@ -1390,6 +1390,8 @@ export abstract class ItemSync { }]; } await this.beforePlanning(acct, [object]); + // An edit is undone in place, so what only the device keeps stays: a star, a ringtone, a joined contact, row ids. + if (objectKind === kind && !p.meta.deleted) return [this.overwriteWork(acct, kind, local, object)]; const insert = objectKind.planDownload(object, null, acct); return [{ group: concatGroups(ref, remove, insert.ops), @@ -1398,6 +1400,23 @@ export abstract class ItemSync { }]; } + /** The rows become `object` where they are and DIRTY is cleared (the planner's write of an accepted version). */ + private overwriteWork(acct: string, kind: Kind, local: unknown, object: ServerObject): Work { + return { + group: kind.planAccepted(local, object, acct).ops, + state: this.listing(kind, acct, object.id, true), + applied: () => this.onDownloaded(kind, acct, object), + replan: async () => { + const again = await this.reload(kind, local); + return again === null ? null : this.overwriteWork(acct, kind, again, object); + }, + failed: (reason, message) => { + this.markStale(acct, object.id); + this.env.report.itemError({ ref: refOf(acct, object.id), side: 'upload', type: reason, description: message }); + }, + }; + } + /** * `invalidProperties` on `uid`: the object exists (a lost response, or the * client's automatic retry of the create). Cards name it in the error; From 474eae9992a9f72856015aa5fef267c42f654758 Mon Sep 17 00:00:00 2001 From: Linus Rath <139418639+rathlinus@users.noreply.github.com> Date: Sun, 27 Sep 2026 07:46:39 +0200 Subject: [PATCH 47/87] fix: never duplicate an event's reminders after a busy calendar database #34 A provider that is busy may commit a batch up to a yield point and then fail it. The engine retried every group of such a batch and relied on the groups' asserts to refuse one that was already committed, but a download into an unchanged event asserts only the event's own columns: when it had only added a reminder or an attendee, the retry added it a second time. A download is now planned again from a new read after a failed batch instead of being sent again, which turns a committed group into an echo. The fake provider can commit a batch up to a yield point to test it. --- .../__tests__/engine/internals.test.ts | 12 ++++++++++ .../__tests__/engine/real-planners.test.ts | 23 ++++++++++++++++++ .../__tests__/fakes/fake-provider.ts | 24 +++++++++++++++++++ src/device-sync/engine/batch.ts | 24 +++++++++++++++---- src/device-sync/engine/item-sync.ts | 1 + 5 files changed, 79 insertions(+), 5 deletions(-) diff --git a/src/device-sync/__tests__/engine/internals.test.ts b/src/device-sync/__tests__/engine/internals.test.ts index 9609e1b4..4fee104b 100644 --- a/src/device-sync/__tests__/engine/internals.test.ts +++ b/src/device-sync/__tests__/engine/internals.test.ts @@ -157,6 +157,18 @@ describe('provider batches', () => { expect(store.committed.accounts.a.groups).toEqual(['a/g1']); }); + it('plans a download again after a failed batch, which may have been committed up to a yield point', async () => { + let failures = 1; + const p = port((ops) => (failures-- > 0 ? { ok: false, reason: 'provider', message: 'failed after a yield point' } : ok(ops))); + const planned: Work = { group: { ref: 'a/1', ops: [{ op: 'update', table: 'events', id: 1, values: { title: 'Planned from the new read' } }] } }; + const download: Work = { group: { ref: 'a/1', ops: [insert('data', { stale: 'group' })] }, fresh: true, replan: async () => planned }; + const other: Work = { group: { ref: 'a/2', ops: [insert('raw_contacts')] } }; + + await writer(p).write([download, other]); + + expect(p.sent.slice(1)).toEqual([planned.group.ops.map((op) => ({ ...op, yieldAllowed: true })), buildBatch([other.group])]); + }); + it('skips groups that only assert', async () => { const p = port(ok); await writer(p).write([{ group: { ref: 'g', ops: [{ op: 'assert', table: 'raw_contacts', id: 1, values: { dirty: 0 } }] } }]); diff --git a/src/device-sync/__tests__/engine/real-planners.test.ts b/src/device-sync/__tests__/engine/real-planners.test.ts index 03291e0b..bf494cdd 100644 --- a/src/device-sync/__tests__/engine/real-planners.test.ts +++ b/src/device-sync/__tests__/engine/real-planners.test.ts @@ -236,6 +236,29 @@ describe('device sync engine with the real planners', () => { }); }); + describe('a batch the provider committed up to a yield point before it failed', () => { + const alert = { al1: { '@type': 'Alert', trigger: { '@type': 'OffsetTrigger', offset: '-PT15M' }, action: 'display' } }; + + it("never inserts a clean event's new reminder twice", async () => { + const h = real(); + const event = (uid: string, title: string) => + h.server.addEvent('a', { uid, title, start: '2026-10-06T12:00:00', duration: 'PT1H', timeZone: 'Europe/Berlin', calendarIds: { [h.calendar]: true } }); + const lunch = event('lunch-uid', 'Lunch'); + const retro = event('retro-uid', 'Retro'); + await h.run(CALENDAR_AUTHORITY); + // Only an alert for Lunch (its row's columns stay), a new title for Retro: one download batch, two groups. + h.server.serverUpdate('CalendarEvent', 'a', lunch, { alerts: alert }); + h.server.serverUpdate('CalendarEvent', 'a', retro, { title: 'Retrospective' }); + h.device.partialCommitNextBatch(1, 'assert', (ops) => ops.some((op) => op.op === 'insert' && op.table === 'reminders')); + + expect((await h.run(CALENDAR_AUTHORITY)).outcome).toBe('ok'); + + const row = h.events().find((e) => e.title === 'Lunch')!; + expect(h.device.rows('reminders').filter((r) => Number(r.event_id) === Number(row._id)).map((r) => r.minutes)).toEqual([15]); + expect(h.events().map((e) => e.title).sort()).toEqual(['Lunch', 'Retrospective']); + }); + }); + describe('a device edit the server refuses (forbidden)', () => { it('puts a contact back in place, keeping what only the device knows', async () => { const h = real(); diff --git a/src/device-sync/__tests__/fakes/fake-provider.ts b/src/device-sync/__tests__/fakes/fake-provider.ts index 8fbad6e2..100d5229 100644 --- a/src/device-sync/__tests__/fakes/fake-provider.ts +++ b/src/device-sync/__tests__/fakes/fake-provider.ts @@ -258,6 +258,7 @@ export class FakeDeviceProviders { private nextId = 1; private nextPhotoFile = 1000; private pendingFailure: { reason: BatchFailure; message: string } | null = null; + private partialFailure: { groups: number; reason: BatchFailure; when: (ops: ProviderOp[]) => boolean } | null = null; private beforeBatch: Array<() => void> = []; /** Every batch applied or refused, for assertions on write counts. */ readonly batches: Array<{ account: string; authority: Authority; ops: ProviderOp[]; ok: boolean }> = []; @@ -301,6 +302,16 @@ export class FakeDeviceProviders { this.beforeBatch.push(fn); } + /** + * The next applyBatch that `when` matches behaves like a provider that + * yielded under contention: the ops before its `groups + 1`-th yield point + * (its first `groups` groups) are committed, then the batch fails with + * `reason` and the rest is rolled back. + */ + partialCommitNextBatch(groups: number, reason: BatchFailure = 'assert', when: (ops: ProviderOp[]) => boolean = () => true): void { + this.partialFailure = { groups, reason, when }; + } + /** Removes an account the way AccountManager does: the providers drop all its rows and SyncState. */ removeAccount(accountName: string): void { for (const table of ['raw_contacts', 'groups', 'settings', 'calendars', 'colors'] as const) { @@ -517,6 +528,19 @@ export class FakeDeviceProviders { } for (const hook of this.beforeBatch.splice(0)) hook(); + if (this.partialFailure?.when(ops)) { + const { groups, reason } = this.partialFailure; + this.partialFailure = null; + const starts = ops.flatMap((op, i) => (i === 0 || (op as { yieldAllowed?: boolean }).yieldAllowed ? [i] : [])); + const cut = starts[groups] ?? 0; + if (cut > 0) { + const committed = this.applyBatch(accountName, authority, ops.slice(0, cut)); + if (!committed.ok) return committed; + } + record(false); + return { ok: false, reason, message: `injected ${reason} after a yield point` }; + } + const snapshot = { tables: cloneTables(this.tables), syncState: new Map(this.syncState), diff --git a/src/device-sync/engine/batch.ts b/src/device-sync/engine/batch.ts index 2375b86f..a7a454ea 100644 --- a/src/device-sync/engine/batch.ts +++ b/src/device-sync/engine/batch.ts @@ -11,10 +11,12 @@ * and never a yield point: the stored state can't get ahead of its rows. * * A failed batch may have been committed up to a yield point the provider - * took, so its groups are retried one by one (their asserts make a retry of - * a committed group fail harmlessly); a group that fails an assert is - * re-read and re-planned up to `maxReplans` times, then given up. `tooLarge` - * splits the batch; `permission` ends the run. + * took, so its groups are retried one by one: a group planned from the rows + * as they are (`fresh`, a download) is planned again from a new read, any + * other is sent again (its asserts make a retry of a committed group fail); + * a group that fails an assert is re-read and re-planned up to `maxReplans` + * times, then given up. `tooLarge` splits the batch; `permission` ends the + * run. * * A group whose rows the SyncState describes (a group row: `groups`) carries * that `state` change: every batch that applies such a group, a retry @@ -36,6 +38,11 @@ export interface Work { failed?(reason: BatchFailure, message: string): void | Promise; /** What the SyncState lists changes with these rows: stored by the batch that applies them. */ state?: StateChange; + /** + * Planned from the rows as they are (a download), so `replan` gives the same group or its echo: after a failed + * batch it is planned again instead of sent again (its asserts need not tell a committed group). + */ + fresh?: boolean; } export interface BatchWriterOptions { @@ -210,7 +217,14 @@ export class BatchWriter { await this.retryAlone(works[0], result.reason, result.message); return; } - for (const work of works) await this.applyAlone(work, 0); + for (const work of works) { + if (work.fresh && work.replan) { + const next = await work.replan(); + if (next) await this.applyAlone(next, 1); + } else { + await this.applyAlone(work, 0); + } + } if (tail) await this.applyTail(tail); } diff --git a/src/device-sync/engine/item-sync.ts b/src/device-sync/engine/item-sync.ts index ee169339..6b4c4c2c 100644 --- a/src/device-sync/engine/item-sync.ts +++ b/src/device-sync/engine/item-sync.ts @@ -642,6 +642,7 @@ export abstract class ItemSync { return { group, state: this.listing(kind, acct, object.id, true), + fresh: true, applied: () => { if (plan.effect === 'insert') this.env.report.stats.downloaded.created++; else this.env.report.stats.downloaded.updated++; From ecbd218c346ef58bd2b96fd2ce21dcd283d7d138 Mon Sep 17 00:00:00 2001 From: Linus Rath <139418639+rathlinus@users.noreply.github.com> Date: Sun, 27 Sep 2026 07:47:17 +0200 Subject: [PATCH 48/87] fix: sync only the primary account's collections by default #34 The settings chooser and the reminder filter treat only the capability's primary account as the user's own, while the engine also counted any account the server marks as personal. On a server that lists a second such account, its address books and calendars synced although the chooser showed them off. The engine now uses the same rule as the app. --- src/device-sync/__tests__/engine/internals.test.ts | 13 +++++++++++++ src/device-sync/jmap/session.ts | 8 ++++++-- 2 files changed, 19 insertions(+), 2 deletions(-) diff --git a/src/device-sync/__tests__/engine/internals.test.ts b/src/device-sync/__tests__/engine/internals.test.ts index 4fee104b..07ec9348 100644 --- a/src/device-sync/__tests__/engine/internals.test.ts +++ b/src/device-sync/__tests__/engine/internals.test.ts @@ -297,6 +297,19 @@ describe('JMAP side', () => { expect(accountsWithCapability(session, JMAP_CALENDARS).map((a) => a.id)).toEqual(['a']); }); + it("counts only the capability's primary account as personal, like the app's chooser", () => { + const server = new FakeJmapServer(); + server.addAccount('a', { name: 'alice@example.com' }); + // Another account the server calls personal (a second mailbox of the same login, say). + server.addAccount('home', { name: 'alice-home@example.com', isPersonal: true }); + const session = server.port().session(); + + expect(accountsWithCapability(session, JMAP_CONTACTS).map((a) => [a.id, a.primary, a.personal])).toEqual([ + ['a', true, true], + ['home', false, false], + ]); + }); + it('takes the login as the owner address and survives a server without identities', async () => { const server = new FakeJmapServer(); server.addAccount('a', { name: 'Alice@Example.com' }); diff --git a/src/device-sync/jmap/session.ts b/src/device-sync/jmap/session.ts index caf93550..b6d6743b 100644 --- a/src/device-sync/jmap/session.ts +++ b/src/device-sync/jmap/session.ts @@ -12,7 +12,11 @@ export interface CapabilityAccount { name: string; /** The capability's primary account (the user's own). */ primary: boolean; - /** Collections of personal accounts are selected by default, shared ones are not. */ + /** + * Collections of the personal account are selected by default, those of the others are not. It is the + * capability's primary account only, whatever `isPersonal` says: one rule with the app's chooser and + * reminder filter, which know only the primary account per authority. + */ personal: boolean; readOnly: boolean; } @@ -31,7 +35,7 @@ export function accountsWithCapability(session: JmapSessionView, capability: str id, name: account.name || id, primary: isPrimary, - personal: isPrimary || account.isPersonal, + personal: isPrimary, readOnly: !!account.isReadOnly, }); }; From 2e1cb5854806322e142d2402e6bfd4278aed5769 Mon Sep 17 00:00:00 2001 From: Linus Rath <139418639+rathlinus@users.noreply.github.com> Date: Sun, 27 Sep 2026 07:47:55 +0200 Subject: [PATCH 49/87] fix: finish loading or removing a book after an interrupted sync #34 The SyncState's `selected` said which collections were on the device, and changed only when a load or a removal had finished. A book turned off and on again while the device was still removing it looked loaded, so the contacts removed so far never came back; a calendar came back empty. A book turned on and off again while it was still loading never looked loaded, so the contacts loaded so far were never removed. `selected` now names only collections whose rows are all on the device: a collection leaves it before its first row is removed, and the contacts SyncState lists collections that are loaded or removed only in part, so the next run loads them again when they are selected, or removes the rest when they are not. A book that still holds a change waiting for its upload stays in that list instead of `selected`. --- .../__tests__/engine/calendar-flow.test.ts | 9 +- .../__tests__/engine/selection-state.test.ts | 143 ++++++++++++++++++ src/device-sync/engine/calendar-sync.ts | 22 ++- src/device-sync/engine/contacts-sync.ts | 19 ++- src/device-sync/engine/item-sync.ts | 15 +- src/device-sync/engine/sync-state.ts | 8 +- 6 files changed, 201 insertions(+), 15 deletions(-) create mode 100644 src/device-sync/__tests__/engine/selection-state.test.ts diff --git a/src/device-sync/__tests__/engine/calendar-flow.test.ts b/src/device-sync/__tests__/engine/calendar-flow.test.ts index 9025737e..8f391630 100644 --- a/src/device-sync/__tests__/engine/calendar-flow.test.ts +++ b/src/device-sync/__tests__/engine/calendar-flow.test.ts @@ -125,7 +125,14 @@ describe('device sync engine: calendar', () => { expect(report.itemErrors).toEqual([expect.objectContaining({ ref: `a/${review}`, type: 'tooLarge' })]); expect(calendarRow(h, work)).toBeDefined(); expect(titles(h)).toEqual(['Code review']); - expect(h.state(CALENDAR_AUTHORITY)?.accounts.a.selected).toEqual([`a/${h.calendar}`, `a/${work}`]); + // Not complete any more (selected again, it is loaded again); its row is dropped again next run. + expect(h.state(CALENDAR_AUTHORITY)?.accounts.a.selected).toEqual([`a/${h.calendar}`]); + + h.clock.now += 2 * 3_600_000; + await h.run(CALENDAR_AUTHORITY); + + expect(h.server.get('CalendarEvent', 'a', review)).toMatchObject({ title: 'Code review' }); + expect(calendarRow(h, work)).toBeUndefined(); }); it('marks a calendar that mirrors a feed subscription read-only', async () => { diff --git a/src/device-sync/__tests__/engine/selection-state.test.ts b/src/device-sync/__tests__/engine/selection-state.test.ts new file mode 100644 index 00000000..01c683ed --- /dev/null +++ b/src/device-sync/__tests__/engine/selection-state.test.ts @@ -0,0 +1,143 @@ +// SyncState `selected` names the collections whose rows are all on the +// device: a newly selected collection is loaded, a deselected one dropped. +// A load or a drop that stopped half-way must neither hide the collection +// from the next load nor keep its rows from the next drop. + +import { describe, expect, it } from 'vitest'; +import { Events } from '../../android-columns'; +import { calendarPlanner } from '../../calendar/planner'; +import { contactsPlanner } from '../../contacts/planner'; +import { addServerCards, CALENDAR_AUTHORITY, createHarness, RUN_BUDGET, type Harness } from './harness'; + +function real(): Harness { + const h = createHarness(); + h.deps.planners = { contacts: contactsPlanner, calendar: calendarPlanner }; + return h; +} + +/** Runs out of time right after checkpoint `n` of the next run. */ +function outOfTimeAt(h: Harness, n: number): void { + h.checkpoints.count = 0; + h.checkpoints.onCheckpoint = (count) => { + if (count === n) h.clock.now += RUN_BUDGET; + }; +} + +describe('device sync engine: collections loaded or dropped in part', () => { + it('brings every card back when a book whose drop stopped half-way is selected again', async () => { + const h = real(); + const work = h.server.addAddressBook('a', { name: 'Work' }); + addServerCards(h, ['Ada']); + addServerCards(h, Array.from({ length: 120 }, (_, i) => `W${String(i).padStart(3, '0')}`), 'a', work); + await h.run(); + expect(h.contacts()).toHaveLength(121); + h.prefs.contactsSelection[`a/${work}`] = false; + // 'changes', 'deselect' (sync), 'deselect' (chunk 1): out of time after the first chunk went. + outOfTimeAt(h, 3); + expect((await h.run()).outcome).toBe('cancelled'); + h.checkpoints.onCheckpoint = undefined; + const left = h.contacts().length; + expect(left).toBeGreaterThan(1); + expect(left).toBeLessThan(121); + + h.prefs.contactsSelection[`a/${work}`] = true; + expect((await h.run()).outcome).toBe('ok'); + + expect(h.contacts()).toHaveLength(121); + expect(h.state()?.accounts.a.selected.sort()).toEqual([`a/${h.book}`, `a/${work}`].sort()); + }); + + it('finishes the drop of a book in the next run when it stopped half-way', async () => { + const h = real(); + const work = h.server.addAddressBook('a', { name: 'Work' }); + addServerCards(h, ['Ada']); + addServerCards(h, Array.from({ length: 120 }, (_, i) => `W${String(i).padStart(3, '0')}`), 'a', work); + await h.run(); + h.prefs.contactsSelection[`a/${work}`] = false; + outOfTimeAt(h, 3); + expect((await h.run()).outcome).toBe('cancelled'); + h.checkpoints.onCheckpoint = undefined; + + expect((await h.run()).outcome).toBe('ok'); + + expect(h.contacts().map((c) => c.name)).toEqual(['Ada']); + expect(h.state()?.accounts.a.selected).toEqual([`a/${h.book}`]); + }); + + it('keeps a deselected book partial while a dirty card of it waits, and drops it once that is uploaded', async () => { + const h = real(); + const work = h.server.addAddressBook('a', { name: 'Work' }); + addServerCards(h, ['Ada']); + const [boss] = addServerCards(h, ['Boss', 'Colleague'], 'a', work); + await h.run(); + const bossRow = h.contactNamed('Boss')!.id; + const name = h.nameDataRow(bossRow)!; + h.device.user.updateData(Number(name._id), { data1: 'The Boss' }); + h.server.setErrorFor('ContactCard', 'a', boss, { type: 'tooLarge' }); + h.prefs.contactsSelection[`a/${work}`] = false; + + await h.run(); + + expect(h.contacts().map((c) => c.name).sort()).toEqual(['Ada', 'The Boss']); + expect(h.state()?.accounts.a).toMatchObject({ selected: [`a/${h.book}`], partial: [`a/${work}`] }); + + h.clock.now += 2 * 3_600_000; + await h.run(); + + expect(h.server.get('ContactCard', 'a', boss)).toMatchObject({ name: { full: 'The Boss' } }); + expect(h.contacts().map((c) => c.name)).toEqual(['Ada']); + expect(h.state()?.accounts.a).toMatchObject({ selected: [`a/${h.book}`], partial: [] }); + }); + + it('drops the cards of a book deselected again while its load had stopped half-way', async () => { + const h = real(); + const work = h.server.addAddressBook('a', { name: 'Work' }); + h.prefs.contactsSelection[`a/${work}`] = false; + addServerCards(h, ['Ada']); + addServerCards(h, Array.from({ length: 120 }, (_, i) => `W${String(i).padStart(3, '0')}`), 'a', work); + await h.run(); + expect(h.contacts()).toHaveLength(1); + h.prefs.contactsSelection[`a/${work}`] = true; + // 'changes', 'select', 'download' (chunk 1): out of time after the first chunk came. + outOfTimeAt(h, 3); + expect((await h.run()).outcome).toBe('cancelled'); + h.checkpoints.onCheckpoint = undefined; + expect(h.contacts().length).toBeGreaterThan(1); + + h.prefs.contactsSelection[`a/${work}`] = false; + expect((await h.run()).outcome).toBe('ok'); + + expect(h.contacts().map((c) => c.name)).toEqual(['Ada']); + }); + + it('loads the events of a calendar dropped by a stopped run when it is selected again', async () => { + const h = real(); + const holidays = h.server.addCalendar('a', { name: 'Holidays' }); + const sports = h.server.addCalendar('a', { name: 'Sports' }); + for (const [calendar, title] of [[holidays, 'Easter'], [holidays, 'Christmas'], [sports, 'Match']] as const) { + h.server.addEvent('a', { uid: `u-${title}`, title, start: '2026-10-01T09:00:00', duration: 'PT1H', timeZone: 'Europe/Berlin', calendarIds: { [calendar]: true } }); + } + await h.run(CALENDAR_AUTHORITY); + expect(h.events()).toHaveLength(3); + // Both deselected; the run is cancelled once the first calendar's rows are gone. + h.prefs.calendarSelection[`a/${holidays}`] = false; + h.prefs.calendarSelection[`a/${sports}`] = false; + const original = h.deps.isCancelled; + let cancelled = false; + h.deps.isCancelled = async (runId) => { + const left = h.events().length; + if (!cancelled && left > 0 && left < 3) cancelled = true; + return cancelled || original(runId); + }; + expect((await h.run(CALENDAR_AUTHORITY)).outcome).toBe('cancelled'); + h.deps.isCancelled = original; + const titles = () => h.events().map((e) => String(e[Events.TITLE])).sort(); + const dropped = titles().includes('Match') ? holidays : sports; + const expected = dropped === holidays ? ['Christmas', 'Easter'] : ['Match']; + + h.prefs.calendarSelection[`a/${dropped}`] = true; + expect((await h.run(CALENDAR_AUTHORITY)).outcome).toBe('ok'); + + expect(titles()).toEqual(expected); + }); +}); diff --git a/src/device-sync/engine/calendar-sync.ts b/src/device-sync/engine/calendar-sync.ts index fab6327c..1cb3d787 100644 --- a/src/device-sync/engine/calendar-sync.ts +++ b/src/device-sync/engine/calendar-sync.ts @@ -546,7 +546,10 @@ export class CalendarSync extends ItemSync { /** * Deselected calendars lose their rows (the provider deletes their events) * once nothing in them waits for an upload; until then only their clean - * events go. Events that left every synced calendar go when clean. + * events go, and the calendar row is dropped again next run. A calendar + * leaves `selected` before its first row goes: stopped half-way and + * selected again, it is loaded again. Events that left every synced + * calendar go when clean. */ protected async dropOutsideSelection(acct: string): Promise { const synced = new Set(this.syncedCollections(acct).map((id) => collectionKey(acct, id))); @@ -554,12 +557,15 @@ export class CalendarSync extends ItemSync { const rows = this.deleteRows.filter((row) => accountOfRef(this.keyByRow.get(row) ?? null) === acct); const outside = [...(this.outside.get(acct) ?? [])]; if (!deselected.length && !rows.length && !outside.length) return; - const kept = new Set(); + if (deselected.length) { + await this.env.writer.write([], this.tail([acct], (next) => { + const a = accountOf(next, acct); + a.selected = a.selected.filter((key) => !deselected.includes(key)); + })); + } for (const rowId of rows) { await this.env.checkpoints.check('deselect'); - const key = this.keyByRow.get(rowId) as string; - if (await this.dropCalendarRow(rowId)) continue; - kept.add(key); + await this.dropCalendarRow(rowId); } const works: Work[] = []; for (const local of (await this.eventKind.loadByRefs(outside)).values()) { @@ -569,9 +575,11 @@ export class CalendarSync extends ItemSync { const work = this.removeWork(acct, id, { kind: this.eventKind, local }, 'outside'); if (work) works.push(work); } - const selected = [...synced, ...kept]; + // Every synced calendar was loaded before this pass; the others go with their calendar rows. await this.env.writer.write(works, this.tail([acct], (next) => { - accountOf(next, acct).selected = selected; + const a = accountOf(next, acct); + a.selected = [...synced]; + if (a.partial) a.partial = []; })); this.outside.delete(acct); } diff --git a/src/device-sync/engine/contacts-sync.ts b/src/device-sync/engine/contacts-sync.ts index 228443e8..0567e7bc 100644 --- a/src/device-sync/engine/contacts-sync.ts +++ b/src/device-sync/engine/contacts-sync.ts @@ -761,12 +761,15 @@ export class ContactsSync extends ItemSync { /** * After the upload: rows of contacts in no synced address book go, when * clean (a deselected book, or a card moved away on the server). Dirty - * ones stay until their upload succeeds, and keep their book selected. + * ones stay until their upload succeeds, and keep their book `partial`, + * so the next run drops it again. A book is `partial` from before its + * first dropped row until the last: a stop half-way leaves it to load + * again when it is selected again, or to drop again when it is not. */ protected async dropOutsideSelection(acct: string): Promise { const synced = new Set(this.syncedCollections(acct).map((id) => collectionKey(acct, id))); const committed = this.env.store.account(acct); - const deselected = committed.selected.filter((key) => !synced.has(key)); + const deselected = [...new Set([...committed.selected, ...(committed.partial ?? [])])].filter((key) => !synced.has(key)); const outside = [...(this.outside.get(acct) ?? [])]; if (!deselected.length && !outside.length) return; @@ -781,6 +784,11 @@ export class ContactsSync extends ItemSync { else for (const key of meta.collections) if (deselected.includes(key)) kept.add(key); }; if (deselected.length) { + await this.env.writer.write([], this.tail([acct], (next) => { + const a = accountOf(next, acct); + a.selected = a.selected.filter((key) => !deselected.includes(key)); + a.partial = [...new Set([...(a.partial ?? []), ...deselected])]; + })); const groupWorks: Work[] = []; for (const group of await this.loadGroupsWhere('sourceid LIKE ?', [`${acct}/%`])) consider({ kind: this.groupKind, local: group }, groupWorks); await this.env.writer.write(groupWorks); @@ -794,10 +802,11 @@ export class ContactsSync extends ItemSync { } await this.env.writer.write(works); } - const selected = [...synced, ...kept]; - // The book leaves `selected` with its last dropped row; one still holding a dirty item stays. + // Every synced book was loaded before this pass; a book still holding a dirty item stays partial. await this.env.writer.write([], this.tail([acct], (next) => { - accountOf(next, acct).selected = selected; + const a = accountOf(next, acct); + a.selected = [...synced]; + a.partial = [...kept]; })); } else { const works: Work[] = []; diff --git a/src/device-sync/engine/item-sync.ts b/src/device-sync/engine/item-sync.ts index 6b4c4c2c..362d5686 100644 --- a/src/device-sync/engine/item-sync.ts +++ b/src/device-sync/engine/item-sync.ts @@ -467,10 +467,16 @@ export abstract class ItemSync { account.itemsState = from; account.reconcile = null; account.selected = selected; + if (account.partial) account.partial = account.partial.filter((key) => !selected.includes(key)); }); } - /** Collections selected since the rows were written: loaded once the account is up to date. */ + /** + * Collections selected since the rows were written (or loaded or dropped + * only in part): loaded once the account is up to date. A collection is + * `partial` from before its first chunk until its last one makes it + * `selected`, so deselected again after a stop, its rows are dropped. + */ protected async loadNewlySelected(acct: string): Promise { const account = this.env.store.account(acct); if (!account.itemsState || account.reconcile) return; @@ -479,9 +485,16 @@ export abstract class ItemSync { if (this.env.store.account(acct).selected.includes(key)) continue; await this.env.checkpoints.check('select'); const ids = await this.env.jmap.queryAll(this.itemType, acct, { [this.parentFilter]: collection }); + if (!(this.env.store.account(acct).partial ?? []).includes(key)) { + await this.env.writer.write([], this.tail([acct], (next) => { + const a = accountOf(next, acct); + a.partial = [...new Set([...(a.partial ?? []), key])]; + })); + } await this.processIds(acct, ids, [], (next) => { const a = accountOf(next, acct); if (!a.selected.includes(key)) a.selected.push(key); + a.partial = (a.partial ?? []).filter((k) => k !== key); }); } } diff --git a/src/device-sync/engine/sync-state.ts b/src/device-sync/engine/sync-state.ts index 2d07f9ab..99ad9089 100644 --- a/src/device-sync/engine/sync-state.ts +++ b/src/device-sync/engine/sync-state.ts @@ -25,8 +25,13 @@ export interface AccountState { collectionsState: string | null; /** ContactCard / CalendarEvent state the rows describe. */ itemsState: string | null; - /** Collection keys the rows were written for. */ + /** Collection keys whose rows are all on the device. */ selected: string[]; + /** + * Collection keys whose rows may be on the device in part: a load or a drop that has not ended. Selected, such + * a collection is loaded again; deselected, it is dropped again. + */ + partial?: string[]; /** Object ids whose rows could not be written; fetched again at the start of every run, nothing uploads for them meanwhile. */ stale: string[]; /** @@ -86,6 +91,7 @@ function parseAccount(value: unknown): AccountState { stale: strings(a.stale), reconcile: parseReconcile(a.reconcile), }; + if (a.partial !== undefined) out.partial = strings(a.partial); if (a.created !== undefined) out.created = strings(a.created); if (a.groups !== undefined) out.groups = strings(a.groups); if (a.taskOnly !== undefined) out.taskOnly = strings(a.taskOnly); From 78700401a2442c7006120c4d266115422b3c9f26 Mon Sep 17 00:00:00 2001 From: Linus Rath <139418639+rathlinus@users.noreply.github.com> Date: Sun, 27 Sep 2026 07:48:34 +0200 Subject: [PATCH 50/87] test: keep a contact's groups when an upload runs without a download #34 A sync with nothing to download, and the upload before sync is turned off, used to plan uploads with an empty group index, so the write after the upload deleted the uploaded contact's membership rows on the phone. The previous commit's refresh of the index before planning uploads fixes both; these tests cover the two paths so it stays fixed. Checked: both tests fail without that refresh. --- .../__tests__/engine/real-planners.test.ts | 36 +++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/src/device-sync/__tests__/engine/real-planners.test.ts b/src/device-sync/__tests__/engine/real-planners.test.ts index bf494cdd..48fa36d9 100644 --- a/src/device-sync/__tests__/engine/real-planners.test.ts +++ b/src/device-sync/__tests__/engine/real-planners.test.ts @@ -217,6 +217,42 @@ describe('device sync engine with the real planners', () => { expect(h.contacts().find((c) => c.name === 'Ada Lovelace')).toMatchObject({ dirty: false }); }); + describe('an upload that downloads nothing first', () => { + /** Ada in Friends (and Grace); Ada's name edited in place (AOSP); nothing new on the server. */ + async function adaRenamedInFriends() { + const h = real(); + const [ada, grace] = addServerCards(h, ['Ada Lovelace', 'Grace Hopper']); + const uid = h.server.get('ContactCard', 'a', ada)!.uid as string; + h.server.addCard('a', { uid: 'g1', kind: 'group', name: { full: 'Friends' }, members: { [uid]: true } }); + expect((await h.run()).outcome).toBe('ok'); + const row = h.contactNamed('Ada Lovelace')!.id; + h.device.user.updateData(Number(h.nameDataRow(row)!._id), { [Data.DATA1]: 'Ada King' }); + const memberships = () => h.device.rows('data').filter((d) => Number(d.raw_contact_id) === row && d.mimetype === MimeType.GROUP_MEMBERSHIP); + return { h, ada, grace, memberships }; + } + + it("keeps the uploaded contact's memberships in a sync run", async () => { + const { h, ada, memberships } = await adaRenamedInFriends(); + + expect((await h.run()).outcome).toBe('ok'); + + expect(h.server.get('ContactCard', 'a', ada)).toMatchObject({ name: { full: 'Ada King' } }); + expect(memberships()).toHaveLength(1); + }); + + it("keeps the uploaded contact's memberships in a teardown's upload", async () => { + const { h, ada, grace, memberships } = await adaRenamedInFriends(); + // Grace's edit cannot go up, so the teardown keeps every row and asks. + renameDeviceContact(h, h.contactNamed('Grace Hopper')!.id, 'Grace B. Hopper'); + h.server.setErrorFor('ContactCard', 'a', grace, { type: 'tooLarge' }); + + expect(await h.teardown(CONTACTS_AUTHORITY)).toEqual({ pending: 1 }); + + expect(h.server.get('ContactCard', 'a', ada)).toMatchObject({ name: { full: 'Ada King' } }); + expect(memberships()).toHaveLength(1); + }); + }); + it('uploads a removal from a group in a run that downloads no contact', async () => { const h = real(); // A keyed phone row shows the editor works in place, so a missing membership row is a removal. From fe0aaa124745d9e3d5fd6f14818480fdd03ddfd3 Mon Sep 17 00:00:00 2001 From: Linus Rath <139418639+rathlinus@users.noreply.github.com> Date: Sun, 27 Sep 2026 07:49:14 +0200 Subject: [PATCH 51/87] fix: load a book selected while the first sync was cut short #34 A full reconcile that ran out of time resumes after the last object it processed, in the sorted order of every listed id. It listed the collections selected at the time it resumed and marked them all loaded at its end, so a book turned on in between lost the objects that sorted before that point, and was never loaded again. The reconcile now keeps the collections it started with; one selected since is loaded after it like any newly selected collection. --- .../__tests__/engine/selection-state.test.ts | 24 +++++++++++++++++++ src/device-sync/engine/item-sync.ts | 10 +++++--- src/device-sync/engine/sync-state.ts | 6 ++++- 3 files changed, 36 insertions(+), 4 deletions(-) diff --git a/src/device-sync/__tests__/engine/selection-state.test.ts b/src/device-sync/__tests__/engine/selection-state.test.ts index 01c683ed..429450a3 100644 --- a/src/device-sync/__tests__/engine/selection-state.test.ts +++ b/src/device-sync/__tests__/engine/selection-state.test.ts @@ -110,6 +110,30 @@ describe('device sync engine: collections loaded or dropped in part', () => { expect(h.contacts().map((c) => c.name)).toEqual(['Ada']); }); + it('downloads every card of a book selected while a first sync was cut short', async () => { + const h = createHarness(); + const extra = h.server.addAddressBook('a', { name: 'Friends' }); + const friends = addServerCards(h, Array.from({ length: 10 }, (_, i) => `F${i}`), 'a', extra); + addServerCards(h, Array.from({ length: 120 }, (_, i) => `P${String(i).padStart(3, '0')}`)); + h.prefs.contactsSelection[`a/${extra}`] = false; + // The first sync (a full reconcile): 'reconcile', 'reconcile:ids', 'reconcile:objects': out of time after chunk 1. + outOfTimeAt(h, 3); + expect((await h.run()).outcome).toBe('cancelled'); + h.checkpoints.onCheckpoint = undefined; + const marker = h.state()!.accounts.a.reconcile!; + expect(friends.filter((id) => id <= (marker.after as string)).length).toBeGreaterThan(0); + + // The user turns Friends on before the next sync. + h.prefs.contactsSelection[`a/${extra}`] = true; + expect((await h.run()).outcome).toBe('ok'); + + const onDevice = new Set(h.contacts().map((c) => c.sourceId)); + expect(friends.filter((id) => !onDevice.has(`a/${id}`))).toEqual([]); + expect(h.contacts()).toHaveLength(130); + expect(h.state()!.accounts.a).toMatchObject({ reconcile: null }); + expect(h.state()!.accounts.a.selected.sort()).toEqual([`a/${h.book}`, `a/${extra}`].sort()); + }); + it('loads the events of a calendar dropped by a stopped run when it is selected again', async () => { const h = real(); const holidays = h.server.addCalendar('a', { name: 'Holidays' }); diff --git a/src/device-sync/engine/item-sync.ts b/src/device-sync/engine/item-sync.ts index 362d5686..af3880b6 100644 --- a/src/device-sync/engine/item-sync.ts +++ b/src/device-sync/engine/item-sync.ts @@ -404,10 +404,11 @@ export abstract class ItemSync { */ protected async fullReconcile(acct: string): Promise { let marker = this.env.store.account(acct).reconcile; + const synced = this.syncedCollections(acct); if (!marker?.from) { await this.env.checkpoints.check('reconcile'); const from = await this.env.jmap.takeState(this.itemType, acct); - const started = { from, phase: 'ids' as const, position: 0, after: null }; + const started = { from, phase: 'ids' as const, position: 0, after: null, collections: synced.map((c) => collectionKey(acct, c)) }; await this.env.writer.write([], this.tail([acct], (next) => { accountOf(next, acct).reconcile = { ...started }; })); @@ -415,7 +416,10 @@ export abstract class ItemSync { } const from = marker.from as string; await this.beforeReconcile(acct); - const collections = this.syncedCollections(acct); + // A resumed reconcile lists the collections it started with (their ids up to `after` are done): + // one selected since then is loaded afterwards, from its first id. + const keys = marker.collections; + const collections = keys ? synced.filter((c) => keys.includes(collectionKey(acct, c))) : synced; const listed = new Set(); for (const collection of collections) { await this.env.checkpoints.check('reconcile:ids'); @@ -441,7 +445,7 @@ export abstract class ItemSync { await this.env.checkpoints.check('reconcile:objects'); const part = todo.slice(i, i + size); position += part.length; - const mark = { from, phase: 'objects' as const, position, after: part[part.length - 1] }; + const mark = { from, phase: 'objects' as const, position, after: part[part.length - 1], ...(keys ? { collections: keys } : {}) }; await this.processChunk(acct, part, [], (next) => { accountOf(next, acct).reconcile = { ...mark }; }); diff --git a/src/device-sync/engine/sync-state.ts b/src/device-sync/engine/sync-state.ts index 99ad9089..896d9d3d 100644 --- a/src/device-sync/engine/sync-state.ts +++ b/src/device-sync/engine/sync-state.ts @@ -18,6 +18,8 @@ export interface ReconcileMarker { position: number; /** The last id processed in that (sorted) order: a resumed reconcile continues after it. */ after: string | null; + /** Keys of the collections the reconcile lists; one selected since is loaded after it. Absent: every synced one. */ + collections?: string[]; } export interface AccountState { @@ -74,12 +76,14 @@ const stringOrNull = (value: unknown): string | null => (typeof value === 'strin function parseReconcile(value: unknown): ReconcileMarker | null { if (!value || typeof value !== 'object') return null; const r = value as Record; - return { + const marker: ReconcileMarker = { from: stringOrNull(r.from), phase: r.phase === 'objects' ? 'objects' : 'ids', position: typeof r.position === 'number' && r.position >= 0 ? r.position : 0, after: stringOrNull(r.after), }; + if (Array.isArray(r.collections)) marker.collections = strings(r.collections); + return marker; } function parseAccount(value: unknown): AccountState { From 1ee6bc459d45fb8a1f0225184af711423d3d0855 Mon Sep 17 00:00:00 2001 From: Linus Rath <139418639+rathlinus@users.noreply.github.com> Date: Sun, 27 Sep 2026 07:49:53 +0200 Subject: [PATCH 52/87] fix: move group members on the phone after an interrupted sync #34 When a group's members change on the server, the download fetches the contacts whose membership flipped in a chunk after the group's. The group's chunk already stored its new version, so when the sync stopped before the members' chunk, the next sync found no flip any more and the phone kept the old members. Those contacts are now listed with the group's rows, as objects to fetch first, until their own chunk ran. --- .../__tests__/engine/group-listing.test.ts | 29 +++++++++++++ src/device-sync/engine/contacts-sync.ts | 41 +++++++++++++------ src/device-sync/engine/item-sync.ts | 24 ++++++++++- 3 files changed, 80 insertions(+), 14 deletions(-) diff --git a/src/device-sync/__tests__/engine/group-listing.test.ts b/src/device-sync/__tests__/engine/group-listing.test.ts index 16e47c49..5dadacb4 100644 --- a/src/device-sync/__tests__/engine/group-listing.test.ts +++ b/src/device-sync/__tests__/engine/group-listing.test.ts @@ -5,6 +5,7 @@ // row it inserted but never listed hides a later hard delete. import { describe, expect, it } from 'vitest'; +import { MimeType } from '../../android-columns'; import { calendarPlanner } from '../../calendar/planner'; import { contactsPlanner } from '../../contacts/planner'; import { serializeSyncState } from '../../engine/sync-state'; @@ -175,6 +176,34 @@ describe('device sync engine: the groups a SyncState lists', () => { expect(h.server.get('ContactCard', 'a', group)).toBeUndefined(); }); + it("moves a group's membership on the device even when time ran out after the group's chunk", async () => { + const h = real(); + const [ada, grace] = addServerCards(h, ['Ada Lovelace', 'Grace Hopper']); + const uid = (id: string) => h.server.get('ContactCard', 'a', id)!.uid as string; + const group = h.server.addCard('a', { uid: 'g1', kind: 'group', name: { full: 'Friends' }, members: { [uid(ada)]: true } }); + await h.run(); + const members = () => + h.device.rows('data') + .filter((d) => d.mimetype === MimeType.GROUP_MEMBERSHIP) + .map((d) => h.contacts().find((c) => c.id === Number(d.raw_contact_id))?.name) + .sort(); + expect(members()).toEqual(['Ada Lovelace']); + // Another client moves the membership from Ada to Grace: only the group card changes. + h.server.serverUpdate('ContactCard', 'a', group, { members: { [uid(grace)]: true } }); + // 'changes', 'download' (the group's chunk), then out of time before the members' chunk. + h.checkpoints.count = 0; + h.checkpoints.onCheckpoint = (n) => { + if (n === 2) h.clock.now += RUN_BUDGET; + }; + expect((await h.run()).outcome).toBe('cancelled'); + h.checkpoints.onCheckpoint = undefined; + + expect((await h.run()).outcome).toBe('ok'); + + expect(members()).toEqual(['Grace Hopper']); + expect(h.state()?.accounts.a.stale).toEqual([]); + }); + it('never destroys a listed group card that is in no synced book or no longer a group, and forgets it', async () => { const h = real(); const archive = h.server.addAddressBook('a', { name: 'Archive' }); diff --git a/src/device-sync/engine/contacts-sync.ts b/src/device-sync/engine/contacts-sync.ts index 0567e7bc..4642c817 100644 --- a/src/device-sync/engine/contacts-sync.ts +++ b/src/device-sync/engine/contacts-sync.ts @@ -123,6 +123,8 @@ export class ContactsSync extends ItemSync { private uidRefs: Map | null = null; /** Per account: raw contacts the upload phases looked at, for their memberships. */ private readonly uploadRows = new Map>(); + /** Group ref → ids of the contacts whose membership in it flips with the version being downloaded. */ + private readonly refreshes = new Map(); constructor(env: RunEnv) { super(env); @@ -409,27 +411,42 @@ export class ContactsSync extends ItemSync { }; } - /** Contacts of `acct` whose membership in one of these group cards flipped since the group's shadow. */ + /** + * Contacts of `acct` whose membership in one of these group cards flipped + * since the group's shadow; kept per group for `dependents`. + */ private async membersToRefresh(acct: string, groups: Array<{ id: string; members?: Record }>): Promise { if (!groups.length) return []; const locals = await this.groupKind.loadByRefs(groups.map((g) => refOf(acct, g.id))); - const flipped = new Set(); + const flippedBy = new Map>(); for (const group of groups) { - const before = new Set(onIds(locals.get(refOf(acct, group.id))?.shadow?.members as Record | undefined)); + const ref = refOf(acct, group.id); + const before = new Set(onIds(locals.get(ref)?.shadow?.members as Record | undefined)); const after = new Set(onIds(group.members)); - for (const uid of before) if (!after.has(uid)) flipped.add(uid); - for (const uid of after) if (!before.has(uid)) flipped.add(uid); + const flipped = new Set([...before].filter((uid) => !after.has(uid)).concat([...after].filter((uid) => !before.has(uid)))); + this.refreshes.delete(ref); + if (flipped.size) flippedBy.set(ref, flipped); } - if (!flipped.size) return []; + if (!flippedBy.size) return []; await this.ensureShadowIndex(); - const out: string[] = []; - for (const uid of flipped) { - for (const ref of this.uidRefs?.get(uid) ?? []) { - const id = idInAccount(ref, acct); - if (id) out.push(id); + const out = new Set(); + for (const [ref, flipped] of flippedBy) { + const ids: string[] = []; + for (const uid of flipped) { + for (const member of this.uidRefs?.get(uid) ?? []) { + const id = idInAccount(member, acct); + if (id) ids.push(id); + } } + this.refreshes.set(ref, ids); + for (const id of ids) out.add(id); } - return out; + return [...out]; + } + + /** The members a group card gained or lost: refreshed in this page, or first thing next run. */ + protected dependents(kind: Kind, acct: string, object: ServerObject): string[] { + return kind === this.groupKind ? this.refreshes.get(refOf(acct, object.id)) ?? [] : []; } protected async beforePlanning(acct: string, objects: ServerObject[]): Promise { diff --git a/src/device-sync/engine/item-sync.ts b/src/device-sync/engine/item-sync.ts index af3880b6..88c0bbe0 100644 --- a/src/device-sync/engine/item-sync.ts +++ b/src/device-sync/engine/item-sync.ts @@ -155,6 +155,23 @@ export abstract class ItemSync { protected listing(_kind: Kind, _acct: string, _id: string, _present: boolean): StateChange | undefined { return undefined; } + /** + * Objects whose rows a download of `object` makes outdated (contacts: the + * members a group card gained or lost). They go on `stale` in the batch + * that writes it, so a stop before their own chunk leaves them to the next + * run's first fetch. + */ + protected dependents(_kind: Kind, _acct: string, _object: ServerObject): string[] { + return []; + } + + private staleWith(acct: string, ids: string[]): StateChange | undefined { + if (!ids.length) return undefined; + return (next) => { + const account = accountOf(next, acct); + account.stale = [...new Set([...account.stale, ...ids])]; + }; + } protected onDiscard(_next: SyncState, _accounts: Set): void {} /** Whether a pending create of `target` may adopt this object (contacts: the target book holds it). */ protected adoptionTargetMatches(acct: string, _object: ServerObject, target: string): boolean { @@ -658,7 +675,7 @@ export abstract class ItemSync { const group = local ? plan.ops : prependOps(plan.ops, [insertGuard(kind, ref)]); return { group, - state: this.listing(kind, acct, object.id, true), + state: both(this.listing(kind, acct, object.id, true), this.staleWith(acct, this.dependents(kind, acct, object))), fresh: true, applied: () => { if (plan.effect === 'insert') this.env.report.stats.downloaded.created++; @@ -694,7 +711,10 @@ export abstract class ItemSync { const plan = kind.planDownload(object, null, acct); return { group: concatGroups(ref, found.kind.planLocalDelete(found.local), plan.ops), - state: both(this.listing(found.kind, acct, object.id, false), this.listing(kind, acct, object.id, true)), + state: both( + both(this.listing(found.kind, acct, object.id, false), this.listing(kind, acct, object.id, true)), + this.staleWith(acct, this.dependents(kind, acct, object)), + ), applied: () => { this.env.report.stats.downloaded.updated++; this.markResolved(acct, object.id); From b86e0a118ea6e12bfd6ed6076df8f8aa80afcd5b Mon Sep 17 00:00:00 2001 From: Linus Rath <139418639+rathlinus@users.noreply.github.com> Date: Sun, 27 Sep 2026 07:50:31 +0200 Subject: [PATCH 53/87] fix: size provider batches by what the Binder transaction carries #34 The batch budget counted JSON characters, but a Parcel carries strings as UTF-16 plus a few hundred bytes per operation, so a full batch was about 700 KB of a 1 MB Binder buffer the whole process shares. Batches are now weighed at two bytes per character plus 300 bytes per op and kept under 500 KB of that. The state op no longer joins a batch whose last item already fills a provider yield window of 499 operations, which ContactsProvider refused. --- .../__tests__/engine/internals.test.ts | 29 ++++++++++++++++- src/device-sync/engine/batch.ts | 31 ++++++++++++++----- src/device-sync/engine/deps.ts | 4 +-- 3 files changed, 53 insertions(+), 11 deletions(-) diff --git a/src/device-sync/__tests__/engine/internals.test.ts b/src/device-sync/__tests__/engine/internals.test.ts index 07ec9348..974f9c9b 100644 --- a/src/device-sync/__tests__/engine/internals.test.ts +++ b/src/device-sync/__tests__/engine/internals.test.ts @@ -3,7 +3,7 @@ // rebuild and how failures map to outcomes. import { describe, expect, it, vi } from 'vitest'; -import { BatchWriter, buildBatch, prependOps, type Work } from '../../engine/batch'; +import { BatchWriter, buildBatch, estimateBatchBytes, MAX_OPS_PER_YIELD, prependOps, type Work } from '../../engine/batch'; import { classifyFailure, RunAbort, StateMismatch, StopRun } from '../../engine/errors'; import { acquireLock } from '../../engine/mutex'; import { isBackedOff, nextMarker } from '../../engine/poison'; @@ -169,6 +169,33 @@ describe('provider batches', () => { expect(p.sent.slice(1)).toEqual([planned.group.ops.map((op) => ({ ...op, yieldAllowed: true })), buildBatch([other.group])]); }); + it('weighs a batch as the Parcel carries it: two bytes per character of its JSON and 300 per op', () => { + const ops: ProviderOp[] = [insert('data', { data1: 'x'.repeat(1000) }), insert('raw_contacts')]; + + expect(estimateBatchBytes(ops)).toBe(2 * JSON.stringify(ops).length + 2 * 300); + expect(estimateBatchBytes(ops)).toBeGreaterThan(2 * 1000); + }); + + it('packs batches by that weight', async () => { + const p = port(ok); + const group = (i: number): Work => ({ group: { ref: `g${i}`, ops: [insert('data', { data1: 'x'.repeat(1000) })] } }); + + await writer(p, { maxBytes: 2 * estimateBatchBytes(group(0).group.ops) }).write([group(0), group(1), group(2)]); + + expect(p.sent.map((b) => b.length)).toEqual([2, 1]); + }); + + it('sends the state op alone after a group that fills a yield window, and with a group that leaves room', async () => { + const tail = () => ({ op: () => ({ op: 'syncState', value: 's' }) as ProviderOp, applied: () => undefined }); + const group = (n: number): Work => ({ group: { ref: `g${n}`, ops: Array.from({ length: n }, () => insert('data')) } }); + const p = port(ok); + + await writer(p, { maxOps: 1000 }).write([group(MAX_OPS_PER_YIELD)], tail()); + await writer(p, { maxOps: 1000 }).write([group(MAX_OPS_PER_YIELD - 1)], tail()); + + expect(p.sent.map((b) => b.length)).toEqual([MAX_OPS_PER_YIELD, 1, MAX_OPS_PER_YIELD]); + }); + it('skips groups that only assert', async () => { const p = port(ok); await writer(p).write([{ group: { ref: 'g', ops: [{ op: 'assert', table: 'raw_contacts', id: 1, values: { dirty: 0 } }] } }]); diff --git a/src/device-sync/engine/batch.ts b/src/device-sync/engine/batch.ts index a7a454ea..334a4419 100644 --- a/src/device-sync/engine/batch.ts +++ b/src/device-sync/engine/batch.ts @@ -4,9 +4,11 @@ * * - a group is never split; `refs` are rebased to the batch and the group's * first op is its yield point, so the provider may commit between groups - * but never inside one; - * - at most `maxOps` ops and about `maxBytes` of JSON per batch; groups that - * carry photo bytes travel alone (Binder budget); + * but never inside one (a group of 499 ops or more is the planner's to + * avoid: ContactsProvider refuses it); + * - at most `maxOps` ops and about `maxBytes` per batch, as + * `estimateBatchBytes` counts them; groups that carry photo bytes travel + * alone (Binder budget); * - the SyncState op, when there is one, is the last op of the last batch * and never a yield point: the stored state can't get ahead of its rows. * @@ -47,12 +49,22 @@ export interface Work { export interface BatchWriterOptions { maxOps: number; + /** Per batch, as `estimateBatchBytes` counts them. */ maxBytes: number; maxReplans: number; } -function sizeOf(ops: readonly ProviderOp[]): number { - return JSON.stringify(ops).length; +/** Ops a provider takes up to and including one yield point (ContactsProvider throws at the 500th). */ +export const MAX_OPS_PER_YIELD = 499; + +/** + * What ops weigh in the Binder transaction of an `applyBatch`, for the batch + * budget and for planners that split big items: a Parcel carries strings as + * UTF-16 (two bytes per character of their JSON) plus about 300 bytes per op + * (URI, flags, value types, back-reference classes). + */ +export function estimateBatchBytes(ops: readonly ProviderOp[]): number { + return 2 * JSON.stringify(ops).length + 300 * ops.length; } function carriesBlob(ops: readonly ProviderOp[]): boolean { @@ -131,8 +143,11 @@ export class BatchWriter { for (let i = 0; i < batches.length; i++) { const last = i === batches.length - 1; const batch = batches[i]; - // A photo batch is already at the Binder budget: its state op goes alone. - const tailHere = last && tail && !(batch.length === 1 && carriesBlob(batch[0].group.ops)) ? tail : undefined; + // A photo batch is already at the Binder budget, and a group that fills a yield window leaves no room: + // the state op goes alone. + const photo = batch.length === 1 && carriesBlob(batch[0].group.ops); + const full = batch[batch.length - 1].group.ops.length >= MAX_OPS_PER_YIELD; + const tailHere = last && tail && !photo && !full ? tail : undefined; await this.applyBatch(batch, tailHere); if (last && tail && !tailHere) await this.applyTail(tail); } @@ -156,7 +171,7 @@ export class BatchWriter { continue; } const n = work.group.ops.length; - const b = sizeOf(work.group.ops); + const b = estimateBatchBytes(work.group.ops); if (current.length && (ops + n > this.options.maxOps || bytes + b > this.options.maxBytes)) flush(); current.push(work); ops += n; diff --git a/src/device-sync/engine/deps.ts b/src/device-sync/engine/deps.ts index f7bbf4a2..8f5bb227 100644 --- a/src/device-sync/engine/deps.ts +++ b/src/device-sync/engine/deps.ts @@ -58,7 +58,7 @@ export interface Tuning { chunkSize: number; /** Provider ops per applyBatch, with a yield point at every item group. */ maxBatchOps: number; - /** Serialized bytes per applyBatch (the Binder budget is 1 MB for the whole transaction). */ + /** Bytes per applyBatch as `estimateBatchBytes` counts them (the Binder budget is 1 MB for the whole transaction). */ maxBatchBytes: number; /** Re-reads and re-plans of an item whose group failed an assert. */ maxReplans: number; @@ -94,7 +94,7 @@ export interface Tuning { export const DEFAULT_TUNING: Tuning = { chunkSize: 50, maxBatchOps: 400, - maxBatchBytes: 300_000, + maxBatchBytes: 500_000, maxReplans: 3, maxCreatesPerSet: 50, maxSetBytes: 1_000_000, From 153e85cb80e38a802c50a9e1c3e9420ad9557559 Mon Sep 17 00:00:00 2001 From: Linus Rath <139418639+rathlinus@users.noreply.github.com> Date: Sun, 27 Sep 2026 07:51:09 +0200 Subject: [PATCH 54/87] fix: finish turning sync off when the app goes to the background #34 Turning sync off first uploads what the device changed, then removes the synced rows. Started from the settings screen, that work yielded the JS thread with timers between its steps, and React Native pauses timers while the app is in the background and no headless task runs: leaving the app stopped it halfway, holding the account's sync lock, until the app came back. The teardown now waits without timers. --- .../__tests__/engine/calendar-flow.test.ts | 22 ++++++++++++++++++- src/device-sync/engine/teardown.ts | 4 +++- 2 files changed, 24 insertions(+), 2 deletions(-) diff --git a/src/device-sync/__tests__/engine/calendar-flow.test.ts b/src/device-sync/__tests__/engine/calendar-flow.test.ts index 8f391630..165bbbf2 100644 --- a/src/device-sync/__tests__/engine/calendar-flow.test.ts +++ b/src/device-sync/__tests__/engine/calendar-flow.test.ts @@ -1,9 +1,10 @@ // Engine runs for calendars, and the teardown the app calls when sync is // turned off (docs/device-sync.md, "Calendar mapping", "App integration"). -import { describe, expect, it } from 'vitest'; +import { describe, expect, it, vi } from 'vitest'; import { Events } from '../../android-columns'; import { parseJsonColumn } from '../../common/json'; +import { teardownAuthority } from '../../engine/teardown'; import type { PoisonMarker } from '../../planner'; import { addDeviceContact, @@ -12,6 +13,7 @@ import { CALENDAR_AUTHORITY, CONTACTS_AUTHORITY, createHarness, + REGISTRY_ID, renameDeviceContact, type Harness, } from './harness'; @@ -239,6 +241,24 @@ describe('device sync engine: teardown', () => { expect(h.serverNames()).toHaveLength(80); }); + it("finishes while JS timers are paused (the UI's teardown with the app in the background)", async () => { + const h = createHarness(); + const [ada] = addServerCards(h, ['Ada']); + await h.run(); + renameDeviceContact(h, h.contactNamed('Ada')!.id, 'Ada King'); + // The app passes neither a yield nor a sleep: the engine's own must not need a timer here. + const { yieldThread: _yield, sleep: _sleep, ...deps } = h.deps; + vi.useFakeTimers(); + try { + await expect(teardownAuthority(deps, REGISTRY_ID, ANDROID_ACCOUNT, CONTACTS_AUTHORITY)).resolves.toEqual({ pending: 0 }); + } finally { + vi.useRealTimers(); + } + + expect(h.server.get('ContactCard', 'a', ada)).toMatchObject({ name: { full: 'Ada King' } }); + expect(h.contacts()).toEqual([]); + }, 3_000); + it('removes the calendar rows, and their events with them', async () => { const h = createHarness(); addServerEvent(h, 'Standup'); diff --git a/src/device-sync/engine/teardown.ts b/src/device-sync/engine/teardown.ts index 507280a9..8e0bbc20 100644 --- a/src/device-sync/engine/teardown.ts +++ b/src/device-sync/engine/teardown.ts @@ -80,7 +80,9 @@ export async function teardownAuthority( if (!stored.owner || stored.owner.origin === connection.origin) { const report = new ReportBuilder('teardown', authority, started, tuning.maxItemErrors); const { sync } = await createEnv({ - deps, + // Started from the UI: JS timers stop while the app is in the background and no headless task + // runs, so the teardown's own waits (checkpoint yields, retry back-offs) must not need one. + deps: { ...deps, yieldThread: deps.yieldThread ?? (async () => undefined), sleep: deps.sleep ?? (async () => undefined) }, tuning: { ...tuning, deadlineMarginMs: 0, createBudgetMs: 0 }, mode: 'teardown', registryId, From 654675d90b99d3f73fa259e2ac246d7d7db100bb Mon Sep 17 00:00:00 2001 From: Linus Rath <139418639+rathlinus@users.noreply.github.com> Date: Sun, 27 Sep 2026 07:51:49 +0200 Subject: [PATCH 55/87] fix: keep unsynced changes waiting when sync is turned off before the first sync ended #34 Turning sync off uploads what the device changed without downloading first; every upload carries the state the device last synced, so the server refuses it when something changed meanwhile and the engine downloads first. Before the first sync of an account had ended there was no such state, and edits and deletions went up unguarded, able to overwrite what another client had changed. They now wait, and the app asks before they are dropped; new items still go up. --- .../__tests__/engine/real-planners.test.ts | 57 +++++++++++++++++++ src/device-sync/engine/item-sync.ts | 13 +++-- 2 files changed, 66 insertions(+), 4 deletions(-) diff --git a/src/device-sync/__tests__/engine/real-planners.test.ts b/src/device-sync/__tests__/engine/real-planners.test.ts index 48fa36d9..d72a6541 100644 --- a/src/device-sync/__tests__/engine/real-planners.test.ts +++ b/src/device-sync/__tests__/engine/real-planners.test.ts @@ -15,6 +15,7 @@ import { createHarness, renameDeviceContact, rowWrites, + RUN_BUDGET, type Harness, } from './harness'; @@ -295,6 +296,62 @@ describe('device sync engine with the real planners', () => { }); }); + describe('a deletion the teardown uploads', () => { + /** The contacts planner as fixed for data finding 4: a deleted contact's shadow follows the server. */ + function refreshingDeletedShadows(h: Harness): void { + h.deps.planners = { + ...h.deps.planners, + contacts: { + ...contactsPlanner, + planDownload: (card, local, ctx) => { + if (!local?.deleted || !local.sourceId) return contactsPlanner.planDownload(card, local, ctx); + const shadow = JSON.stringify({ ...card, '~memberOf': local.shadow?.['~memberOf'] ?? [] }); + const ops = [{ op: 'update' as const, table: 'raw_contacts' as const, id: local.rawContactId, values: { sync2: shadow } }]; + return { ops: { ref: local.sourceId, ops }, conflicts: 0, stillDirty: true, effect: 'update' as const, writes: 1 }; + }, + }, + }; + } + + it('is decided on the server version when the server changed since the last sync', async () => { + const h = real(); + const archive = h.server.addAddressBook('a', { name: 'Archive' }); + h.prefs.contactsSelection[`a/${archive}`] = false; + const [ada] = addServerCards(h, ['Ada Lovelace']); + await h.run(); + refreshingDeletedShadows(h); + h.device.user.deleteContact(h.contactNamed('Ada Lovelace')!.id); + // Meanwhile another client files Ada in Archive too. + h.server.serverUpdate('ContactCard', 'a', ada, { [`addressBookIds/${archive}`]: true }); + + expect(await h.teardown(CONTACTS_AUTHORITY)).toEqual({ pending: 0 }); + + expect(h.server.get('ContactCard', 'a', ada)).toMatchObject({ addressBookIds: { [archive]: true } }); + expect((h.server.get('ContactCard', 'a', ada)!.addressBookIds as object)).not.toHaveProperty(h.book); + }); + + it('waits, with the edits, while no state guards the upload (the first sync never ended)', async () => { + const h = real(); + const [ada] = addServerCards(h, ['Ada Lovelace', 'Grace Hopper', ...Array.from({ length: 60 }, (_, i) => `P${i}`)]); + // The first sync (a full reconcile) runs out of time after its first chunk (Ada and Grace): no state yet. + h.checkpoints.onCheckpoint = (n) => { + if (n === 3) h.clock.now += RUN_BUDGET; + }; + expect((await h.run()).outcome).toBe('cancelled'); + h.checkpoints.onCheckpoint = undefined; + expect(h.state()?.accounts.a.itemsState).toBeNull(); + renameDeviceContact(h, h.contactNamed('Ada Lovelace')!.id, 'Ada King'); + h.device.user.deleteContact(h.contactNamed('Grace Hopper')!.id); + addDeviceContact(h, 'Hedy Lamarr'); + // Meanwhile another client changes Ada. + h.server.serverUpdate('ContactCard', 'a', ada, { 'name/full': 'Ada, Countess of Lovelace' }); + + expect(await h.teardown(CONTACTS_AUTHORITY)).toEqual({ pending: 2 }); + + expect(h.serverNames().filter((n) => !/^P\d+$/.test(n))).toEqual(['Ada, Countess of Lovelace', 'Grace Hopper', 'Hedy Lamarr']); + }); + }); + describe('a device edit the server refuses (forbidden)', () => { it('puts a contact back in place, keeping what only the device knows', async () => { const h = real(); diff --git a/src/device-sync/engine/item-sync.ts b/src/device-sync/engine/item-sync.ts index 88c0bbe0..31c75eaa 100644 --- a/src/device-sync/engine/item-sync.ts +++ b/src/device-sync/engine/item-sync.ts @@ -800,14 +800,19 @@ export abstract class ItemSync { */ protected async uploadAll(): Promise { const accounts = this.accountIds(); - for (const acct of accounts) await this.withRetries(acct, () => this.pairPhase(acct)); + // A teardown downloads nothing first: `ifInState` is what keeps its uploads from overwriting a server change. + // An account without a stored state (its first sync never ended) uploads only new items; the rest waits. + const guarded = accounts.filter((acct) => this.env.mode !== 'teardown' || !!this.env.store.account(acct).itemsState); + for (const acct of guarded) await this.withRetries(acct, () => this.pairPhase(acct)); await this.claimNewItems(); const deletionsAllowed = await this.deletionPolicy(await this.loadUploadItems()); - for (const acct of accounts) await this.withRetries(acct, () => this.prePhase(acct)); + for (const acct of guarded) await this.withRetries(acct, () => this.prePhase(acct)); for (const phase of ['create', 'update', 'delete'] as const) { - for (const acct of accounts) await this.withRetries(acct, () => this.phase(acct, phase, deletionsAllowed)); + for (const acct of phase === 'create' ? accounts : guarded) { + await this.withRetries(acct, () => this.phase(acct, phase, deletionsAllowed)); + } } - for (const acct of accounts) await this.withRetries(acct, () => this.extraPhase(acct, deletionsAllowed)); + for (const acct of guarded) await this.withRetries(acct, () => this.extraPhase(acct, deletionsAllowed)); for (const acct of accounts) await this.adoptExtraCreated(acct); } From 0faee568cc6d46a606370650f39d20db688fb5c8 Mon Sep 17 00:00:00 2001 From: Linus Rath <139418639+rathlinus@users.noreply.github.com> Date: Sun, 27 Sep 2026 07:52:29 +0200 Subject: [PATCH 56/87] fix: let a contact deleted on the phone and moved away meanwhile survive #34 When another client moved a card out of every address book the device syncs while the device deleted the contact, the engine noted the card for dropping but never showed its new version to the planner, so the deletion was decided on the old copy and destroyed the card. A deleted item is now merged like any changed one first; the contacts planner then leaves a card that is in no synced book. When the planner cannot place the object, the deletion stays as it was. --- .../__tests__/engine/real-planners.test.ts | 50 +++++++++++++++++-- src/device-sync/engine/item-sync.ts | 31 ++++++++---- 2 files changed, 67 insertions(+), 14 deletions(-) diff --git a/src/device-sync/__tests__/engine/real-planners.test.ts b/src/device-sync/__tests__/engine/real-planners.test.ts index d72a6541..cae21f0c 100644 --- a/src/device-sync/__tests__/engine/real-planners.test.ts +++ b/src/device-sync/__tests__/engine/real-planners.test.ts @@ -296,7 +296,7 @@ describe('device sync engine with the real planners', () => { }); }); - describe('a deletion the teardown uploads', () => { + describe('a deletion decided on the server version', () => { /** The contacts planner as fixed for data finding 4: a deleted contact's shadow follows the server. */ function refreshingDeletedShadows(h: Harness): void { h.deps.planners = { @@ -313,7 +313,7 @@ describe('device sync engine with the real planners', () => { }; } - it('is decided on the server version when the server changed since the last sync', async () => { + it('in a teardown, when the server changed since the last sync', async () => { const h = real(); const archive = h.server.addAddressBook('a', { name: 'Archive' }); h.prefs.contactsSelection[`a/${archive}`] = false; @@ -330,7 +330,51 @@ describe('device sync engine with the real planners', () => { expect((h.server.get('ContactCard', 'a', ada)!.addressBookIds as object)).not.toHaveProperty(h.book); }); - it('waits, with the edits, while no state guards the upload (the first sync never ended)', async () => { + it('lets the planner see the server version of a contact deleted on the device and filed away meanwhile', async () => { + const h = real(); + const archive = h.server.addAddressBook('a', { name: 'Archive' }); + h.prefs.contactsSelection[`a/${archive}`] = false; + const [ada] = addServerCards(h, ['Ada Lovelace']); + await h.run(); + refreshingDeletedShadows(h); + const planned: string[][] = []; + const contacts = h.deps.planners.contacts; + h.deps.planners = { + ...h.deps.planners, + contacts: { + ...contacts, + planUpload: (local, ctx) => { + if (local.deleted) planned.push(Object.keys(local.shadow?.addressBookIds ?? {})); + return contacts.planUpload(local, ctx); + }, + }, + }; + h.device.user.deleteContact(h.contactNamed('Ada Lovelace')!.id); + // Another client files Ada in Archive only: she left every synced book. + h.server.serverUpdate('ContactCard', 'a', ada, { addressBookIds: { [archive]: true } }); + + expect((await h.run()).outcome).toBe('ok'); + + expect(planned).toEqual([[archive]]); + }); + + it('still uploads the deletion of an event whose calendar the planner cannot place', async () => { + const h = real(); + const archive = h.server.addCalendar('a', { name: 'Archive' }); + h.prefs.calendarSelection[`a/${archive}`] = false; + const lunch = h.server.addEvent('a', { uid: 'lunch-uid', title: 'Lunch', start: '2026-10-06T12:00:00', duration: 'PT1H', timeZone: 'Europe/Berlin', calendarIds: { [h.calendar]: true } }); + await h.run(CALENDAR_AUTHORITY); + h.device.user.deleteEvent(Number(h.events()[0]._id)); + h.server.serverUpdate('CalendarEvent', 'a', lunch, { calendarIds: { [archive]: true } }); + + const report = await h.run(CALENDAR_AUTHORITY); + + expect(report).toMatchObject({ outcome: 'ok', itemErrors: [], stats: { uploaded: { deleted: 1 } } }); + expect(h.server.get('CalendarEvent', 'a', lunch)).toBeUndefined(); + expect(h.events()).toEqual([]); + }); + + it('waits in a teardown, with the edits, while no state guards the upload (the first sync never ended)', async () => { const h = real(); const [ada] = addServerCards(h, ['Ada Lovelace', 'Grace Hopper', ...Array.from({ length: 60 }, (_, i) => `P${i}`)]); // The first sync (a full reconcile) runs out of time after its first chunk (Ada and Grace): no state yet. diff --git a/src/device-sync/engine/item-sync.ts b/src/device-sync/engine/item-sync.ts index 31c75eaa..8473f1e5 100644 --- a/src/device-sync/engine/item-sync.ts +++ b/src/device-sync/engine/item-sync.ts @@ -164,14 +164,6 @@ export abstract class ItemSync { protected dependents(_kind: Kind, _acct: string, _object: ServerObject): string[] { return []; } - - private staleWith(acct: string, ids: string[]): StateChange | undefined { - if (!ids.length) return undefined; - return (next) => { - const account = accountOf(next, acct); - account.stale = [...new Set([...account.stale, ...ids])]; - }; - } protected onDiscard(_next: SyncState, _accounts: Set): void {} /** Whether a pending create of `target` may adopt this object (contacts: the target book holds it). */ protected adoptionTargetMatches(acct: string, _object: ServerObject, target: string): boolean { @@ -279,6 +271,15 @@ export abstract class ItemSync { if (this.marks(acct).has(id) || this.env.store.account(acct).stale.includes(id)) this.marks(acct).set(id, false); } + /** Puts `ids` on `stale` in the batch whose rows make them outdated (see `dependents`). */ + private staleWith(acct: string, ids: string[]): StateChange | undefined { + if (!ids.length) return undefined; + return (next) => { + const account = accountOf(next, acct); + account.stale = [...new Set([...account.stale, ...ids])]; + }; + } + protected isStale(acct: string, sourceId: string | null): boolean { const id = idInAccount(sourceId, acct); if (!id) return false; @@ -638,10 +639,18 @@ export abstract class ItemSync { if (!kind || !this.inSelection(acct, object)) { if (!found) return null; const meta = found.kind.meta(found.local); - if (found.kind === kind && meta.dirty && !meta.deleted && !meta.isNew) { - // Uploaded first, dropped afterwards: merged like any dirty item, so the upload builds on the server's version. + if (found.kind === kind && (meta.dirty || meta.deleted) && !meta.isNew) { + // Uploaded first, dropped afterwards: merged like any changed item (a deleted one's shadow follows + // too), so the upload builds on the server's version. this.outsideOf(acct).add(refOf(acct, object.id)); - return this.downloadWork(kind, acct, object, found.local, written); + if (!meta.deleted) return this.downloadWork(kind, acct, object, found.local, written); + try { + return this.downloadWork(kind, acct, object, found.local, written); + } catch (error) { + // A planner that cannot place it (no synced calendar) leaves the deletion as it was planned. + this.env.log(`deleted ${refOf(acct, object.id)} outside the selection keeps its shadow`, error); + return null; + } } return this.removeWork(acct, object.id, found, 'outside'); } From f24ff7da9ead0bc9bb20f00aac35d9f9e8e5d5a9 Mon Sep 17 00:00:00 2001 From: Linus Rath <139418639+rathlinus@users.noreply.github.com> Date: Sun, 27 Sep 2026 08:40:14 +0200 Subject: [PATCH 57/87] fix: keep the other edits of an event moved to another calendar #34 Etar moves an event, or turns a series into a single event, by deleting the row and inserting a copy. The pair uploaded the move as a patch of the event, but then took the copy as it stood for the new baseline and wrote the server's version over it, so everything else saved with the move was dropped: a changed reminder came back, and an edit made before the move never reached the server. The synced occurrences, which Etar does not copy, were deleted with the old row, losing an occurrence edit that was still waiting. The copy now starts from the old row's baseline with only what the move changed, so any other difference stays dirty and uploads right after the move as an ordinary update. The old row's occurrences move to the new row with what they wait to upload. Attendee rows of the copy are the app's re-insert (Etar writes everyone as required, without a status) and are not read as edits. Checked with planner tests and runs through the engine and the fake server: a reminder saved with the move and a pending occurrence edit both reach the server, the event keeps its uid, and the echo writes nothing. --- .../__tests__/calendar/engine.test.ts | 85 ++++++++++++++++++ .../__tests__/calendar/lifecycle.test.ts | 88 +++++++++++++++++++ src/device-sync/calendar/pairs.ts | 85 +++++++++++++++--- src/device-sync/calendar/planner.ts | 2 +- src/device-sync/engine/calendar-sync.ts | 6 ++ 5 files changed, 254 insertions(+), 12 deletions(-) diff --git a/src/device-sync/__tests__/calendar/engine.test.ts b/src/device-sync/__tests__/calendar/engine.test.ts index 76f4c661..c0e3dcc5 100644 --- a/src/device-sync/__tests__/calendar/engine.test.ts +++ b/src/device-sync/__tests__/calendar/engine.test.ts @@ -166,6 +166,91 @@ describe('calendar device edits through the engine', () => { expect(h.server.all('CalendarEvent', 'team')).toEqual([]); }); + describe("Etar's move (delete + insert) with other edits", () => { + /** The event columns Etar copies into the row it inserts when it moves an event. */ + const ETAR_COPY = [ + Events.TITLE, Events.DESCRIPTION, Events.EVENT_LOCATION, Events.STATUS, Events.AVAILABILITY, Events.ACCESS_LEVEL, + Events.EVENT_COLOR, Events.DTSTART, Events.DTEND, Events.DURATION, Events.EVENT_TIMEZONE, Events.ALL_DAY, Events.RRULE, Events.EXDATE, + ]; + function etarMove(h: Harness, masterId: number, calendarRowId: number, reminders: Array<{ minutes: number }>): number { + const old = h.events().find((e) => Number(e._id) === masterId)!; + const values = Object.fromEntries(ETAR_COPY.filter((c) => old[c] !== undefined).map((c) => [c, old[c]])); + h.device.user.deleteEvent(masterId); + return h.device.user.insertEvent(calendarRowId, values, { + reminders: reminders.map((r) => ({ [Reminders.MINUTES]: r.minutes, [Reminders.METHOD]: 1 })), + }); + } + const minutes = (h: Harness, eventId: number) => + h.device.rows('reminders').filter((r) => Number(r[Reminders.EVENT_ID]) === eventId).map((r) => Number(r[Reminders.MINUTES])); + + it('uploads a reminder saved with the move, and keeps the event', async () => { + const h = real(); + const work = h.server.addCalendar('a', { name: 'Work' }); + const id = h.server.addEvent('a', { + uid: 'lunch-uid', + title: 'Lunch', + start: '2026-10-06T12:00:00', + duration: 'PT1H', + timeZone: 'Europe/Berlin', + keywords: { important: true }, + alerts: { al1: { '@type': 'Alert', trigger: { '@type': 'OffsetTrigger', offset: '-PT15M' }, action: 'display' } }, + calendarIds: { [h.calendar]: true }, + }); + await h.run(CALENDAR_AUTHORITY); + const master = Number(h.events().find((e) => e[Events._SYNC_ID] === `a/${id}`)!._id); + const workRow = Number(h.device.rows('calendars').find((c) => c._sync_id === `a/${work}`)!._id); + + const moved = etarMove(h, master, workRow, [{ minutes: 60 }]); + const report = await h.run(CALENDAR_AUTHORITY); + + expect(report).toMatchObject({ outcome: 'ok', itemErrors: [], stats: { uploaded: { created: 0, deleted: 0 } } }); + const event = h.server.get('CalendarEvent', 'a', id)!; + expect(event).toMatchObject({ uid: 'lunch-uid', calendarIds: { [work]: true }, keywords: { important: true } }); + const alerts = Object.values(event.alerts as Record).map((a) => a.trigger.offset); + expect(alerts).toEqual(['-PT1H']); + expect(h.events()).toEqual([expect.objectContaining({ _id: moved, _sync_id: `a/${id}`, [Events.DIRTY]: 0 })]); + expect(minutes(h, moved)).toEqual([60]); + + const before = h.batches.log.length; + await h.run(CALENDAR_AUTHORITY); + expect(h.batches.log.slice(before).filter((ops) => ops.some((op) => op.op !== 'syncState' && op.op !== 'assert'))).toEqual([]); + }); + + it('keeps an occurrence edit that had not reached the server when the series moves', async () => { + const h = real(); + const work = h.server.addCalendar('a', { name: 'Work' }); + const id = h.server.addEvent('a', { + uid: 'standup-uid', + title: 'Standup', + start: '2026-09-28T09:00:00', + duration: 'PT15M', + timeZone: 'Europe/Berlin', + recurrenceRule: { '@type': 'RecurrenceRule', frequency: 'daily', count: 5 }, + recurrenceOverrides: { '2026-09-29T09:00:00': { title: 'Planning' } }, + calendarIds: { [h.calendar]: true }, + }); + await h.run(CALENDAR_AUTHORITY); + const master = Number(h.events().find((e) => e[Events._SYNC_ID] === `a/${id}`)!._id); + const exception = Number(h.events().find((e) => e[Events.ORIGINAL_SYNC_ID] === `a/${id}`)!._id); + const workRow = Number(h.device.rows('calendars').find((c) => c._sync_id === `a/${work}`)!._id); + // The occurrence is renamed on the device; before it syncs, Etar moves the series (it copies only unsynced exceptions). + h.device.user.updateEvent(exception, { [Events.TITLE]: 'Sprint planning' }); + + const moved = etarMove(h, master, workRow, []); + const report = await h.run(CALENDAR_AUTHORITY); + + expect(report).toMatchObject({ outcome: 'ok', itemErrors: [], stats: { uploaded: { created: 0, deleted: 0 } } }); + const event = h.server.get('CalendarEvent', 'a', id)!; + expect(event.calendarIds).toEqual({ [work]: true }); + expect((event.recurrenceOverrides as Overrides)['2026-09-29T09:00:00']).toMatchObject({ title: 'Sprint planning' }); + const rows = h.events(); + expect(rows.filter((e) => e[Events.ORIGINAL_ID] === null)).toEqual([expect.objectContaining({ _id: moved, _sync_id: `a/${id}`, [Events.DIRTY]: 0 })]); + expect(rows.filter((e) => e[Events.ORIGINAL_ID] !== null)).toEqual([ + expect.objectContaining({ [Events.ORIGINAL_ID]: moved, [Events.CALENDAR_ID]: workRow, [Events.TITLE]: 'Sprint planning', [Events.DIRTY]: 0 }), + ]); + }); + }); + it('keeps a description and location cleared on one occurrence cleared', async () => { const h = real(); const id = h.server.addEvent('a', { diff --git a/src/device-sync/__tests__/calendar/lifecycle.test.ts b/src/device-sync/__tests__/calendar/lifecycle.test.ts index 5acc036e..a3cb9182 100644 --- a/src/device-sync/__tests__/calendar/lifecycle.test.ts +++ b/src/device-sync/__tests__/calendar/lifecycle.test.ts @@ -425,6 +425,94 @@ describe('calendar planner: pairs', () => { const events = await h.events(); expect(calendarPlanner.planPairs(events.filter((e) => e.deleted), events.filter((e) => !e.syncId), h.ctx)).toEqual([]); }); + + /** Etar's move of `single()` (delete + insert into Work), with `extra` columns and children on the new row. */ + const lunchColumns = { + [Events.TITLE]: 'Lunch', [Events.DTSTART]: utc('2026-10-06T10:00:00Z'), [Events.DTEND]: utc('2026-10-06T11:00:00Z'), [Events.EVENT_TIMEZONE]: 'Europe/Berlin', [Events.STATUS]: 1, [Events.AVAILABILITY]: 0, [Events.ACCESS_LEVEL]: 0, + }; + async function pairAndApply(h: Harness) { + const events = await h.events(); + const pairs = calendarPlanner.planPairs(events.filter((e) => e.deleted), events.filter((e) => !e.syncId && !e.deleted), h.ctx); + expect(pairs).toHaveLength(1); + await h.apply(pairs[0].ops); + return pairs[0]; + } + const alert15 = { al1: { '@type': 'Alert', trigger: { '@type': 'OffsetTrigger', offset: '-PT15M', relativeTo: 'start' }, action: 'display' } } as CalendarEventWire['alerts']; + + it('keeps a reminder saved with the move for the next upload instead of taking it as the baseline', async () => { + const { h, id } = await synced(single({ alerts: alert15 })); + h.fake.user.deleteEvent(id); + const moved = h.fake.user.insertEvent(h.rowIds.get('w')!, lunchColumns, { reminders: [{ [Reminders.MINUTES]: 60, [Reminders.METHOD]: 1 }] }); + + const pair = await pairAndApply(h); + + expect(pair.actions).toEqual([{ kind: 'update', id: 'e1', patch: { 'calendarIds/b': null, 'calendarIds/w': true } }]); + const after = (await h.local('e1'))!; + expect(after).toMatchObject({ eventId: moved, dirty: true }); + const [update] = actionsOf(calendarPlanner.planUpload(after, h.ctx)); + if (update.kind !== 'update') throw new Error('not an update'); + // Only the reminder is left to send: the move went up with the pair. + expect(Object.keys(update.patch).every((k) => k.startsWith('alerts/'))).toBe(true); + const alerts = serverApply(after.shadow!, update.patch).alerts!; + expect(Object.values(alerts).map((a) => (a.trigger as { offset?: string }).offset)).toEqual(['-PT1H']); + }); + + it('uploads an edit made before the move, which the new row copied', async () => { + const { h, id } = await synced(single({ description: 'Bring salad' })); + h.fake.user.updateEvent(id, { [Events.DESCRIPTION]: 'Bring soup' }); + h.fake.user.deleteEvent(id); + h.fake.user.insertEvent(h.rowIds.get('w')!, { ...lunchColumns, [Events.DESCRIPTION]: 'Bring soup' }); + + await pairAndApply(h); + + const after = (await h.local('e1'))!; + expect(after.dirty).toBe(true); + expect(actionsOf(calendarPlanner.planUpload(after, h.ctx))).toEqual([{ kind: 'update', id: 'e1', patch: { description: 'Bring soup' } }]); + }); + + it('takes attendee rows an app re-inserted with constant details as they are, uploading nothing for them', async () => { + const participants = { + bob: { '@type': 'Participant', calendarAddress: 'mailto:bob@example.net', name: 'Bob', roles: { attendee: true, optional: true }, participationStatus: 'accepted' }, + carol: { '@type': 'Participant', calendarAddress: 'mailto:carol@example.net', name: 'Carol', roles: { attendee: true }, participationStatus: 'declined' }, + } as CalendarEventWire['participants']; + const { h, id } = await synced(single({ participants, organizerCalendarAddress: 'mailto:boss@example.net' })); + const attendees = h.fake.rows('attendees', `${Attendees.EVENT_ID} = ?`, [id]).map((a) => ({ + // Etar writes every attendee it re-inserts as a required attendee without a status. + [Attendees.ATTENDEE_EMAIL]: a[Attendees.ATTENDEE_EMAIL], [Attendees.ATTENDEE_NAME]: a[Attendees.ATTENDEE_NAME], + [Attendees.ATTENDEE_RELATIONSHIP]: 1, [Attendees.ATTENDEE_TYPE]: 1, [Attendees.ATTENDEE_STATUS]: 0, + })); + const organizer = h.row(id)[Events.ORGANIZER]; + h.fake.user.deleteEvent(id); + h.fake.user.insertEvent(h.rowIds.get('w')!, { ...lunchColumns, [Events.ORGANIZER]: organizer }, { attendees }); + + await pairAndApply(h); + + const after = (await h.local('e1'))!; + expect(after.dirty).toBe(false); + expect(calendarPlanner.planUpload({ ...after, dirty: true }, h.ctx).kind).toBe('clean'); + }); + + it('keeps the timing of a series turned single against the series, so a start moved before uploads', async () => { + const { h, id } = await synced(single({ recurrenceRule: { frequency: 'daily' } })); + // Moved an hour later in place, then turned into a single event (Etar: delete + insert). + h.fake.user.updateEvent(id, { [Events.DTSTART]: utc('2026-10-06T11:00:00Z') }); + h.fake.user.deleteEvent(id); + h.fake.user.insertEvent(h.rowIds.get('b')!, { ...lunchColumns, [Events.DTSTART]: utc('2026-10-06T11:00:00Z'), [Events.DTEND]: utc('2026-10-06T12:00:00Z') }); + + const pair = await pairAndApply(h); + + expect(pair.actions).toEqual([{ kind: 'update', id: 'e1', patch: { recurrenceRule: null } }]); + const after = (await h.local('e1'))!; + expect(after.dirty).toBe(true); + expect(actionsOf(calendarPlanner.planUpload(after, h.ctx))).toEqual([{ kind: 'update', id: 'e1', patch: { start: '2026-10-06T13:00:00' } }]); + + // Without the earlier edit, the single event matches what the server holds after the pair: nothing left. + const plain = await synced(single({ recurrenceRule: { frequency: 'daily' } })); + plain.h.fake.user.deleteEvent(plain.id); + plain.h.fake.user.insertEvent(plain.h.rowIds.get('b')!, lunchColumns); + await pairAndApply(plain.h); + expect((await plain.h.local('e1'))!.dirty).toBe(false); + }); }); describe('calendar planner: baselines, zones and reminder owners', () => { diff --git a/src/device-sync/calendar/pairs.ts b/src/device-sync/calendar/pairs.ts index 80513bd2..1788bfcb 100644 --- a/src/device-sync/calendar/pairs.ts +++ b/src/device-sync/calendar/pairs.ts @@ -9,17 +9,20 @@ * target account and a destroy in the source. */ import { Attendees, Events } from '../android-columns'; -import type { CalendarContext, LocalEvent, LocalEventRow, OpGroup } from '../planner'; +import type { CalendarContext, EventBaseline, LocalEvent, LocalEventRow, OpGroup } from '../planner'; import type { Row } from '../types'; import type { CalendarEventWire } from '../wire'; import { exceptionRef, parseObjectRef } from '../common/ids'; +import { clone } from '../common/json'; import { applyPatch, type PatchObject } from '../common/patch'; import { EXCEPTION_CELLS, MASTER_CELLS, encodeBaseline, makeBaseline } from './columns'; import { durationTextSeconds } from './duration'; import { isExcluded, overridesOf } from './exceptions'; -import { GroupBuilder, assertRow, deleteRow, updateRow } from './rows'; +import { linkedExceptionCount } from './merge'; +import { GroupBuilder, assertExceptionCount, assertRow, deleteRow, updateRow } from './rows'; import { rruleParts } from './rrule'; import { sendsSchedulingMessages } from './scheduling'; +import { MASTER_COLUMN_UNITS, deviceChangedUnit, exdateEntries, remindersDiffer, sideOfBaseline, sideOfRow } from './units'; import { put } from './upload'; import { isUnsetStatus } from './values'; @@ -101,24 +104,80 @@ export function pairPatch( return { patch, sendSchedulingMessages: scheduling }; } +const children = (row: LocalEventRow) => ({ attendees: row.attendees.map((a) => a.cells), reminders: row.reminders.map((r) => r.cells) }); + /** - * Applied once the pair's patch is accepted: the old row and its exception - * rows go, the new row takes the identity (its copied exception rows the - * overrides that still exist), and the shadow is the old one with the patch - * applied until the next download brings the server's. + * The new row's baseline: the old row's (what the server held, as last + * written) with what the patch changed, the calendar and, for a series turned + * single, its rule and its timing as a single event. So an edit saved with the + * move, or made before it, still differs from the baseline and uploads next. + * Attendee rows are the app's re-insert (Etar writes every attendee it copies + * as required, without a status), so their details are no edits: they are + * taken as they are, like the reminders of a row that copied none. */ -export function pairOps(deleted: LocalEvent, fresh: LocalEvent, patch: PatchObject): OpGroup { +function pairedBaseline(deleted: LocalEvent, fresh: LocalEvent, kind: PairKind, ctx: CalendarContext): EventBaseline { + const own = makeBaseline(fresh.cells, MASTER_CELLS, children(fresh).attendees, children(fresh).reminders); + if (!deleted.baseline) return own; + const next = clone(deleted.baseline); + next.cells[Events.CALENDAR_ID] = own.cells[Events.CALENDAR_ID]; + next.attendees = own.attendees; + if (ctx.reminderOwner !== 'device' || !fresh.reminders.length) next.reminders = own.reminders; + if (kind.unrecur) { + const start = num(deleted.baseline.cells[Events.DTSTART]); + const length = lengthMs(deleted.baseline.cells); + next.cells[Events.DTEND] = start !== null && length !== null ? start + length : own.cells[Events.DTEND]; + next.cells[Events.DURATION] = null; + next.cells[Events.RRULE] = own.cells[Events.RRULE]; + next.cells[Events.EXDATE] = own.cells[Events.EXDATE]; + } + return next; +} + +/** Whether a master holds a change against `baseline` that an upload would look at. */ +function changedSince(row: LocalEventRow, baseline: EventBaseline, ctx: CalendarContext): boolean { + const cur = sideOfRow(row); + const bl = sideOfBaseline(baseline); + if (MASTER_COLUMN_UNITS.some((unit) => deviceChangedUnit(unit, cur, bl))) return true; + const [now, before] = [exdateEntries(cur), exdateEntries(bl)]; + if (now.size !== before.size || [...now].some((e) => !before.has(e))) return true; + return ctx.reminderOwner === 'device' && remindersDiffer(cur, bl); +} + +/** + * Applied once the pair's patch is accepted: the new row takes the identity + * and the old row goes. The old row's exception rows move to the new row with + * what they wait to upload (Etar copies only the exceptions it never synced; + * an app's copy of one the old row has goes). A copied exception takes the + * identity of its override. The shadow is the old one with the patch applied + * until the server's comes back. A row that still differs from its baseline + * (see `pairedBaseline`) stays dirty and uploads next. + */ +export function pairOps(deleted: LocalEvent, fresh: LocalEvent, kind: PairKind, patch: PatchObject, ctx: CalendarContext): OpGroup { const syncId = deleted.syncId!; const shadow: CalendarEventWire = applyPatch(deleted.shadow!, patch) ?? deleted.shadow!; const group = new GroupBuilder(syncId); assertRow(group, deleted, false, false); + for (const x of deleted.exceptions) assertRow(group, x, true, x.dirty || x.deleted); + assertExceptionCount(group, deleted.eventId, linkedExceptionCount(deleted)); assertRow(group, fresh, false, true); for (const x of fresh.exceptions) assertRow(group, x, true, true); - for (const x of deleted.exceptions) deleteRow(group, x.eventId); + assertExceptionCount(group, fresh.eventId, linkedExceptionCount(fresh)); + const kept = new Set(); + for (const x of deleted.exceptions) { + // A series turned single has no occurrences left. + if (kind.unrecur || !x.recurrenceId) { + deleteRow(group, x.eventId); + continue; + } + kept.add(x.recurrenceId); + const values: Row = { [Events.ORIGINAL_ID]: fresh.eventId, [Events.CALENDAR_ID]: fresh.calendarRowId }; + if (x.baseline) values[Events.SYNC_DATA4] = encodeBaseline({ ...x.baseline, cells: { ...x.baseline.cells, [Events.CALENDAR_ID]: fresh.calendarRowId } }); + group.write({ op: 'update', table: 'events', id: x.eventId, values, expectCount: 1 }); + } deleteRow(group, deleted.eventId); - const children = (row: LocalEventRow) => ({ attendees: row.attendees.map((a) => a.cells), reminders: row.reminders.map((r) => r.cells) }); const cells: Row = {}; for (const c of MASTER_CELLS) cells[c] = fresh.cells[c] ?? null; + const baseline = pairedBaseline(deleted, fresh, kind, ctx); updateRow( group, { ...fresh, shadow: null }, @@ -126,18 +185,22 @@ export function pairOps(deleted: LocalEvent, fresh: LocalEvent, patch: PatchObje isException: false, cells, ...children(fresh), - baseline: makeBaseline(fresh.cells, MASTER_CELLS, children(fresh).attendees, children(fresh).reminders), + baseline, syncId, uid: typeof shadow.uid === 'string' ? shadow.uid : null, shadow, pending: null, - clearDirty: true, + clearDirty: !changedSince(fresh, baseline, ctx), hasParticipants: fresh.attendees.length > 0, }, ); const overrides = overridesOf(shadow); for (const x of fresh.exceptions) { const key = x.recurrenceId; + if (key && kept.has(key)) { + deleteRow(group, x.eventId); + continue; + } const values: Row = { [Events.ORIGINAL_SYNC_ID]: syncId }; if (key && overrides[key] && !isExcluded(overrides[key])) { Object.assign(values, { diff --git a/src/device-sync/calendar/planner.ts b/src/device-sync/calendar/planner.ts index 08a2ac15..e5051242 100644 --- a/src/device-sync/calendar/planner.ts +++ b/src/device-sync/calendar/planner.ts @@ -385,7 +385,7 @@ function planPairs( if (!pair) continue; const action: UploadAction = { kind: 'update', id: ref.id, patch: pair.patch }; if (pair.sendSchedulingMessages) action.sendSchedulingMessages = true; - pairs.push({ deleted: d, fresh: f, actions: [action], ops: pairOps(d, f, pair.patch) }); + pairs.push({ deleted: d, fresh: f, actions: [action], ops: pairOps(d, f, kind, pair.patch, ctx) }); used.add(f.eventId); break; } diff --git a/src/device-sync/engine/calendar-sync.ts b/src/device-sync/engine/calendar-sync.ts index 1cb3d787..a4112297 100644 --- a/src/device-sync/engine/calendar-sync.ts +++ b/src/device-sync/engine/calendar-sync.ts @@ -492,6 +492,12 @@ export class CalendarSync extends ItemSync { if (ownId) this.markStale(acct, ownId); return []; } + if (this.eventKind.meta(moved).dirty) { + // Edits saved with the move, or made before it, are still to go up: the update phase + // sends them as the moved event's, and its accepted write brings the server's version. + this.env.report.stats.uploaded.updated++; + return []; + } const held: Held = { kind: this.eventKind, local: moved }; try { const work = this.acceptWork(acct, { held, meta: this.eventKind.meta(moved), fingerprint: '', actions: [] }, server, false); From 1a348792c0c6c84ec191731d78ec613e3af0af09 Mon Sep 17 00:00:00 2001 From: Linus Rath <139418639+rathlinus@users.noreply.github.com> Date: Sun, 27 Sep 2026 08:40:57 +0200 Subject: [PATCH 58/87] fix: let the server's deletion win over an event moved on the phone #34 Etar moves an event by deleting the row and inserting a copy. When the event was deleted on the server meanwhile, the sync removed the deleted row but kept the copy, which then uploaded as a new event: the event the server had deleted came back in the target calendar, for everyone. That happened whether the download found the event gone first, which is the usual case, or the patch of the move did. A move is an edit, and the server's deletion wins over an edit: the copy that pairs with the deleted row now goes with it, and the report counts the conflict. A copy edited beyond the move stays a new event. Checked through the engine with the event destroyed before the run, while the move was sent, and with the patch answered notFound. --- .../__tests__/calendar/engine.test.ts | 59 +++++++++++++++++++ src/device-sync/engine/calendar-sync.ts | 34 +++++++++++ src/device-sync/engine/item-sync.ts | 20 +++++-- 3 files changed, 109 insertions(+), 4 deletions(-) diff --git a/src/device-sync/__tests__/calendar/engine.test.ts b/src/device-sync/__tests__/calendar/engine.test.ts index c0e3dcc5..6aacfcf8 100644 --- a/src/device-sync/__tests__/calendar/engine.test.ts +++ b/src/device-sync/__tests__/calendar/engine.test.ts @@ -216,6 +216,65 @@ describe('calendar device edits through the engine', () => { expect(h.batches.log.slice(before).filter((ops) => ops.some((op) => op.op !== 'syncState' && op.op !== 'assert'))).toEqual([]); }); + describe('when the server deleted the event meanwhile', () => { + async function movedLunch() { + const h = real(); + const work = h.server.addCalendar('a', { name: 'Work' }); + const id = h.server.addEvent('a', { + uid: 'lunch-uid', + title: 'Lunch', + start: '2026-10-06T12:00:00', + duration: 'PT1H', + timeZone: 'Europe/Berlin', + calendarIds: { [h.calendar]: true }, + }); + await h.run(CALENDAR_AUTHORITY); + const master = Number(h.events().find((e) => e[Events._SYNC_ID] === `a/${id}`)!._id); + const workRow = Number(h.device.rows('calendars').find((c) => c._sync_id === `a/${work}`)!._id); + etarMove(h, master, workRow, [{ minutes: 30 }]); + return { h, id }; + } + const creates = (h: Harness) => + h.server.calls('CalendarEvent/set').filter(([, args]) => Object.keys((args as { create?: object }).create ?? {}).length); + + it('removes the moved event when the patch of the move finds it gone', async () => { + const { h, id } = await movedLunch(); + h.server.setErrorFor('CalendarEvent', 'a', id, { type: 'notFound' }); + + const report = await h.run(CALENDAR_AUTHORITY); + + expect(report).toMatchObject({ outcome: 'ok', conflicts: 1, stats: { uploaded: { created: 0 } } }); + expect(creates(h)).toEqual([]); + expect(h.events()).toEqual([]); + }); + + it('removes the moved event when the download finds it destroyed', async () => { + const { h, id } = await movedLunch(); + h.server.serverDestroy('CalendarEvent', 'a', id); + + const report = await h.run(CALENDAR_AUTHORITY); + + expect(report).toMatchObject({ outcome: 'ok', itemErrors: [], conflicts: 1, stats: { uploaded: { created: 0 } } }); + expect(h.server.all('CalendarEvent', 'a')).toEqual([]); + expect(h.events()).toEqual([]); + }); + + it('removes the moved event when it is destroyed while the move is on its way', async () => { + const { h, id } = await movedLunch(); + const stop = h.server.onBeforeRequest((request) => { + if (!request.methods.includes('CalendarEvent/set')) return; + stop(); + h.server.serverDestroy('CalendarEvent', 'a', id); + }); + + const report = await h.run(CALENDAR_AUTHORITY); + + expect(report).toMatchObject({ outcome: 'ok', itemErrors: [], stats: { uploaded: { created: 0 } } }); + expect(h.server.all('CalendarEvent', 'a')).toEqual([]); + expect(h.events()).toEqual([]); + }); + }); + it('keeps an occurrence edit that had not reached the server when the series moves', async () => { const h = real(); const work = h.server.addCalendar('a', { name: 'Work' }); diff --git a/src/device-sync/engine/calendar-sync.ts b/src/device-sync/engine/calendar-sync.ts index a4112297..12dcb233 100644 --- a/src/device-sync/engine/calendar-sync.ts +++ b/src/device-sync/engine/calendar-sync.ts @@ -515,6 +515,40 @@ export class CalendarSync extends ItemSync { await this.send(acct, pending); } + /** + * A deleted row whose event the server destroyed may be half of a move the + * device made (a pair not uploaded yet): the server's deletion wins over the + * move, an edit, so the new row goes too instead of being created (a conflict). + */ + protected async alongDestroyed(acct: string, removed: Held[]): Promise { + const deleted = removed.map((h) => h.local as LocalEvent).filter((e) => e.deleted); + if (!deleted.length) return []; + const fresh = (await this.eventKind.loadNew()).filter((e) => { + const m = this.eventKind.meta(e); + return !m.deleted && !m.pending && this.accountOfItem({ kind: this.eventKind, local: e }) === acct; + }); + if (!fresh.length) return []; + let pairs: PairPlan[]; + try { + pairs = this.planner.planPairs(deleted, fresh, this.ctx(acct)); + } catch (error) { + this.plannerFailed(acct, `pairs:${acct}`, 'download', error); + return []; + } + const conflict = (work: Work): Work => ({ + ...work, + applied: async (results) => { + await work.applied?.(results); + this.env.report.conflicts++; + }, + replan: work.replan && (async () => { + const next = await work.replan!(); + return next && conflict(next); + }), + }); + return pairs.map((pair) => conflict(this.purgeWork(acct, { kind: this.eventKind, local: pair.fresh }, false))); + } + /** * A pair's rows after the server took its patch. When their assert fails * (an app changed them during the upload), both rows are read and paired diff --git a/src/device-sync/engine/item-sync.ts b/src/device-sync/engine/item-sync.ts index 8473f1e5..76ad8fa6 100644 --- a/src/device-sync/engine/item-sync.ts +++ b/src/device-sync/engine/item-sync.ts @@ -165,6 +165,14 @@ export abstract class ItemSync { return []; } protected onDiscard(_next: SyncState, _accounts: Set): void {} + /** + * Local items that go with the rows of objects the server destroyed + * (calendar: the new row of a move the device made of a deleted event; the + * server's deletion wins over the move). Written with those rows. + */ + protected async alongDestroyed(_acct: string, _removed: Held[]): Promise { + return []; + } /** Whether a pending create of `target` may adopt this object (contacts: the target book holds it). */ protected adoptionTargetMatches(acct: string, _object: ServerObject, target: string): boolean { return accountOfRef(target) === acct; @@ -582,6 +590,7 @@ export abstract class ItemSync { if (work) works.push(work); else this.markResolved(acct, object.id); } + const removed: Held[] = []; for (const id of allGone) { const found = held.get(refOf(acct, id)); let work: Work | null; @@ -591,9 +600,12 @@ export abstract class ItemSync { this.plannerFailed(acct, refOf(acct, id), 'download', error, id); continue; } - if (work) works.push(work); - else this.markResolved(acct, id); + if (work && found) { + works.push(work); + removed.push(found); + } else this.markResolved(acct, id); } + if (removed.length) works.push(...(await this.alongDestroyed(acct, removed))); await this.env.writer.write(works, mutate ? this.tail([acct], mutate) : undefined); await this.readBack(written); } @@ -1320,10 +1332,10 @@ export abstract class ItemSync { return this.succeeded(acct, p, done, objects); } if (failure.action.kind === 'update') { - // Deleted on the server meanwhile: the server delete wins, the rows go. + // Deleted on the server meanwhile: the server delete wins, the rows go (a pair's new row too). const id = idInAccount(p.meta.sourceId, acct); const work = id ? this.removeWork(acct, id, p.held, 'destroyed') : null; - return work ? [work] : []; + return work ? [work, ...(await this.alongDestroyed(acct, [p.held]))] : []; } return [this.poisonWork(p, failure.error)]; case 'forbidden': From 1c75da873343224f06b670d68efc199e687d9e8a Mon Sep 17 00:00:00 2001 From: Linus Rath <139418639+rathlinus@users.noreply.github.com> Date: Sun, 27 Sep 2026 08:41:39 +0200 Subject: [PATCH 59/87] fix: remove an event created in a read-only calendar and say so #34 An event created on the phone in a calendar the user may only read was meant to be removed, since the server takes no changes there, but the claim step ignored that answer: the event stayed on the phone, was never sent or reported, and turning sync off always warned that a change had not reached the server. The claim step now applies it: the row goes, the report lists it as read-only, and nothing is left waiting. --- .../__tests__/calendar/engine.test.ts | 38 +++++++++++++++++++ src/device-sync/engine/item-sync.ts | 5 +++ 2 files changed, 43 insertions(+) diff --git a/src/device-sync/__tests__/calendar/engine.test.ts b/src/device-sync/__tests__/calendar/engine.test.ts index 6aacfcf8..fd298087 100644 --- a/src/device-sync/__tests__/calendar/engine.test.ts +++ b/src/device-sync/__tests__/calendar/engine.test.ts @@ -310,6 +310,44 @@ describe('calendar device edits through the engine', () => { }); }); + it('removes an event created in a read-only calendar and reports it', async () => { + const h = real({ accounts: 'withShared' }); + const team = h.server.addCalendar('team', { + name: 'Team events', + myRights: { mayReadFreeBusy: true, mayReadItems: true, mayWriteAll: false, mayWriteOwn: false, mayUpdatePrivate: false, mayRSVP: false, mayShare: false, mayDelete: false }, + }); + h.prefs.calendarSelection[`team/${team}`] = true; + await h.run(CALENDAR_AUTHORITY); + const teamRow = Number(h.device.rows('calendars').find((c) => c._sync_id === `team/${team}`)!._id); + + const id = h.device.user.insertEvent(teamRow, { + [Events.TITLE]: 'Mine', + [Events.DTSTART]: Date.UTC(2026, 9, 14, 13), + [Events.DTEND]: Date.UTC(2026, 9, 14, 14), + [Events.EVENT_TIMEZONE]: 'Europe/Berlin', + }); + const report = await h.run(CALENDAR_AUTHORITY); + + expect(report).toMatchObject({ outcome: 'ok', stats: { uploaded: { created: 0 }, skipped: 1 } }); + expect(report.itemErrors).toEqual([expect.objectContaining({ ref: `row:${id}`, side: 'upload', type: 'readOnly' })]); + expect(h.events().find((e) => Number(e._id) === id)).toBeUndefined(); + expect(h.server.all('CalendarEvent', 'team')).toEqual([]); + // Nothing is left waiting: the next run has nothing to say about it. + expect((await h.run(CALENDAR_AUTHORITY)).itemErrors).toEqual([]); + }); + + it('leaves nothing waiting for turning sync off after an event was created in a read-only calendar', async () => { + const h = real({ accounts: 'withShared' }); + const team = h.server.addCalendar('team', { name: 'Team events', myRights: { mayReadItems: true, mayWriteAll: false, mayWriteOwn: false, mayRSVP: false } }); + h.prefs.calendarSelection[`team/${team}`] = true; + await h.run(CALENDAR_AUTHORITY); + const teamRow = Number(h.device.rows('calendars').find((c) => c._sync_id === `team/${team}`)!._id); + h.device.user.insertEvent(teamRow, { [Events.TITLE]: 'Mine', [Events.DTSTART]: Date.UTC(2026, 9, 14, 13), [Events.DTEND]: Date.UTC(2026, 9, 14, 14), [Events.EVENT_TIMEZONE]: 'Europe/Berlin' }); + + expect(await h.teardown(CALENDAR_AUTHORITY)).toEqual({ pending: 0 }); + expect(h.server.all('CalendarEvent', 'team')).toEqual([]); + }); + it('keeps a description and location cleared on one occurrence cleared', async () => { const h = real(); const id = h.server.addEvent('a', { diff --git a/src/device-sync/engine/item-sync.ts b/src/device-sync/engine/item-sync.ts index 76ad8fa6..f4c8275f 100644 --- a/src/device-sync/engine/item-sync.ts +++ b/src/device-sync/engine/item-sync.ts @@ -912,6 +912,11 @@ export abstract class ItemSync { if (plan.kind === 'skip') { this.env.report.stats.skipped++; this.env.report.itemError({ ref, side: 'upload', type: plan.reason }); + } else if (plan.kind === 'revert') { + // It may not be created where it was made (a read-only calendar): the row goes, and the report says so. + this.env.report.stats.skipped++; + this.env.report.itemError({ ref, side: 'upload', type: 'readOnly' }); + works.push(this.planWork(acct, held, 'revert', plan.ops)); } else if (plan.kind === 'claim' || plan.kind === 'purge' || plan.kind === 'clean') { works.push(this.planWork(acct, held, plan.kind, plan.ops)); } From d1ae713b1f12cbe65e2824e5e17e532e8c8726cb Mon Sep 17 00:00:00 2001 From: Linus Rath <139418639+rathlinus@users.noreply.github.com> Date: Sun, 27 Sep 2026 08:42:23 +0200 Subject: [PATCH 60/87] fix: report changes the phone puts back because the server can't take them #34 A change the server cannot take was put back on the phone without a word: an edit in a read-only calendar or address book, an answer where answers are not allowed, an event moved to another account's calendar, an occurrence the server keeps without its series, a contact added to a group this device can't change. The change just vanished. The planners now say why they put a change back, and the sync report lists the item with that reason (`readOnly`, `rsvpRefused`, `instanceOnly`, `crossAccountMove`, `groupNotWritable`). A contact whose only change still uploads through a group card is not listed. A refused edit was also reported as "invitations not sent", as if it had been stored without them; the note now appears only when the change was stored without its scheduling messages. --- .../__tests__/calendar/edge-cases.test.ts | 4 +-- .../__tests__/calendar/engine.test.ts | 33 ++++++++++++++++++- .../__tests__/calendar/exceptions.test.ts | 2 +- .../__tests__/calendar/lifecycle.test.ts | 2 +- .../__tests__/calendar/upload.test.ts | 6 +++- .../__tests__/contacts/groups.test.ts | 13 +++++--- .../__tests__/contacts/real-engine.test.ts | 1 + .../__tests__/contacts/upload-editors.test.ts | 4 +-- src/device-sync/calendar/planner.ts | 14 ++++---- src/device-sync/calendar/upload.ts | 7 +++- src/device-sync/contacts/groups.ts | 6 ++-- src/device-sync/contacts/upload.ts | 9 +++-- src/device-sync/engine/item-sync.ts | 13 +++++--- src/device-sync/planner.ts | 5 +-- 14 files changed, 88 insertions(+), 31 deletions(-) diff --git a/src/device-sync/__tests__/calendar/edge-cases.test.ts b/src/device-sync/__tests__/calendar/edge-cases.test.ts index b32cb6e4..2f9b1da3 100644 --- a/src/device-sync/__tests__/calendar/edge-cases.test.ts +++ b/src/device-sync/__tests__/calendar/edge-cases.test.ts @@ -15,7 +15,7 @@ describe('calendar planner: edge cases', () => { const { h, id } = await synced(single({ calendarIds: { r: true } })); h.fake.user.deleteEvent(id); const plan = calendarPlanner.planUpload((await h.local('e1'))!, h.ctx); - expect(plan).toMatchObject({ kind: 'revert', refetch: true }); + expect(plan).toMatchObject({ kind: 'revert', refetch: true, reason: 'readOnly' }); if (plan.kind === 'revert') await h.apply(plan.ops); expect(await h.events()).toEqual([]); }); @@ -24,7 +24,7 @@ describe('calendar planner: edge cases', () => { const { h, id } = await synced(single({ recurrenceId: '2026-10-06T12:00:00', recurrenceIdTimeZone: 'Europe/Berlin' })); h.fake.user.updateEvent(id, { [Events.TITLE]: 'Mine' }); const plan = calendarPlanner.planUpload((await h.local('e1'))!, h.ctx); - expect(plan.kind).toBe('revert'); + expect(plan).toMatchObject({ kind: 'revert', reason: 'instanceOnly' }); if (plan.kind === 'revert') await h.apply(plan.ops); expect(h.row(id)).toMatchObject({ [Events.TITLE]: 'Lunch', [Events.DIRTY]: 0 }); }); diff --git a/src/device-sync/__tests__/calendar/engine.test.ts b/src/device-sync/__tests__/calendar/engine.test.ts index fd298087..10418fd6 100644 --- a/src/device-sync/__tests__/calendar/engine.test.ts +++ b/src/device-sync/__tests__/calendar/engine.test.ts @@ -129,6 +129,36 @@ describe('calendar device edits through the engine', () => { expect(report.itemErrors).toEqual([expect.objectContaining({ ref: `team/${id}`, side: 'upload', type: 'forbidden' })]); expect(h.events().filter((e) => e[Events.ORIGINAL_SYNC_ID] === `team/${id}`)).toEqual([]); expect(h.events().every((e) => Number(e[Events.DIRTY] ?? 0) === 0)).toBe(true); + // Nothing was stored at all: the report does not claim it was, without invitations. + expect(report.message ?? '').not.toContain('invitations not sent'); + }); + + it('notes that invitations were not sent only when the change was stored without them', async () => { + const h = real(); + const id = h.server.addEvent('a', { + uid: 'review-uid', + title: 'Review', + start: '2026-10-06T12:00:00', + duration: 'PT1H', + timeZone: 'Europe/Berlin', + organizerCalendarAddress: 'mailto:alice@example.com', + participants: { + me: { '@type': 'Participant', calendarAddress: 'mailto:alice@example.com', roles: { owner: true, attendee: true }, participationStatus: 'accepted' }, + bob: { '@type': 'Participant', calendarAddress: 'mailto:bob@example.com', roles: { attendee: true }, participationStatus: 'needs-action' }, + }, + calendarIds: { [h.calendar]: true }, + }); + await h.run(CALENDAR_AUTHORITY); + const row = Number(h.events().find((e) => e[Events._SYNC_ID] === `a/${id}`)!._id); + h.device.user.updateEvent(row, { [Events.TITLE]: 'Design review' }); + // The server refuses the scheduling messages once (e.g. scheduling turned off): the change goes up without them. + h.server.setErrorFor('CalendarEvent', 'a', id, { type: 'forbidden' }); + + const report = await h.run(CALENDAR_AUTHORITY); + + expect(report).toMatchObject({ outcome: 'ok', itemErrors: [], stats: { uploaded: { updated: 1 } } }); + expect(report.message).toContain('invitations not sent'); + expect(h.server.get('CalendarEvent', 'a', id)).toMatchObject({ title: 'Design review' }); }); it('puts an event an app moved in place to a calendar of another account back into its calendar', async () => { @@ -155,8 +185,9 @@ describe('calendar device edits through the engine', () => { expect(row()).toMatchObject({ [Events.CALENDAR_ID]: calendarRow(`a/${h.calendar}`), [Events.DIRTY]: 0 }); expect(h.server.get('CalendarEvent', 'a', id)!.calendarIds).toEqual({ [h.calendar]: true }); expect(h.server.all('CalendarEvent', 'team')).toEqual([]); - // Nothing was sent; the change counts as skipped (a report entry needs the engine to report reverts). + // Nothing was sent; the change counts as skipped, and the report says why. expect(report).toMatchObject({ outcome: 'ok', stats: { uploaded: { created: 0, updated: 0, deleted: 0 }, skipped: 1 } }); + expect(report.itemErrors).toEqual([expect.objectContaining({ ref: `a/${id}`, side: 'upload', type: 'crossAccountMove' })]); // Together with another edit: that one uploads, and the row goes back all the same. h.device.user.updateEvent(Number(row()._id), { [Events.CALENDAR_ID]: calendarRow(`team/${team}`), [Events.TITLE]: 'Team lunch' }); diff --git a/src/device-sync/__tests__/calendar/exceptions.test.ts b/src/device-sync/__tests__/calendar/exceptions.test.ts index 965bbf38..2163d4c1 100644 --- a/src/device-sync/__tests__/calendar/exceptions.test.ts +++ b/src/device-sync/__tests__/calendar/exceptions.test.ts @@ -249,7 +249,7 @@ describe('calendar planner: recurrence edits made on the device', () => { expect(action.sendSchedulingMessages).toBe(true); // Anything more than the answer is still the calendar owner's to change. h.fake.user.updateEvent(added, { [Events.TITLE]: 'Mine now' }); - expect(calendarPlanner.planUpload((await h.local('bl'))!, h.ctx).kind).toBe('revert'); + expect(calendarPlanner.planUpload((await h.local('bl'))!, h.ctx)).toMatchObject({ kind: 'revert', reason: 'readOnly' }); }); it('turns EXDATE entries an app adds or removes into exclusions and re-inclusions', async () => { diff --git a/src/device-sync/__tests__/calendar/lifecycle.test.ts b/src/device-sync/__tests__/calendar/lifecycle.test.ts index a3cb9182..52546c45 100644 --- a/src/device-sync/__tests__/calendar/lifecycle.test.ts +++ b/src/device-sync/__tests__/calendar/lifecycle.test.ts @@ -189,7 +189,7 @@ describe('calendar planner: new events', () => { const h = await new Harness().setup(); const id = h.fake.user.insertEvent(h.rowIds.get('r')!, { [Events.TITLE]: 'x', [Events.DTSTART]: 0, [Events.DTEND]: 1000, [Events.EVENT_TIMEZONE]: 'UTC', [Events.STATUS]: 1 }); const plan = calendarPlanner.planUpload(await h.byRow(id), h.ctx); - expect(plan.kind).toBe('revert'); + expect(plan).toMatchObject({ kind: 'revert', reason: 'readOnly' }); if (plan.kind === 'revert') await h.apply(plan.ops); expect(h.row(id)).toBeUndefined(); }); diff --git a/src/device-sync/__tests__/calendar/upload.test.ts b/src/device-sync/__tests__/calendar/upload.test.ts index 5e22fcfb..edd940c6 100644 --- a/src/device-sync/__tests__/calendar/upload.test.ts +++ b/src/device-sync/__tests__/calendar/upload.test.ts @@ -285,12 +285,16 @@ describe('calendar planner: uploads of edited events', () => { const { h, id } = await synced(event); h.fake.user.updateEvent(id, { [Events.TITLE]: 'Mine now' }); const plan = calendarPlanner.planUpload((await h.local('e1'))!, h.ctx); - expect(plan.kind).toBe('revert'); + expect(plan).toMatchObject({ kind: 'revert', reason: 'readOnly' }); if (plan.kind === 'revert') await h.apply(plan.ops); expect(h.row(id)).toMatchObject({ [Events.TITLE]: 'Lunch', [Events.DIRTY]: 0 }); h.fake.user.setAttendeeStatus(id, ME, 2); expect(updateOf(calendarPlanner.planUpload((await h.local('e1'))!, h.ctx)).patch).toEqual({ 'participants/me/participationStatus': 'declined' }); + + // Without mayRSVP, the answer is put back too, and says why. + const noRsvp = { ...h.ctx, calendar: (cid: string) => (cid === 'r' ? { ...h.ctx.calendar(cid)!, myRights: { mayReadItems: true } } : h.ctx.calendar(cid)) }; + expect(calendarPlanner.planUpload((await h.local('e1'))!, noRsvp)).toMatchObject({ kind: 'revert', reason: 'rsvpRefused' }); }); it('skips a poisoned item until it changes or the back-off ends', async () => { diff --git a/src/device-sync/__tests__/contacts/groups.test.ts b/src/device-sync/__tests__/contacts/groups.test.ts index 786271b9..14323ae2 100644 --- a/src/device-sync/__tests__/contacts/groups.test.ts +++ b/src/device-sync/__tests__/contacts/groups.test.ts @@ -182,7 +182,7 @@ describe('contacts groups: membership edits', () => { const gid = await readOnlyGroup(h, { 'urn:uuid:someone': true }); const { rawId } = await h.seed(appleCard()); h.device.user.addToGroup(rawId, (await localGroup(h, gid)).groupId); - expect((await h.upload(rawId)).kind).toBe('revert'); + expect(await h.upload(rawId)).toMatchObject({ kind: 'revert', reason: 'groupNotWritable' }); expect(h.membershipGroups(rawId)).toEqual([]); expect(h.dirty(rawId)).toBe(false); expect(h.planner.planMembershipUploads([await h.contact(rawId)], await h.groups(), h.ctx)).toEqual([]); @@ -194,7 +194,7 @@ describe('contacts groups: membership edits', () => { const { rawId } = await h.seed(appleCard()); const membership = (await h.contact(rawId)).rows.find((r) => r.mimetype === MimeType.GROUP_MEMBERSHIP)!; h.device.user.deleteData(membership.id); - expect((await h.upload(rawId)).kind).toBe('revert'); + expect(await h.upload(rawId)).toMatchObject({ kind: 'revert', reason: 'groupNotWritable' }); expect(h.membershipGroups(rawId)).toEqual([`${JMAP}/${gid}`]); expect(h.dirty(rawId)).toBe(false); }); @@ -209,7 +209,7 @@ describe('contacts groups: membership edits', () => { const { rawId } = await h.seed(appleCard()); h.device.user.addToGroup(rawId, (await h.groups()).find((g) => g.sourceId === `t/${team}`)!.groupId); expect(h.membershipGroups(rawId)).toEqual([`t/${team}`]); - expect((await h.upload(rawId)).kind).toBe('revert'); + expect(await h.upload(rawId)).toMatchObject({ kind: 'revert', reason: 'groupNotWritable' }); expect(h.membershipGroups(rawId)).toEqual([]); expect(h.dirty(rawId)).toBe(false); }); @@ -222,7 +222,10 @@ describe('contacts groups: membership edits', () => { await h.download(gid); const { rawId } = await h.seed({ ...appleCard(), addressBookIds: { [other]: true } }); h.device.user.addToGroup(rawId, (await localGroup(h, gid)).groupId); - expect((await h.upload(rawId)).kind).toBe('revert'); + const plan = await h.upload(rawId); + expect(plan.kind).toBe('revert'); + // Nothing of its own was put back: nothing to report. + expect(plan).not.toHaveProperty('reason'); expect(h.membershipGroups(rawId)).toEqual([`${JMAP}/${gid}`]); expect(h.dirty(rawId)).toBe(true); expect(h.planner.planMembershipUploads([await h.contact(rawId)], await h.groups(), h.ctx)).toEqual([ @@ -319,7 +322,7 @@ describe('contacts groups: device edits of groups', () => { const g = await localGroup(h, gid); h.device.user.updateGroup(g.groupId, { [Groups.TITLE]: 'Mine now' }); const plan = h.planner.planGroupUpload(await localGroup(h, gid), h.ctx); - expect(plan.kind).toBe('revert'); + expect(plan).toMatchObject({ kind: 'revert', reason: 'readOnly' }); await h.applyOk(opsOf(plan)); expect(await localGroup(h, gid)).toMatchObject({ title: 'Friends', dirty: false }); }); diff --git a/src/device-sync/__tests__/contacts/real-engine.test.ts b/src/device-sync/__tests__/contacts/real-engine.test.ts index 2edfebe5..373c3aa6 100644 --- a/src/device-sync/__tests__/contacts/real-engine.test.ts +++ b/src/device-sync/__tests__/contacts/real-engine.test.ts @@ -73,6 +73,7 @@ describe('contacts through the engine', () => { const report = await h.run(); expect(report).toMatchObject({ outcome: 'ok', stats: { skipped: 1 } }); + expect(report.itemErrors).toEqual([expect.objectContaining({ side: 'upload', type: 'groupNotWritable' })]); expect(h.contactNamed('Ada Lovelace')).toMatchObject({ dirty: false }); expect(memberships(h, ada)).toEqual([]); expect(h.server.get('ContactCard', 'team', team)?.members).toBeUndefined(); diff --git a/src/device-sync/__tests__/contacts/upload-editors.test.ts b/src/device-sync/__tests__/contacts/upload-editors.test.ts index 0fcc6a4c..d4783bef 100644 --- a/src/device-sync/__tests__/contacts/upload-editors.test.ts +++ b/src/device-sync/__tests__/contacts/upload-editors.test.ts @@ -359,7 +359,7 @@ describe('contacts upload: read-only and poisoned contacts', () => { const { rawId } = await h.seed(appleCard()); h.device.user.updateData(h.rowsByKey(rawId)['emails:k1']._id as number, { data1: 'hacked@example.org' }); const plan = await uploadPlan(h, rawId); - expect(plan.kind).toBe('revert'); + expect(plan).toMatchObject({ kind: 'revert', reason: 'readOnly' }); await h.upload(rawId); expect(h.rowsByKey(rawId)['emails:k1'].data1).toBe('anna@example.com'); expect(h.dirty(rawId)).toBe(false); @@ -370,7 +370,7 @@ describe('contacts upload: read-only and poisoned contacts', () => { h.readOnlyBooks.add(h.book); const { rawId } = await h.seed(appleCard()); h.device.user.deleteContact(rawId); - expect(await uploadPlan(h, rawId)).toMatchObject({ kind: 'revert', refetch: true }); + expect(await uploadPlan(h, rawId)).toMatchObject({ kind: 'revert', refetch: true, reason: 'readOnly' }); }); it('skips a poisoned contact until it changes or the back-off ends', async () => { diff --git a/src/device-sync/calendar/planner.ts b/src/device-sync/calendar/planner.ts index e5051242..8348e6ef 100644 --- a/src/device-sync/calendar/planner.ts +++ b/src/device-sync/calendar/planner.ts @@ -261,12 +261,12 @@ function planUpload(local: LocalEvent, ctx: CalendarContext): Plan { return { kind: 'upload', actions: [{ kind: 'destroy', id: null, uid }] }; } if (!parseObjectRef(local.syncId)) return { kind: 'purge', ops: purgeGroup(local) }; - if (ctx.isReadOnly(calendar.calendarId)) return { kind: 'revert', ops: purgeGroup(local), refetch: true }; + if (ctx.isReadOnly(calendar.calendarId)) return { kind: 'revert', ops: purgeGroup(local), refetch: true, reason: SKIP.readOnly }; return { kind: 'upload', actions: deleteActions(local, ctx) }; } if (isNew) { - if (ctx.isReadOnly(calendar.calendarId)) return { kind: 'revert', ops: purgeGroup(local) }; + if (ctx.isReadOnly(calendar.calendarId)) return { kind: 'revert', ops: purgeGroup(local), reason: SKIP.readOnly }; const uid = claimedUid(local); const target = createTarget(local, ctx); if (!target) return { kind: 'skip', reason: SKIP.notOurCalendar }; @@ -292,7 +292,7 @@ function planUpload(local: LocalEvent, ctx: CalendarContext): Plan { // An instance Stalwart holds without its series can't be changed on its own. if (shadow.recurrenceId && !shadow.recurrenceRule) { const ops = revertGroup(local, ctx); - return ops ? { kind: 'revert', ops } : { kind: 'skip', reason: SKIP.instanceOnly }; + return ops ? { kind: 'revert', ops, reason: SKIP.instanceOnly } : { kind: 'skip', reason: SKIP.instanceOnly }; } let computation: UploadComputation; @@ -310,9 +310,11 @@ function planUpload(local: LocalEvent, ctx: CalendarContext): Plan { }); if ((readOnly && keys.length) || movedToReadOnly) { // RSVP, to the series or to occurrences, is the one change a calendar without write rights may take. - if (!(computation.rsvpOnly && ctx.calendar(calendar.calendarId)?.myRights?.mayRSVP && !movedToReadOnly)) { + const rsvp = computation.rsvpOnly && !movedToReadOnly; + if (!(rsvp && ctx.calendar(calendar.calendarId)?.myRights?.mayRSVP)) { const ops = revertGroup(local, ctx); - return ops ? { kind: 'revert', ops } : { kind: 'skip', reason: 'readOnly' }; + const reason = rsvp ? SKIP.rsvpRefused : SKIP.readOnly; + return ops ? { kind: 'revert', ops, reason } : { kind: 'skip', reason }; } } if (!keys.length) { @@ -322,7 +324,7 @@ function planUpload(local: LocalEvent, ctx: CalendarContext): Plan { // write puts it back. if (computation.dropped.includes('calendar')) { const ops = revertGroup(local, ctx); - if (ops) return { kind: 'revert', ops }; + if (ops) return { kind: 'revert', ops, reason: SKIP.crossAccountMove }; } return { kind: 'clean', ops: cleanGroup(local, computation) }; } diff --git a/src/device-sync/calendar/upload.ts b/src/device-sync/calendar/upload.ts index 269924f9..f39210cc 100644 --- a/src/device-sync/calendar/upload.ts +++ b/src/device-sync/calendar/upload.ts @@ -79,12 +79,17 @@ import { wallMs, } from './zoned-time'; -/** Why an item is not uploaded now. */ +/** Why an item is not uploaded now (a skip), or why its change was put back (a revert). */ export const SKIP = { dstAmbiguous: 'dstAmbiguous', ruleNotRepresentable: 'ruleNotRepresentable', instanceOnly: 'instanceOnly', notOurCalendar: 'notOurCalendar', + readOnly: 'readOnly', + /** The user's answer, in a calendar that takes no answers either. */ + rsvpRefused: 'rsvpRefused', + /** Moved in place to a calendar of another JMAP account: no patch can do that. */ + crossAccountMove: 'crossAccountMove', } as const; export class SkipUpload extends Error { diff --git a/src/device-sync/contacts/groups.ts b/src/device-sync/contacts/groups.ts index 45829807..9dde6caa 100644 --- a/src/device-sync/contacts/groups.ts +++ b/src/device-sync/contacts/groups.ts @@ -141,7 +141,7 @@ export function planGroupUpload(local: LocalGroup, ctx: Ctx): UploadPlan on).map(([id]) => id); const selected = books.filter((id) => ctx.isSelected(collectionKey(parsed.accountId, id))); if (selected.length && selected.length < books.length) { @@ -175,7 +175,9 @@ export function planGroupUpload(local: LocalGroup, ctx: Ctx): UploadPlan const purge = (): OpGroup => group(local, [deleteWhereId('raw_contacts', local.rawContactId)]); const ref = parseObjectRef(local.sourceId); if (ref) { - if (readOnlyOnServer(local, ctx)) return { kind: 'revert', ops: purge(), refetch: true }; + if (readOnlyOnServer(local, ctx)) return { kind: 'revert', ops: purge(), refetch: true, reason: 'readOnly' }; const books = Object.entries(local.shadow?.addressBookIds ?? {}).filter(([, on]) => on).map(([id]) => id); const selected = books.filter((id) => ctx.isSelected(collectionKey(ref.accountId, id))); // A card that is also in books this device does not sync only leaves the synced ones. @@ -257,7 +257,10 @@ export function planContactUpload(local: LocalContact, ctx: Ctx): UploadPlan[] = [{ kind: 'update', id: parseObjectRef(local.sourceId)!.id, patch }]; if (backedOff(local, ctx, actions)) return { kind: 'skip', reason: `poisoned:${local.poison!.type}` }; diff --git a/src/device-sync/engine/item-sync.ts b/src/device-sync/engine/item-sync.ts index f4c8275f..c5856617 100644 --- a/src/device-sync/engine/item-sync.ts +++ b/src/device-sync/engine/item-sync.ts @@ -913,9 +913,9 @@ export abstract class ItemSync { this.env.report.stats.skipped++; this.env.report.itemError({ ref, side: 'upload', type: plan.reason }); } else if (plan.kind === 'revert') { - // It may not be created where it was made (a read-only calendar): the row goes, and the report says so. + // It may not be created where it was made (a read-only calendar): the row goes, and the report says why. this.env.report.stats.skipped++; - this.env.report.itemError({ ref, side: 'upload', type: 'readOnly' }); + if (plan.reason) this.env.report.itemError({ ref, side: 'upload', type: plan.reason }); works.push(this.planWork(acct, held, 'revert', plan.ops)); } else if (plan.kind === 'claim' || plan.kind === 'purge' || plan.kind === 'clean') { works.push(this.planWork(acct, held, plan.kind, plan.ops)); @@ -1047,6 +1047,7 @@ export abstract class ItemSync { } case 'revert': report.stats.skipped++; + if (plan.reason) report.itemError({ ref, side: 'upload', type: plan.reason }); works.push(this.planWork(acct, held, 'revert', plan.ops)); if (plan.refetch) { const id = idInAccount(meta.sourceId, acct); @@ -1328,7 +1329,11 @@ export abstract class ItemSync { retry: Pending[], ): Promise { const failure = outcomes.find((o): o is Extract => !o.ok); - if (!failure) return this.succeeded(acct, p, outcomes, objects); + if (!failure) { + // Stored without the scheduling messages it asked for (see `forbidden` below). + if (p.noScheduling) this.env.report.note('invitations not sent'); + return this.succeeded(acct, p, outcomes, objects); + } switch (classifySetError(failure.error, failure.action.kind)) { case 'notFound': if (failure.action.kind === 'destroy') { @@ -1346,7 +1351,7 @@ export abstract class ItemSync { case 'forbidden': if (failure.scheduling && !p.noScheduling) { // Scheduling is off, the account has no calendar address, or no permission: store the change without it. - this.env.report.note('invitations not sent'); + // A second refusal is about rights, not invitations: the note waits for the change to be stored. retry.push({ ...p, noScheduling: true }); return []; } diff --git a/src/device-sync/planner.ts b/src/device-sync/planner.ts index 93349e8d..4b56cc5d 100644 --- a/src/device-sync/planner.ts +++ b/src/device-sync/planner.ts @@ -76,7 +76,8 @@ export type UploadAction = * - `claim`: a new item without a pending uid: apply `ops` (they write the uid), then re-read and plan again. * - `upload`: send `actions`; `local` is what they were computed from. * - `purge`: a deleted item that never reached the server: apply `ops`. - * - `revert`: the item may not be changed on the server (read-only): apply `ops` (rewrite from the shadow). + * - `revert`: the item may not be changed on the server (read-only): apply `ops` (rewrite from the shadow); + * `reason`, set when a device change is put back, goes to the report. * - `skip`: poisoned and backed off, or not representable; `reason` goes to the report. */ export type UploadPlan = @@ -84,7 +85,7 @@ export type UploadPlan = | { kind: 'claim'; ops: OpGroup } | { kind: 'upload'; actions: UploadAction[] } | { kind: 'purge'; ops: OpGroup } - | { kind: 'revert'; ops: OpGroup; refetch?: boolean } + | { kind: 'revert'; ops: OpGroup; refetch?: boolean; reason?: string } | { kind: 'skip'; reason: string }; export interface DownloadPlan { From 7453513e08a45b7384bd380923cf4c570589ce36 Mon Sep 17 00:00:00 2001 From: Linus Rath <139418639+rathlinus@users.noreply.github.com> Date: Sun, 27 Sep 2026 08:43:08 +0200 Subject: [PATCH 61/87] fix: stop waiting for group memberships in read-only address books #34 A contact added on the phone to a group in a read-only address book (one shared with the user) could never reach the server, yet it stayed waiting on every run: the planner puts such a membership back, but the engine never told it which groups are read-only, so only groups of another account were recognised. Turning sync off or signing out then always asked about a change that could never go up. The engine now tells the planner which synced groups are in read-only books only, from the group cards it holds and the books' rights. The membership is put back and reported, and nothing is left waiting. --- src/device-sync/__tests__/contacts/harness.ts | 7 +++---- .../__tests__/contacts/real-engine.test.ts | 21 +++++++++++++++++++ src/device-sync/contacts/members.ts | 16 ++++---------- src/device-sync/engine/contacts-sync.ts | 8 ++++++- src/device-sync/planner.ts | 6 ++++++ 5 files changed, 41 insertions(+), 17 deletions(-) diff --git a/src/device-sync/__tests__/contacts/harness.ts b/src/device-sync/__tests__/contacts/harness.ts index 27f1a243..8705ef99 100644 --- a/src/device-sync/__tests__/contacts/harness.ts +++ b/src/device-sync/__tests__/contacts/harness.ts @@ -6,7 +6,6 @@ */ import { Data, GroupMembership, Groups, RawContacts } from '../../android-columns'; import { makeKeyMinter, parseObjectRef, uuidFrom } from '../../common/ids'; -import type { GroupRights } from '../../contacts/members'; import { contactsPlanner } from '../../contacts/planner'; import type { ContactCardWire, @@ -85,7 +84,7 @@ export class Harness { groupIndex = true; /** When true the photos the planner writes come out thumbnail-only (see `keepThumbnailsOnly`). */ thumbnailPhotos = false; - readonly ctx: ContactsContext & GroupRights; + readonly ctx: ContactsContext; constructor() { this.server.addAccount(JMAP, { name: ACCOUNT }); @@ -126,10 +125,10 @@ export class Harness { } /** The context as the engine may build it without a group index. */ - context(): ContactsContext & GroupRights { + context(): ContactsContext { if (this.groupIndex) return this.ctx; const { groupsOf: _drop, ...rest } = this.ctx; - return rest as ContactsContext & GroupRights; + return rest as ContactsContext; } async apply(group: OpGroup): Promise { diff --git a/src/device-sync/__tests__/contacts/real-engine.test.ts b/src/device-sync/__tests__/contacts/real-engine.test.ts index 373c3aa6..d39e6bb4 100644 --- a/src/device-sync/__tests__/contacts/real-engine.test.ts +++ b/src/device-sync/__tests__/contacts/real-engine.test.ts @@ -81,6 +81,27 @@ describe('contacts through the engine', () => { expect(await h.teardown(CONTACTS_AUTHORITY)).toEqual({ pending: 0 }); }); + it('puts back a membership in a group of a read-only book, so no change waits for ever', async () => { + const h = real(); + const shared = h.server.addAddressBook('a', { name: 'Shared', myRights: { mayRead: true, mayWrite: false, mayShare: false, mayDelete: false } }); + h.prefs.contactsSelection[`a/${shared}`] = true; + const team = h.server.addCard('a', { uid: 'g-team', kind: 'group', name: { full: 'Team' }, members: { other: true }, addressBookIds: { [shared]: true } }); + addServerCards(h, ['Ada Lovelace']); + expect((await h.run()).outcome).toBe('ok'); + + const ada = h.contactNamed('Ada Lovelace')!.id; + h.device.user.addToGroup(ada, Number(h.device.rows('groups').find((g) => g.sourceid === `a/${team}`)!._id)); + const report = await h.run(); + + expect(report).toMatchObject({ outcome: 'ok', stats: { skipped: 1 } }); + expect(report.itemErrors).toEqual([expect.objectContaining({ side: 'upload', type: 'groupNotWritable' })]); + expect(h.contactNamed('Ada Lovelace')).toMatchObject({ dirty: false }); + expect(memberships(h, ada)).toEqual([]); + expect(h.server.get('ContactCard', 'a', team)?.members).toEqual({ other: true }); + // Turning sync off finds nothing waiting. + expect(await h.teardown(CONTACTS_AUTHORITY)).toEqual({ pending: 0 }); + }); + it('puts a contact created on the device into the group it was added to', async () => { const h = real(); const group = h.server.addCard('a', { uid: 'g1', kind: 'group', name: { full: 'Friends' }, members: { other: true }, addressBookIds: { [h.book]: true } }); diff --git a/src/device-sync/contacts/members.ts b/src/device-sync/contacts/members.ts index 3a331d1a..e5e9218b 100644 --- a/src/device-sync/contacts/members.ts +++ b/src/device-sync/contacts/members.ts @@ -27,21 +27,13 @@ import { deleteData, insertUnit, updateData } from './ops'; import { NO_PHOTO } from './photo'; import { MEMBER_OF, type Unit } from './project'; -/** - * Group rights the engine can add to the context (not part of - * ContactsContext yet): whether a group card, by SOURCE_ID, is in read-only - * address books only. Without it every synced group counts as writable. - */ -export interface GroupRights { - groupReadOnly?(sourceId: string): boolean; -} - /** * The groups whose members an upload for a contact of `account` can change: - * that account's groups outside read-only books. A membership change of any - * other group can never reach the server (`planMembershipUploads` skips it). + * that account's groups outside read-only books (`groupReadOnly`). A + * membership change of any other group can never reach the server + * (`planMembershipUploads` skips it). */ -export function editableGroups(ctx: ContactsContext & GroupRights, account: string): (sourceId: string) => boolean { +export function editableGroups(ctx: ContactsContext, account: string): (sourceId: string) => boolean { return (sourceId) => parseObjectRef(sourceId)?.accountId === account && !(ctx.groupReadOnly?.(sourceId) ?? false); } diff --git a/src/device-sync/engine/contacts-sync.ts b/src/device-sync/engine/contacts-sync.ts index 4642c817..2e3469ba 100644 --- a/src/device-sync/engine/contacts-sync.ts +++ b/src/device-sync/engine/contacts-sync.ts @@ -112,6 +112,8 @@ export class ContactsSync extends ItemSync { /** Group SOURCE_ID → Groups row id, and member uid → group SOURCE_IDs; rebuilt after group writes. */ private readonly groupRows = new Map(); private readonly membership = new Map>(); + /** Group SOURCE_IDs whose cards are in read-only address books only: the device may not change their members. */ + private readonly readOnlyGroups = new Set(); private groupsStale = true; /** Per account: group refs that appeared (true) or went (false), for `SyncState.groups`. */ private readonly groupChanges = new Map>(); @@ -255,13 +257,16 @@ export class ContactsSync extends ItemSync { return rows.map((r) => this.planner.decodeGroup(r)); } - /** Group rows by SOURCE_ID and who is a member of which group, from the group cards' shadows. */ + /** Group rows by SOURCE_ID, who is a member of which group, and which groups are read-only, from the group cards' shadows. */ private async refreshGroups(): Promise { this.groupRows.clear(); this.membership.clear(); + this.readOnlyGroups.clear(); for (const group of await this.loadGroupsWhere()) { if (!group.sourceId || group.deleted) continue; this.groupRows.set(group.sourceId, group.groupId); + const keys = this.cardKeys(accountOfRef(group.sourceId), group.shadow); + if (keys.length > 0 && keys.every((k) => this.book(k)?.myRights?.mayWrite === false)) this.readOnlyGroups.add(group.sourceId); for (const uid of onIds(group.shadow?.members as Record | undefined)) { let set = this.membership.get(uid); if (!set) this.membership.set(uid, (set = new Set())); @@ -379,6 +384,7 @@ export class ContactsSync extends ItemSync { nameForUid: (uid) => this.uidNames?.get(uid) ?? null, groupRowIdBySourceId: (sourceId) => this.groupRows.get(sourceId) ?? null, groupsOf: (uid) => [...(this.membership.get(uid) ?? [])], + groupReadOnly: (sourceId) => this.readOnlyGroups.has(sourceId), }; } diff --git a/src/device-sync/planner.ts b/src/device-sync/planner.ts index 4b56cc5d..97de9ef4 100644 --- a/src/device-sync/planner.ts +++ b/src/device-sync/planner.ts @@ -189,6 +189,12 @@ export interface ContactsContext extends MapContext { * it; without it the planner leaves memberships as they are. */ groupsOf?(uid: string): string[]; + /** + * Whether a synced group card (by SOURCE_ID) is in read-only address books + * only, so the device may not change its members. The engine provides it + * from the group rows' shadows; without it every group counts as writable. + */ + groupReadOnly?(sourceId: string): boolean; } export interface ContactsPlanner { From 84f6b5777e4793e9d4250bffc661180ff8a132b9 Mon Sep 17 00:00:00 2001 From: Linus Rath <139418639+rathlinus@users.noreply.github.com> Date: Sun, 27 Sep 2026 08:50:15 +0200 Subject: [PATCH 62/87] feat: translate device sync into all 26 languages #34 The strings of device sync were English only: the "Sync to this device" sections in the Contacts and Calendar settings with their dialogs, and the notifications about signing in again and about sync problems. Each language keeps the form of address its file already uses, the app's own terms, and Android's names for its settings, auto-sync and the Contacts and Calendar apps. Checked: every message parses as ICU with the same arguments as the English one and the plural categories of its language, `node scripts/rn-missing-translations.mjs ` lists no device sync key for any language, `npm run i18n:check` and the i18n tests pass. --- locales/rn/ar.json | 113 ++++++++++++++++++++++++++++++++++++++++++ locales/rn/ca.json | 113 ++++++++++++++++++++++++++++++++++++++++++ locales/rn/cs.json | 113 ++++++++++++++++++++++++++++++++++++++++++ locales/rn/da.json | 113 ++++++++++++++++++++++++++++++++++++++++++ locales/rn/de.json | 113 ++++++++++++++++++++++++++++++++++++++++++ locales/rn/es.json | 113 ++++++++++++++++++++++++++++++++++++++++++ locales/rn/fa.json | 113 ++++++++++++++++++++++++++++++++++++++++++ locales/rn/fr.json | 113 ++++++++++++++++++++++++++++++++++++++++++ locales/rn/he.json | 113 ++++++++++++++++++++++++++++++++++++++++++ locales/rn/hu.json | 113 ++++++++++++++++++++++++++++++++++++++++++ locales/rn/it.json | 113 ++++++++++++++++++++++++++++++++++++++++++ locales/rn/ja.json | 113 ++++++++++++++++++++++++++++++++++++++++++ locales/rn/ko.json | 113 ++++++++++++++++++++++++++++++++++++++++++ locales/rn/lv.json | 113 ++++++++++++++++++++++++++++++++++++++++++ locales/rn/mn.json | 113 ++++++++++++++++++++++++++++++++++++++++++ locales/rn/nb.json | 113 ++++++++++++++++++++++++++++++++++++++++++ locales/rn/nl.json | 113 ++++++++++++++++++++++++++++++++++++++++++ locales/rn/pl.json | 113 ++++++++++++++++++++++++++++++++++++++++++ locales/rn/pt.json | 113 ++++++++++++++++++++++++++++++++++++++++++ locales/rn/ro.json | 113 ++++++++++++++++++++++++++++++++++++++++++ locales/rn/ru.json | 113 ++++++++++++++++++++++++++++++++++++++++++ locales/rn/sk.json | 113 ++++++++++++++++++++++++++++++++++++++++++ locales/rn/tr.json | 113 ++++++++++++++++++++++++++++++++++++++++++ locales/rn/uk.json | 113 ++++++++++++++++++++++++++++++++++++++++++ locales/rn/zh-TW.json | 113 ++++++++++++++++++++++++++++++++++++++++++ locales/rn/zh.json | 113 ++++++++++++++++++++++++++++++++++++++++++ 26 files changed, 2938 insertions(+) diff --git a/locales/rn/ar.json b/locales/rn/ar.json index 502ef925..cedb796f 100644 --- a/locales/rn/ar.json +++ b/locales/rn/ar.json @@ -93,6 +93,8 @@ "until_date": "حتى {date}" }, "settings": { + "device_sync_desc": "اعرض تقويماتك في تطبيق التقويم والتطبيقات الأخرى على هذا الجهاز. تتم مزامنة التغييرات في الاتجاهين.", + "device_sync_title": "المزامنة مع هذا الجهاز", "show_week_numbers_mobile_desc": "عرض أرقام الأسابيع في عرض الشهر.", "time_zone": "المنطقة الزمنية", "time_zone_auto": "المنطقة الزمنية للجهاز", @@ -249,6 +251,18 @@ "color_tag": "وسم", "remove_color": "إزالة الوسم" }, + "device_sync": { + "auth_problem_text": "لم يعد الخادم يقبل تسجيل الدخول إلى {account}، لذلك توقفت مزامنة جهات الاتصال والتقويمات الخاصة به على هذا الجهاز. سجّل الدخول مرة أخرى لاستئناف المزامنة. يتم الاحتفاظ بالتغييرات التي أُجريت على هذا الجهاز.", + "auth_problem_title": "سجّل الدخول مرة أخرى لمواصلة المزامنة", + "problem_channel": "مشكلات المزامنة", + "sign_out_anyway": "تسجيل الخروج", + "sign_out_pending_message": "{count, plural, zero {# تغيير أُجري على هذا الجهاز لم يصل إلى الخادم. سيُفقد إذا سجّلت الخروج الآن.} one {تغيير واحد أُجري على هذا الجهاز لم يصل إلى الخادم. سيُفقد إذا سجّلت الخروج الآن.} two {تغييران أُجريا على هذا الجهاز لم يصلا إلى الخادم. سيُفقدان إذا سجّلت الخروج الآن.} few {# تغييرات أُجريت على هذا الجهاز لم تصل إلى الخادم. ستُفقد إذا سجّلت الخروج الآن.} many {# تغييرًا أُجريت على هذا الجهاز لم تصل إلى الخادم. ستُفقد إذا سجّلت الخروج الآن.} other {# تغيير أُجريت على هذا الجهاز لم تصل إلى الخادم. ستُفقد إذا سجّلت الخروج الآن.}}", + "sign_out_pending_title": "تسجيل الخروج على أي حال؟", + "sign_out_unknown_message": "ربما لم تصل التغييرات التي أُجريت على هذا الجهاز إلى الخادم. ستُفقد إذا سجّلت الخروج الآن.", + "signed_out_text": "تم تسجيل الخروج من {account}، لذلك توقفت مزامنة جهات الاتصال والتقويمات الخاصة به على هذا الجهاز. سجّل الدخول مرة أخرى لاستئناف المزامنة. يتم الاحتفاظ بالتغييرات التي أُجريت على هذا الجهاز.", + "signed_out_title": "سجّل الدخول لمواصلة المزامنة", + "signing_out": "جارٍ إزالة جهات الاتصال والتقويمات المتزامنة من هذا الجهاز…" + }, "email_composer": { "attach_camera": "الكاميرا", "attachment_reminder_body": "تذكر رسالتك \"{keyword}\" لكن لم يتم إرفاق أي ملف. هل تريد الإرسال على أي حال؟", @@ -598,11 +612,110 @@ "book_count": "{count, plural, zero {# جهة اتصال} one {جهة اتصال واحدة} two {جهتا اتصال} few {# جهات اتصال} many {# جهة اتصال} other {# جهة اتصال}}", "default_book": "دفتر العناوين الافتراضي", "description_mobile": "تفضيلات العرض وأدوات دفاتر العناوين.", + "device_sync_desc": "اعرض دفاتر عناوينك في تطبيق جهات الاتصال والتطبيقات الأخرى على هذا الجهاز. تتم مزامنة التغييرات في الاتجاهين.", + "device_sync_title": "المزامنة مع هذا الجهاز", "export_description_count": "{count, plural, zero {تصدير # جهة اتصال في ملف vCard واحد.} one {تصدير جهة اتصال واحدة في ملف vCard واحد.} two {تصدير جهتي اتصال في ملف vCard واحد.} few {تصدير # جهات اتصال في ملف vCard واحد.} many {تصدير # جهة اتصال في ملف vCard واحد.} other {تصدير # جهة اتصال في ملف vCard واحد.}}", "export_empty": "لا توجد جهات اتصال للتصدير", "import_description_mobile": "استيراد جهات الاتصال من ملف vCard (.vcf) مع اكتشاف التكرارات.", "manage_description_mobile": "أنشئ دفاتر العناوين التي تنظّم جهات اتصالك، وأعد تسميتها، واحذفها." }, + "device_sync": { + "also_synced_by": "تتم مزامنته أيضًا عبر {accounts}، لذلك يظهر مرتين على هذا الجهاز.", + "android_settings": "إعدادات حساب Android", + "blocked_calendar": "لم يعد Android يطلب الوصول إلى التقويمات. اسمح به من إعدادات التطبيق، ثم فعّل المزامنة مرة أخرى.", + "blocked_contacts": "لم يعد Android يطلب الوصول إلى جهات الاتصال. اسمح به من إعدادات التطبيق، ثم فعّل المزامنة مرة أخرى.", + "blocked_title": "الوصول متوقف", + "books_default": "تتم مزامنة دفاتر عناوينك، أما المشتركة فلا تتم مزامنتها إلا إذا اخترتها.", + "books_hint": "تتم مزامنة دفاتر عناوينك ما لم توقفها، أما المشتركة فلا تتم مزامنتها إلا إذا فعّلتها.", + "books_label": "دفاتر العناوين", + "calendars_default": "تتم مزامنة تقويماتك، أما المشتركة فلا تتم مزامنتها إلا إذا اخترتها.", + "calendars_hint": "تتم مزامنة تقويماتك ما لم توقفها، أما المشتركة فلا تتم مزامنتها إلا إذا فعّلتها. تبقى قوائم المهام في Bulwark.", + "calendars_label": "التقويمات", + "choose": "اختيار", + "choose_books_title": "دفاتر العناوين المراد مزامنتها", + "choose_calendars_title": "التقويمات المراد مزامنتها", + "collections_count": "تتم مزامنة {selected} من {total}", + "continue": "متابعة", + "deletions_delete": "الحذف من الخادم", + "deletions_message": "حُذفت عناصر كثيرة دفعة واحدة على هذا الجهاز، لذلك لم تُحذف من الخادم. احذفها من هناك أيضًا، أو استعدها على هذا الجهاز.", + "deletions_restore": "استعادتها", + "deletions_title": "هل تريد حذفها من الخادم أيضًا؟", + "denied_calendar": "لا يستطيع Bulwark وضع تقويماتك على هذا الجهاز بدون الوصول إلى التقويمات.", + "denied_contacts": "لا يستطيع Bulwark وضع دفاتر عناوينك على هذا الجهاز بدون الوصول إلى جهات الاتصال.", + "denied_title": "تبقى المزامنة متوقفة", + "failed_title": "تعذّر تشغيل المزامنة", + "grant_access": "منح الوصول", + "interval_15m": "كل 15 دقيقة", + "interval_1h": "كل ساعة", + "interval_30m": "كل 30 دقيقة", + "interval_6h": "كل 6 ساعات", + "interval_desc": "عدد مرات المزامنة التي يجريها Android من تلقاء نفسه. تتم مزامنة التعديلات في Bulwark والتغييرات الواردة عبر الإشعارات الفورية بشكل أسرع.", + "interval_label": "فترة المزامنة", + "interval_manual": "بدون مزامنة منتظمة", + "keep_syncing": "مواصلة المزامنة", + "list_error_auth": "لم يقبل الخادم تسجيل الدخول.", + "list_error_network": "تعذّر الوصول إلى الخادم.", + "list_error_server": "تعذّر على الخادم عرض القائمة.", + "list_error_signed_out": "لم يتم تسجيل الدخول إلى هذا الحساب على هذا الجهاز.", + "master_off": "المزامنة التلقائية متوقفة على هذا الجهاز، لذلك لا تتم المزامنة إلا عند النقر على \"مزامنة الآن\".", + "new_contacts_default": "دفتر العناوين الافتراضي", + "new_contacts_desc": "دفتر العناوين لجهات الاتصال التي تُنشأ في تطبيقات أخرى على هذا الجهاز.", + "new_contacts_label": "حفظ جهات الاتصال الجديدة في", + "no_books": "لا توجد دفاتر عناوين في هذا الحساب.", + "no_calendars": "لا توجد تقويمات للمزامنة في هذا الحساب.", + "ok": "حسنًا", + "open_app_settings": "فتح إعدادات التطبيق", + "own_collections": "الخاصة بك", + "rationale_calendar": "لعرض تقويماتك في تطبيق التقويم، يحتاج Bulwark إلى الوصول إلى التقويمات على هذا الجهاز. ولا يغيّر إلا تقويمات حسابه.", + "rationale_contacts": "لعرض دفاتر عناوينك في تطبيق جهات الاتصال، يحتاج Bulwark إلى الوصول إلى جهات الاتصال على هذا الجهاز. ولا يغيّر إلا جهات الاتصال في حسابه.", + "rationale_title_calendar": "السماح بالوصول إلى التقويمات", + "rationale_title_contacts": "السماح بالوصول إلى جهات الاتصال", + "read_only": "للقراءة فقط", + "reminder_owner_bulwark": "Bulwark", + "reminder_owner_device": "تطبيق التقويم", + "reminder_question_message": "يمكن أن تصلك تذكيرات الأحداث المتزامنة من تطبيق التقويم على هذا الجهاز أو من Bulwark. اختر أحدهما حتى لا تصلك التذكيرات مرتين. يمكنك تغيير ذلك لاحقًا.", + "reminder_question_title": "من يجب أن يذكّرك؟", + "reminders_desc": "الجهة التي تذكّرك بالأحداث المتزامنة. يبقى الآخر صامتًا، فلا تصلك التذكيرات مرتين.", + "reminders_label": "التذكيرات", + "reminders_nobody": "إشعارات الأحداث في Bulwark متوقفة، لذلك لن يذكّرك شيء بالأحداث المتزامنة.", + "remove": "إزالة", + "remove_from_device": "إزالة من هذا الجهاز", + "remove_message": "ستُزال جهات الاتصال والتقويمات الخاصة بالحساب من هذا الجهاز. وستُفقد التغييرات التي أُجريت هنا ولم تصل إلى الخادم.", + "remove_title": "إزالة من هذا الجهاز؟", + "review_deletions": "مراجعة المحذوفات", + "shared_by": "تمت المشاركة بواسطة {name}", + "sign_in_again": "تسجيل الدخول مرة أخرى", + "status_auth": "لم يعد الخادم يقبل تسجيل الدخول. سجّل الدخول مرة أخرى لمواصلة المزامنة.", + "status_cancelled": "توقفت آخر مزامنة قبل اكتمالها. وستُستكمل في المرة القادمة.", + "status_failed": "فشلت آخر مزامنة: {message}", + "status_failed_plain": "فشلت آخر مزامنة.", + "status_io": "تعذّر الوصول إلى الخادم {when}. سيحاول Android مرة أخرى.", + "status_item_errors": "{count, plural, zero {تعذّرت مزامنة # عنصر.} one {تعذّرت مزامنة عنصر واحد.} two {تعذّرت مزامنة عنصرين.} few {تعذّرت مزامنة # عناصر.} many {تعذّرت مزامنة # عنصرًا.} other {تعذّرت مزامنة # عنصر.}}", + "status_off": "متوقفة", + "status_paused": "متوقفة مؤقتًا في إعدادات Android", + "status_permission_calendar": "تم إيقاف الوصول إلى التقويمات، لذلك لا تتم مزامنة أي شيء.", + "status_permission_contacts": "تم إيقاف الوصول إلى جهات الاتصال، لذلك لا تتم مزامنة أي شيء.", + "status_removed": "تمت الإزالة في إعدادات Android. فعّل المزامنة لإضافة الحساب مرة أخرى.", + "status_safety_abort": "تم الإيقاف: لم يُرجع الخادم أي شيء رغم أن هذا الجهاز لا يزال يحتوي على عناصر. تحقّق من الخادم، ثم أعد المزامنة.", + "status_synced": "آخر مزامنة: {when}", + "status_syncing": "جارٍ المزامنة…", + "status_too_many_deletions": "حُذفت عناصر كثيرة على هذا الجهاز. لن تُحذف من الخادم حتى تؤكد ذلك.", + "status_unsupported_calendar": "لا يوفّر الخادم أي تقويمات لهذا الحساب.", + "status_unsupported_contacts": "لا يوفّر الخادم أي دفاتر عناوين لهذا الحساب.", + "status_waiting": "بانتظار المزامنة الأولى", + "status_waiting_next": "بانتظار المزامنة التالية", + "suspended_desc": "تم تسجيل الخروج. تبقى جهات الاتصال والتقويمات الخاصة به على هذا الجهاز مع التغييرات التي أُجريت هنا، وستتم مزامنتها مجددًا بعد تسجيل الدخول.", + "switch_label": "مزامنة {account} مع هذا الجهاز", + "sync_now": "مزامنة الآن", + "turn_off_anyway": "إيقاف", + "turn_off_failed": "تعذّرت إزالة البيانات المتزامنة من هذا الجهاز. المزامنة متوقفة مؤقتًا؛ حاول مرة أخرى لاحقًا.", + "turn_off_failed_title": "تعذّر إيقاف المزامنة", + "turn_off_pending": "{count, plural, zero {# تغيير أُجري على هذا الجهاز لم يصل إلى الخادم بعد. سيُفقد إذا أوقفت المزامنة الآن.} one {تغيير واحد أُجري على هذا الجهاز لم يصل إلى الخادم بعد. سيُفقد إذا أوقفت المزامنة الآن.} two {تغييران أُجريا على هذا الجهاز لم يصلا إلى الخادم بعد. سيُفقدان إذا أوقفت المزامنة الآن.} few {# تغييرات أُجريت على هذا الجهاز لم تصل إلى الخادم بعد. ستُفقد إذا أوقفت المزامنة الآن.} many {# تغييرًا أُجريت على هذا الجهاز لم تصل إلى الخادم بعد. ستُفقد إذا أوقفت المزامنة الآن.} other {# تغيير أُجريت على هذا الجهاز لم تصل إلى الخادم بعد. ستُفقد إذا أوقفت المزامنة الآن.}}", + "turn_off_title": "الإيقاف على أي حال؟", + "turn_off_unknown": "تعذّر على Bulwark التحقق مما إذا كانت التغييرات التي أُجريت على هذا الجهاز قد وصلت إلى الخادم. ستُفقد إذا أوقفت المزامنة الآن.", + "turning_off": "جارٍ تحميل التغييرات ثم إزالة البيانات من هذا الجهاز…", + "turning_on": "جارٍ الإعداد…" + }, "downloads": { "attachment_template": { "description_mobile": "القالب المستخدم عند حفظ مرفق أو مشاركته." diff --git a/locales/rn/ca.json b/locales/rn/ca.json index 1f3d0c8e..df83812e 100644 --- a/locales/rn/ca.json +++ b/locales/rn/ca.json @@ -93,6 +93,8 @@ "until_date": "Fins al {date}" }, "settings": { + "device_sync_desc": "Mostra els vostres calendaris a l'aplicació Calendari i a altres aplicacions d'aquest dispositiu. Els canvis se sincronitzen en tots dos sentits.", + "device_sync_title": "Sincronitza amb aquest dispositiu", "show_week_numbers_mobile_desc": "Mostra els números de setmana a la vista de mes.", "time_zone": "Zona horària", "time_zone_auto": "Zona horària del dispositiu", @@ -249,6 +251,18 @@ "color_tag": "Etiqueta", "remove_color": "Elimina l'etiqueta" }, + "device_sync": { + "auth_problem_text": "El servidor ja no accepta l'inici de sessió de {account}, de manera que els seus contactes i calendaris d'aquest dispositiu han deixat de sincronitzar-se. Torneu a iniciar la sessió per reprendre la sincronització. Els canvis fets en aquest dispositiu es conserven.", + "auth_problem_title": "Torneu a iniciar la sessió per continuar sincronitzant", + "problem_channel": "Problemes de sincronització", + "sign_out_anyway": "Tanca la sessió", + "sign_out_pending_message": "{count, plural, one {# canvi fet en aquest dispositiu no ha arribat al servidor. Es perdrà si tanqueu la sessió ara.} other {# canvis fets en aquest dispositiu no han arribat al servidor. Es perdran si tanqueu la sessió ara.}}", + "sign_out_pending_title": "Voleu tancar la sessió igualment?", + "sign_out_unknown_message": "És possible que els canvis fets en aquest dispositiu no hagin arribat al servidor. Es perdran si tanqueu la sessió ara.", + "signed_out_text": "S'ha tancat la sessió de {account}, de manera que els seus contactes i calendaris d'aquest dispositiu ja no se sincronitzen. Torneu a iniciar la sessió per reprendre la sincronització. Els canvis fets en aquest dispositiu es conserven.", + "signed_out_title": "Inicieu la sessió per continuar sincronitzant", + "signing_out": "S'estan eliminant d'aquest dispositiu els contactes i calendaris sincronitzats…" + }, "email_composer": { "attach_camera": "Càmera", "attachment_reminder_body": "El vostre missatge esmenta «{keyword}», però no hi ha cap fitxer adjunt. Voleu enviar-lo igualment?", @@ -598,11 +612,110 @@ "book_count": "{count, plural, one {# contacte} other {# contactes}}", "default_book": "Llibreta d'adreces predeterminada", "description_mobile": "Preferències de visualització i eines de la llibreta d'adreces.", + "device_sync_desc": "Mostra les vostres llibretes d'adreces a l'aplicació Contactes i a altres aplicacions d'aquest dispositiu. Els canvis se sincronitzen en tots dos sentits.", + "device_sync_title": "Sincronitza amb aquest dispositiu", "export_description_count": "{count, plural, one {Exporta # contacte com un únic fitxer vCard.} other {Exporta # contactes com un únic fitxer vCard.}}", "export_empty": "No hi ha contactes per exportar", "import_description_mobile": "Importa contactes des d'un fitxer vCard (.vcf) amb detecció de duplicats.", "manage_description_mobile": "Creeu, canvieu el nom i suprimiu les llibretes d'adreces que organitzen els contactes." }, + "device_sync": { + "also_synced_by": "També se sincronitza per a {accounts}, per això apareix dues vegades en aquest dispositiu.", + "android_settings": "Configuració del compte d'Android", + "blocked_calendar": "Android ja no demana accés als calendaris. Permeteu-lo a la configuració de l'aplicació i torneu a activar la sincronització.", + "blocked_contacts": "Android ja no demana accés als contactes. Permeteu-lo a la configuració de l'aplicació i torneu a activar la sincronització.", + "blocked_title": "L'accés està desactivat", + "books_default": "Les vostres llibretes d'adreces se sincronitzen; les compartides, només si les trieu.", + "books_hint": "Les vostres llibretes d'adreces se sincronitzen tret que les desactiveu; les compartides, només si les activeu.", + "books_label": "Llibretes d'adreces", + "calendars_default": "Els vostres calendaris se sincronitzen; els compartits, només si els trieu.", + "calendars_hint": "Els vostres calendaris se sincronitzen tret que els desactiveu; els compartits, només si els activeu. Les llistes de tasques es queden al Bulwark.", + "calendars_label": "Calendaris", + "choose": "Tria", + "choose_books_title": "Llibretes d'adreces per sincronitzar", + "choose_calendars_title": "Calendaris per sincronitzar", + "collections_count": "Sincronització: {selected} de {total}", + "continue": "Continua", + "deletions_delete": "Suprimeix al servidor", + "deletions_message": "S'han suprimit molts elements alhora en aquest dispositiu, de manera que no s'han suprimit al servidor. Suprimiu-los també allà o recupereu-los en aquest dispositiu.", + "deletions_restore": "Recupera'ls", + "deletions_title": "Voleu suprimir-los també al servidor?", + "denied_calendar": "Sense accés als calendaris, el Bulwark no pot posar els vostres calendaris en aquest dispositiu.", + "denied_contacts": "Sense accés als contactes, el Bulwark no pot posar les vostres llibretes d'adreces en aquest dispositiu.", + "denied_title": "La sincronització continua desactivada", + "failed_title": "No s'ha pogut activar la sincronització", + "grant_access": "Concedeix l'accés", + "interval_15m": "Cada 15 minuts", + "interval_1h": "Cada hora", + "interval_30m": "Cada 30 minuts", + "interval_6h": "Cada 6 hores", + "interval_desc": "Amb quina freqüència sincronitza Android pel seu compte. Els canvis fets al Bulwark i els que arriben per push se sincronitzen abans.", + "interval_label": "Interval de sincronització", + "interval_manual": "Sense sincronització periòdica", + "keep_syncing": "Continua sincronitzant", + "list_error_auth": "El servidor no ha acceptat l'inici de sessió.", + "list_error_network": "No s'ha pogut connectar amb el servidor.", + "list_error_server": "El servidor no n'ha pogut mostrar la llista.", + "list_error_signed_out": "Aquest compte no té la sessió iniciada en aquest dispositiu.", + "master_off": "La sincronització automàtica està desactivada en aquest dispositiu, de manera que només se sincronitza quan toqueu «Sincronitza ara».", + "new_contacts_default": "Llibreta d'adreces predeterminada", + "new_contacts_desc": "La llibreta d'adreces per als contactes creats en altres aplicacions d'aquest dispositiu.", + "new_contacts_label": "Desa els contactes nous a", + "no_books": "Aquest compte no té cap llibreta d'adreces.", + "no_calendars": "Aquest compte no té cap calendari per sincronitzar.", + "ok": "D'acord", + "open_app_settings": "Obre la configuració", + "own_collections": "Personals", + "rationale_calendar": "Per mostrar els vostres calendaris a l'aplicació Calendari, el Bulwark necessita accés als calendaris d'aquest dispositiu. Només modifica els calendaris del seu propi compte.", + "rationale_contacts": "Per mostrar les vostres llibretes d'adreces a l'aplicació Contactes, el Bulwark necessita accés als contactes d'aquest dispositiu. Només modifica els contactes del seu propi compte.", + "rationale_title_calendar": "Permet l'accés als calendaris", + "rationale_title_contacts": "Permet l'accés als contactes", + "read_only": "Només lectura", + "reminder_owner_bulwark": "Bulwark", + "reminder_owner_device": "Aplicació Calendari", + "reminder_question_message": "Els recordatoris dels esdeveniments sincronitzats poden venir de l'aplicació de calendari d'aquest dispositiu o del Bulwark. Trieu-ne un perquè no rebeu els recordatoris dues vegades. Ho podeu canviar més endavant.", + "reminder_question_title": "Qui us ha d'enviar els recordatoris?", + "reminders_desc": "Qui us recorda els esdeveniments sincronitzats. L'altre no avisa, perquè no rebeu els recordatoris dues vegades.", + "reminders_label": "Recordatoris", + "reminders_nobody": "Les notificacions d'esdeveniments del Bulwark estan desactivades, de manera que res no us recorda els esdeveniments sincronitzats.", + "remove": "Elimina", + "remove_from_device": "Elimina d'aquest dispositiu", + "remove_message": "Els contactes i calendaris del compte s'eliminen d'aquest dispositiu. Els canvis fets aquí que no hagin arribat al servidor es perden.", + "remove_title": "Voleu eliminar-ho d'aquest dispositiu?", + "review_deletions": "Revisa les supressions", + "shared_by": "Compartit per {name}", + "sign_in_again": "Torna a iniciar la sessió", + "status_auth": "El servidor ja no accepta l'inici de sessió. Torneu a iniciar la sessió per continuar sincronitzant.", + "status_cancelled": "L'última sincronització s'ha aturat abans d'hora. Continuarà la propera vegada.", + "status_failed": "Ha fallat l'última sincronització: {message}", + "status_failed_plain": "Ha fallat l'última sincronització.", + "status_io": "No s'ha pogut connectar amb el servidor {when}. Android ho tornarà a provar.", + "status_item_errors": "{count, plural, one {No s'ha pogut sincronitzar # element.} other {No s'han pogut sincronitzar # elements.}}", + "status_off": "Desactivada", + "status_paused": "En pausa a la configuració d'Android", + "status_permission_calendar": "S'ha desactivat l'accés als calendaris, de manera que no se sincronitza res.", + "status_permission_contacts": "S'ha desactivat l'accés als contactes, de manera que no se sincronitza res.", + "status_removed": "S'ha eliminat a la configuració d'Android. Activeu la sincronització per tornar a afegir el compte.", + "status_safety_abort": "S'ha aturat: el servidor no ha retornat res, però aquest dispositiu encara conté elements. Comproveu el servidor i torneu a sincronitzar.", + "status_synced": "Sincronitzat {when}", + "status_syncing": "S'està sincronitzant…", + "status_too_many_deletions": "S'han suprimit molts elements en aquest dispositiu. No se suprimiran al servidor fins que no ho confirmeu.", + "status_unsupported_calendar": "El servidor no ofereix calendaris per a aquest compte.", + "status_unsupported_contacts": "El servidor no ofereix llibretes d'adreces per a aquest compte.", + "status_waiting": "S'està esperant la primera sincronització", + "status_waiting_next": "S'està esperant la propera sincronització", + "suspended_desc": "Sessió tancada. Els seus contactes i calendaris es queden en aquest dispositiu, amb els canvis fets aquí, i es tornaran a sincronitzar quan inicieu la sessió.", + "switch_label": "Sincronitza {account} amb aquest dispositiu", + "sync_now": "Sincronitza ara", + "turn_off_anyway": "Desactiva", + "turn_off_failed": "No s'han pogut eliminar les dades sincronitzades d'aquest dispositiu. La sincronització està en pausa; torneu-ho a provar més tard.", + "turn_off_failed_title": "No s'ha pogut desactivar la sincronització", + "turn_off_pending": "{count, plural, one {# canvi fet en aquest dispositiu encara no ha arribat al servidor. Es perdrà si desactiveu la sincronització ara.} other {# canvis fets en aquest dispositiu encara no han arribat al servidor. Es perdran si desactiveu la sincronització ara.}}", + "turn_off_title": "Voleu desactivar-la igualment?", + "turn_off_unknown": "El Bulwark no ha pogut comprovar si els canvis fets en aquest dispositiu han arribat al servidor. Es perdran si desactiveu la sincronització ara.", + "turning_off": "S'estan pujant els canvis i després s'eliminaran les dades d'aquest dispositiu…", + "turning_on": "S'està configurant…" + }, "downloads": { "attachment_template": { "description_mobile": "Plantilla utilitzada en desar o compartir un fitxer adjunt." diff --git a/locales/rn/cs.json b/locales/rn/cs.json index 9c527427..d8cd5e19 100644 --- a/locales/rn/cs.json +++ b/locales/rn/cs.json @@ -93,6 +93,8 @@ "until_date": "Do {date}" }, "settings": { + "device_sync_desc": "Zobrazí vaše kalendáře v aplikaci Kalendář a dalších aplikacích v tomto zařízení. Změny se synchronizují oběma směry.", + "device_sync_title": "Synchronizovat s tímto zařízením", "show_week_numbers_mobile_desc": "Zobrazovat čísla týdnů v měsíčním zobrazení.", "time_zone": "Časové pásmo", "time_zone_auto": "Časové pásmo zařízení", @@ -249,6 +251,18 @@ "color_tag": "Štítek", "remove_color": "Odebrat štítek" }, + "device_sync": { + "auth_problem_text": "Server už nepřijímá přihlášení účtu {account}, proto se jeho kontakty a kalendáře v tomto zařízení přestaly synchronizovat. Chcete-li pokračovat, přihlaste se znovu. Změny provedené v tomto zařízení zůstanou zachovány.", + "auth_problem_title": "Pro další synchronizaci se znovu přihlaste", + "problem_channel": "Problémy se synchronizací", + "sign_out_anyway": "Odhlásit se", + "sign_out_pending_message": "{count, plural, one {# změna provedená v tomto zařízení se nedostala na server. Pokud se teď odhlásíte, bude ztracena.} few {# změny provedené v tomto zařízení se nedostaly na server. Pokud se teď odhlásíte, budou ztraceny.} many {# změny provedené v tomto zařízení se nedostalo na server. Pokud se teď odhlásíte, budou ztraceny.} other {# změn provedených v tomto zařízení se nedostalo na server. Pokud se teď odhlásíte, budou ztraceny.}}", + "sign_out_pending_title": "Přesto se odhlásit?", + "sign_out_unknown_message": "Změny provedené v tomto zařízení se možná nedostaly na server. Pokud se teď odhlásíte, budou ztraceny.", + "signed_out_text": "Účet {account} byl odhlášen, proto se jeho kontakty a kalendáře v tomto zařízení už nesynchronizují. Chcete-li pokračovat, přihlaste se znovu. Změny provedené v tomto zařízení zůstanou zachovány.", + "signed_out_title": "Pro další synchronizaci se přihlaste", + "signing_out": "Odebírání synchronizovaných kontaktů a kalendářů z tohoto zařízení…" + }, "email_composer": { "attach_camera": "Fotoaparát", "attachment_reminder_body": "Vaše zpráva obsahuje slovo „{keyword}“, ale není k ní připojen žádný soubor. Přesto odeslat?", @@ -598,11 +612,110 @@ "book_count": "{count, plural, one {# kontakt} few {# kontakty} many {# kontaktu} other {# kontaktů}}", "default_book": "Výchozí adresář", "description_mobile": "Předvolby zobrazení a nástroje pro adresáře.", + "device_sync_desc": "Zobrazí vaše adresáře v aplikaci Kontakty a dalších aplikacích v tomto zařízení. Změny se synchronizují oběma směry.", + "device_sync_title": "Synchronizovat s tímto zařízením", "export_description_count": "{count, plural, one {Exportovat # kontakt do jednoho souboru vCard.} few {Exportovat # kontakty do jednoho souboru vCard.} many {Exportovat # kontaktu do jednoho souboru vCard.} other {Exportovat # kontaktů do jednoho souboru vCard.}}", "export_empty": "Žádné kontakty k exportu", "import_description_mobile": "Importovat kontakty ze souboru vCard (.vcf) s rozpoznáním duplicit.", "manage_description_mobile": "Vytvářejte, přejmenovávejte a odstraňujte adresáře, ve kterých máte uspořádané kontakty." }, + "device_sync": { + "also_synced_by": "Synchronizuje se také pro {accounts}, proto se v tomto zařízení zobrazuje dvakrát.", + "android_settings": "Nastavení účtu v Androidu", + "blocked_calendar": "Android už o přístup ke kalendářům nežádá. Povolte ho v nastavení aplikace a pak synchronizaci znovu zapněte.", + "blocked_contacts": "Android už o přístup ke kontaktům nežádá. Povolte ho v nastavení aplikace a pak synchronizaci znovu zapněte.", + "blocked_title": "Přístup je vypnutý", + "books_default": "Vaše vlastní adresáře se synchronizují, sdílené jen tehdy, když je vyberete.", + "books_hint": "Vaše vlastní adresáře se synchronizují, pokud je nevypnete; sdílené jen tehdy, když je zapnete.", + "books_label": "Adresáře", + "calendars_default": "Vaše vlastní kalendáře se synchronizují, sdílené jen tehdy, když je vyberete.", + "calendars_hint": "Vaše vlastní kalendáře se synchronizují, pokud je nevypnete; sdílené jen tehdy, když je zapnete. Seznamy úkolů zůstávají v Bulwarku.", + "calendars_label": "Kalendáře", + "choose": "Vybrat", + "choose_books_title": "Adresáře k synchronizaci", + "choose_calendars_title": "Kalendáře k synchronizaci", + "collections_count": "Synchronizováno: {selected} z {total}", + "continue": "Pokračovat", + "deletions_delete": "Smazat na serveru", + "deletions_message": "V tomto zařízení bylo najednou smazáno mnoho položek, proto nebyly smazány na serveru. Smažte je i tam, nebo je v tomto zařízení obnovte.", + "deletions_restore": "Obnovit", + "deletions_title": "Smazat je i na serveru?", + "denied_calendar": "Bez přístupu ke kalendářům nemůže Bulwark přenést vaše kalendáře do tohoto zařízení.", + "denied_contacts": "Bez přístupu ke kontaktům nemůže Bulwark přenést vaše adresáře do tohoto zařízení.", + "denied_title": "Synchronizace zůstává vypnutá", + "failed_title": "Synchronizaci se nepodařilo zapnout", + "grant_access": "Udělit přístup", + "interval_15m": "Každých 15 minut", + "interval_1h": "Každou hodinu", + "interval_30m": "Každých 30 minut", + "interval_6h": "Každých 6 hodin", + "interval_desc": "Jak často Android synchronizuje sám. Úpravy v Bulwarku a změny oznámené přes push se synchronizují dříve.", + "interval_label": "Interval synchronizace", + "interval_manual": "Bez pravidelné synchronizace", + "keep_syncing": "Dál synchronizovat", + "list_error_auth": "Server nepřijal přihlášení.", + "list_error_network": "Nepodařilo se spojit se serverem.", + "list_error_server": "Server nedokázal poskytnout seznam.", + "list_error_signed_out": "Tento účet není v tomto zařízení přihlášen.", + "master_off": "Automatická synchronizace je v tomto zařízení vypnutá, proto se synchronizuje jen po klepnutí na „Synchronizovat nyní“.", + "new_contacts_default": "Výchozí adresář", + "new_contacts_desc": "Adresář pro kontakty vytvořené v jiných aplikacích v tomto zařízení.", + "new_contacts_label": "Nové kontakty ukládat do", + "no_books": "Tento účet nemá žádné adresáře.", + "no_calendars": "Tento účet nemá žádné kalendáře k synchronizaci.", + "ok": "OK", + "open_app_settings": "Otevřít nastavení", + "own_collections": "Vlastní", + "rationale_calendar": "Aby mohl Bulwark zobrazit vaše kalendáře v aplikaci Kalendář, potřebuje přístup ke kalendářům v tomto zařízení. Mění jen kalendáře svého vlastního účtu.", + "rationale_contacts": "Aby mohl Bulwark zobrazit vaše adresáře v aplikaci Kontakty, potřebuje přístup ke kontaktům v tomto zařízení. Mění jen kontakty svého vlastního účtu.", + "rationale_title_calendar": "Povolit přístup ke kalendářům", + "rationale_title_contacts": "Povolit přístup ke kontaktům", + "read_only": "Jen pro čtení", + "reminder_owner_bulwark": "Bulwark", + "reminder_owner_device": "Aplikace Kalendář", + "reminder_question_message": "Připomenutí synchronizovaných událostí může zobrazovat aplikace kalendáře v tomto zařízení, nebo Bulwark. Vyberte jednu možnost, abyste nedostávali připomenutí dvakrát. Později to můžete změnit.", + "reminder_question_title": "Kdo vám má připomínat události?", + "reminders_desc": "Kdo vám připomíná synchronizované události. Druhá možnost mlčí, abyste nedostávali připomenutí dvakrát.", + "reminders_label": "Připomenutí", + "reminders_nobody": "Oznámení událostí v Bulwarku jsou vypnutá, takže vám synchronizované události nic nepřipomene.", + "remove": "Odebrat", + "remove_from_device": "Odebrat z tohoto zařízení", + "remove_message": "Kontakty a kalendáře účtu budou z tohoto zařízení odebrány. Změny provedené zde, které se nedostaly na server, budou ztraceny.", + "remove_title": "Odebrat z tohoto zařízení?", + "review_deletions": "Zkontrolovat smazání", + "shared_by": "Sdílí {name}", + "sign_in_again": "Znovu se přihlásit", + "status_auth": "Server už nepřijímá přihlášení. Pro další synchronizaci se znovu přihlaste.", + "status_cancelled": "Poslední synchronizace skončila předčasně. Příště bude pokračovat.", + "status_failed": "Poslední synchronizace selhala: {message}", + "status_failed_plain": "Poslední synchronizace selhala.", + "status_io": "Server byl {when} nedostupný. Android to zkusí znovu.", + "status_item_errors": "{count, plural, one {# položku se nepodařilo synchronizovat.} few {# položky se nepodařilo synchronizovat.} many {# položky se nepodařilo synchronizovat.} other {# položek se nepodařilo synchronizovat.}}", + "status_off": "Vypnuto", + "status_paused": "Pozastaveno v nastavení Androidu", + "status_permission_calendar": "Přístup ke kalendářům byl vypnut, proto se nic nesynchronizuje.", + "status_permission_contacts": "Přístup ke kontaktům byl vypnut, proto se nic nesynchronizuje.", + "status_removed": "Odebráno v nastavení Androidu. Chcete-li účet znovu přidat, zapněte synchronizaci.", + "status_safety_abort": "Zastaveno: server nic nevrátil, přestože toto zařízení stále obsahuje položky. Zkontrolujte server a pak synchronizujte znovu.", + "status_synced": "Synchronizováno {when}", + "status_syncing": "Synchronizace…", + "status_too_many_deletions": "V tomto zařízení bylo smazáno mnoho položek. Na serveru budou smazány až po vašem potvrzení.", + "status_unsupported_calendar": "Server pro tento účet nenabízí žádné kalendáře.", + "status_unsupported_contacts": "Server pro tento účet nenabízí žádné adresáře.", + "status_waiting": "Čeká se na první synchronizaci", + "status_waiting_next": "Čeká se na další synchronizaci", + "suspended_desc": "Odhlášeno. Kontakty a kalendáře účtu zůstanou v tomto zařízení i se zde provedenými změnami a po přihlášení se znovu synchronizují.", + "switch_label": "Synchronizovat {account} s tímto zařízením", + "sync_now": "Synchronizovat nyní", + "turn_off_anyway": "Vypnout", + "turn_off_failed": "Synchronizovaná data se nepodařilo z tohoto zařízení odebrat. Synchronizace je pozastavena; zkuste to později.", + "turn_off_failed_title": "Synchronizaci se nepodařilo vypnout", + "turn_off_pending": "{count, plural, one {# změna provedená v tomto zařízení se ještě nedostala na server. Pokud teď synchronizaci vypnete, bude ztracena.} few {# změny provedené v tomto zařízení se ještě nedostaly na server. Pokud teď synchronizaci vypnete, budou ztraceny.} many {# změny provedené v tomto zařízení se ještě nedostalo na server. Pokud teď synchronizaci vypnete, budou ztraceny.} other {# změn provedených v tomto zařízení se ještě nedostalo na server. Pokud teď synchronizaci vypnete, budou ztraceny.}}", + "turn_off_title": "Přesto vypnout?", + "turn_off_unknown": "Bulwark nemohl ověřit, zda se změny provedené v tomto zařízení dostaly na server. Pokud teď synchronizaci vypnete, budou ztraceny.", + "turning_off": "Nahrávání změn a poté odebírání dat z tohoto zařízení…", + "turning_on": "Nastavování…" + }, "downloads": { "attachment_template": { "description_mobile": "Šablona použitá při ukládání nebo sdílení přílohy." diff --git a/locales/rn/da.json b/locales/rn/da.json index e80afe4d..a2866552 100644 --- a/locales/rn/da.json +++ b/locales/rn/da.json @@ -93,6 +93,8 @@ "until_date": "Indtil {date}" }, "settings": { + "device_sync_desc": "Vis dine kalendere i Kalender-appen og andre apps på denne enhed. Ændringer synkroniseres begge veje.", + "device_sync_title": "Synkronisér med denne enhed", "show_week_numbers_mobile_desc": "Vis ugenumre i månedsvisningen.", "time_zone": "Tidszone", "time_zone_auto": "Enhedens tidszone", @@ -249,6 +251,18 @@ "color_tag": "Tag", "remove_color": "Fjern tag" }, + "device_sync": { + "auth_problem_text": "Serveren accepterer ikke længere login for {account}, så kontoens kontakter og kalendere på denne enhed synkroniseres ikke længere. Log ind igen for at fortsætte. Ændringer foretaget på denne enhed bevares.", + "auth_problem_title": "Log ind igen for at fortsætte synkroniseringen", + "problem_channel": "Synkroniseringsproblemer", + "sign_out_anyway": "Log ud", + "sign_out_pending_message": "{count, plural, one {# ændring foretaget på denne enhed er ikke nået frem til serveren. Den går tabt, hvis du logger ud nu.} other {# ændringer foretaget på denne enhed er ikke nået frem til serveren. De går tabt, hvis du logger ud nu.}}", + "sign_out_pending_title": "Vil du logge ud alligevel?", + "sign_out_unknown_message": "Ændringer foretaget på denne enhed er måske ikke nået frem til serveren. De går tabt, hvis du logger ud nu.", + "signed_out_text": "{account} blev logget ud, så kontoens kontakter og kalendere på denne enhed synkroniseres ikke længere. Log ind igen for at fortsætte. Ændringer foretaget på denne enhed bevares.", + "signed_out_title": "Log ind for at fortsætte synkroniseringen", + "signing_out": "Fjerner synkroniserede kontakter og kalendere fra denne enhed…" + }, "email_composer": { "attach_camera": "Kamera", "attachment_reminder_body": "Din besked nævner \"{keyword}\", men ingen fil er vedhæftet. Send alligevel?", @@ -598,11 +612,110 @@ "book_count": "{count, plural, one {# kontakt} other {# kontakter}}", "default_book": "Standardadressebog", "description_mobile": "Visningsindstillinger og værktøjer til adressebøger.", + "device_sync_desc": "Vis dine adressebøger i Kontakter-appen og andre apps på denne enhed. Ændringer synkroniseres begge veje.", + "device_sync_title": "Synkronisér med denne enhed", "export_description_count": "{count, plural, one {Eksportér # kontakt som én vCard-fil.} other {Eksportér # kontakter som én vCard-fil.}}", "export_empty": "Ingen kontakter at eksportere", "import_description_mobile": "Importér kontakter fra en vCard-fil (.vcf) med registrering af dubletter.", "manage_description_mobile": "Opret, omdøb og fjern de adressebøger, der organiserer dine kontakter." }, + "device_sync": { + "also_synced_by": "Synkroniseres også for {accounts}, så den vises to gange på denne enhed.", + "android_settings": "Android-kontoindstillinger", + "blocked_calendar": "Android spørger ikke længere om adgang til kalendere. Tillad det i appens indstillinger, og slå derefter synkronisering til igen.", + "blocked_contacts": "Android spørger ikke længere om adgang til kontakter. Tillad det i appens indstillinger, og slå derefter synkronisering til igen.", + "blocked_title": "Adgang er slået fra", + "books_default": "Dine egne adressebøger synkroniseres; delte kun, når du vælger dem.", + "books_hint": "Dine egne adressebøger synkroniseres, medmindre du slår dem fra; delte kun, når du slår dem til.", + "books_label": "Adressebøger", + "calendars_default": "Dine egne kalendere synkroniseres; delte kun, når du vælger dem.", + "calendars_hint": "Dine egne kalendere synkroniseres, medmindre du slår dem fra; delte kun, når du slår dem til. Opgavelister bliver i Bulwark.", + "calendars_label": "Kalendere", + "choose": "Vælg", + "choose_books_title": "Adressebøger, der skal synkroniseres", + "choose_calendars_title": "Kalendere, der skal synkroniseres", + "collections_count": "{selected} af {total} synkroniseres", + "continue": "Fortsæt", + "deletions_delete": "Slet på serveren", + "deletions_message": "Mange elementer blev slettet på én gang på denne enhed, så de blev ikke slettet på serveren. Slet dem også der, eller hent dem tilbage på denne enhed.", + "deletions_restore": "Hent dem tilbage", + "deletions_title": "Vil du også slette dem på serveren?", + "denied_calendar": "Uden adgang til kalendere kan Bulwark ikke lægge dine kalendere på denne enhed.", + "denied_contacts": "Uden adgang til kontakter kan Bulwark ikke lægge dine adressebøger på denne enhed.", + "denied_title": "Synkronisering forbliver slået fra", + "failed_title": "Synkronisering kunne ikke slås til", + "grant_access": "Giv adgang", + "interval_15m": "Hvert 15. minut", + "interval_1h": "Hver time", + "interval_30m": "Hvert 30. minut", + "interval_6h": "Hver 6. time", + "interval_desc": "Hvor ofte Android synkroniserer af sig selv. Ændringer i Bulwark og ændringer, der meldes via push, synkroniseres hurtigere.", + "interval_label": "Synkroniseringsinterval", + "interval_manual": "Ingen regelmæssig synkronisering", + "keep_syncing": "Fortsæt synkronisering", + "list_error_auth": "Serveren accepterede ikke login.", + "list_error_network": "Serveren kunne ikke nås.", + "list_error_server": "Serveren kunne ikke vise listen.", + "list_error_signed_out": "Denne konto er ikke logget ind på denne enhed.", + "master_off": "Automatisk synkronisering er slået fra på denne enhed, så der synkroniseres kun, når du trykker på \"Synkronisér nu\".", + "new_contacts_default": "Standardadressebog", + "new_contacts_desc": "Adressebogen til kontakter, der oprettes i andre apps på denne enhed.", + "new_contacts_label": "Gem nye kontakter i", + "no_books": "Denne konto har ingen adressebøger.", + "no_calendars": "Denne konto har ingen kalendere at synkronisere.", + "ok": "OK", + "open_app_settings": "Åbn appindstillinger", + "own_collections": "Dine egne", + "rationale_calendar": "For at vise dine kalendere i Kalender-appen skal Bulwark have adgang til kalenderne på denne enhed. Bulwark ændrer kun kalenderne for sin egen konto.", + "rationale_contacts": "For at vise dine adressebøger i Kontakter-appen skal Bulwark have adgang til kontakterne på denne enhed. Bulwark ændrer kun kontakterne for sin egen konto.", + "rationale_title_calendar": "Giv adgang til kalendere", + "rationale_title_contacts": "Giv adgang til kontakter", + "read_only": "Skrivebeskyttet", + "reminder_owner_bulwark": "Bulwark", + "reminder_owner_device": "Kalender-app", + "reminder_question_message": "Påmindelser om synkroniserede begivenheder kan komme fra kalenderappen på denne enhed eller fra Bulwark. Vælg én af dem, så du ikke får to påmindelser. Du kan ændre det senere.", + "reminder_question_title": "Hvem skal minde dig om begivenheder?", + "reminders_desc": "Hvem der minder dig om synkroniserede begivenheder. Den anden forholder sig tavs, så du ikke får to påmindelser.", + "reminders_label": "Påmindelser", + "reminders_nobody": "Bulwarks notifikationer om begivenheder er slået fra, så intet minder dig om synkroniserede begivenheder.", + "remove": "Fjern", + "remove_from_device": "Fjern fra denne enhed", + "remove_message": "Kontoens kontakter og kalendere fjernes fra denne enhed. Ændringer foretaget her, som ikke er nået frem til serveren, går tabt.", + "remove_title": "Fjern fra denne enhed?", + "review_deletions": "Gennemgå sletninger", + "shared_by": "Delt af {name}", + "sign_in_again": "Log ind igen", + "status_auth": "Serveren accepterer ikke længere login. Log ind igen for at fortsætte synkroniseringen.", + "status_cancelled": "Den seneste synkronisering stoppede før tid. Den fortsætter næste gang.", + "status_failed": "Den seneste synkronisering mislykkedes: {message}", + "status_failed_plain": "Den seneste synkronisering mislykkedes.", + "status_io": "Serveren kunne ikke nås {when}. Android prøver igen.", + "status_item_errors": "{count, plural, one {# element kunne ikke synkroniseres.} other {# elementer kunne ikke synkroniseres.}}", + "status_off": "Fra", + "status_paused": "Sat på pause i Android-indstillinger", + "status_permission_calendar": "Adgangen til kalendere blev slået fra, så intet synkroniseres.", + "status_permission_contacts": "Adgangen til kontakter blev slået fra, så intet synkroniseres.", + "status_removed": "Fjernet i Android-indstillinger. Slå synkronisering til for at tilføje kontoen igen.", + "status_safety_abort": "Stoppet: Serveren returnerede intet, selvom denne enhed stadig har elementer. Tjek serveren, og synkronisér derefter igen.", + "status_synced": "Sidst synkroniseret: {when}", + "status_syncing": "Synkroniserer…", + "status_too_many_deletions": "Mange elementer blev slettet på denne enhed. De slettes ikke på serveren, før du bekræfter.", + "status_unsupported_calendar": "Serveren tilbyder ingen kalendere til denne konto.", + "status_unsupported_contacts": "Serveren tilbyder ingen adressebøger til denne konto.", + "status_waiting": "Venter på den første synkronisering", + "status_waiting_next": "Venter på den næste synkronisering", + "suspended_desc": "Logget ud. Kontoens kontakter og kalendere bliver på denne enhed sammen med de ændringer, der er foretaget her, og synkroniseres igen, når du logger ind.", + "switch_label": "Synkronisér {account} med denne enhed", + "sync_now": "Synkronisér nu", + "turn_off_anyway": "Slå fra", + "turn_off_failed": "De synkroniserede data kunne ikke fjernes fra denne enhed. Synkroniseringen er sat på pause; prøv igen senere.", + "turn_off_failed_title": "Synkronisering kunne ikke slås fra", + "turn_off_pending": "{count, plural, one {# ændring foretaget på denne enhed er endnu ikke nået frem til serveren. Den går tabt, hvis du slår synkronisering fra nu.} other {# ændringer foretaget på denne enhed er endnu ikke nået frem til serveren. De går tabt, hvis du slår synkronisering fra nu.}}", + "turn_off_title": "Vil du slå fra alligevel?", + "turn_off_unknown": "Bulwark kunne ikke kontrollere, om ændringer foretaget på denne enhed er nået frem til serveren. De går tabt, hvis du slår synkronisering fra nu.", + "turning_off": "Uploader ændringer og fjerner derefter data fra denne enhed…", + "turning_on": "Konfigurerer…" + }, "downloads": { "attachment_template": { "description_mobile": "Skabelon, der bruges, når en vedhæftning gemmes eller deles." diff --git a/locales/rn/de.json b/locales/rn/de.json index 67f7861e..6528fd2d 100644 --- a/locales/rn/de.json +++ b/locales/rn/de.json @@ -93,6 +93,8 @@ "until_date": "Bis {date}" }, "settings": { + "device_sync_desc": "Zeigt Ihre Kalender in der Kalender-App und in anderen Apps auf diesem Gerät an. Änderungen werden in beide Richtungen synchronisiert.", + "device_sync_title": "Mit diesem Gerät synchronisieren", "show_week_numbers_mobile_desc": "Kalenderwochen in der Monatsansicht anzeigen.", "time_zone": "Zeitzone", "time_zone_auto": "Gerätezeitzone", @@ -249,6 +251,18 @@ "color_tag": "Label", "remove_color": "Label entfernen" }, + "device_sync": { + "auth_problem_text": "Der Server akzeptiert die Anmeldung von {account} nicht mehr, daher werden die Kontakte und Kalender dieses Kontos auf diesem Gerät nicht mehr synchronisiert. Melden Sie sich erneut an, um fortzufahren. Auf diesem Gerät vorgenommene Änderungen bleiben erhalten.", + "auth_problem_title": "Erneut anmelden, um weiter zu synchronisieren", + "problem_channel": "Synchronisierungsprobleme", + "sign_out_anyway": "Abmelden", + "sign_out_pending_message": "{count, plural, one {# auf diesem Gerät vorgenommene Änderung hat den Server nicht erreicht. Sie geht verloren, wenn Sie sich jetzt abmelden.} other {# auf diesem Gerät vorgenommene Änderungen haben den Server nicht erreicht. Sie gehen verloren, wenn Sie sich jetzt abmelden.}}", + "sign_out_pending_title": "Trotzdem abmelden?", + "sign_out_unknown_message": "Auf diesem Gerät vorgenommene Änderungen haben den Server möglicherweise nicht erreicht. Sie gehen verloren, wenn Sie sich jetzt abmelden.", + "signed_out_text": "{account} wurde abgemeldet, daher werden die Kontakte und Kalender dieses Kontos auf diesem Gerät nicht mehr synchronisiert. Melden Sie sich erneut an, um fortzufahren. Auf diesem Gerät vorgenommene Änderungen bleiben erhalten.", + "signed_out_title": "Anmelden, um weiter zu synchronisieren", + "signing_out": "Synchronisierte Kontakte und Kalender werden von diesem Gerät entfernt…" + }, "email_composer": { "attach_camera": "Kamera", "attachment_reminder_body": "Ihre Nachricht erwähnt \"{keyword}\", aber es ist keine Datei angehängt. Trotzdem senden?", @@ -598,11 +612,110 @@ "book_count": "{count, plural, one {# Kontakt} other {# Kontakte}}", "default_book": "Standardadressbuch", "description_mobile": "Anzeigeeinstellungen und Adressbuch-Werkzeuge.", + "device_sync_desc": "Zeigt Ihre Adressbücher in der Kontakte-App und in anderen Apps auf diesem Gerät an. Änderungen werden in beide Richtungen synchronisiert.", + "device_sync_title": "Mit diesem Gerät synchronisieren", "export_description_count": "{count, plural, one {# Kontakt als eine vCard-Datei exportieren.} other {# Kontakte als eine vCard-Datei exportieren.}}", "export_empty": "Keine Kontakte zum Exportieren", "import_description_mobile": "Kontakte aus einer vCard-Datei (.vcf) importieren, mit Duplikaterkennung.", "manage_description_mobile": "Die Adressbücher erstellen, umbenennen und entfernen, in denen Ihre Kontakte organisiert sind." }, + "device_sync": { + "also_synced_by": "Wird auch für {accounts} synchronisiert und erscheint daher doppelt auf diesem Gerät.", + "android_settings": "Android-Kontoeinstellungen", + "blocked_calendar": "Android fragt nicht mehr nach dem Zugriff auf Kalender. Erlauben Sie ihn in den App-Einstellungen und schalten Sie die Synchronisierung dann wieder ein.", + "blocked_contacts": "Android fragt nicht mehr nach dem Zugriff auf Kontakte. Erlauben Sie ihn in den App-Einstellungen und schalten Sie die Synchronisierung dann wieder ein.", + "blocked_title": "Zugriff ist ausgeschaltet", + "books_default": "Ihre eigenen Adressbücher werden synchronisiert, geteilte nur, wenn Sie sie auswählen.", + "books_hint": "Ihre eigenen Adressbücher werden synchronisiert, außer Sie schalten sie aus; geteilte nur, wenn Sie sie einschalten.", + "books_label": "Adressbücher", + "calendars_default": "Ihre eigenen Kalender werden synchronisiert, geteilte nur, wenn Sie sie auswählen.", + "calendars_hint": "Ihre eigenen Kalender werden synchronisiert, außer Sie schalten sie aus; geteilte nur, wenn Sie sie einschalten. Aufgabenlisten bleiben in Bulwark.", + "calendars_label": "Kalender", + "choose": "Auswählen", + "choose_books_title": "Zu synchronisierende Adressbücher", + "choose_calendars_title": "Zu synchronisierende Kalender", + "collections_count": "{selected} von {total} synchronisiert", + "continue": "Weiter", + "deletions_delete": "Auf dem Server löschen", + "deletions_message": "Auf diesem Gerät wurden viele Elemente auf einmal gelöscht, daher wurden sie auf dem Server nicht gelöscht. Löschen Sie sie auch dort oder stellen Sie sie auf diesem Gerät wieder her.", + "deletions_restore": "Wiederherstellen", + "deletions_title": "Auch auf dem Server löschen?", + "denied_calendar": "Ohne Zugriff auf Kalender kann Bulwark Ihre Kalender nicht auf dieses Gerät bringen.", + "denied_contacts": "Ohne Zugriff auf Kontakte kann Bulwark Ihre Adressbücher nicht auf dieses Gerät bringen.", + "denied_title": "Synchronisierung bleibt aus", + "failed_title": "Synchronisierung konnte nicht eingeschaltet werden", + "grant_access": "Zugriff erlauben", + "interval_15m": "Alle 15 Minuten", + "interval_1h": "Jede Stunde", + "interval_30m": "Alle 30 Minuten", + "interval_6h": "Alle 6 Stunden", + "interval_desc": "Wie oft Android von selbst synchronisiert. Änderungen in Bulwark und per Push gemeldete Änderungen werden früher synchronisiert.", + "interval_label": "Synchronisierungsintervall", + "interval_manual": "Keine regelmäßige Synchronisierung", + "keep_syncing": "Weiter synchronisieren", + "list_error_auth": "Der Server hat die Anmeldung nicht akzeptiert.", + "list_error_network": "Der Server war nicht erreichbar.", + "list_error_server": "Der Server konnte sie nicht auflisten.", + "list_error_signed_out": "Dieses Konto ist auf diesem Gerät nicht angemeldet.", + "master_off": "Die automatische Synchronisierung ist auf diesem Gerät ausgeschaltet, daher wird nur synchronisiert, wenn Sie auf \"Jetzt synchronisieren\" tippen.", + "new_contacts_default": "Standardadressbuch", + "new_contacts_desc": "Das Adressbuch für Kontakte, die in anderen Apps auf diesem Gerät erstellt werden.", + "new_contacts_label": "Neue Kontakte speichern in", + "no_books": "Dieses Konto hat keine Adressbücher.", + "no_calendars": "Dieses Konto hat keine Kalender zum Synchronisieren.", + "ok": "OK", + "open_app_settings": "App-Einstellungen öffnen", + "own_collections": "Ihre eigenen", + "rationale_calendar": "Um Ihre Kalender in der Kalender-App anzuzeigen, benötigt Bulwark Zugriff auf die Kalender auf diesem Gerät. Bulwark ändert nur die Kalender seines eigenen Kontos.", + "rationale_contacts": "Um Ihre Adressbücher in der Kontakte-App anzuzeigen, benötigt Bulwark Zugriff auf die Kontakte auf diesem Gerät. Bulwark ändert nur die Kontakte seines eigenen Kontos.", + "rationale_title_calendar": "Zugriff auf Kalender erlauben", + "rationale_title_contacts": "Zugriff auf Kontakte erlauben", + "read_only": "Schreibgeschützt", + "reminder_owner_bulwark": "Bulwark", + "reminder_owner_device": "Kalender-App", + "reminder_question_message": "Synchronisierte Termine können Sie über die Kalender-App auf diesem Gerät oder über Bulwark erinnern. Wählen Sie eines davon, damit Sie nicht doppelt erinnert werden. Sie können dies später ändern.", + "reminder_question_title": "Wer soll Sie erinnern?", + "reminders_desc": "Wer Sie an synchronisierte Termine erinnert. Die jeweils andere App bleibt still, damit Sie nicht doppelt erinnert werden.", + "reminders_label": "Erinnerungen", + "reminders_nobody": "Die Terminbenachrichtigungen von Bulwark sind ausgeschaltet, daher erinnert Sie nichts an synchronisierte Termine.", + "remove": "Entfernen", + "remove_from_device": "Von diesem Gerät entfernen", + "remove_message": "Die Kontakte und Kalender des Kontos werden von diesem Gerät entfernt. Hier vorgenommene Änderungen, die den Server noch nicht erreicht haben, gehen verloren.", + "remove_title": "Von diesem Gerät entfernen?", + "review_deletions": "Löschungen prüfen", + "shared_by": "Von {name} geteilt", + "sign_in_again": "Erneut anmelden", + "status_auth": "Der Server akzeptiert die Anmeldung nicht mehr. Melden Sie sich erneut an, um weiter zu synchronisieren.", + "status_cancelled": "Die letzte Synchronisierung wurde vorzeitig beendet. Sie wird beim nächsten Mal fortgesetzt.", + "status_failed": "Die letzte Synchronisierung ist fehlgeschlagen: {message}", + "status_failed_plain": "Die letzte Synchronisierung ist fehlgeschlagen.", + "status_io": "Der Server war {when} nicht erreichbar. Android versucht es erneut.", + "status_item_errors": "{count, plural, one {# Element konnte nicht synchronisiert werden.} other {# Elemente konnten nicht synchronisiert werden.}}", + "status_off": "Aus", + "status_paused": "In den Android-Einstellungen pausiert", + "status_permission_calendar": "Der Zugriff auf Kalender wurde ausgeschaltet, daher wird nichts synchronisiert.", + "status_permission_contacts": "Der Zugriff auf Kontakte wurde ausgeschaltet, daher wird nichts synchronisiert.", + "status_removed": "In den Android-Einstellungen entfernt. Schalten Sie die Synchronisierung ein, um das Konto wieder hinzuzufügen.", + "status_safety_abort": "Angehalten: Der Server hat nichts zurückgegeben, obwohl dieses Gerät noch Elemente enthält. Prüfen Sie den Server und synchronisieren Sie dann erneut.", + "status_synced": "Zuletzt synchronisiert: {when}", + "status_syncing": "Wird synchronisiert…", + "status_too_many_deletions": "Auf diesem Gerät wurden viele Elemente gelöscht. Auf dem Server werden sie erst gelöscht, wenn Sie dies bestätigen.", + "status_unsupported_calendar": "Der Server bietet für dieses Konto keine Kalender an.", + "status_unsupported_contacts": "Der Server bietet für dieses Konto keine Adressbücher an.", + "status_waiting": "Warten auf die erste Synchronisierung", + "status_waiting_next": "Warten auf die nächste Synchronisierung", + "suspended_desc": "Abgemeldet. Die Kontakte und Kalender des Kontos bleiben mit den hier vorgenommenen Änderungen auf diesem Gerät und werden wieder synchronisiert, sobald Sie sich anmelden.", + "switch_label": "{account} mit diesem Gerät synchronisieren", + "sync_now": "Jetzt synchronisieren", + "turn_off_anyway": "Ausschalten", + "turn_off_failed": "Die synchronisierten Daten konnten nicht von diesem Gerät entfernt werden. Die Synchronisierung ist pausiert; versuchen Sie es später erneut.", + "turn_off_failed_title": "Synchronisierung konnte nicht ausgeschaltet werden", + "turn_off_pending": "{count, plural, one {# auf diesem Gerät vorgenommene Änderung hat den Server noch nicht erreicht. Sie geht verloren, wenn Sie die Synchronisierung jetzt ausschalten.} other {# auf diesem Gerät vorgenommene Änderungen haben den Server noch nicht erreicht. Sie gehen verloren, wenn Sie die Synchronisierung jetzt ausschalten.}}", + "turn_off_title": "Trotzdem ausschalten?", + "turn_off_unknown": "Bulwark konnte nicht prüfen, ob auf diesem Gerät vorgenommene Änderungen den Server erreicht haben. Sie gehen verloren, wenn Sie die Synchronisierung jetzt ausschalten.", + "turning_off": "Änderungen werden hochgeladen, dann werden die Daten von diesem Gerät entfernt…", + "turning_on": "Wird eingerichtet…" + }, "downloads": { "attachment_template": { "description_mobile": "Vorlage für das Speichern oder Teilen eines Anhangs." diff --git a/locales/rn/es.json b/locales/rn/es.json index b601d8a3..f0f42034 100644 --- a/locales/rn/es.json +++ b/locales/rn/es.json @@ -93,6 +93,8 @@ "until_date": "Hasta el {date}" }, "settings": { + "device_sync_desc": "Muestra tus calendarios en la app Calendario y en otras apps de este dispositivo. Los cambios se sincronizan en ambos sentidos.", + "device_sync_title": "Sincronizar con este dispositivo", "show_week_numbers_mobile_desc": "Mostrar los números de semana en la vista mensual.", "time_zone": "Zona horaria", "time_zone_auto": "Zona horaria del dispositivo", @@ -249,6 +251,18 @@ "color_tag": "Etiqueta", "remove_color": "Eliminar etiqueta" }, + "device_sync": { + "auth_problem_text": "El servidor ya no acepta el inicio de sesión de {account}, así que sus contactos y calendarios de este dispositivo han dejado de sincronizarse. Vuelve a iniciar sesión para reanudar la sincronización. Los cambios hechos en este dispositivo se conservan.", + "auth_problem_title": "Vuelve a iniciar sesión para seguir sincronizando", + "problem_channel": "Problemas de sincronización", + "sign_out_anyway": "Cerrar sesión", + "sign_out_pending_message": "{count, plural, one {# cambio hecho en este dispositivo no ha llegado al servidor. Se perderá si cierras sesión ahora.} other {# cambios hechos en este dispositivo no han llegado al servidor. Se perderán si cierras sesión ahora.}}", + "sign_out_pending_title": "¿Cerrar sesión de todos modos?", + "sign_out_unknown_message": "Puede que los cambios hechos en este dispositivo no hayan llegado al servidor. Se perderán si cierras sesión ahora.", + "signed_out_text": "Se cerró la sesión de {account}, así que sus contactos y calendarios de este dispositivo ya no se sincronizan. Vuelve a iniciar sesión para reanudar la sincronización. Los cambios hechos en este dispositivo se conservan.", + "signed_out_title": "Inicia sesión para seguir sincronizando", + "signing_out": "Quitando de este dispositivo los contactos y calendarios sincronizados…" + }, "email_composer": { "attach_camera": "Cámara", "attachment_reminder_body": "Tu mensaje menciona \"{keyword}\" pero no hay ningún archivo adjunto. ¿Enviar de todos modos?", @@ -598,11 +612,110 @@ "book_count": "{count, plural, one {# contacto} other {# contactos}}", "default_book": "Libreta de direcciones predeterminada", "description_mobile": "Preferencias de visualización y herramientas de la libreta de direcciones.", + "device_sync_desc": "Muestra tus libretas de direcciones en la app Contactos y en otras apps de este dispositivo. Los cambios se sincronizan en ambos sentidos.", + "device_sync_title": "Sincronizar con este dispositivo", "export_description_count": "{count, plural, one {Exporta # contacto como un único archivo vCard.} other {Exporta # contactos como un único archivo vCard.}}", "export_empty": "No hay contactos para exportar", "import_description_mobile": "Importa contactos desde un archivo vCard (.vcf) con detección de duplicados.", "manage_description_mobile": "Crea, renombra y elimina las libretas de direcciones que organizan tus contactos." }, + "device_sync": { + "also_synced_by": "También se sincroniza para {accounts}, por lo que aparece dos veces en este dispositivo.", + "android_settings": "Ajustes de cuenta de Android", + "blocked_calendar": "Android ya no pide acceso a los calendarios. Permítelo en los ajustes de la app y vuelve a activar la sincronización.", + "blocked_contacts": "Android ya no pide acceso a los contactos. Permítelo en los ajustes de la app y vuelve a activar la sincronización.", + "blocked_title": "El acceso está desactivado", + "books_default": "Tus libretas de direcciones se sincronizan; las compartidas, solo si las eliges.", + "books_hint": "Tus libretas de direcciones se sincronizan salvo que las desactives; las compartidas, solo si las activas.", + "books_label": "Libretas de direcciones", + "calendars_default": "Tus calendarios se sincronizan; los compartidos, solo si los eliges.", + "calendars_hint": "Tus calendarios se sincronizan salvo que los desactives; los compartidos, solo si los activas. Las listas de tareas se quedan en Bulwark.", + "calendars_label": "Calendarios", + "choose": "Elegir", + "choose_books_title": "Libretas de direcciones que sincronizar", + "choose_calendars_title": "Calendarios que sincronizar", + "collections_count": "Sincronización: {selected} de {total}", + "continue": "Continuar", + "deletions_delete": "Eliminar en el servidor", + "deletions_message": "Se eliminaron muchos elementos a la vez en este dispositivo, así que no se eliminaron en el servidor. Elimínalos también allí o recupéralos en este dispositivo.", + "deletions_restore": "Recuperarlos", + "deletions_title": "¿Eliminarlos también en el servidor?", + "denied_calendar": "Sin acceso a los calendarios, Bulwark no puede poner tus calendarios en este dispositivo.", + "denied_contacts": "Sin acceso a los contactos, Bulwark no puede poner tus libretas de direcciones en este dispositivo.", + "denied_title": "La sincronización sigue desactivada", + "failed_title": "No se pudo activar la sincronización", + "grant_access": "Conceder acceso", + "interval_15m": "Cada 15 minutos", + "interval_1h": "Cada hora", + "interval_30m": "Cada 30 minutos", + "interval_6h": "Cada 6 horas", + "interval_desc": "Con qué frecuencia sincroniza Android por su cuenta. Los cambios hechos en Bulwark y los que llegan por push se sincronizan antes.", + "interval_label": "Intervalo de sincronización", + "interval_manual": "Sin sincronización periódica", + "keep_syncing": "Seguir sincronizando", + "list_error_auth": "El servidor no aceptó el inicio de sesión.", + "list_error_network": "No se pudo conectar con el servidor.", + "list_error_server": "El servidor no pudo mostrar la lista.", + "list_error_signed_out": "Esta cuenta no tiene la sesión iniciada en este dispositivo.", + "master_off": "La sincronización automática está desactivada en este dispositivo, así que solo se sincroniza cuando tocas \"Sincronizar ahora\".", + "new_contacts_default": "Libreta de direcciones predeterminada", + "new_contacts_desc": "La libreta de direcciones para los contactos creados en otras apps de este dispositivo.", + "new_contacts_label": "Guardar contactos nuevos en", + "no_books": "Esta cuenta no tiene libretas de direcciones.", + "no_calendars": "Esta cuenta no tiene calendarios que sincronizar.", + "ok": "Aceptar", + "open_app_settings": "Abrir ajustes de la app", + "own_collections": "Personales", + "rationale_calendar": "Para mostrar tus calendarios en la app Calendario, Bulwark necesita acceso a los calendarios de este dispositivo. Solo modifica los calendarios de su propia cuenta.", + "rationale_contacts": "Para mostrar tus libretas de direcciones en la app Contactos, Bulwark necesita acceso a los contactos de este dispositivo. Solo modifica los contactos de su propia cuenta.", + "rationale_title_calendar": "Permitir acceso a los calendarios", + "rationale_title_contacts": "Permitir acceso a los contactos", + "read_only": "Solo lectura", + "reminder_owner_bulwark": "Bulwark", + "reminder_owner_device": "App Calendario", + "reminder_question_message": "Los recordatorios de los eventos sincronizados pueden venir de la app de calendario de este dispositivo o de Bulwark. Elige una opción para no recibirlos dos veces. Puedes cambiarlo más tarde.", + "reminder_question_title": "¿Quién debe avisarte?", + "reminders_desc": "Quién te avisa de los eventos sincronizados. El otro se queda en silencio para que no recibas dos avisos.", + "reminders_label": "Recordatorios", + "reminders_nobody": "Las notificaciones de eventos de Bulwark están desactivadas, así que nada te avisará de los eventos sincronizados.", + "remove": "Quitar", + "remove_from_device": "Quitar de este dispositivo", + "remove_message": "Los contactos y calendarios de la cuenta se quitarán de este dispositivo. Los cambios hechos aquí que no hayan llegado al servidor se perderán.", + "remove_title": "¿Quitar de este dispositivo?", + "review_deletions": "Revisar eliminaciones", + "shared_by": "Compartido por {name}", + "sign_in_again": "Volver a iniciar sesión", + "status_auth": "El servidor ya no acepta el inicio de sesión. Vuelve a iniciar sesión para seguir sincronizando.", + "status_cancelled": "La última sincronización se detuvo antes de tiempo. Continuará la próxima vez.", + "status_failed": "La última sincronización falló: {message}", + "status_failed_plain": "La última sincronización falló.", + "status_io": "No se pudo conectar con el servidor {when}. Android lo volverá a intentar.", + "status_item_errors": "{count, plural, one {No se pudo sincronizar # elemento.} other {No se pudieron sincronizar # elementos.}}", + "status_off": "Desactivada", + "status_paused": "En pausa en los ajustes de Android", + "status_permission_calendar": "Se desactivó el acceso a los calendarios, así que no se sincroniza nada.", + "status_permission_contacts": "Se desactivó el acceso a los contactos, así que no se sincroniza nada.", + "status_removed": "Se quitó en los ajustes de Android. Activa la sincronización para volver a añadir la cuenta.", + "status_safety_abort": "Detenida: el servidor no devolvió nada, aunque este dispositivo aún tiene elementos. Revisa el servidor y vuelve a sincronizar.", + "status_synced": "Sincronizado {when}", + "status_syncing": "Sincronizando…", + "status_too_many_deletions": "Se eliminaron muchos elementos en este dispositivo. No se eliminarán en el servidor hasta que lo confirmes.", + "status_unsupported_calendar": "El servidor no ofrece calendarios para esta cuenta.", + "status_unsupported_contacts": "El servidor no ofrece libretas de direcciones para esta cuenta.", + "status_waiting": "Esperando la primera sincronización", + "status_waiting_next": "Esperando la próxima sincronización", + "suspended_desc": "Sesión cerrada. Sus contactos y calendarios se quedan en este dispositivo, con los cambios hechos aquí, y se volverán a sincronizar cuando inicies sesión.", + "switch_label": "Sincronizar {account} con este dispositivo", + "sync_now": "Sincronizar ahora", + "turn_off_anyway": "Desactivar", + "turn_off_failed": "No se pudieron quitar los datos sincronizados de este dispositivo. La sincronización está en pausa; inténtalo de nuevo más tarde.", + "turn_off_failed_title": "No se pudo desactivar la sincronización", + "turn_off_pending": "{count, plural, one {# cambio hecho en este dispositivo aún no ha llegado al servidor. Se perderá si desactivas la sincronización ahora.} other {# cambios hechos en este dispositivo aún no han llegado al servidor. Se perderán si desactivas la sincronización ahora.}}", + "turn_off_title": "¿Desactivar de todos modos?", + "turn_off_unknown": "Bulwark no pudo comprobar si los cambios hechos en este dispositivo llegaron al servidor. Se perderán si desactivas la sincronización ahora.", + "turning_off": "Subiendo los cambios y quitando después los datos de este dispositivo…", + "turning_on": "Configurando…" + }, "downloads": { "attachment_template": { "description_mobile": "Plantilla usada al guardar o compartir un adjunto." diff --git a/locales/rn/fa.json b/locales/rn/fa.json index e5afa8f0..19841e58 100644 --- a/locales/rn/fa.json +++ b/locales/rn/fa.json @@ -93,6 +93,8 @@ "until_date": "تا {date}" }, "settings": { + "device_sync_desc": "تقویم‌های خود را در برنامه تقویم و برنامه‌های دیگر این دستگاه نشان دهید. تغییرات در هر دو جهت همگام‌سازی می‌شوند.", + "device_sync_title": "همگام‌سازی با این دستگاه", "show_week_numbers_mobile_desc": "نمایش شماره هفته‌ها در نمای ماه.", "time_zone": "منطقه زمانی", "time_zone_auto": "منطقه زمانی دستگاه", @@ -249,6 +251,18 @@ "color_tag": "برچسب", "remove_color": "حذف برچسب" }, + "device_sync": { + "auth_problem_text": "سرور دیگر ورود {account} را نمی‌پذیرد، بنابراین همگام‌سازی مخاطبین و تقویم‌های این حساب در این دستگاه متوقف شد. برای ادامه، دوباره وارد شوید. تغییراتی که در این دستگاه ایجاد شده‌اند حفظ می‌شوند.", + "auth_problem_title": "برای ادامه همگام‌سازی دوباره وارد شوید", + "problem_channel": "مشکلات همگام‌سازی", + "sign_out_anyway": "خروج", + "sign_out_pending_message": "{count, plural, one {# تغییری که در این دستگاه ایجاد شده به سرور نرسیده است. اگر اکنون خارج شوید، از دست می‌رود.} other {# تغییری که در این دستگاه ایجاد شده به سرور نرسیده است. اگر اکنون خارج شوید، از دست می‌روند.}}", + "sign_out_pending_title": "به هر حال خارج می‌شوید؟", + "sign_out_unknown_message": "ممکن است تغییراتی که در این دستگاه ایجاد شده‌اند به سرور نرسیده باشند. اگر اکنون خارج شوید، از دست می‌روند.", + "signed_out_text": "خروج از {account} انجام شد، بنابراین مخاطبین و تقویم‌های این حساب در این دستگاه دیگر همگام‌سازی نمی‌شوند. برای ادامه، دوباره وارد شوید. تغییراتی که در این دستگاه ایجاد شده‌اند حفظ می‌شوند.", + "signed_out_title": "برای ادامه همگام‌سازی وارد شوید", + "signing_out": "در حال حذف مخاطبین و تقویم‌های همگام‌سازی‌شده از این دستگاه…" + }, "email_composer": { "attach_camera": "دوربین", "attachment_reminder_body": "در پیام شما به «{keyword}» اشاره شده اما فایلی پیوست نشده است. با این حال ارسال شود؟", @@ -598,11 +612,110 @@ "book_count": "{count, plural, one {# مخاطب} other {# مخاطب}}", "default_book": "دفترچه آدرس پیش‌فرض", "description_mobile": "تنظیمات نمایش و ابزارهای دفترچه آدرس.", + "device_sync_desc": "دفترچه‌های آدرس خود را در برنامه مخاطبین و برنامه‌های دیگر این دستگاه نشان دهید. تغییرات در هر دو جهت همگام‌سازی می‌شوند.", + "device_sync_title": "همگام‌سازی با این دستگاه", "export_description_count": "{count, plural, one {خروجی گرفتن از # مخاطب در قالب یک فایل vCard.} other {خروجی گرفتن از # مخاطب در قالب یک فایل vCard.}}", "export_empty": "مخاطبی برای خروجی گرفتن نیست", "import_description_mobile": "وارد کردن مخاطبین از فایل vCard (.vcf) با تشخیص موارد تکراری.", "manage_description_mobile": "دفترچه‌های آدرسی را که مخاطبین شما را سازمان‌دهی می‌کنند ایجاد، تغییر نام و حذف کنید." }, + "device_sync": { + "also_synced_by": "برای {accounts} هم همگام‌سازی می‌شود، به همین دلیل در این دستگاه دو بار نمایش داده می‌شود.", + "android_settings": "تنظیمات حساب Android", + "blocked_calendar": "Android دیگر برای دسترسی به تقویم‌ها درخواست نمی‌دهد. آن را در تنظیمات برنامه مجاز کنید، سپس همگام‌سازی را دوباره روشن کنید.", + "blocked_contacts": "Android دیگر برای دسترسی به مخاطبین درخواست نمی‌دهد. آن را در تنظیمات برنامه مجاز کنید، سپس همگام‌سازی را دوباره روشن کنید.", + "blocked_title": "دسترسی خاموش است", + "books_default": "دفترچه‌های آدرس خودتان همگام‌سازی می‌شوند؛ دفترچه‌های اشتراکی فقط وقتی آن‌ها را انتخاب کنید.", + "books_hint": "دفترچه‌های آدرس خودتان همگام‌سازی می‌شوند، مگر اینکه آن‌ها را خاموش کنید؛ دفترچه‌های اشتراکی فقط وقتی آن‌ها را روشن کنید.", + "books_label": "دفترچه‌های آدرس", + "calendars_default": "تقویم‌های خودتان همگام‌سازی می‌شوند؛ تقویم‌های اشتراکی فقط وقتی آن‌ها را انتخاب کنید.", + "calendars_hint": "تقویم‌های خودتان همگام‌سازی می‌شوند، مگر اینکه آن‌ها را خاموش کنید؛ تقویم‌های اشتراکی فقط وقتی آن‌ها را روشن کنید. فهرست‌های وظایف در Bulwark می‌مانند.", + "calendars_label": "تقویم‌ها", + "choose": "انتخاب", + "choose_books_title": "دفترچه‌های آدرس برای همگام‌سازی", + "choose_calendars_title": "تقویم‌ها برای همگام‌سازی", + "collections_count": "{selected} از {total} همگام‌سازی می‌شوند", + "continue": "ادامه", + "deletions_delete": "حذف در سرور", + "deletions_message": "موارد زیادی یکجا در این دستگاه حذف شدند، بنابراین در سرور حذف نشدند. آن‌ها را در سرور هم حذف کنید یا در این دستگاه بازگردانید.", + "deletions_restore": "بازگرداندن", + "deletions_title": "در سرور هم حذف شوند؟", + "denied_calendar": "بدون دسترسی به تقویم‌ها، Bulwark نمی‌تواند تقویم‌های شما را در این دستگاه قرار دهد.", + "denied_contacts": "بدون دسترسی به مخاطبین، Bulwark نمی‌تواند دفترچه‌های آدرس شما را در این دستگاه قرار دهد.", + "denied_title": "همگام‌سازی خاموش می‌ماند", + "failed_title": "روشن کردن همگام‌سازی ممکن نشد", + "grant_access": "اعطای دسترسی", + "interval_15m": "هر 15 دقیقه", + "interval_1h": "هر ساعت", + "interval_30m": "هر 30 دقیقه", + "interval_6h": "هر 6 ساعت", + "interval_desc": "اینکه Android هر چند وقت یک بار خودش همگام‌سازی کند. ویرایش‌ها در Bulwark و تغییراتی که با push اعلام می‌شوند زودتر همگام‌سازی می‌شوند.", + "interval_label": "فاصله همگام‌سازی", + "interval_manual": "بدون همگام‌سازی منظم", + "keep_syncing": "ادامه همگام‌سازی", + "list_error_auth": "سرور ورود را نپذیرفت.", + "list_error_network": "اتصال به سرور ممکن نشد.", + "list_error_server": "سرور نتوانست فهرست را ارائه کند.", + "list_error_signed_out": "در این دستگاه به این حساب وارد نشده‌اید.", + "master_off": "همگام‌سازی خودکار در این دستگاه خاموش است، بنابراین همگام‌سازی فقط وقتی انجام می‌شود که روی «همگام‌سازی اکنون» ضربه بزنید.", + "new_contacts_default": "دفترچه آدرس پیش‌فرض", + "new_contacts_desc": "دفترچه آدرس برای مخاطبینی که در برنامه‌های دیگر این دستگاه ایجاد می‌شوند.", + "new_contacts_label": "محل ذخیره مخاطبین جدید", + "no_books": "این حساب هیچ دفترچه آدرسی ندارد.", + "no_calendars": "این حساب هیچ تقویمی برای همگام‌سازی ندارد.", + "ok": "تأیید", + "open_app_settings": "باز کردن تنظیمات برنامه", + "own_collections": "متعلق به شما", + "rationale_calendar": "Bulwark برای نمایش تقویم‌های شما در برنامه تقویم به تقویم‌های این دستگاه دسترسی نیاز دارد. فقط تقویم‌های حساب خودش را تغییر می‌دهد.", + "rationale_contacts": "Bulwark برای نمایش دفترچه‌های آدرس شما در برنامه مخاطبین به مخاطبین این دستگاه دسترسی نیاز دارد. فقط مخاطبین حساب خودش را تغییر می‌دهد.", + "rationale_title_calendar": "اجازه دسترسی به تقویم‌ها", + "rationale_title_contacts": "اجازه دسترسی به مخاطبین", + "read_only": "فقط خواندنی", + "reminder_owner_bulwark": "Bulwark", + "reminder_owner_device": "برنامه تقویم", + "reminder_question_message": "یادآور رویدادهای همگام‌سازی‌شده می‌تواند از برنامه تقویم این دستگاه یا از Bulwark بیاید. یکی را انتخاب کنید تا دو بار یادآوری دریافت نکنید. بعداً می‌توانید آن را تغییر دهید.", + "reminder_question_title": "چه کسی به شما یادآوری کند؟", + "reminders_desc": "چه کسی رویدادهای همگام‌سازی‌شده را به شما یادآوری می‌کند. دیگری ساکت می‌ماند تا دو بار یادآوری دریافت نکنید.", + "reminders_label": "یادآورها", + "reminders_nobody": "اعلان‌های رویداد Bulwark خاموش است، بنابراین هیچ چیز رویدادهای همگام‌سازی‌شده را به شما یادآوری نمی‌کند.", + "remove": "حذف", + "remove_from_device": "حذف از این دستگاه", + "remove_message": "مخاطبین و تقویم‌های این حساب از این دستگاه حذف می‌شوند. تغییراتی که اینجا ایجاد شده و به سرور نرسیده‌اند از دست می‌روند.", + "remove_title": "از این دستگاه حذف شود؟", + "review_deletions": "بررسی حذف‌ها", + "shared_by": "به اشتراک گذاشته شده توسط {name}", + "sign_in_again": "ورود دوباره", + "status_auth": "سرور دیگر ورود را نمی‌پذیرد. برای ادامه همگام‌سازی دوباره وارد شوید.", + "status_cancelled": "آخرین همگام‌سازی زودتر متوقف شد. دفعه بعد ادامه می‌یابد.", + "status_failed": "آخرین همگام‌سازی ناموفق بود: {message}", + "status_failed_plain": "آخرین همگام‌سازی ناموفق بود.", + "status_io": "اتصال به سرور ممکن نشد ({when}). Android دوباره تلاش می‌کند.", + "status_item_errors": "{count, plural, one {# مورد همگام‌سازی نشد.} other {# مورد همگام‌سازی نشد.}}", + "status_off": "خاموش", + "status_paused": "در تنظیمات Android متوقف شده", + "status_permission_calendar": "دسترسی به تقویم‌ها خاموش شد، بنابراین چیزی همگام‌سازی نمی‌شود.", + "status_permission_contacts": "دسترسی به مخاطبین خاموش شد، بنابراین چیزی همگام‌سازی نمی‌شود.", + "status_removed": "در تنظیمات Android حذف شد. برای افزودن دوباره حساب، همگام‌سازی را روشن کنید.", + "status_safety_abort": "متوقف شد: سرور چیزی برنگرداند، در حالی که این دستگاه هنوز موارد دارد. سرور را بررسی کنید، سپس دوباره همگام‌سازی کنید.", + "status_synced": "آخرین همگام‌سازی: {when}", + "status_syncing": "در حال همگام‌سازی…", + "status_too_many_deletions": "موارد زیادی در این دستگاه حذف شدند. تا وقتی تأیید نکنید، در سرور حذف نمی‌شوند.", + "status_unsupported_calendar": "سرور هیچ تقویمی برای این حساب ارائه نمی‌کند.", + "status_unsupported_contacts": "سرور هیچ دفترچه آدرسی برای این حساب ارائه نمی‌کند.", + "status_waiting": "در انتظار نخستین همگام‌سازی", + "status_waiting_next": "در انتظار همگام‌سازی بعدی", + "suspended_desc": "خارج شده. مخاطبین و تقویم‌های این حساب همراه با تغییراتی که اینجا ایجاد شده در این دستگاه می‌مانند و پس از ورود دوباره همگام‌سازی می‌شوند.", + "switch_label": "همگام‌سازی {account} با این دستگاه", + "sync_now": "همگام‌سازی اکنون", + "turn_off_anyway": "خاموش کردن", + "turn_off_failed": "حذف داده‌های همگام‌سازی‌شده از این دستگاه ممکن نشد. همگام‌سازی متوقف شده است؛ بعداً دوباره تلاش کنید.", + "turn_off_failed_title": "خاموش کردن همگام‌سازی ممکن نشد", + "turn_off_pending": "{count, plural, one {# تغییری که در این دستگاه ایجاد شده هنوز به سرور نرسیده است. اگر اکنون همگام‌سازی را خاموش کنید، از دست می‌رود.} other {# تغییری که در این دستگاه ایجاد شده هنوز به سرور نرسیده است. اگر اکنون همگام‌سازی را خاموش کنید، از دست می‌روند.}}", + "turn_off_title": "به هر حال خاموش شود؟", + "turn_off_unknown": "Bulwark نتوانست بررسی کند که آیا تغییراتی که در این دستگاه ایجاد شده‌اند به سرور رسیده‌اند یا نه. اگر اکنون همگام‌سازی را خاموش کنید، از دست می‌روند.", + "turning_off": "در حال بارگذاری تغییرات و سپس حذف داده‌ها از این دستگاه…", + "turning_on": "در حال راه‌اندازی…" + }, "downloads": { "attachment_template": { "description_mobile": "الگویی که هنگام ذخیره یا اشتراک‌گذاری پیوست استفاده می‌شود." diff --git a/locales/rn/fr.json b/locales/rn/fr.json index a58dbf6d..2577ac4a 100644 --- a/locales/rn/fr.json +++ b/locales/rn/fr.json @@ -93,6 +93,8 @@ "until_date": "Jusqu'au {date}" }, "settings": { + "device_sync_desc": "Affichez vos calendriers dans l'application Agenda et dans les autres applications de cet appareil. Les modifications sont synchronisées dans les deux sens.", + "device_sync_title": "Synchroniser avec cet appareil", "show_week_numbers_mobile_desc": "Afficher les numéros de semaine dans la vue mensuelle.", "time_zone": "Fuseau horaire", "time_zone_auto": "Fuseau horaire de l'appareil", @@ -249,6 +251,18 @@ "color_tag": "Étiquette", "remove_color": "Retirer l'étiquette" }, + "device_sync": { + "auth_problem_text": "Le serveur n'accepte plus la connexion de {account} : ses contacts et calendriers ne sont donc plus synchronisés sur cet appareil. Reconnectez-vous pour reprendre la synchronisation. Les modifications effectuées sur cet appareil sont conservées.", + "auth_problem_title": "Reconnectez-vous pour continuer la synchronisation", + "problem_channel": "Problèmes de synchronisation", + "sign_out_anyway": "Se déconnecter", + "sign_out_pending_message": "{count, plural, one {# modification effectuée sur cet appareil n'a pas atteint le serveur. Elle sera perdue si vous vous déconnectez maintenant.} other {# modifications effectuées sur cet appareil n'ont pas atteint le serveur. Elles seront perdues si vous vous déconnectez maintenant.}}", + "sign_out_pending_title": "Se déconnecter quand même ?", + "sign_out_unknown_message": "Les modifications effectuées sur cet appareil n'ont peut-être pas atteint le serveur. Elles seront perdues si vous vous déconnectez maintenant.", + "signed_out_text": "{account} a été déconnecté : ses contacts et calendriers ne sont donc plus synchronisés sur cet appareil. Reconnectez-vous pour reprendre la synchronisation. Les modifications effectuées sur cet appareil sont conservées.", + "signed_out_title": "Connectez-vous pour continuer la synchronisation", + "signing_out": "Suppression des contacts et calendriers synchronisés de cet appareil…" + }, "email_composer": { "attach_camera": "Appareil photo", "attachment_reminder_body": "Votre message mentionne « {keyword} » mais aucun fichier n'est joint. Envoyer quand même ?", @@ -598,11 +612,110 @@ "book_count": "{count, plural, one {# contact} other {# contacts}}", "default_book": "Carnet d'adresses par défaut", "description_mobile": "Préférences d'affichage et outils pour les carnets d'adresses.", + "device_sync_desc": "Affichez vos carnets d'adresses dans l'application Contacts et dans les autres applications de cet appareil. Les modifications sont synchronisées dans les deux sens.", + "device_sync_title": "Synchroniser avec cet appareil", "export_description_count": "{count, plural, one {Exporter # contact dans un seul fichier vCard.} other {Exporter # contacts dans un seul fichier vCard.}}", "export_empty": "Aucun contact à exporter", "import_description_mobile": "Importer des contacts depuis un fichier vCard (.vcf) avec détection des doublons.", "manage_description_mobile": "Créez, renommez et supprimez les carnets d'adresses qui organisent vos contacts." }, + "device_sync": { + "also_synced_by": "Également synchronisé pour {accounts} : il apparaît donc deux fois sur cet appareil.", + "android_settings": "Paramètres du compte Android", + "blocked_calendar": "Android ne demande plus l'accès aux calendriers. Autorisez-le dans les paramètres de l'application, puis réactivez la synchronisation.", + "blocked_contacts": "Android ne demande plus l'accès aux contacts. Autorisez-le dans les paramètres de l'application, puis réactivez la synchronisation.", + "blocked_title": "L'accès est désactivé", + "books_default": "Vos propres carnets d'adresses sont synchronisés ; les carnets partagés, uniquement si vous les choisissez.", + "books_hint": "Vos propres carnets d'adresses sont synchronisés, sauf si vous les désactivez ; les carnets partagés, uniquement si vous les activez.", + "books_label": "Carnets d'adresses", + "calendars_default": "Vos propres calendriers sont synchronisés ; les calendriers partagés, uniquement si vous les choisissez.", + "calendars_hint": "Vos propres calendriers sont synchronisés, sauf si vous les désactivez ; les calendriers partagés, uniquement si vous les activez. Les listes de tâches restent dans Bulwark.", + "calendars_label": "Calendriers", + "choose": "Choisir", + "choose_books_title": "Carnets d'adresses à synchroniser", + "choose_calendars_title": "Calendriers à synchroniser", + "collections_count": "Synchronisation : {selected} sur {total}", + "continue": "Continuer", + "deletions_delete": "Supprimer sur le serveur", + "deletions_message": "De nombreux éléments ont été supprimés d'un coup sur cet appareil ; ils n'ont donc pas été supprimés sur le serveur. Supprimez-les aussi sur le serveur ou restaurez-les sur cet appareil.", + "deletions_restore": "Restaurer", + "deletions_title": "Les supprimer aussi sur le serveur ?", + "denied_calendar": "Sans accès aux calendriers, Bulwark ne peut pas placer vos calendriers sur cet appareil.", + "denied_contacts": "Sans accès aux contacts, Bulwark ne peut pas placer vos carnets d'adresses sur cet appareil.", + "denied_title": "La synchronisation reste désactivée", + "failed_title": "Impossible d'activer la synchronisation", + "grant_access": "Autoriser l'accès", + "interval_15m": "Toutes les 15 minutes", + "interval_1h": "Toutes les heures", + "interval_30m": "Toutes les 30 minutes", + "interval_6h": "Toutes les 6 heures", + "interval_desc": "Fréquence à laquelle Android synchronise de lui-même. Les modifications faites dans Bulwark et celles signalées par push sont synchronisées plus tôt.", + "interval_label": "Fréquence de synchronisation", + "interval_manual": "Pas de synchronisation régulière", + "keep_syncing": "Continuer à synchroniser", + "list_error_auth": "Le serveur n'a pas accepté la connexion.", + "list_error_network": "Impossible de joindre le serveur.", + "list_error_server": "Le serveur n'a pas pu en fournir la liste.", + "list_error_signed_out": "Ce compte n'est pas connecté sur cet appareil.", + "master_off": "La synchronisation automatique est désactivée sur cet appareil : la synchronisation n'a lieu que lorsque vous appuyez sur « Synchroniser maintenant ».", + "new_contacts_default": "Carnet d'adresses par défaut", + "new_contacts_desc": "Le carnet d'adresses des contacts créés dans d'autres applications de cet appareil.", + "new_contacts_label": "Enregistrer les nouveaux contacts dans", + "no_books": "Ce compte n'a aucun carnet d'adresses.", + "no_calendars": "Ce compte n'a aucun calendrier à synchroniser.", + "ok": "OK", + "open_app_settings": "Ouvrir les paramètres", + "own_collections": "Les vôtres", + "rationale_calendar": "Pour afficher vos calendriers dans l'application Agenda, Bulwark doit accéder aux calendriers de cet appareil. Il ne modifie que les calendriers de son propre compte.", + "rationale_contacts": "Pour afficher vos carnets d'adresses dans l'application Contacts, Bulwark doit accéder aux contacts de cet appareil. Il ne modifie que les contacts de son propre compte.", + "rationale_title_calendar": "Autoriser l'accès aux calendriers", + "rationale_title_contacts": "Autoriser l'accès aux contacts", + "read_only": "Lecture seule", + "reminder_owner_bulwark": "Bulwark", + "reminder_owner_device": "Application Agenda", + "reminder_question_message": "Les événements synchronisés peuvent vous être rappelés par l'application d'agenda de cet appareil ou par Bulwark. Choisissez l'un des deux pour ne pas recevoir deux rappels. Vous pourrez modifier ce choix plus tard.", + "reminder_question_title": "Qui doit vous envoyer les rappels ?", + "reminders_desc": "Qui vous rappelle les événements synchronisés. L'autre reste silencieux pour que vous ne receviez pas deux rappels.", + "reminders_label": "Rappels", + "reminders_nobody": "Les notifications d'événements de Bulwark sont désactivées : rien ne vous rappelle donc les événements synchronisés.", + "remove": "Supprimer", + "remove_from_device": "Supprimer de cet appareil", + "remove_message": "Les contacts et calendriers du compte sont supprimés de cet appareil. Les modifications faites ici qui n'ont pas atteint le serveur sont perdues.", + "remove_title": "Supprimer de cet appareil ?", + "review_deletions": "Vérifier les suppressions", + "shared_by": "Partagé par {name}", + "sign_in_again": "Se reconnecter", + "status_auth": "Le serveur n'accepte plus la connexion. Reconnectez-vous pour continuer la synchronisation.", + "status_cancelled": "La dernière synchronisation s'est arrêtée avant la fin. Elle reprendra la prochaine fois.", + "status_failed": "Échec de la dernière synchronisation : {message}", + "status_failed_plain": "Échec de la dernière synchronisation.", + "status_io": "Le serveur était injoignable {when}. Android va réessayer.", + "status_item_errors": "{count, plural, one {# élément n'a pas pu être synchronisé.} other {# éléments n'ont pas pu être synchronisés.}}", + "status_off": "Désactivée", + "status_paused": "En pause dans les paramètres Android", + "status_permission_calendar": "L'accès aux calendriers a été désactivé : rien n'est synchronisé.", + "status_permission_contacts": "L'accès aux contacts a été désactivé : rien n'est synchronisé.", + "status_removed": "Supprimé dans les paramètres Android. Activez la synchronisation pour ajouter à nouveau le compte.", + "status_safety_abort": "Arrêtée : le serveur n'a rien renvoyé alors que cet appareil contient encore des éléments. Vérifiez le serveur, puis relancez la synchronisation.", + "status_synced": "Synchronisé {when}", + "status_syncing": "Synchronisation…", + "status_too_many_deletions": "De nombreux éléments ont été supprimés sur cet appareil. Ils ne seront pas supprimés sur le serveur tant que vous n'aurez pas confirmé.", + "status_unsupported_calendar": "Le serveur ne propose aucun calendrier pour ce compte.", + "status_unsupported_contacts": "Le serveur ne propose aucun carnet d'adresses pour ce compte.", + "status_waiting": "En attente de la première synchronisation", + "status_waiting_next": "En attente de la prochaine synchronisation", + "suspended_desc": "Déconnecté. Ses contacts et calendriers restent sur cet appareil, avec les modifications faites ici, et seront de nouveau synchronisés dès que vous vous connecterez.", + "switch_label": "Synchroniser {account} avec cet appareil", + "sync_now": "Synchroniser maintenant", + "turn_off_anyway": "Désactiver", + "turn_off_failed": "Les données synchronisées n'ont pas pu être supprimées de cet appareil. La synchronisation est en pause ; réessayez plus tard.", + "turn_off_failed_title": "Impossible de désactiver la synchronisation", + "turn_off_pending": "{count, plural, one {# modification effectuée sur cet appareil n'a pas encore atteint le serveur. Elle sera perdue si vous désactivez la synchronisation maintenant.} other {# modifications effectuées sur cet appareil n'ont pas encore atteint le serveur. Elles seront perdues si vous désactivez la synchronisation maintenant.}}", + "turn_off_title": "Désactiver quand même ?", + "turn_off_unknown": "Bulwark n'a pas pu vérifier si les modifications effectuées sur cet appareil ont atteint le serveur. Elles seront perdues si vous désactivez la synchronisation maintenant.", + "turning_off": "Envoi des modifications, puis suppression des données de cet appareil…", + "turning_on": "Configuration…" + }, "downloads": { "attachment_template": { "description_mobile": "Modèle utilisé pour enregistrer ou partager une pièce jointe." diff --git a/locales/rn/he.json b/locales/rn/he.json index e4ded47f..7cfd2cd3 100644 --- a/locales/rn/he.json +++ b/locales/rn/he.json @@ -93,6 +93,8 @@ "until_date": "עד {date}" }, "settings": { + "device_sync_desc": "הצג את לוחות השנה שלך באפליקציית היומן ובאפליקציות אחרות במכשיר הזה. השינויים מסתנכרנים בשני הכיוונים.", + "device_sync_title": "סנכרון למכשיר הזה", "show_week_numbers_mobile_desc": "הצג מספרי שבועות בתצוגת החודש.", "time_zone": "אזור זמן", "time_zone_auto": "אזור הזמן של המכשיר", @@ -249,6 +251,18 @@ "color_tag": "תג", "remove_color": "הסר תג" }, + "device_sync": { + "auth_problem_text": "השרת כבר לא מקבל את ההתחברות של {account}, ולכן אנשי הקשר ולוחות השנה שלו במכשיר הזה הפסיקו להסתנכרן. התחבר מחדש כדי לחדש את הסנכרון. שינויים שבוצעו במכשיר הזה נשמרים.", + "auth_problem_title": "התחבר מחדש כדי להמשיך לסנכרן", + "problem_channel": "בעיות סנכרון", + "sign_out_anyway": "התנתק", + "sign_out_pending_message": "{count, plural, one {שינוי אחד שבוצע במכשיר הזה לא הגיע לשרת. הוא יאבד אם תתנתק עכשיו.} two {שני שינויים שבוצעו במכשיר הזה לא הגיעו לשרת. הם יאבדו אם תתנתק עכשיו.} other {# שינויים שבוצעו במכשיר הזה לא הגיעו לשרת. הם יאבדו אם תתנתק עכשיו.}}", + "sign_out_pending_title": "להתנתק בכל זאת?", + "sign_out_unknown_message": "ייתכן ששינויים שבוצעו במכשיר הזה לא הגיעו לשרת. הם יאבדו אם תתנתק עכשיו.", + "signed_out_text": "בוצעה התנתקות מ-{account}, ולכן אנשי הקשר ולוחות השנה שלו במכשיר הזה כבר לא מסתנכרנים. התחבר מחדש כדי לחדש את הסנכרון. שינויים שבוצעו במכשיר הזה נשמרים.", + "signed_out_title": "התחבר כדי להמשיך לסנכרן", + "signing_out": "מסיר מהמכשיר הזה את אנשי הקשר ולוחות השנה המסונכרנים…" + }, "email_composer": { "attach_camera": "מצלמה", "attachment_reminder_body": "ההודעה שלך מזכירה \"{keyword}\" אבל לא צורף קובץ. לשלוח בכל זאת?", @@ -598,11 +612,110 @@ "book_count": "{count, plural, one {איש קשר אחד} two {שני אנשי קשר} other {# אנשי קשר}}", "default_book": "ספר כתובות ברירת מחדל", "description_mobile": "העדפות תצוגה וכלים לספרי כתובות.", + "device_sync_desc": "הצג את ספרי הכתובות שלך באפליקציית אנשי הקשר ובאפליקציות אחרות במכשיר הזה. השינויים מסתנכרנים בשני הכיוונים.", + "device_sync_title": "סנכרון למכשיר הזה", "export_description_count": "{count, plural, one {ייצוא איש קשר אחד כקובץ vCard יחיד.} two {ייצוא שני אנשי קשר כקובץ vCard יחיד.} other {ייצוא # אנשי קשר כקובץ vCard יחיד.}}", "export_empty": "אין אנשי קשר לייצוא", "import_description_mobile": "ייבוא אנשי קשר מקובץ vCard (.vcf) עם זיהוי כפילויות.", "manage_description_mobile": "צור, שנה שם והסר את ספרי הכתובות שמארגנים את אנשי הקשר שלך." }, + "device_sync": { + "also_synced_by": "מסתנכרן גם עבור {accounts}, ולכן מופיע פעמיים במכשיר הזה.", + "android_settings": "הגדרות החשבון ב-Android", + "blocked_calendar": "Android כבר לא מבקש גישה ללוחות השנה. אפשר את הגישה בהגדרות האפליקציה ואז הפעל שוב את הסנכרון.", + "blocked_contacts": "Android כבר לא מבקש גישה לאנשי הקשר. אפשר את הגישה בהגדרות האפליקציה ואז הפעל שוב את הסנכרון.", + "blocked_title": "הגישה כבויה", + "books_default": "ספרי הכתובות שלך מסתנכרנים; ספרים משותפים רק אם תבחר בהם.", + "books_hint": "ספרי הכתובות שלך מסתנכרנים אלא אם תכבה אותם; ספרים משותפים רק אם תפעיל אותם.", + "books_label": "ספרי כתובות", + "calendars_default": "לוחות השנה שלך מסתנכרנים; לוחות משותפים רק אם תבחר בהם.", + "calendars_hint": "לוחות השנה שלך מסתנכרנים אלא אם תכבה אותם; לוחות משותפים רק אם תפעיל אותם. רשימות המשימות נשארות ב-Bulwark.", + "calendars_label": "לוחות שנה", + "choose": "בחירה", + "choose_books_title": "ספרי כתובות לסנכרון", + "choose_calendars_title": "לוחות שנה לסנכרון", + "collections_count": "מסתנכרנים: {selected} מתוך {total}", + "continue": "המשך", + "deletions_delete": "מחיקה בשרת", + "deletions_message": "פריטים רבים נמחקו בבת אחת במכשיר הזה, ולכן הם לא נמחקו בשרת. מחק אותם גם שם או החזר אותם למכשיר הזה.", + "deletions_restore": "החזרה", + "deletions_title": "למחוק אותם גם בשרת?", + "denied_calendar": "ללא גישה ללוחות השנה, Bulwark לא יכולה להעביר את לוחות השנה שלך למכשיר הזה.", + "denied_contacts": "ללא גישה לאנשי הקשר, Bulwark לא יכולה להעביר את ספרי הכתובות שלך למכשיר הזה.", + "denied_title": "הסנכרון נשאר כבוי", + "failed_title": "לא ניתן היה להפעיל את הסנכרון", + "grant_access": "מתן גישה", + "interval_15m": "כל 15 דקות", + "interval_1h": "כל שעה", + "interval_30m": "כל 30 דקות", + "interval_6h": "כל 6 שעות", + "interval_desc": "באיזו תדירות Android מסנכרן מעצמו. עריכות ב-Bulwark ושינויים שמתקבלים ב-Push מסתנכרנים מוקדם יותר.", + "interval_label": "תדירות סנכרון", + "interval_manual": "ללא סנכרון קבוע", + "keep_syncing": "המשך לסנכרן", + "list_error_auth": "השרת לא קיבל את ההתחברות.", + "list_error_network": "לא ניתן היה להתחבר לשרת.", + "list_error_server": "השרת לא הצליח להציג את הרשימה.", + "list_error_signed_out": "החשבון הזה לא מחובר במכשיר הזה.", + "master_off": "הסנכרון האוטומטי כבוי במכשיר הזה, ולכן הסנכרון מתבצע רק כשאתה מקיש על \"סנכרן עכשיו\".", + "new_contacts_default": "ספר כתובות ברירת מחדל", + "new_contacts_desc": "ספר הכתובות לאנשי קשר שנוצרים באפליקציות אחרות במכשיר הזה.", + "new_contacts_label": "יעד לאנשי קשר חדשים", + "no_books": "לחשבון הזה אין ספרי כתובות.", + "no_calendars": "לחשבון הזה אין לוחות שנה לסנכרון.", + "ok": "אישור", + "open_app_settings": "פתיחת הגדרות האפליקציה", + "own_collections": "שלך", + "rationale_calendar": "כדי להציג את לוחות השנה שלך באפליקציית היומן, Bulwark צריכה גישה ללוחות השנה במכשיר הזה. היא משנה רק את לוחות השנה של החשבון שלה.", + "rationale_contacts": "כדי להציג את ספרי הכתובות שלך באפליקציית אנשי הקשר, Bulwark צריכה גישה לאנשי הקשר במכשיר הזה. היא משנה רק את אנשי הקשר של החשבון שלה.", + "rationale_title_calendar": "מתן גישה ללוחות השנה", + "rationale_title_contacts": "מתן גישה לאנשי הקשר", + "read_only": "קריאה בלבד", + "reminder_owner_bulwark": "Bulwark", + "reminder_owner_device": "אפליקציית היומן", + "reminder_question_message": "תזכורות לאירועים מסונכרנים יכולות להגיע מאפליקציית היומן במכשיר הזה או מ-Bulwark. בחר אחת מהן כדי שלא תקבל תזכורות כפולות. אפשר לשנות זאת מאוחר יותר.", + "reminder_question_title": "מי ישלח לך תזכורות?", + "reminders_desc": "מי מזכיר לך על אירועים מסונכרנים. השני שותק, כך שלא תקבל תזכורות כפולות.", + "reminders_label": "תזכורות", + "reminders_nobody": "התראות האירועים של Bulwark כבויות, ולכן שום דבר לא יזכיר לך על אירועים מסונכרנים.", + "remove": "הסרה", + "remove_from_device": "הסרה מהמכשיר הזה", + "remove_message": "אנשי הקשר ולוחות השנה של החשבון יוסרו מהמכשיר הזה. שינויים שבוצעו כאן ועדיין לא הגיעו לשרת יאבדו.", + "remove_title": "להסיר מהמכשיר הזה?", + "review_deletions": "בדיקת המחיקות", + "shared_by": "משותף על ידי {name}", + "sign_in_again": "התחברות מחדש", + "status_auth": "השרת כבר לא מקבל את ההתחברות. התחבר מחדש כדי להמשיך לסנכרן.", + "status_cancelled": "הסנכרון האחרון נעצר מוקדם. הוא ימשיך בפעם הבאה.", + "status_failed": "הסנכרון האחרון נכשל: {message}", + "status_failed_plain": "הסנכרון האחרון נכשל.", + "status_io": "לא ניתן היה להתחבר לשרת {when}. Android ינסה שוב.", + "status_item_errors": "{count, plural, one {לא ניתן היה לסנכרן פריט אחד.} two {לא ניתן היה לסנכרן שני פריטים.} other {לא ניתן היה לסנכרן # פריטים.}}", + "status_off": "כבוי", + "status_paused": "מושהה בהגדרות Android", + "status_permission_calendar": "הגישה ללוחות השנה כובתה, ולכן שום דבר לא מסתנכרן.", + "status_permission_contacts": "הגישה לאנשי הקשר כובתה, ולכן שום דבר לא מסתנכרן.", + "status_removed": "הוסר בהגדרות Android. הפעל את הסנכרון כדי להוסיף את החשבון מחדש.", + "status_safety_abort": "נעצר: השרת לא החזיר דבר, אף שבמכשיר הזה עדיין יש פריטים. בדוק את השרת ואז סנכרן שוב.", + "status_synced": "סונכרן {when}", + "status_syncing": "מסנכרן…", + "status_too_many_deletions": "פריטים רבים נמחקו במכשיר הזה. הם לא יימחקו בשרת עד שתאשר.", + "status_unsupported_calendar": "השרת לא מציע לוחות שנה לחשבון הזה.", + "status_unsupported_contacts": "השרת לא מציע ספרי כתובות לחשבון הזה.", + "status_waiting": "ממתין לסנכרון הראשון", + "status_waiting_next": "ממתין לסנכרון הבא", + "suspended_desc": "מנותק. אנשי הקשר ולוחות השנה שלו נשארים במכשיר הזה, עם השינויים שבוצעו כאן, ויסתנכרנו שוב אחרי שתתחבר.", + "switch_label": "סנכרון {account} למכשיר הזה", + "sync_now": "סנכרן עכשיו", + "turn_off_anyway": "כיבוי", + "turn_off_failed": "לא ניתן היה להסיר את הנתונים המסונכרנים מהמכשיר הזה. הסנכרון מושהה; נסה שוב מאוחר יותר.", + "turn_off_failed_title": "לא ניתן היה לכבות את הסנכרון", + "turn_off_pending": "{count, plural, one {שינוי אחד שבוצע במכשיר הזה עדיין לא הגיע לשרת. הוא יאבד אם תכבה את הסנכרון עכשיו.} two {שני שינויים שבוצעו במכשיר הזה עדיין לא הגיעו לשרת. הם יאבדו אם תכבה את הסנכרון עכשיו.} other {# שינויים שבוצעו במכשיר הזה עדיין לא הגיעו לשרת. הם יאבדו אם תכבה את הסנכרון עכשיו.}}", + "turn_off_title": "לכבות בכל זאת?", + "turn_off_unknown": "Bulwark לא הצליחה לבדוק אם השינויים שבוצעו במכשיר הזה הגיעו לשרת. הם יאבדו אם תכבה את הסנכרון עכשיו.", + "turning_off": "מעלה שינויים ואז מסיר את הנתונים מהמכשיר הזה…", + "turning_on": "מגדיר…" + }, "downloads": { "attachment_template": { "description_mobile": "התבנית שמשמשת בעת שמירה או שיתוף של קובץ מצורף." diff --git a/locales/rn/hu.json b/locales/rn/hu.json index 41a1e34e..62af8f21 100644 --- a/locales/rn/hu.json +++ b/locales/rn/hu.json @@ -93,6 +93,8 @@ "until_date": "Eddig: {date}" }, "settings": { + "device_sync_desc": "A naptáraid megjelennek a Naptár alkalmazásban és az eszköz más alkalmazásaiban. A módosítások mindkét irányban szinkronizálódnak.", + "device_sync_title": "Szinkronizálás ezzel az eszközzel", "show_week_numbers_mobile_desc": "Hétszámok megjelenítése a havi nézetben.", "time_zone": "Időzóna", "time_zone_auto": "Eszköz időzónája", @@ -249,6 +251,18 @@ "color_tag": "Címke", "remove_color": "Címke eltávolítása" }, + "device_sync": { + "auth_problem_text": "A szerver már nem fogadja el ennek a fióknak a bejelentkezését: {account}. Emiatt a fiók névjegyei és naptárai ezen az eszközön már nem szinkronizálódnak. A folytatáshoz jelentkezz be újra. Az ezen az eszközön végzett módosítások megmaradnak.", + "auth_problem_title": "Jelentkezz be újra a szinkronizálás folytatásához", + "problem_channel": "Szinkronizálási problémák", + "sign_out_anyway": "Kijelentkezés", + "sign_out_pending_message": "{count, plural, one {Ezen az eszközön # módosítás nem jutott el a szerverre. Elvész, ha most kijelentkezel.} other {Ezen az eszközön # módosítás nem jutott el a szerverre. Elvesznek, ha most kijelentkezel.}}", + "sign_out_pending_title": "Mindenképp kijelentkezel?", + "sign_out_unknown_message": "Előfordulhat, hogy az ezen az eszközön végzett módosítások nem jutottak el a szerverre. Elvesznek, ha most kijelentkezel.", + "signed_out_text": "Ez a fiók már nincs bejelentkezve: {account}. Emiatt a fiók névjegyei és naptárai ezen az eszközön már nem szinkronizálódnak. A folytatáshoz jelentkezz be újra. Az ezen az eszközön végzett módosítások megmaradnak.", + "signed_out_title": "Jelentkezz be a szinkronizálás folytatásához", + "signing_out": "A szinkronizált névjegyek és naptárak eltávolítása erről az eszközről…" + }, "email_composer": { "attach_camera": "Kamera", "attachment_reminder_body": "Az üzeneted említi a(z) „{keyword}” kifejezést, de nincs fájl csatolva. Elküldöd így is?", @@ -598,11 +612,110 @@ "book_count": "{count, plural, one {# névjegy} other {# névjegy}}", "default_book": "Alapértelmezett címjegyzék", "description_mobile": "Megjelenítési beállítások és címjegyzék-eszközök.", + "device_sync_desc": "A címjegyzékeid megjelennek a Névjegyek alkalmazásban és az eszköz más alkalmazásaiban. A módosítások mindkét irányban szinkronizálódnak.", + "device_sync_title": "Szinkronizálás ezzel az eszközzel", "export_description_count": "{count, plural, one {# névjegy exportálása egyetlen vCard-fájlba.} other {# névjegy exportálása egyetlen vCard-fájlba.}}", "export_empty": "Nincs exportálható névjegy", "import_description_mobile": "Névjegyek importálása vCard (.vcf) fájlból, a duplikátumok felismerésével.", "manage_description_mobile": "Itt hozhatod létre, nevezheted át és távolíthatod el a névjegyeidet rendező címjegyzékeket." }, + "device_sync": { + "also_synced_by": "Más fiókhoz is szinkronizálódik ({accounts}), ezért kétszer jelenik meg ezen az eszközön.", + "android_settings": "Android-fiókbeállítások", + "blocked_calendar": "Az Android már nem kér hozzáférést a naptárakhoz. Engedélyezd az alkalmazás beállításaiban, majd kapcsold be újra a szinkronizálást.", + "blocked_contacts": "Az Android már nem kér hozzáférést a névjegyekhez. Engedélyezd az alkalmazás beállításaiban, majd kapcsold be újra a szinkronizálást.", + "blocked_title": "A hozzáférés ki van kapcsolva", + "books_default": "A saját címjegyzékeid szinkronizálódnak, a megosztottak csak akkor, ha kiválasztod őket.", + "books_hint": "A saját címjegyzékeid szinkronizálódnak, hacsak ki nem kapcsolod őket; a megosztottak csak akkor, ha bekapcsolod őket.", + "books_label": "Címjegyzékek", + "calendars_default": "A saját naptáraid szinkronizálódnak, a megosztottak csak akkor, ha kiválasztod őket.", + "calendars_hint": "A saját naptáraid szinkronizálódnak, hacsak ki nem kapcsolod őket; a megosztottak csak akkor, ha bekapcsolod őket. A feladatlisták a Bulwarkban maradnak.", + "calendars_label": "Naptárak", + "choose": "Kiválasztás", + "choose_books_title": "Szinkronizálandó címjegyzékek", + "choose_calendars_title": "Szinkronizálandó naptárak", + "collections_count": "Szinkronizálva: {selected}/{total}", + "continue": "Tovább", + "deletions_delete": "Törlés a szerveren", + "deletions_message": "Ezen az eszközön egyszerre sok elemet töröltek, ezért a szerveren nem lettek törölve. Töröld őket ott is, vagy állítsd vissza őket ezen az eszközön.", + "deletions_restore": "Visszaállítás", + "deletions_title": "A szerveren is törlöd őket?", + "denied_calendar": "A naptárakhoz való hozzáférés nélkül a Bulwark nem tudja a naptáraidat erre az eszközre hozni.", + "denied_contacts": "A névjegyekhez való hozzáférés nélkül a Bulwark nem tudja a címjegyzékeidet erre az eszközre hozni.", + "denied_title": "A szinkronizálás kikapcsolva marad", + "failed_title": "Nem sikerült bekapcsolni a szinkronizálást", + "grant_access": "Hozzáférés megadása", + "interval_15m": "15 percenként", + "interval_1h": "Óránként", + "interval_30m": "30 percenként", + "interval_6h": "6 óránként", + "interval_desc": "Milyen gyakran szinkronizál az Android magától. A Bulwarkban végzett szerkesztések és a push útján jelzett módosítások hamarabb szinkronizálódnak.", + "interval_label": "Szinkronizálási időköz", + "interval_manual": "Nincs rendszeres szinkronizálás", + "keep_syncing": "Szinkronizálás folytatása", + "list_error_auth": "A szerver nem fogadta el a bejelentkezést.", + "list_error_network": "Nem sikerült elérni a szervert.", + "list_error_server": "A szerver nem tudta visszaadni a listát.", + "list_error_signed_out": "Ez a fiók nincs bejelentkezve ezen az eszközön.", + "master_off": "Az automatikus szinkronizálás ki van kapcsolva ezen az eszközön, ezért szinkronizálás csak akkor történik, ha a „Szinkronizálás most” gombra koppintasz.", + "new_contacts_default": "Alapértelmezett címjegyzék", + "new_contacts_desc": "A címjegyzék az eszköz más alkalmazásaiban létrehozott névjegyekhez.", + "new_contacts_label": "Új névjegyek helye", + "no_books": "Ennek a fióknak nincs címjegyzéke.", + "no_calendars": "Ennek a fióknak nincs szinkronizálható naptára.", + "ok": "OK", + "open_app_settings": "Beállítások megnyitása", + "own_collections": "Saját", + "rationale_calendar": "Ahhoz, hogy a naptáraid megjelenjenek a Naptár alkalmazásban, a Bulwarknak hozzá kell férnie az eszköz naptáraihoz. Csak a saját fiókja naptárait módosítja.", + "rationale_contacts": "Ahhoz, hogy a címjegyzékeid megjelenjenek a Névjegyek alkalmazásban, a Bulwarknak hozzá kell férnie az eszköz névjegyeihez. Csak a saját fiókja névjegyeit módosítja.", + "rationale_title_calendar": "Hozzáférés engedélyezése a naptárakhoz", + "rationale_title_contacts": "Hozzáférés engedélyezése a névjegyekhez", + "read_only": "Csak olvasható", + "reminder_owner_bulwark": "Bulwark", + "reminder_owner_device": "Naptár alkalmazás", + "reminder_question_message": "A szinkronizált eseményekre az eszköz naptáralkalmazása vagy a Bulwark emlékeztethet. Válassz egyet, hogy ne kapj kétszer emlékeztetőt. Később módosíthatod.", + "reminder_question_title": "Ki emlékeztessen?", + "reminders_desc": "Ki emlékeztet a szinkronizált eseményekre. A másik csendben marad, így nem kapsz kétszer emlékeztetőt.", + "reminders_label": "Emlékeztetők", + "reminders_nobody": "A Bulwark eseményértesítései ki vannak kapcsolva, így semmi sem emlékeztet a szinkronizált eseményekre.", + "remove": "Eltávolítás", + "remove_from_device": "Eltávolítás erről az eszközről", + "remove_message": "A fiók névjegyei és naptárai törlődnek erről az eszközről. Az itt végzett, a szerverre még el nem jutott módosítások elvesznek.", + "remove_title": "Eltávolítod erről az eszközről?", + "review_deletions": "Törlések áttekintése", + "shared_by": "Megosztotta: {name}", + "sign_in_again": "Újbóli bejelentkezés", + "status_auth": "A szerver már nem fogadja el a bejelentkezést. Jelentkezz be újra a szinkronizálás folytatásához.", + "status_cancelled": "A legutóbbi szinkronizálás idő előtt leállt. Legközelebb folytatódik.", + "status_failed": "A legutóbbi szinkronizálás sikertelen: {message}", + "status_failed_plain": "A legutóbbi szinkronizálás sikertelen.", + "status_io": "A szerver nem volt elérhető ({when}). Az Android újra megpróbálja.", + "status_item_errors": "{count, plural, one {# elemet nem sikerült szinkronizálni.} other {# elemet nem sikerült szinkronizálni.}}", + "status_off": "Kikapcsolva", + "status_paused": "Szüneteltetve az Android-beállításokban", + "status_permission_calendar": "A naptárakhoz való hozzáférés ki lett kapcsolva, ezért semmi sem szinkronizálódik.", + "status_permission_contacts": "A névjegyekhez való hozzáférés ki lett kapcsolva, ezért semmi sem szinkronizálódik.", + "status_removed": "Eltávolítva az Android-beállításokban. Kapcsold be a szinkronizálást a fiók újbóli hozzáadásához.", + "status_safety_abort": "Leállítva: a szerver semmit sem adott vissza, pedig ezen az eszközön még vannak elemek. Ellenőrizd a szervert, majd szinkronizálj újra.", + "status_synced": "Szinkronizálva: {when}", + "status_syncing": "Szinkronizálás…", + "status_too_many_deletions": "Ezen az eszközön sok elemet töröltek. A szerveren csak akkor törlődnek, ha megerősíted.", + "status_unsupported_calendar": "A szerver nem kínál naptárakat ehhez a fiókhoz.", + "status_unsupported_contacts": "A szerver nem kínál címjegyzékeket ehhez a fiókhoz.", + "status_waiting": "Várakozás az első szinkronizálásra", + "status_waiting_next": "Várakozás a következő szinkronizálásra", + "suspended_desc": "Kijelentkezve. A fiók névjegyei és naptárai az itt végzett módosításokkal együtt ezen az eszközön maradnak, és bejelentkezés után újra szinkronizálódnak.", + "switch_label": "{account} szinkronizálása ezzel az eszközzel", + "sync_now": "Szinkronizálás most", + "turn_off_anyway": "Kikapcsolás", + "turn_off_failed": "A szinkronizált adatokat nem sikerült eltávolítani erről az eszközről. A szinkronizálás szünetel; próbáld újra később.", + "turn_off_failed_title": "Nem sikerült kikapcsolni a szinkronizálást", + "turn_off_pending": "{count, plural, one {Ezen az eszközön # módosítás még nem jutott el a szerverre. Elvész, ha most kikapcsolod a szinkronizálást.} other {Ezen az eszközön # módosítás még nem jutott el a szerverre. Elvesznek, ha most kikapcsolod a szinkronizálást.}}", + "turn_off_title": "Mindenképp kikapcsolod?", + "turn_off_unknown": "A Bulwark nem tudta ellenőrizni, hogy az ezen az eszközön végzett módosítások eljutottak-e a szerverre. Elvesznek, ha most kikapcsolod a szinkronizálást.", + "turning_off": "Módosítások feltöltése, majd az adatok eltávolítása erről az eszközről…", + "turning_on": "Beállítás folyamatban…" + }, "downloads": { "attachment_template": { "description_mobile": "Csatolmány mentésekor vagy megosztásakor használt sablon." diff --git a/locales/rn/it.json b/locales/rn/it.json index dd947280..a9ead67c 100644 --- a/locales/rn/it.json +++ b/locales/rn/it.json @@ -93,6 +93,8 @@ "until_date": "Fino al {date}" }, "settings": { + "device_sync_desc": "Mostra i tuoi calendari nell'app Calendario e nelle altre app di questo dispositivo. Le modifiche vengono sincronizzate in entrambe le direzioni.", + "device_sync_title": "Sincronizza con questo dispositivo", "show_week_numbers_mobile_desc": "Mostra i numeri di settimana nella vista mensile.", "time_zone": "Fuso orario", "time_zone_auto": "Fuso orario del dispositivo", @@ -249,6 +251,18 @@ "color_tag": "Etichetta", "remove_color": "Rimuovi etichetta" }, + "device_sync": { + "auth_problem_text": "Il server non accetta più l'accesso di {account}, quindi i suoi contatti e calendari su questo dispositivo non vengono più sincronizzati. Accedi di nuovo per riprendere. Le modifiche fatte su questo dispositivo vengono mantenute.", + "auth_problem_title": "Accedi di nuovo per continuare a sincronizzare", + "problem_channel": "Problemi di sincronizzazione", + "sign_out_anyway": "Esci", + "sign_out_pending_message": "{count, plural, one {# modifica fatta su questo dispositivo non ha raggiunto il server. Andrà persa se esci ora.} other {# modifiche fatte su questo dispositivo non hanno raggiunto il server. Andranno perse se esci ora.}}", + "sign_out_pending_title": "Uscire comunque?", + "sign_out_unknown_message": "Le modifiche fatte su questo dispositivo potrebbero non aver raggiunto il server. Andranno perse se esci ora.", + "signed_out_text": "{account} è stato disconnesso, quindi i suoi contatti e calendari su questo dispositivo non vengono più sincronizzati. Accedi di nuovo per riprendere. Le modifiche fatte su questo dispositivo vengono mantenute.", + "signed_out_title": "Accedi per continuare a sincronizzare", + "signing_out": "Rimozione dei contatti e dei calendari sincronizzati da questo dispositivo…" + }, "email_composer": { "attach_camera": "Fotocamera", "attachment_reminder_body": "Il tuo messaggio menziona \"{keyword}\" ma nessun file è allegato. Inviare comunque?", @@ -598,11 +612,110 @@ "book_count": "{count, plural, one {# contatto} other {# contatti}}", "default_book": "Rubrica predefinita", "description_mobile": "Preferenze di visualizzazione e strumenti per le rubriche.", + "device_sync_desc": "Mostra le tue rubriche nell'app Contatti e nelle altre app di questo dispositivo. Le modifiche vengono sincronizzate in entrambe le direzioni.", + "device_sync_title": "Sincronizza con questo dispositivo", "export_description_count": "{count, plural, one {Esporta # contatto come un unico file vCard.} other {Esporta # contatti come un unico file vCard.}}", "export_empty": "Nessun contatto da esportare", "import_description_mobile": "Importa contatti da un file vCard (.vcf) con rilevamento dei duplicati.", "manage_description_mobile": "Crea, rinomina e rimuovi le rubriche che organizzano i tuoi contatti." }, + "device_sync": { + "also_synced_by": "Si sincronizza anche per {accounts}, quindi compare due volte su questo dispositivo.", + "android_settings": "Impostazioni account Android", + "blocked_calendar": "Android non chiede più l'accesso ai calendari. Consentilo nelle impostazioni dell'app, poi riattiva la sincronizzazione.", + "blocked_contacts": "Android non chiede più l'accesso ai contatti. Consentilo nelle impostazioni dell'app, poi riattiva la sincronizzazione.", + "blocked_title": "L'accesso è disattivato", + "books_default": "Le tue rubriche vengono sincronizzate; quelle condivise solo se le scegli.", + "books_hint": "Le tue rubriche vengono sincronizzate a meno che tu non le disattivi; quelle condivise solo se le attivi.", + "books_label": "Rubriche", + "calendars_default": "I tuoi calendari vengono sincronizzati; quelli condivisi solo se li scegli.", + "calendars_hint": "I tuoi calendari vengono sincronizzati a meno che tu non li disattivi; quelli condivisi solo se li attivi. Gli elenchi di attività restano in Bulwark.", + "calendars_label": "Calendari", + "choose": "Scegli", + "choose_books_title": "Rubriche da sincronizzare", + "choose_calendars_title": "Calendari da sincronizzare", + "collections_count": "Sincronizzazione: {selected} su {total}", + "continue": "Continua", + "deletions_delete": "Elimina dal server", + "deletions_message": "Molti elementi sono stati eliminati contemporaneamente su questo dispositivo, quindi non sono stati eliminati sul server. Eliminali anche lì o ripristinali su questo dispositivo.", + "deletions_restore": "Ripristina", + "deletions_title": "Eliminarli anche sul server?", + "denied_calendar": "Senza l'accesso ai calendari, Bulwark non può portare i tuoi calendari su questo dispositivo.", + "denied_contacts": "Senza l'accesso ai contatti, Bulwark non può portare le tue rubriche su questo dispositivo.", + "denied_title": "La sincronizzazione resta disattivata", + "failed_title": "Impossibile attivare la sincronizzazione", + "grant_access": "Consenti l'accesso", + "interval_15m": "Ogni 15 minuti", + "interval_1h": "Ogni ora", + "interval_30m": "Ogni 30 minuti", + "interval_6h": "Ogni 6 ore", + "interval_desc": "La frequenza con cui Android sincronizza da solo. Le modifiche fatte in Bulwark e quelle notificate via push vengono sincronizzate prima.", + "interval_label": "Intervallo di sincronizzazione", + "interval_manual": "Nessuna sincronizzazione periodica", + "keep_syncing": "Continua a sincronizzare", + "list_error_auth": "Il server non ha accettato l'accesso.", + "list_error_network": "Impossibile raggiungere il server.", + "list_error_server": "Il server non è riuscito a fornire l'elenco.", + "list_error_signed_out": "Su questo dispositivo non hai eseguito l'accesso a questo account.", + "master_off": "La sincronizzazione automatica è disattivata su questo dispositivo, quindi si sincronizza solo quando tocchi \"Sincronizza ora\".", + "new_contacts_default": "Rubrica predefinita", + "new_contacts_desc": "La rubrica per i contatti creati in altre app su questo dispositivo.", + "new_contacts_label": "Salva i nuovi contatti in", + "no_books": "Questo account non ha rubriche.", + "no_calendars": "Questo account non ha calendari da sincronizzare.", + "ok": "OK", + "open_app_settings": "Apri impostazioni app", + "own_collections": "Personali", + "rationale_calendar": "Per mostrare i tuoi calendari nell'app Calendario, Bulwark ha bisogno dell'accesso ai calendari di questo dispositivo. Modifica solo i calendari del proprio account.", + "rationale_contacts": "Per mostrare le tue rubriche nell'app Contatti, Bulwark ha bisogno dell'accesso ai contatti di questo dispositivo. Modifica solo i contatti del proprio account.", + "rationale_title_calendar": "Consenti l'accesso ai calendari", + "rationale_title_contacts": "Consenti l'accesso ai contatti", + "read_only": "Sola lettura", + "reminder_owner_bulwark": "Bulwark", + "reminder_owner_device": "App Calendario", + "reminder_question_message": "I promemoria degli eventi sincronizzati possono arrivare dall'app di calendario di questo dispositivo o da Bulwark. Scegline uno per non riceverli due volte. Puoi cambiare idea in seguito.", + "reminder_question_title": "Chi deve inviarti i promemoria?", + "reminders_desc": "Chi ti ricorda gli eventi sincronizzati. L'altro resta in silenzio, così non ricevi due promemoria.", + "reminders_label": "Promemoria", + "reminders_nobody": "Le notifiche degli eventi di Bulwark sono disattivate, quindi niente ti ricorda gli eventi sincronizzati.", + "remove": "Rimuovi", + "remove_from_device": "Rimuovi da questo dispositivo", + "remove_message": "I contatti e i calendari dell'account vengono rimossi da questo dispositivo. Le modifiche fatte qui che non hanno raggiunto il server andranno perse.", + "remove_title": "Rimuovere da questo dispositivo?", + "review_deletions": "Controlla eliminazioni", + "shared_by": "Condiviso da {name}", + "sign_in_again": "Accedi di nuovo", + "status_auth": "Il server non accetta più l'accesso. Accedi di nuovo per continuare a sincronizzare.", + "status_cancelled": "L'ultima sincronizzazione si è interrotta prima del termine. Riprenderà la prossima volta.", + "status_failed": "Ultima sincronizzazione non riuscita: {message}", + "status_failed_plain": "Ultima sincronizzazione non riuscita.", + "status_io": "Il server non era raggiungibile {when}. Android riproverà.", + "status_item_errors": "{count, plural, one {Impossibile sincronizzare # elemento.} other {Impossibile sincronizzare # elementi.}}", + "status_off": "Disattivata", + "status_paused": "In pausa nelle impostazioni di Android", + "status_permission_calendar": "L'accesso ai calendari è stato disattivato, quindi non viene sincronizzato nulla.", + "status_permission_contacts": "L'accesso ai contatti è stato disattivato, quindi non viene sincronizzato nulla.", + "status_removed": "Rimosso nelle impostazioni di Android. Attiva la sincronizzazione per aggiungere di nuovo l'account.", + "status_safety_abort": "Interrotta: il server non ha restituito nulla mentre questo dispositivo contiene ancora elementi. Controlla il server, poi sincronizza di nuovo.", + "status_synced": "Sincronizzato {when}", + "status_syncing": "Sincronizzazione…", + "status_too_many_deletions": "Molti elementi sono stati eliminati su questo dispositivo. Non verranno eliminati sul server finché non confermi.", + "status_unsupported_calendar": "Il server non offre calendari per questo account.", + "status_unsupported_contacts": "Il server non offre rubriche per questo account.", + "status_waiting": "In attesa della prima sincronizzazione", + "status_waiting_next": "In attesa della prossima sincronizzazione", + "suspended_desc": "Disconnesso. I suoi contatti e calendari restano su questo dispositivo, con le modifiche fatte qui, e verranno sincronizzati di nuovo quando accedi.", + "switch_label": "Sincronizza {account} con questo dispositivo", + "sync_now": "Sincronizza ora", + "turn_off_anyway": "Disattiva", + "turn_off_failed": "Impossibile rimuovere i dati sincronizzati da questo dispositivo. La sincronizzazione è in pausa; riprova più tardi.", + "turn_off_failed_title": "Impossibile disattivare la sincronizzazione", + "turn_off_pending": "{count, plural, one {# modifica fatta su questo dispositivo non ha ancora raggiunto il server. Andrà persa se disattivi ora la sincronizzazione.} other {# modifiche fatte su questo dispositivo non hanno ancora raggiunto il server. Andranno perse se disattivi ora la sincronizzazione.}}", + "turn_off_title": "Disattivare comunque?", + "turn_off_unknown": "Bulwark non è riuscito a verificare se le modifiche fatte su questo dispositivo hanno raggiunto il server. Andranno perse se disattivi ora la sincronizzazione.", + "turning_off": "Caricamento delle modifiche, poi rimozione dei dati da questo dispositivo…", + "turning_on": "Configurazione…" + }, "downloads": { "attachment_template": { "description_mobile": "Modello usato quando salvi o condividi un allegato." diff --git a/locales/rn/ja.json b/locales/rn/ja.json index 0b517a77..1d661c4d 100644 --- a/locales/rn/ja.json +++ b/locales/rn/ja.json @@ -93,6 +93,8 @@ "until_date": "{date}まで" }, "settings": { + "device_sync_desc": "カレンダーアプリやこのデバイスの他のアプリにカレンダーを表示します。変更は双方向に同期されます。", + "device_sync_title": "このデバイスに同期", "show_week_numbers_mobile_desc": "月表示に週番号を表示する", "time_zone": "タイムゾーン", "time_zone_auto": "デバイスのタイムゾーン", @@ -249,6 +251,18 @@ "color_tag": "ラベル", "remove_color": "ラベルを削除" }, + "device_sync": { + "auth_problem_text": "サーバーが {account} のサインインを受け付けなくなったため、このデバイス上のこのアカウントの連絡先とカレンダーの同期が停止しました。再開するには、もう一度サインインしてください。このデバイスで行った変更は保持されます。", + "auth_problem_title": "同期を続けるには再度サインインしてください", + "problem_channel": "同期の問題", + "sign_out_anyway": "サインアウト", + "sign_out_pending_message": "{count, plural, other {このデバイスで行った#件の変更がサーバーに届いていません。今サインアウトすると失われます。}}", + "sign_out_pending_title": "それでもサインアウトしますか？", + "sign_out_unknown_message": "このデバイスで行った変更がサーバーに届いていない可能性があります。今サインアウトすると失われます。", + "signed_out_text": "{account} からサインアウトされたため、このデバイス上のこのアカウントの連絡先とカレンダーは同期されなくなりました。再開するには、もう一度サインインしてください。このデバイスで行った変更は保持されます。", + "signed_out_title": "同期を続けるにはサインインしてください", + "signing_out": "同期した連絡先とカレンダーをこのデバイスから削除しています…" + }, "email_composer": { "attach_camera": "カメラ", "attachment_reminder_body": "メッセージに「{keyword}」が含まれていますが、ファイルが添付されていません。このまま送信しますか？", @@ -598,11 +612,110 @@ "book_count": "{count, plural, other {#件の連絡先}}", "default_book": "デフォルトのアドレス帳", "description_mobile": "表示設定とアドレス帳のツールです。", + "device_sync_desc": "連絡帳アプリやこのデバイスの他のアプリにアドレス帳を表示します。変更は双方向に同期されます。", + "device_sync_title": "このデバイスに同期", "export_description_count": "{count, plural, other {#件の連絡先を1つの vCard ファイルとしてエクスポートします。}}", "export_empty": "エクスポートする連絡先がありません", "import_description_mobile": "重複を検出しながら、vCard（.vcf）ファイルから連絡先をインポートします。", "manage_description_mobile": "連絡先を整理するアドレス帳の作成、名前変更、削除ができます。" }, + "device_sync": { + "also_synced_by": "{accounts} でも同期されているため、このデバイスには2回表示されます。", + "android_settings": "Android のアカウント設定", + "blocked_calendar": "Android はカレンダーへのアクセスを求めなくなりました。アプリの設定で許可してから、同期をもう一度オンにしてください。", + "blocked_contacts": "Android は連絡先へのアクセスを求めなくなりました。アプリの設定で許可してから、同期をもう一度オンにしてください。", + "blocked_title": "アクセスがオフになっています", + "books_default": "自分のアドレス帳は同期されます。共有されたアドレス帳は選択した場合のみ同期されます。", + "books_hint": "自分のアドレス帳はオフにしない限り同期されます。共有されたアドレス帳はオンにした場合のみ同期されます。", + "books_label": "アドレス帳", + "calendars_default": "自分のカレンダーは同期されます。共有されたカレンダーは選択した場合のみ同期されます。", + "calendars_hint": "自分のカレンダーはオフにしない限り同期されます。共有されたカレンダーはオンにした場合のみ同期されます。タスクリストは Bulwark に残ります。", + "calendars_label": "カレンダー", + "choose": "選択", + "choose_books_title": "同期するアドレス帳", + "choose_calendars_title": "同期するカレンダー", + "collections_count": "{total}件中{selected}件を同期", + "continue": "続行", + "deletions_delete": "サーバーで削除", + "deletions_message": "このデバイスで多数の項目が一度に削除されたため、サーバーでは削除されていません。サーバーでも削除するか、このデバイスに復元してください。", + "deletions_restore": "復元", + "deletions_title": "サーバーでも削除しますか？", + "denied_calendar": "カレンダーへのアクセスがないと、Bulwark はカレンダーをこのデバイスに追加できません。", + "denied_contacts": "連絡先へのアクセスがないと、Bulwark はアドレス帳をこのデバイスに追加できません。", + "denied_title": "同期はオフのままです", + "failed_title": "同期をオンにできませんでした", + "grant_access": "アクセスを許可", + "interval_15m": "15分ごと", + "interval_1h": "1時間ごと", + "interval_30m": "30分ごと", + "interval_6h": "6時間ごと", + "interval_desc": "Android が自動で同期する頻度です。Bulwark での編集やプッシュで通知された変更は、より早く同期されます。", + "interval_label": "同期の間隔", + "interval_manual": "定期的に同期しない", + "keep_syncing": "同期を続ける", + "list_error_auth": "サーバーがサインインを受け付けませんでした。", + "list_error_network": "サーバーに接続できませんでした。", + "list_error_server": "サーバーが一覧を返せませんでした。", + "list_error_signed_out": "このデバイスではこのアカウントにサインインしていません。", + "master_off": "このデバイスでは自動同期がオフになっているため、「今すぐ同期」をタップしたときにのみ同期されます。", + "new_contacts_default": "デフォルトのアドレス帳", + "new_contacts_desc": "このデバイスの他のアプリで作成した連絡先の保存先となるアドレス帳です。", + "new_contacts_label": "新しい連絡先の保存先", + "no_books": "このアカウントにはアドレス帳がありません。", + "no_calendars": "このアカウントには同期できるカレンダーがありません。", + "ok": "OK", + "open_app_settings": "アプリの設定を開く", + "own_collections": "自分のもの", + "rationale_calendar": "カレンダーアプリにカレンダーを表示するには、Bulwark にこのデバイスのカレンダーへのアクセスを許可する必要があります。Bulwark が変更するのは自分のアカウントのカレンダーだけです。", + "rationale_contacts": "連絡帳アプリにアドレス帳を表示するには、Bulwark にこのデバイスの連絡先へのアクセスを許可する必要があります。Bulwark が変更するのは自分のアカウントの連絡先だけです。", + "rationale_title_calendar": "カレンダーへのアクセスを許可", + "rationale_title_contacts": "連絡先へのアクセスを許可", + "read_only": "読み取り専用", + "reminder_owner_bulwark": "Bulwark", + "reminder_owner_device": "カレンダーアプリ", + "reminder_question_message": "同期した予定のリマインダーは、このデバイスのカレンダーアプリか Bulwark のどちらかから通知できます。重複して通知されないよう、どちらか一方を選んでください。後で変更できます。", + "reminder_question_title": "どちらから通知しますか？", + "reminders_desc": "同期した予定のリマインダーを通知するアプリです。もう一方は通知しないため、重複して通知されません。", + "reminders_label": "リマインダー", + "reminders_nobody": "Bulwark の予定の通知がオフになっているため、同期した予定のリマインダーは通知されません。", + "remove": "削除", + "remove_from_device": "このデバイスから削除", + "remove_message": "このアカウントの連絡先とカレンダーがこのデバイスから削除されます。ここで行った変更のうち、サーバーに届いていないものは失われます。", + "remove_title": "このデバイスから削除しますか？", + "review_deletions": "削除を確認", + "shared_by": "{name} が共有", + "sign_in_again": "再度サインイン", + "status_auth": "サーバーがサインインを受け付けなくなりました。同期を続けるには、再度サインインしてください。", + "status_cancelled": "前回の同期は途中で停止しました。次回に続きから同期します。", + "status_failed": "前回の同期に失敗しました：{message}", + "status_failed_plain": "前回の同期に失敗しました。", + "status_io": "サーバーに接続できませんでした（{when}）。Android が再試行します。", + "status_item_errors": "{count, plural, other {#件の項目を同期できませんでした。}}", + "status_off": "オフ", + "status_paused": "Android の設定で一時停止中", + "status_permission_calendar": "カレンダーへのアクセスがオフになったため、何も同期されません。", + "status_permission_contacts": "連絡先へのアクセスがオフになったため、何も同期されません。", + "status_removed": "Android の設定で削除されました。アカウントを再度追加するには、同期をオンにしてください。", + "status_safety_abort": "停止しました：このデバイスに項目が残っているのに、サーバーから何も返されませんでした。サーバーを確認してから、もう一度同期してください。", + "status_synced": "最終同期：{when}", + "status_syncing": "同期中…", + "status_too_many_deletions": "このデバイスで多数の項目が削除されました。確認するまでサーバーでは削除されません。", + "status_unsupported_calendar": "サーバーはこのアカウントのカレンダーを提供していません。", + "status_unsupported_contacts": "サーバーはこのアカウントのアドレス帳を提供していません。", + "status_waiting": "初回の同期を待っています", + "status_waiting_next": "次回の同期を待っています", + "suspended_desc": "サインアウトしました。このアカウントの連絡先とカレンダーは、ここで行った変更とともにこのデバイスに残り、サインインすると再び同期されます。", + "switch_label": "{account} をこのデバイスに同期", + "sync_now": "今すぐ同期", + "turn_off_anyway": "オフにする", + "turn_off_failed": "同期したデータをこのデバイスから削除できませんでした。同期は一時停止しています。後でもう一度お試しください。", + "turn_off_failed_title": "同期をオフにできませんでした", + "turn_off_pending": "{count, plural, other {このデバイスで行った#件の変更がまだサーバーに届いていません。今同期をオフにすると失われます。}}", + "turn_off_title": "それでもオフにしますか？", + "turn_off_unknown": "このデバイスで行った変更がサーバーに届いたかどうかを Bulwark で確認できませんでした。今同期をオフにすると失われます。", + "turning_off": "変更をアップロードしてから、このデバイスのデータを削除しています…", + "turning_on": "設定しています…" + }, "downloads": { "attachment_template": { "description_mobile": "添付ファイルを保存または共有するときに使用するテンプレートです。" diff --git a/locales/rn/ko.json b/locales/rn/ko.json index e6f26abf..b32fa6f1 100644 --- a/locales/rn/ko.json +++ b/locales/rn/ko.json @@ -93,6 +93,8 @@ "until_date": "{date}까지" }, "settings": { + "device_sync_desc": "캘린더 앱과 이 기기의 다른 앱에 내 캘린더를 표시해요. 변경 사항은 양방향으로 동기화돼요.", + "device_sync_title": "이 기기에 동기화", "show_week_numbers_mobile_desc": "월간 보기에 주 차 번호를 표시해요.", "time_zone": "시간대", "time_zone_auto": "기기 시간대", @@ -249,6 +251,18 @@ "color_tag": "태그", "remove_color": "태그 제거" }, + "device_sync": { + "auth_problem_text": "서버가 더 이상 {account} 계정의 로그인을 허용하지 않아서 이 기기에 있는 이 계정의 연락처와 캘린더 동기화가 중단되었어요. 다시 시작하려면 다시 로그인해 주세요. 이 기기에서 변경한 내용은 유지돼요.", + "auth_problem_title": "계속 동기화하려면 다시 로그인해 주세요", + "problem_channel": "동기화 문제", + "sign_out_anyway": "로그아웃", + "sign_out_pending_message": "{count, plural, other {이 기기에서 변경한 내용 #개가 서버에 전달되지 않았어요. 지금 로그아웃하면 사라져요.}}", + "sign_out_pending_title": "그래도 로그아웃할까요?", + "sign_out_unknown_message": "이 기기에서 변경한 내용이 서버에 전달되지 않았을 수 있어요. 지금 로그아웃하면 사라져요.", + "signed_out_text": "{account} 계정에서 로그아웃되어 이 기기에 있는 이 계정의 연락처와 캘린더가 더 이상 동기화되지 않아요. 다시 시작하려면 다시 로그인해 주세요. 이 기기에서 변경한 내용은 유지돼요.", + "signed_out_title": "계속 동기화하려면 로그인해 주세요", + "signing_out": "동기화된 연락처와 캘린더를 이 기기에서 삭제하는 중…" + }, "email_composer": { "attach_camera": "카메라", "attachment_reminder_body": "메시지에 \"{keyword}\"이(가) 언급되어 있지만 첨부된 파일이 없어요. 그래도 보낼까요?", @@ -598,11 +612,110 @@ "book_count": "{count, plural, other {연락처 #개}}", "default_book": "기본 주소록", "description_mobile": "표시 설정과 주소록 도구예요.", + "device_sync_desc": "연락처 앱과 이 기기의 다른 앱에 내 주소록을 표시해요. 변경 사항은 양방향으로 동기화돼요.", + "device_sync_title": "이 기기에 동기화", "export_description_count": "{count, plural, other {연락처 #개를 하나의 vCard 파일로 내보내요.}}", "export_empty": "내보낼 연락처가 없어요", "import_description_mobile": "vCard(.vcf) 파일에서 연락처를 가져오고 중복을 감지해요.", "manage_description_mobile": "연락처를 정리하는 주소록을 만들고, 이름을 바꾸고, 삭제할 수 있어요." }, + "device_sync": { + "also_synced_by": "{accounts} 계정에서도 동기화되어 이 기기에 두 번 표시돼요.", + "android_settings": "Android 계정 설정", + "blocked_calendar": "Android가 더 이상 캘린더 접근 권한을 요청하지 않아요. 앱 설정에서 허용한 후 동기화를 다시 켜 주세요.", + "blocked_contacts": "Android가 더 이상 연락처 접근 권한을 요청하지 않아요. 앱 설정에서 허용한 후 동기화를 다시 켜 주세요.", + "blocked_title": "접근 권한이 꺼져 있어요", + "books_default": "내 주소록은 동기화되고, 공유된 주소록은 선택한 경우에만 동기화돼요.", + "books_hint": "내 주소록은 끄지 않는 한 동기화되고, 공유된 주소록은 켠 경우에만 동기화돼요.", + "books_label": "주소록", + "calendars_default": "내 캘린더는 동기화되고, 공유된 캘린더는 선택한 경우에만 동기화돼요.", + "calendars_hint": "내 캘린더는 끄지 않는 한 동기화되고, 공유된 캘린더는 켠 경우에만 동기화돼요. 할 일 목록은 Bulwark에 남아요.", + "calendars_label": "캘린더", + "choose": "선택", + "choose_books_title": "동기화할 주소록", + "choose_calendars_title": "동기화할 캘린더", + "collections_count": "{total}개 중 {selected}개 동기화", + "continue": "계속", + "deletions_delete": "서버에서 삭제", + "deletions_message": "이 기기에서 많은 항목이 한꺼번에 삭제되어 서버에서는 삭제되지 않았어요. 서버에서도 삭제하거나 이 기기에 다시 복원해 주세요.", + "deletions_restore": "복원", + "deletions_title": "서버에서도 삭제할까요?", + "denied_calendar": "캘린더 접근 권한이 없으면 Bulwark가 내 캘린더를 이 기기에 추가할 수 없어요.", + "denied_contacts": "연락처 접근 권한이 없으면 Bulwark가 내 주소록을 이 기기에 추가할 수 없어요.", + "denied_title": "동기화가 꺼진 상태로 유지돼요", + "failed_title": "동기화를 켤 수 없어요", + "grant_access": "접근 권한 허용", + "interval_15m": "15분마다", + "interval_1h": "1시간마다", + "interval_30m": "30분마다", + "interval_6h": "6시간마다", + "interval_desc": "Android가 자동으로 동기화하는 주기예요. Bulwark에서 수정한 내용과 푸시로 알려진 변경 사항은 더 빨리 동기화돼요.", + "interval_label": "동기화 주기", + "interval_manual": "정기 동기화 안 함", + "keep_syncing": "계속 동기화", + "list_error_auth": "서버가 로그인을 허용하지 않았어요.", + "list_error_network": "서버에 연결할 수 없어요.", + "list_error_server": "서버에서 목록을 가져올 수 없어요.", + "list_error_signed_out": "이 기기에서 이 계정에 로그인되어 있지 않아요.", + "master_off": "이 기기에서 자동 동기화가 꺼져 있어서 \"지금 동기화\"를 탭할 때만 동기화돼요.", + "new_contacts_default": "기본 주소록", + "new_contacts_desc": "이 기기의 다른 앱에서 만든 연락처를 저장할 주소록이에요.", + "new_contacts_label": "새 연락처 저장 위치", + "no_books": "이 계정에는 주소록이 없어요.", + "no_calendars": "이 계정에는 동기화할 캘린더가 없어요.", + "ok": "확인", + "open_app_settings": "앱 설정 열기", + "own_collections": "내 항목", + "rationale_calendar": "캘린더 앱에 내 캘린더를 표시하려면 Bulwark에 이 기기의 캘린더 접근 권한이 필요해요. Bulwark는 자기 계정의 캘린더만 변경해요.", + "rationale_contacts": "연락처 앱에 내 주소록을 표시하려면 Bulwark에 이 기기의 연락처 접근 권한이 필요해요. Bulwark는 자기 계정의 연락처만 변경해요.", + "rationale_title_calendar": "캘린더 접근 허용", + "rationale_title_contacts": "연락처 접근 허용", + "read_only": "읽기 전용", + "reminder_owner_bulwark": "Bulwark", + "reminder_owner_device": "캘린더 앱", + "reminder_question_message": "동기화된 일정 알림은 이 기기의 캘린더 앱이나 Bulwark에서 받을 수 있어요. 알림을 두 번 받지 않도록 하나를 선택해 주세요. 나중에 변경할 수 있어요.", + "reminder_question_title": "어디에서 알림을 받을까요?", + "reminders_desc": "동기화된 일정 알림을 보내는 앱이에요. 다른 쪽은 알림을 보내지 않아 알림을 두 번 받지 않아요.", + "reminders_label": "알림", + "reminders_nobody": "Bulwark의 일정 알림이 꺼져 있어서 동기화된 일정 알림을 받을 수 없어요.", + "remove": "제거", + "remove_from_device": "이 기기에서 제거", + "remove_message": "이 계정의 연락처와 캘린더가 이 기기에서 제거돼요. 여기에서 변경한 내용 중 서버에 전달되지 않은 내용은 사라져요.", + "remove_title": "이 기기에서 제거할까요?", + "review_deletions": "삭제 항목 검토", + "shared_by": "{name} 님이 공유함", + "sign_in_again": "다시 로그인", + "status_auth": "서버가 더 이상 로그인을 허용하지 않아요. 계속 동기화하려면 다시 로그인해 주세요.", + "status_cancelled": "마지막 동기화가 도중에 멈췄어요. 다음번에 이어서 진행해요.", + "status_failed": "마지막 동기화 실패: {message}", + "status_failed_plain": "마지막 동기화에 실패했어요.", + "status_io": "서버에 연결할 수 없었어요({when}). Android가 다시 시도해요.", + "status_item_errors": "{count, plural, other {항목 #개를 동기화할 수 없었어요.}}", + "status_off": "끔", + "status_paused": "Android 설정에서 일시중지됨", + "status_permission_calendar": "캘린더 접근 권한이 꺼져서 아무것도 동기화되지 않아요.", + "status_permission_contacts": "연락처 접근 권한이 꺼져서 아무것도 동기화되지 않아요.", + "status_removed": "Android 설정에서 삭제되었어요. 계정을 다시 추가하려면 동기화를 켜 주세요.", + "status_safety_abort": "중지됨: 이 기기에 아직 항목이 있는데 서버가 아무것도 반환하지 않았어요. 서버를 확인한 후 다시 동기화해 주세요.", + "status_synced": "마지막 동기화: {when}", + "status_syncing": "동기화 중…", + "status_too_many_deletions": "이 기기에서 많은 항목이 삭제되었어요. 확인하기 전까지 서버에서는 삭제되지 않아요.", + "status_unsupported_calendar": "서버가 이 계정에 캘린더를 제공하지 않아요.", + "status_unsupported_contacts": "서버가 이 계정에 주소록을 제공하지 않아요.", + "status_waiting": "첫 동기화를 기다리는 중", + "status_waiting_next": "다음 동기화를 기다리는 중", + "suspended_desc": "로그아웃됨. 이 계정의 연락처와 캘린더는 여기에서 변경한 내용과 함께 이 기기에 남아 있고, 로그인하면 다시 동기화돼요.", + "switch_label": "{account} 계정을 이 기기에 동기화", + "sync_now": "지금 동기화", + "turn_off_anyway": "끄기", + "turn_off_failed": "동기화된 데이터를 이 기기에서 제거할 수 없었어요. 동기화가 일시중지되었어요. 나중에 다시 시도해 주세요.", + "turn_off_failed_title": "동기화를 끌 수 없어요", + "turn_off_pending": "{count, plural, other {이 기기에서 변경한 내용 #개가 아직 서버에 전달되지 않았어요. 지금 동기화를 끄면 사라져요.}}", + "turn_off_title": "그래도 끌까요?", + "turn_off_unknown": "이 기기에서 변경한 내용이 서버에 전달되었는지 Bulwark에서 확인할 수 없었어요. 지금 동기화를 끄면 사라져요.", + "turning_off": "변경 사항을 업로드한 후 이 기기에서 데이터를 제거하는 중…", + "turning_on": "설정하는 중…" + }, "downloads": { "attachment_template": { "description_mobile": "첨부 파일을 저장하거나 공유할 때 사용하는 템플릿이에요." diff --git a/locales/rn/lv.json b/locales/rn/lv.json index e5c449c7..ea5131ef 100644 --- a/locales/rn/lv.json +++ b/locales/rn/lv.json @@ -93,6 +93,8 @@ "until_date": "Līdz {date}" }, "settings": { + "device_sync_desc": "Rādiet savus kalendārus lietotnē Kalendārs un citās šīs ierīces lietotnēs. Izmaiņas tiek sinhronizētas abos virzienos.", + "device_sync_title": "Sinhronizēt ar šo ierīci", "show_week_numbers_mobile_desc": "Rādīt nedēļu numurus mēneša skatā.", "time_zone": "Laika josla", "time_zone_auto": "Ierīces laika josla", @@ -249,6 +251,18 @@ "color_tag": "Tags", "remove_color": "Noņemt tagu" }, + "device_sync": { + "auth_problem_text": "Serveris vairs nepieņem konta {account} pierakstīšanos, tāpēc tā kontakti un kalendāri šajā ierīcē vairs netiek sinhronizēti. Lai turpinātu, pierakstieties vēlreiz. Šajā ierīcē veiktās izmaiņas tiek saglabātas.", + "auth_problem_title": "Pierakstieties vēlreiz, lai turpinātu sinhronizāciju", + "problem_channel": "Sinhronizācijas problēmas", + "sign_out_anyway": "Izrakstīties", + "sign_out_pending_message": "{count, plural, zero {# izmaiņas, kas veiktas šajā ierīcē, nav nonākušas serverī. Tās tiks zaudētas, ja tagad izrakstīsieties.} one {# izmaiņa, kas veikta šajā ierīcē, nav nonākusi serverī. Tā tiks zaudēta, ja tagad izrakstīsieties.} other {# izmaiņas, kas veiktas šajā ierīcē, nav nonākušas serverī. Tās tiks zaudētas, ja tagad izrakstīsieties.}}", + "sign_out_pending_title": "Tomēr izrakstīties?", + "sign_out_unknown_message": "Šajā ierīcē veiktās izmaiņas, iespējams, nav nonākušas serverī. Tās tiks zaudētas, ja tagad izrakstīsieties.", + "signed_out_text": "Konts {account} tika izrakstīts, tāpēc tā kontakti un kalendāri šajā ierīcē vairs netiek sinhronizēti. Lai turpinātu, pierakstieties vēlreiz. Šajā ierīcē veiktās izmaiņas tiek saglabātas.", + "signed_out_title": "Pierakstieties, lai turpinātu sinhronizāciju", + "signing_out": "Sinhronizētie kontakti un kalendāri tiek noņemti no šīs ierīces…" + }, "email_composer": { "attach_camera": "Kamera", "attachment_reminder_body": "Jūsu ziņojumā minēts \"{keyword}\", bet nav pievienots neviens fails. Vai tomēr sūtīt?", @@ -598,11 +612,110 @@ "book_count": "{count, plural, zero {# kontakti} one {# kontakts} other {# kontakti}}", "default_book": "Noklusējuma adrešu grāmata", "description_mobile": "Attēlošanas preferences un adrešu grāmatu rīki.", + "device_sync_desc": "Rādiet savas adrešu grāmatas lietotnē Kontakti un citās šīs ierīces lietotnēs. Izmaiņas tiek sinhronizētas abos virzienos.", + "device_sync_title": "Sinhronizēt ar šo ierīci", "export_description_count": "{count, plural, zero {Eksportēt # kontaktus vienā vCard failā.} one {Eksportēt # kontaktu vienā vCard failā.} other {Eksportēt # kontaktus vienā vCard failā.}}", "export_empty": "Nav kontaktu, ko eksportēt", "import_description_mobile": "Importēt kontaktus no vCard (.vcf) faila ar dublikātu noteikšanu.", "manage_description_mobile": "Izveidojiet, pārdēvējiet un noņemiet adrešu grāmatas, kurās kārtoti jūsu kontakti." }, + "device_sync": { + "also_synced_by": "Sinhronizējas arī kontiem {accounts}, tāpēc šajā ierīcē parādās divreiz.", + "android_settings": "Android konta iestatījumi", + "blocked_calendar": "Android vairs neprasa piekļuvi kalendāriem. Atļaujiet to lietotnes iestatījumos un pēc tam atkal ieslēdziet sinhronizāciju.", + "blocked_contacts": "Android vairs neprasa piekļuvi kontaktiem. Atļaujiet to lietotnes iestatījumos un pēc tam atkal ieslēdziet sinhronizāciju.", + "blocked_title": "Piekļuve ir izslēgta", + "books_default": "Jūsu adrešu grāmatas tiek sinhronizētas, koplietotās — tikai tad, ja tās izvēlaties.", + "books_hint": "Jūsu adrešu grāmatas tiek sinhronizētas, ja vien tās neizslēdzat; koplietotās — tikai tad, ja tās ieslēdzat.", + "books_label": "Adrešu grāmatas", + "calendars_default": "Jūsu kalendāri tiek sinhronizēti, koplietotie — tikai tad, ja tos izvēlaties.", + "calendars_hint": "Jūsu kalendāri tiek sinhronizēti, ja vien tos neizslēdzat; koplietotie — tikai tad, ja tos ieslēdzat. Uzdevumu saraksti paliek Bulwark.", + "calendars_label": "Kalendāri", + "choose": "Izvēlēties", + "choose_books_title": "Sinhronizējamās adrešu grāmatas", + "choose_calendars_title": "Sinhronizējamie kalendāri", + "collections_count": "Sinhronizē: {selected} no {total}", + "continue": "Turpināt", + "deletions_delete": "Dzēst serverī", + "deletions_message": "Šajā ierīcē vienlaikus tika izdzēsts daudz vienumu, tāpēc tie netika izdzēsti serverī. Izdzēsiet tos arī tur vai atjaunojiet tos šajā ierīcē.", + "deletions_restore": "Atjaunot", + "deletions_title": "Vai dzēst tos arī serverī?", + "denied_calendar": "Bez piekļuves kalendāriem Bulwark nevar ievietot jūsu kalendārus šajā ierīcē.", + "denied_contacts": "Bez piekļuves kontaktiem Bulwark nevar ievietot jūsu adrešu grāmatas šajā ierīcē.", + "denied_title": "Sinhronizācija paliek izslēgta", + "failed_title": "Sinhronizāciju neizdevās ieslēgt", + "grant_access": "Piešķirt piekļuvi", + "interval_15m": "Ik pēc 15 minūtēm", + "interval_1h": "Ik pēc stundas", + "interval_30m": "Ik pēc 30 minūtēm", + "interval_6h": "Ik pēc 6 stundām", + "interval_desc": "Cik bieži Android sinhronizē patstāvīgi. Labojumi Bulwark un ar push paziņotās izmaiņas tiek sinhronizēti ātrāk.", + "interval_label": "Sinhronizācijas intervāls", + "interval_manual": "Bez regulāras sinhronizācijas", + "keep_syncing": "Turpināt sinhronizāciju", + "list_error_auth": "Serveris nepieņēma pierakstīšanos.", + "list_error_network": "Neizdevās sazināties ar serveri.", + "list_error_server": "Serveris nespēja sniegt sarakstu.", + "list_error_signed_out": "Šis konts šajā ierīcē nav pierakstīts.", + "master_off": "Automātiskā sinhronizācija šajā ierīcē ir izslēgta, tāpēc sinhronizācija notiek tikai tad, kad pieskaraties \"Sinhronizēt tagad\".", + "new_contacts_default": "Noklusējuma adrešu grāmata", + "new_contacts_desc": "Adrešu grāmata kontaktiem, kas izveidoti citās šīs ierīces lietotnēs.", + "new_contacts_label": "Kur saglabāt jaunos kontaktus", + "no_books": "Šim kontam nav adrešu grāmatu.", + "no_calendars": "Šim kontam nav sinhronizējamu kalendāru.", + "ok": "Labi", + "open_app_settings": "Atvērt iestatījumus", + "own_collections": "Jūsu", + "rationale_calendar": "Lai rādītu jūsu kalendārus lietotnē Kalendārs, Bulwark nepieciešama piekļuve šīs ierīces kalendāriem. Bulwark maina tikai sava konta kalendārus.", + "rationale_contacts": "Lai rādītu jūsu adrešu grāmatas lietotnē Kontakti, Bulwark nepieciešama piekļuve šīs ierīces kontaktiem. Bulwark maina tikai sava konta kontaktus.", + "rationale_title_calendar": "Atļaut piekļuvi kalendāriem", + "rationale_title_contacts": "Atļaut piekļuvi kontaktiem", + "read_only": "Tikai lasāms", + "reminder_owner_bulwark": "Bulwark", + "reminder_owner_device": "Lietotne Kalendārs", + "reminder_question_message": "Atgādinājumus par sinhronizētajiem notikumiem var sūtīt šīs ierīces kalendāra lietotne vai Bulwark. Izvēlieties vienu, lai nesaņemtu atgādinājumus divreiz. Vēlāk to var mainīt.", + "reminder_question_title": "Kas jums atgādinās?", + "reminders_desc": "Kas jums atgādina par sinhronizētajiem notikumiem. Otrs klusē, lai jūs nesaņemtu atgādinājumus divreiz.", + "reminders_label": "Atgādinājumi", + "reminders_nobody": "Bulwark notikumu paziņojumi ir izslēgti, tāpēc nekas neatgādina par sinhronizētajiem notikumiem.", + "remove": "Noņemt", + "remove_from_device": "Noņemt no šīs ierīces", + "remove_message": "Konta kontakti un kalendāri tiks noņemti no šīs ierīces. Šeit veiktās izmaiņas, kas nav nonākušas serverī, tiks zaudētas.", + "remove_title": "Vai noņemt no šīs ierīces?", + "review_deletions": "Pārskatīt dzēšanas", + "shared_by": "Kopīgoja: {name}", + "sign_in_again": "Pierakstīties vēlreiz", + "status_auth": "Serveris vairs nepieņem pierakstīšanos. Pierakstieties vēlreiz, lai turpinātu sinhronizāciju.", + "status_cancelled": "Pēdējā sinhronizācija apstājās priekšlaicīgi. Nākamreiz tā turpināsies.", + "status_failed": "Pēdējā sinhronizācija neizdevās: {message}", + "status_failed_plain": "Pēdējā sinhronizācija neizdevās.", + "status_io": "Serveris {when} nebija sasniedzams. Android mēģinās vēlreiz.", + "status_item_errors": "{count, plural, zero {# vienumus neizdevās sinhronizēt.} one {# vienumu neizdevās sinhronizēt.} other {# vienumus neizdevās sinhronizēt.}}", + "status_off": "Izslēgta", + "status_paused": "Apturēta Android iestatījumos", + "status_permission_calendar": "Piekļuve kalendāriem tika izslēgta, tāpēc nekas netiek sinhronizēts.", + "status_permission_contacts": "Piekļuve kontaktiem tika izslēgta, tāpēc nekas netiek sinhronizēts.", + "status_removed": "Noņemts Android iestatījumos. Ieslēdziet sinhronizāciju, lai kontu pievienotu vēlreiz.", + "status_safety_abort": "Apturēts: serveris neatgrieza neko, lai gan šajā ierīcē joprojām ir vienumi. Pārbaudiet serveri un pēc tam sinhronizējiet vēlreiz.", + "status_synced": "Sinhronizēts {when}", + "status_syncing": "Sinhronizē…", + "status_too_many_deletions": "Šajā ierīcē tika izdzēsts daudz vienumu. Serverī tie netiks izdzēsti, kamēr jūs to neapstiprināsiet.", + "status_unsupported_calendar": "Serveris šim kontam nepiedāvā kalendārus.", + "status_unsupported_contacts": "Serveris šim kontam nepiedāvā adrešu grāmatas.", + "status_waiting": "Gaida pirmo sinhronizāciju", + "status_waiting_next": "Gaida nākamo sinhronizāciju", + "suspended_desc": "Izrakstīts. Konta kontakti un kalendāri paliek šajā ierīcē kopā ar šeit veiktajām izmaiņām un tiks atkal sinhronizēti, tiklīdz pierakstīsieties.", + "switch_label": "Sinhronizēt {account} ar šo ierīci", + "sync_now": "Sinhronizēt tagad", + "turn_off_anyway": "Izslēgt", + "turn_off_failed": "Sinhronizētos datus neizdevās noņemt no šīs ierīces. Sinhronizācija ir apturēta; vēlāk mēģiniet vēlreiz.", + "turn_off_failed_title": "Sinhronizāciju neizdevās izslēgt", + "turn_off_pending": "{count, plural, zero {# izmaiņas, kas veiktas šajā ierīcē, vēl nav nonākušas serverī. Tās tiks zaudētas, ja tagad izslēgsiet sinhronizāciju.} one {# izmaiņa, kas veikta šajā ierīcē, vēl nav nonākusi serverī. Tā tiks zaudēta, ja tagad izslēgsiet sinhronizāciju.} other {# izmaiņas, kas veiktas šajā ierīcē, vēl nav nonākušas serverī. Tās tiks zaudētas, ja tagad izslēgsiet sinhronizāciju.}}", + "turn_off_title": "Tomēr izslēgt?", + "turn_off_unknown": "Bulwark nevarēja pārbaudīt, vai šajā ierīcē veiktās izmaiņas ir nonākušas serverī. Tās tiks zaudētas, ja tagad izslēgsiet sinhronizāciju.", + "turning_off": "Augšupielādē izmaiņas, pēc tam noņem datus no šīs ierīces…", + "turning_on": "Notiek iestatīšana…" + }, "downloads": { "attachment_template": { "description_mobile": "Veidne, ko izmanto, saglabājot vai kopīgojot pielikumu." diff --git a/locales/rn/mn.json b/locales/rn/mn.json index eaf3f827..c6df1948 100644 --- a/locales/rn/mn.json +++ b/locales/rn/mn.json @@ -93,6 +93,8 @@ "until_date": "{date} хүртэл" }, "settings": { + "device_sync_desc": "Хуанлиудаа энэ төхөөрөмжийн Хуанли апп болон бусад аппуудад харуулна. Өөрчлөлтүүд хоёр чиглэлд синк хийгдэнэ.", + "device_sync_title": "Энэ төхөөрөмжтэй синк хийх", "show_week_numbers_mobile_desc": "Сарын харагдацад долоо хоногийн дугаарыг харуулах.", "time_zone": "Цагийн бүс", "time_zone_auto": "Төхөөрөмжийн цагийн бүс", @@ -249,6 +251,18 @@ "color_tag": "Таг", "remove_color": "Тагийг хасах" }, + "device_sync": { + "auth_problem_text": "Сервер {account} бүртгэлийн нэвтрэлтийг хүлээн авахаа больсон тул энэ төхөөрөмж дээрх түүний харилцагчид болон хуанлиуд синк хийгдэхээ болилоо. Үргэлжлүүлэхийн тулд дахин нэвтэрнэ үү. Энэ төхөөрөмж дээр хийсэн өөрчлөлтүүд хадгалагдана.", + "auth_problem_title": "Синкийг үргэлжлүүлэхийн тулд дахин нэвтэрнэ үү", + "problem_channel": "Синкийн асуудлууд", + "sign_out_anyway": "Гарах", + "sign_out_pending_message": "{count, plural, one {Энэ төхөөрөмж дээр хийсэн # өөрчлөлт серверт хүрээгүй байна. Одоо гарвал энэ нь алга болно.} other {Энэ төхөөрөмж дээр хийсэн # өөрчлөлт серверт хүрээгүй байна. Одоо гарвал эдгээр нь алга болно.}}", + "sign_out_pending_title": "Ямар ч байсан гарах уу?", + "sign_out_unknown_message": "Энэ төхөөрөмж дээр хийсэн өөрчлөлтүүд серверт хүрээгүй байж магадгүй. Одоо гарвал тэдгээр нь алга болно.", + "signed_out_text": "{account} бүртгэлээс гарсан тул энэ төхөөрөмж дээрх түүний харилцагчид болон хуанлиуд синк хийгдэхээ болилоо. Үргэлжлүүлэхийн тулд дахин нэвтэрнэ үү. Энэ төхөөрөмж дээр хийсэн өөрчлөлтүүд хадгалагдана.", + "signed_out_title": "Синкийг үргэлжлүүлэхийн тулд нэвтэрнэ үү", + "signing_out": "Синк хийсэн харилцагчид болон хуанлиудыг энэ төхөөрөмжөөс устгаж байна…" + }, "email_composer": { "attach_camera": "Камер", "attachment_reminder_body": "Таны имэйлд \"{keyword}\" гэж дурдсан ч файл хавсаргаагүй байна. Ямар ч байсан илгээх үү?", @@ -598,11 +612,110 @@ "book_count": "{count, plural, one {# харилцагч} other {# харилцагч}}", "default_book": "Өгөгдмөл хаягийн дэвтэр", "description_mobile": "Харагдах байдлын тохиргоо болон хаягийн дэвтрийн хэрэгслүүд.", + "device_sync_desc": "Хаягийн дэвтрүүдээ энэ төхөөрөмжийн Харилцагчид апп болон бусад аппуудад харуулна. Өөрчлөлтүүд хоёр чиглэлд синк хийгдэнэ.", + "device_sync_title": "Энэ төхөөрөмжтэй синк хийх", "export_description_count": "{count, plural, one {# харилцагчийг нэг vCard файл болгон экспортлох.} other {# харилцагчийг нэг vCard файл болгон экспортлох.}}", "export_empty": "Экспортлох харилцагч алга", "import_description_mobile": "vCard (.vcf) файлаас давхардлыг илрүүлэн харилцагчдыг импортлох.", "manage_description_mobile": "Харилцагчдаа цэгцлэх хаягийн дэвтрүүдийг үүсгэх, нэрийг нь өөрчлөх, устгах." }, + "device_sync": { + "also_synced_by": "{accounts} бүртгэлд мөн синк хийгддэг тул энэ төхөөрөмж дээр хоёр удаа харагдана.", + "android_settings": "Android бүртгэлийн тохиргоо", + "blocked_calendar": "Android хуанлид хандах зөвшөөрөл дахин асуухгүй. Аппын тохиргооноос зөвшөөрөөд синкийг дахин асаана уу.", + "blocked_contacts": "Android харилцагчдад хандах зөвшөөрөл дахин асуухгүй. Аппын тохиргооноос зөвшөөрөөд синкийг дахин асаана уу.", + "blocked_title": "Хандалт унтарсан байна", + "books_default": "Таны өөрийн хаягийн дэвтрүүд синк хийгдэнэ; хуваалцсан нь зөвхөн та сонгосон үед.", + "books_hint": "Таны өөрийн хаягийн дэвтрүүд таныг унтраахаас бусад тохиолдолд синк хийгдэнэ; хуваалцсан нь зөвхөн та асаасан үед.", + "books_label": "Хаягийн дэвтрүүд", + "calendars_default": "Таны өөрийн хуанлиуд синк хийгдэнэ; хуваалцсан нь зөвхөн та сонгосон үед.", + "calendars_hint": "Таны өөрийн хуанлиуд таныг унтраахаас бусад тохиолдолд синк хийгдэнэ; хуваалцсан нь зөвхөн та асаасан үед. Даалгаврын жагсаалтууд Bulwark-д үлдэнэ.", + "calendars_label": "Хуанлиуд", + "choose": "Сонгох", + "choose_books_title": "Синк хийх хаягийн дэвтрүүд", + "choose_calendars_title": "Синк хийх хуанлиуд", + "collections_count": "Синк хийгдэж буй: {selected}/{total}", + "continue": "Үргэлжлүүлэх", + "deletions_delete": "Сервер дээр устгах", + "deletions_message": "Энэ төхөөрөмж дээр олон зүйлийг нэг дор устгасан тул сервер дээр устгаагүй. Тэдгээрийг тэнд ч мөн устгах эсвэл энэ төхөөрөмж дээр сэргээнэ үү.", + "deletions_restore": "Сэргээх", + "deletions_title": "Сервер дээр ч мөн устгах уу?", + "denied_calendar": "Хуанлид хандах эрхгүйгээр Bulwark таны хуанлиудыг энэ төхөөрөмж дээр байршуулж чадахгүй.", + "denied_contacts": "Харилцагчдад хандах эрхгүйгээр Bulwark таны хаягийн дэвтрүүдийг энэ төхөөрөмж дээр байршуулж чадахгүй.", + "denied_title": "Синк унтраалттай хэвээр байна", + "failed_title": "Синкийг асааж чадсангүй", + "grant_access": "Хандалт олгох", + "interval_15m": "15 минут тутамд", + "interval_1h": "Цаг бүр", + "interval_30m": "30 минут тутамд", + "interval_6h": "6 цаг тутамд", + "interval_desc": "Android өөрөө хэр олон удаа синк хийхийг заана. Bulwark дахь засварууд болон push-аар мэдэгдсэн өөрчлөлтүүд илүү эрт синк хийгдэнэ.", + "interval_label": "Синкийн давтамж", + "interval_manual": "Тогтмол синк хийхгүй", + "keep_syncing": "Синкийг үргэлжлүүлэх", + "list_error_auth": "Сервер нэвтрэлтийг хүлээн аваагүй.", + "list_error_network": "Сервертэй холбогдож чадсангүй.", + "list_error_server": "Сервер жагсаалтыг гаргаж чадсангүй.", + "list_error_signed_out": "Энэ төхөөрөмж дээр энэ бүртгэлээр нэвтрээгүй байна.", + "master_off": "Энэ төхөөрөмж дээр автомат синк унтарсан байгаа тул зөвхөн \"Одоо синк хийх\" дээр товшиход синк хийгдэнэ.", + "new_contacts_default": "Өгөгдмөл хаягийн дэвтэр", + "new_contacts_desc": "Энэ төхөөрөмжийн бусад аппад үүсгэсэн харилцагчдыг хадгалах хаягийн дэвтэр.", + "new_contacts_label": "Шинэ харилцагчдыг хадгалах газар", + "no_books": "Энэ бүртгэлд хаягийн дэвтэр алга.", + "no_calendars": "Энэ бүртгэлд синк хийх хуанли алга.", + "ok": "OK", + "open_app_settings": "Аппын тохиргоог нээх", + "own_collections": "Таных", + "rationale_calendar": "Таны хуанлиудыг Хуанли аппад харуулахын тулд Bulwark-д энэ төхөөрөмжийн хуанлид хандах эрх хэрэгтэй. Энэ нь зөвхөн өөрийн бүртгэлийн хуанлиудыг өөрчилнө.", + "rationale_contacts": "Таны хаягийн дэвтрүүдийг Харилцагчид аппад харуулахын тулд Bulwark-д энэ төхөөрөмжийн харилцагчдад хандах эрх хэрэгтэй. Энэ нь зөвхөн өөрийн бүртгэлийн харилцагчдыг өөрчилнө.", + "rationale_title_calendar": "Хуанлид хандахыг зөвшөөрөх", + "rationale_title_contacts": "Харилцагчдад хандахыг зөвшөөрөх", + "read_only": "Зөвхөн унших", + "reminder_owner_bulwark": "Bulwark", + "reminder_owner_device": "Хуанли апп", + "reminder_question_message": "Синк хийсэн үйл явдлын сануулгыг энэ төхөөрөмжийн хуанли апп эсвэл Bulwark илгээж болно. Сануулга хоёр удаа ирэхгүйн тулд аль нэгийг нь сонгоно уу. Үүнийг дараа нь өөрчилж болно.", + "reminder_question_title": "Хэн танд сануулах вэ?", + "reminders_desc": "Синк хийсэн үйл явдлын талаар хэн танд сануулах. Нөгөө нь чимээгүй байх тул сануулга хоёр удаа ирэхгүй.", + "reminders_label": "Сануулгууд", + "reminders_nobody": "Bulwark-ын үйл явдлын мэдэгдэл унтарсан тул синк хийсэн үйл явдлын талаар юу ч танд сануулахгүй.", + "remove": "Хасах", + "remove_from_device": "Энэ төхөөрөмжөөс хасах", + "remove_message": "Бүртгэлийн харилцагчид болон хуанлиуд энэ төхөөрөмжөөс хасагдана. Энд хийсэн, серверт хүрээгүй өөрчлөлтүүд алга болно.", + "remove_title": "Энэ төхөөрөмжөөс хасах уу?", + "review_deletions": "Устгалуудыг хянах", + "shared_by": "Хуваалцсан: {name}", + "sign_in_again": "Дахин нэвтрэх", + "status_auth": "Сервер нэвтрэлтийг хүлээн авахаа больсон. Синкийг үргэлжлүүлэхийн тулд дахин нэвтэрнэ үү.", + "status_cancelled": "Сүүлийн синк эрт зогссон. Дараагийн удаа үргэлжилнэ.", + "status_failed": "Сүүлийн синк амжилтгүй: {message}", + "status_failed_plain": "Сүүлийн синк амжилтгүй боллоо.", + "status_io": "Сервертэй холбогдож чадсангүй ({when}). Android дахин оролдоно.", + "status_item_errors": "{count, plural, one {# зүйлийг синк хийж чадсангүй.} other {# зүйлийг синк хийж чадсангүй.}}", + "status_off": "Унтраалттай", + "status_paused": "Android-ийн тохиргоонд түр зогсоосон", + "status_permission_calendar": "Хуанлид хандах эрх унтарсан тул юу ч синк хийгдэхгүй.", + "status_permission_contacts": "Харилцагчдад хандах эрх унтарсан тул юу ч синк хийгдэхгүй.", + "status_removed": "Android-ийн тохиргооноос хасагдсан. Бүртгэлийг дахин нэмэхийн тулд синкийг асаана уу.", + "status_safety_abort": "Зогссон: энэ төхөөрөмж дээр зүйлс байсаар байхад сервер юу ч буцаасангүй. Серверийг шалгаад дахин синк хийнэ үү.", + "status_synced": "Сүүлийн синк: {when}", + "status_syncing": "Синк хийж байна…", + "status_too_many_deletions": "Энэ төхөөрөмж дээр олон зүйл устгагдсан. Таныг баталгаажуулах хүртэл сервер дээр устгахгүй.", + "status_unsupported_calendar": "Сервер энэ бүртгэлд хуанли санал болгодоггүй.", + "status_unsupported_contacts": "Сервер энэ бүртгэлд хаягийн дэвтэр санал болгодоггүй.", + "status_waiting": "Анхны синкийг хүлээж байна", + "status_waiting_next": "Дараагийн синкийг хүлээж байна", + "suspended_desc": "Гарсан. Түүний харилцагчид болон хуанлиуд энд хийсэн өөрчлөлтүүдийн хамт энэ төхөөрөмж дээр үлдэх бөгөөд таныг нэвтэрмэгц дахин синк хийгдэнэ.", + "switch_label": "{account} бүртгэлийг энэ төхөөрөмжтэй синк хийх", + "sync_now": "Одоо синк хийх", + "turn_off_anyway": "Унтраах", + "turn_off_failed": "Синк хийсэн өгөгдлийг энэ төхөөрөмжөөс хасаж чадсангүй. Синк түр зогссон; дараа дахин оролдоно уу.", + "turn_off_failed_title": "Синкийг унтрааж чадсангүй", + "turn_off_pending": "{count, plural, one {Энэ төхөөрөмж дээр хийсэн # өөрчлөлт серверт хараахан хүрээгүй байна. Одоо синкийг унтраавал энэ нь алга болно.} other {Энэ төхөөрөмж дээр хийсэн # өөрчлөлт серверт хараахан хүрээгүй байна. Одоо синкийг унтраавал эдгээр нь алга болно.}}", + "turn_off_title": "Ямар ч байсан унтраах уу?", + "turn_off_unknown": "Энэ төхөөрөмж дээр хийсэн өөрчлөлтүүд серверт хүрсэн эсэхийг Bulwark шалгаж чадсангүй. Одоо синкийг унтраавал тэдгээр нь алга болно.", + "turning_off": "Өөрчлөлтүүдийг илгээж, дараа нь өгөгдлийг энэ төхөөрөмжөөс хасаж байна…", + "turning_on": "Тохируулж байна…" + }, "downloads": { "attachment_template": { "description_mobile": "Хавсралтыг хадгалах эсвэл хуваалцахад ашиглах загвар." diff --git a/locales/rn/nb.json b/locales/rn/nb.json index 54042dde..f790ab68 100644 --- a/locales/rn/nb.json +++ b/locales/rn/nb.json @@ -93,6 +93,8 @@ "until_date": "Til {date}" }, "settings": { + "device_sync_desc": "Vis kalenderne dine i Kalender-appen og andre apper på denne enheten. Endringer synkroniseres begge veier.", + "device_sync_title": "Synkroniser med denne enheten", "show_week_numbers_mobile_desc": "Vis ukenummer i månedsvisningen.", "time_zone": "Tidssone", "time_zone_auto": "Enhetens tidssone", @@ -249,6 +251,18 @@ "color_tag": "Etikett", "remove_color": "Fjern etiketten" }, + "device_sync": { + "auth_problem_text": "Serveren godtar ikke lenger påloggingen for {account}, så kontaktene og kalenderne til kontoen på denne enheten synkroniseres ikke lenger. Logg på igjen for å fortsette. Endringer som er gjort på denne enheten, beholdes.", + "auth_problem_title": "Logg på igjen for å fortsette synkroniseringen", + "problem_channel": "Synkroniseringsproblemer", + "sign_out_anyway": "Logg ut", + "sign_out_pending_message": "{count, plural, one {# endring som er gjort på denne enheten, har ikke nådd serveren. Den går tapt hvis du logger ut nå.} other {# endringer som er gjort på denne enheten, har ikke nådd serveren. De går tapt hvis du logger ut nå.}}", + "sign_out_pending_title": "Vil du logge ut likevel?", + "sign_out_unknown_message": "Endringer som er gjort på denne enheten, har kanskje ikke nådd serveren. De går tapt hvis du logger ut nå.", + "signed_out_text": "{account} ble logget ut, så kontaktene og kalenderne til kontoen på denne enheten synkroniseres ikke lenger. Logg på igjen for å fortsette. Endringer som er gjort på denne enheten, beholdes.", + "signed_out_title": "Logg på for å fortsette synkroniseringen", + "signing_out": "Fjerner synkroniserte kontakter og kalendere fra denne enheten…" + }, "email_composer": { "attach_camera": "Kamera", "attachment_reminder_body": "Meldingen din nevner «{keyword}», men ingen fil er vedlagt. Sende likevel?", @@ -598,11 +612,110 @@ "book_count": "{count, plural, one {# kontakt} other {# kontakter}}", "default_book": "Standardadressebok", "description_mobile": "Visningsvalg og verktøy for adressebøker.", + "device_sync_desc": "Vis adressebøkene dine i Kontakter-appen og andre apper på denne enheten. Endringer synkroniseres begge veier.", + "device_sync_title": "Synkroniser med denne enheten", "export_description_count": "{count, plural, one {Eksporter # kontakt som én vCard-fil.} other {Eksporter # kontakter som én vCard-fil.}}", "export_empty": "Ingen kontakter å eksportere", "import_description_mobile": "Importer kontakter fra en vCard-fil (.vcf) med gjenkjenning av duplikater.", "manage_description_mobile": "Opprett, gi nytt navn til og fjern adressebøkene som organiserer kontaktene dine." }, + "device_sync": { + "also_synced_by": "Synkroniseres også for {accounts}, så den vises to ganger på denne enheten.", + "android_settings": "Android-kontoinnstillinger", + "blocked_calendar": "Android spør ikke lenger om tilgang til kalendere. Gi tilgang i appinnstillingene, og slå deretter på synkronisering igjen.", + "blocked_contacts": "Android spør ikke lenger om tilgang til kontakter. Gi tilgang i appinnstillingene, og slå deretter på synkronisering igjen.", + "blocked_title": "Tilgangen er slått av", + "books_default": "Dine egne adressebøker synkroniseres; delte bare når du velger dem.", + "books_hint": "Dine egne adressebøker synkroniseres med mindre du slår dem av; delte bare når du slår dem på.", + "books_label": "Adressebøker", + "calendars_default": "Dine egne kalendere synkroniseres; delte bare når du velger dem.", + "calendars_hint": "Dine egne kalendere synkroniseres med mindre du slår dem av; delte bare når du slår dem på. Oppgavelister blir værende i Bulwark.", + "calendars_label": "Kalendere", + "choose": "Velg", + "choose_books_title": "Adressebøker som skal synkroniseres", + "choose_calendars_title": "Kalendere som skal synkroniseres", + "collections_count": "{selected} av {total} synkroniseres", + "continue": "Fortsett", + "deletions_delete": "Slett på serveren", + "deletions_message": "Mange elementer ble slettet på én gang på denne enheten, så de ble ikke slettet på serveren. Slett dem der også, eller hent dem tilbake på denne enheten.", + "deletions_restore": "Hent dem tilbake", + "deletions_title": "Vil du slette dem på serveren også?", + "denied_calendar": "Uten tilgang til kalendere kan ikke Bulwark legge kalenderne dine på denne enheten.", + "denied_contacts": "Uten tilgang til kontakter kan ikke Bulwark legge adressebøkene dine på denne enheten.", + "denied_title": "Synkroniseringen forblir av", + "failed_title": "Kunne ikke slå på synkronisering", + "grant_access": "Gi tilgang", + "interval_15m": "Hvert 15. minutt", + "interval_1h": "Hver time", + "interval_30m": "Hvert 30. minutt", + "interval_6h": "Hver 6. time", + "interval_desc": "Hvor ofte Android synkroniserer av seg selv. Endringer i Bulwark og endringer som varsles via push, synkroniseres tidligere.", + "interval_label": "Synkroniseringsintervall", + "interval_manual": "Ingen jevnlig synkronisering", + "keep_syncing": "Fortsett synkroniseringen", + "list_error_auth": "Serveren godtok ikke påloggingen.", + "list_error_network": "Kunne ikke nå serveren.", + "list_error_server": "Serveren kunne ikke vise listen.", + "list_error_signed_out": "Denne kontoen er ikke pålogget på denne enheten.", + "master_off": "Automatisk synkronisering er slått av på denne enheten, så synkronisering skjer bare når du trykker på «Synkroniser nå».", + "new_contacts_default": "Standardadressebok", + "new_contacts_desc": "Adresseboken for kontakter som opprettes i andre apper på denne enheten.", + "new_contacts_label": "Lagre nye kontakter i", + "no_books": "Denne kontoen har ingen adressebøker.", + "no_calendars": "Denne kontoen har ingen kalendere å synkronisere.", + "ok": "OK", + "open_app_settings": "Åpne appinnstillingene", + "own_collections": "Dine egne", + "rationale_calendar": "For å vise kalenderne dine i Kalender-appen trenger Bulwark tilgang til kalenderne på denne enheten. Bulwark endrer bare kalenderne til sin egen konto.", + "rationale_contacts": "For å vise adressebøkene dine i Kontakter-appen trenger Bulwark tilgang til kontaktene på denne enheten. Bulwark endrer bare kontaktene til sin egen konto.", + "rationale_title_calendar": "Gi tilgang til kalendere", + "rationale_title_contacts": "Gi tilgang til kontakter", + "read_only": "Skrivebeskyttet", + "reminder_owner_bulwark": "Bulwark", + "reminder_owner_device": "Kalender-appen", + "reminder_question_message": "Påminnelser om synkroniserte hendelser kan komme fra kalenderappen på denne enheten eller fra Bulwark. Velg én av dem, så du ikke får to påminnelser. Du kan endre dette senere.", + "reminder_question_title": "Hvem skal minne deg på hendelser?", + "reminders_desc": "Hvem som minner deg på synkroniserte hendelser. Den andre er stille, så du ikke får to påminnelser.", + "reminders_label": "Påminnelser", + "reminders_nobody": "Hendelsesvarslene i Bulwark er slått av, så ingenting minner deg på synkroniserte hendelser.", + "remove": "Fjern", + "remove_from_device": "Fjern fra denne enheten", + "remove_message": "Kontoens kontakter og kalendere fjernes fra denne enheten. Endringer som er gjort her og ikke har nådd serveren, går tapt.", + "remove_title": "Vil du fjerne fra denne enheten?", + "review_deletions": "Gå gjennom slettinger", + "shared_by": "Delt av {name}", + "sign_in_again": "Logg på igjen", + "status_auth": "Serveren godtar ikke lenger påloggingen. Logg på igjen for å fortsette synkroniseringen.", + "status_cancelled": "Den siste synkroniseringen stoppet før den var ferdig. Den fortsetter neste gang.", + "status_failed": "Den siste synkroniseringen mislyktes: {message}", + "status_failed_plain": "Den siste synkroniseringen mislyktes.", + "status_io": "Kunne ikke nå serveren {when}. Android prøver igjen.", + "status_item_errors": "{count, plural, one {# element kunne ikke synkroniseres.} other {# elementer kunne ikke synkroniseres.}}", + "status_off": "Av", + "status_paused": "Satt på pause i Android-innstillingene", + "status_permission_calendar": "Tilgangen til kalendere ble slått av, så ingenting synkroniseres.", + "status_permission_contacts": "Tilgangen til kontakter ble slått av, så ingenting synkroniseres.", + "status_removed": "Fjernet i Android-innstillingene. Slå på synkronisering for å legge til kontoen igjen.", + "status_safety_abort": "Stoppet: Serveren returnerte ingenting, selv om denne enheten fortsatt har elementer. Sjekk serveren, og synkroniser deretter på nytt.", + "status_synced": "Synkronisert {when}", + "status_syncing": "Synkroniserer…", + "status_too_many_deletions": "Mange elementer ble slettet på denne enheten. De slettes ikke på serveren før du bekrefter.", + "status_unsupported_calendar": "Serveren tilbyr ingen kalendere for denne kontoen.", + "status_unsupported_contacts": "Serveren tilbyr ingen adressebøker for denne kontoen.", + "status_waiting": "Venter på den første synkroniseringen", + "status_waiting_next": "Venter på neste synkronisering", + "suspended_desc": "Logget ut. Kontoens kontakter og kalendere blir liggende på denne enheten sammen med endringene som er gjort her, og synkroniseres igjen når du logger på.", + "switch_label": "Synkroniser {account} med denne enheten", + "sync_now": "Synkroniser nå", + "turn_off_anyway": "Slå av", + "turn_off_failed": "De synkroniserte dataene kunne ikke fjernes fra denne enheten. Synkroniseringen er satt på pause. Prøv igjen senere.", + "turn_off_failed_title": "Kunne ikke slå av synkronisering", + "turn_off_pending": "{count, plural, one {# endring som er gjort på denne enheten, har ikke nådd serveren ennå. Den går tapt hvis du slår av synkronisering nå.} other {# endringer som er gjort på denne enheten, har ikke nådd serveren ennå. De går tapt hvis du slår av synkronisering nå.}}", + "turn_off_title": "Vil du slå av likevel?", + "turn_off_unknown": "Bulwark kunne ikke sjekke om endringer som er gjort på denne enheten, har nådd serveren. De går tapt hvis du slår av synkronisering nå.", + "turning_off": "Laster opp endringer og fjerner deretter dataene fra denne enheten…", + "turning_on": "Konfigurerer…" + }, "downloads": { "attachment_template": { "description_mobile": "Malen som brukes når et vedlegg lagres eller deles." diff --git a/locales/rn/nl.json b/locales/rn/nl.json index c657a088..859934ab 100644 --- a/locales/rn/nl.json +++ b/locales/rn/nl.json @@ -93,6 +93,8 @@ "until_date": "Tot {date}" }, "settings": { + "device_sync_desc": "Toon uw agenda's in de Agenda-app en andere apps op dit apparaat. Wijzigingen worden in beide richtingen gesynchroniseerd.", + "device_sync_title": "Synchroniseren met dit apparaat", "show_week_numbers_mobile_desc": "Weeknummers weergeven in de maandweergave.", "time_zone": "Tijdzone", "time_zone_auto": "Tijdzone van apparaat", @@ -249,6 +251,18 @@ "color_tag": "Label", "remove_color": "Label verwijderen" }, + "device_sync": { + "auth_problem_text": "De server accepteert de aanmelding van {account} niet meer, dus de contacten en agenda's van dit account op dit apparaat worden niet meer gesynchroniseerd. Meld u opnieuw aan om verder te gaan. Wijzigingen op dit apparaat blijven behouden.", + "auth_problem_title": "Meld u opnieuw aan om te blijven synchroniseren", + "problem_channel": "Synchronisatieproblemen", + "sign_out_anyway": "Afmelden", + "sign_out_pending_message": "{count, plural, one {# wijziging op dit apparaat heeft de server niet bereikt. Die gaat verloren als u zich nu afmeldt.} other {# wijzigingen op dit apparaat hebben de server niet bereikt. Die gaan verloren als u zich nu afmeldt.}}", + "sign_out_pending_title": "Toch afmelden?", + "sign_out_unknown_message": "Wijzigingen op dit apparaat hebben de server mogelijk niet bereikt. Ze gaan verloren als u zich nu afmeldt.", + "signed_out_text": "{account} is afgemeld, dus de contacten en agenda's van dit account op dit apparaat worden niet meer gesynchroniseerd. Meld u opnieuw aan om verder te gaan. Wijzigingen op dit apparaat blijven behouden.", + "signed_out_title": "Meld u aan om te blijven synchroniseren", + "signing_out": "Gesynchroniseerde contacten en agenda's worden van dit apparaat verwijderd…" + }, "email_composer": { "attach_camera": "Camera", "attachment_reminder_body": "Uw bericht vermeldt \"{keyword}\" maar er is geen bestand bijgevoegd. Toch verzenden?", @@ -598,11 +612,110 @@ "book_count": "{count, plural, one {# contact} other {# contacten}}", "default_book": "Standaardadresboek", "description_mobile": "Weergavevoorkeuren en hulpmiddelen voor adresboeken.", + "device_sync_desc": "Toon uw adresboeken in de Contacten-app en andere apps op dit apparaat. Wijzigingen worden in beide richtingen gesynchroniseerd.", + "device_sync_title": "Synchroniseren met dit apparaat", "export_description_count": "{count, plural, one {# contact exporteren als één vCard-bestand.} other {# contacten exporteren als één vCard-bestand.}}", "export_empty": "Geen contacten om te exporteren", "import_description_mobile": "Contacten importeren uit een vCard-bestand (.vcf) met detectie van duplicaten.", "manage_description_mobile": "Maak, hernoem en verwijder de adresboeken waarin uw contacten zijn ingedeeld." }, + "device_sync": { + "also_synced_by": "Wordt ook gesynchroniseerd voor {accounts} en verschijnt daarom twee keer op dit apparaat.", + "android_settings": "Android-accountinstellingen", + "blocked_calendar": "Android vraagt niet meer om toegang tot agenda's. Sta de toegang toe in de app-instellingen en zet synchronisatie daarna weer aan.", + "blocked_contacts": "Android vraagt niet meer om toegang tot contacten. Sta de toegang toe in de app-instellingen en zet synchronisatie daarna weer aan.", + "blocked_title": "Toegang staat uit", + "books_default": "Uw eigen adresboeken worden gesynchroniseerd; gedeelde alleen als u ze kiest.", + "books_hint": "Uw eigen adresboeken worden gesynchroniseerd, tenzij u ze uitzet; gedeelde alleen als u ze aanzet.", + "books_label": "Adresboeken", + "calendars_default": "Uw eigen agenda's worden gesynchroniseerd; gedeelde alleen als u ze kiest.", + "calendars_hint": "Uw eigen agenda's worden gesynchroniseerd, tenzij u ze uitzet; gedeelde alleen als u ze aanzet. Takenlijsten blijven in Bulwark.", + "calendars_label": "Agenda's", + "choose": "Kiezen", + "choose_books_title": "Te synchroniseren adresboeken", + "choose_calendars_title": "Te synchroniseren agenda's", + "collections_count": "{selected} van {total} gesynchroniseerd", + "continue": "Doorgaan", + "deletions_delete": "Op server verwijderen", + "deletions_message": "Er zijn op dit apparaat veel items tegelijk verwijderd, daarom zijn ze niet op de server verwijderd. Verwijder ze daar ook of zet ze terug op dit apparaat.", + "deletions_restore": "Terugzetten", + "deletions_title": "Ook op de server verwijderen?", + "denied_calendar": "Zonder toegang tot agenda's kan Bulwark uw agenda's niet op dit apparaat zetten.", + "denied_contacts": "Zonder toegang tot contacten kan Bulwark uw adresboeken niet op dit apparaat zetten.", + "denied_title": "Synchronisatie blijft uit", + "failed_title": "Synchronisatie kon niet worden aangezet", + "grant_access": "Toegang geven", + "interval_15m": "Elke 15 minuten", + "interval_1h": "Elk uur", + "interval_30m": "Elke 30 minuten", + "interval_6h": "Elke 6 uur", + "interval_desc": "Hoe vaak Android uit zichzelf synchroniseert. Wijzigingen in Bulwark en via push gemelde wijzigingen worden eerder gesynchroniseerd.", + "interval_label": "Synchronisatie-interval", + "interval_manual": "Geen regelmatige synchronisatie", + "keep_syncing": "Blijven synchroniseren", + "list_error_auth": "De server heeft de aanmelding niet geaccepteerd.", + "list_error_network": "Kan de server niet bereiken.", + "list_error_server": "De server kon ze niet weergeven.", + "list_error_signed_out": "Dit account is niet aangemeld op dit apparaat.", + "master_off": "Automatisch synchroniseren staat uit op dit apparaat, dus er wordt alleen gesynchroniseerd als u op \"Nu synchroniseren\" tikt.", + "new_contacts_default": "Standaardadresboek", + "new_contacts_desc": "Het adresboek voor contacten die in andere apps op dit apparaat worden gemaakt.", + "new_contacts_label": "Nieuwe contacten opslaan in", + "no_books": "Dit account heeft geen adresboeken.", + "no_calendars": "Dit account heeft geen agenda's om te synchroniseren.", + "ok": "OK", + "open_app_settings": "App-instellingen openen", + "own_collections": "Uw eigen", + "rationale_calendar": "Om uw agenda's in de Agenda-app te tonen, heeft Bulwark toegang nodig tot de agenda's op dit apparaat. Bulwark wijzigt alleen de agenda's van zijn eigen account.", + "rationale_contacts": "Om uw adresboeken in de Contacten-app te tonen, heeft Bulwark toegang nodig tot de contacten op dit apparaat. Bulwark wijzigt alleen de contacten van zijn eigen account.", + "rationale_title_calendar": "Toegang tot agenda's toestaan", + "rationale_title_contacts": "Toegang tot contacten toestaan", + "read_only": "Alleen lezen", + "reminder_owner_bulwark": "Bulwark", + "reminder_owner_device": "Agenda-app", + "reminder_question_message": "Gesynchroniseerde evenementen kunnen u herinneren via de agenda-app op dit apparaat of via Bulwark. Kies er één, zodat u niet dubbel wordt herinnerd. U kunt dit later wijzigen.", + "reminder_question_title": "Wie moet u herinneren?", + "reminders_desc": "Wie u herinnert aan gesynchroniseerde evenementen. De andere blijft stil, zodat u niet dubbel wordt herinnerd.", + "reminders_label": "Herinneringen", + "reminders_nobody": "De evenementmeldingen van Bulwark staan uit, dus niets herinnert u aan gesynchroniseerde evenementen.", + "remove": "Verwijderen", + "remove_from_device": "Verwijderen van dit apparaat", + "remove_message": "De contacten en agenda's van het account worden van dit apparaat verwijderd. Wijzigingen die hier zijn gemaakt en de server nog niet hebben bereikt, gaan verloren.", + "remove_title": "Verwijderen van dit apparaat?", + "review_deletions": "Verwijderingen bekijken", + "shared_by": "Gedeeld door {name}", + "sign_in_again": "Opnieuw aanmelden", + "status_auth": "De server accepteert de aanmelding niet meer. Meld u opnieuw aan om te blijven synchroniseren.", + "status_cancelled": "De laatste synchronisatie is voortijdig gestopt. Die gaat de volgende keer verder.", + "status_failed": "Laatste synchronisatie mislukt: {message}", + "status_failed_plain": "Laatste synchronisatie mislukt.", + "status_io": "De server was {when} niet bereikbaar. Android probeert het opnieuw.", + "status_item_errors": "{count, plural, one {# item kon niet worden gesynchroniseerd.} other {# items konden niet worden gesynchroniseerd.}}", + "status_off": "Uit", + "status_paused": "Gepauzeerd in Android-instellingen", + "status_permission_calendar": "Toegang tot agenda's is uitgezet, dus er wordt niets gesynchroniseerd.", + "status_permission_contacts": "Toegang tot contacten is uitgezet, dus er wordt niets gesynchroniseerd.", + "status_removed": "Verwijderd in Android-instellingen. Zet synchronisatie aan om het account opnieuw toe te voegen.", + "status_safety_abort": "Gestopt: de server gaf niets terug terwijl dit apparaat nog items bevat. Controleer de server en synchroniseer daarna opnieuw.", + "status_synced": "Laatst gesynchroniseerd: {when}", + "status_syncing": "Synchroniseren…", + "status_too_many_deletions": "Er zijn veel items verwijderd op dit apparaat. Ze worden pas op de server verwijderd als u dat bevestigt.", + "status_unsupported_calendar": "De server biedt geen agenda's aan voor dit account.", + "status_unsupported_contacts": "De server biedt geen adresboeken aan voor dit account.", + "status_waiting": "Wachten op de eerste synchronisatie", + "status_waiting_next": "Wachten op de volgende synchronisatie", + "suspended_desc": "Afgemeld. De contacten en agenda's van het account blijven met de hier gemaakte wijzigingen op dit apparaat en worden weer gesynchroniseerd zodra u zich aanmeldt.", + "switch_label": "{account} synchroniseren met dit apparaat", + "sync_now": "Nu synchroniseren", + "turn_off_anyway": "Uitzetten", + "turn_off_failed": "De gesynchroniseerde gegevens konden niet van dit apparaat worden verwijderd. Synchronisatie is gepauzeerd; probeer het later opnieuw.", + "turn_off_failed_title": "Synchronisatie kon niet worden uitgezet", + "turn_off_pending": "{count, plural, one {# wijziging op dit apparaat heeft de server nog niet bereikt. Die gaat verloren als u synchronisatie nu uitzet.} other {# wijzigingen op dit apparaat hebben de server nog niet bereikt. Die gaan verloren als u synchronisatie nu uitzet.}}", + "turn_off_title": "Toch uitzetten?", + "turn_off_unknown": "Bulwark kon niet controleren of wijzigingen op dit apparaat de server hebben bereikt. Ze gaan verloren als u synchronisatie nu uitzet.", + "turning_off": "Wijzigingen uploaden en daarna de gegevens van dit apparaat verwijderen…", + "turning_on": "Instellen…" + }, "downloads": { "attachment_template": { "description_mobile": "Sjabloon voor het opslaan of delen van een bijlage." diff --git a/locales/rn/pl.json b/locales/rn/pl.json index 317a88c0..4e514042 100644 --- a/locales/rn/pl.json +++ b/locales/rn/pl.json @@ -93,6 +93,8 @@ "until_date": "Do {date}" }, "settings": { + "device_sync_desc": "Pokazuj swoje kalendarze w aplikacji Kalendarz i innych aplikacjach na tym urządzeniu. Zmiany są synchronizowane w obie strony.", + "device_sync_title": "Synchronizuj z tym urządzeniem", "show_week_numbers_mobile_desc": "Wyświetlaj numery tygodni w widoku miesiąca.", "time_zone": "Strefa czasowa", "time_zone_auto": "Strefa czasowa urządzenia", @@ -249,6 +251,18 @@ "color_tag": "Etykieta", "remove_color": "Usuń etykietę" }, + "device_sync": { + "auth_problem_text": "Serwer nie akceptuje już logowania konta {account}, więc jego kontakty i kalendarze na tym urządzeniu przestały się synchronizować. Zaloguj się ponownie, aby wznowić synchronizację. Zmiany wprowadzone na tym urządzeniu zostaną zachowane.", + "auth_problem_title": "Zaloguj się ponownie, aby dalej synchronizować", + "problem_channel": "Problemy z synchronizacją", + "sign_out_anyway": "Wyloguj się", + "sign_out_pending_message": "{count, plural, one {# zmiana wprowadzona na tym urządzeniu nie dotarła do serwera. Zostanie utracona, jeśli wylogujesz się teraz.} few {# zmiany wprowadzone na tym urządzeniu nie dotarły do serwera. Zostaną utracone, jeśli wylogujesz się teraz.} many {# zmian wprowadzonych na tym urządzeniu nie dotarło do serwera. Zostaną utracone, jeśli wylogujesz się teraz.} other {# zmiany wprowadzonej na tym urządzeniu nie dotarło do serwera. Zostaną utracone, jeśli wylogujesz się teraz.}}", + "sign_out_pending_title": "Wylogować się mimo to?", + "sign_out_unknown_message": "Zmiany wprowadzone na tym urządzeniu mogły nie dotrzeć do serwera. Zostaną utracone, jeśli wylogujesz się teraz.", + "signed_out_text": "Konto {account} zostało wylogowane, więc jego kontakty i kalendarze na tym urządzeniu nie są już synchronizowane. Zaloguj się ponownie, aby wznowić synchronizację. Zmiany wprowadzone na tym urządzeniu zostaną zachowane.", + "signed_out_title": "Zaloguj się, aby dalej synchronizować", + "signing_out": "Usuwanie zsynchronizowanych kontaktów i kalendarzy z tego urządzenia…" + }, "email_composer": { "attach_camera": "Aparat", "attachment_reminder_body": "Twoja wiadomość wspomina o „{keyword}”, ale nie załączono żadnego pliku. Wysłać mimo to?", @@ -598,11 +612,110 @@ "book_count": "{count, plural, one {# kontakt} few {# kontakty} many {# kontaktów} other {# kontaktu}}", "default_book": "Domyślna książka adresowa", "description_mobile": "Preferencje wyświetlania i narzędzia książek adresowych.", + "device_sync_desc": "Pokazuj swoje książki adresowe w aplikacji Kontakty i innych aplikacjach na tym urządzeniu. Zmiany są synchronizowane w obie strony.", + "device_sync_title": "Synchronizuj z tym urządzeniem", "export_description_count": "{count, plural, one {Eksportuj # kontakt jako jeden plik vCard.} few {Eksportuj # kontakty jako jeden plik vCard.} many {Eksportuj # kontaktów jako jeden plik vCard.} other {Eksportuj # kontaktu jako jeden plik vCard.}}", "export_empty": "Brak kontaktów do eksportu", "import_description_mobile": "Importuj kontakty z pliku vCard (.vcf) z wykrywaniem duplikatów.", "manage_description_mobile": "Twórz, zmieniaj nazwy i usuwaj książki adresowe, w których porządkujesz kontakty." }, + "device_sync": { + "also_synced_by": "Synchronizuje się także dla {accounts}, więc pojawia się na tym urządzeniu dwa razy.", + "android_settings": "Ustawienia konta w Androidzie", + "blocked_calendar": "Android nie prosi już o dostęp do kalendarzy. Zezwól na niego w ustawieniach aplikacji, a potem ponownie włącz synchronizację.", + "blocked_contacts": "Android nie prosi już o dostęp do kontaktów. Zezwól na niego w ustawieniach aplikacji, a potem ponownie włącz synchronizację.", + "blocked_title": "Dostęp jest wyłączony", + "books_default": "Twoje własne książki adresowe są synchronizowane, a udostępnione tylko wtedy, gdy je wybierzesz.", + "books_hint": "Twoje własne książki adresowe są synchronizowane, chyba że je wyłączysz, a udostępnione tylko wtedy, gdy je włączysz.", + "books_label": "Książki adresowe", + "calendars_default": "Twoje własne kalendarze są synchronizowane, a udostępnione tylko wtedy, gdy je wybierzesz.", + "calendars_hint": "Twoje własne kalendarze są synchronizowane, chyba że je wyłączysz, a udostępnione tylko wtedy, gdy je włączysz. Listy zadań pozostają w Bulwark.", + "calendars_label": "Kalendarze", + "choose": "Wybierz", + "choose_books_title": "Książki adresowe do synchronizacji", + "choose_calendars_title": "Kalendarze do synchronizacji", + "collections_count": "Synchronizowane: {selected} z {total}", + "continue": "Dalej", + "deletions_delete": "Usuń na serwerze", + "deletions_message": "Na tym urządzeniu usunięto naraz wiele elementów, więc nie zostały usunięte na serwerze. Usuń je także tam albo przywróć je na tym urządzeniu.", + "deletions_restore": "Przywróć je", + "deletions_title": "Usunąć je także na serwerze?", + "denied_calendar": "Bez dostępu do kalendarzy Bulwark nie może umieścić Twoich kalendarzy na tym urządzeniu.", + "denied_contacts": "Bez dostępu do kontaktów Bulwark nie może umieścić Twoich książek adresowych na tym urządzeniu.", + "denied_title": "Synchronizacja pozostaje wyłączona", + "failed_title": "Nie udało się włączyć synchronizacji", + "grant_access": "Przyznaj dostęp", + "interval_15m": "Co 15 minut", + "interval_1h": "Co godzinę", + "interval_30m": "Co 30 minut", + "interval_6h": "Co 6 godzin", + "interval_desc": "Jak często Android synchronizuje samodzielnie. Zmiany w Bulwark i zmiany zgłoszone przez push są synchronizowane szybciej.", + "interval_label": "Częstotliwość synchronizacji", + "interval_manual": "Bez regularnej synchronizacji", + "keep_syncing": "Synchronizuj dalej", + "list_error_auth": "Serwer nie zaakceptował logowania.", + "list_error_network": "Nie udało się połączyć z serwerem.", + "list_error_server": "Serwer nie mógł udostępnić listy.", + "list_error_signed_out": "To konto nie jest zalogowane na tym urządzeniu.", + "master_off": "Autosynchronizacja jest wyłączona na tym urządzeniu, więc synchronizacja odbywa się tylko po stuknięciu „Synchronizuj teraz”.", + "new_contacts_default": "Domyślna książka adresowa", + "new_contacts_desc": "Książka adresowa dla kontaktów utworzonych w innych aplikacjach na tym urządzeniu.", + "new_contacts_label": "Zapisuj nowe kontakty w", + "no_books": "To konto nie ma książek adresowych.", + "no_calendars": "To konto nie ma kalendarzy do synchronizacji.", + "ok": "OK", + "open_app_settings": "Otwórz ustawienia", + "own_collections": "Własne", + "rationale_calendar": "Aby pokazywać Twoje kalendarze w aplikacji Kalendarz, Bulwark potrzebuje dostępu do kalendarzy na tym urządzeniu. Zmienia tylko kalendarze własnego konta.", + "rationale_contacts": "Aby pokazywać Twoje książki adresowe w aplikacji Kontakty, Bulwark potrzebuje dostępu do kontaktów na tym urządzeniu. Zmienia tylko kontakty własnego konta.", + "rationale_title_calendar": "Zezwól na dostęp do kalendarzy", + "rationale_title_contacts": "Zezwól na dostęp do kontaktów", + "read_only": "Tylko do odczytu", + "reminder_owner_bulwark": "Bulwark", + "reminder_owner_device": "Aplikacja Kalendarz", + "reminder_question_message": "Przypomnienia o zsynchronizowanych wydarzeniach mogą pochodzić z aplikacji kalendarza na tym urządzeniu albo z Bulwark. Wybierz jedno źródło, aby nie dostawać przypomnień podwójnie. Możesz to później zmienić.", + "reminder_question_title": "Kto ma Ci przypominać?", + "reminders_desc": "Kto przypomina Ci o zsynchronizowanych wydarzeniach. Drugie źródło milczy, więc nie dostajesz przypomnień podwójnie.", + "reminders_label": "Przypomnienia", + "reminders_nobody": "Powiadomienia o wydarzeniach w Bulwark są wyłączone, więc nic nie przypomina Ci o zsynchronizowanych wydarzeniach.", + "remove": "Usuń", + "remove_from_device": "Usuń z tego urządzenia", + "remove_message": "Kontakty i kalendarze tego konta zostaną usunięte z tego urządzenia. Zmiany wprowadzone tutaj, które nie dotarły do serwera, zostaną utracone.", + "remove_title": "Usunąć z tego urządzenia?", + "review_deletions": "Przejrzyj usunięcia", + "shared_by": "Udostępnione przez: {name}", + "sign_in_again": "Zaloguj się ponownie", + "status_auth": "Serwer nie akceptuje już logowania. Zaloguj się ponownie, aby dalej synchronizować.", + "status_cancelled": "Ostatnia synchronizacja zatrzymała się przedwcześnie. Będzie kontynuowana następnym razem.", + "status_failed": "Ostatnia synchronizacja nie powiodła się: {message}", + "status_failed_plain": "Ostatnia synchronizacja nie powiodła się.", + "status_io": "Nie udało się połączyć z serwerem {when}. Android spróbuje ponownie.", + "status_item_errors": "{count, plural, one {Nie udało się zsynchronizować # elementu.} few {Nie udało się zsynchronizować # elementów.} many {Nie udało się zsynchronizować # elementów.} other {Nie udało się zsynchronizować # elementu.}}", + "status_off": "Wyłączona", + "status_paused": "Wstrzymana w ustawieniach Androida", + "status_permission_calendar": "Dostęp do kalendarzy został wyłączony, więc nic nie jest synchronizowane.", + "status_permission_contacts": "Dostęp do kontaktów został wyłączony, więc nic nie jest synchronizowane.", + "status_removed": "Usunięte w ustawieniach Androida. Włącz synchronizację, aby ponownie dodać konto.", + "status_safety_abort": "Zatrzymano: serwer nic nie zwrócił, choć na tym urządzeniu wciąż są elementy. Sprawdź serwer, a potem zsynchronizuj ponownie.", + "status_synced": "Zsynchronizowano {when}", + "status_syncing": "Synchronizowanie…", + "status_too_many_deletions": "Na tym urządzeniu usunięto wiele elementów. Nie zostaną usunięte na serwerze, dopóki tego nie potwierdzisz.", + "status_unsupported_calendar": "Serwer nie udostępnia kalendarzy dla tego konta.", + "status_unsupported_contacts": "Serwer nie udostępnia książek adresowych dla tego konta.", + "status_waiting": "Oczekiwanie na pierwszą synchronizację", + "status_waiting_next": "Oczekiwanie na następną synchronizację", + "suspended_desc": "Wylogowano. Kontakty i kalendarze tego konta pozostają na tym urządzeniu razem ze zmianami wprowadzonymi tutaj i zostaną ponownie zsynchronizowane po zalogowaniu.", + "switch_label": "Synchronizuj {account} z tym urządzeniem", + "sync_now": "Synchronizuj teraz", + "turn_off_anyway": "Wyłącz", + "turn_off_failed": "Nie udało się usunąć zsynchronizowanych danych z tego urządzenia. Synchronizacja jest wstrzymana; spróbuj ponownie później.", + "turn_off_failed_title": "Nie udało się wyłączyć synchronizacji", + "turn_off_pending": "{count, plural, one {# zmiana wprowadzona na tym urządzeniu nie dotarła jeszcze do serwera. Zostanie utracona, jeśli teraz wyłączysz synchronizację.} few {# zmiany wprowadzone na tym urządzeniu nie dotarły jeszcze do serwera. Zostaną utracone, jeśli teraz wyłączysz synchronizację.} many {# zmian wprowadzonych na tym urządzeniu nie dotarło jeszcze do serwera. Zostaną utracone, jeśli teraz wyłączysz synchronizację.} other {# zmiany wprowadzonej na tym urządzeniu nie dotarło jeszcze do serwera. Zostaną utracone, jeśli teraz wyłączysz synchronizację.}}", + "turn_off_title": "Wyłączyć mimo to?", + "turn_off_unknown": "Bulwark nie mógł sprawdzić, czy zmiany wprowadzone na tym urządzeniu dotarły do serwera. Zostaną utracone, jeśli teraz wyłączysz synchronizację.", + "turning_off": "Wysyłanie zmian, a potem usuwanie danych z tego urządzenia…", + "turning_on": "Konfigurowanie…" + }, "downloads": { "attachment_template": { "description_mobile": "Szablon używany przy zapisywaniu lub udostępnianiu załącznika." diff --git a/locales/rn/pt.json b/locales/rn/pt.json index bfd322e6..13a601b1 100644 --- a/locales/rn/pt.json +++ b/locales/rn/pt.json @@ -93,6 +93,8 @@ "until_date": "Até {date}" }, "settings": { + "device_sync_desc": "Mostre seus calendários no app Agenda e em outros apps deste dispositivo. As alterações são sincronizadas nos dois sentidos.", + "device_sync_title": "Sincronizar com este dispositivo", "show_week_numbers_mobile_desc": "Exibir números da semana na visualização mensal.", "time_zone": "Fuso horário", "time_zone_auto": "Fuso horário do dispositivo", @@ -249,6 +251,18 @@ "color_tag": "Etiqueta", "remove_color": "Remover etiqueta" }, + "device_sync": { + "auth_problem_text": "O servidor não aceita mais o login de {account}, então os contatos e calendários dessa conta neste dispositivo pararam de ser sincronizados. Faça login novamente para retomar. As alterações feitas neste dispositivo são mantidas.", + "auth_problem_title": "Faça login novamente para continuar sincronizando", + "problem_channel": "Problemas de sincronização", + "sign_out_anyway": "Sair", + "sign_out_pending_message": "{count, plural, one {# alteração feita neste dispositivo não chegou ao servidor. Ela será perdida se você sair agora.} other {# alterações feitas neste dispositivo não chegaram ao servidor. Elas serão perdidas se você sair agora.}}", + "sign_out_pending_title": "Sair mesmo assim?", + "sign_out_unknown_message": "As alterações feitas neste dispositivo podem não ter chegado ao servidor. Elas serão perdidas se você sair agora.", + "signed_out_text": "A sessão de {account} foi encerrada, então os contatos e calendários dessa conta neste dispositivo não são mais sincronizados. Faça login novamente para retomar. As alterações feitas neste dispositivo são mantidas.", + "signed_out_title": "Faça login para continuar sincronizando", + "signing_out": "Removendo contatos e calendários sincronizados deste dispositivo…" + }, "email_composer": { "attach_camera": "Câmera", "attachment_reminder_body": "Sua mensagem menciona \"{keyword}\", mas nenhum arquivo está anexado. Enviar mesmo assim?", @@ -598,11 +612,110 @@ "book_count": "{count, plural, one {# contato} other {# contatos}}", "default_book": "Catálogo de endereços padrão", "description_mobile": "Preferências de exibição e ferramentas do catálogo de endereços.", + "device_sync_desc": "Mostre seus catálogos de endereços no app Contatos e em outros apps deste dispositivo. As alterações são sincronizadas nos dois sentidos.", + "device_sync_title": "Sincronizar com este dispositivo", "export_description_count": "{count, plural, one {Exporta # contato como um único arquivo vCard.} other {Exporta # contatos como um único arquivo vCard.}}", "export_empty": "Nenhum contato para exportar", "import_description_mobile": "Importa contatos de um arquivo vCard (.vcf) com detecção de duplicados.", "manage_description_mobile": "Crie, renomeie e remova os catálogos de endereços que organizam seus contatos." }, + "device_sync": { + "also_synced_by": "Também é sincronizado para {accounts}, por isso aparece duas vezes neste dispositivo.", + "android_settings": "Configurações de conta do Android", + "blocked_calendar": "O Android não pede mais acesso aos calendários. Permita o acesso nas configurações do app e ative a sincronização de novo.", + "blocked_contacts": "O Android não pede mais acesso aos contatos. Permita o acesso nas configurações do app e ative a sincronização de novo.", + "blocked_title": "O acesso está desativado", + "books_default": "Seus catálogos de endereços são sincronizados; os compartilhados, só quando você os escolher.", + "books_hint": "Seus catálogos de endereços são sincronizados, a menos que você os desative; os compartilhados, só quando você os ativar.", + "books_label": "Catálogos de endereços", + "calendars_default": "Seus calendários são sincronizados; os compartilhados, só quando você os escolher.", + "calendars_hint": "Seus calendários são sincronizados, a menos que você os desative; os compartilhados, só quando você os ativar. As listas de tarefas ficam no Bulwark.", + "calendars_label": "Calendários", + "choose": "Escolher", + "choose_books_title": "Catálogos de endereços para sincronizar", + "choose_calendars_title": "Calendários para sincronizar", + "collections_count": "{selected} de {total} sincronizados", + "continue": "Continuar", + "deletions_delete": "Excluir no servidor", + "deletions_message": "Muitos itens foram excluídos de uma vez neste dispositivo, por isso não foram excluídos no servidor. Exclua-os lá também ou restaure-os neste dispositivo.", + "deletions_restore": "Restaurar", + "deletions_title": "Excluir também no servidor?", + "denied_calendar": "Sem acesso aos calendários, o Bulwark não consegue colocar seus calendários neste dispositivo.", + "denied_contacts": "Sem acesso aos contatos, o Bulwark não consegue colocar seus catálogos de endereços neste dispositivo.", + "denied_title": "A sincronização continua desativada", + "failed_title": "Não foi possível ativar a sincronização", + "grant_access": "Conceder acesso", + "interval_15m": "A cada 15 minutos", + "interval_1h": "A cada hora", + "interval_30m": "A cada 30 minutos", + "interval_6h": "A cada 6 horas", + "interval_desc": "Com que frequência o Android sincroniza por conta própria. Edições no Bulwark e alterações avisadas por push são sincronizadas antes.", + "interval_label": "Intervalo de sincronização", + "interval_manual": "Sem sincronização periódica", + "keep_syncing": "Continuar sincronizando", + "list_error_auth": "O servidor não aceitou o login.", + "list_error_network": "Não foi possível acessar o servidor.", + "list_error_server": "O servidor não conseguiu listá-los.", + "list_error_signed_out": "Esta conta não está conectada neste dispositivo.", + "master_off": "A sincronização automática está desativada neste dispositivo, então só há sincronização quando você toca em \"Sincronizar agora\".", + "new_contacts_default": "Catálogo de endereços padrão", + "new_contacts_desc": "O catálogo de endereços para contatos criados em outros apps deste dispositivo.", + "new_contacts_label": "Salvar novos contatos em", + "no_books": "Esta conta não tem catálogos de endereços.", + "no_calendars": "Esta conta não tem calendários para sincronizar.", + "ok": "OK", + "open_app_settings": "Abrir configurações do app", + "own_collections": "Seus", + "rationale_calendar": "Para mostrar seus calendários no app Agenda, o Bulwark precisa de acesso aos calendários deste dispositivo. Ele só altera os calendários da própria conta.", + "rationale_contacts": "Para mostrar seus catálogos de endereços no app Contatos, o Bulwark precisa de acesso aos contatos deste dispositivo. Ele só altera os contatos da própria conta.", + "rationale_title_calendar": "Permitir acesso aos calendários", + "rationale_title_contacts": "Permitir acesso aos contatos", + "read_only": "Somente leitura", + "reminder_owner_bulwark": "Bulwark", + "reminder_owner_device": "App Agenda", + "reminder_question_message": "Os lembretes de eventos sincronizados podem vir do app de agenda deste dispositivo ou do Bulwark. Escolha um para não receber lembretes em dobro. Você pode mudar isso depois.", + "reminder_question_title": "Quem deve lembrar você?", + "reminders_desc": "Quem lembra você dos eventos sincronizados. O outro fica em silêncio, para que você não receba lembretes em dobro.", + "reminders_label": "Lembretes", + "reminders_nobody": "As notificações de eventos do Bulwark estão desativadas, então nada vai lembrar você dos eventos sincronizados.", + "remove": "Remover", + "remove_from_device": "Remover deste dispositivo", + "remove_message": "Os contatos e calendários da conta são removidos deste dispositivo. As alterações feitas aqui que não chegaram ao servidor são perdidas.", + "remove_title": "Remover deste dispositivo?", + "review_deletions": "Revisar exclusões", + "shared_by": "Compartilhado por {name}", + "sign_in_again": "Fazer login novamente", + "status_auth": "O servidor não aceita mais o login. Faça login novamente para continuar sincronizando.", + "status_cancelled": "A última sincronização parou antes do fim. Ela continua na próxima vez.", + "status_failed": "Falha na última sincronização: {message}", + "status_failed_plain": "Falha na última sincronização.", + "status_io": "Não foi possível acessar o servidor {when}. O Android vai tentar de novo.", + "status_item_errors": "{count, plural, one {Não foi possível sincronizar # item.} other {Não foi possível sincronizar # itens.}}", + "status_off": "Desativada", + "status_paused": "Pausada nas configurações do Android", + "status_permission_calendar": "O acesso aos calendários foi desativado, então nada é sincronizado.", + "status_permission_contacts": "O acesso aos contatos foi desativado, então nada é sincronizado.", + "status_removed": "Conta removida nas configurações do Android. Ative a sincronização para adicioná-la de novo.", + "status_safety_abort": "Interrompida: o servidor não retornou nada, mas este dispositivo ainda tem itens. Verifique o servidor e sincronize de novo.", + "status_synced": "Sincronizado {when}", + "status_syncing": "Sincronizando…", + "status_too_many_deletions": "Muitos itens foram excluídos neste dispositivo. Eles só serão excluídos no servidor quando você confirmar.", + "status_unsupported_calendar": "O servidor não oferece calendários para esta conta.", + "status_unsupported_contacts": "O servidor não oferece catálogos de endereços para esta conta.", + "status_waiting": "Aguardando a primeira sincronização", + "status_waiting_next": "Aguardando a próxima sincronização", + "suspended_desc": "Sessão encerrada. Os contatos e calendários da conta continuam neste dispositivo, com as alterações feitas aqui, e voltam a sincronizar quando você fizer login.", + "switch_label": "Sincronizar {account} com este dispositivo", + "sync_now": "Sincronizar agora", + "turn_off_anyway": "Desativar", + "turn_off_failed": "Não foi possível remover os dados sincronizados deste dispositivo. A sincronização está pausada; tente novamente mais tarde.", + "turn_off_failed_title": "Não foi possível desativar a sincronização", + "turn_off_pending": "{count, plural, one {# alteração feita neste dispositivo ainda não chegou ao servidor. Ela será perdida se você desativar a sincronização agora.} other {# alterações feitas neste dispositivo ainda não chegaram ao servidor. Elas serão perdidas se você desativar a sincronização agora.}}", + "turn_off_title": "Desativar mesmo assim?", + "turn_off_unknown": "O Bulwark não conseguiu verificar se as alterações feitas neste dispositivo chegaram ao servidor. Elas serão perdidas se você desativar a sincronização agora.", + "turning_off": "Enviando alterações e depois removendo os dados deste dispositivo…", + "turning_on": "Configurando…" + }, "downloads": { "attachment_template": { "description_mobile": "Modelo usado ao salvar ou compartilhar um anexo." diff --git a/locales/rn/ro.json b/locales/rn/ro.json index 0123cafc..0403fef9 100644 --- a/locales/rn/ro.json +++ b/locales/rn/ro.json @@ -93,6 +93,8 @@ "until_date": "Până la {date}" }, "settings": { + "device_sync_desc": "Afișați calendarele în aplicația Calendar și în alte aplicații de pe acest dispozitiv. Modificările se sincronizează în ambele sensuri.", + "device_sync_title": "Sincronizare cu acest dispozitiv", "show_week_numbers_mobile_desc": "Afișați numerele săptămânilor în vizualizarea lunară.", "time_zone": "Fus orar", "time_zone_auto": "Fusul orar al dispozitivului", @@ -249,6 +251,18 @@ "color_tag": "Etichetă", "remove_color": "Eliminați eticheta" }, + "device_sync": { + "auth_problem_text": "Serverul nu mai acceptă autentificarea pentru {account}, așa că contactele și calendarele acestui cont de pe acest dispozitiv nu se mai sincronizează. Autentificați-vă din nou pentru a relua sincronizarea. Modificările făcute pe acest dispozitiv se păstrează.", + "auth_problem_title": "Autentificați-vă din nou pentru a continua sincronizarea", + "problem_channel": "Probleme de sincronizare", + "sign_out_anyway": "Deconectare", + "sign_out_pending_message": "{count, plural, one {# modificare făcută pe acest dispozitiv nu a ajuns pe server. Se va pierde dacă vă deconectați acum.} few {# modificări făcute pe acest dispozitiv nu au ajuns pe server. Se vor pierde dacă vă deconectați acum.} other {# de modificări făcute pe acest dispozitiv nu au ajuns pe server. Se vor pierde dacă vă deconectați acum.}}", + "sign_out_pending_title": "Vă deconectați totuși?", + "sign_out_unknown_message": "Este posibil ca modificările făcute pe acest dispozitiv să nu fi ajuns pe server. Se vor pierde dacă vă deconectați acum.", + "signed_out_text": "{account} a fost deconectat, așa că contactele și calendarele acestui cont de pe acest dispozitiv nu se mai sincronizează. Autentificați-vă din nou pentru a relua sincronizarea. Modificările făcute pe acest dispozitiv se păstrează.", + "signed_out_title": "Autentificați-vă pentru a continua sincronizarea", + "signing_out": "Se elimină de pe acest dispozitiv contactele și calendarele sincronizate…" + }, "email_composer": { "attach_camera": "Cameră", "attachment_reminder_body": "Mesajul dvs. menționează „{keyword}”, dar nu este atașat niciun fișier. Doriți să trimiteți oricum?", @@ -598,11 +612,110 @@ "book_count": "{count, plural, one {# contact} few {# contacte} other {# de contacte}}", "default_book": "Agendă implicită", "description_mobile": "Preferințe de afișare și instrumente pentru agende.", + "device_sync_desc": "Afișați agendele de adrese în aplicația Contacte și în alte aplicații de pe acest dispozitiv. Modificările se sincronizează în ambele sensuri.", + "device_sync_title": "Sincronizare cu acest dispozitiv", "export_description_count": "{count, plural, one {Exportă # contact ca un singur fișier vCard.} few {Exportă # contacte ca un singur fișier vCard.} other {Exportă # de contacte ca un singur fișier vCard.}}", "export_empty": "Nu există contacte de exportat", "import_description_mobile": "Importă contacte dintr-un fișier vCard (.vcf), cu detectarea duplicatelor.", "manage_description_mobile": "Creați, redenumiți și eliminați agendele care vă organizează contactele." }, + "device_sync": { + "also_synced_by": "Se sincronizează și pentru {accounts}, așa că apare de două ori pe acest dispozitiv.", + "android_settings": "Setările contului Android", + "blocked_calendar": "Android nu mai solicită accesul la calendare. Permiteți-l din setările aplicației, apoi activați din nou sincronizarea.", + "blocked_contacts": "Android nu mai solicită accesul la contacte. Permiteți-l din setările aplicației, apoi activați din nou sincronizarea.", + "blocked_title": "Accesul este dezactivat", + "books_default": "Agendele dvs. de adrese se sincronizează; cele partajate doar dacă le alegeți.", + "books_hint": "Agendele dvs. de adrese se sincronizează dacă nu le dezactivați; cele partajate doar dacă le activați.", + "books_label": "Agende de adrese", + "calendars_default": "Calendarele dvs. se sincronizează; cele partajate doar dacă le alegeți.", + "calendars_hint": "Calendarele dvs. se sincronizează dacă nu le dezactivați; cele partajate doar dacă le activați. Listele de sarcini rămân în Bulwark.", + "calendars_label": "Calendare", + "choose": "Alegeți", + "choose_books_title": "Agende de adrese de sincronizat", + "choose_calendars_title": "Calendare de sincronizat", + "collections_count": "Sincronizate: {selected} din {total}", + "continue": "Continuați", + "deletions_delete": "Ștergeți de pe server", + "deletions_message": "Multe elemente au fost șterse deodată pe acest dispozitiv, așa că nu au fost șterse de pe server. Ștergeți-le și acolo sau readuceți-le pe acest dispozitiv.", + "deletions_restore": "Readuceți-le", + "deletions_title": "Le ștergeți și de pe server?", + "denied_calendar": "Fără acces la calendare, Bulwark nu poate aduce calendarele dvs. pe acest dispozitiv.", + "denied_contacts": "Fără acces la contacte, Bulwark nu poate aduce agendele dvs. de adrese pe acest dispozitiv.", + "denied_title": "Sincronizarea rămâne dezactivată", + "failed_title": "Sincronizarea nu a putut fi activată", + "grant_access": "Acordați acces", + "interval_15m": "La fiecare 15 minute", + "interval_1h": "La fiecare oră", + "interval_30m": "La fiecare 30 de minute", + "interval_6h": "La fiecare 6 ore", + "interval_desc": "Cât de des sincronizează Android de la sine. Modificările din Bulwark și cele anunțate prin push se sincronizează mai repede.", + "interval_label": "Interval de sincronizare", + "interval_manual": "Fără sincronizare periodică", + "keep_syncing": "Continuați sincronizarea", + "list_error_auth": "Serverul nu a acceptat autentificarea.", + "list_error_network": "Serverul nu a putut fi contactat.", + "list_error_server": "Serverul nu a putut furniza lista.", + "list_error_signed_out": "Acest cont nu este autentificat pe acest dispozitiv.", + "master_off": "Sincronizarea automată este dezactivată pe acest dispozitiv, așa că sincronizarea are loc doar când atingeți „Sincronizați acum”.", + "new_contacts_default": "Agendă implicită", + "new_contacts_desc": "Agenda de adrese pentru contactele create în alte aplicații de pe acest dispozitiv.", + "new_contacts_label": "Contactele noi se salvează în", + "no_books": "Acest cont nu are agende de adrese.", + "no_calendars": "Acest cont nu are calendare de sincronizat.", + "ok": "OK", + "open_app_settings": "Deschideți setările", + "own_collections": "Ale dvs.", + "rationale_calendar": "Pentru a afișa calendarele dvs. în aplicația Calendar, Bulwark are nevoie de acces la calendarele de pe acest dispozitiv. Modifică doar calendarele propriului cont.", + "rationale_contacts": "Pentru a afișa agendele dvs. de adrese în aplicația Contacte, Bulwark are nevoie de acces la contactele de pe acest dispozitiv. Modifică doar contactele propriului cont.", + "rationale_title_calendar": "Permiteți accesul la calendare", + "rationale_title_contacts": "Permiteți accesul la contacte", + "read_only": "Doar citire", + "reminder_owner_bulwark": "Bulwark", + "reminder_owner_device": "Aplicația Calendar", + "reminder_question_message": "Mementourile pentru evenimentele sincronizate pot veni din aplicația de calendar de pe acest dispozitiv sau din Bulwark. Alegeți una dintre ele ca să nu primiți mementouri de două ori. Puteți schimba ulterior.", + "reminder_question_title": "Cine să vă trimită mementouri?", + "reminders_desc": "Cine vă amintește de evenimentele sincronizate. Cealaltă aplicație rămâne silențioasă, ca să nu primiți mementouri de două ori.", + "reminders_label": "Mementouri", + "reminders_nobody": "Notificările de evenimente din Bulwark sunt dezactivate, așa că nimic nu vă amintește de evenimentele sincronizate.", + "remove": "Eliminați", + "remove_from_device": "Eliminați de pe acest dispozitiv", + "remove_message": "Contactele și calendarele contului vor fi eliminate de pe acest dispozitiv. Modificările făcute aici care nu au ajuns pe server se vor pierde.", + "remove_title": "Eliminați de pe acest dispozitiv?", + "review_deletions": "Verificați ștergerile", + "shared_by": "Distribuit de {name}", + "sign_in_again": "Autentificați-vă din nou", + "status_auth": "Serverul nu mai acceptă autentificarea. Autentificați-vă din nou pentru a continua sincronizarea.", + "status_cancelled": "Ultima sincronizare s-a oprit prea devreme. Va continua data viitoare.", + "status_failed": "Ultima sincronizare a eșuat: {message}", + "status_failed_plain": "Ultima sincronizare a eșuat.", + "status_io": "Serverul nu a putut fi contactat {when}. Android va încerca din nou.", + "status_item_errors": "{count, plural, one {# element nu a putut fi sincronizat.} few {# elemente nu au putut fi sincronizate.} other {# de elemente nu au putut fi sincronizate.}}", + "status_off": "Dezactivată", + "status_paused": "Întreruptă în setările Android", + "status_permission_calendar": "Accesul la calendare a fost dezactivat, așa că nu se sincronizează nimic.", + "status_permission_contacts": "Accesul la contacte a fost dezactivat, așa că nu se sincronizează nimic.", + "status_removed": "Eliminat în setările Android. Activați sincronizarea pentru a adăuga din nou contul.", + "status_safety_abort": "Oprită: serverul nu a returnat nimic, deși acest dispozitiv încă are elemente. Verificați serverul, apoi sincronizați din nou.", + "status_synced": "Sincronizat {when}", + "status_syncing": "Se sincronizează…", + "status_too_many_deletions": "Multe elemente au fost șterse pe acest dispozitiv. Nu vor fi șterse de pe server până nu confirmați.", + "status_unsupported_calendar": "Serverul nu oferă calendare pentru acest cont.", + "status_unsupported_contacts": "Serverul nu oferă agende de adrese pentru acest cont.", + "status_waiting": "Se așteaptă prima sincronizare", + "status_waiting_next": "Se așteaptă următoarea sincronizare", + "suspended_desc": "Deconectat. Contactele și calendarele contului rămân pe acest dispozitiv, împreună cu modificările făcute aici, și se vor sincroniza din nou după ce vă autentificați.", + "switch_label": "Sincronizați {account} cu acest dispozitiv", + "sync_now": "Sincronizați acum", + "turn_off_anyway": "Dezactivați", + "turn_off_failed": "Datele sincronizate nu au putut fi eliminate de pe acest dispozitiv. Sincronizarea este întreruptă; încercați din nou mai târziu.", + "turn_off_failed_title": "Sincronizarea nu a putut fi dezactivată", + "turn_off_pending": "{count, plural, one {# modificare făcută pe acest dispozitiv nu a ajuns încă pe server. Se va pierde dacă dezactivați acum sincronizarea.} few {# modificări făcute pe acest dispozitiv nu au ajuns încă pe server. Se vor pierde dacă dezactivați acum sincronizarea.} other {# de modificări făcute pe acest dispozitiv nu au ajuns încă pe server. Se vor pierde dacă dezactivați acum sincronizarea.}}", + "turn_off_title": "Dezactivați totuși?", + "turn_off_unknown": "Bulwark nu a putut verifica dacă modificările făcute pe acest dispozitiv au ajuns pe server. Se vor pierde dacă dezactivați acum sincronizarea.", + "turning_off": "Se încarcă modificările, apoi se elimină datele de pe acest dispozitiv…", + "turning_on": "Se configurează…" + }, "downloads": { "attachment_template": { "description_mobile": "Șablon utilizat la salvarea sau partajarea unui atașament." diff --git a/locales/rn/ru.json b/locales/rn/ru.json index 62bd0b80..71c16dbc 100644 --- a/locales/rn/ru.json +++ b/locales/rn/ru.json @@ -93,6 +93,8 @@ "until_date": "До {date}" }, "settings": { + "device_sync_desc": "Ваши календари будут видны в приложении «Календарь» и других приложениях на этом устройстве. Изменения синхронизируются в обе стороны.", + "device_sync_title": "Синхронизация с этим устройством", "show_week_numbers_mobile_desc": "Показывать номера недель в представлении «Месяц».", "time_zone": "Часовой пояс", "time_zone_auto": "Часовой пояс устройства", @@ -249,6 +251,18 @@ "color_tag": "Тег", "remove_color": "Убрать тег" }, + "device_sync": { + "auth_problem_text": "Сервер больше не принимает вход в аккаунт {account}, поэтому его контакты и календари на этом устройстве больше не синхронизируются. Войдите снова, чтобы возобновить синхронизацию. Изменения, сделанные на этом устройстве, сохранятся.", + "auth_problem_title": "Войдите снова, чтобы продолжить синхронизацию", + "problem_channel": "Проблемы синхронизации", + "sign_out_anyway": "Выйти", + "sign_out_pending_message": "{count, plural, one {# изменение, сделанное на этом устройстве, не дошло до сервера. Оно будет потеряно, если вы сейчас выйдете.} few {# изменения, сделанные на этом устройстве, не дошли до сервера. Они будут потеряны, если вы сейчас выйдете.} many {# изменений, сделанных на этом устройстве, не дошли до сервера. Они будут потеряны, если вы сейчас выйдете.} other {# изменения, сделанного на этом устройстве, не дошли до сервера. Они будут потеряны, если вы сейчас выйдете.}}", + "sign_out_pending_title": "Всё равно выйти?", + "sign_out_unknown_message": "Изменения, сделанные на этом устройстве, могли не дойти до сервера. Они будут потеряны, если вы сейчас выйдете.", + "signed_out_text": "Выполнен выход из аккаунта {account}, поэтому его контакты и календари на этом устройстве больше не синхронизируются. Войдите снова, чтобы возобновить синхронизацию. Изменения, сделанные на этом устройстве, сохранятся.", + "signed_out_title": "Войдите, чтобы продолжить синхронизацию", + "signing_out": "Удаление синхронизированных контактов и календарей с этого устройства…" + }, "email_composer": { "attach_camera": "Камера", "attachment_reminder_body": "В письме упоминается «{keyword}», но файл не прикреплён. Всё равно отправить?", @@ -598,11 +612,110 @@ "book_count": "{count, plural, one {# контакт} few {# контакта} many {# контактов} other {# контакта}}", "default_book": "Адресная книга по умолчанию", "description_mobile": "Параметры отображения и инструменты адресных книг.", + "device_sync_desc": "Ваши адресные книги будут видны в приложении «Контакты» и других приложениях на этом устройстве. Изменения синхронизируются в обе стороны.", + "device_sync_title": "Синхронизация с этим устройством", "export_description_count": "{count, plural, one {Экспортировать # контакт в один файл vCard.} few {Экспортировать # контакта в один файл vCard.} many {Экспортировать # контактов в один файл vCard.} other {Экспортировать # контакта в один файл vCard.}}", "export_empty": "Нет контактов для экспорта", "import_description_mobile": "Импорт контактов из файла vCard (.vcf) с поиском дубликатов.", "manage_description_mobile": "Создавайте, переименовывайте и удаляйте адресные книги, в которых хранятся ваши контакты." }, + "device_sync": { + "also_synced_by": "Также синхронизируется для {accounts}, поэтому отображается на этом устройстве дважды.", + "android_settings": "Настройки аккаунта Android", + "blocked_calendar": "Android больше не запрашивает доступ к календарям. Разрешите его в настройках приложения, а затем снова включите синхронизацию.", + "blocked_contacts": "Android больше не запрашивает доступ к контактам. Разрешите его в настройках приложения, а затем снова включите синхронизацию.", + "blocked_title": "Доступ отключён", + "books_default": "Ваши собственные адресные книги синхронизируются, а общие — только если вы их выберете.", + "books_hint": "Ваши собственные адресные книги синхронизируются, если вы их не отключите, а общие — только если вы их включите.", + "books_label": "Адресные книги", + "calendars_default": "Ваши собственные календари синхронизируются, а общие — только если вы их выберете.", + "calendars_hint": "Ваши собственные календари синхронизируются, если вы их не отключите, а общие — только если вы их включите. Списки задач остаются в Bulwark.", + "calendars_label": "Календари", + "choose": "Выбрать", + "choose_books_title": "Адресные книги для синхронизации", + "choose_calendars_title": "Календари для синхронизации", + "collections_count": "Синхронизируется: {selected} из {total}", + "continue": "Продолжить", + "deletions_delete": "Удалить на сервере", + "deletions_message": "На этом устройстве сразу было удалено много элементов, поэтому на сервере они не удалены. Удалите их и там или верните их на это устройство.", + "deletions_restore": "Вернуть", + "deletions_title": "Удалить их и на сервере?", + "denied_calendar": "Без доступа к календарям Bulwark не сможет перенести ваши календари на это устройство.", + "denied_contacts": "Без доступа к контактам Bulwark не сможет перенести ваши адресные книги на это устройство.", + "denied_title": "Синхронизация остаётся выключенной", + "failed_title": "Не удалось включить синхронизацию", + "grant_access": "Предоставить доступ", + "interval_15m": "Каждые 15 минут", + "interval_1h": "Каждый час", + "interval_30m": "Каждые 30 минут", + "interval_6h": "Каждые 6 часов", + "interval_desc": "Как часто Android синхронизирует данные самостоятельно. Изменения в Bulwark и изменения, о которых сообщает push, синхронизируются раньше.", + "interval_label": "Интервал синхронизации", + "interval_manual": "Без регулярной синхронизации", + "keep_syncing": "Продолжить синхронизацию", + "list_error_auth": "Сервер не принял вход.", + "list_error_network": "Не удалось связаться с сервером.", + "list_error_server": "Сервер не смог вернуть список.", + "list_error_signed_out": "На этом устройстве не выполнен вход в этот аккаунт.", + "master_off": "Автосинхронизация на этом устройстве выключена, поэтому синхронизация выполняется только при нажатии «Синхронизировать».", + "new_contacts_default": "Адресная книга по умолчанию", + "new_contacts_desc": "Адресная книга для контактов, созданных в других приложениях на этом устройстве.", + "new_contacts_label": "Сохранять новые контакты в", + "no_books": "В этом аккаунте нет адресных книг.", + "no_calendars": "В этом аккаунте нет календарей для синхронизации.", + "ok": "ОК", + "open_app_settings": "Открыть настройки", + "own_collections": "Ваши", + "rationale_calendar": "Чтобы показывать ваши календари в приложении «Календарь», Bulwark нужен доступ к календарям на этом устройстве. Он изменяет только календари своего аккаунта.", + "rationale_contacts": "Чтобы показывать ваши адресные книги в приложении «Контакты», Bulwark нужен доступ к контактам на этом устройстве. Он изменяет только контакты своего аккаунта.", + "rationale_title_calendar": "Разрешить доступ к календарям", + "rationale_title_contacts": "Разрешить доступ к контактам", + "read_only": "Только для чтения", + "reminder_owner_bulwark": "Bulwark", + "reminder_owner_device": "Приложение «Календарь»", + "reminder_question_message": "Напоминания о синхронизированных событиях может показывать приложение календаря на этом устройстве или Bulwark. Выберите что-то одно, чтобы не получать напоминания дважды. Это можно изменить позже.", + "reminder_question_title": "Кто будет напоминать о событиях?", + "reminders_desc": "Кто напоминает о синхронизированных событиях. Второе приложение молчит, чтобы напоминания не приходили дважды.", + "reminders_label": "Напоминания", + "reminders_nobody": "Уведомления о событиях в Bulwark выключены, поэтому о синхронизированных событиях ничто не напомнит.", + "remove": "Удалить", + "remove_from_device": "Удалить с этого устройства", + "remove_message": "Контакты и календари аккаунта будут удалены с этого устройства. Сделанные здесь изменения, которые не дошли до сервера, будут потеряны.", + "remove_title": "Удалить с этого устройства?", + "review_deletions": "Проверить удаления", + "shared_by": "Владелец: {name}", + "sign_in_again": "Войти снова", + "status_auth": "Сервер больше не принимает вход. Войдите снова, чтобы продолжить синхронизацию.", + "status_cancelled": "Последняя синхронизация прервалась. Она продолжится в следующий раз.", + "status_failed": "Ошибка последней синхронизации: {message}", + "status_failed_plain": "Последняя синхронизация не удалась.", + "status_io": "Не удалось связаться с сервером {when}. Android повторит попытку.", + "status_item_errors": "{count, plural, one {Не удалось синхронизировать # элемент.} few {Не удалось синхронизировать # элемента.} many {Не удалось синхронизировать # элементов.} other {Не удалось синхронизировать # элемента.}}", + "status_off": "Выкл.", + "status_paused": "Приостановлено в настройках Android", + "status_permission_calendar": "Доступ к календарям отключён, поэтому ничего не синхронизируется.", + "status_permission_contacts": "Доступ к контактам отключён, поэтому ничего не синхронизируется.", + "status_removed": "Удалено в настройках Android. Включите синхронизацию, чтобы снова добавить аккаунт.", + "status_safety_abort": "Остановлено: сервер ничего не вернул, хотя на этом устройстве ещё есть элементы. Проверьте сервер и повторите синхронизацию.", + "status_synced": "Синхронизировано {when}", + "status_syncing": "Синхронизация…", + "status_too_many_deletions": "На этом устройстве удалено много элементов. На сервере они будут удалены только после вашего подтверждения.", + "status_unsupported_calendar": "Сервер не предоставляет календари для этого аккаунта.", + "status_unsupported_contacts": "Сервер не предоставляет адресные книги для этого аккаунта.", + "status_waiting": "Ожидание первой синхронизации", + "status_waiting_next": "Ожидание следующей синхронизации", + "suspended_desc": "Выполнен выход. Контакты и календари аккаунта остаются на этом устройстве вместе со сделанными здесь изменениями и снова синхронизируются после входа.", + "switch_label": "Синхронизировать {account} с этим устройством", + "sync_now": "Синхронизировать", + "turn_off_anyway": "Выключить", + "turn_off_failed": "Не удалось удалить синхронизированные данные с этого устройства. Синхронизация приостановлена; повторите попытку позже.", + "turn_off_failed_title": "Не удалось выключить синхронизацию", + "turn_off_pending": "{count, plural, one {# изменение, сделанное на этом устройстве, ещё не дошло до сервера. Оно будет потеряно, если сейчас выключить синхронизацию.} few {# изменения, сделанные на этом устройстве, ещё не дошли до сервера. Они будут потеряны, если сейчас выключить синхронизацию.} many {# изменений, сделанных на этом устройстве, ещё не дошли до сервера. Они будут потеряны, если сейчас выключить синхронизацию.} other {# изменения, сделанного на этом устройстве, ещё не дошли до сервера. Они будут потеряны, если сейчас выключить синхронизацию.}}", + "turn_off_title": "Всё равно выключить?", + "turn_off_unknown": "Bulwark не удалось проверить, дошли ли изменения, сделанные на этом устройстве, до сервера. Они будут потеряны, если сейчас выключить синхронизацию.", + "turning_off": "Отправка изменений, затем удаление данных с этого устройства…", + "turning_on": "Настройка…" + }, "downloads": { "attachment_template": { "description_mobile": "Шаблон имени при сохранении вложения или при отправке его в другое приложение." diff --git a/locales/rn/sk.json b/locales/rn/sk.json index befaa934..ca165034 100644 --- a/locales/rn/sk.json +++ b/locales/rn/sk.json @@ -93,6 +93,8 @@ "until_date": "Do {date}" }, "settings": { + "device_sync_desc": "Zobrazí vaše kalendáre v aplikácii Kalendár a ďalších aplikáciách v tomto zariadení. Zmeny sa synchronizujú oboma smermi.", + "device_sync_title": "Synchronizovať s týmto zariadením", "show_week_numbers_mobile_desc": "Zobrazovať čísla týždňov v mesačnom zobrazení.", "time_zone": "Časové pásmo", "time_zone_auto": "Časové pásmo zariadenia", @@ -249,6 +251,18 @@ "color_tag": "Štítok", "remove_color": "Odstrániť štítok" }, + "device_sync": { + "auth_problem_text": "Server už neprijíma prihlásenie účtu {account}, preto sa jeho kontakty a kalendáre v tomto zariadení prestali synchronizovať. Ak chcete pokračovať, znova sa prihláste. Zmeny vykonané v tomto zariadení zostanú zachované.", + "auth_problem_title": "Ak chcete ďalej synchronizovať, znova sa prihláste", + "problem_channel": "Problémy so synchronizáciou", + "sign_out_anyway": "Odhlásiť sa", + "sign_out_pending_message": "{count, plural, one {# zmena vykonaná v tomto zariadení sa nedostala na server. Ak sa teraz odhlásite, stratí sa.} few {# zmeny vykonané v tomto zariadení sa nedostali na server. Ak sa teraz odhlásite, stratia sa.} many {# zmeny vykonanej v tomto zariadení sa nedostalo na server. Ak sa teraz odhlásite, stratia sa.} other {# zmien vykonaných v tomto zariadení sa nedostalo na server. Ak sa teraz odhlásite, stratia sa.}}", + "sign_out_pending_title": "Napriek tomu sa odhlásiť?", + "sign_out_unknown_message": "Zmeny vykonané v tomto zariadení sa možno nedostali na server. Ak sa teraz odhlásite, stratia sa.", + "signed_out_text": "Účet {account} bol odhlásený, preto sa jeho kontakty a kalendáre v tomto zariadení už nesynchronizujú. Ak chcete pokračovať, znova sa prihláste. Zmeny vykonané v tomto zariadení zostanú zachované.", + "signed_out_title": "Ak chcete ďalej synchronizovať, prihláste sa", + "signing_out": "Odstraňujú sa synchronizované kontakty a kalendáre z tohto zariadenia…" + }, "email_composer": { "attach_camera": "Fotoaparát", "attachment_reminder_body": "Vaša správa spomína „{keyword}“, ale nie je priložený žiadny súbor. Odoslať aj tak?", @@ -598,11 +612,110 @@ "book_count": "{count, plural, one {# kontakt} few {# kontakty} many {# kontaktu} other {# kontaktov}}", "default_book": "Predvolený adresár", "description_mobile": "Predvoľby zobrazenia a nástroje pre adresáre.", + "device_sync_desc": "Zobrazí vaše adresáre v aplikácii Kontakty a ďalších aplikáciách v tomto zariadení. Zmeny sa synchronizujú oboma smermi.", + "device_sync_title": "Synchronizovať s týmto zariadením", "export_description_count": "{count, plural, one {Exportovať # kontakt do jedného súboru vCard.} few {Exportovať # kontakty do jedného súboru vCard.} many {Exportovať # kontaktu do jedného súboru vCard.} other {Exportovať # kontaktov do jedného súboru vCard.}}", "export_empty": "Žiadne kontakty na export", "import_description_mobile": "Importovať kontakty zo súboru vCard (.vcf) s rozpoznaním duplicít.", "manage_description_mobile": "Vytvárajte, premenúvajte a odstraňujte adresáre, v ktorých máte kontakty usporiadané." }, + "device_sync": { + "also_synced_by": "Synchronizuje sa aj pre {accounts}, preto sa v tomto zariadení zobrazuje dvakrát.", + "android_settings": "Nastavenia účtu v Androide", + "blocked_calendar": "Android už o prístup ku kalendárom nežiada. Povoľte ho v nastaveniach aplikácie a potom synchronizáciu znova zapnite.", + "blocked_contacts": "Android už o prístup ku kontaktom nežiada. Povoľte ho v nastaveniach aplikácie a potom synchronizáciu znova zapnite.", + "blocked_title": "Prístup je vypnutý", + "books_default": "Vaše vlastné adresáre sa synchronizujú, zdieľané iba vtedy, keď ich vyberiete.", + "books_hint": "Vaše vlastné adresáre sa synchronizujú, ak ich nevypnete; zdieľané iba vtedy, keď ich zapnete.", + "books_label": "Adresáre", + "calendars_default": "Vaše vlastné kalendáre sa synchronizujú, zdieľané iba vtedy, keď ich vyberiete.", + "calendars_hint": "Vaše vlastné kalendáre sa synchronizujú, ak ich nevypnete; zdieľané iba vtedy, keď ich zapnete. Zoznamy úloh zostávajú v Bulwarku.", + "calendars_label": "Kalendáre", + "choose": "Vybrať", + "choose_books_title": "Adresáre na synchronizáciu", + "choose_calendars_title": "Kalendáre na synchronizáciu", + "collections_count": "Synchronizované: {selected} z {total}", + "continue": "Pokračovať", + "deletions_delete": "Odstrániť na serveri", + "deletions_message": "V tomto zariadení sa naraz odstránilo veľa položiek, preto sa neodstránili na serveri. Odstráňte ich aj tam, alebo ich v tomto zariadení obnovte.", + "deletions_restore": "Obnoviť", + "deletions_title": "Odstrániť ich aj na serveri?", + "denied_calendar": "Bez prístupu ku kalendárom nemôže Bulwark preniesť vaše kalendáre do tohto zariadenia.", + "denied_contacts": "Bez prístupu ku kontaktom nemôže Bulwark preniesť vaše adresáre do tohto zariadenia.", + "denied_title": "Synchronizácia zostáva vypnutá", + "failed_title": "Synchronizáciu sa nepodarilo zapnúť", + "grant_access": "Udeliť prístup", + "interval_15m": "Každých 15 minút", + "interval_1h": "Každú hodinu", + "interval_30m": "Každých 30 minút", + "interval_6h": "Každých 6 hodín", + "interval_desc": "Ako často Android synchronizuje sám. Úpravy v Bulwarku a zmeny oznámené cez push sa synchronizujú skôr.", + "interval_label": "Interval synchronizácie", + "interval_manual": "Bez pravidelnej synchronizácie", + "keep_syncing": "Ďalej synchronizovať", + "list_error_auth": "Server neprijal prihlásenie.", + "list_error_network": "Nepodarilo sa spojiť so serverom.", + "list_error_server": "Server nedokázal poskytnúť zoznam.", + "list_error_signed_out": "Tento účet nie je v tomto zariadení prihlásený.", + "master_off": "Automatická synchronizácia je v tomto zariadení vypnutá, preto sa synchronizuje iba po klepnutí na „Synchronizovať teraz“.", + "new_contacts_default": "Predvolený adresár", + "new_contacts_desc": "Adresár pre kontakty vytvorené v iných aplikáciách v tomto zariadení.", + "new_contacts_label": "Nové kontakty ukladať do", + "no_books": "Tento účet nemá žiadne adresáre.", + "no_calendars": "Tento účet nemá žiadne kalendáre na synchronizáciu.", + "ok": "OK", + "open_app_settings": "Otvoriť nastavenia", + "own_collections": "Vlastné", + "rationale_calendar": "Aby mohol Bulwark zobraziť vaše kalendáre v aplikácii Kalendár, potrebuje prístup ku kalendárom v tomto zariadení. Mení iba kalendáre svojho vlastného účtu.", + "rationale_contacts": "Aby mohol Bulwark zobraziť vaše adresáre v aplikácii Kontakty, potrebuje prístup ku kontaktom v tomto zariadení. Mení iba kontakty svojho vlastného účtu.", + "rationale_title_calendar": "Povoliť prístup ku kalendárom", + "rationale_title_contacts": "Povoliť prístup ku kontaktom", + "read_only": "Iba na čítanie", + "reminder_owner_bulwark": "Bulwark", + "reminder_owner_device": "Aplikácia Kalendár", + "reminder_question_message": "Pripomenutia synchronizovaných udalostí môže zobrazovať aplikácia kalendára v tomto zariadení alebo Bulwark. Vyberte jednu možnosť, aby ste nedostávali pripomenutia dvakrát. Neskôr to môžete zmeniť.", + "reminder_question_title": "Kto vám má pripomínať udalosti?", + "reminders_desc": "Kto vám pripomína synchronizované udalosti. Druhá možnosť mlčí, aby ste nedostávali pripomenutia dvakrát.", + "reminders_label": "Pripomenutia", + "reminders_nobody": "Oznámenia udalostí v Bulwarku sú vypnuté, takže vám synchronizované udalosti nič nepripomenie.", + "remove": "Odstrániť", + "remove_from_device": "Odstrániť z tohto zariadenia", + "remove_message": "Kontakty a kalendáre účtu sa z tohto zariadenia odstránia. Zmeny vykonané tu, ktoré sa nedostali na server, sa stratia.", + "remove_title": "Odstrániť z tohto zariadenia?", + "review_deletions": "Skontrolovať odstránenia", + "shared_by": "Zdieľané od {name}", + "sign_in_again": "Znova sa prihlásiť", + "status_auth": "Server už neprijíma prihlásenie. Ak chcete ďalej synchronizovať, znova sa prihláste.", + "status_cancelled": "Posledná synchronizácia sa skončila predčasne. Nabudúce bude pokračovať.", + "status_failed": "Posledná synchronizácia zlyhala: {message}", + "status_failed_plain": "Posledná synchronizácia zlyhala.", + "status_io": "Server bol {when} nedostupný. Android to skúsi znova.", + "status_item_errors": "{count, plural, one {# položku sa nepodarilo synchronizovať.} few {# položky sa nepodarilo synchronizovať.} many {# položky sa nepodarilo synchronizovať.} other {# položiek sa nepodarilo synchronizovať.}}", + "status_off": "Vypnuté", + "status_paused": "Pozastavené v nastaveniach Androidu", + "status_permission_calendar": "Prístup ku kalendárom bol vypnutý, preto sa nič nesynchronizuje.", + "status_permission_contacts": "Prístup ku kontaktom bol vypnutý, preto sa nič nesynchronizuje.", + "status_removed": "Odstránené v nastaveniach Androidu. Ak chcete účet znova pridať, zapnite synchronizáciu.", + "status_safety_abort": "Zastavené: server nič nevrátil, hoci toto zariadenie stále obsahuje položky. Skontrolujte server a potom synchronizujte znova.", + "status_synced": "Synchronizované {when}", + "status_syncing": "Synchronizuje sa…", + "status_too_many_deletions": "V tomto zariadení sa odstránilo veľa položiek. Na serveri sa odstránia až po vašom potvrdení.", + "status_unsupported_calendar": "Server pre tento účet neponúka žiadne kalendáre.", + "status_unsupported_contacts": "Server pre tento účet neponúka žiadne adresáre.", + "status_waiting": "Čaká sa na prvú synchronizáciu", + "status_waiting_next": "Čaká sa na ďalšiu synchronizáciu", + "suspended_desc": "Odhlásené. Kontakty a kalendáre účtu zostanú v tomto zariadení aj so zmenami vykonanými tu a po prihlásení sa znova synchronizujú.", + "switch_label": "Synchronizovať {account} s týmto zariadením", + "sync_now": "Synchronizovať teraz", + "turn_off_anyway": "Vypnúť", + "turn_off_failed": "Synchronizované údaje sa nepodarilo z tohto zariadenia odstrániť. Synchronizácia je pozastavená; skúste to neskôr.", + "turn_off_failed_title": "Synchronizáciu sa nepodarilo vypnúť", + "turn_off_pending": "{count, plural, one {# zmena vykonaná v tomto zariadení sa ešte nedostala na server. Ak teraz synchronizáciu vypnete, stratí sa.} few {# zmeny vykonané v tomto zariadení sa ešte nedostali na server. Ak teraz synchronizáciu vypnete, stratia sa.} many {# zmeny vykonanej v tomto zariadení sa ešte nedostalo na server. Ak teraz synchronizáciu vypnete, stratia sa.} other {# zmien vykonaných v tomto zariadení sa ešte nedostalo na server. Ak teraz synchronizáciu vypnete, stratia sa.}}", + "turn_off_title": "Napriek tomu vypnúť?", + "turn_off_unknown": "Bulwark nemohol overiť, či sa zmeny vykonané v tomto zariadení dostali na server. Ak teraz synchronizáciu vypnete, stratia sa.", + "turning_off": "Nahrávajú sa zmeny a potom sa z tohto zariadenia odstránia údaje…", + "turning_on": "Nastavuje sa…" + }, "downloads": { "attachment_template": { "description_mobile": "Šablóna použitá pri ukladaní alebo zdieľaní prílohy." diff --git a/locales/rn/tr.json b/locales/rn/tr.json index cd1c496e..456ae1fe 100644 --- a/locales/rn/tr.json +++ b/locales/rn/tr.json @@ -93,6 +93,8 @@ "until_date": "{date} tarihine kadar" }, "settings": { + "device_sync_desc": "Takvimlerinizi bu cihazdaki Takvim uygulamasında ve diğer uygulamalarda gösterin. Değişiklikler iki yönlü senkronize edilir.", + "device_sync_title": "Bu cihazla senkronize et", "show_week_numbers_mobile_desc": "Ay görünümünde hafta numaralarını göster.", "time_zone": "Saat dilimi", "time_zone_auto": "Cihaz saat dilimi", @@ -249,6 +251,18 @@ "color_tag": "Etiket", "remove_color": "Etiketi kaldır" }, + "device_sync": { + "auth_problem_text": "Sunucu artık {account} hesabının girişini kabul etmiyor, bu nedenle hesabın bu cihazdaki kişileri ve takvimleri artık senkronize edilmiyor. Devam etmek için tekrar giriş yapın. Bu cihazda yapılan değişiklikler korunur.", + "auth_problem_title": "Senkronizasyona devam etmek için tekrar giriş yapın", + "problem_channel": "Senkronizasyon sorunları", + "sign_out_anyway": "Çıkış yap", + "sign_out_pending_message": "{count, plural, one {Bu cihazda yapılan # değişiklik sunucuya ulaşmadı. Şimdi çıkış yaparsanız kaybolur.} other {Bu cihazda yapılan # değişiklik sunucuya ulaşmadı. Şimdi çıkış yaparsanız kaybolur.}}", + "sign_out_pending_title": "Yine de çıkış yapılsın mı?", + "sign_out_unknown_message": "Bu cihazda yapılan değişiklikler sunucuya ulaşmamış olabilir. Şimdi çıkış yaparsanız kaybolur.", + "signed_out_text": "{account} hesabından çıkış yapıldı, bu nedenle hesabın bu cihazdaki kişileri ve takvimleri artık senkronize edilmiyor. Devam etmek için tekrar giriş yapın. Bu cihazda yapılan değişiklikler korunur.", + "signed_out_title": "Senkronizasyona devam etmek için giriş yapın", + "signing_out": "Senkronize edilen kişiler ve takvimler bu cihazdan kaldırılıyor…" + }, "email_composer": { "attach_camera": "Kamera", "attachment_reminder_body": "İletinizde \"{keyword}\" geçiyor ancak hiçbir dosya eklenmemiş. Yine de gönderilsin mi?", @@ -598,11 +612,110 @@ "book_count": "{count, plural, one {# kişi} other {# kişi}}", "default_book": "Varsayılan adres defteri", "description_mobile": "Görüntüleme tercihleri ve adres defteri araçları.", + "device_sync_desc": "Adres defterlerinizi bu cihazdaki Kişiler uygulamasında ve diğer uygulamalarda gösterin. Değişiklikler iki yönlü senkronize edilir.", + "device_sync_title": "Bu cihazla senkronize et", "export_description_count": "{count, plural, one {# kişiyi tek bir vCard dosyası olarak dışa aktarın.} other {# kişiyi tek bir vCard dosyası olarak dışa aktarın.}}", "export_empty": "Dışa aktarılacak kişi yok", "import_description_mobile": "Kişileri, yinelenenleri algılayarak bir vCard (.vcf) dosyasından içe aktarın.", "manage_description_mobile": "Kişilerinizi düzenleyen adres defterlerini oluşturun, yeniden adlandırın ve kaldırın." }, + "device_sync": { + "also_synced_by": "{accounts} için de senkronize ediliyor, bu nedenle bu cihazda iki kez görünüyor.", + "android_settings": "Android hesap ayarları", + "blocked_calendar": "Android artık takvimlere erişim istemiyor. Uygulama ayarlarından erişime izin verin, ardından senkronizasyonu tekrar açın.", + "blocked_contacts": "Android artık kişilere erişim istemiyor. Uygulama ayarlarından erişime izin verin, ardından senkronizasyonu tekrar açın.", + "blocked_title": "Erişim kapalı", + "books_default": "Kendi adres defterleriniz senkronize edilir; paylaşılanlar yalnızca siz seçtiğinizde.", + "books_hint": "Kendi adres defterleriniz siz kapatmadığınız sürece senkronize edilir; paylaşılanlar yalnızca siz açtığınızda.", + "books_label": "Adres defterleri", + "calendars_default": "Kendi takvimleriniz senkronize edilir; paylaşılanlar yalnızca siz seçtiğinizde.", + "calendars_hint": "Kendi takvimleriniz siz kapatmadığınız sürece senkronize edilir; paylaşılanlar yalnızca siz açtığınızda. Görev listeleri Bulwark'ta kalır.", + "calendars_label": "Takvimler", + "choose": "Seç", + "choose_books_title": "Senkronize edilecek adres defterleri", + "choose_calendars_title": "Senkronize edilecek takvimler", + "collections_count": "Senkronize edilen: {selected}/{total}", + "continue": "Devam", + "deletions_delete": "Sunucuda sil", + "deletions_message": "Bu cihazda aynı anda çok sayıda öğe silindi, bu nedenle sunucuda silinmediler. Bunları sunucuda da silin veya bu cihaza geri getirin.", + "deletions_restore": "Geri getir", + "deletions_title": "Sunucuda da silinsin mi?", + "denied_calendar": "Takvimlere erişim olmadan Bulwark takvimlerinizi bu cihaza getiremez.", + "denied_contacts": "Kişilere erişim olmadan Bulwark adres defterlerinizi bu cihaza getiremez.", + "denied_title": "Senkronizasyon kapalı kalıyor", + "failed_title": "Senkronizasyon açılamadı", + "grant_access": "Erişim izni ver", + "interval_15m": "Her 15 dakikada bir", + "interval_1h": "Her saat", + "interval_30m": "Her 30 dakikada bir", + "interval_6h": "Her 6 saatte bir", + "interval_desc": "Android'in kendi kendine ne sıklıkla senkronize edeceği. Bulwark'taki düzenlemeler ve push ile bildirilen değişiklikler daha erken senkronize edilir.", + "interval_label": "Senkronizasyon aralığı", + "interval_manual": "Düzenli senkronizasyon yok", + "keep_syncing": "Senkronizasyona devam et", + "list_error_auth": "Sunucu girişi kabul etmedi.", + "list_error_network": "Sunucuya ulaşılamadı.", + "list_error_server": "Sunucu listeyi sağlayamadı.", + "list_error_signed_out": "Bu cihazda bu hesapla giriş yapılmamış.", + "master_off": "Bu cihazda otomatik senkronizasyon kapalı, bu nedenle senkronizasyon yalnızca \"Şimdi senkronize et\" düğmesine dokunduğunuzda yapılır.", + "new_contacts_default": "Varsayılan adres defteri", + "new_contacts_desc": "Bu cihazdaki diğer uygulamalarda oluşturulan kişilerin kaydedileceği adres defteri.", + "new_contacts_label": "Yeni kişilerin kaydedileceği yer", + "no_books": "Bu hesabın adres defteri yok.", + "no_calendars": "Bu hesabın senkronize edilecek takvimi yok.", + "ok": "Tamam", + "open_app_settings": "Uygulama ayarlarını aç", + "own_collections": "Size ait", + "rationale_calendar": "Takvimlerinizi Takvim uygulamasında göstermek için Bulwark'ın bu cihazdaki takvimlere erişmesi gerekir. Yalnızca kendi hesabının takvimlerini değiştirir.", + "rationale_contacts": "Adres defterlerinizi Kişiler uygulamasında göstermek için Bulwark'ın bu cihazdaki kişilere erişmesi gerekir. Yalnızca kendi hesabının kişilerini değiştirir.", + "rationale_title_calendar": "Takvimlere erişim izni verin", + "rationale_title_contacts": "Kişilere erişim izni verin", + "read_only": "Salt okunur", + "reminder_owner_bulwark": "Bulwark", + "reminder_owner_device": "Takvim uygulaması", + "reminder_question_message": "Senkronize edilen etkinlikler için hatırlatmalar bu cihazdaki takvim uygulamasından veya Bulwark'tan gelebilir. İki kez hatırlatılmamak için birini seçin. Bunu daha sonra değiştirebilirsiniz.", + "reminder_question_title": "Size kim hatırlatsın?", + "reminders_desc": "Senkronize edilen etkinlikleri size kimin hatırlatacağı. Diğeri sessiz kalır, böylece iki kez hatırlatılmazsınız.", + "reminders_label": "Hatırlatıcılar", + "reminders_nobody": "Bulwark'ın etkinlik bildirimleri kapalı, bu nedenle senkronize edilen etkinlikleri size hiçbir şey hatırlatmaz.", + "remove": "Kaldır", + "remove_from_device": "Bu cihazdan kaldır", + "remove_message": "Hesabın kişileri ve takvimleri bu cihazdan kaldırılır. Burada yapılan ve sunucuya ulaşmamış değişiklikler kaybolur.", + "remove_title": "Bu cihazdan kaldırılsın mı?", + "review_deletions": "Silmeleri incele", + "shared_by": "Paylaşan: {name}", + "sign_in_again": "Tekrar giriş yap", + "status_auth": "Sunucu artık girişi kabul etmiyor. Senkronizasyona devam etmek için tekrar giriş yapın.", + "status_cancelled": "Son senkronizasyon erken durdu. Bir dahaki sefere devam edecek.", + "status_failed": "Son senkronizasyon başarısız: {message}", + "status_failed_plain": "Son senkronizasyon başarısız oldu.", + "status_io": "Sunucuya {when} ulaşılamadı. Android tekrar deneyecek.", + "status_item_errors": "{count, plural, one {# öğe senkronize edilemedi.} other {# öğe senkronize edilemedi.}}", + "status_off": "Kapalı", + "status_paused": "Android ayarlarında duraklatıldı", + "status_permission_calendar": "Takvimlere erişim kapatıldı, bu nedenle hiçbir şey senkronize edilmiyor.", + "status_permission_contacts": "Kişilere erişim kapatıldı, bu nedenle hiçbir şey senkronize edilmiyor.", + "status_removed": "Android ayarlarında kaldırıldı. Hesabı yeniden eklemek için senkronizasyonu açın.", + "status_safety_abort": "Durduruldu: bu cihazda hâlâ öğeler varken sunucu hiçbir şey döndürmedi. Sunucuyu kontrol edin, ardından yeniden senkronize edin.", + "status_synced": "Son senkronizasyon: {when}", + "status_syncing": "Senkronize ediliyor…", + "status_too_many_deletions": "Bu cihazda çok sayıda öğe silindi. Siz onaylayana kadar sunucuda silinmezler.", + "status_unsupported_calendar": "Sunucu bu hesap için takvim sunmuyor.", + "status_unsupported_contacts": "Sunucu bu hesap için adres defteri sunmuyor.", + "status_waiting": "İlk senkronizasyon bekleniyor", + "status_waiting_next": "Sonraki senkronizasyon bekleniyor", + "suspended_desc": "Çıkış yapıldı. Hesabın kişileri ve takvimleri, burada yapılan değişikliklerle birlikte bu cihazda kalır ve giriş yaptığınızda yeniden senkronize edilir.", + "switch_label": "{account} hesabını bu cihazla senkronize et", + "sync_now": "Şimdi senkronize et", + "turn_off_anyway": "Kapat", + "turn_off_failed": "Senkronize edilen veriler bu cihazdan kaldırılamadı. Senkronizasyon duraklatıldı; daha sonra tekrar deneyin.", + "turn_off_failed_title": "Senkronizasyon kapatılamadı", + "turn_off_pending": "{count, plural, one {Bu cihazda yapılan # değişiklik henüz sunucuya ulaşmadı. Senkronizasyonu şimdi kapatırsanız kaybolur.} other {Bu cihazda yapılan # değişiklik henüz sunucuya ulaşmadı. Senkronizasyonu şimdi kapatırsanız kaybolur.}}", + "turn_off_title": "Yine de kapatılsın mı?", + "turn_off_unknown": "Bulwark, bu cihazda yapılan değişikliklerin sunucuya ulaşıp ulaşmadığını kontrol edemedi. Senkronizasyonu şimdi kapatırsanız kaybolur.", + "turning_off": "Değişiklikler yükleniyor, ardından veriler bu cihazdan kaldırılıyor…", + "turning_on": "Ayarlanıyor…" + }, "downloads": { "attachment_template": { "description_mobile": "Bir eki kaydederken veya paylaşırken kullanılan şablon." diff --git a/locales/rn/uk.json b/locales/rn/uk.json index a885fa36..968dc329 100644 --- a/locales/rn/uk.json +++ b/locales/rn/uk.json @@ -93,6 +93,8 @@ "until_date": "До {date}" }, "settings": { + "device_sync_desc": "Ваші календарі відображатимуться в застосунку «Календар» та інших застосунках на цьому пристрої. Зміни синхронізуються в обидва боки.", + "device_sync_title": "Синхронізація з цим пристроєм", "show_week_numbers_mobile_desc": "Показувати номери тижнів у поданні «Місяць».", "time_zone": "Часовий пояс", "time_zone_auto": "Часовий пояс пристрою", @@ -249,6 +251,18 @@ "color_tag": "Тег", "remove_color": "Прибрати тег" }, + "device_sync": { + "auth_problem_text": "Сервер більше не приймає вхід в обліковий запис {account}, тому його контакти й календарі на цьому пристрої більше не синхронізуються. Увійдіть знову, щоб відновити синхронізацію. Зміни, внесені на цьому пристрої, збережуться.", + "auth_problem_title": "Увійдіть знову, щоб продовжити синхронізацію", + "problem_channel": "Проблеми із синхронізацією", + "sign_out_anyway": "Вийти", + "sign_out_pending_message": "{count, plural, one {# зміна, внесена на цьому пристрої, не дійшла до сервера. Її буде втрачено, якщо ви зараз вийдете.} few {# зміни, внесені на цьому пристрої, не дійшли до сервера. Їх буде втрачено, якщо ви зараз вийдете.} many {# змін, внесених на цьому пристрої, не дійшли до сервера. Їх буде втрачено, якщо ви зараз вийдете.} other {# зміни, внесеної на цьому пристрої, не дійшли до сервера. Їх буде втрачено, якщо ви зараз вийдете.}}", + "sign_out_pending_title": "Усе одно вийти?", + "sign_out_unknown_message": "Зміни, внесені на цьому пристрої, могли не дійти до сервера. Їх буде втрачено, якщо ви зараз вийдете.", + "signed_out_text": "Виконано вихід з облікового запису {account}, тому його контакти й календарі на цьому пристрої більше не синхронізуються. Увійдіть знову, щоб відновити синхронізацію. Зміни, внесені на цьому пристрої, збережуться.", + "signed_out_title": "Увійдіть, щоб продовжити синхронізацію", + "signing_out": "Видалення синхронізованих контактів і календарів із цього пристрою…" + }, "email_composer": { "attach_camera": "Камера", "attachment_reminder_body": "У листі згадано «{keyword}», але файл не вкладено. Усе одно надіслати?", @@ -598,11 +612,110 @@ "book_count": "{count, plural, one {# контакт} few {# контакти} many {# контактів} other {# контакту}}", "default_book": "Адресна книга за замовчуванням", "description_mobile": "Параметри відображення та інструменти адресних книг.", + "device_sync_desc": "Ваші адресні книги відображатимуться в застосунку «Контакти» та інших застосунках на цьому пристрої. Зміни синхронізуються в обидва боки.", + "device_sync_title": "Синхронізація з цим пристроєм", "export_description_count": "{count, plural, one {Експортувати # контакт в один файл vCard.} few {Експортувати # контакти в один файл vCard.} many {Експортувати # контактів в один файл vCard.} other {Експортувати # контакту в один файл vCard.}}", "export_empty": "Немає контактів для експорту", "import_description_mobile": "Імпорт контактів із файлу vCard (.vcf) з виявленням дублікатів.", "manage_description_mobile": "Створюйте, перейменовуйте та видаляйте адресні книги, у яких зберігаються ваші контакти." }, + "device_sync": { + "also_synced_by": "Також синхронізується для {accounts}, тому відображається на цьому пристрої двічі.", + "android_settings": "Налаштування облікового запису Android", + "blocked_calendar": "Android більше не запитує доступ до календарів. Дозвольте його в налаштуваннях застосунку, а потім знову ввімкніть синхронізацію.", + "blocked_contacts": "Android більше не запитує доступ до контактів. Дозвольте його в налаштуваннях застосунку, а потім знову ввімкніть синхронізацію.", + "blocked_title": "Доступ вимкнено", + "books_default": "Ваші власні адресні книги синхронізуються, а спільні — лише якщо ви їх виберете.", + "books_hint": "Ваші власні адресні книги синхронізуються, якщо ви їх не вимкнете, а спільні — лише якщо ви їх увімкнете.", + "books_label": "Адресні книги", + "calendars_default": "Ваші власні календарі синхронізуються, а спільні — лише якщо ви їх виберете.", + "calendars_hint": "Ваші власні календарі синхронізуються, якщо ви їх не вимкнете, а спільні — лише якщо ви їх увімкнете. Списки завдань залишаються в Bulwark.", + "calendars_label": "Календарі", + "choose": "Вибрати", + "choose_books_title": "Адресні книги для синхронізації", + "choose_calendars_title": "Календарі для синхронізації", + "collections_count": "Синхронізується: {selected} з {total}", + "continue": "Продовжити", + "deletions_delete": "Видалити на сервері", + "deletions_message": "На цьому пристрої одночасно видалено багато елементів, тому на сервері їх не видалено. Видаліть їх і там або поверніть їх на цей пристрій.", + "deletions_restore": "Повернути", + "deletions_title": "Видалити їх і на сервері?", + "denied_calendar": "Без доступу до календарів Bulwark не зможе перенести ваші календарі на цей пристрій.", + "denied_contacts": "Без доступу до контактів Bulwark не зможе перенести ваші адресні книги на цей пристрій.", + "denied_title": "Синхронізація залишається вимкненою", + "failed_title": "Не вдалося ввімкнути синхронізацію", + "grant_access": "Надати доступ", + "interval_15m": "Кожні 15 хвилин", + "interval_1h": "Щогодини", + "interval_30m": "Кожні 30 хвилин", + "interval_6h": "Кожні 6 годин", + "interval_desc": "Як часто Android синхронізує дані самостійно. Зміни в Bulwark і зміни, про які повідомляє push, синхронізуються швидше.", + "interval_label": "Інтервал синхронізації", + "interval_manual": "Без регулярної синхронізації", + "keep_syncing": "Продовжити синхронізацію", + "list_error_auth": "Сервер не прийняв вхід.", + "list_error_network": "Не вдалося зв'язатися із сервером.", + "list_error_server": "Сервер не зміг повернути список.", + "list_error_signed_out": "На цьому пристрої не виконано вхід у цей обліковий запис.", + "master_off": "Автосинхронізацію на цьому пристрої вимкнено, тому синхронізація виконується лише після натискання «Синхронізувати».", + "new_contacts_default": "Адресна книга за замовчуванням", + "new_contacts_desc": "Адресна книга для контактів, створених в інших застосунках на цьому пристрої.", + "new_contacts_label": "Зберігати нові контакти в", + "no_books": "У цьому обліковому записі немає адресних книг.", + "no_calendars": "У цьому обліковому записі немає календарів для синхронізації.", + "ok": "OK", + "open_app_settings": "Відкрити налаштування", + "own_collections": "Ваші", + "rationale_calendar": "Щоб показувати ваші календарі в застосунку «Календар», Bulwark потрібен доступ до календарів на цьому пристрої. Він змінює лише календарі свого облікового запису.", + "rationale_contacts": "Щоб показувати ваші адресні книги в застосунку «Контакти», Bulwark потрібен доступ до контактів на цьому пристрої. Він змінює лише контакти свого облікового запису.", + "rationale_title_calendar": "Дозволити доступ до календарів", + "rationale_title_contacts": "Дозволити доступ до контактів", + "read_only": "Лише для читання", + "reminder_owner_bulwark": "Bulwark", + "reminder_owner_device": "Застосунок «Календар»", + "reminder_question_message": "Нагадування про синхронізовані події може показувати застосунок календаря на цьому пристрої або Bulwark. Виберіть щось одне, щоб не отримувати нагадування двічі. Це можна змінити пізніше.", + "reminder_question_title": "Хто нагадуватиме про події?", + "reminders_desc": "Хто нагадує про синхронізовані події. Інший застосунок мовчить, щоб нагадування не надходили двічі.", + "reminders_label": "Нагадування", + "reminders_nobody": "Сповіщення про події в Bulwark вимкнено, тому про синхронізовані події ніщо не нагадає.", + "remove": "Видалити", + "remove_from_device": "Видалити з цього пристрою", + "remove_message": "Контакти й календарі облікового запису буде видалено з цього пристрою. Внесені тут зміни, які не дійшли до сервера, буде втрачено.", + "remove_title": "Видалити з цього пристрою?", + "review_deletions": "Переглянути видалення", + "shared_by": "Власник: {name}", + "sign_in_again": "Увійти знову", + "status_auth": "Сервер більше не приймає вхід. Увійдіть знову, щоб продовжити синхронізацію.", + "status_cancelled": "Остання синхронізація перервалася. Вона продовжиться наступного разу.", + "status_failed": "Помилка останньої синхронізації: {message}", + "status_failed_plain": "Остання синхронізація не вдалася.", + "status_io": "Не вдалося зв'язатися із сервером {when}. Android повторить спробу.", + "status_item_errors": "{count, plural, one {Не вдалося синхронізувати # елемент.} few {Не вдалося синхронізувати # елементи.} many {Не вдалося синхронізувати # елементів.} other {Не вдалося синхронізувати # елемента.}}", + "status_off": "Вимкнено", + "status_paused": "Призупинено в налаштуваннях Android", + "status_permission_calendar": "Доступ до календарів вимкнено, тому нічого не синхронізується.", + "status_permission_contacts": "Доступ до контактів вимкнено, тому нічого не синхронізується.", + "status_removed": "Видалено в налаштуваннях Android. Увімкніть синхронізацію, щоб знову додати обліковий запис.", + "status_safety_abort": "Зупинено: сервер нічого не повернув, хоча на цьому пристрої ще є елементи. Перевірте сервер і повторіть синхронізацію.", + "status_synced": "Синхронізовано {when}", + "status_syncing": "Синхронізація…", + "status_too_many_deletions": "На цьому пристрої видалено багато елементів. На сервері їх буде видалено лише після вашого підтвердження.", + "status_unsupported_calendar": "Сервер не надає календарів для цього облікового запису.", + "status_unsupported_contacts": "Сервер не надає адресних книг для цього облікового запису.", + "status_waiting": "Очікування першої синхронізації", + "status_waiting_next": "Очікування наступної синхронізації", + "suspended_desc": "Виконано вихід. Контакти й календарі облікового запису залишаються на цьому пристрої разом із внесеними тут змінами й знову синхронізуються після входу.", + "switch_label": "Синхронізувати {account} з цим пристроєм", + "sync_now": "Синхронізувати", + "turn_off_anyway": "Вимкнути", + "turn_off_failed": "Не вдалося видалити синхронізовані дані з цього пристрою. Синхронізацію призупинено; спробуйте пізніше.", + "turn_off_failed_title": "Не вдалося вимкнути синхронізацію", + "turn_off_pending": "{count, plural, one {# зміна, внесена на цьому пристрої, ще не дійшла до сервера. Її буде втрачено, якщо зараз вимкнути синхронізацію.} few {# зміни, внесені на цьому пристрої, ще не дійшли до сервера. Їх буде втрачено, якщо зараз вимкнути синхронізацію.} many {# змін, внесених на цьому пристрої, ще не дійшли до сервера. Їх буде втрачено, якщо зараз вимкнути синхронізацію.} other {# зміни, внесеної на цьому пристрої, ще не дійшли до сервера. Їх буде втрачено, якщо зараз вимкнути синхронізацію.}}", + "turn_off_title": "Усе одно вимкнути?", + "turn_off_unknown": "Bulwark не вдалося перевірити, чи дійшли зміни, внесені на цьому пристрої, до сервера. Їх буде втрачено, якщо зараз вимкнути синхронізацію.", + "turning_off": "Надсилання змін, потім видалення даних із цього пристрою…", + "turning_on": "Налаштовування…" + }, "downloads": { "attachment_template": { "description_mobile": "Шаблон назви під час збереження вкладення або передавання його в інший застосунок." diff --git a/locales/rn/zh-TW.json b/locales/rn/zh-TW.json index c8d7e800..07f72b5d 100644 --- a/locales/rn/zh-TW.json +++ b/locales/rn/zh-TW.json @@ -93,6 +93,8 @@ "until_date": "直到 {date}" }, "settings": { + "device_sync_desc": "在此裝置上的日曆應用程式和其他應用程式中顯示您的行事曆。變更會雙向同步。", + "device_sync_title": "同步到此裝置", "show_week_numbers_mobile_desc": "在月檢視中顯示週數", "time_zone": "時區", "time_zone_auto": "裝置時區", @@ -249,6 +251,18 @@ "color_tag": "標籤", "remove_color": "移除標籤" }, + "device_sync": { + "auth_problem_text": "伺服器不再接受 {account} 的登入，因此此帳號在這部裝置上的聯絡人和行事曆已停止同步。請重新登入以繼續同步。在此裝置上所做的變更會保留。", + "auth_problem_title": "請重新登入以繼續同步", + "problem_channel": "同步問題", + "sign_out_anyway": "登出", + "sign_out_pending_message": "{count, plural, other {在此裝置上所做的 # 項變更尚未傳送到伺服器。如果現在登出，這些變更將會遺失。}}", + "sign_out_pending_title": "仍要登出嗎？", + "sign_out_unknown_message": "在此裝置上所做的變更可能尚未傳送到伺服器。如果現在登出，這些變更將會遺失。", + "signed_out_text": "{account} 已登出，因此此帳號在這部裝置上的聯絡人和行事曆不再同步。請重新登入以繼續同步。在此裝置上所做的變更會保留。", + "signed_out_title": "請登入以繼續同步", + "signing_out": "正在從此裝置移除已同步的聯絡人和行事曆…" + }, "email_composer": { "attach_camera": "相機", "attachment_reminder_body": "您的郵件提到「{keyword}」，但沒有附加檔案。仍要傳送嗎？", @@ -598,11 +612,110 @@ "book_count": "{count, plural, other {# 位聯絡人}}", "default_book": "預設通訊錄", "description_mobile": "顯示偏好設定與通訊錄工具。", + "device_sync_desc": "在此裝置上的聯絡人應用程式和其他應用程式中顯示您的通訊錄。變更會雙向同步。", + "device_sync_title": "同步到此裝置", "export_description_count": "{count, plural, other {將 # 位聯絡人匯出為單一 vCard 檔案。}}", "export_empty": "沒有可匯出的聯絡人", "import_description_mobile": "從 vCard（.vcf）檔案匯入聯絡人，並偵測重複項目。", "manage_description_mobile": "建立、重新命名及移除用來整理聯絡人的通訊錄。" }, + "device_sync": { + "also_synced_by": "也為 {accounts} 同步，因此會在此裝置上顯示兩次。", + "android_settings": "Android 帳號設定", + "blocked_calendar": "Android 不會再要求行事曆存取權。請在應用程式設定中允許，然後重新開啟同步。", + "blocked_contacts": "Android 不會再要求聯絡人存取權。請在應用程式設定中允許，然後重新開啟同步。", + "blocked_title": "存取權已關閉", + "books_default": "您自己的通訊錄會同步；共用的通訊錄只有在您選擇後才會同步。", + "books_hint": "您自己的通訊錄會同步，除非您將其關閉；共用的通訊錄只有在您開啟後才會同步。", + "books_label": "通訊錄", + "calendars_default": "您自己的行事曆會同步；共用的行事曆只有在您選擇後才會同步。", + "calendars_hint": "您自己的行事曆會同步，除非您將其關閉；共用的行事曆只有在您開啟後才會同步。任務清單會保留在 Bulwark 中。", + "calendars_label": "行事曆", + "choose": "選擇", + "choose_books_title": "要同步的通訊錄", + "choose_calendars_title": "要同步的行事曆", + "collections_count": "同步 {selected} 個，共 {total} 個", + "continue": "繼續", + "deletions_delete": "在伺服器上刪除", + "deletions_message": "此裝置上一次刪除了許多項目，因此未在伺服器上刪除。請同時在伺服器上刪除，或在此裝置上還原這些項目。", + "deletions_restore": "還原", + "deletions_title": "也要在伺服器上刪除嗎？", + "denied_calendar": "沒有行事曆存取權，Bulwark 就無法將您的行事曆放到此裝置上。", + "denied_contacts": "沒有聯絡人存取權，Bulwark 就無法將您的通訊錄放到此裝置上。", + "denied_title": "同步維持關閉", + "failed_title": "無法開啟同步", + "grant_access": "授予存取權", + "interval_15m": "每 15 分鐘", + "interval_1h": "每小時", + "interval_30m": "每 30 分鐘", + "interval_6h": "每 6 小時", + "interval_desc": "Android 自動同步的頻率。在 Bulwark 中的編輯和透過推送通知的變更會更快同步。", + "interval_label": "同步間隔", + "interval_manual": "不進行定期同步", + "keep_syncing": "繼續同步", + "list_error_auth": "伺服器未接受登入。", + "list_error_network": "無法連線到伺服器。", + "list_error_server": "伺服器無法提供清單。", + "list_error_signed_out": "此帳號未在此裝置上登入。", + "master_off": "此裝置的自動同步已關閉，因此只有在您輕觸「立即同步」時才會同步。", + "new_contacts_default": "預設通訊錄", + "new_contacts_desc": "用來儲存在此裝置上其他應用程式中建立的聯絡人的通訊錄。", + "new_contacts_label": "新聯絡人儲存位置", + "no_books": "此帳號沒有通訊錄。", + "no_calendars": "此帳號沒有可同步的行事曆。", + "ok": "確定", + "open_app_settings": "開啟應用程式設定", + "own_collections": "您自己的", + "rationale_calendar": "如要在日曆應用程式中顯示您的行事曆，Bulwark 需要存取此裝置上的行事曆。它只會變更自己帳號的行事曆。", + "rationale_contacts": "如要在聯絡人應用程式中顯示您的通訊錄，Bulwark 需要存取此裝置上的聯絡人。它只會變更自己帳號的聯絡人。", + "rationale_title_calendar": "允許存取行事曆", + "rationale_title_contacts": "允許存取聯絡人", + "read_only": "唯讀", + "reminder_owner_bulwark": "Bulwark", + "reminder_owner_device": "日曆應用程式", + "reminder_question_message": "已同步活動的提醒可以來自此裝置上的日曆應用程式或 Bulwark。請擇一使用，以免收到兩次提醒。您之後可以變更。", + "reminder_question_title": "要由誰提醒您？", + "reminders_desc": "由誰提醒您已同步的活動。另一方會保持靜音，您就不會收到兩次提醒。", + "reminders_label": "提醒", + "reminders_nobody": "Bulwark 的活動通知已關閉，因此已同步的活動不會有任何提醒。", + "remove": "移除", + "remove_from_device": "從此裝置移除", + "remove_message": "此帳號的聯絡人和行事曆將從此裝置移除。在此處所做但尚未傳送到伺服器的變更將會遺失。", + "remove_title": "要從此裝置移除嗎？", + "review_deletions": "檢查刪除項目", + "shared_by": "由 {name} 分享", + "sign_in_again": "重新登入", + "status_auth": "伺服器不再接受登入。請重新登入以繼續同步。", + "status_cancelled": "上次同步提前停止。下次會繼續。", + "status_failed": "上次同步失敗：{message}", + "status_failed_plain": "上次同步失敗。", + "status_io": "無法連線到伺服器（{when}）。Android 會重試。", + "status_item_errors": "{count, plural, other {有 # 個項目無法同步。}}", + "status_off": "關閉", + "status_paused": "已在 Android 設定中暫停", + "status_permission_calendar": "行事曆存取權已關閉，因此不會同步任何內容。", + "status_permission_contacts": "聯絡人存取權已關閉，因此不會同步任何內容。", + "status_removed": "已在 Android 設定中移除。開啟同步即可重新新增此帳號。", + "status_safety_abort": "已停止：此裝置上仍有項目，但伺服器未傳回任何內容。請檢查伺服器，然後重新同步。", + "status_synced": "上次同步：{when}", + "status_syncing": "正在同步…", + "status_too_many_deletions": "此裝置上刪除了許多項目。在您確認之前，這些項目不會在伺服器上刪除。", + "status_unsupported_calendar": "伺服器未為此帳號提供行事曆。", + "status_unsupported_contacts": "伺服器未為此帳號提供通訊錄。", + "status_waiting": "正在等待首次同步", + "status_waiting_next": "正在等待下次同步", + "suspended_desc": "已登出。此帳號的聯絡人和行事曆會連同在此處所做的變更一起保留在此裝置上，並在您登入後重新同步。", + "switch_label": "將 {account} 同步到此裝置", + "sync_now": "立即同步", + "turn_off_anyway": "關閉", + "turn_off_failed": "無法從此裝置移除已同步的資料。同步已暫停；請稍後再試。", + "turn_off_failed_title": "無法關閉同步", + "turn_off_pending": "{count, plural, other {在此裝置上所做的 # 項變更尚未傳送到伺服器。如果現在關閉同步，這些變更將會遺失。}}", + "turn_off_title": "仍要關閉嗎？", + "turn_off_unknown": "Bulwark 無法確認在此裝置上所做的變更是否已傳送到伺服器。如果現在關閉同步，這些變更將會遺失。", + "turning_off": "正在上傳變更，然後從此裝置移除資料…", + "turning_on": "正在設定…" + }, "downloads": { "attachment_template": { "description_mobile": "儲存或分享附件時使用的範本。" diff --git a/locales/rn/zh.json b/locales/rn/zh.json index 6bb8f899..2768f649 100644 --- a/locales/rn/zh.json +++ b/locales/rn/zh.json @@ -93,6 +93,8 @@ "until_date": "直到 {date}" }, "settings": { + "device_sync_desc": "在此设备上的日历应用和其他应用中显示您的日历。更改会双向同步。", + "device_sync_title": "同步到此设备", "show_week_numbers_mobile_desc": "在月视图中显示周数", "time_zone": "时区", "time_zone_auto": "设备时区", @@ -249,6 +251,18 @@ "color_tag": "标签", "remove_color": "删除标签" }, + "device_sync": { + "auth_problem_text": "服务器不再接受 {account} 的登录，因此该账户在此设备上的联系人和日历已停止同步。请重新登录以恢复同步。在此设备上所做的更改会保留。", + "auth_problem_title": "请重新登录以继续同步", + "problem_channel": "同步问题", + "sign_out_anyway": "退出登录", + "sign_out_pending_message": "{count, plural, other {在此设备上所做的 # 项更改尚未到达服务器。如果现在退出登录，这些更改将丢失。}}", + "sign_out_pending_title": "仍要退出登录？", + "sign_out_unknown_message": "在此设备上所做的更改可能尚未到达服务器。如果现在退出登录，这些更改将丢失。", + "signed_out_text": "{account} 已退出登录，因此该账户在此设备上的联系人和日历不再同步。请重新登录以恢复同步。在此设备上所做的更改会保留。", + "signed_out_title": "请登录以继续同步", + "signing_out": "正在从此设备移除已同步的联系人和日历…" + }, "email_composer": { "attach_camera": "相机", "attachment_reminder_body": "您的邮件中提到了“{keyword}”，但未附加任何文件。仍然发送吗？", @@ -598,11 +612,110 @@ "book_count": "{count, plural, other {# 位联系人}}", "default_book": "默认通讯录", "description_mobile": "显示偏好设置和通讯录工具。", + "device_sync_desc": "在此设备上的联系人应用和其他应用中显示您的通讯录。更改会双向同步。", + "device_sync_title": "同步到此设备", "export_description_count": "{count, plural, other {将 # 位联系人导出为单个 vCard 文件。}}", "export_empty": "没有可导出的联系人", "import_description_mobile": "从 vCard (.vcf) 文件导入联系人，并检测重复项。", "manage_description_mobile": "创建、重命名和删除用于整理联系人的通讯录。" }, + "device_sync": { + "also_synced_by": "也为 {accounts} 同步，因此会在此设备上显示两次。", + "android_settings": "Android 账户设置", + "blocked_calendar": "Android 不再请求日历访问权限。请在应用设置中允许，然后重新开启同步。", + "blocked_contacts": "Android 不再请求联系人访问权限。请在应用设置中允许，然后重新开启同步。", + "blocked_title": "访问权限已关闭", + "books_default": "您自己的通讯录会同步；共享的通讯录仅在您选择后同步。", + "books_hint": "您自己的通讯录会同步，除非您将其关闭；共享的通讯录仅在您开启后同步。", + "books_label": "通讯录", + "calendars_default": "您自己的日历会同步；共享的日历仅在您选择后同步。", + "calendars_hint": "您自己的日历会同步，除非您将其关闭；共享的日历仅在您开启后同步。任务列表保留在 Bulwark 中。", + "calendars_label": "日历", + "choose": "选择", + "choose_books_title": "要同步的通讯录", + "choose_calendars_title": "要同步的日历", + "collections_count": "同步 {selected} 个，共 {total} 个", + "continue": "继续", + "deletions_delete": "在服务器上删除", + "deletions_message": "此设备上一次删除了许多项目，因此未在服务器上删除。请同时在服务器上删除它们，或在此设备上恢复它们。", + "deletions_restore": "恢复", + "deletions_title": "也在服务器上删除？", + "denied_calendar": "没有日历访问权限，Bulwark 无法将您的日历放到此设备上。", + "denied_contacts": "没有联系人访问权限，Bulwark 无法将您的通讯录放到此设备上。", + "denied_title": "同步保持关闭", + "failed_title": "无法开启同步", + "grant_access": "授予访问权限", + "interval_15m": "每 15 分钟", + "interval_1h": "每小时", + "interval_30m": "每 30 分钟", + "interval_6h": "每 6 小时", + "interval_desc": "Android 自动同步的频率。在 Bulwark 中的编辑和通过推送通知的更改会更快同步。", + "interval_label": "同步间隔", + "interval_manual": "不进行定期同步", + "keep_syncing": "继续同步", + "list_error_auth": "服务器未接受登录。", + "list_error_network": "无法连接到服务器。", + "list_error_server": "服务器无法提供列表。", + "list_error_signed_out": "此账户未在此设备上登录。", + "master_off": "此设备的自动同步已关闭，因此只有在您点按“立即同步”时才会同步。", + "new_contacts_default": "默认通讯录", + "new_contacts_desc": "用于保存在此设备上其他应用中创建的联系人的通讯录。", + "new_contacts_label": "新联系人保存到", + "no_books": "此账户没有通讯录。", + "no_calendars": "此账户没有可同步的日历。", + "ok": "确定", + "open_app_settings": "打开应用设置", + "own_collections": "您自己的", + "rationale_calendar": "要在日历应用中显示您的日历，Bulwark 需要访问此设备上的日历。它只会更改自己账户的日历。", + "rationale_contacts": "要在联系人应用中显示您的通讯录，Bulwark 需要访问此设备上的联系人。它只会更改自己账户的联系人。", + "rationale_title_calendar": "允许访问日历", + "rationale_title_contacts": "允许访问联系人", + "read_only": "只读", + "reminder_owner_bulwark": "Bulwark", + "reminder_owner_device": "日历应用", + "reminder_question_message": "已同步活动的提醒可以来自此设备上的日历应用或 Bulwark。请选择其一，以免收到两次提醒。您之后可以更改。", + "reminder_question_title": "由谁来提醒您？", + "reminders_desc": "由谁提醒您已同步的活动。另一方保持静默，这样您就不会收到两次提醒。", + "reminders_label": "提醒", + "reminders_nobody": "Bulwark 的活动通知已关闭，因此已同步的活动不会有任何提醒。", + "remove": "移除", + "remove_from_device": "从此设备移除", + "remove_message": "该账户的联系人和日历将从此设备移除。在此处所做但尚未到达服务器的更改将丢失。", + "remove_title": "从此设备移除？", + "review_deletions": "查看删除项", + "shared_by": "由 {name} 共享", + "sign_in_again": "重新登录", + "status_auth": "服务器不再接受登录。请重新登录以继续同步。", + "status_cancelled": "上次同步提前停止。下次会继续。", + "status_failed": "上次同步失败：{message}", + "status_failed_plain": "上次同步失败。", + "status_io": "无法连接到服务器（{when}）。Android 会重试。", + "status_item_errors": "{count, plural, other {有 # 个项目无法同步。}}", + "status_off": "关闭", + "status_paused": "已在 Android 设置中暂停", + "status_permission_calendar": "日历访问权限已关闭，因此不会同步任何内容。", + "status_permission_contacts": "联系人访问权限已关闭，因此不会同步任何内容。", + "status_removed": "已在 Android 设置中移除。开启同步即可重新添加该账户。", + "status_safety_abort": "已停止：此设备上仍有项目，但服务器未返回任何内容。请检查服务器，然后重新同步。", + "status_synced": "上次同步：{when}", + "status_syncing": "正在同步…", + "status_too_many_deletions": "此设备上删除了许多项目。在您确认之前，它们不会在服务器上被删除。", + "status_unsupported_calendar": "服务器没有为此账户提供日历。", + "status_unsupported_contacts": "服务器没有为此账户提供通讯录。", + "status_waiting": "正在等待首次同步", + "status_waiting_next": "正在等待下次同步", + "suspended_desc": "已退出登录。该账户的联系人和日历会连同在此处所做的更改一起保留在此设备上，并在您登录后重新同步。", + "switch_label": "将 {account} 同步到此设备", + "sync_now": "立即同步", + "turn_off_anyway": "关闭", + "turn_off_failed": "无法从此设备移除已同步的数据。同步已暂停；请稍后重试。", + "turn_off_failed_title": "无法关闭同步", + "turn_off_pending": "{count, plural, other {在此设备上所做的 # 项更改尚未到达服务器。如果现在关闭同步，这些更改将丢失。}}", + "turn_off_title": "仍要关闭？", + "turn_off_unknown": "Bulwark 无法确认在此设备上所做的更改是否已到达服务器。如果现在关闭同步，这些更改将丢失。", + "turning_off": "正在上传更改，然后从此设备移除数据…", + "turning_on": "正在设置…" + }, "downloads": { "attachment_template": { "description_mobile": "保存或分享附件时使用的模板。" From 13d5d1733be9fb0aeeff564f0c0ce47d65af3747 Mon Sep 17 00:00:00 2001 From: Linus Rath <139418639+rathlinus@users.noreply.github.com> Date: Sun, 27 Sep 2026 08:50:59 +0200 Subject: [PATCH 63/87] fix: say "1 of 3 syncs" when one collection of several syncs #34 --- locales/rn/en.json | 2 +- src/components/settings/device-sync/DeviceSyncAccountRow.tsx | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/locales/rn/en.json b/locales/rn/en.json index dff3fc6b..7a6011c5 100644 --- a/locales/rn/en.json +++ b/locales/rn/en.json @@ -644,7 +644,7 @@ "choose": "Choose", "choose_books_title": "Address books to sync", "choose_calendars_title": "Calendars to sync", - "collections_count": "{selected} of {total} sync", + "collections_count": "{selected, plural, one {# of {total} syncs} other {# of {total} sync}}", "continue": "Continue", "deletions_delete": "Delete on server", "deletions_message": "Many items were deleted on this device at once, so they were not deleted on the server. Delete them there as well, or bring them back on this device.", diff --git a/src/components/settings/device-sync/DeviceSyncAccountRow.tsx b/src/components/settings/device-sync/DeviceSyncAccountRow.tsx index e83b9910..b7053ec9 100644 --- a/src/components/settings/device-sync/DeviceSyncAccountRow.tsx +++ b/src/components/settings/device-sync/DeviceSyncAccountRow.tsx @@ -288,7 +288,7 @@ export function DeviceSyncAccountRow({ account, authority }: Props) { : ''; const collectionsDescription = collections && selectedKeys - ? t('settings.device_sync.collections_count', '{selected} of {total} sync', { + ? t('settings.device_sync.collections_count', '{selected, plural, one {# of {total} syncs} other {# of {total} sync}}', { selected: selectedKeys.length, total: collections.length, }) From f24d919fdbaf1adc632702cfad04c010e0baadc7 Mon Sep 17 00:00:00 2001 From: Linus Rath <139418639+rathlinus@users.noreply.github.com> Date: Sun, 27 Sep 2026 09:15:53 +0200 Subject: [PATCH 64/87] docs: describe device sync after the review fixes and how it was verified #34 The device sync doc still described the feature as first built. The review fixes since then changed pairs, merges of items outside the synced collections, the order of uploads, group handling, photos, the provider-op scope checks, the settings actions and what the report says about changes put back on the phone. Each fix's doc notes are applied against the code as it is now, the known limitations gain what the device runs found (an identity as organizer, Stalwart refusing answers in read-only calendars, Fossify Calendar around DST changes, AOSP's one-line address edits), and a new Verification section lists what was checked on emulators and the scale run's numbers. --- docs/device-sync.md | 582 +++++++++++++++++++++++++++++++++----------- 1 file changed, 436 insertions(+), 146 deletions(-) diff --git a/docs/device-sync.md b/docs/device-sync.md index ef5ad0dc..b1ae634d 100644 --- a/docs/device-sync.md +++ b/docs/device-sync.md @@ -10,10 +10,11 @@ server. This document is for maintainers. It describes the design: the architecture, what every provider column holds, the mapping tables, the merge rules, the -native API, the failure matrix and the known limitations. Behaviour it relies -on was checked against Stalwart v0.16.23 (the server Bulwark targets) and -AOSP's ContactsProvider, CalendarProvider and the calendar/contacts apps; the -notable findings are quoted where they shape a rule. +native API, the failure matrix, the known limitations and what was checked on +devices. Behaviour it relies on was checked against Stalwart v0.16.23 (the +server Bulwark targets) and AOSP's ContactsProvider, CalendarProvider and the +calendar/contacts apps; the notable findings are quoted where they shape a +rule. ## Contents @@ -33,6 +34,7 @@ notable findings are quoted where they shape a rule. 14. [Testing](#testing) 15. [Decisions and limitations](#decisions-and-limitations) 16. [Out of scope](#out-of-scope) +17. [Verification](#verification) ## Architecture @@ -251,7 +253,7 @@ event of its own. |---|---| | `RawContacts.SOURCE_ID` | identity (above) | | `RawContacts.SYNC1` | the card's selected address books: `/` joined by `,` | -| `RawContacts.SYNC2` | the shadow: the last server card, wire format, with each `data:` photo URI replaced by `sha256:` of its bytes, plus a `~memberOf` member: the group cards whose memberships the rows held after our last write, so a removal on the device and an addition on the server can be told apart (stripped before any projection or patch) | +| `RawContacts.SYNC2` | the shadow: the last server card, wire format, with each `data:` photo URI replaced by `sha256:` of its bytes, plus a `~memberOf` member: the group cards whose memberships the rows held after our last write, so a removal on the device and an addition on the server can be told apart (stripped before any projection or patch), and `~noPhoto: true` when the card's photo has no row on the device because it could not be written | | `RawContacts.SYNC3` | the pending create: `{"uid":…,"target":"/"}`, written before the `/set` and kept until the identity is | | `RawContacts.SYNC4` | poison marker `{"fp":…,"type":…,"n":…,"until":…}` | | `RawContacts.RAW_CONTACT_IS_READ_ONLY` | 1 when none of the card's address books grants `mayWrite`. Only written, and only along with other raw contact writes, so an echo stays write-free: ContactsProvider refuses the column in every projection and selection ("Invalid column", API 32 and 35). Whether device edits are put back is decided from the server's current rights on the stored card | @@ -306,7 +308,9 @@ JSON, versioned. "collectionsState": "s12eq", "itemsState": "syufa", "selected": ["c/b"], + "partial": [], "stale": [], + "created": [], "reconcile": null, "groups": ["c/e12"], "taskOnly": [] @@ -331,21 +335,37 @@ JSON, versioned. refusal that re-planning did not fix) goes on `stale` in the same batch, is fetched again first in every run, and uploads nothing until it has been. The state still advances, so one bad object never blocks the pages after - it. + it. It also names the contacts whose membership a group card just written + changed, until their own chunk ran. +- `created` lists the objects our uploads created after `itemsState`, written + in the batch that stores their identity. The next run fetches them first, + with `stale`: `/changes` from the stored state omits an object that was + created and destroyed since (Stalwart does, as RFC 8620 allows), so a create + that another client destroyed before the next download would otherwise stay + on the device. - `reconcile` marks a full reconcile in progress: `{ "from": , "phase": "ids" | "objects", "position": n, "after": }`, - written with every chunk of objects. `after` is the last id processed in - the reconcile's sorted order: the next run lists the ids again and - continues after it (`phase` and `position` are informative only). `from: - null` asks for a reconcile that has not started (after - `cannotCalculateChanges`, or when local deletions were discarded). Its - deletions are applied only when the complete id list is known, and + first>, "phase": "ids" | "objects", "position": n, "after": , + "collections": [] }`, written with every chunk of objects. `after` is + the last id processed in the reconcile's sorted order: the next run lists + the ids again and continues after it (`phase` and `position` are + informative only). `collections` names the collections it lists, fixed when + it starts; a collection selected while it runs is loaded after it, like any + newly selected one. `from: null` asks for a reconcile that has not started + (after `cannotCalculateChanges`, or when local deletions were discarded). + Its deletions are applied only when the complete id list is known, and `itemsState` becomes `from` when it ends. -- `selected` is the selection the rows were written for; a collection is added - in the batch with its last loaded chunk, and removed with its last deleted - row. +- `selected` names the collections whose rows are all on the device: a + collection is added in the batch with its last loaded chunk, and removed in + a batch of its own before its first row is dropped. `partial` names + collections loaded or dropped only in part: added before the first chunk of + a load or the first row a contacts drop removes (a calendar is dropped with + its calendar row), removed with the last. Selected, such a collection is + loaded again; deselected, it is dropped again. - `groups` (contacts): the group cards present on the device, so a group an app - hard-deleted (Fossify does) is recognised by its absence. + hard-deleted (Fossify does) is recognised by its absence. It changes in the + batch that inserts or deletes the group's row (such a batch ends with a + state op, also on a retry), and a group a download finds unchanged is listed + too. - `taskOnly` lists calendars found to hold only tasks. - `deviceZone` (calendar) is the zone floating events were written in; `deviceZonePending` is set while a zone-change pass runs and cleared, with @@ -375,7 +395,8 @@ deadline reports `cancelled` with `moreRecordsToGet`. for our account ends here with `ok`, before JMAP is opened: ContactsProvider and CalendarProvider schedule an upload sync for every account 30 s after any app write, other accounts' included. A pending - reconcile or objects on `stale` make it a full run. + reconcile, objects on `stale` or objects in `created` make it a full + run. - `new JMAPClient().loadAccount(registryId)`: `false` or `AuthenticationError` → rebuild once (another client may have rotated the refresh token); still failing → `auth`, plus one notification that @@ -395,7 +416,8 @@ deadline reports `cancelled` with `moreRecordsToGet`. never skips changes of the other collections. Deselected collections are dropped in step 5. 3. **Download.** - - Objects on `stale` are fetched again first. + - Objects on `stale`, and the objects our last uploads created + (`created`), are fetched again first. - With a stored `itemsState`: `/changes` with `maxChanges: 256` and a `hasMoreChanges` loop; each page's changed ids are fetched with explicit `properties` in batches of `maxObjectsInGet`. @@ -412,15 +434,25 @@ deadline reports `cancelled` with `moreRecordsToGet`. - Contacts: a page's group cards are written before its contacts. When a group card's `members` changed, its old and new member contacts are fetched in the same page, before the page's state op: their membership - rows change although their own cards did not. + rows change although their own cards did not. They go on `stale` in the + batch that writes the group, so a stop before their chunk leaves them to + be fetched first by the next run. - An object without rows first looks for a local item it may already be: a new row whose pending create targets a collection the object is in and whose pending uid is the object's `uid` (our own create whose identity never got written) is adopted: it takes the identity and shadow and is - merged as a dirty item. + merged as a dirty item. A contact or group the user deleted meanwhile + takes only the identity and shadow, and its deletion then uploads by id. - Each object is [merged](#merge-rules) into its rows. An object that left - every selected collection loses its rows only when they are clean; dirty - units are uploaded first. + every selected collection loses its rows only when they are clean; a + dirty or deleted item is merged like any change first (a deleted one's + shadow follows the server, so its deletion is decided on the server's + version), so its upload builds on the server's version, and its rows go + once the upload is accepted. A full reconcile downloads such objects the + same way. The calendar planner places an event only in a synced + calendar, so an edited event that left all of them is not merged: it goes + on `stale` (reported as `plannerError`) and its changes wait on the + device; a deleted one's deletion is decided on its last shadow. - An object whose rows could not be written (a planner error, a provider refusal that re-planning did not fix) goes on `stale`, and the page's state still advances. @@ -430,11 +462,13 @@ deadline reports `cancelled` with `moreRecordsToGet`. [Uploads](#uploads). 5. **After the upload.** Deselected collections are dropped, and only their clean rows go: dirty items there were uploaded first (they are still valid - objects on the server), and an item whose upload failed keeps its rows, - and its collection stays in `selected`. Rows of objects that left every - selected collection go the same way. Calendar: then the device-zone pass - and the reminder-owner pass rewrite clean events (see [Timing](#timing) - and [Reminder owner](#reminder-owner)). + objects on the server), and an item whose upload failed keeps its rows; + its collection is no longer complete (not in `selected`, `partial` for + contacts), so the next run drops it again, or loads it again when it is + selected again. Rows of objects that left every selected collection go the + same way. Calendar: then the device-zone pass and the reminder-owner pass + rewrite clean events (see [Timing](#timing) and [Reminder + owner](#reminder-owner)). 6. **Report** counts, conflicts, per-item errors and duration; persist the status for the settings UI; call `finishRun` before the task's promise settles (JS timers stop once the task is over). @@ -477,13 +511,15 @@ Every reviewer checks against these. entry it dropped) is never read as a deletion. 2. **No silent device-edit loss.** A dirty row is cleared only after the server accepted the change, and only if it wasn't edited again during the upload: - the clearing batch asserts `RawContacts.VERSION` for contacts and the full - read projection (event row, attendees, reminders) for events, so check and - clear are one provider transaction. A row whose mapped projection equals its - baseline (only STARRED changed, say) is cleared without uploading. No dirty, - deleted or pending row is removed locally (deselection, move-away, disable, - logout, account reconciliation) before its upload succeeded or the user - confirmed losing it. + the clearing batch asserts `RawContacts.VERSION` for contacts and the read + projection for events (every event row, with the attendees and reminders + of the rows that are dirty: a clean row's DIRTY flag guards them, since + CalendarProvider sets it on every app write to an event, its attendees or + reminders), so check and clear are one provider transaction. A row whose + mapped projection equals its baseline (only STARRED changed, say) is + cleared without uploading. No dirty, deleted or pending row is removed + locally (deselection, move-away, disable, logout, account reconciliation) + before its upload succeeded or the user confirmed losing it. 3. **Crash-safe and idempotent.** Every step can be re-run. The JMAP state is written in the same `applyBatch` as the last chunk of rows it describes, as that batch's last op. Creates carry a client-minted `uid`, written to the @@ -562,7 +598,10 @@ not model (DEPARTMENT, labels, a website's type), `IS_PRIMARY`, and every nickname and organization but one; AOSP Contacts updates rows in place. So, per data kind: -1. Empty rows (an editor's blank nickname or note) are ignored. +1. Empty rows (an editor's blank nickname or note) are ignored. A Photo row + is never empty: ContactsProvider keeps a photo within its 96 px thumbnail + as the thumbnail alone, without `PHOTO_FILE_ID`, and the thumbnail does + not read back. 2. Rows whose `DATA_SYNC1` key is in the shadow keep that key. 3. Rows without a usable key are matched to the shadow's unmatched entries by the kind's primary value (email address case-insensitively, phone digits, @@ -579,7 +618,8 @@ per data kind: birthday). When every row of the kind is keyless, an editor rewrote them all, and a missing entry may just be one it could not show: it stays on the server. A missing name row is never a deletion; a missing photo row always - is (no editor re-inserts photo rows). + is (no editor re-inserts photo rows), unless the device never got the + photo (`~noPhoto`). A row matched by its key is compared column by column with its baseline; an emptied column is a real deletion. A row matched by value has no baseline: its @@ -645,31 +685,40 @@ patches only the paths the user changed and the server keeps its own. Whole-object rules: - Edit on the device + delete on the server → **the server delete wins**; the - rows are deleted and counted as a conflict. + rows are deleted and counted as a conflict. A move made on the device as a + delete plus an insert (a pair not uploaded yet) is such an edit: the + inserted row goes with the deleted one. - Delete on the device + edit on the server → **the device delete wins**; the object is destroyed. A deletion is an explicit act; restoring the object - would resurrect something the user removed. + would resurrect something the user removed. The download writes only the + new version into the item's shadow, so the deletion is judged by the + object's current collections: one the server meanwhile also filed in a + collection this device does not sync only leaves the synced ones. - Delete on both → purge, no conflict. - A server object whose `uid` differs from the shadow's is a different object: its rows are replaced, never patched. Every download write into an existing item is guarded: the group asserts the -DIRTY flag and VERSION (contacts) or projection (events) the plan was made from, -and ops that address one row by id expect exactly one row. Exception rows are +DIRTY flag and VERSION (contacts) or projection (events) the plan was made from +(for events, attendees and reminders only for the rows that are dirty), and +ops that address one row by id expect exactly one row. Exception rows are deleted by id only. A failed assert re-reads and re-plans the item (up to 3 -times); after that the item goes on `stale`. +times); after that the item goes on `stale`. A batch that failed as a whole +may have been committed up to a provider yield point: its downloads are then +planned again from a new read rather than sent again, so a committed one +becomes an echo. ## Uploads -The upload runs phase by phase across the JMAP accounts: new rows are claimed -(below), each account resolves its series splits and pairs, then every -account's creates go up, then every account's updates, then every account's -deletions, then (contacts) groups deleted by absence and group memberships. So -a move between accounts never destroys before it created. Changes go in `/set` -calls of at most `maxObjectsInSet` objects (creates: 50) and about 1 MB of -JSON (Stalwart splits bigger calls into several commits, which can apply -partially), one request in flight, and one `sendSchedulingMessages` value per -call (it is a request-level flag). +The upload runs phase by phase across the JMAP accounts: each account uploads +its pairs (below), new rows are claimed, each account resolves its series +splits, then every account's creates go up, then every account's updates, +then every account's deletions, then (contacts) groups deleted by absence and +group memberships. So a move between accounts never destroys before it +created. Changes go in `/set` calls of at most `maxObjectsInSet` objects +(creates: 50) and about 1 MB of JSON (Stalwart splits bigger calls into +several commits, which can apply partially), one request in flight, and one +`sendSchedulingMessages` value per call (it is a request-level flag). - **ifInState.** Every `/set` carries `ifInState` = the item state the merge was based on (then the `newState` of our previous `/set`). A `stateMismatch` @@ -687,21 +736,40 @@ call (it is a request-level flag). `accountNotSupportedByMethod`) holds back only that account: its items wait, the report lists the account, and the other accounts go on. - **Splits.** A `CONTENT_EXCEPTION_URI` split (two masters sharing a - `_SYNC_ID`, see [Recurrence](#recurrence)) is resolved first: the older - master's capped rule is planned while the split is still visible, the newer - row is claimed as a new event, then the rule is sent. The new event is - created with the other creates. + `_SYNC_ID`, see [Recurrence](#recurrence)) is resolved before the creates: + the older master's capped rule is planned while the split is still visible, + the newer row is claimed as a new event, then the rule is sent. The new + event is created with the other creates. - **Pairs.** Some apps turn an edit into a delete plus an insert: Etar moves an event to another calendar that way, and turns a recurring series into a single event (or edits "this and following" from its first instance) the same way. A deleted row and a new row of the same run whose projections match apart from the calendar (or the rule) are uploaded as a patch of the existing object (`calendarIds` swapped; `recurrenceRule: null` with its - overrides removed); the new row takes the identity and exceptions of the old - one, which is purged. Rows of different JMAP accounts never pair: that move - is a create in the target account, identity written, and later in the run a - destroy in the source, whose echo finds no rows left to change. Unpaired - rows are ordinary creates and deletions. + overrides removed); the new row takes the identity and the exception rows + of the old one, which is purged. Rows of different JMAP accounts never + pair: that move is a create in the target account, identity written, and + later in the run a destroy in the source, whose echo finds no rows left to + change. Unpaired rows are ordinary creates and deletions. Pairs go up + before any new row is claimed: the new row takes the old row's identity + instead of a uid of its own, and a paired deletion is a move, which the + deletion threshold does not count. When an app changes the rows between the + patch and the identity write, they are read and paired again; rows that no + longer pair do not get the identity: the object is fetched again first next + run, and the rows upload as a create and a deletion. The new row's baseline + is the old row's, with what the patch changed (the calendar; for a series + turned single the rule and its timing as a single event), so anything else + saved with the move or made before it, a reminder say, keeps the row dirty + and uploads right after the pair as an ordinary update; the pair's accepted + write then waits for that upload. Such a change keeps the device's value + even where the server changed the same field meanwhile: the download of the + deleted row refreshes only its shadow, so no merge sees the conflict. + Attendee rows of the new row are the app's re-insert (Etar writes every + attendee as required, without a status): only their addresses count, for + the pairing. A new row without reminder rows is taken to have copied none, + not to have deleted them. The old row's exception rows (Etar copies only + those it never synced) move to the new row with their pending edits; a + series turned single has none left. - **New rows** (see [Identity](#identity) for what counts as new): 1. Claim: write the pending create (`SYNC3` / `SYNC_DATA3`: the uid and the target collection; for events also `UID_2445` and @@ -713,10 +781,15 @@ call (it is a request-level flag). A server object with that uid does not count, because it may be this row's own earlier create. The target is fixed at the claim: contacts go to the address book chosen in the settings if it is selected and - writable, else to the personal account's default book (or its first by + writable, else to the primary account's default book (or its first by sort order) among the selected writable ones; with none, the item is skipped and reported as `noWritableAddressBook` in every run. Events go - to the calendar of their row. + to the calendar of their row. A claim whose collection is gone from the + server (its account was listed and no longer has it) is made again for + contacts and groups, since nothing of an earlier create can exist there; + an event's create into a calendar that is gone is not sent and is + reported as `collectionGone` until the user moves the event to another + calendar. 2. Look the uid up: contacts in the target address book (card uids are unique per book), events in the whole account (event uids are unique per account). A hit in the create's target collection is adopted: the row @@ -756,19 +829,25 @@ call (it is a request-level flag). uploaded, and DIRTY stays: the newer edit uploads next time, and so does a revert to the old value. For contacts these baseline writes address their data rows with `where _id = ?`, so a row deleted meanwhile does not fail the - group. + group. A group card accepted for a membership patch keeps a rename the + device made that the card does not show: only its shadow is written, and + the rename uploads next. - **SetErrors**: - - `notFound` on update → treated as a remote delete (the rows go); on a - destroy it counts as done; on a create the item is poisoned; + - `notFound` on update → treated as a remote delete (the rows go, and for + a pair the new row too); on a destroy it counts as done; on a create the + item is poisoned; - `forbidden` in a call that asked for scheduling messages → the item is - retried once without them and the report notes "invitations not sent" - (Stalwart refuses scheduling when it is disabled, when the account has no - calendar address, or without the scheduling permission). `forbidden` - otherwise: on a create the item is poisoned (a revert would delete the - user's new item); on an update or a deletion the object is read-only on - the server after all, and its rows are rewritten from the server's - current version, fetched for it (the shadow holds photos only as hashes), - or removed when the object is gone or outside the selection; + retried once without them, and the report notes "invitations not sent" + once that retry is accepted (Stalwart refuses scheduling when it is + disabled, when the account has no calendar address, or without the + scheduling permission). `forbidden` otherwise: on a create the item is + poisoned (a revert would delete the user's new item); on an update or a + deletion the object is read-only on the server after all, and the report + lists the item as `forbidden`: its rows are rewritten in place from the + server's current version, fetched for it, so device-only columns (a star, + a ringtone, joins, row ids) stay; a refused deletion brings the item back + as a new copy, and the rows are removed when the object is gone or + outside the selection; - `stateMismatch` → as above; - `invalidProperties` on `uid` → as above; - `invalidProperties`, `invalidPatch`, `tooLarge`, `overQuota` and the rest → @@ -788,14 +867,17 @@ call (it is a request-level flag). More than 50 local deletions of an authority in one run, and more than 20 % of that authority's synced objects, set `tooManyDeletions` unless the run carries `overrideTooManyDeletions`. Deletions inferred from absence (groups) count. -No deletion of that run is uploaded; everything else proceeds. With -`discardLocalDeletions` (the user chose "undo" in Android's too-many-deletions -notification), the affected JMAP accounts first get a `reconcile` marker in a -batch of its own, then the deleted rows are purged, so the next runs download -the objects again whatever their extras; a crash in between only asks the -question again. A teardown applies the same threshold: deletions above it stay -on the device and count as waiting changes, so the app asks before turning -sync off (see [Lifecycle](#lifecycle)). +No deletion of that run is uploaded; everything else proceeds. The user +resolves it in Android's too-many-deletions notification, or with "Review +deletions" in the app's sync settings, which asks whether to delete the items +on the server too ("Delete on server", a sync with `overrideTooManyDeletions`) +or to bring them back ("Bring them back", a sync with `discardLocalDeletions`). +With `discardLocalDeletions`, the affected JMAP accounts first get a +`reconcile` marker in a batch of its own, then the deleted rows are purged, so +the next runs download the objects again whatever their extras; a crash in +between only asks the question again. A teardown applies the same threshold: +deletions above it stay on the device and count as waiting changes, so the +app asks before turning sync off (see [Lifecycle](#lifecycle)). ## Contacts mapping @@ -833,11 +915,11 @@ as "Ada King" on the device sends both. | JSContact | Android kind | Columns | |---|---|---| | `nicknames/` | Nickname | `NAME`, `TYPE=DEFAULT` | -| `emails/` | Email | `ADDRESS`; `TYPE` from contexts: `private`→HOME, `work`→WORK, none→OTHER, `label`→CUSTOM+`LABEL`; `pref=1`→`IS_PRIMARY` | +| `emails/` | Email | `ADDRESS`; `TYPE` from contexts: `private`→HOME, `work`→WORK, none→OTHER, `label`→CUSTOM+`LABEL` (`mobile`→MOBILE, which uploads that label); `pref=1`→`IS_PRIMARY` | | `phones/` | Phone | `NUMBER`; `TYPE` from features + contexts (table below); `label`→CUSTOM+`LABEL`; `pref=1`→`IS_PRIMARY` | | `addresses/` | StructuredPostal | components → `STREET`, `POBOX`, `NEIGHBORHOOD`, `CITY`, `REGION`, `POSTCODE`, `COUNTRY` (table below), `full`→`FORMATTED_ADDRESS` (a card without `full` gets a one-line address joined from the components, so the provider leaves the row as written); contexts → TYPE HOME/WORK/OTHER, `label`→CUSTOM | | `organizations/` + `titles` pointing at it | Organization | `name`→`COMPANY`, `units` joined with `, `→`DEPARTMENT`, title with `kind: title`→`TITLE`, title with `kind: role`→`JOB_DESCRIPTION`; key `organizations:|titles:,`; titles without an organization get their own row | -| `links/` | Website | `URL` ← `uri`; contexts `private`→HOME, `work`→WORK, else OTHER; `label`→CUSTOM | +| `links/` | Website | `URL` ← `uri`; contexts `private`→HOME, `work`→WORK, else OTHER; `label` `homepage`/`blog`/`profile`/`ftp`→HOMEPAGE/BLOG/PROFILE/FTP (they upload that label), any other `label`→CUSTOM | | `anniversaries/` | Event | `START_DATE`: `YYYY-MM-DD`, `--MM-DD` without year (`partialDateToString`); kind `birth`→BIRTHDAY, `wedding`→ANNIVERSARY, `other`→OTHER, anything else → CUSTOM + its kind as `LABEL`; a custom type uploads its label, lowercased, as the kind | | `relatedTo/` | Relation | `NAME` ← the key (a `urn:uuid:` of a synced card shows that card's name), `TYPE` from the relation set (table below) | | first entry of `notes` | Note | `NOTE` ← `note` | @@ -858,11 +940,17 @@ Phone types (first match wins; reverse: the table read right to left, and | pager + work | WORK_PAGER | | pager | PAGER | | textphone | TTY_TDD | +| main-number + work | COMPANY_MAIN | | main-number | MAIN | | work (voice) | WORK | | private (voice) | HOME | | anything else | OTHER | +CALLBACK, CAR, ISDN, RADIO, TELEX and ASSISTANT upload as `voice` with the +label `callback`, `car`, `isdn`, `radio`, `telex` or `assistant`, and MMS as +`text` with the label `mms`; such a label next to that feature reads back as +the type (a custom label uploads without the feature and stays CUSTOM). + Postal components (reverse: an edited field replaces all components it was built from with one component of the first kind, keeps the rest): @@ -878,7 +966,16 @@ built from with one component of the first kind, keeps the rest): Legacy flat address fields (`street`, `locality`, …) are read when there are no components and never written. When an editor wrote only the formatted address -and the provider copied it into `STREET`, only `full` uploads. +(AOSP Contacts edits it as one line), the provider copies it into `STREET` and +empties the other parts. An unchanged line uploads nothing. A changed line +uploads as `full` with the parts it implies, read back along the old line: +each line or comma-separated segment of the old one must be some of the old +parts in a row, and a segment whose change is confined to one part (the parts +around it keep their words) rewrites that part as an edit of its field would. +When the layout changed or a change spans two parts, the address keeps the +whole line as its street (`name`) component. Either way `full` and the +components agree; an address without components would show empty in the +app's editor and in CardDAV clients. Relations: `friend`↔FRIEND, `spouse`↔SPOUSE, `child`↔CHILD, `parent`↔PARENT, `sibling`→RELATIVE, `kin`→RELATIVE, `co-worker`/`colleague`→CUSTOM with the @@ -912,9 +1009,15 @@ with the detached client. Upload: the display photo is read back, scaled to at most 512 px, JPEG, and sent as a `data:image/jpeg;base64,…` URI in `media//uri`, matching the app's picker. `DATA_SYNC2` holds the hash of the server photo last applied, the baseline holds `PHOTO_FILE_ID`; a new file -id means the user changed the photo. Photos travel in batches of their own -(Binder budget). A photo whose bytes could not be fetched keeps its row and -counts as unchanged. +id means the user changed the photo. A photo within the 96 px thumbnail has no +file id, so replacing it on the device with another photo that small is not +seen. Photos travel in batches of their own (Binder budget). A photo larger +than 700,000 base64 characters (about 512 KiB) is not written: a contact's +rows travel in one provider batch, one Binder transaction of at most 1 MB. +Such a photo, and one whose bytes could not be fetched, keeps an existing row +and counts as unchanged; without a row the shadow's `~noPhoto` records it, so +the missing row is not read as a deletion, and the photo stays on the server +until the device sets one of its own. Groups: group cards are written before the contacts of the same download, and membership rows name their group by row id (`GROUP_ROW_ID`, looked up by the @@ -924,15 +1027,30 @@ membership whose group has no row is not written. A contact's memberships are merged from three sets: what the rows held after our last write (`~memberOf` in the shadow), the rows now, and the group cards that list the card's uid now; both sides can only flip a membership the same way, so memberships never -conflict. Membership edits upload after the deletions, as patches of the -group cards' `members` (`members/: true` or `null`; the whole map for a -group without members). A contact stays dirty until the server's group cards -show its membership edits (the clean and accepted writes keep DIRTY); the -next run's clean write clears it. +conflict. Membership edits, also those of a contact created on the device +(after its create), upload after the deletions as patches of the group cards' +`members` (`members/: true` or `null`; the whole map for a group without +members). Edits for a group on `stale` wait until it has been fetched again. A +contact stays dirty until the server's group cards show its membership edits +(the clean and accepted writes keep DIRTY); once the run's membership patches +are accepted, a contact with nothing else to upload is cleaned in the same +run. A membership edit the server can't take, of a group in read-only address +books or of another JMAP account, does not wait: the contact's next clean or +accepted write puts it back to what the server has, like an edit of a +read-only contact, and when that was the contact's only change the upload +counts it as skipped and the report lists the contact as `groupNotWritable`. A group deleted on the device is found by `DELETED`, or by its absence from the rows while `SyncState.groups` still lists it (Fossify hard-deletes groups -through a sync-adapter URI); both count toward the deletion threshold. +through a sync-adapter URI); both count toward the deletion threshold. An +absent group whose card is no longer a group, or is in no address book that +syncs, is only dropped from the list: it is not one the device showed. + +A contact or group whose address books are all read-only takes no device +changes: an edit (a group's rename) is put back from the shadow, a deletion +is purged and fetched again, and the report lists either as `readOnly`. +Joining a writable group is no edit of the contact: it uploads through that +group's card. ## Calendar mapping @@ -941,7 +1059,7 @@ through a sync-adapter URI); both count toward the deletion threshold. | JMAP `Calendar` | Android `Calendars` | |---|---| | `id` | `_SYNC_ID` = `/` | -| `name` | `NAME`, `CALENDAR_DISPLAY_NAME`; a shared account's calendars show ` ()` (the JMAP account's name, unless it is one of the user's addresses), the personal account's get no suffix | +| `name` | `NAME`, `CALENDAR_DISPLAY_NAME`; a shared account's calendars show ` ()` (the JMAP account's name, unless it is one of the user's addresses), the primary account's get no suffix | | `color` | `CALENDAR_COLOR` (CSS color → ARGB; absent → the app's default palette colour) | | `myRights` | `CALENDAR_ACCESS_LEVEL`: `mayWriteAll` → OWNER (700); `mayWriteOwn` → CONTRIBUTOR (500); `mayRSVP` → RESPOND (300); else READ (200); subscribed iCal feed → READ | | the user's calendar address | `OWNER_ACCOUNT` (never null: Etar crashes on it) | @@ -959,10 +1077,18 @@ first uploads the calendar's dirty events. A read-only calendar (without `mayWriteAll` and `mayWriteOwn`, or a subscribed feed) takes no device changes: a dirty event is rewritten from its shadow, a deleted one is purged and fetched again, a new one is removed. The one -exception is the user's own RSVP on the master -(`participants//participationStatus`), which uploads when the calendar -grants `mayRSVP`. Edits of an event Stalwart holds only as an instance (no -series) are reverted too, in any calendar. +exception is the user's own answer, to the series +(`participants//participationStatus`) or to occurrences (their +overrides' participants, changed in nothing but the user's status), which +uploads when the calendar grants `mayRSVP`. Stalwart 0.16.23 refuses it +anyway (`forbidden`: its `/set` requires write rights in a shared calendar); +the rows are then rewritten from the server and the item is reported +(`forbidden`). Edits of an event Stalwart holds only as an instance (no +series) are reverted too, in any calendar. Every change put back this way is +listed in the run's report: `readOnly` (an edit, a deletion or a new event in +a read-only calendar, or a move into one), `rsvpRefused` (an answer where the +calendar takes none), `instanceOnly`, and `crossAccountMove` for an event an +app moved in place to another account's calendar. ### Timing @@ -1033,12 +1159,12 @@ Device-side changes map back as follows: | exception row inserted (Etar "this event": `ORIGINAL_SYNC_ID` + `ORIGINAL_INSTANCE_TIME`, `DIRTY=1`) or edited | `recurrenceOverrides/`; the key is `ORIGINAL_INSTANCE_TIME` in the master's zone (a date at midnight for all-day). A new override carries the master's title, locations, participants and sequence (Stalwart stores a bare override as a separate event otherwise); an existing one is patched one level deep (`recurrenceOverrides//`), never deeper: Stalwart turns a deeper pointer into a partial override (other attendees vanish for that instance) | | exception row with `STATUS_CANCELED` (Etar and Fossify "delete this event"), a deleted exception row, or an EXDATE added to the master | `recurrenceOverrides/: {excluded: true}` | | EXDATE removed | `recurrenceOverrides/: null` | -| master `RRULE` capped with `UNTIL`/`COUNT` + a new event without identity (Etar "this and following") | the master uploads only `recurrenceRule` (whole object) with the new `until`/`count` and removes the overrides after the new end (`recurrenceOverrides/: null`; for a `COUNT` the end is the start of the last instance, CalendarProvider's `LAST_DATE` − `DURATION`); the new series is a row of its own, claimed and created as a new event with a fresh uid | +| master `RRULE` capped with `UNTIL`/`COUNT` + a new event without identity (Etar "this and following") | the master uploads only `recurrenceRule` (whole object) with the new `until`/`count` and removes the overrides after the new end (`recurrenceOverrides/: null`; for a `COUNT` the end is the start of the last instance, CalendarProvider's `LAST_DATE` − `DURATION`); the new series is a row of its own, claimed and created as a new event with a fresh uid; exclusions Etar copied from the old series' EXDATE that fall before the new series' start are left out | | master start moved (Etar "all events") | `start`/`duration`/`timeZone`, and every override key moved by the same local offset (overrides whose `start` equalled their old key move too), sent as the whole `recurrenceOverrides` map | -| two masters with the same `_SYNC_ID` (a `CONTENT_EXCEPTION_URI` split clones `_SYNC_ID`, `SYNC_DATA*` and `UID_2445`, and caps the old master's rule without `DIRTY`) | both masters are always loaded together. The download merges into the older master as a dirty item, leaves the newer one alone and heals no baseline of either. The upload resolves the split in its first step: the older master's current RRULE is uploaded, and the newer row is claimed as a new event with a fresh uid; that claim points the source's exception rows' `ORIGINAL_SYNC_ID` back at the source and sets the source `DIRTY=1`, so its capped rule uploads | +| two masters with the same `_SYNC_ID` (a `CONTENT_EXCEPTION_URI` split clones `_SYNC_ID`, `SYNC_DATA*` and `UID_2445`, and caps the old master's rule without `DIRTY`) | both masters are always loaded together. The download merges into the older master as a dirty item, leaves the newer one alone and heals no baseline of either. The upload resolves the split before the creates: the older master's current RRULE is uploaded, and the newer row is claimed as a new event with a fresh uid; that claim points the source's exception rows' `ORIGINAL_SYNC_ID` back at the source and sets the source `DIRTY=1`, so its capped rule uploads | | a new master carrying another master's `UID_2445` (an app's "duplicate" copies it) | a clone without a source: claimed with a fresh uid and created as a new event, never adopting the other master's object (the engine loads every master that shares a new master's `UID_2445`) | | event moved to another calendar, or a series turned into a single event (Etar deletes and re-inserts both) | paired with the deleted row and uploaded as a patch of the same object (see [Uploads](#uploads)); never across JMAP accounts, where the move is a create in the target account and a destroy in the source | -| `CALENDAR_ID` changed in place to a calendar of another JMAP account | not uploaded; the next download of the object puts the row back | +| `CALENDAR_ID` changed in place to a calendar of another JMAP account | not uploaded: the row is put back into its calendar, rewritten from the shadow and reported as `crossAccountMove` (changes made with it upload, and their accepted write puts it back) | | an RSVP for one instance (Etar goes through `CONTENT_EXCEPTION_URI`) | the override's whole `participants` map with the user's new status. An exception row's status uploads only when the user organizes the event: Etar also puts `STATUS_CONFIRMED` on the instance, and Stalwart refuses an attendee's change to an event's status | On an exception row an app created, an empty title, description, location or @@ -1046,6 +1172,13 @@ colour, or no attendee or reminder rows, means "inherit from the series", never a deletion. An override never carries `privacy`: Stalwart drops it from overrides. +On an exception row device sync wrote, clearing the description or the +location clears it for that occurrence. An override without them shows the +series' ones, so while the series has a description or location the override +gets the empty text, or one location without a name (Stalwart writes no +LOCATION line for an empty map, which would read back as the series' +locations); otherwise the override's own value is removed. + The first override of an event with none sends the whole `recurrenceOverrides` map (a patch below a missing property fails). @@ -1056,7 +1189,7 @@ map (a patch below a missing property fails). | `title` | `TITLE` (exception rows always carry the master's title: Fossify Calendar treats an exception with an empty title as deleted) | | `description` | `DESCRIPTION`; `descriptionContentType: text/html` is written as plain text and uploaded only when edited, then as `text/plain` | | first entry of `locations` (server order) | `EVENT_LOCATION` ← `name`; edit patches `locations//name`, a new one adds `locations/`, clearing deletes the entry | -| `status` confirmed / tentative / cancelled | `STATUS` 1 / 0 / 2; absent → 1. Never NULL (the provider throws) | +| `status` confirmed / tentative / cancelled | `STATUS` 1 / 0 / 2; absent → 1. Never written NULL (the provider throws). A NULL STATUS (CalendarProvider has no default; Google Calendar leaves it NULL on the events and exceptions it inserts) sets no status: a create leaves `status` out, an override carries none, and it is never a change or a reason not to pair | | `freeBusyStatus` busy / free | `AVAILABILITY` 0 / 1; device TENTATIVE (2) uploads as `busy` | | `privacy` absent / public / private / secret | `ACCESS_LEVEL` DEFAULT / PUBLIC / PRIVATE / CONFIDENTIAL | | `color` | `EVENT_COLOR` (CSS name or `#rrggbb` → ARGB; upload `#rrggbb`); no Colors table in v1 | @@ -1065,7 +1198,7 @@ map (a patch below a missing property fails). | — | `HAS_ATTENDEE_DATA=1`; `SELF_ATTENDEE_STATUS` is never written (the provider derives it and throws on update) | | alerts with an offset before start (`relativeTo` start or absent, offset ≤ 0) | Reminders: `MINUTES` = −offset, `METHOD` = EMAIL for `action: email`, else ALERT | | alerts relative to the end, absolute alerts, offsets after the start | preserved, not shown | -| `useDefaultAlerts: true` | the calendar's `defaultAlertsWithTime`/`WithoutTime` become the reminders (as the app's own scheduler does); a device edit uploads `useDefaultAlerts: false` plus explicit `alerts` | +| `useDefaultAlerts: true` | the calendar's `defaultAlertsWithTime`/`WithoutTime` become the reminders (as the app's own scheduler does); a device edit uploads `useDefaultAlerts: false` plus explicit `alerts`; an edit of one occurrence's reminders does so in that occurrence's override only (Stalwart keeps an override's `useDefaultAlerts` as a JSPROP), so the series and its other occurrences stay on the defaults | | `uid` | `UID_2445` | | `keywords`, `categories`, `priority`, `virtualLocations`, `links`, `relatedTo`, `sequence`, `locale`, `replyTo`, locations after the first, participants without a `mailto:` address | preserved only | @@ -1170,7 +1303,7 @@ nested maps. Every provider call is scoped to one account of our type. | `setSyncEnabled(name, authority, on)` | syncable = 1 and `setSyncAutomatically` | | `setPeriodicSync(name, authority, seconds)` | replaces our periodic syncs (including the framework's default daily one); 0 removes them; the framework clamps to 15 min | | `requestSync(name, authority, optionsJson)` | `{manual, expedited, upload, overrideTooManyDeletions, discardLocalDeletions}` as SYNC_EXTRAS_*; constant extras so repeated requests coalesce; flags an active run for `fullSyncRequested` | -| `openAccountSettings(name)` | the account's sync screen (`android.settings.ACCOUNT_SYNC_SETTINGS` with the Account), else `SYNC_SETTINGS` filtered to our type | +| `openAccountSettings(name)` | the account's sync screen (`android.settings.ACCOUNT_SYNC_SETTINGS` with the Account as the `account` extra and in `:settings:show_fragment_args`: the Settings app of Android 12L and later passes only those arguments on to the screen, which closes at once without an account), else `SYNC_SETTINGS` filtered to our type | | `finishRun(runId, reportJson)` / `isRunCancelled(runId)` | the run handshake | | `setPushRoutes(json)` | `{ jmapAccountId: [{ accountName, authorities }] }` for the push router, in SharedPreferences | | `showSyncProblem(name, title, text, uri, channelName)` / `clearSyncProblem(name)` | the account's one sync-problem notification; a tap opens `uri` (`bulwarkmobile://` only) in the app; `channelName` names the channel in the app's language | @@ -1205,6 +1338,12 @@ Scoping, enforced in Kotlin for every op - attendees, reminders and extended properties get `event_id IN` the account's events among those the op touches (resolved first; the providers may compile selections strictly, so no subqueries). +- Selections and sort orders are checked before the account condition is + ANDed on as `(where) AND (scope)`: their parentheses must balance outside + quoted literals (`1) OR (1` would close the scope's own), and they may not + end the statement, carry a comment or use `SELECT`, `UNION`, `ATTACH`, + `DETACH` or `PRAGMA` (`scope` otherwise). The engine sends only constant + selections with `?` arguments; the check guards against bugs. - Ops by `id` verify the row belongs to the account first (`scope` otherwise) and expect exactly one row unless they set `expectCount`, so a row that vanished fails with `assert`. `id` and `where` together address @@ -1217,8 +1356,11 @@ Scoping, enforced in Kotlin for every op parent (a back-reference to an insert of the same batch, or an id); an insert or update naming a parent, exception master (`original_id`) or group (`data1` of a membership row) checks it: gone fails with `assert` (re-read - and re-plan), another account's with `scope`. A back-reference must point - to an earlier insert of the right table; settings inserts return no id. + and re-plan), another account's with `scope`. An update of a data row by id + that writes `data1` without restating the mimetype has the row's mimetype + looked up, so a membership's group is checked there too; such a write by + `where` is refused (`scope`). A back-reference must point to an earlier + insert of the right table; settings inserts return no id. - Inserts into the tables with account columns (raw contacts, groups, settings, calendars, colors) get `account_name` and `account_type` added, so calendar inserts carry them too. Otherwise @@ -1227,12 +1369,18 @@ Scoping, enforced in Kotlin for every op - Tables are an enumerated set per authority; anything else is refused (`scope`). - Blobs (`{ b64 }`) are only accepted for `data.data15` (photos). + `data.data14`, a photo row's file id, is never written (`scope`): the + provider sets it, and another account's file id would let `readPhoto` read + that account's photo. - A `syncState` op must be the last op of its batch and not `yieldAllowed`, so a provider yield never commits the state ahead of the rows (refused as `scope` otherwise). -- Batches: at most 499 ops up to and including each yield point +- Batches: for contacts, at most 499 ops up to and including each yield point (ContactsProvider counts an op before resetting at a yield point and throws - at 500), about 300 KB per call; `TransactionTooLargeException` resolves as + at 500), so the SyncState op goes in a batch of its own after a group of + 499 ops or more; about 500 KB per call as `estimateBatchBytes` weighs it (2 + bytes per character of the ops' JSON plus 300 bytes per op: the Parcel + carries UTF-16 strings); `TransactionTooLargeException` resolves as `tooLarge` so JS splits the batch. `OperationApplicationException` resolves as `assert` (except the provider's own "too many operations" and failed inserts, `provider`), `SecurityException` as `permission`. The whole batch @@ -1255,10 +1403,11 @@ What the providers do with values, which the planners account for: `device-sync:v1` with `createPersistStorage({ writeDelayMs: 0 })` (not the 1-second delayed storage) and not part of the settings export. Per account: `contactsSelection` / `calendarSelection` (explicit choices only; unlisted -collections follow the default: the personal account's on, shared accounts' -off, and the webmail's "Trusted Senders" address book off too: it is the -allow-list of senders whose images load, not an address book people expect -in their phone's contacts), `newContactsAddressBook`, `reminderOwner`, +collections follow the default: those of the capability's primary account +on, every other account's off (also one the server marks as personal), and +the webmail's "Trusted Senders" address book off too: it is the allow-list of +senders whose images load, not an address book people expect in their +phone's contacts), `newContactsAddressBook`, `reminderOwner`, `intervalSeconds` per authority, the last `RunStatus` per authority, `androidAccountName` (the Android account it uses), `knownStates` (the latest states the engine synced, per JMAP account and type, for the echo skip), @@ -1277,16 +1426,19 @@ native module present). Settings → Contacts and Settings → Calendar each get - an on/off switch (reflects `getSyncAutomatically`; "Paused in Android settings" when the account exists but the authority is off there; a hint when the device's master auto-sync is off); -- a chooser (sheet) for address books or calendars, with personal ones on and - shared ones (and Trusted Senders) off by default, read-only ones marked, and - a warning when another signed-in account already syncs the same shared - collection; +- a chooser (sheet) for address books or calendars, with the primary + account's on and shared ones (and Trusted Senders) off by default, + read-only ones marked, and a warning when another signed-in account already + syncs the same shared collection; - contacts: the address book for new contacts; - calendar: who reminds you (calendar app or Bulwark); - the interval: 15 min, 30 min, 1 h (default), 6 h, manual; - "Sync now" (`requestSync` with manual + expedited); -- the last sync time and its result, or the error with an action ("Sign in - again", "Grant access", "Review deletions"); +- the last sync time (its "… ago" refreshed every 30 s while the section is + open) and its result, or the error with an action ("Sign in again", "Grant + access", "Review deletions", which asks whether to delete the items on the + server too or bring them back; see [Deletion + threshold](#deletion-threshold)); - "Android account settings" (`openAccountSettings`). Turning sync on asks for `READ/WRITE_CONTACTS` or `READ/WRITE_CALENDAR` after a @@ -1306,16 +1458,22 @@ app's system settings when Android no longer asks. deleted and new items (bounded to 30 s; it is not a sync run, so the "disabled" preflight does not stop it), with the [deletion threshold](#deletion-threshold) applied, and never uploads rows written for - another registry account or server (SyncState `owner`). Then it deletes the - authority's rows and clears the SyncState by writing `''`. When items could - not be uploaded (offline, errors, deletions above the threshold), nothing is - deleted and the app asks: "N changes made on this device haven't reached - the server. Turn off anyway?" Turning off anyway still makes one short - upload attempt (5 s) before the rows go; "Keep syncing" turns automatic sync - back on. With nothing waiting the rows go without a question: the server - keeps everything. When both authorities are off the Android account is - removed. Turning calendar sync off also forgets who reminds of synced - events, so the question comes again the next time. + another registry account or server (SyncState `owner`). It downloads + nothing first: its uploads carry the stored state (`ifInState`), so a + server change makes it download before it decides; for an account whose + first sync never ended (no stored state) only new items go up, and the rest + counts as waiting. Then it deletes the authority's rows and clears the + SyncState by writing `''`. When items could not be uploaded (offline, + errors, deletions above the threshold), nothing is deleted and the app + asks: "N changes made on this device haven't reached the server. Turn off + anyway?" Turning off anyway still makes one short upload attempt (5 s) + before the rows go; "Keep syncing" turns automatic sync back on. With + nothing waiting the rows go without a question: the server keeps + everything. When both authorities are off the Android account is removed. + Turning calendar sync off also forgets who reminds of synced events, so the + question comes again the next time. Turning sync on, off and back on for + one app account runs one step at a time: a turn-on made while a turn-off + still tears down waits for it, then sets everything up again. - **Logout** (`auth-store.logout`, `logoutAll`, `removeAccount`): the same as disabling both, before the credentials go, with the same question when changes would be lost; "stay signed in" keeps the accounts syncing. Several @@ -1340,7 +1498,7 @@ app's system settings when Android no longer asks. on in Android Settings is adopted as on, one switched off there shows as "Paused in Android settings" (not "Off"), and "Sync now" is disabled while paused, since a manual run would only report `disabled`. `suspended` marks - an evicted account, `primaryJmapAccounts` the personal JMAP account per + an evicted account, `primaryJmapAccounts` the primary JMAP account per authority (push routes and the trigger filter). - An Android account that exists without a registry id (created, then the app died before writing it) is adopted by the next `ensureAccount`; one @@ -1404,13 +1562,14 @@ app's system settings when Android no longer asks. | Password revoked | `auth` (hard, no retry) + "Sign in again" | user signs in; the next trigger syncs | | Permission revoked | Android fails the sync (`databaseError`) | "Grant access" in settings | | `cannotCalculateChanges` | full reconcile of that JMAP account | automatic | -| Server `SetError` on an item | notFound on an update → local delete; forbidden with scheduling → retried without it ("invitations not sent"); forbidden on an update or deletion → rows rewritten from the server's current version; forbidden on a create, and the rest → poisoned with back-off, listed in the report | automatic / user fixes the item | +| Server `SetError` on an item | notFound on an update → local delete (a pair's new row too); forbidden with scheduling → retried without it ("invitations not sent" once that is accepted); forbidden on an update → rows rewritten in place from the server's current version, on a deletion → the item comes back from the server, both reported as `forbidden`; forbidden on a create, and the rest → poisoned with back-off, listed in the report | automatic / user fixes the item | +| A device change the server can't take (a read-only collection, an answer where none is allowed, an instance without its series, an in-place move to another account's calendar, a membership of a group the device can't change) | put back on the device without a request, listed in the report as `readOnly`, `rsvpRefused`, `instanceOnly`, `crossAccountMove` or `groupNotWritable` | — | | A whole `/set` refused for an account (`accountReadOnly`, `forbidden`, `accountNotFound`, `accountNotSupportedByMethod`) | that account's items wait, the report lists the account; the other accounts go on | automatic once the account takes writes | | A shared account refuses to list its address books or calendars (`forbidden`, `accountNotFound`, `accountNotSupportedByMethod`) | the account is left out of the run and only logged; its rows stay untouched | automatic once it lists them again | | A planner throws on an item | download: the object goes on `stale`; upload: the item is skipped (`plannerError`); the rest of the run goes on | fixed in code; listed in the report | | `stateMismatch` or `serverUnavailable` on a `/set` | re-download, retry up to 3 times, then `io` without progress | automatic (back-off) | | 429 | `io` with `delayUntil` = Retry-After (60 s without one) | automatic | -| Too many local deletions | `tooManyDeletions` + system notification | user picks delete or undo | +| Too many local deletions | `tooManyDeletions` + system notification | user picks delete or undo (in the notification, or "Review deletions" in the settings) | | Empty server on a full reconcile | `safetyAbort`, device untouched | user checks the server; turning sync off and on starts over | | Edit during an upload | clearing assert fails → DIRTY stays, shadow updated | next run uploads the newer edit | | JS crash or reload mid-run | task ends without report → soft error | automatic | @@ -1429,11 +1588,21 @@ app's system settings when Android no longer asks. in flight), `maxSizeRequest` (10 MB; the engine targets 1 MB), queries of at most 5000 ids per call, `/changes` pages of 256 (Stalwart caps at 5000), cards and events of at most 512 KiB. -- Provider: ≤ 499 ops between yield points (the engine packs at most 400 per - batch), ~300 KB per `applyBatch`, photos in their own batch; 50 raw contacts +- Provider: ≤ 499 ops between yield points in ContactsProvider (the engine + packs at most 400 per batch; CalendarProvider has no such limit), about + 500 KB per `applyBatch` as `estimateBatchBytes` weighs it, photos in their + own batch (photos over about 512 KiB stay on the server); 50 raw contacts or 50 events per chunk. +- A recurring meeting with many attendees on many occurrences is big: 25 + attendees on 52 overrides weigh about 2 MB of provider ops on its first + download (the shadow alone about 0.56 MB), more than one Binder transaction + (1 MB). Writes after an edit take about 0.65 MB. The engine writes an event + in one group, so such an event does not reach a device that downloads it + for the first time: it stays on `stale`. - JS thread: a `setTimeout(0)` yield between chunks, so the UI keeps rendering - when the app is open; JSON crosses the bridge as strings. + when the app is open (a teardown, which the UI starts, yields without + timers: React Native pauses them while the app is in the background and no + headless task runs); JSON crosses the bridge as strings. - Time: 9.5 min per run (2.5 min expedited), checkpoints between chunks, continuation through `fullSyncRequested`. Checkpoints stop 30 s before the payload's deadline and a create batch needs 60 s left (see [A sync @@ -1443,7 +1612,7 @@ app's system settings when Android no longer asks. group's members, reads every synced contact's shadow (`SYNC2`) once per run to index the cards by uid. -The 2,000 + 2,000 scale run is recorded in the parity notes when #34 closes. +The 2,000 + 2,000 scale run is in [Verification](#verification). ## Testing @@ -1454,18 +1623,19 @@ The 2,000 + 2,000 scale run is recorded in the parity notes when #34 closes. attendees; merge rules; the import-graph purity test. - **Engine** (vitest): `src/device-sync/__tests__/fakes/fake-provider.ts`, an in-memory ProviderPort with DIRTY/DELETED/VERSION, soft/hard deletes, - asserts, back-references, yield points and user-edit helpers (AOSP-style - in-place edits and Fossify-style delete+reinsert); and a fake JMAP server + asserts, back-references, yield points (and a batch committed up to a yield + point before it fails) and user-edit helpers (AOSP-style in-place edits and + Fossify-style delete+reinsert); and a fake JMAP server for ContactCard/CalendarEvent/AddressBook/Calendar with `/changes`, `cannotCalculateChanges`, `ifInState`, SetError injection, the async uid index and Stalwart's key behaviour. Crash injection at every checkpoint; poison items; the deletion guards; echo suppression; an edit during upload. - **Kotlin** (JUnit, `gradlew :app:testDebugUnitTest`): report mapping, op and URI translation, scoping. -- **Device** (API 34 and 32 emulators, local Stalwart only): the scenario matrix - in the #34 parity note, verified on three sides: the provider (`adb shell - content query …`), the server (JMAP calls) and the apps' UI. Debug builds - create accounts without UI: +- **Device** (API 34 and 32 emulators, local Stalwart only): the scenarios in + [Verification](#verification), checked on three sides: the provider (`adb + shell content query …`), the server (JMAP calls) and the apps' UI. Debug + builds create accounts without UI: ``` adb shell am broadcast -n com.anonymous.bulwarkmobile/.sync.DeviceSyncDebugReceiver \ @@ -1523,13 +1693,34 @@ Known limitations: include that instance in the old series again (better than Stalwart dropping the `UNTIL`). - A single occurrence that Stalwart holds only as an instance (no master) is - shown but read-only. -- In a calendar that grants RSVP but no writes, only the user's answer to the - whole series uploads; an answer for one instance is reverted. + shown but read-only: an edit is put back and reported as `instanceOnly`. +- Stalwart refuses every change to an event in a calendar shared without + write rights, even the user's own answer to an invitation (to the series or + to one occurrence). Where the calendar grants RSVP the sync sends the + answer; after the refusal (`forbidden`, also on the retry without + scheduling messages) it rewrites the rows from the server's version, so the + answer is put back on the device, and the report lists the item as + `forbidden`. Where the calendar grants no RSVP, the answer is put back + without a request and reported as `rsvpRefused`. +- When the organizer of an event is one of the user's own identities (for + example a shared mailbox the user may send as, which Stalwart lists among + the user's identities), the device shows the user as the organizer: no + answer buttons, and the user's own attendee entry is folded into the + organizer's row. Every identity address counts as "me" + (`calendarAddresses` in `src/device-sync/jmap/session.ts`, the address keys + in `src/device-sync/calendar/attendees.ts`). - Moving an event to a calendar of another JMAP account uploads as a create in the target account and a destroy in the source (a new uid; participants are notified accordingly). An app that moves it by changing `CALENDAR_ID` in - place gets the row put back by the next download of the event. + place gets the row put back at the next sync (reported as + `crossAccountMove`). +- Fossify Calendar shows the occurrences of a zoned weekly series one hour + off after a daylight-saving change (it expands the rule without the zone's + DST change); Google Calendar and Etar show them right. The rows are + correct. +- A recurring meeting too big for one provider transaction on its first + download (about 25 attendees on 52 overrides) does not reach the device; + see [Limits and performance](#limits-and-performance). - Descriptions over 64 KB are shown truncated and are not uploaded when edited. - A shared address book or calendar that two signed-in accounts can both see appears once per account when both sync it (each Android account holds its @@ -1537,7 +1728,22 @@ Known limitations: - Floating events follow the device zone; alerts relative to the end, absolute alerts and alerts after the start are kept on the server but not shown. - Photos travel as 512 px JPEG `data:` URIs; a larger original is replaced by - its scaled copy only when the user changes the photo on the device. + its scaled copy only when the user changes the photo on the device. A photo + over about 512 KiB is not shown on the device (it stays on the server), and + replacing a photo within the 96 px thumbnail by another one that small on + the device is not uploaded. +- AOSP Contacts edits an address as one formatted line. The sync reads the + parts back along the old line; when the edit changes the line's layout or + spans two parts, the server's address keeps the whole line as its street: + `full` and the components agree, but city, postcode and country are no + longer separate parts. +- The settings show the status of the last run only, and an upload sync the + providers scheduled after an app write replaces it even when it finds + nothing to do: the conflicts and item errors of the run before can + disappear from the section moments after it. +- Read-only address books and calendars and subscribed calendars were not + verified on a device (the test server had none); their handling is covered + by tests only. Open: @@ -1560,3 +1766,87 @@ Possible follow-ups: - **Importing device-local contacts** into Bulwark. - An account-scoped Colors table so calendar apps can offer event colours. - Re-basing old daily/hourly series past AOSP's 2000-period expansion cap. + +## Verification + +Checked on Android emulators against a local Stalwart 0.16.23, each time on +three sides (provider rows, server objects, the apps' screens): contacts on +API 32 with AOSP Contacts and Fossify Contacts, calendars on API 34 with +Google Calendar, Etar and Fossify Calendar. What these runs found in other +apps or in Stalwart is under [known limitations](#decisions-and-limitations). + +- **Contacts.** Every mapped field of a card (name parts, nickname, + organization with department and title, typed and labelled emails, phones + and addresses, a website, a birthday without year, an anniversary, a + relation, a note, a photo) shows in both apps. An edit in AOSP Contacts (in + place) and one in Fossify Contacts (delete and re-insert) upload only the + changed entries and leave the rest of the card alone; the next clean write + puts back on the device what Fossify dropped (a department, the primary + flag, the address parts). A contact created in AOSP Contacts goes up once; + deletions go both ways; a conflict on different fields keeps both changes, + one on the same field takes the server's and is counted. A photo within the + 96 px thumbnail keeps one row through server edits and stays on the server + after a device edit. +- **Calendar.** Timed events in two zones, a two-day all-day event, a + floating event, a weekly series with an excluded and a moved occurrence, + two alerts and a meeting show right in Google Calendar and Etar. An edit, a + new event and a deletion in Google Calendar go up, and the edit keeps the + event's unmapped properties (keywords, locale, priority, links). Etar's + "only this event", deleting one occurrence and "this and all future + events", and "only this occurrence" in Fossify Calendar and Google Calendar + become the right overrides, exclusions and series, with no instance + duplicated or lost. "Yes" and "Maybe" in Google Calendar change only the + user's `participationStatus`. Tasks and a tasks-only calendar stay off the + device; switching the reminder owner both ways rewrites only the reminders; + a calendar deselected and selected again leaves and comes back without + server writes; an event created in a shared account's writable calendar + lands in that account; a change of the device zone rewrites only floating + events. +- **No duplicates.** With the process killed, or the network cut, between a + create's `/set` and its identity write (30 contacts or 20 events at a + time), the next run adopts every row by uid, and server and device hold + each item once. +- **Deletion threshold.** 60 contacts deleted at once stay on the server + until the user chooses in Android's notification: "undo" brings them back + on the device, "delete" deletes them on the server. +- **Settings and lifecycle.** The switch, the chooser, "Sync now", "Paused in + Android settings", "Grant access" after the permissions were revoked, and + "Android account settings" (API 32 and 34) work as described. Turning sync + off with an edit waiting offline asks first, and "Keep syncing" keeps the + edit, which goes up once the network is back; turning sync off and on + removes and restores the rows without server writes or duplicates; an + account removed in Android Settings is not added back by the app; signing + out removes the rows and the Android account. +- **Covered by tests only:** contact groups and memberships, photos too + large for the device, moves between calendars and a series turned into a + single event (Etar's delete and insert), events and exceptions Google + Calendar creates without a status, a device create the server deletes + before the next download, changes put back because the server can't take + them, an address edited as one line in AOSP Contacts, and the "Review + deletions" question in the settings. + +Scale, with a debug build (JavaScript from Metro in dev mode) on an API 34 +emulator given 4 GB of memory: 2,000 cards and 2,000 events (100 of them +recurring, with overrides) on top of 152 cards and 35 events. Times are the +adapter's, from the start of the sync to the report; memory is the peak PSS +of the app process. + +| Run | Contacts | Calendar | Peak PSS | +|---|---|---|---| +| First sync, contacts alone | 2,000 inserts, 57.6 s | — | 464 MB | +| First sync, calendar alone | — | 2,000 inserts + 68 exception rows, 38.7 s | 450 MB | +| First sync, both at once (they share the JS thread) | 117.1 s | 117.8 s | 455 MB | +| Both at once, while scrolling the mail list | 150.3 s | 154.0 s | 433 MB | +| 1 card and 1 event changed on the server | 0.83 s | 1.03 s | 448 MB | +| No changes | 0.38 s | 0.70 s | 436 MB | +| Both collections deleted on the server | 2,000 deletes, 27-36 s | 2,000 deletes, 36-41 s | 452 MB | + +The app used about 412 MB before the runs (a debug build with its dev +tooling); syncing added 20-50 MB at the peak. No run crashed or caused an +ANR. During the first syncs, scrolling the mail list stayed as smooth as +without them (2.85 % janky frames against 3.05 %), and switching between the +app's tabs took as long as without a sync, except once, when the app's own +Contacts screen loaded the 2,000 new cards at the same time. The sync shares +the JS thread with the app's screens (it peaked at 96 %), so a first sync +takes longer while the app is in use (74 s instead of 57.6 s for the +contacts). A release build and a real phone were not measured. From 0110eebb7802d188292be5a87c60a1888a57ca26 Mon Sep 17 00:00:00 2001 From: Linus Rath <139418639+rathlinus@users.noreply.github.com> Date: Sun, 27 Sep 2026 09:54:03 +0200 Subject: [PATCH 65/87] fix: upload an edited event another client moved out of synced calendars #34 When another client moved an event into a calendar this device does not sync while the device had an unsent edit of it, the calendar planner refused to merge the server's version, since the event is in no synced calendar any more. The edit then waited on every run as a planner error, and the event never left the device. The server's version now merges into the rows where they are, so the edit uploads against it without undoing the other client's changes, and the rows go once the server has taken the edit. Checked with a planner test and a run through the engine and the fake server: the series rescheduled on the device keeps the occurrence the other client added, and no rows are left. --- .../__tests__/calendar/download.test.ts | 22 +++++++++++++++++++ .../__tests__/engine/real-planners.test.ts | 8 +++---- src/device-sync/calendar/planner.ts | 17 +++++++++----- 3 files changed, 37 insertions(+), 10 deletions(-) diff --git a/src/device-sync/__tests__/calendar/download.test.ts b/src/device-sync/__tests__/calendar/download.test.ts index ab00eea5..b4884312 100644 --- a/src/device-sync/__tests__/calendar/download.test.ts +++ b/src/device-sync/__tests__/calendar/download.test.ts @@ -235,6 +235,28 @@ describe('calendar planner: downloads', () => { expect(() => calendarPlanner.planDownload({ ...single(), '@type': undefined, due: '2026-10-01T00:00:00' } as unknown as CalendarEventWire, null, h.ctx)).toThrow(/tasks/); }); + it('merges an edited event another client moved out of every synced calendar where its rows are', async () => { + const h = await new Harness().setup(); + await h.download(single()); + const id = Number(h.masterRow('e1')._id); + h.fake.user.updateEvent(id, { [Events.TITLE]: 'Team lunch' }); + // Filed in a calendar this device does not sync, with a new description. + const moved = single({ calendarIds: { archive: true }, description: 'Bring salad' }); + + const plan = calendarPlanner.planDownload(moved, await h.local('e1'), h.ctx); + await h.apply(plan.ops); + + expect(plan).toMatchObject({ effect: 'update', stillDirty: true, conflicts: 0 }); + expect(h.masterRow('e1')).toMatchObject({ [Events.CALENDAR_ID]: h.rowIds.get('b'), [Events.TITLE]: 'Team lunch', [Events.DESCRIPTION]: 'Bring salad', [Events.DIRTY]: 1 }); + const local = (await h.local('e1'))!; + expect(local.shadow).toEqual(moved); + // Only the device's edit goes up (no calendar move), and the accepted version takes the rows away. + const plan2 = calendarPlanner.planUpload(local, h.ctx); + expect(plan2.kind === 'upload' && plan2.actions).toEqual([{ kind: 'update', id: 'e1', patch: { title: 'Team lunch' } }]); + await h.apply(calendarPlanner.planAccepted(local, { ...moved, title: 'Team lunch' }, h.ctx).ops); + expect(await h.events()).toEqual([]); + }); + it('places an event in several calendars in the first selected one, and keeps it where it is', async () => { const h = await new Harness().setup(); await h.download(single({ calendarIds: { w: true, b: true } })); diff --git a/src/device-sync/__tests__/engine/real-planners.test.ts b/src/device-sync/__tests__/engine/real-planners.test.ts index cae21f0c..1f95bb6c 100644 --- a/src/device-sync/__tests__/engine/real-planners.test.ts +++ b/src/device-sync/__tests__/engine/real-planners.test.ts @@ -574,14 +574,14 @@ describe('device sync engine with the real planners', () => { 'recurrenceOverrides/2026-10-01T09:00:00': { title: 'Retro' }, }); - expect((await h.run(CALENDAR_AUTHORITY)).outcome).toBe('ok'); + expect(await h.run(CALENDAR_AUTHORITY)).toMatchObject({ outcome: 'ok', itemErrors: [], stats: { uploaded: { updated: 1 } } }); const event = h.server.get('CalendarEvent', 'a', series)!; const titles = Object.values((event.recurrenceOverrides ?? {}) as Record).map((o) => o.title).sort(); expect(titles).toEqual(['Planning', 'Retro']); - // The move went up, or it still waits on the device: it is not lost. - const row = h.events().find((e) => Number(e._id) === Number(master._id)); - expect(event.start === '2026-09-28T10:00:00' || (Number(row?.dirty) === 1 && Number(row?.[Events.DTSTART]) === later)).toBe(true); + // The move went up against the server's version, and the rows went with the event. + expect(event).toMatchObject({ start: '2026-09-28T10:00:00', calendarIds: { [archive]: true } }); + expect(h.events()).toEqual([]); }); }); diff --git a/src/device-sync/calendar/planner.ts b/src/device-sync/calendar/planner.ts index 8348e6ef..5db04219 100644 --- a/src/device-sync/calendar/planner.ts +++ b/src/device-sync/calendar/planner.ts @@ -31,7 +31,7 @@ import { claimOps, claimUid, claimedUid, computeCreate, createTarget, deleteActi import { decodeCalendar, decodeEvents, isExceptionRow, isNewMaster } from './decode'; import { isExcluded, overridesOf } from './exceptions'; import { eventImage } from './image'; -import { insertEvent, isRemovedInstance, linkedExceptionCount, pickCalendar, writeEvent } from './merge'; +import { insertEvent, isRemovedInstance, linkedExceptionCount, pickCalendar, writeEvent, type CalendarTarget } from './merge'; import { pairKind, pairOps, pairPatch } from './pairs'; import { GroupBuilder, assertExceptionCount, assertRow, deleteRow, writesOf } from './rows'; import { eventZone } from './timing'; @@ -52,6 +52,12 @@ function refOf(local: LocalEventRow | null, event?: CalendarEventWire, jmapAccou return local ? `row:${local.eventId}` : 'new'; } +/** The calendar a row is in, when it is one of ours. */ +function rowCalendar(local: LocalEventRow, ctx: CalendarContext): CalendarTarget | null { + const cal = ctx.calendarIdOfRow(local.calendarRowId); + return cal ? { calendarRowId: local.calendarRowId, calendarId: cal.calendarId } : null; +} + // ─── Downloads ────────────────────────────────────────── function planDownload(event: CalendarEventWire, local: LocalEvent | null, ctx: CalendarContext): DownloadPlan { @@ -59,7 +65,9 @@ function planDownload(event: CalendarEventWire, local: LocalEvent | null, ctx: C refuseTask(local?.shadow); const ref = refOf(local, event, ctx.jmapAccountId); const none = (stillDirty: boolean): DownloadPlan => ({ ops: { ref, ops: [] }, conflicts: 0, stillDirty, effect: 'none', writes: 0 }); - const target = pickCalendar(event, local, ctx); + // An event another client moved out of every synced calendar still merges into its rows where they are + // (they wait for an upload; the accepted write then removes them). + const target = pickCalendar(event, local, ctx) ?? (local ? rowCalendar(local, ctx) : null); if (!target) throw new Error(`calendar planner: ${ref} is in no synced calendar (the engine removes such rows)`); const image = eventImage(event, target.calendarRowId, target.calendarId, ctx); // Stalwart drops a start it can't resolve (a DST gap or overlap): nothing to show. @@ -210,10 +218,7 @@ function purgeGroup(local: LocalEvent): OpGroup { function revertGroup(local: LocalEvent, ctx: CalendarContext): OpGroup | null { const shadow = local.shadow; if (!shadow) return null; - const target = pickCalendar(shadow, local, ctx) ?? (() => { - const cal = ctx.calendarIdOfRow(local.calendarRowId); - return cal ? { calendarRowId: local.calendarRowId, calendarId: cal.calendarId } : null; - })(); + const target = pickCalendar(shadow, local, ctx) ?? rowCalendar(local, ctx); if (!target) return null; const image = eventImage(shadow, target.calendarRowId, target.calendarId, ctx); if (!image) return null; From 4bcca705485789907bd8fe42e6dd412b67619a78 Mon Sep 17 00:00:00 2001 From: Linus Rath <139418639+rathlinus@users.noreply.github.com> Date: Sun, 27 Sep 2026 09:55:03 +0200 Subject: [PATCH 66/87] fix: write big meetings in several provider transactions #34 A recurring meeting with many attendees on many occurrences (25 on 52, 100 on 12) is about 2 MB of provider operations, and one Android provider transaction carries 1 MB. Every event was written as one group that the engine never splits, so such a meeting was refused every run and never reached a phone that synced it for the first time. An item too big for one group now comes as a chain of groups that the engine applies one by one, each in a transaction of its own. A new event is written master first, with its identity, then its occurrences in groups that fit, then its shadow; a write into an existing event writes the occurrences first and the series with its new shadow last. When a run stops between two groups, the next one finds the event by its identity and writes only what is missing, planned from the old shadow, so no row is written twice and no server change is lost. Checked with planner tests for both meeting sizes and every cut point, and runs through the engine with a provider that refuses transactions over 1 MB, a crash after every batch included. --- .../__tests__/calendar/big-events.test.ts | 158 +++++++++++++++++- src/device-sync/__tests__/calendar/harness.ts | 7 +- .../__tests__/engine/internals.test.ts | 51 ++++++ src/device-sync/calendar/merge.ts | 73 ++++++-- src/device-sync/calendar/planner.ts | 12 +- src/device-sync/calendar/rows.ts | 143 +++++++++++++++- src/device-sync/engine/batch.ts | 81 ++++++++- src/device-sync/engine/calendar-sync.ts | 6 +- src/device-sync/engine/item-sync.ts | 7 +- src/device-sync/planner.ts | 9 + 10 files changed, 505 insertions(+), 42 deletions(-) diff --git a/src/device-sync/__tests__/calendar/big-events.test.ts b/src/device-sync/__tests__/calendar/big-events.test.ts index d367ab0f..0b1942ba 100644 --- a/src/device-sync/__tests__/calendar/big-events.test.ts +++ b/src/device-sync/__tests__/calendar/big-events.test.ts @@ -4,17 +4,23 @@ import { describe, expect, it } from 'vitest'; import { Attendees, Events } from '../../android-columns'; import { calendarPlanner } from '../../calendar/planner'; +import { contactsPlanner } from '../../contacts/planner'; +import { estimateBatchBytes } from '../../engine/batch'; import type { OpGroup } from '../../planner'; +import { CALENDAR_AUTHORITY, type ProviderPort } from '../../types'; import type { CalendarEventWire } from '../../wire'; +import { createHarness, rowWrites, type Harness as EngineHarness } from '../engine/harness'; import { Harness, ME, serverApply } from './harness'; const TRANSACTION = 1_000_000; const estimate = (group: OpGroup) => 2 * JSON.stringify(group.ops).length + 300 * group.ops.length; +/** A plan's groups: the group and the ones applied after it. */ +const chain = (group: OpGroup): OpGroup[] => [group, ...(group.next ?? [])]; /** A daily meeting with `attendees` guests whose `overrides` occurrences each carry the whole participants map. */ -function meeting(attendees: number, overrides: number): CalendarEventWire { +function meeting(attendees: number, overrides: number, me = ME, calendarId = 'b'): CalendarEventWire { const participants: Record = { - me: { '@type': 'Participant', calendarAddress: `mailto:${ME}`, roles: { owner: true, attendee: true }, participationStatus: 'accepted' }, + me: { '@type': 'Participant', calendarAddress: `mailto:${me}`, roles: { owner: true, attendee: true }, participationStatus: 'accepted' }, }; for (let i = 0; i < attendees; i++) { participants[`p${i}`] = { '@type': 'Participant', calendarAddress: `mailto:person${i}@example.com`, name: `Person ${i}`, roles: { attendee: true }, participationStatus: 'needs-action', expectReply: true }; @@ -28,19 +34,54 @@ function meeting(attendees: number, overrides: number): CalendarEventWire { id: 'big', '@type': 'Event', uid: 'big@example.org', - calendarIds: { b: true }, + calendarIds: { [calendarId]: true }, title: 'Standup', description: 'Daily standup', start: '2026-10-05T09:00:00', timeZone: 'Europe/Berlin', duration: 'PT15M', recurrenceRule: { frequency: 'daily', count: 300 }, - organizerCalendarAddress: `mailto:${ME}`, + organizerCalendarAddress: `mailto:${me}`, participants, recurrenceOverrides, } as unknown as CalendarEventWire; } +/** The engine with the real planners, its provider refusing a batch bigger than one Binder transaction (as Android does). */ +function engineWithBinderLimit(): EngineHarness { + const h = createHarness(); + h.deps.planners = { contacts: contactsPlanner, calendar: calendarPlanner }; + const base = h.deps.provider; + h.deps.provider = (name, authority): ProviderPort => { + const port = base(name, authority); + return { + accountName: port.accountName, + authority: port.authority, + query: (q) => port.query(q), + readSyncState: () => port.readSyncState(), + readPhoto: (id, px) => port.readPhoto(id, px), + applyBatch: async (ops) => + estimateBatchBytes(ops) > TRANSACTION ? { ok: false, reason: 'tooLarge', message: 'TransactionTooLargeException' } : port.applyBatch(ops), + }; + }; + return h; +} + +/** The big meeting's rows on the engine's device: one master, one exception row per override, every attendee once. */ +function expectWholeMeeting(h: EngineHarness, id: string, attendees: number, overrides: number, context = ''): void { + const rows = h.events(); + const masters = rows.filter((e) => e[Events._SYNC_ID] === `a/${id}`); + expect(masters, context).toHaveLength(1); + const exceptions = rows.filter((e) => e[Events.ORIGINAL_SYNC_ID] === `a/${id}`); + expect(exceptions, context).toHaveLength(overrides); + expect(new Set(exceptions.map((x) => x[Events._SYNC_ID])).size, context).toBe(overrides); + expect(exceptions.every((x) => Number(x[Events.ORIGINAL_ID]) === Number(masters[0]._id)), context).toBe(true); + const attendeesOf = (eventId: unknown) => h.device.rows('attendees').filter((a) => Number(a.event_id) === Number(eventId)).length; + expect([masters[0], ...exceptions].every((row) => attendeesOf(row._id) === attendees + 1), context).toBe(true); + expect(Object.keys(JSON.parse(String(masters[0][Events.SYNC_DATA1])).recurrenceOverrides), context).toHaveLength(overrides); + expect(rows.filter((e) => e[Events.ORIGINAL_SYNC_ID] === null), context).toHaveLength(1); +} + async function synced(event: CalendarEventWire) { const h = await new Harness().setup(); await h.download(event); @@ -87,4 +128,113 @@ describe('calendar planner: big meetings', () => { expect(await h.tryApply(accepted.ops)).toMatchObject({ ok: false, reason: 'assert' }); expect(h.fake.rows('attendees', `${Attendees.EVENT_ID} = ? AND ${Attendees.ATTENDEE_EMAIL} = ?`, [x.eventId, 'person1@example.com'])[0][Attendees.ATTENDEE_STATUS]).toBe(1); }); + + it.each([ + [25, 52], + [100, 12], + ])('writes the first download of %i attendees on %i overrides in groups that each fit one transaction', async (attendees, overrides) => { + const h = await new Harness().setup(); + const event = meeting(attendees, overrides); + + const plan = calendarPlanner.planDownload(event, null, h.ctx); + + expect(plan.effect).toBe('insert'); + expect(chain(plan.ops).length).toBeGreaterThan(1); + for (const group of chain(plan.ops)) expect(estimate(group)).toBeLessThan(TRANSACTION); + await h.apply(plan.ops); + const local = (await h.local('big'))!; + expect(local.shadow).toEqual(event); + expect(local.exceptions.map((x) => x.recurrenceId).sort()).toEqual(Object.keys(event.recurrenceOverrides!).sort()); + expect(local.exceptions.every((x) => x.attendees.length === attendees + 1 && !x.dirty)).toBe(true); + // Its echo writes nothing. + expect(calendarPlanner.planDownload(event, local, h.ctx).effect).toBe('none'); + }); + + it('finishes a first download cut off between its groups, without writing a row twice', async () => { + const event = meeting(25, 52); + const groups = chain(calendarPlanner.planDownload(event, null, (await new Harness().setup()).ctx).ops).length; + for (let cut = 1; cut < groups; cut++) { + const h = await new Harness().setup(); + for (const group of chain(calendarPlanner.planDownload(event, null, h.ctx).ops).slice(0, cut)) await h.apply({ ref: group.ref, ops: group.ops }); + + // Planned again from what the first groups wrote. + const again = calendarPlanner.planDownload(event, await h.local('big'), h.ctx); + for (const group of chain(again.ops)) expect(estimate(group)).toBeLessThan(TRANSACTION); + await h.apply(again.ops); + + const local = (await h.local('big'))!; + expect(local.shadow, `cut after ${cut}`).toEqual(event); + expect(new Set(local.exceptions.map((x) => x.recurrenceId)).size, `cut after ${cut}`).toBe(52); + expect(local.exceptions, `cut after ${cut}`).toHaveLength(52); + expect(local.exceptions.every((x) => x.attendees.length === 26), `cut after ${cut}`).toBe(true); + expect(calendarPlanner.planDownload(event, local, h.ctx).effect, `cut after ${cut}`).toBe('none'); + } + // Every cut point writes a 2 MB meeting again. + }, 60_000); + + it('writes a change of every occurrence in groups, the series with its new shadow last', async () => { + const h = await new Harness().setup(); + const event = meeting(25, 52); + await h.download(event); + // Another client adds a guest to the meeting and to every occurrence. + const guest = { '@type': 'Participant', calendarAddress: 'mailto:guest@example.com', name: 'Guest', roles: { attendee: true }, participationStatus: 'needs-action' }; + const changed = JSON.parse(JSON.stringify(event)) as CalendarEventWire; + (changed.participants as Record).guest = guest; + for (const override of Object.values(changed.recurrenceOverrides!)) (override as { participants: Record }).participants.guest = guest; + + const plan = calendarPlanner.planDownload(changed, await h.local('big'), h.ctx); + + const groups = chain(plan.ops); + expect(groups.length).toBeGreaterThan(1); + for (const group of groups) expect(estimate(group)).toBeLessThan(TRANSACTION); + const writesShadow = (group: OpGroup) => group.ops.some((op) => op.op === 'update' && Events.SYNC_DATA1 in op.values); + expect(groups.map(writesShadow)).toEqual(groups.map((_, i) => i === groups.length - 1)); + // Cut off after the first group: planned again from what it wrote, the rest follows. + await h.apply({ ref: groups[0].ref, ops: groups[0].ops }); + await h.apply(calendarPlanner.planDownload(changed, await h.local('big'), h.ctx).ops); + const local = (await h.local('big'))!; + expect(local.shadow).toEqual(changed); + expect(local.exceptions.every((x) => x.attendees.length === 27)).toBe(true); + expect(calendarPlanner.planDownload(changed, local, h.ctx).effect).toBe('none'); + }); +}); + +describe('device sync engine: big meetings', () => { + it.each([ + [25, 52], + [100, 12], + ])('downloads a meeting of %i attendees on %i overrides in several transactions', async (attendees, overrides) => { + const h = engineWithBinderLimit(); + const { id: _drop, ...event } = meeting(attendees, overrides, 'alice@example.com', h.calendar) as unknown as Record; + const id = h.server.addEvent('a', event); + + const report = await h.run(CALENDAR_AUTHORITY); + + expect(report).toMatchObject({ outcome: 'ok', itemErrors: [], stats: { downloaded: { created: 1 } } }); + expectWholeMeeting(h, id, attendees, overrides); + const before = h.batches.log.length; + expect((await h.run(CALENDAR_AUTHORITY)).outcome).toBe('ok'); + expect(rowWrites(h.batches.log.slice(before))).toEqual([]); + }); + + it('finishes a big meeting cut off between its groups by a crash, without duplicating rows', async () => { + const setup = () => { + const h = engineWithBinderLimit(); + const { id: _drop, ...event } = meeting(25, 52, 'alice@example.com', h.calendar) as unknown as Record; + return { h, id: h.server.addEvent('a', event) }; + }; + const probe = setup(); + await probe.h.run(CALENDAR_AUTHORITY); + const batches = probe.h.batches.applied; + + for (let n = 1; n < batches; n++) { + const { h, id } = setup(); + h.batches.crashAfter = n; + await h.run(CALENDAR_AUTHORITY); + h.batches.crashAfter = null; + expect((await h.run(CALENDAR_AUTHORITY)).outcome, `crash after batch ${n}`).toBe('ok'); + expectWholeMeeting(h, id, 25, 52, `crash after batch ${n}`); + } + // Two runs of a 2 MB meeting per crash point. + }, 60_000); }); diff --git a/src/device-sync/__tests__/calendar/harness.ts b/src/device-sync/__tests__/calendar/harness.ts index e8f5e7af..1ec0873c 100644 --- a/src/device-sync/__tests__/calendar/harness.ts +++ b/src/device-sync/__tests__/calendar/harness.ts @@ -86,9 +86,12 @@ export class Harness { return this; } + /** Applies a group and the groups after it (`next`), each in a batch of its own, as the engine does. */ async apply(group: OpGroup): Promise { - const result = await this.port.applyBatch(group.ops); - if (!result.ok) throw new Error(`batch ${group.ref} failed: ${result.reason} ${result.message}`); + for (const g of [group, ...(group.next ?? [])]) { + const result = await this.port.applyBatch(g.ops); + if (!result.ok) throw new Error(`batch ${group.ref} failed: ${result.reason} ${result.message}`); + } } async tryApply(group: OpGroup) { diff --git a/src/device-sync/__tests__/engine/internals.test.ts b/src/device-sync/__tests__/engine/internals.test.ts index 974f9c9b..33323444 100644 --- a/src/device-sync/__tests__/engine/internals.test.ts +++ b/src/device-sync/__tests__/engine/internals.test.ts @@ -201,6 +201,57 @@ describe('provider batches', () => { await writer(p).write([{ group: { ref: 'g', ops: [{ op: 'assert', table: 'raw_contacts', id: 1, values: { dirty: 0 } }] } }]); expect(p.sent).toEqual([]); }); + + describe('an item too big for one group (a chain)', () => { + const named = (n: string) => insert('data', { n }); + const namesOf = (batches: ProviderOp[][]) => batches.map((b) => b.map((op) => (op.op === 'insert' ? op.values.n : op.op))); + const chained = (extra: Partial = {}): Work => ({ + group: { ref: 'big', ops: [named('1')], next: [{ ref: 'big', ops: [named('2')] }, { ref: 'big', ops: [named('3')] }] }, + ...extra, + }); + + it('applies its groups in order, each in a batch of its own, between the other works, and counts it after the last', async () => { + const p = port(ok); + const applied: string[] = []; + const store = new StateStore(emptySyncState(), { registryId: 'r', origin: 'o' }); + const big = chained({ + applied: (results) => void applied.push(`big:${results.length}`), + state: (next) => { + accountOf(next, 'a').stale = ['big']; + }, + }); + const other = (n: string): Work => ({ group: { ref: n, ops: [named(n)] }, applied: () => void applied.push(n) }); + + await new BatchWriter(p, { maxOps: 400, maxBytes: 300_000, maxReplans: 3 }, store).write( + [other('a'), big, other('b')], + store.tail(() => undefined), + ); + + expect(namesOf(p.sent)).toEqual([['a'], ['1'], ['2'], ['3', 'syncState'], ['b', 'syncState']]); + expect(applied).toEqual(['a', 'big:3', 'b']); + expect(store.committed.accounts.a.stale).toEqual(['big']); + }); + + it('plans the item again from what its first groups wrote when a later one fails, never sending that one again', async () => { + let failures = 1; + const p = port((ops) => (namesOf([ops])[0].includes('2') && failures-- > 0 ? { ok: false, reason: 'assert', message: 'changed' } : ok(ops))); + const rest: Work = { group: { ref: 'big', ops: [named('rest')] } }; + + await writer(p).write([chained({ replan: async () => rest })]); + + expect(namesOf(p.sent)).toEqual([['1'], ['2'], ['rest']]); + }); + + it('gives the item up when one of its groups is refused for good', async () => { + const p = port((ops) => (namesOf([ops])[0].includes('2') ? { ok: false, reason: 'tooLarge', message: 'TransactionTooLargeException' } : ok(ops))); + const failed = vi.fn(); + + await writer(p).write([chained({ failed, replan: async () => null })]); + + expect(namesOf(p.sent)).toEqual([['1'], ['2']]); + expect(failed).toHaveBeenCalledWith('tooLarge', 'TransactionTooLargeException'); + }); + }); }); describe('SyncState', () => { diff --git a/src/device-sync/calendar/merge.ts b/src/device-sync/calendar/merge.ts index eadd6b71..f03a9834 100644 --- a/src/device-sync/calendar/merge.ts +++ b/src/device-sync/calendar/merge.ts @@ -19,7 +19,7 @@ * edit of them) and the number of exception rows. */ import { Events } from '../android-columns'; -import type { CalendarContext, EventBaseline, LocalEvent, LocalEventRow, LocalException } from '../planner'; +import type { CalendarContext, EventBaseline, LocalEvent, LocalEventRow, LocalException, OpGroup } from '../planner'; import type { Row } from '../types'; import type { CalendarEventWire } from '../wire'; import { collectionKey, exceptionRef, objectRef } from '../common/ids'; @@ -31,6 +31,8 @@ import { assertExceptionCount, assertRow, deleteRow, + eventInsert, + eventWrite, insertRow, updateRow, type RowState, @@ -240,7 +242,8 @@ export type WriteMode = | 'overwrite'; export interface EventWriteResult { - group: GroupBuilder; + /** One group, or a chain for an event too big for one (see `eventWrite`). */ + ops: OpGroup; conflicts: number; stillDirty: boolean; } @@ -277,7 +280,6 @@ export function writeEvent( options: EventWriteOptions, ): EventWriteResult { const { ctx } = options; - const group = new GroupBuilder(options.ref); const overwrite = options.mode === 'overwrite'; const masterRef = objectRef(ctx.jmapAccountId, event.id); const uid = typeof event.uid === 'string' && event.uid ? event.uid : null; @@ -288,9 +290,14 @@ export function writeEvent( // Asserts first: the rows as read. Attendees and reminders only for dirty rows (see `assertRow`): a big // meeting's occurrences would otherwise assert every attendee and outgrow one provider transaction. - assertRow(group, local, false, local.dirty || local.deleted); - for (const x of local.exceptions) assertRow(group, x, true, x.dirty || x.deleted); - assertExceptionCount(group, local.eventId, linkedExceptionCount(local)); + const opsOf = (build: (group: GroupBuilder) => void) => { + const group = new GroupBuilder(options.ref); + build(group); + return group.build().ops; + }; + const masterAsserts = opsOf((g) => assertRow(g, local, false, local.dirty || local.deleted)); + const rowAsserts = new Map(local.exceptions.map((x) => [x.eventId, opsOf((g) => assertRow(g, x, true, x.dirty || x.deleted))])); + const countAssert = opsOf((g) => assertExceptionCount(g, local.eventId, linkedExceptionCount(local))); let conflicts = 0; let stillDirty = false; @@ -412,12 +419,17 @@ export function writeEvent( updates.push({ row, state: exceptionState(key, merged, left, row, !!r) }); } - for (const row of deletes) deleteRow(group, row.eventId); - updateRow(group, local, masterState); - for (const { row, state } of updates) updateRow(group, row, state); - for (const state of inserts) insertRow(group, state, { id: local.eventId }); + const ops = eventWrite(options.ref, { + master: masterAsserts, + rows: rowAsserts, + count: countAssert, + deletes: deletes.map((row) => ({ row: row.eventId, asserts: [], writes: opsOf((g) => deleteRow(g, row.eventId)) })), + masterWrite: opsOf((g) => updateRow(g, local, masterState)), + updates: updates.map(({ row, state }) => ({ row: row.eventId, asserts: [], writes: opsOf((g) => updateRow(g, row, state)) })), + inserts: inserts.map((state) => ({ asserts: [], writes: opsOf((g) => insertRow(g, state, { id: local.eventId })) })), + }); - return { group, conflicts, stillDirty: !overwrite && (stillDirty || local.deleted) }; + return { ops, conflicts, stillDirty: !overwrite && (stillDirty || local.deleted) }; } /** The row an exception image becomes, written clean. */ @@ -438,12 +450,8 @@ export function imageState(x: ExceptionImage, calendarRowId: number, masterRef: }; } -/** A new event's rows: the master and one row per exception, attendees and reminders included. */ -export function insertEvent(event: CalendarEventWire, image: EventImage, ctx: CalendarContext, ref: string): GroupBuilder { - const group = new GroupBuilder(ref); - const masterRef = objectRef(ctx.jmapAccountId, event.id); - const uid = typeof event.uid === 'string' && event.uid ? event.uid : null; - const master = insertRow(group, { +function masterInsertState(event: CalendarEventWire, image: EventImage, masterRef: string, uid: string | null): RowState { + return { isException: false, cells: image.master.cells, attendees: image.master.attendees, @@ -453,10 +461,39 @@ export function insertEvent(event: CalendarEventWire, image: EventImage, ctx: Ca uid, shadow: event, hasParticipants: image.master.hasParticipants, - }); + }; +} + +/** A new event's rows: the master and one row per exception, attendees and reminders included. */ +export function insertEvent(event: CalendarEventWire, image: EventImage, ctx: CalendarContext, ref: string): GroupBuilder { + const group = new GroupBuilder(ref); + const masterRef = objectRef(ctx.jmapAccountId, event.id); + const uid = typeof event.uid === 'string' && event.uid ? event.uid : null; + const master = insertRow(group, masterInsertState(event, image, masterRef, uid)); for (const key of [...image.exceptions.keys()].sort()) { const x = image.exceptions.get(key)!; insertRow(group, imageState(x, Number(image.master.cells[Events.CALENDAR_ID]), masterRef, uid), { ref: master }); } return group; } + +/** A new event's rows as one group, or as a chain for an event too big for one (see `eventInsert`). */ +export function insertEventOps(event: CalendarEventWire, image: EventImage, ctx: CalendarContext, ref: string): OpGroup { + const masterRef = objectRef(ctx.jmapAccountId, event.id); + const uid = typeof event.uid === 'string' && event.uid ? event.uid : null; + const calendarRowId = Number(image.master.cells[Events.CALENDAR_ID]); + const opsOf = (build: (group: GroupBuilder) => void) => { + const group = new GroupBuilder(ref); + build(group); + return group.build().ops; + }; + return eventInsert(ref, insertEvent(event, image, ctx, ref).build(), () => ({ + masterRef, + master: opsOf((g) => insertRow(g, { ...masterInsertState(event, image, masterRef, uid), shadow: undefined })), + exceptions: [...image.exceptions.keys()].sort().map((key) => ({ + asserts: [], + writes: opsOf((g) => insertRow(g, imageState(image.exceptions.get(key)!, calendarRowId, masterRef, uid))), + })), + shadow: event, + })); +} diff --git a/src/device-sync/calendar/planner.ts b/src/device-sync/calendar/planner.ts index 5db04219..4ee3b26b 100644 --- a/src/device-sync/calendar/planner.ts +++ b/src/device-sync/calendar/planner.ts @@ -31,7 +31,7 @@ import { claimOps, claimUid, claimedUid, computeCreate, createTarget, deleteActi import { decodeCalendar, decodeEvents, isExceptionRow, isNewMaster } from './decode'; import { isExcluded, overridesOf } from './exceptions'; import { eventImage } from './image'; -import { insertEvent, isRemovedInstance, linkedExceptionCount, pickCalendar, writeEvent, type CalendarTarget } from './merge'; +import { insertEvent, insertEventOps, isRemovedInstance, linkedExceptionCount, pickCalendar, writeEvent, type CalendarTarget } from './merge'; import { pairKind, pairOps, pairPatch } from './pairs'; import { GroupBuilder, assertExceptionCount, assertRow, deleteRow, writesOf } from './rows'; import { eventZone } from './timing'; @@ -74,7 +74,7 @@ function planDownload(event: CalendarEventWire, local: LocalEvent | null, ctx: C if (!image) return none(!!local && (local.dirty || local.deleted)); if (!local) { - const ops = insertEvent(event, image, ctx, ref).build(); + const ops = insertEventOps(event, image, ctx, ref); return { ops, conflicts: 0, stillDirty: false, effect: 'insert', writes: writesOf(ops) }; } // A split clone only looks like this object: it is a new event and uploads as one. The source row is the object's. @@ -118,7 +118,7 @@ function planDownload(event: CalendarEventWire, local: LocalEvent | null, ctx: C ...(local.shadow ? {} : { pending: null }), clearPoison: !itemDirty, }); - const ops = result.group.build(); + const ops = result.ops; const writes = writesOf(ops); return { ops: writes ? ops : { ref, ops: [] }, @@ -222,7 +222,7 @@ function revertGroup(local: LocalEvent, ctx: CalendarContext): OpGroup | null { if (!target) return null; const image = eventImage(shadow, target.calendarRowId, target.calendarId, ctx); if (!image) return null; - return writeEvent(shadow, image, target, local, { mode: 'overwrite', ref: refOf(local), ctx, base: shadow }).group.build(); + return writeEvent(shadow, image, target, local, { mode: 'overwrite', ref: refOf(local), ctx, base: shadow }).ops; } /** Clears DIRTY where nothing uploadable changed, taking the rows as they are as the new baselines. */ @@ -368,7 +368,7 @@ function planAccepted(local: LocalEvent, server: CalendarEventWire, ctx: Calenda base: local.shadow ?? server, pending: null, clearPoison: true, - }).group.build(); + }).ops; return { ops, keepDirtyOps }; } @@ -411,7 +411,7 @@ function planZoneChange(local: LocalEvent, previousZone: string, ctx: CalendarCo if (!target) return null; const image = eventImage(shadow, target.calendarRowId, target.calendarId, ctx); if (!image) return null; - const ops = writeEvent(shadow, image, target, local, { mode: 'overwrite', ref: refOf(local), ctx, base: shadow }).group.build(); + const ops = writeEvent(shadow, image, target, local, { mode: 'overwrite', ref: refOf(local), ctx, base: shadow }).ops; return writesOf(ops) ? ops : null; } diff --git a/src/device-sync/calendar/rows.ts b/src/device-sync/calendar/rows.ts index 8c877a6c..79111f57 100644 --- a/src/device-sync/calendar/rows.ts +++ b/src/device-sync/calendar/rows.ts @@ -14,6 +14,8 @@ import type { EventBaseline, LocalEventRow, OpGroup, PendingCreate, PoisonMarker import type { ProviderOp, Row, WriteRow } from '../types'; import type { CalendarEventWire } from '../wire'; import { deepEqual } from '../common/json'; +import { MAX_OPS_PER_YIELD, estimateBatchBytes } from '../engine/batch'; +import { DEFAULT_TUNING } from '../engine/deps'; import { ATTENDEE_CELLS, MASTER_CELLS, EXCEPTION_CELLS, REMINDER_CELLS, encodeBaseline, sameCell } from './columns'; import { rowReminder } from './reminders'; @@ -245,7 +247,144 @@ function writeReminders(group: GroupBuilder, current: LocalEventRow, wanted: Row for (const add of inserts) group.write({ op: 'insert', table: 'reminders', values: { ...add, [Reminders.EVENT_ID]: current.eventId } }); } -/** Counts the provider writes of a group (asserts are not writes). */ +/** Counts the provider writes of a group and the groups after it (asserts are not writes). */ export function writesOf(group: OpGroup): number { - return group.ops.filter((op) => op.op === 'insert' || op.op === 'update' || op.op === 'delete').length; + return [group, ...(group.next ?? [])].reduce( + (n, g) => n + g.ops.filter((op) => op.op === 'insert' || op.op === 'update' || op.op === 'delete').length, + 0, + ); +} + +// ─── Items too big for one group ──────────────────────── + +/** + * What one group may hold: the engine's batch budget as `estimateBatchBytes` + * weighs it, and one op less than a provider yield window (room for the + * engine's insert guard). A bigger event is written as a chain of groups + * (`OpGroup.next`); a master's shadow is one column and may take a group of + * its own beyond the budget (under one transaction for any event that syncs). + */ +const GROUP_BYTES = DEFAULT_TUNING.maxBatchBytes; +const GROUP_OPS = MAX_OPS_PER_YIELD - 1; + +const weight = (ops: readonly ProviderOp[]) => (ops.length ? estimateBatchBytes(ops) : 0); + +/** One group from asserts and write blocks whose `refs` index into their own block, asserts first. */ +function assemble(ref: string, asserts: readonly ProviderOp[], blocks: ReadonlyArray): OpGroup { + const ops: ProviderOp[] = [...asserts]; + for (const block of blocks) { + const offset = ops.length; + for (const op of block) { + if (op.op !== 'insert' || !op.refs) ops.push(op); + else ops.push({ ...op, refs: Object.fromEntries(Object.entries(op.refs).map(([column, i]) => [column, i + offset])) }); + } + } + return { ref, ops }; +} + +/** Writes of one row (`refs` index into them), and the asserts that guard that row (none for a new row). */ +export interface RowWrite { + asserts: ProviderOp[]; + writes: ProviderOp[]; +} + +/** + * Cuts row writes into groups of the budget, in order, a row never split: + * each group starts with `guard`, the first also with `first`. + */ +function cutRows(ref: string, rows: readonly RowWrite[], guard: readonly ProviderOp[], first: readonly ProviderOp[] = []): OpGroup[] { + const groups: OpGroup[] = []; + let asserts: ProviderOp[] = [...guard, ...first]; + let blocks: ProviderOp[][] = []; + let ops = asserts.length; + let bytes = weight(asserts); + for (const row of rows) { + const n = row.asserts.length + row.writes.length; + const b = weight(row.asserts) + weight(row.writes); + if (blocks.length && (ops + n > GROUP_OPS || bytes + b > GROUP_BYTES)) { + groups.push(assemble(ref, asserts, blocks)); + asserts = [...guard]; + blocks = []; + ops = asserts.length; + bytes = weight(asserts); + } + asserts.push(...row.asserts); + blocks.push(row.writes); + ops += n; + bytes += b; + } + if (blocks.length) groups.push(assemble(ref, asserts, blocks)); + return groups; +} + +function chained(groups: readonly OpGroup[]): OpGroup { + const [first, ...next] = groups; + return next.length ? { ...first, next } : first; +} + +/** + * The writes into an existing event as one group: asserts first (`master`, + * the exception rows' `rows` in order, `count`), then the writes (`deletes`, + * the master's `master`, `updates`, `inserts`). When that is more than a group + * may hold, a chain: the exception rows' writes in groups of the budget, each + * asserting the master row and the rows it writes, the first also the number + * of exception rows; then the master's write, with its new shadow, last, + * asserting the other rows. Cut off in between, the item is planned again + * from the old shadow, so no server change to a row not written yet is taken + * as done. + */ +export function eventWrite( + ref: string, + parts: { + master: ProviderOp[]; + rows: Map; + count: ProviderOp[]; + deletes: Array; + masterWrite: ProviderOp[]; + updates: Array; + inserts: RowWrite[]; + }, +): OpGroup { + const rowAsserts = [...parts.rows.values()].flat(); + const single = assemble(ref, [...parts.master, ...rowAsserts, ...parts.count], [ + ...parts.deletes.map((d) => d.writes), + parts.masterWrite, + ...parts.updates.map((u) => u.writes), + ...parts.inserts.map((i) => i.writes), + ]); + if (single.ops.length <= GROUP_OPS && estimateBatchBytes(single.ops) <= GROUP_BYTES) return single; + const writing = [...parts.deletes, ...parts.updates].filter((w) => w.writes.length); + if (!writing.length && !parts.inserts.length) return single; + const written = new Set(writing.map((w) => w.row)); + const rows: RowWrite[] = [ + ...writing.map((w) => ({ asserts: parts.rows.get(w.row) ?? [], writes: w.writes })), + ...parts.inserts, + ]; + // The master row's own assert guards every group; its attendees and reminders go with its write. + const groups = cutRows(ref, rows, parts.master.slice(0, 1), parts.count); + const rest = [...parts.rows].filter(([row]) => !written.has(row)).flatMap(([, asserts]) => asserts); + groups.push(assemble(ref, [...parts.master, ...rest], [parts.masterWrite])); + return chained(groups); +} + +/** + * A new event's rows as one group, or, when they are more than a group may + * hold, as a chain: `master` (the master's insert with its identity but + * without its shadow), its exception rows in groups of the budget, each + * asserting the master by `masterRef` and linked to it by ORIGINAL_SYNC_ID + * (the provider fills ORIGINAL_ID from it and CALENDAR_ID), then `shadow`. + * Cut off in between, the next download finds the master by its identity and + * writes what is missing; a master without a shadow merges like an adopted one. + */ +export function eventInsert( + ref: string, + single: OpGroup, + partsOf: () => { masterRef: string; master: ProviderOp[]; exceptions: RowWrite[]; shadow: CalendarEventWire }, +): OpGroup { + if (single.ops.length <= GROUP_OPS && estimateBatchBytes(single.ops) <= GROUP_BYTES) return single; + const parts = partsOf(); + const byIdentity = { table: 'events' as const, where: `${Events._SYNC_ID} = ?`, args: [parts.masterRef], expectCount: 1 }; + const groups = [assemble(ref, [], [parts.master]), ...cutRows(ref, parts.exceptions, [{ op: 'assert', ...byIdentity }])]; + groups.push({ ref, ops: [{ op: 'update', ...byIdentity, values: { [Events.SYNC_DATA1]: JSON.stringify(parts.shadow) } }] }); + return chained(groups); } diff --git a/src/device-sync/engine/batch.ts b/src/device-sync/engine/batch.ts index 334a4419..686d7aea 100644 --- a/src/device-sync/engine/batch.ts +++ b/src/device-sync/engine/batch.ts @@ -23,6 +23,12 @@ * A group whose rows the SyncState describes (a group row: `groups`) carries * that `state` change: every batch that applies such a group, a retry * included, ends with a state op holding it. + * + * An item too big for one transaction comes as a chain (`OpGroup.next`): its + * groups go in order, each in a batch of its own, between the batches of the + * works before and after it; its `state` and `applied` go with the last. A + * group of the chain that fails leaves the earlier ones written: after an + * assert the item is read and planned again from them, else it is given up. */ import type { OpGroup } from '../planner'; import type { BatchFailure, OpResult, ProviderOp, ProviderPort } from '../types'; @@ -67,6 +73,17 @@ export function estimateBatchBytes(ops: readonly ProviderOp[]): number { return 2 * JSON.stringify(ops).length + 300 * ops.length; } +/** A group and the groups of its item that follow it (`next`), each without `next`. */ +export function chainOf(group: OpGroup): OpGroup[] { + const { next, ...first } = group; + return [first, ...(next ?? [])]; +} + +/** Whether a group, or one of the groups after it, writes anything. */ +export function chainHasWrites(group: OpGroup): boolean { + return chainOf(group).some((g) => hasWrites(g.ops)); +} + function carriesBlob(ops: readonly ProviderOp[]): boolean { return ops.some( (op) => @@ -100,19 +117,30 @@ export function buildBatch(groups: readonly OpGroup[], tailOp?: ProviderOp): Pro return out; } -/** Prepends ops to a group (e.g. an engine-side assert), shifting the group's refs. */ +/** Prepends ops to a group (e.g. an engine-side assert), shifting the group's refs; the groups after it stay. */ export function prependOps(group: OpGroup, ops: ProviderOp[]): OpGroup { - return { ref: group.ref, ops: [...ops, ...placeGroup(group.ops, ops.length).map(stripYield)] }; + return { ...group, ops: [...ops, ...placeGroup(group.ops, ops.length).map(stripYield)] }; } -/** One group running `first` then `second` atomically. */ +/** One group running `first` then `second` atomically (then the groups after `second`). */ export function concatGroups(ref: string, first: OpGroup, second: OpGroup): OpGroup { return { ref, ops: [...first.ops.map(stripYield), ...placeGroup(second.ops, first.ops.length).map(stripYield)], + ...(second.next ? { next: second.next } : {}), }; } +/** Appends ops to the last group of a chain (e.g. clearing a poison marker once the item is written). */ +export function appendToLast(group: OpGroup, ops: ProviderOp[]): OpGroup { + if (!ops.length) return group; + if (!group.next?.length) return { ...group, ops: [...group.ops, ...ops] }; + const next = [...group.next]; + const last = next[next.length - 1]; + next[next.length - 1] = { ...last, ops: [...last.ops, ...ops] }; + return { ...group, next }; +} + function stripYield(op: ProviderOp): ProviderOp { const { yieldAllowed: _y, ...rest } = op as ProviderOp & { yieldAllowed?: boolean }; return rest as ProviderOp; @@ -134,7 +162,22 @@ export class BatchWriter { * every group of `works` was applied or given up. */ async write(works: readonly Work[], tail?: Tail): Promise { - const todo = works.filter((w) => hasWrites(w.group.ops)); + let run: Work[] = []; + for (const work of works) { + if (!chainHasWrites(work.group)) continue; + if (!work.group.next?.length) { + run.push(work); + continue; + } + await this.writeRun(run); + run = []; + await this.applyChain(work, 0); + } + await this.writeRun(run, tail); + } + + /** Works of one group each, packed into batches. */ + private async writeRun(todo: readonly Work[], tail?: Tail): Promise { const batches = this.pack(todo); if (batches.length === 0) { if (tail) await this.applyTail(tail); @@ -243,7 +286,37 @@ export class BatchWriter { if (tail) await this.applyTail(tail); } + /** + * An item's chain of groups, each in a batch of its own and so committed + * whole or not at all; the item's state and `applied` go with the last. + */ + private async applyChain(work: Work, attempt: number): Promise { + const groups = chainOf(work.group).filter((g) => hasWrites(g.ops)); + const results: OpResult[] = []; + for (let i = 0; i < groups.length; i++) { + const last = i === groups.length - 1; + const stated = last ? this.stage([work]) : { staged: [], tail: undefined }; + let result; + try { + result = await this.send(buildBatch([groups[i]], stated.tail?.op())); + } finally { + this.unstage(stated.staged); + } + if (!result.ok) { + // The groups before it are written: planned again from them, never sent again as they are. + await this.retryAlone(work, result.reason, result.message, attempt); + return; + } + results.push(...result.results.slice(0, groups[i].ops.length)); + if (last) { + await work.applied?.(results); + stated.tail?.applied(); + } + } + } + private async applyAlone(work: Work, attempt: number): Promise { + if (work.group.next?.length) return this.applyChain(work, attempt); if (!hasWrites(work.group.ops)) return; const stated = this.stage([work]); let result; diff --git a/src/device-sync/engine/calendar-sync.ts b/src/device-sync/engine/calendar-sync.ts index 12dcb233..6fecd598 100644 --- a/src/device-sync/engine/calendar-sync.ts +++ b/src/device-sync/engine/calendar-sync.ts @@ -10,13 +10,13 @@ import type { CalendarContext, CalendarEventWire, CalendarLike, CalendarPlanner, import { CALENDAR_AUTHORITY, type ReminderOwner, type Row } from '../types'; import { CALENDAR_EVENT_PROPERTIES, CALENDAR_PROPERTIES } from '../wire'; import { calendarAddresses, type CalendarAddresses, type CapabilityAccount } from '../jmap/session'; -import type { Work } from './batch'; +import { chainHasWrites, type Work } from './batch'; import type { RunEnv } from './context'; import type { SubscriptionCalendar } from './deps'; import { ItemSync, type Pending } from './item-sync'; import { accountOfRef, idInAccount, refOf, type Held, type Kind, type ServerObject } from './kinds'; import { poisonFingerprint } from './poison'; -import { flag, hasWrites, num, str } from './provider'; +import { flag, num, str } from './provider'; import { isCollectionSelected } from './selection'; import { accountOf, type SyncState } from './sync-state'; @@ -501,7 +501,7 @@ export class CalendarSync extends ItemSync { const held: Held = { kind: this.eventKind, local: moved }; try { const work = this.acceptWork(acct, { held, meta: this.eventKind.meta(moved), fingerprint: '', actions: [] }, server, false); - if (hasWrites(work.group.ops)) return [work]; + if (chainHasWrites(work.group)) return [work]; // The pair's ops already wrote what the server holds. this.env.report.stats.uploaded.updated++; return []; diff --git a/src/device-sync/engine/item-sync.ts b/src/device-sync/engine/item-sync.ts index c5856617..ce7947f5 100644 --- a/src/device-sync/engine/item-sync.ts +++ b/src/device-sync/engine/item-sync.ts @@ -16,7 +16,7 @@ import { type SetResponse, } from '../jmap/caller'; import { isMethodError } from '../jmap/errors'; -import { concatGroups, prependOps, type Work } from './batch'; +import { appendToLast, chainHasWrites, concatGroups, prependOps, type Work } from './batch'; import type { RunEnv } from './context'; import { RunAbort, StateMismatch } from './errors'; import { @@ -687,7 +687,7 @@ export abstract class ItemSync { ): Work | null { const ref = refOf(acct, object.id); const plan = kind.planDownload(object, local ?? null, acct); - if (plan.effect === 'none' || !hasWrites(plan.ops.ops)) { + if (plan.effect === 'none' || !chainHasWrites(plan.ops)) { if (local) this.onEcho(kind, acct, object); if (!local || kind.meta(local).dirty) return null; const heal = kind.planBaselineHeal(local); @@ -1405,7 +1405,8 @@ export abstract class ItemSync { const { kind, local } = p.held; const plan = kind.planAccepted(local, server, acct); const extra = p.meta.poison ? [poisonOp(kind, p.meta.rowId, null)] : []; - const withExtra = (group: OpGroup): OpGroup => (extra.length ? { ref: group.ref, ops: [...group.ops, ...extra] } : group); + // With the item's last group: a big one comes in several (OpGroup.next). + const withExtra = (group: OpGroup): OpGroup => appendToLast(group, extra); const counted = () => { if (created) this.env.report.stats.uploaded.created++; else this.env.report.stats.uploaded.updated++; diff --git a/src/device-sync/planner.ts b/src/device-sync/planner.ts index 97de9ef4..65f8a659 100644 --- a/src/device-sync/planner.ts +++ b/src/device-sync/planner.ts @@ -25,6 +25,15 @@ export interface OpGroup { /** For logs and reports: SOURCE_ID/_SYNC_ID or `row:<_id>`. */ ref: string; ops: ProviderOp[]; + /** + * Further groups of the same item, for one too big for a provider + * transaction (a big meeting): the engine applies `[this, ...next]` in + * order, each atomically in a batch of its own, and counts the item once + * the last one is in. When one fails, the item is read and planned again + * from what the earlier ones wrote, so a planner orders them for that: a + * new row's identity first, a master's shadow last. They have no `next`. + */ + next?: OpGroup[]; } /** Why an item is left alone, with the fingerprint that lifts the back-off when the item changes. */ From 9439a4ffc0ed5517ef2b8dccbfcea6dd9b289a0c Mon Sep 17 00:00:00 2001 From: Linus Rath <139418639+rathlinus@users.noreply.github.com> Date: Sun, 27 Sep 2026 09:56:12 +0200 Subject: [PATCH 67/87] fix: keep an occurrence's colour cleared, and report reminders that can't go #34 Clearing the colour of one occurrence of a coloured series removed the occurrence's own colour, so it showed the series' colour again after the sync. The occurrence now gets an empty colour, which Stalwart keeps and which shows as the calendar's colour, as on the phone. Removing every reminder of one occurrence sent an empty set of alerts, which Stalwart does not store: the occurrence then showed the series' reminders again, without a word. When the series uses the calendar's default reminders, the occurrence leaves the defaults and has none, as before. When the series has reminders of its own, the occurrence would show them whatever is sent, so the change is put back and the report says why (`remindersNotRepresentable`). Checked against calcard 0.3.14, with planner tests and runs through the engine and the fake server. --- .../__tests__/calendar/engine.test.ts | 47 ++++++++++++ .../__tests__/calendar/exceptions.test.ts | 75 +++++++++++++++++++ src/device-sync/__tests__/calendar/harness.ts | 5 ++ src/device-sync/calendar/planner.ts | 5 ++ src/device-sync/calendar/upload.ts | 46 ++++++++---- 5 files changed, 162 insertions(+), 16 deletions(-) diff --git a/src/device-sync/__tests__/calendar/engine.test.ts b/src/device-sync/__tests__/calendar/engine.test.ts index 10418fd6..8e30e46b 100644 --- a/src/device-sync/__tests__/calendar/engine.test.ts +++ b/src/device-sync/__tests__/calendar/engine.test.ts @@ -379,6 +379,53 @@ describe('calendar device edits through the engine', () => { expect(h.server.all('CalendarEvent', 'team')).toEqual([]); }); + describe('an occurrence\'s own colour and reminders removed on the device', () => { + async function standup(extra: Record, override: Record) { + const h = real(); + const id = h.server.addEvent('a', { + uid: 'standup-uid', + title: 'Standup', + start: '2026-09-28T09:00:00', + duration: 'PT15M', + timeZone: 'Europe/Berlin', + recurrenceRule: { '@type': 'RecurrenceRule', frequency: 'daily', count: 5 }, + recurrenceOverrides: { '2026-09-29T09:00:00': { title: 'Standup (late)', ...override } }, + calendarIds: { [h.calendar]: true }, + ...extra, + }); + await h.run(CALENDAR_AUTHORITY); + const exception = () => h.events().find((e) => e[Events.ORIGINAL_SYNC_ID] === `a/${id}`)!; + const minutes = () => h.device.rows('reminders').filter((r) => Number(r[Reminders.EVENT_ID]) === Number(exception()._id)).map((r) => Number(r[Reminders.MINUTES])); + return { h, id, exception, minutes }; + } + + it('keeps a colour cleared on one occurrence cleared', async () => { + const { h, id, exception } = await standup({ color: 'steelblue' }, { color: 'red' }); + + h.device.user.updateEvent(Number(exception()._id), { [Events.EVENT_COLOR]: null }); + expect(await h.run(CALENDAR_AUTHORITY)).toMatchObject({ outcome: 'ok', itemErrors: [], stats: { uploaded: { updated: 1 } } }); + + expect(exception()).toMatchObject({ [Events.EVENT_COLOR]: null, [Events.DIRTY]: 0 }); + expect((h.server.get('CalendarEvent', 'a', id)!.recurrenceOverrides as Overrides)['2026-09-29T09:00:00']).toMatchObject({ color: '' }); + }); + + it('puts back the removal of every reminder of an occurrence whose series shows its own, and reports it', async () => { + const alert = { '@type': 'Alert', trigger: { '@type': 'OffsetTrigger', offset: '-PT15M' }, action: 'display' }; + const { h, id, exception, minutes } = await standup({ alerts: { al1: alert } }, {}); + expect(minutes()).toEqual([15]); + const sets = () => h.server.requests.filter((r) => r.methods.includes('CalendarEvent/set')).length; + const before = sets(); + + h.device.user.setReminders(Number(exception()._id), []); + const report = await h.run(CALENDAR_AUTHORITY); + + expect(report.itemErrors).toEqual([expect.objectContaining({ ref: `a/${id}`, side: 'upload', type: 'remindersNotRepresentable' })]); + expect(sets()).toBe(before); + expect(minutes()).toEqual([15]); + expect(exception()[Events.DIRTY]).toBe(0); + }); + }); + it('keeps a description and location cleared on one occurrence cleared', async () => { const h = real(); const id = h.server.addEvent('a', { diff --git a/src/device-sync/__tests__/calendar/exceptions.test.ts b/src/device-sync/__tests__/calendar/exceptions.test.ts index 2163d4c1..da473d36 100644 --- a/src/device-sync/__tests__/calendar/exceptions.test.ts +++ b/src/device-sync/__tests__/calendar/exceptions.test.ts @@ -1,6 +1,7 @@ import { describe, expect, it } from 'vitest'; import { Attendees, Events, Reminders } from '../../android-columns'; import { calendarPlanner } from '../../calendar/planner'; +import { cssColorToArgb } from '../../calendar/values'; import type { LocalEvent, UploadPlan } from '../../planner'; import type { CalendarEventWire } from '../../wire'; import { ACCT, Harness, ME, serverApply, single, utc, weekly } from './harness'; @@ -194,6 +195,80 @@ describe('calendar planner: recurrence edits made on the device', () => { expect(patch['recurrenceOverrides/2026-10-14T09:00:00']).toMatchObject({ useDefaultAlerts: false }); }); + it('clears an occurrence\'s own colour instead of letting the series\' come back', async () => { + const event = weekly(); + event.recurrenceOverrides!['2026-10-07T09:00:00'] = { ...event.recurrenceOverrides!['2026-10-07T09:00:00'], color: 'red' }; + const { h } = await synced(event); + const x = (await h.local('bl'))!.exceptions[0]; + expect(h.row(x.eventId)[Events.EVENT_COLOR]).toBe(cssColorToArgb('red')); + h.fake.user.updateEvent(x.eventId, { [Events.EVENT_COLOR]: null }); + const local = (await h.local('bl'))!; + + const plan = calendarPlanner.planUpload(local, h.ctx); + + // Without a colour of its own the occurrence would show the series' steelblue again: an empty one, which Stalwart keeps. + expect(patchOf(plan).patch).toEqual({ 'recurrenceOverrides/2026-10-07T09:00:00/color': '' }); + const server = await accept(h, event, local, plan); + expect(h.row(x.eventId)).toMatchObject({ [Events.EVENT_COLOR]: null, [Events.DIRTY]: 0 }); + expect(calendarPlanner.planDownload(server, (await h.local('bl'))!, h.ctx).effect).toBe('none'); + + // A series without a colour has none to show: the occurrence's own is removed. + const plain = weekly(); + delete plain.color; + plain.recurrenceOverrides!['2026-10-07T09:00:00'] = { ...plain.recurrenceOverrides!['2026-10-07T09:00:00'], color: 'red' }; + const p = await synced(plain); + const px = (await p.h.local('bl'))!.exceptions[0]; + p.h.fake.user.updateEvent(px.eventId, { [Events.EVENT_COLOR]: null }); + expect(patchOf(calendarPlanner.planUpload((await p.h.local('bl'))!, p.h.ctx)).patch).toEqual({ 'recurrenceOverrides/2026-10-07T09:00:00/color': null }); + }); + + it('removes every reminder of one occurrence of a series on the calendar\'s defaults', async () => { + const event = weekly(); + delete event.alerts; + event.useDefaultAlerts = true; + event.recurrenceOverrides!['2026-10-07T09:00:00'] = { + ...event.recurrenceOverrides!['2026-10-07T09:00:00'], + useDefaultAlerts: false, + alerts: { a60: { '@type': 'Alert', trigger: { '@type': 'OffsetTrigger', offset: '-PT1H', relativeTo: 'start' }, action: 'display' } }, + }; + const { h } = await synced(event); + const x = (await h.local('bl'))!.exceptions[0]; + h.fake.user.setReminders(x.eventId, []); + const local = (await h.local('bl'))!; + + const plan = calendarPlanner.planUpload(local, h.ctx); + const server = await accept(h, event, local, plan); + + // Stalwart drops the empty map, but the series has no alerts of its own to show instead. + expect(server.recurrenceOverrides!['2026-10-07T09:00:00']).toMatchObject({ useDefaultAlerts: false }); + expect(server.recurrenceOverrides!['2026-10-07T09:00:00']).not.toHaveProperty('alerts'); + expect(h.fake.rows('reminders', `${Reminders.EVENT_ID} = ?`, [x.eventId])).toEqual([]); + expect(h.row(x.eventId)[Events.DIRTY]).toBe(0); + expect(calendarPlanner.planDownload(server, (await h.local('bl'))!, h.ctx).effect).toBe('none'); + }); + + it('puts back and reports the removal of every reminder of one occurrence whose series would show its own again', async () => { + const event = weekly(); + event.alerts = { al1: { '@type': 'Alert', trigger: { '@type': 'OffsetTrigger', offset: '-PT15M', relativeTo: 'start' }, action: 'display' } }; + const { h } = await synced(event); + const minutes = (eventId: number) => h.fake.rows('reminders', `${Reminders.EVENT_ID} = ?`, [eventId]).map((r) => Number(r[Reminders.MINUTES])); + const x = (await h.local('bl'))!.exceptions[0]; + expect(minutes(x.eventId)).toEqual([15]); + h.fake.user.setReminders(x.eventId, []); + + // An override without alerts shows the series' (Stalwart writes no VALARM for an empty map): none can't be stored. + const plan = calendarPlanner.planUpload((await h.local('bl'))!, h.ctx); + expect(plan).toMatchObject({ kind: 'revert', reason: 'remindersNotRepresentable' }); + if (plan.kind === 'revert') await h.apply(plan.ops); + expect(minutes(x.eventId)).toEqual([15]); + expect(h.row(x.eventId)[Events.DIRTY]).toBe(0); + + // With another edit, that one goes up alone. + h.fake.user.setReminders(x.eventId, []); + h.fake.user.updateEvent(x.eventId, { [Events.TITLE]: 'Moved again' }); + expect(patchOf(calendarPlanner.planUpload((await h.local('bl'))!, h.ctx)).patch).toEqual({ 'recurrenceOverrides/2026-10-07T09:00:00/title': 'Moved again' }); + }); + it('patches an existing override one level deep, never deeper', async () => { const { h } = await synced(weekly()); const x = (await h.local('bl'))!.exceptions[0]; diff --git a/src/device-sync/__tests__/calendar/harness.ts b/src/device-sync/__tests__/calendar/harness.ts index 1ec0873c..b073e75b 100644 --- a/src/device-sync/__tests__/calendar/harness.ts +++ b/src/device-sync/__tests__/calendar/harness.ts @@ -169,6 +169,11 @@ export function serverApply(event: CalendarEventWire, patch: PatchObject): Calen for (const forbidden of ['@type', 'method', 'organizerCalendarAddress', 'privacy', 'prodId', 'recurrenceId', 'recurrenceIdTimeZone', 'sentBy', 'uid', 'recurrenceRule', 'recurrenceOverrides']) { delete (value as Record)[forbidden]; } + // An override is stored as its own VEVENT: an empty map writes nothing, so it reads back without it. + for (const map of ['participants', 'alerts', 'locations']) { + const inner = (value as Record)[map]; + if (inner && typeof inner === 'object' && !Object.keys(inner).length) delete (value as Record)[map]; + } } if (!Object.keys(overrides).length) delete next.recurrenceOverrides; } diff --git a/src/device-sync/calendar/planner.ts b/src/device-sync/calendar/planner.ts index 4ee3b26b..ac65d474 100644 --- a/src/device-sync/calendar/planner.ts +++ b/src/device-sync/calendar/planner.ts @@ -331,6 +331,11 @@ function planUpload(local: LocalEvent, ctx: CalendarContext): Plan { const ops = revertGroup(local, ctx); if (ops) return { kind: 'revert', ops, reason: SKIP.crossAccountMove }; } + // Every reminder of an occurrence removed where the server can't store none: back to what it holds, reported. + if (computation.dropped.includes('reminders')) { + const ops = revertGroup(local, ctx); + if (ops) return { kind: 'revert', ops, reason: SKIP.remindersNotRepresentable }; + } return { kind: 'clean', ops: cleanGroup(local, computation) }; } const action: UploadAction = { kind: 'update', id: ref.id, patch: computation.patch }; diff --git a/src/device-sync/calendar/upload.ts b/src/device-sync/calendar/upload.ts index f39210cc..73d63a73 100644 --- a/src/device-sync/calendar/upload.ts +++ b/src/device-sync/calendar/upload.ts @@ -50,6 +50,7 @@ import { } from './exceptions'; import { eventImage, plainInstanceImage } from './image'; import { isRemovedInstance } from './merge'; +import { reminderRows } from './reminders'; import { isRuleRepresentable, partsToRule, ruleToRRule, rruleParts, untilUtc, type RuleZone } from './rrule'; import { NOTIFYING_UNITS, sendsSchedulingMessages } from './scheduling'; import { fixedOffsetTiming, rowTiming, type RowTiming } from './timing'; @@ -90,6 +91,8 @@ export const SKIP = { rsvpRefused: 'rsvpRefused', /** Moved in place to a calendar of another JMAP account: no patch can do that. */ crossAccountMove: 'crossAccountMove', + /** Every reminder removed from one occurrence of a series with alerts of its own: the occurrence would show those. */ + remindersNotRepresentable: 'remindersNotRepresentable', } as const; export class SkipUpload extends Error { @@ -373,6 +376,7 @@ export function computeUpdate(local: LocalEvent, ctx: CalendarContext): UploadCo if (!ruleRemoved && shadow.recurrenceRule) { for (const x of local.exceptions) { const units = exceptionChanges(x, local, shadow, calendarId, zone, overrides, removedExceptions, ctx, lossy); + if (units.dropped) dropped.push(...units.dropped); if (units.units.size) { exceptionUnits.set(x.eventId, units.units); overridesChanged = true; @@ -423,7 +427,7 @@ function exceptionChanges( removed: Set, ctx: CalendarContext, lossy: boolean, -): { units: Set; rsvp: boolean; others?: boolean } { +): { units: Set; rsvp: boolean; others?: boolean; dropped?: string[] } { const units = new Set(); const key = x.recurrenceId; if (!key) return { units, rsvp: false }; @@ -478,7 +482,7 @@ function exceptionChanges( values.freeBusyStatus = availabilityFromDevice(cells[Events.AVAILABILITY]); break; case 'color': - values.color = argbToCss(cells[Events.EVENT_COLOR]); + values.color = argbToCss(cells[Events.EVENT_COLOR]) ?? clearedInOverride('color', instance, shadow); break; case 'timing': { const moved = columnUnitDiffers('timing', cur, reference); @@ -519,22 +523,30 @@ function exceptionChanges( units.add('attendees'); } } + let dropped: string[] | undefined; if (ctx.reminderOwner === 'device' && (existing || cur.reminders!.length) && remindersDiffer(cur, reference)) { - values.alerts = reminderChanges(instance, ctx.calendar(calendarId), cur.reminders!, ctx.mintKey).alerts; - // An occurrence on the calendar's defaults ignores its own alerts: it leaves them, as a series does - // (Stalwart keeps the override's `useDefaultAlerts` as a JSPROP; the series stays on the defaults). - if (instance.useDefaultAlerts) values.useDefaultAlerts = false; - units.add('reminders'); + const alerts = reminderChanges(instance, ctx.calendar(calendarId), cur.reminders!, ctx.mintKey).alerts; + if (!Object.keys(alerts).length && reminderRows(shadow.alerts).length) { + // An override is stored as a VEVENT of its own and an empty map writes no VALARM, so an override without + // alerts shows the series' again: none at all can't be stored while the series has alerts of its own. + dropped = ['reminders']; + } else { + values.alerts = alerts; + // An occurrence on the calendar's defaults ignores its own alerts: it leaves them, as a series does + // (Stalwart keeps the override's `useDefaultAlerts` as a JSPROP; the series stays on the defaults). + if (instance.useDefaultAlerts) values.useDefaultAlerts = false; + units.add('reminders'); + } } if (existing) { if (units.size) work.props.set(key, { ...(work.props.get(key) ?? {}), ...values }); - return { units, rsvp, others }; + return { units, rsvp, others, dropped }; } // A new override: nothing but its pinned time is still an override (an app saving the instance unchanged). if (!units.size) units.add('new'); work.whole.set(key, newOverride(shadow, key, values)); - return { units, rsvp, others }; + return { units, rsvp, others, dropped }; } /** Units a new exception row inherits from its series when it leaves them empty. */ @@ -547,15 +559,17 @@ function isEmptyUnit(unit: string, cells: Row): boolean { } /** - * A description or location cleared on an existing override. An override - * without one of its own shows the series' again, so while the series has - * one the override gets the empty value Stalwart keeps: the empty text, or a - * location without a name (an empty map writes no LOCATION line and reads - * back as none of its own). Null, a clean removal, when nothing would come - * back. + * A description, location or colour cleared on an existing override. An + * override without one of its own shows the series' again, so while the + * series has one the override gets the empty value Stalwart keeps: the empty + * text or colour (an empty DESCRIPTION or COLOR; an empty colour is none, the + * calendar's, on the device), or a location without a name (an empty map + * writes no LOCATION line and reads back as none of its own). Null, a clean + * removal, when nothing would come back. */ -function clearedInOverride(unit: 'description' | 'location', instance: CalendarEventWire, master: CalendarEventWire): unknown { +function clearedInOverride(unit: 'description' | 'location' | 'color', instance: CalendarEventWire, master: CalendarEventWire): unknown { if (unit === 'description') return typeof master.description === 'string' && master.description ? '' : null; + if (unit === 'color') return typeof master.color === 'string' && master.color ? '' : null; const series = Object.keys(master.locations ?? {}); if (!series.length) return null; const key = Object.keys(instance.locations ?? {})[0] ?? series[0]; From 3ab131dbe85ca56a0d6568abd18f696bc70aef60 Mon Sep 17 00:00:00 2001 From: Linus Rath <139418639+rathlinus@users.noreply.github.com> Date: Sun, 27 Sep 2026 09:57:06 +0200 Subject: [PATCH 68/87] fix: keep the last sync's result when an upload sync finds nothing to do #34 Android starts an upload-only sync right after a manual sync, and 30 s after any app write. When nothing waited on the device it ended at once, but it still stored its status, so the settings showed a plain "synced" and lost the conflicts and item errors of the run just before it. Such a run did nothing and now leaves the stored status alone. --- .../__tests__/engine/contacts-flow.test.ts | 19 +++++++++++++++++- src/device-sync/engine/run.ts | 20 ++++++++++++++++--- 2 files changed, 35 insertions(+), 4 deletions(-) diff --git a/src/device-sync/__tests__/engine/contacts-flow.test.ts b/src/device-sync/__tests__/engine/contacts-flow.test.ts index d786b20e..54c7dfa2 100644 --- a/src/device-sync/__tests__/engine/contacts-flow.test.ts +++ b/src/device-sync/__tests__/engine/contacts-flow.test.ts @@ -2,7 +2,7 @@ // server: first sync, incremental /changes, uploads, echoes, selection // changes and groups (docs/device-sync.md, "A sync run"). -import { describe, expect, it } from 'vitest'; +import { describe, expect, it, vi } from 'vitest'; import { Data, MimeType } from '../../android-columns'; import { runDeviceSync } from '../../engine/run'; import { @@ -295,6 +295,23 @@ describe('device sync engine: contacts', () => { expect(h.server.requests.length).toBe(requests); }); + it("keeps the last run's status (its conflicts and item errors) when an upload-only sync finds nothing to do", async () => { + const h = createHarness(); + addServerCards(h, ['Ada']); + await h.run(); + const recordStatus = vi.mocked(h.deps.recordStatus); + const recorded = recordStatus.mock.calls.length; + + // The upload sync Android starts right after a manual one: nothing waits on the device. + expect((await h.run(CONTACTS_AUTHORITY, { upload: true })).outcome).toBe('ok'); + + expect(recordStatus.mock.calls.length).toBe(recorded); + // One that has something to upload is a run like any other. + renameDeviceContact(h, h.contactNamed('Ada')!.id, 'Ada King'); + await h.run(CONTACTS_AUTHORITY, { upload: true }); + expect(recordStatus.mock.calls.length).toBe(recorded + 1); + }); + it('reports disabled, unsupported and rows of another owner without touching anything', async () => { const off = createHarness(); off.deps.isSyncEnabled = async () => false; diff --git a/src/device-sync/engine/run.ts b/src/device-sync/engine/run.ts index 883be9c5..4fb8c348 100644 --- a/src/device-sync/engine/run.ts +++ b/src/device-sync/engine/run.ts @@ -157,15 +157,19 @@ export async function runDeviceSync(payload: RunPayload, deps: EngineDeps): Prom const key = lockKey(payload.registryId, payload.authority); const release = await acquireLock(key, Math.max(0, payload.deadline - deps.now() - tuning.deadlineMarginMs)); let result: RunReport; + const run = { idle: false }; if (!release) { result = report.build('cancelled', deps.now(), { message: 'Another sync of this account is still running', moreRecordsToGet: true }); } else { try { - result = await runLocked(payload, deps, tuning, report, key); + result = await runLocked(payload, deps, tuning, report, key, run); } finally { release(); } } + // An upload sync with nothing to upload (Android starts one right after a manual sync) did nothing: + // the last run's status, with its conflicts and item errors, stays what the settings show. + if (run.idle) return result; try { await deps.recordStatus(payload.registryId, payload.authority, statusOf(result, report.itemErrorCount)); } catch (error) { @@ -174,7 +178,14 @@ export async function runDeviceSync(payload: RunPayload, deps: EngineDeps): Prom return result; } -async function runLocked(payload: RunPayload, deps: EngineDeps, tuning: Tuning, report: ReportBuilder, key: string): Promise { +async function runLocked( + payload: RunPayload, + deps: EngineDeps, + tuning: Tuning, + report: ReportBuilder, + key: string, + run: { idle: boolean }, +): Promise { const finish = (outcome: RunReport['outcome'], extra: Parameters[2] = {}) => report.build(outcome, deps.now(), extra); const extras = payload.extras ?? {}; @@ -189,7 +200,10 @@ async function runLocked(payload: RunPayload, deps: EngineDeps, tuning: Tuning, return finish('internal', { message: 'The rows on this device were written for another account' }); } // Providers schedule an upload sync for every account 30 s after any app write. - if (extras.upload && !(await hasLocalWork(reader, payload.authority, stored))) return finish('ok'); + if (extras.upload && !(await hasLocalWork(reader, payload.authority, stored))) { + run.idle = true; + return finish('ok'); + } const connection = await deps.jmap(payload.registryId); if (stored.owner && stored.owner.origin !== connection.origin) { From 117823f746e50dd63b3a4c08bd132a848d2d37f9 Mon Sep 17 00:00:00 2001 From: Linus Rath <139418639+rathlinus@users.noreply.github.com> Date: Sun, 27 Sep 2026 09:58:01 +0200 Subject: [PATCH 69/87] test: make the device sync fakes as strict as Android and Stalwart #34 The fake providers and the fake server were more lenient than the real ones, so three bugs showed only on a device: an event inserted without a status, small contact photos, and meetings too big for one provider transaction. The fake CalendarProvider leaves STATUS NULL when an app writes none. The fake ContactsProvider keeps a photo within the 96 px thumbnail as the thumbnail alone, without a PHOTO_FILE_ID, and serves it to readPhoto from there. A batch fails with tooLarge when it weighs more than one Binder transaction, a photo counted by its bytes. The fake server drops empty maps inside recurrence overrides, as Stalwart does. Tests now read photos the way the engine does, and the helpers that made up for the old fakes are gone. --- .../__tests__/calendar/big-events.test.ts | 24 ++---- .../__tests__/contacts/download.test.ts | 4 +- src/device-sync/__tests__/contacts/harness.ts | 44 +---------- .../__tests__/contacts/lifecycle.test.ts | 7 +- .../__tests__/contacts/real-engine.test.ts | 3 - .../__tests__/fake-jmap-server.test.ts | 23 ++++++ .../__tests__/fake-provider.test.ts | 39 ++++++++- .../__tests__/fakes/fake-jmap-server.ts | 2 + .../__tests__/fakes/fake-provider.ts | 79 +++++++++++++++++-- 9 files changed, 148 insertions(+), 77 deletions(-) diff --git a/src/device-sync/__tests__/calendar/big-events.test.ts b/src/device-sync/__tests__/calendar/big-events.test.ts index 0b1942ba..6075134c 100644 --- a/src/device-sync/__tests__/calendar/big-events.test.ts +++ b/src/device-sync/__tests__/calendar/big-events.test.ts @@ -5,9 +5,8 @@ import { describe, expect, it } from 'vitest'; import { Attendees, Events } from '../../android-columns'; import { calendarPlanner } from '../../calendar/planner'; import { contactsPlanner } from '../../contacts/planner'; -import { estimateBatchBytes } from '../../engine/batch'; import type { OpGroup } from '../../planner'; -import { CALENDAR_AUTHORITY, type ProviderPort } from '../../types'; +import { CALENDAR_AUTHORITY } from '../../types'; import type { CalendarEventWire } from '../../wire'; import { createHarness, rowWrites, type Harness as EngineHarness } from '../engine/harness'; import { Harness, ME, serverApply } from './harness'; @@ -47,23 +46,10 @@ function meeting(attendees: number, overrides: number, me = ME, calendarId = 'b' } as unknown as CalendarEventWire; } -/** The engine with the real planners, its provider refusing a batch bigger than one Binder transaction (as Android does). */ -function engineWithBinderLimit(): EngineHarness { +/** The engine with the real planners (the fake provider refuses a batch over one Binder transaction, as Android does). */ +function engineWithRealPlanners(): EngineHarness { const h = createHarness(); h.deps.planners = { contacts: contactsPlanner, calendar: calendarPlanner }; - const base = h.deps.provider; - h.deps.provider = (name, authority): ProviderPort => { - const port = base(name, authority); - return { - accountName: port.accountName, - authority: port.authority, - query: (q) => port.query(q), - readSyncState: () => port.readSyncState(), - readPhoto: (id, px) => port.readPhoto(id, px), - applyBatch: async (ops) => - estimateBatchBytes(ops) > TRANSACTION ? { ok: false, reason: 'tooLarge', message: 'TransactionTooLargeException' } : port.applyBatch(ops), - }; - }; return h; } @@ -204,7 +190,7 @@ describe('device sync engine: big meetings', () => { [25, 52], [100, 12], ])('downloads a meeting of %i attendees on %i overrides in several transactions', async (attendees, overrides) => { - const h = engineWithBinderLimit(); + const h = engineWithRealPlanners(); const { id: _drop, ...event } = meeting(attendees, overrides, 'alice@example.com', h.calendar) as unknown as Record; const id = h.server.addEvent('a', event); @@ -219,7 +205,7 @@ describe('device sync engine: big meetings', () => { it('finishes a big meeting cut off between its groups by a crash, without duplicating rows', async () => { const setup = () => { - const h = engineWithBinderLimit(); + const h = engineWithRealPlanners(); const { id: _drop, ...event } = meeting(25, 52, 'alice@example.com', h.calendar) as unknown as Record; return { h, id: h.server.addEvent('a', event) }; }; diff --git a/src/device-sync/__tests__/contacts/download.test.ts b/src/device-sync/__tests__/contacts/download.test.ts index 4fd6e639..40c02b7f 100644 --- a/src/device-sync/__tests__/contacts/download.test.ts +++ b/src/device-sync/__tests__/contacts/download.test.ts @@ -52,7 +52,9 @@ describe('contacts download: golden rows', () => { expect(cells(rows['relatedTo:urn:uuid:other-person'], TYPED)).toEqual({ data1: 'Other Person', data2: '6', data3: null }); expect(cells(rows['notes:n1'], ['data1'])).toEqual({ data1: 'first note' }); expect(rows['media:ph'].data_sync2).toBe(photoHash(JPEG)); - expect(h.device.photo(Number(rows['media:ph'].data14))).toBe(JPEG); + // A 1×1 photo is kept as the thumbnail alone (no PHOTO_FILE_ID), and read from it. + expect(rows['media:ph'].data14).toBeNull(); + expect((await h.port.readPhoto(rawId, 512))?.jpegBase64).toBe(JPEG); const raw = h.rawContact(rawId); expect(raw).toMatchObject({ diff --git a/src/device-sync/__tests__/contacts/harness.ts b/src/device-sync/__tests__/contacts/harness.ts index 8705ef99..544aa89a 100644 --- a/src/device-sync/__tests__/contacts/harness.ts +++ b/src/device-sync/__tests__/contacts/harness.ts @@ -4,7 +4,7 @@ * way the engine applies them (one op group per batch); uploads go through * the fake server's patch handling and come back through its normalisation. */ -import { Data, GroupMembership, Groups, RawContacts } from '../../android-columns'; +import { GroupMembership, Groups, RawContacts } from '../../android-columns'; import { makeKeyMinter, parseObjectRef, uuidFrom } from '../../common/ids'; import { contactsPlanner } from '../../contacts/planner'; import type { @@ -16,7 +16,7 @@ import type { UploadAction, UploadPlan, } from '../../planner'; -import { CONTACTS_AUTHORITY, type BatchResult, type OpResult, type ProviderOp, type ProviderPort, type Row } from '../../types'; +import { CONTACTS_AUTHORITY, type BatchResult, type ProviderPort, type Row } from '../../types'; import { FakeJmapServer } from '../fakes/fake-jmap-server'; import { FakeDeviceProviders } from '../fakes/fake-provider'; @@ -34,40 +34,6 @@ function rowsOf(res: { columns: string[]; rows: unknown[][] }): Row[] { return res.rows.map((cells) => Object.fromEntries(res.columns.map((c, i) => [c, cells[i]])) as Row); } -/** - * ContactsProvider keeps a photo no larger than its 96 px thumbnail as the - * thumbnail only: PHOTO_FILE_ID (DATA14) stays NULL, and the thumbnail - * (DATA15) does not read back. The fake stores every photo with a display - * photo; this makes the photo rows a batch wrote thumbnail-only, as a device - * does with the test fixtures' 1×1 JPEGs. - */ -export function keepThumbnailsOnly(device: FakeDeviceProviders, ops: ProviderOp[], results: OpResult[]): void { - ops.forEach((op, i) => { - if ((op.op !== 'insert' && op.op !== 'update') || op.table !== 'data') return; - const bytes = op.values[Data.DATA15]; - if (!bytes || typeof bytes !== 'object') return; - const id = op.op === 'insert' ? results[i]?.id : op.id; - const row = id === undefined ? undefined : device.table('data').get(id); - if (row) row[Data.DATA14] = null; - }); -} - -/** A provider port whose photo writes come out thumbnail-only (see `keepThumbnailsOnly`). */ -export function thumbnailOnlyPort(device: FakeDeviceProviders, port: ProviderPort): ProviderPort { - return { - accountName: port.accountName, - authority: port.authority, - query: (q) => port.query(q), - readSyncState: () => port.readSyncState(), - readPhoto: (id, px) => port.readPhoto(id, px), - applyBatch: async (ops) => { - const res = await port.applyBatch(ops); - if (res.ok) keepThumbnailsOnly(device, ops, res.results); - return res; - }, - }; -} - export class Harness { readonly device = new FakeDeviceProviders(); readonly port: ProviderPort = this.device.port(ACCOUNT, CONTACTS_AUTHORITY); @@ -82,8 +48,6 @@ export class Harness { readonly unselected = new Set(); /** When false the context has no `groupsOf` (memberships not synced). */ groupIndex = true; - /** When true the photos the planner writes come out thumbnail-only (see `keepThumbnailsOnly`). */ - thumbnailPhotos = false; readonly ctx: ContactsContext; constructor() { @@ -133,9 +97,7 @@ export class Harness { async apply(group: OpGroup): Promise { if (!group.ops.length) return { ok: true, results: [] }; - const res = await this.port.applyBatch(group.ops); - if (res.ok && this.thumbnailPhotos) keepThumbnailsOnly(this.device, group.ops, res.results); - return res; + return this.port.applyBatch(group.ops); } async applyOk(group: OpGroup): Promise { diff --git a/src/device-sync/__tests__/contacts/lifecycle.test.ts b/src/device-sync/__tests__/contacts/lifecycle.test.ts index 8d66eff4..e8583e32 100644 --- a/src/device-sync/__tests__/contacts/lifecycle.test.ts +++ b/src/device-sync/__tests__/contacts/lifecycle.test.ts @@ -80,8 +80,7 @@ describe('contacts lifecycle', () => { const h = new Harness(); h.blobs.set('B1', JPEG); const { id, rawId } = await h.seed({ name: { full: 'Blob Photo' }, media: { p: { kind: 'photo', blobId: 'B1', mediaType: 'image/jpeg' } } }); - const photo = h.rowsByKey(rawId)['media:p']; - expect(h.device.photo(Number(photo.data14))).toBe(JPEG); + expect((await h.port.readPhoto(rawId, 512))?.jpegBase64).toBe(JPEG); expect((await h.contact(rawId)).shadow?.media).toEqual({ p: { kind: 'photo', blobId: 'B1', mediaType: 'image/jpeg' } }); await expectSettled(h, id, rawId); }); @@ -164,7 +163,7 @@ describe('contacts lifecycle', () => { h.blobs.set('B1', JPEG); h.server.serverUpdate('ContactCard', JMAP, id, { 'emails/e1/address': 'c@example.org' }); await h.download(id); - expect(h.device.photo(Number(h.rowsByKey(rawId)['media:p'].data14))).toBe(JPEG); + expect((await h.port.readPhoto(rawId, 512))?.jpegBase64).toBe(JPEG); await expectSettled(h, id, rawId); }); @@ -188,7 +187,6 @@ describe('contacts lifecycle', () => { it('keeps one row through downloads and merges, and never reads it as deleted', async () => { const h = new Harness(); - h.thumbnailPhotos = true; const { id, rawId } = await h.seed(small()); expect(photos(h, rawId)).toMatchObject([{ data14: null, data_sync1: 'media:ph' }]); await expectSettled(h, id, rawId); @@ -220,7 +218,6 @@ describe('contacts lifecycle', () => { it('heals the photo rows the same photo was inserted as again, keeping one and the server photo', async () => { const h = new Harness(); - h.thumbnailPhotos = true; const { id, rawId } = await h.seed(small()); const [row] = photos(h, rawId); const again = { [Data.MIMETYPE]: MimeType.PHOTO, [Data.DATA15]: { b64: JPEG }, data_sync1: row.data_sync1, data_sync2: row.data_sync2, data_sync3: row.data_sync3 }; diff --git a/src/device-sync/__tests__/contacts/real-engine.test.ts b/src/device-sync/__tests__/contacts/real-engine.test.ts index d39e6bb4..3e1ee1c3 100644 --- a/src/device-sync/__tests__/contacts/real-engine.test.ts +++ b/src/device-sync/__tests__/contacts/real-engine.test.ts @@ -9,7 +9,6 @@ import { contactsPlanner } from '../../contacts/planner'; import { addServerCards, ANDROID_ACCOUNT, CONTACTS_AUTHORITY, createHarness, rowWrites, type Harness } from '../engine/harness'; import type { SetTarget } from '../fakes/fake-jmap-server'; import { JPEG } from './fixtures'; -import { thumbnailOnlyPort } from './harness'; function real(options: Parameters[0] = {}): Harness { const h = createHarness(options); @@ -119,8 +118,6 @@ describe('contacts through the engine', () => { it('keeps one row for a small photo the provider stores as its thumbnail, and never deletes it on the server', async () => { const h = real(); - const base = h.deps.provider; - h.deps.provider = (name, authority) => thumbnailOnlyPort(h.device, base(name, authority)); const media = { ph: { kind: 'photo', uri: `data:image/jpeg;base64,${JPEG}`, mediaType: 'image/jpeg' } }; const card = h.server.addCard('a', { uid: 'u-small', name: { full: 'Photo Smalltest' }, notes: { n1: { note: 'v1' } }, addressBookIds: { [h.book]: true }, media }); expect((await h.run()).outcome).toBe('ok'); diff --git a/src/device-sync/__tests__/fake-jmap-server.test.ts b/src/device-sync/__tests__/fake-jmap-server.test.ts index ceae4551..426830f4 100644 --- a/src/device-sync/__tests__/fake-jmap-server.test.ts +++ b/src/device-sync/__tests__/fake-jmap-server.test.ts @@ -429,6 +429,29 @@ describe('FakeJmapServer', () => { expect(server.get('CalendarEvent', 'a', id)?.updated).toBe('2026-09-27T10:02:00Z'); }); + it('drops empty maps inside overrides too, which an override VEVENT cannot hold', async () => { + const { server, port, cal } = setup(); + const id = server.addEvent('a', { + title: 'Standup', + start: '2026-09-28T09:00:00', + recurrenceRule: { frequency: 'daily' }, + alerts: { al1: { '@type': 'Alert', trigger: { '@type': 'OffsetTrigger', offset: '-PT15M' } } }, + recurrenceOverrides: { '2026-09-29T09:00:00': { title: 'Late', alerts: { a2: { '@type': 'Alert', trigger: { '@type': 'OffsetTrigger', offset: '-PT1H' } } } } }, + calendarIds: { [cal]: true }, + }); + + await call(port, 'CalendarEvent/set', { + accountId: 'a', + update: { [id]: { 'recurrenceOverrides/2026-09-29T09:00:00/alerts': {}, 'recurrenceOverrides/2026-09-29T09:00:00/locations': {} } }, + }); + + // Read back without them, so the occurrence shows the series' alerts again, as on Stalwart. + const override = (server.get('CalendarEvent', 'a', id)?.recurrenceOverrides as Record>)['2026-09-29T09:00:00']; + expect(override).toMatchObject({ title: 'Late' }); + expect(override).not.toHaveProperty('alerts'); + expect(override).not.toHaveProperty('locations'); + }); + it('leaves useDefaultAlerts out of properties: null and returns it when listed', async () => { const { server, port, cal } = setup(); const id = server.addEvent('a', { title: 'x', useDefaultAlerts: true, calendarIds: { [cal]: true } }); diff --git a/src/device-sync/__tests__/fake-provider.test.ts b/src/device-sync/__tests__/fake-provider.test.ts index 98803ef9..41afcc47 100644 --- a/src/device-sync/__tests__/fake-provider.test.ts +++ b/src/device-sync/__tests__/fake-provider.test.ts @@ -10,6 +10,7 @@ import { StructuredPostal, } from '../android-columns'; import { CALENDAR_AUTHORITY, CONTACTS_AUTHORITY, type ProviderOp } from '../types'; +import { JPEG } from './contacts/fixtures'; import { FakeDeviceProviders, aospDurationValid, aospRruleValid } from './fakes/fake-provider'; import { compileWhere } from './fakes/fake-sql'; @@ -129,13 +130,13 @@ describe('FakeDeviceProviders: contacts', () => { }); it('refuses 500 contacts ops without a yield point and oversized batches', async () => { - const fake = new FakeDeviceProviders({ maxBatchBytes: 50_000 }); + const fake = new FakeDeviceProviders({ maxBatchBytes: 500_000 }); const port = fake.port(ME, CONTACTS_AUTHORITY); const ops: ProviderOp[] = Array.from({ length: 500 }, () => ({ op: 'insert', table: 'raw_contacts', values: {} }) as ProviderOp); expect(await port.applyBatch(ops)).toMatchObject({ ok: false, reason: 'provider' }); ops[250] = { ...ops[250], yieldAllowed: true }; expect((await port.applyBatch(ops)).ok).toBe(true); - const big = await port.applyBatch([{ op: 'insert', table: 'raw_contacts', values: { [RawContacts.SYNC2]: 'x'.repeat(60_000) } }]); + const big = await port.applyBatch([{ op: 'insert', table: 'raw_contacts', values: { [RawContacts.SYNC2]: 'x'.repeat(300_000) } }]); expect(big).toMatchObject({ ok: false, reason: 'tooLarge' }); }); @@ -166,6 +167,31 @@ describe('FakeDeviceProviders: contacts', () => { expect(rows.rows[0]).toEqual([String(photo!.fileId), null]); }); + it('keeps a photo within the 96 px thumbnail as the thumbnail alone, and serves that to readPhoto', async () => { + const fake = new FakeDeviceProviders(); + const { port, rawId } = await contactWithEmail(fake); + const res = await port.applyBatch([ + { op: 'insert', table: 'data', values: { [Data.RAW_CONTACT_ID]: rawId, [Data.MIMETYPE]: MimeType.PHOTO, [Data.DATA15]: { b64: JPEG } } }, + ]); + expect(res.ok).toBe(true); + const rows = await port.query({ table: 'data', columns: [Data.DATA14, Data.DATA15], where: `${Data.MIMETYPE} = ?`, args: [MimeType.PHOTO] }); + // No PHOTO_FILE_ID for a 1×1 JPEG, and the thumbnail does not read back. + expect(rows.rows[0]).toEqual([null, null]); + expect(await port.readPhoto(rawId, 512)).toEqual({ jpegBase64: JPEG, fileId: null }); + }); + + it('weighs a batch as its Binder transaction does: two bytes per character of its JSON, a photo by its bytes', async () => { + const fake = new FakeDeviceProviders(); + const { port, rawId } = await contactWithEmail(fake); + // 600,000 characters of text are about 1.2 MB in the Parcel; the same number of base64 characters are 450 KB of photo. + const text = await port.applyBatch([{ op: 'update', table: 'raw_contacts', id: rawId, values: { [RawContacts.SYNC2]: 'x'.repeat(600_000) } }]); + expect(text).toMatchObject({ ok: false, reason: 'tooLarge' }); + const photo = await port.applyBatch([ + { op: 'insert', table: 'data', values: { [Data.RAW_CONTACT_ID]: rawId, [Data.MIMETYPE]: MimeType.PHOTO, [Data.DATA15]: { b64: 'A'.repeat(600_000) } } }, + ]); + expect(photo.ok).toBe(true); + }); + it('models Fossify saves: rows re-inserted without DATA_SYNC', async () => { const fake = new FakeDeviceProviders(); const { rawId, dataId } = await contactWithEmail(fake); @@ -246,6 +272,15 @@ describe('FakeDeviceProviders: calendar', () => { expect(fake.row('events', exceptionId)).toBeDefined(); }); + it('leaves STATUS NULL on an event an app inserts without one (the column has no default)', async () => { + const fake = new FakeDeviceProviders(); + const { calendarId } = await calendar(fake); + const id = fake.user.insertEvent(calendarId, { [Events.DTSTART]: 0, [Events.DTEND]: 1000, [Events.EVENT_TIMEZONE]: 'UTC', [Events.TITLE]: 'x' }); + expect(fake.row('events', id)![Events.STATUS]).toBeNull(); + // Unlike AVAILABILITY and ACCESS_LEVEL (NOT NULL DEFAULT 0). + expect(fake.row('events', id)).toMatchObject({ [Events.AVAILABILITY]: 0, [Events.ACCESS_LEVEL]: 0 }); + }); + it('hard-deletes app-deleted events without _SYNC_ID and soft-deletes synced ones', async () => { const fake = new FakeDeviceProviders(); const { calendarId } = await calendar(fake); diff --git a/src/device-sync/__tests__/fakes/fake-jmap-server.ts b/src/device-sync/__tests__/fakes/fake-jmap-server.ts index 015a161d..892f4c7f 100644 --- a/src/device-sync/__tests__/fakes/fake-jmap-server.ts +++ b/src/device-sync/__tests__/fakes/fake-jmap-server.ts @@ -965,6 +965,8 @@ function normalizeEvent(input: Obj, now: string, mode: 'jmap' | 'server'): Obj { } privacy = Math.max(privacy, PRIVACY_RANK.indexOf(String(value.privacy))); for (const forbidden of FORBIDDEN_OVERRIDE_PROPERTIES) delete value[forbidden]; + // Stored as a VEVENT of its own: an empty map writes nothing there either. + dropEmptyMaps(value, EVENT_MAPS); if (Object.keys(value).length > 0) { normalizeEventTimes(value); addNestedTypes(value); diff --git a/src/device-sync/__tests__/fakes/fake-provider.ts b/src/device-sync/__tests__/fakes/fake-provider.ts index 100d5229..17c238f6 100644 --- a/src/device-sync/__tests__/fakes/fake-provider.ts +++ b/src/device-sync/__tests__/fakes/fake-provider.ts @@ -25,6 +25,12 @@ * owner, all-day times are zeroed in UTC and `PS` all-day durations become * `PD`, while `PT0S`-style all-day durations, NULL STATUS updates, * SELF_ATTENDEE_STATUS updates and RRULEs with unknown parts fail the batch. + * STATUS has no default: an event an app inserts without one holds NULL. + * - Photos, as PhotoProcessor keeps them: one within the 96 px thumbnail is + * the thumbnail alone (PHOTO_FILE_ID NULL), a bigger one (or bytes that are + * no JPEG or PNG) a display photo with a file id; DATA15 never reads back. + * - A batch over one Binder transaction (`estimateBatchBytes` over 1 MB, by + * default) fails with `tooLarge`. * - Scoping: rows of other accounts are invisible and untouchable. * - The native bridge's rules (android/.../sync/ProviderOps.kt and * ProviderScope.kt): an op with `id` expects exactly one row unless it says @@ -74,6 +80,7 @@ import { type WriteCell, type WriteRow, } from '../../types'; +import { estimateBatchBytes } from '../../engine/batch'; import { compileOrderBy, compileWhere, whereColumns } from './fake-sql'; export const FAKE_ACCOUNT_TYPE = 'com.anonymous.bulwarkmobile.account'; @@ -240,7 +247,7 @@ export interface FakeProviderOptions { accountType?: string; /** Derive/split names and addresses like ContactsProvider (default true). */ normalize?: boolean; - /** Byte budget of one applyBatch before it fails with `tooLarge` (default 1 MB). */ + /** Bytes of one applyBatch, as `estimateBatchBytes` weighs them, before it fails with `tooLarge` (default 1 MB). */ maxBatchBytes?: number; } @@ -517,7 +524,7 @@ export class FakeDeviceProviders { record(false); return { ok: false, reason, message }; } - if (JSON.stringify(ops).length > this.maxBatchBytes) { + if (parcelBytes(ops) > this.maxBatchBytes) { record(false); return { ok: false, reason: 'tooLarge', message: 'TransactionTooLargeException' }; } @@ -569,6 +576,26 @@ export class FakeDeviceProviders { } } +/** + * What a batch weighs in its Binder transaction: `estimateBatchBytes`, with a + * photo blob counted by its bytes (the native side hands ContentValues a + * decoded `byte[]`, not the base64 text). + */ +function parcelBytes(ops: readonly ProviderOp[]): number { + let blobs = 0; + const texts = ops.map((op) => { + if (op.op !== 'insert' && op.op !== 'update') return op; + const values = { ...op.values }; + for (const [column, value] of Object.entries(values)) { + if (value === null || typeof value !== 'object') continue; + blobs += Math.floor((value.b64.length * 3) / 4); + values[column] = null; + } + return { ...op, values }; + }); + return estimateBatchBytes(texts) + blobs; +} + function cloneTables(tables: Tables): Tables { const out = {} as Tables; for (const [name, rows] of Object.entries(tables) as Array<[ProviderTable, RowMap]>) { @@ -642,8 +669,11 @@ class FakePort implements ProviderPort { } for (const row of this.fake.table('data').values()) { if (row[Data.RAW_CONTACT_ID] === rawContactId && row[Data.MIMETYPE] === MimeType.PHOTO) { - const fileId = row[Data.DATA14] === null ? null : Number(row[Data.DATA14]); - const b64 = fileId === null ? undefined : this.fake.photo(fileId); + const fileId = row[Data.DATA14] === null || row[Data.DATA14] === undefined ? null : Number(row[Data.DATA14]); + // A photo kept as its thumbnail alone is read from the thumbnail. + const thumbnail = row[Data.DATA15]; + if (fileId === null) return typeof thumbnail === 'string' && thumbnail.startsWith('blob:') ? { jpegBase64: thumbnail.slice(5), fileId: null } : null; + const b64 = this.fake.photo(fileId); return b64 === undefined ? null : { jpegBase64: b64, fileId }; } } @@ -755,10 +785,19 @@ class BatchApplier { return out; } - /** A photo row's full-size bytes become a stored display photo plus a file id, like PhotoProcessor. */ + /** + * A photo row's full-size bytes, as PhotoProcessor keeps them: within the + * 96 px thumbnail as the thumbnail alone (PHOTO_FILE_ID NULL, the bytes kept + * for readPhoto), else as a stored display photo plus a file id. + */ private processPhoto(row: StoredRow): void { const v = row[Data.DATA15]; if (typeof v === 'string' && v.startsWith('blob:')) { + const size = imageSize(v.slice(5)); + if (size && Math.max(size.width, size.height) <= THUMBNAIL_PX) { + row[Data.DATA14] = null; + return; + } row[Data.DATA14] = this.fake.storePhoto(v.slice(5)); row[Data.DATA15] = 'thumbnail'; } @@ -996,6 +1035,34 @@ class BatchApplier { const defaults = (d: StoredRow): StoredRow => ({ ...d }); +/** ContactsProvider's thumbnail size: a photo no bigger is kept as the thumbnail alone. */ +const THUMBNAIL_PX = 96; + +/** The pixel size of JPEG (its SOFn segment) or PNG (its IHDR chunk) bytes; null for anything else. */ +function imageSize(b64: string): { width: number; height: number } | null { + let bytes: Uint8Array; + try { + bytes = Uint8Array.from(atob(b64), (c) => c.charCodeAt(0)); + } catch { + return null; + } + const u16 = (i: number) => (bytes[i] << 8) | bytes[i + 1]; + const u32 = (i: number) => ((bytes[i] << 24) >>> 0) + (bytes[i + 1] << 16) + (bytes[i + 2] << 8) + bytes[i + 3]; + if (bytes.length >= 24 && bytes[0] === 0x89 && String.fromCharCode(...bytes.slice(12, 16)) === 'IHDR') { + return { width: u32(16), height: u32(20) }; + } + if (bytes[0] !== 0xff || bytes[1] !== 0xd8) return null; + for (let i = 2; i + 9 < bytes.length; ) { + if (bytes[i] !== 0xff) return null; + const marker = bytes[i + 1]; + if (marker >= 0xc0 && marker <= 0xcf && marker !== 0xc4 && marker !== 0xc8 && marker !== 0xcc) { + return { height: u16(i + 5), width: u16(i + 7) }; + } + i += 2 + u16(i + 2); + } + return null; +} + const RAW_CONTACT_DEFAULTS: StoredRow = { [RawContacts.SOURCE_ID]: null, [RawContacts.VERSION]: 1, @@ -1045,7 +1112,7 @@ const EVENT_DEFAULTS: StoredRow = { [Events.GUESTS_CAN_INVITE_OTHERS]: 1, [Events.GUESTS_CAN_SEE_GUESTS]: 1, [Events.ALL_DAY]: 0, - [Events.STATUS]: 0, + // No default for STATUS (`eventStatus INTEGER`): an app that writes none leaves NULL. }; /** From 66789251e3635fb7a2527c386a48f3713875e95e Mon Sep 17 00:00:00 2001 From: Linus Rath <139418639+rathlinus@users.noreply.github.com> Date: Sun, 27 Sep 2026 09:58:56 +0200 Subject: [PATCH 70/87] fix: keep a meeting moved in Etar the same event instead of cancelling and re-inviting #34 Etar moves an event by inserting a copy from its editor and deleting the old row. Its editor keeps the user's own attendee row out of the attendee list, so the copy of a meeting that lists the user has no row for the user, and the copy of one that does not gets the user as its organizer. The pairing compared the attendee addresses of both rows exactly, so a moved meeting never paired: it was deleted on the server and created again under a new uid, the other attendees' answers were lost, and everyone got a cancellation and a new invitation. The pairing now leaves the user's own addresses out of that comparison. Anyone else still has to be on both rows, so a copy that lost or gained another attendee is not taken for the same event. Checked with planner tests for both kinds of meeting and a run through the engine with a fake Etar move. --- .../__tests__/calendar/lifecycle.test.ts | 50 +++++++++++++++++++ .../__tests__/engine/real-planners.test.ts | 41 +++++++++++++++ .../__tests__/fakes/fake-provider.ts | 46 +++++++++++++++++ src/device-sync/calendar/pairs.ts | 18 +++++-- src/device-sync/calendar/planner.ts | 2 +- 5 files changed, 152 insertions(+), 5 deletions(-) diff --git a/src/device-sync/__tests__/calendar/lifecycle.test.ts b/src/device-sync/__tests__/calendar/lifecycle.test.ts index 52546c45..038d2d00 100644 --- a/src/device-sync/__tests__/calendar/lifecycle.test.ts +++ b/src/device-sync/__tests__/calendar/lifecycle.test.ts @@ -492,6 +492,56 @@ describe('calendar planner: pairs', () => { expect(calendarPlanner.planUpload({ ...after, dirty: true }, h.ctx).kind).toBe('clean'); }); + describe("Etar's move of a meeting (its copy leaves the user's own row out, or adds the user as organizer)", () => { + const emailsOf = (h: Harness, eventId: number) => + h.fake.rows('attendees', `${Attendees.EVENT_ID} = ?`, [eventId]).map((a) => String(a[Attendees.ATTENDEE_EMAIL])).sort(); + const person = (address: string, extra: Record = {}) => ({ '@type': 'Participant', calendarAddress: `mailto:${address}`, roles: { attendee: true }, ...extra }); + const moveToWork = { kind: 'update', id: 'e1', patch: { 'calendarIds/b': null, 'calendarIds/w': true } }; + + it('pairs a meeting that lists the user', async () => { + const participants = { + me: person(ME, { roles: { owner: true, attendee: true }, participationStatus: 'accepted' }), + bob: person('bob@example.net', { name: 'Bob', participationStatus: 'accepted' }), + } as CalendarEventWire['participants']; + const { h, id } = await synced(single({ participants, organizerCalendarAddress: `mailto:${ME}` })); + + const moved = h.fake.user.etarMove(id, h.rowIds.get('w')!); + + expect(emailsOf(h, moved)).toEqual(['bob@example.net']); + expect((await pairAndApply(h)).actions).toEqual([moveToWork]); + expect((await h.local('e1'))!).toMatchObject({ eventId: moved, dirty: false }); + }); + + it('pairs a meeting that does not list the user', async () => { + const participants = { + boss: person('boss@example.net', { roles: { owner: true, attendee: true }, participationStatus: 'accepted' }), + bob: person('bob@example.net', { participationStatus: 'needs-action' }), + } as CalendarEventWire['participants']; + const { h, id } = await synced(single({ participants, organizerCalendarAddress: 'mailto:boss@example.net' })); + + const moved = h.fake.user.etarMove(id, h.rowIds.get('w')!); + + expect(emailsOf(h, moved)).toEqual(['bob@example.net', 'boss@example.net', ME]); + expect((await pairAndApply(h)).actions).toEqual([moveToWork]); + }); + + it('never pairs a copy that lost or gained someone else', async () => { + const participants = { + me: person(ME, { roles: { owner: true, attendee: true }, participationStatus: 'accepted' }), + bob: person('bob@example.net'), + carol: person('carol@example.net'), + } as CalendarEventWire['participants']; + const attendee = (email: string) => ({ [Attendees.ATTENDEE_EMAIL]: email, [Attendees.ATTENDEE_RELATIONSHIP]: 1, [Attendees.ATTENDEE_TYPE]: 1, [Attendees.ATTENDEE_STATUS]: 0 }); + for (const copied of [['bob@example.net'], ['bob@example.net', 'carol@example.net', 'dave@example.net']]) { + const { h, id } = await synced(single({ participants, organizerCalendarAddress: `mailto:${ME}` })); + h.fake.user.deleteEvent(id); + h.fake.user.insertEvent(h.rowIds.get('w')!, lunchColumns, { attendees: copied.map(attendee) }); + const events = await h.events(); + expect(calendarPlanner.planPairs(events.filter((e) => e.deleted), events.filter((e) => !e.syncId), h.ctx), copied.join()).toEqual([]); + } + }); + }); + it('keeps the timing of a series turned single against the series, so a start moved before uploads', async () => { const { h, id } = await synced(single({ recurrenceRule: { frequency: 'daily' } })); // Moved an hour later in place, then turned into a single event (Etar: delete + insert). diff --git a/src/device-sync/__tests__/engine/real-planners.test.ts b/src/device-sync/__tests__/engine/real-planners.test.ts index 1f95bb6c..1f466ac9 100644 --- a/src/device-sync/__tests__/engine/real-planners.test.ts +++ b/src/device-sync/__tests__/engine/real-planners.test.ts @@ -697,6 +697,47 @@ describe('device sync engine with the real planners', () => { expect(rowWrites(h.batches.log.slice(before))).toEqual([]); }); + it('keeps a meeting Etar moved the same event, without cancelling and inviting anyone again', async () => { + const h = real(); + const work = h.server.addCalendar('a', { name: 'Work' }); + const lunch = h.server.addEvent('a', { + uid: 'lunch-uid', + title: 'Lunch', + start: '2026-10-06T12:00:00', + duration: 'PT1H', + timeZone: 'Europe/Berlin', + organizerCalendarAddress: 'mailto:alice@example.com', + participants: { + me: { '@type': 'Participant', calendarAddress: 'mailto:alice@example.com', roles: { owner: true, attendee: true }, participationStatus: 'accepted' }, + bob: { '@type': 'Participant', calendarAddress: 'mailto:bob@example.com', name: 'Bob', roles: { attendee: true, optional: true }, participationStatus: 'accepted' }, + }, + calendarIds: { [h.calendar]: true }, + }); + await h.run(CALENDAR_AUTHORITY); + const old = h.events().find((e) => e._sync_id === `a/${lunch}`)!; + const workRow = Number(h.device.rows('calendars').find((c) => c._sync_id === `a/${work}`)!._id); + + // What Etar really inserts: Bob as a required attendee without an answer, and no row for the user. + const moved = h.device.user.etarMove(Number(old._id), workRow); + const report = await h.run(CALENDAR_AUTHORITY); + + expect(report).toMatchObject({ outcome: 'ok', itemErrors: [], stats: { uploaded: { created: 0, updated: 1, deleted: 0 } } }); + expect(h.server.all('CalendarEvent', 'a')).toHaveLength(1); + expect(h.server.get('CalendarEvent', 'a', lunch)).toMatchObject({ + uid: 'lunch-uid', + calendarIds: { [work]: true }, + participants: { bob: { roles: { attendee: true, optional: true }, participationStatus: 'accepted' } }, + }); + expect(h.server.scheduling).toEqual([]); + // The rows show the meeting as the server holds it again: the user's own row and Bob's answer. + const attendees = h.device.rows('attendees').filter((a) => Number(a.event_id) === moved); + expect(attendees.map((a) => [a[Attendees.ATTENDEE_EMAIL], a[Attendees.ATTENDEE_STATUS], a[Attendees.ATTENDEE_TYPE]]).sort()).toEqual([ + ['alice@example.com', 1, 1], + ['bob@example.com', 1, 2], + ]); + expect(h.events()).toEqual([expect.objectContaining({ _id: moved, _sync_id: `a/${lunch}`, dirty: 0 })]); + }); + it('uploads a series Etar turned into a single event as the removal of its rule', async () => { const h = real(); const series = h.server.addEvent('a', { diff --git a/src/device-sync/__tests__/fakes/fake-provider.ts b/src/device-sync/__tests__/fakes/fake-provider.ts index 17c238f6..8e1fa716 100644 --- a/src/device-sync/__tests__/fakes/fake-provider.ts +++ b/src/device-sync/__tests__/fakes/fake-provider.ts @@ -45,6 +45,9 @@ * Pure TypeScript, for vitest only. */ import { + AttendeeRelationship, + AttendeeStatus, + AttendeeType, Attendees, Calendars, Colors, @@ -1448,4 +1451,47 @@ export class FakeUser { for (const a of attendees) this.fake.table('attendees').set(this.fake.allocId(), { ...a, event_id: eventId }); this.setReminders(eventId, reminders); } + + /** + * Etar's move to another calendar (EditEventHelper.saveEvent → moveEventToCalendar): a copy of the + * event from its editor model is inserted there, then the event is deleted. The model leaves the + * owner's own attendee row out (EditEventFragment: ATTENDEE_EMAIL equal to the calendar's + * OWNER_ACCOUNT, ignoring case), so the copy gets every other attendee as a required attendee + * without a status, and, when the owner had no row, the owner as its organizer. The reminders are + * copied. (Exceptions Etar never synced, which it moves too on API 30+, are not modelled.) + */ + etarMove(eventId: number, calendarId: number, changes: Row = {}): number { + const old = this.event(eventId); + const owner = String(this.fake.table('calendars').get(Number(old[Events.CALENDAR_ID]))?.[Calendars.OWNER_ACCOUNT] ?? ''); + const isOwner = (a: StoredRow) => String(a[Attendees.ATTENDEE_EMAIL] ?? '').toLowerCase() === owner.toLowerCase(); + const values: Row = {}; + for (const c of ETAR_COPY) if (old[c] !== undefined) values[c] = old[c]; + const rows = [...this.fake.table('attendees').values()].filter((a) => a.event_id === eventId); + const others: Row[] = rows.filter((a) => !isOwner(a)).map((a) => ({ + [Attendees.ATTENDEE_NAME]: a[Attendees.ATTENDEE_NAME] ?? null, + [Attendees.ATTENDEE_EMAIL]: a[Attendees.ATTENDEE_EMAIL], + [Attendees.ATTENDEE_RELATIONSHIP]: AttendeeRelationship.ATTENDEE, + [Attendees.ATTENDEE_TYPE]: AttendeeType.REQUIRED, + [Attendees.ATTENDEE_STATUS]: AttendeeStatus.NONE, + })); + const organizer: Row[] = rows.some(isOwner) || !others.length ? [] : [{ + [Attendees.ATTENDEE_EMAIL]: owner, + [Attendees.ATTENDEE_RELATIONSHIP]: AttendeeRelationship.ORGANIZER, + [Attendees.ATTENDEE_TYPE]: AttendeeType.REQUIRED, + [Attendees.ATTENDEE_STATUS]: AttendeeStatus.ACCEPTED, + }]; + const reminders = [...this.fake.table('reminders').values()] + .filter((r) => r.event_id === eventId) + .map((r) => ({ [Reminders.MINUTES]: r[Reminders.MINUTES], [Reminders.METHOD]: r[Reminders.METHOD] })); + const id = this.insertEvent(calendarId, { ...values, ...changes }, { attendees: [...organizer, ...others], reminders }); + this.deleteEvent(eventId); + return id; + } } + +/** The event columns Etar's editor model writes into the copy of an event it moves (getContentValuesFromModel). */ +const ETAR_COPY = [ + Events.TITLE, Events.DESCRIPTION, Events.EVENT_LOCATION, Events.STATUS, Events.AVAILABILITY, Events.ACCESS_LEVEL, + Events.EVENT_COLOR, Events.DTSTART, Events.DTEND, Events.DURATION, Events.EVENT_TIMEZONE, Events.ALL_DAY, Events.RRULE, + Events.EXDATE, +]; diff --git a/src/device-sync/calendar/pairs.ts b/src/device-sync/calendar/pairs.ts index 1788bfcb..267066f6 100644 --- a/src/device-sync/calendar/pairs.ts +++ b/src/device-sync/calendar/pairs.ts @@ -15,6 +15,7 @@ import type { CalendarEventWire } from '../wire'; import { exceptionRef, parseObjectRef } from '../common/ids'; import { clone } from '../common/json'; import { applyPatch, type PatchObject } from '../common/patch'; +import { isSelfAddress, type SelfContext } from './attendees'; import { EXCEPTION_CELLS, MASTER_CELLS, encodeBaseline, makeBaseline } from './columns'; import { durationTextSeconds } from './duration'; import { isExcluded, overridesOf } from './exceptions'; @@ -49,8 +50,17 @@ function lengthMs(cells: Row): number | null { return seconds === null ? null : seconds * 1000; } -const attendeeSet = (row: LocalEventRow) => - row.attendees.map((a) => String(a.cells[Attendees.ATTENDEE_EMAIL] ?? '').toLowerCase()).sort().join(','); +/** + * The attendees' addresses, the user's own left out: Etar's copy of a meeting + * has no row for the user when the user was listed (its editor keeps the + * owner's row out of its attendee list), and one as the organizer when not. + */ +const attendeeSet = (row: LocalEventRow, self: SelfContext) => + row.attendees + .map((a) => String(a.cells[Attendees.ATTENDEE_EMAIL] ?? '').toLowerCase()) + .filter((email) => !isSelfAddress(email, self)) + .sort() + .join(','); export interface PairKind { /** The same event in another calendar. */ @@ -60,7 +70,7 @@ export interface PairKind { } /** How a deleted and a new row are one edit, or null. */ -export function pairKind(deleted: LocalEvent, fresh: LocalEvent): PairKind | null { +export function pairKind(deleted: LocalEvent, fresh: LocalEvent, self: SelfContext): PairKind | null { for (const c of PAIR_COLUMNS) { // A copy inserted without STATUS (Google Calendar) says nothing about it. if (c === Events.STATUS && (isUnsetStatus(deleted.cells[c]) || isUnsetStatus(fresh.cells[c]))) continue; @@ -69,7 +79,7 @@ export function pairKind(deleted: LocalEvent, fresh: LocalEvent): PairKind | nul const allDay = Number(fresh.cells[Events.ALL_DAY] ?? 0) === 1; if (!allDay && String(deleted.cells[Events.EVENT_TIMEZONE] ?? '') !== String(fresh.cells[Events.EVENT_TIMEZONE] ?? '')) return null; if (lengthMs(deleted.cells) !== lengthMs(fresh.cells)) return null; - if (attendeeSet(deleted) !== attendeeSet(fresh)) return null; + if (attendeeSet(deleted, self) !== attendeeSet(fresh, self)) return null; const move = deleted.calendarRowId !== fresh.calendarRowId; const deletedRule = rruleParts(deleted.cells[Events.RRULE]); const freshRule = rruleParts(fresh.cells[Events.RRULE]); diff --git a/src/device-sync/calendar/planner.ts b/src/device-sync/calendar/planner.ts index ac65d474..7a01bbfd 100644 --- a/src/device-sync/calendar/planner.ts +++ b/src/device-sync/calendar/planner.ts @@ -391,7 +391,7 @@ function planPairs( refuseTask(d.shadow); for (const f of fresh) { if (used.has(f.eventId) || f.syncId !== null || f.pending || f.deleted || isExceptionRow(f.cells) || !inAccount(f)) continue; - const kind = pairKind(d, f); + const kind = pairKind(d, f, ctx); if (!kind) continue; const pair = pairPatch(d, f, kind, ctx); if (!pair) continue; From 029eba4562a22090cd046bd63f6af312bba8a128 Mon Sep 17 00:00:00 2001 From: Linus Rath <139418639+rathlinus@users.noreply.github.com> Date: Sun, 27 Sep 2026 09:59:53 +0200 Subject: [PATCH 71/87] fix: keep the last sync's result when a sync after it finds nothing to do #34 The settings show the stored status of the last sync. A sync that finds nothing to do still replaced it, so a change the server refused and the phone put back, or a conflict, was shown for a second or two and then gone. The previous fix covered only the upload sync Android starts right after a manual one. When changes arrive during a sync, the adapter runs a full sync right after it, which finds nothing new on the server and replaced the status 1-2 s after the real one; a manual sync right after another does the same. Now what counts is what a run did, not how it started. A run that wrote nothing either way and reported no conflict, item error, note or failure keeps the details of the run before it and moves only the time on. After a failed run it shows the sync as working again, without the failure's message. Any other run replaces the status, so a problem stays only until the next run that does something. --- .../__tests__/engine/contacts-flow.test.ts | 13 ++-- src/device-sync/__tests__/engine/harness.ts | 10 ++- .../__tests__/engine/internals.test.ts | 47 +++++++++++++- .../__tests__/engine/real-planners.test.ts | 65 +++++++++++++++++++ src/device-sync/__tests__/engine/task.test.ts | 14 +++- src/device-sync/engine/deps.ts | 2 + src/device-sync/engine/report.ts | 34 ++++++++++ src/device-sync/engine/run.ts | 52 +++++++-------- src/device-sync/task.ts | 6 +- 9 files changed, 206 insertions(+), 37 deletions(-) diff --git a/src/device-sync/__tests__/engine/contacts-flow.test.ts b/src/device-sync/__tests__/engine/contacts-flow.test.ts index 54c7dfa2..35db0a97 100644 --- a/src/device-sync/__tests__/engine/contacts-flow.test.ts +++ b/src/device-sync/__tests__/engine/contacts-flow.test.ts @@ -2,7 +2,7 @@ // server: first sync, incremental /changes, uploads, echoes, selection // changes and groups (docs/device-sync.md, "A sync run"). -import { describe, expect, it, vi } from 'vitest'; +import { describe, expect, it } from 'vitest'; import { Data, MimeType } from '../../android-columns'; import { runDeviceSync } from '../../engine/run'; import { @@ -295,21 +295,22 @@ describe('device sync engine: contacts', () => { expect(h.server.requests.length).toBe(requests); }); - it("keeps the last run's status (its conflicts and item errors) when an upload-only sync finds nothing to do", async () => { + it("keeps the last run's status when an upload-only sync finds nothing to do, moving only its time on", async () => { const h = createHarness(); addServerCards(h, ['Ada']); await h.run(); - const recordStatus = vi.mocked(h.deps.recordStatus); - const recorded = recordStatus.mock.calls.length; + const status = h.lastStatus()!; + expect(status).toMatchObject({ outcome: 'ok', stats: { downloaded: { created: 1 } } }); // The upload sync Android starts right after a manual one: nothing waits on the device. + h.clock.now += 1_000; expect((await h.run(CONTACTS_AUTHORITY, { upload: true })).outcome).toBe('ok'); - expect(recordStatus.mock.calls.length).toBe(recorded); + expect(h.lastStatus()).toEqual({ ...status, at: h.clock.now }); // One that has something to upload is a run like any other. renameDeviceContact(h, h.contactNamed('Ada')!.id, 'Ada King'); await h.run(CONTACTS_AUTHORITY, { upload: true }); - expect(recordStatus.mock.calls.length).toBe(recorded + 1); + expect(h.lastStatus()).toMatchObject({ stats: { downloaded: { created: 0 }, uploaded: { updated: 1 } } }); }); it('reports disabled, unsupported and rows of another owner without touching anything', async () => { diff --git a/src/device-sync/__tests__/engine/harness.ts b/src/device-sync/__tests__/engine/harness.ts index dc13849e..c86135c2 100644 --- a/src/device-sync/__tests__/engine/harness.ts +++ b/src/device-sync/__tests__/engine/harness.ts @@ -19,6 +19,7 @@ import { type ProviderPort, type RunExtras, type RunReport, + type RunStatus, type Row, } from '../../types'; import { FakeDeviceProviders } from '../fakes/fake-provider'; @@ -67,6 +68,8 @@ export interface Harness { nameDataRow(rawContactId: number): Row | undefined; events(): Row[]; serverNames(accountId?: string): string[]; + /** The status the runs recorded last for the settings (what `deps.lastStatus` answers). */ + lastStatus(authority?: Authority): RunStatus | undefined; } export function createHarness(options: { tuning?: Partial; accounts?: 'personal' | 'withShared'; rebuild?: boolean } = {}): Harness { @@ -84,6 +87,7 @@ export function createHarness(options: { tuning?: Partial; accounts?: 'p const prefs: AccountSyncPrefs = { contactsSelection: {}, calendarSelection: {}, reminderOwner: 'device' }; const checkpoints: Harness['checkpoints'] = { count: 0, crashAt: null }; const batches: Harness['batches'] = { applied: 0, crashAfter: null, log: [] }; + const statuses = new Map(); const wrap = (port: ProviderPort): ProviderPort => ({ accountName: port.accountName, @@ -119,7 +123,10 @@ export function createHarness(options: { tuning?: Partial; accounts?: 'p subscriptionCalendars: async () => [], isSyncEnabled: async () => true, deviceZone: () => 'Europe/Berlin', - recordStatus: vi.fn(), + recordStatus: vi.fn((_registryId: string, authority: Authority, status: RunStatus) => { + statuses.set(authority, status); + }), + lastStatus: (_registryId, authority) => statuses.get(authority), recordKnownState: vi.fn(), notifyAuthProblem: vi.fn(), yieldThread: async () => undefined, @@ -168,6 +175,7 @@ export function createHarness(options: { tuning?: Partial; accounts?: 'p .filter((c) => c.kind !== 'group') .map((c) => String((c.name as { full?: string } | undefined)?.full ?? '')) .sort(), + lastStatus: (authority = CONTACTS_AUTHORITY) => statuses.get(authority), }; return harness; } diff --git a/src/device-sync/__tests__/engine/internals.test.ts b/src/device-sync/__tests__/engine/internals.test.ts index 33323444..32c20531 100644 --- a/src/device-sync/__tests__/engine/internals.test.ts +++ b/src/device-sync/__tests__/engine/internals.test.ts @@ -7,13 +7,14 @@ import { BatchWriter, buildBatch, estimateBatchBytes, MAX_OPS_PER_YIELD, prepend import { classifyFailure, RunAbort, StateMismatch, StopRun } from '../../engine/errors'; import { acquireLock } from '../../engine/mutex'; import { isBackedOff, nextMarker } from '../../engine/poison'; +import { emptyStats, isQuietRun, statusToRecord } from '../../engine/report'; import { collectionDefaultOn, isCollectionSelected } from '../../engine/selection'; import { accountOf, emptySyncState, parseSyncState, serializeSyncState, StateStore, type SyncState } from '../../engine/sync-state'; import { classifySetError, existingIdFromUidError, JmapCaller } from '../../jmap/caller'; import { openWithAuthRebuild } from '../../jmap/connection'; import { JMAPMethodError } from '../../jmap/errors'; import { accountsWithCapability, calendarAddresses } from '../../jmap/session'; -import { JMAP_CALENDARS, JMAP_CONTACTS, type BatchResult, type ProviderOp, type ProviderPort } from '../../types'; +import { JMAP_CALENDARS, JMAP_CONTACTS, type BatchResult, type ProviderOp, type ProviderPort, type RunStats, type RunStatus } from '../../types'; import { FakeJmapServer } from '../fakes/fake-jmap-server'; function port(apply: (ops: ProviderOp[]) => BatchResult | Promise): ProviderPort & { sent: ProviderOp[][] } { @@ -459,3 +460,47 @@ describe('outcomes', () => { expect(classifyFailure(new TypeError('x is undefined'), 0)).toMatchObject({ outcome: 'internal' }); }); }); + +describe('the status a run leaves for the settings', () => { + const status = (over: Partial = {}): RunStatus => ({ at: 2_000, outcome: 'ok', durationMs: 50, conflicts: 0, itemErrors: 0, stats: emptyStats(), ...over }); + const withStats = (edit: (stats: RunStats) => void) => { + const stats = emptyStats(); + edit(stats); + return status({ stats }); + }; + const reported = status({ at: 1_000, durationMs: 900, conflicts: 1, itemErrors: 2, message: 'invitations not sent', stats: { ...emptyStats(), skipped: 2 } }); + + it('keeps what the run before reported through a run that did nothing, moving the time on', () => { + expect(isQuietRun(status())).toBe(true); + expect(statusToRecord(status(), reported)).toEqual({ ...reported, at: 2_000 }); + expect(statusToRecord(status(), undefined)).toEqual(status()); + }); + + it('shows a failure followed by a run that did nothing as recovered, with what the failed run reported', () => { + const failed = { ...reported, outcome: 'io' as const, message: 'Could not reach the server' }; + const recorded = statusToRecord(status(), failed); + + expect(recorded).toEqual({ at: 2_000, outcome: 'ok', durationMs: 900, conflicts: 1, itemErrors: 2, stats: reported.stats }); + }); + + it('replaces it with a run that wrote, skipped or reported anything, or failed', () => { + const busy = [ + withStats((s) => (s.downloaded.created = 1)), + withStats((s) => (s.downloaded.deleted = 1)), + withStats((s) => (s.uploaded.updated = 1)), + withStats((s) => (s.skipped = 1)), + status({ conflicts: 1 }), + status({ itemErrors: 1 }), + status({ message: 'invitations not sent' }), + status({ outcome: 'io', message: 'Could not reach the server' }), + status({ outcome: 'cancelled' }), + status({ outcome: 'disabled', message: 'Sync is off for this account' }), + ]; + for (const run of busy) { + expect(isQuietRun(run)).toBe(false); + expect(statusToRecord(run, reported)).toBe(run); + } + // Items only looked at are no work. + expect(isQuietRun(withStats((s) => (s.entries = 3)))).toBe(true); + }); +}); diff --git a/src/device-sync/__tests__/engine/real-planners.test.ts b/src/device-sync/__tests__/engine/real-planners.test.ts index 1f466ac9..0652f345 100644 --- a/src/device-sync/__tests__/engine/real-planners.test.ts +++ b/src/device-sync/__tests__/engine/real-planners.test.ts @@ -452,6 +452,71 @@ describe('device sync engine with the real planners', () => { }); }); + describe('the last run status the settings show', () => { + /** A run that puts back an edit the server refuses: reported once, nothing waits afterwards. */ + async function refusedEdit() { + const h = real(); + const [ada] = addServerCards(h, ['Ada Lovelace', 'Grace Hopper']); + await h.run(); + renameDeviceContact(h, h.contactNamed('Ada Lovelace')!.id, 'Ada King'); + h.server.setErrorFor('ContactCard', 'a', ada, { type: 'forbidden' }); + h.clock.now += 60_000; + expect((await h.run()).itemErrors).toEqual([expect.objectContaining({ type: 'forbidden' })]); + const status = h.lastStatus()!; + expect(status).toMatchObject({ outcome: 'ok', itemErrors: 1 }); + return { h, status }; + } + + it('keeps what a run reported through an upload sync after it that finds nothing to do', async () => { + const { h, status } = await refusedEdit(); + h.clock.now += 2_000; + + await h.run(CONTACTS_AUTHORITY, { upload: true }); + + expect(h.lastStatus()).toEqual({ ...status, at: expect.any(Number) }); + expect(h.lastStatus()!.at).toBeGreaterThan(status.at); + }); + + it('keeps what a run reported through a follow-up sync that finds nothing new on the server', async () => { + const { h, status } = await refusedEdit(); + h.clock.now += 2_000; + + expect((await h.run()).itemErrors).toEqual([]); + + expect(h.lastStatus()).toEqual({ ...status, at: expect.any(Number) }); + expect(h.lastStatus()!.at).toBeGreaterThan(status.at); + }); + + it('replaces it with a run that did something', async () => { + const { h } = await refusedEdit(); + addServerCards(h, ['Linus Torvalds']); + + await h.run(); + + expect(h.lastStatus()).toMatchObject({ outcome: 'ok', itemErrors: 0, stats: { downloaded: { created: 1 } } }); + }); + + it('shows a sync that finds nothing to do after a failed one as recovered', async () => { + const h = real(); + addServerCards(h, ['Ada Lovelace']); + await h.run(); + renameDeviceContact(h, h.contactNamed('Ada Lovelace')!.id, 'Ada King'); + h.server.failNextRequest('network', { match: 'ContactCard/set' }); + expect((await h.run()).outcome).toBe('io'); + expect(h.lastStatus()).toMatchObject({ outcome: 'io' }); + // Uploaded by the next sync; the one after it finds nothing to do. + expect((await h.run()).outcome).toBe('ok'); + h.clock.now += 2_000; + h.server.failNextRequest('network', { match: 'ContactCard/changes' }); + expect((await h.run()).outcome).toBe('io'); + + expect((await h.run()).outcome).toBe('ok'); + + expect(h.lastStatus()).toMatchObject({ outcome: 'ok', itemErrors: 0, conflicts: 0 }); + expect(h.lastStatus()).not.toHaveProperty('message'); + }); + }); + describe('a create whose target the server deleted after the claim', () => { it('claims a new contact again for a book that exists', async () => { const h = real(); diff --git a/src/device-sync/__tests__/engine/task.test.ts b/src/device-sync/__tests__/engine/task.test.ts index 384fc21c..5bc8adaa 100644 --- a/src/device-sync/__tests__/engine/task.test.ts +++ b/src/device-sync/__tests__/engine/task.test.ts @@ -15,7 +15,8 @@ vi.mock('../../native', () => ({ })); import * as native from '../../native'; -import { runDeviceSyncTask, teardownAuthority } from '../../task'; +import { emptyStats } from '../../engine/report'; +import { createTaskDeps, runDeviceSyncTask, teardownAuthority } from '../../task'; import { useDeviceSyncStore } from '../../../stores/device-sync-store'; import type { RunPayload } from '../../types'; @@ -42,6 +43,17 @@ describe('device sync task', () => { }); }); + it('reads back the status it records, for a run that did nothing to keep', async () => { + const deps = createTaskDeps(); + const status = { at: 5, outcome: 'ok' as const, durationMs: 3, conflicts: 1, itemErrors: 2, stats: emptyStats() }; + + await deps.recordStatus('bob@mail.example.com', 'com.android.calendar', status); + + await expect(deps.lastStatus('bob@mail.example.com', 'com.android.calendar')).resolves.toEqual(status); + await expect(deps.lastStatus('bob@mail.example.com', 'com.android.contacts')).resolves.toBeUndefined(); + await expect(deps.lastStatus('nobody@mail.example.com', 'com.android.calendar')).resolves.toBeUndefined(); + }); + it('reports a failure instead of throwing, and survives a refused finishRun', async () => { vi.mocked(native.getSyncSettings).mockRejectedValueOnce(new Error('module gone')); vi.mocked(native.finishRun).mockRejectedValueOnce(new Error('bridge down')); diff --git a/src/device-sync/engine/deps.ts b/src/device-sync/engine/deps.ts index 8f5bb227..28c9b98d 100644 --- a/src/device-sync/engine/deps.ts +++ b/src/device-sync/engine/deps.ts @@ -39,6 +39,8 @@ export interface EngineDeps { deviceZone(): string; /** The last run's status for the settings UI. */ recordStatus(registryId: string, authority: Authority, status: RunStatus): void | Promise; + /** The status `recordStatus` stored last, if any (a run that did nothing keeps its details). */ + lastStatus(registryId: string, authority: Authority): RunStatus | undefined | Promise; /** The latest state the engine synced for a JMAP account and type, so the app skips the echo of our own writes. */ recordKnownState(registryId: string, jmapAccountId: string, type: string, state: string): void | Promise; /** One notification that deep-links to sign-in, after an auth failure the rebuild did not fix. */ diff --git a/src/device-sync/engine/report.ts b/src/device-sync/engine/report.ts index 6f810607..b324e652 100644 --- a/src/device-sync/engine/report.ts +++ b/src/device-sync/engine/report.ts @@ -93,3 +93,37 @@ export function statusOf(report: RunReport, itemErrorCount = report.itemErrors.l if (report.message) status.message = report.message; return status; } + +const wrote =(counts: RunStats['downloaded']) => counts.created + counts.updated + counts.deleted > 0; + +/** A run that wrote nothing either way and had nothing to report: no conflict, item error, note or failure. */ +export function isQuietRun(status: RunStatus): boolean { + const { stats } = status; + return ( + status.outcome === 'ok' && + !status.conflicts && + !status.itemErrors && + !status.message && + !stats.skipped && + !wrote(stats.downloaded) && + !wrote(stats.uploaded) + ); +} + +/** + * The status the settings keep after a run. A quiet run (`isQuietRun`: the + * upload sync Android starts after a manual one, the follow-up sync the + * adapter asks for when changes arrived during a run, a manual sync right + * after an automatic one) keeps what the run before it reported, its + * conflicts, item errors and counts, and moves only the time on: otherwise + * a put-back edit or a conflict is reported for a second and gone. Its + * outcome is its own, so a failure before it shows as recovered, without the + * failure's message. Any other run replaces the status, so a problem stays + * only until the next run that does something. + */ +export function statusToRecord(status: RunStatus, previous: RunStatus | undefined): RunStatus { + if (!previous || !isQuietRun(status)) return status; + const kept: RunStatus = { ...previous, at: status.at, outcome: status.outcome }; + if (previous.outcome !== 'ok') delete kept.message; + return kept; +} diff --git a/src/device-sync/engine/run.ts b/src/device-sync/engine/run.ts index 4fb8c348..238d0aaa 100644 --- a/src/device-sync/engine/run.ts +++ b/src/device-sync/engine/run.ts @@ -16,6 +16,7 @@ import { type RunExtras, type RunPayload, type RunReport, + type RunStatus, } from '../types'; import { BatchWriter } from './batch'; import { CalendarSync } from './calendar-sync'; @@ -26,7 +27,7 @@ import { DEFAULT_TUNING, type EngineDeps, type Tuning } from './deps'; import { classifyFailure, RunAbort } from './errors'; import { acquireLock, lockKey, stopRequested } from './mutex'; import { ProviderReader, str } from './provider'; -import { ReportBuilder, statusOf } from './report'; +import { isQuietRun, ReportBuilder, statusOf, statusToRecord } from './report'; import { emptySyncState, parseSyncState, StateStore, type SyncState } from './sync-state'; import type { ItemSync } from './item-sync'; @@ -156,36 +157,36 @@ export async function runDeviceSync(payload: RunPayload, deps: EngineDeps): Prom const report = new ReportBuilder(payload.runId, payload.authority, deps.now(), tuning.maxItemErrors); const key = lockKey(payload.registryId, payload.authority); const release = await acquireLock(key, Math.max(0, payload.deadline - deps.now() - tuning.deadlineMarginMs)); - let result: RunReport; - const run = { idle: false }; if (!release) { - result = report.build('cancelled', deps.now(), { message: 'Another sync of this account is still running', moreRecordsToGet: true }); - } else { - try { - result = await runLocked(payload, deps, tuning, report, key, run); - } finally { - release(); - } + const result = report.build('cancelled', deps.now(), { message: 'Another sync of this account is still running', moreRecordsToGet: true }); + await recordStatus(payload, deps, statusOf(result, report.itemErrorCount)); + return result; } - // An upload sync with nothing to upload (Android starts one right after a manual sync) did nothing: - // the last run's status, with its conflicts and item errors, stays what the settings show. - if (run.idle) return result; try { - await deps.recordStatus(payload.registryId, payload.authority, statusOf(result, report.itemErrorCount)); + const result = await runLocked(payload, deps, tuning, report, key); + // Still under the lock: no other run of the account and authority records between the read and the write. + await recordStatus(payload, deps, statusOf(result, report.itemErrorCount)); + return result; + } finally { + release(); + } +} + +/** + * Stores the status the settings show. A run that did nothing (the upload + * sync Android starts after a manual one, the follow-up of a run during which + * changes arrived) keeps the details of the run before it (`statusToRecord`). + */ +async function recordStatus(payload: RunPayload, deps: EngineDeps, status: RunStatus): Promise { + try { + const previous = isQuietRun(status) ? await deps.lastStatus(payload.registryId, payload.authority) : undefined; + await deps.recordStatus(payload.registryId, payload.authority, statusToRecord(status, previous)); } catch (error) { deps.log?.('recording the run status failed', error); } - return result; } -async function runLocked( - payload: RunPayload, - deps: EngineDeps, - tuning: Tuning, - report: ReportBuilder, - key: string, - run: { idle: boolean }, -): Promise { +async function runLocked(payload: RunPayload, deps: EngineDeps, tuning: Tuning, report: ReportBuilder, key: string): Promise { const finish = (outcome: RunReport['outcome'], extra: Parameters[2] = {}) => report.build(outcome, deps.now(), extra); const extras = payload.extras ?? {}; @@ -200,10 +201,7 @@ async function runLocked( return finish('internal', { message: 'The rows on this device were written for another account' }); } // Providers schedule an upload sync for every account 30 s after any app write. - if (extras.upload && !(await hasLocalWork(reader, payload.authority, stored))) { - run.idle = true; - return finish('ok'); - } + if (extras.upload && !(await hasLocalWork(reader, payload.authority, stored))) return finish('ok'); const connection = await deps.jmap(payload.registryId); if (stored.owner && stored.owner.origin !== connection.origin) { diff --git a/src/device-sync/task.ts b/src/device-sync/task.ts index 4500875b..cb0f1009 100644 --- a/src/device-sync/task.ts +++ b/src/device-sync/task.ts @@ -16,7 +16,7 @@ import AsyncStorage from '@react-native-async-storage/async-storage'; import { detectDeviceLocale, isSupportedLocale, translate, type LocaleCode } from '../i18n'; import { randomBytes } from '../lib/random'; -import { accountSyncPrefs, useDeviceSyncStore, waitForDeviceSyncHydration } from '../stores/device-sync-store'; +import { accountDeviceSync, accountSyncPrefs, useDeviceSyncStore, waitForDeviceSyncHydration } from '../stores/device-sync-store'; import type { RandomSource } from './common/ids'; import { runDeviceSync, @@ -124,6 +124,10 @@ export function createTaskDeps(): EngineDeps { await waitForDeviceSyncHydration(); useDeviceSyncStore.getState().recordRunStatus(registryId, authority, status); }, + lastStatus: async (registryId, authority) => { + await waitForDeviceSyncHydration(); + return accountDeviceSync(registryId).lastRun[authority]; + }, recordKnownState: async (registryId, jmapAccountId, type, state) => { await waitForDeviceSyncHydration(); useDeviceSyncStore.getState().recordKnownState(registryId, jmapAccountId, type, state); From 11b897177d718f88aa6f91cebbb237091f15294f Mon Sep 17 00:00:00 2001 From: Linus Rath <139418639+rathlinus@users.noreply.github.com> Date: Sun, 27 Sep 2026 09:32:35 +0200 Subject: [PATCH 72/87] fix: keep mail push on servers without contacts or calendars #34 The push subscription got the contact and calendar types whenever the account had device sync on, without asking whether its server has contacts or calendars. Sync can be on for such an account: the settings list every signed-in account, and a server can drop a capability later. A server that refuses unknown types then refused the update and the replacement, and push setup failed: a new account got no mail push, an existing one lost it when its subscription expired. The types are now added only for a capability the account's session advertises, and a write the server refuses while it carries device sync's types is retried with EmailDelivery alone, so mail push never depends on device sync. Checked with the push setup tests against a fake server that refuses unknown types: a new subscription is created mail-only, an existing one is kept and refreshed, and another account's subscription is left alone when its server lacks the capability. --- src/lib/__tests__/push-notifications.test.ts | 115 +++++++++++++++++-- src/lib/push-notifications.ts | 95 ++++++++++----- 2 files changed, 172 insertions(+), 38 deletions(-) diff --git a/src/lib/__tests__/push-notifications.test.ts b/src/lib/__tests__/push-notifications.test.ts index 519b47ce..11db452c 100644 --- a/src/lib/__tests__/push-notifications.test.ts +++ b/src/lib/__tests__/push-notifications.test.ts @@ -43,10 +43,21 @@ const { CREATED } = vi.hoisted(() => ({ })); // A client of another signed-in account (device sync patches its push types -// without the singleton). -const { DETACHED } = vi.hoisted(() => ({ - DETACHED: { loaded: true, types: ['EmailDelivery'] as string[], log: [] as unknown[] }, -})); +// without the singleton), on a server with contacts and calendars unless a +// test says otherwise. +const { DETACHED, SYNC_SESSION } = vi.hoisted(() => { + const SYNC_SESSION = { + capabilities: { + 'urn:ietf:params:jmap:core': {}, + 'urn:ietf:params:jmap:contacts': {}, + 'urn:ietf:params:jmap:calendars': {}, + } as Record, + }; + return { + SYNC_SESSION, + DETACHED: { loaded: true, types: ['EmailDelivery'] as string[], log: [] as unknown[], session: SYNC_SESSION }, + }; +}); vi.mock('../../api/jmap-client', () => ({ jmapClient: { @@ -56,6 +67,9 @@ vi.mock('../../api/jmap-client', () => ({ currentSession: { capabilities: { 'urn:ietf:params:jmap:core': {} } }, }, JMAPClient: class { + get currentSession() { + return DETACHED.session; + } async loadAccount(id: string) { DETACHED.log.push(['load', id]); return DETACHED.loaded; @@ -112,7 +126,7 @@ import { } from '../../api/push'; import { getSharedMailboxes } from '../../api/email'; import { JMAPMethodError } from '../../api/jmap-result'; -import type { EmailPushConfig } from '../../api/types'; +import type { EmailPushConfig, JMAPSession } from '../../api/types'; import { jmapClient } from '../../api/jmap-client'; import { NativeModules } from 'react-native'; import { generateAccountId } from '../account-utils'; @@ -394,26 +408,103 @@ describe('push types for device sync (#34)', () => { })); } + // A server that does not know a type refuses the whole write. + const refusedTypes = (types: string[] | undefined) => { + const unknown = (types ?? []).filter((type) => type !== 'EmailDelivery'); + if (unknown.length > 0) throw new JMAPMethodError('invalidProperties', `Unknown types: ${unknown.join(', ')}`); + }; + beforeEach(async () => { vi.clearAllMocks(); await AsyncStorage.clear(); await AsyncStorage.setItem(deviceClientIdKey(ACCOUNT_ID), OUR_DCID); - (jmapClient as { currentSession: unknown }).currentSession = { - capabilities: { 'urn:ietf:params:jmap:core': {} }, - }; + (jmapClient as { currentSession: unknown }).currentSession = SYNC_SESSION; + DETACHED.session = SYNC_SESSION; listMock.mockResolvedValue([]); updateMock.mockResolvedValue(true); installFetch({}); }); + afterEach(() => { + (jmapClient as { currentSession: unknown }).currentSession = { + capabilities: { 'urn:ietf:params:jmap:core': {} }, + }; + DETACHED.session = SYNC_SESSION; + createMock.mockImplementation(async () => CREATED); + updateMock.mockImplementation(async () => undefined); + }); + it('adds the contact and calendar types of what the account syncs to the device', async () => { - expect(await pushTypesFor(ACCOUNT_ID)).toEqual(['EmailDelivery']); + expect(await pushTypesFor(ACCOUNT_ID, SYNC_SESSION as unknown as JMAPSession)).toEqual(['EmailDelivery']); await syncing({ [CONTACTS]: true, [CALENDAR]: false }); - expect(await pushTypesFor(ACCOUNT_ID)).toEqual(['EmailDelivery', 'ContactCard', 'AddressBook']); - expect(await pushTypesFor('someone@else.example')).toEqual(['EmailDelivery']); + expect(await pushTypesFor(ACCOUNT_ID, SYNC_SESSION as unknown as JMAPSession)) + .toEqual(['EmailDelivery', 'ContactCard', 'AddressBook']); + expect(await pushTypesFor('someone@else.example', SYNC_SESSION as unknown as JMAPSession)).toEqual(['EmailDelivery']); // Removed in Android Settings: nothing to route any more. await syncing({ [CONTACTS]: true }, { removedInAndroidSettings: true }); - expect(await pushTypesFor(ACCOUNT_ID)).toEqual(['EmailDelivery']); + expect(await pushTypesFor(ACCOUNT_ID, SYNC_SESSION as unknown as JMAPSession)).toEqual(['EmailDelivery']); + }); + + it('adds only the types of what the server offers', async () => { + await syncing({ [CONTACTS]: true, [CALENDAR]: true }); + const calendarsOnly = { capabilities: { 'urn:ietf:params:jmap:core': {}, 'urn:ietf:params:jmap:calendars': {} } }; + expect(await pushTypesFor(ACCOUNT_ID, calendarsOnly as unknown as JMAPSession)) + .toEqual(['EmailDelivery', 'CalendarEvent', 'Calendar']); + expect(await pushTypesFor(ACCOUNT_ID, null)).toEqual(['EmailDelivery']); + + // Sync was turned on while another account's server was served; this one + // has neither: its subscription stays mail-only. + (jmapClient as { currentSession: unknown }).currentSession = { + capabilities: { 'urn:ietf:params:jmap:core': {} }, + }; + createMock.mockImplementation(async (params: { types: string[] }) => { + refusedTypes(params.types); + return CREATED; + }); + const result = await setupPushNotifications({ relayBaseUrl: RELAY }); + expect(result.subscriptionId).toBe('new-server-id'); + expect(createMock.mock.calls.map((c) => c[0].types)).toEqual([['EmailDelivery']]); + }); + + it("does not add types another account's server does not offer", async () => { + const OTHER = 'bob@other.example.net'; + await AsyncStorage.setItem(`push:subscriptionId:v2:${OTHER}`, 'bob-sub'); + await AsyncStorage.setItem(DEVICE_SYNC_KEY, JSON.stringify({ + state: { accounts: { [OTHER]: { enabled: { [CONTACTS]: true } } } }, + version: 0, + })); + DETACHED.session = { capabilities: { 'urn:ietf:params:jmap:core': {}, 'urn:ietf:params:jmap:calendars': {} } }; + DETACHED.log = []; + DETACHED.types = ['EmailDelivery']; + await refreshPushSubscriptionTypes(OTHER); + expect(DETACHED.log.map((entry) => (entry as unknown[])[0])).toEqual(['load', 'PushSubscription/get']); + }); + + it('keeps mail push on a server that refuses the device sync types', async () => { + await syncing({ [CONTACTS]: true }); + createMock.mockImplementation(async (params: { types: string[] }) => { + refusedTypes(params.types); + return CREATED; + }); + updateMock.mockImplementation(async (_id: string, patch: { types?: string[] }) => refusedTypes(patch.types)); + + // A new subscription: mail-only rather than none. + const created = await setupPushNotifications({ relayBaseUrl: RELAY }); + expect(created).toEqual({ subscriptionId: 'new-server-id', verified: true }); + expect(createMock.mock.calls.map((c) => c[0].types)) + .toEqual([['EmailDelivery', 'ContactCard', 'AddressBook'], ['EmailDelivery']]); + + // The working subscription is kept and its expiry pushed forward, never + // replaced by one the server refuses as well. + vi.clearAllMocks(); + await AsyncStorage.setItem(SUB_KEY, 'existing'); + listMock.mockResolvedValue([healthy(['EmailDelivery'])]); + const refreshed = await setupPushNotifications({ relayBaseUrl: RELAY }); + expect(refreshed.subscriptionId).toBe('existing'); + expect(updateMock.mock.calls.map((c) => c[1].types)).toEqual([['EmailDelivery', 'ContactCard', 'AddressBook'], undefined]); + expect(updateMock.mock.calls[1][1].expires).toBeDefined(); + expect(createMock).not.toHaveBeenCalled(); + expect(destroyMock).not.toHaveBeenCalled(); }); it('subscribes with them', async () => { diff --git a/src/lib/push-notifications.ts b/src/lib/push-notifications.ts index 4151665b..97c48971 100644 --- a/src/lib/push-notifications.ts +++ b/src/lib/push-notifications.ts @@ -12,10 +12,16 @@ import { loadedMailboxes } from './mailbox-source'; import { jmapClient, JMAPClient } from '../api/jmap-client'; import { JMAPMethodError, requireMethodResult } from '../api/jmap-result'; import { CAPABILITIES } from '../api/types'; -import type { EmailPushConfig, JMAPAccountInfo, Mailbox } from '../api/types'; +import type { EmailPushConfig, JMAPAccountInfo, JMAPSession, Mailbox } from '../api/types'; import { generateAccountId } from './account-utils'; import { t } from '../stores/locale-store'; -import { DEVICE_SYNC_STORAGE_KEY, deviceSyncPushTypes, persistedAccounts } from '../device-sync/app/prefs'; +import { + authorityOfType, + DEVICE_SYNC_STORAGE_KEY, + deviceSyncPushTypes, + persistedAccounts, +} from '../device-sync/app/prefs'; +import { CALENDAR_AUTHORITY, CONTACTS_AUTHORITY, type Authority } from '../device-sync/types'; import { getUnifiedPushDistributors, isUnifiedPushSupported, @@ -207,19 +213,32 @@ export function isValidRelayUrl(value: string): boolean { // In-app sync uses the separate SSE channel and is unaffected. export const PUSH_TYPES = ['EmailDelivery'] as const; +// The capability a server advertises for the data types of an authority. +const DEVICE_SYNC_CAPABILITIES: Record = { + [CONTACTS_AUTHORITY]: CAPABILITIES.CONTACTS, + [CALENDAR_AUTHORITY]: CAPABILITIES.CALENDARS, +}; + /** * The push types of one account's subscription: `EmailDelivery`, plus the * contact and calendar types while the account syncs them to the device * (#34, docs/device-sync.md "Triggers"): the native push router turns those - * into device syncs, and a push carrying only them never notifies. + * into device syncs, and a push carrying only them never notifies. Only the + * types of a capability the account's session advertises: a server without + * contacts or calendars may refuse the whole subscription over them. * * Read from the device sync store's persisted JSON rather than the store, so * code that runs without it (the push task) never creates it. */ -export async function pushTypesFor(accountId: string): Promise { +export async function pushTypesFor(accountId: string, session: JMAPSession | null): Promise { let extra: string[] = []; try { - extra = deviceSyncPushTypes(persistedAccounts(await AsyncStorage.getItem(DEVICE_SYNC_STORAGE_KEY))[accountId]); + const capabilities = session?.capabilities ?? {}; + extra = deviceSyncPushTypes(persistedAccounts(await AsyncStorage.getItem(DEVICE_SYNC_STORAGE_KEY))[accountId]) + .filter((type) => { + const authority = authorityOfType(type); + return !!authority && DEVICE_SYNC_CAPABILITIES[authority] in capabilities; + }); } catch { extra = []; } @@ -404,6 +423,24 @@ async function writeWithEmailPush( } } +/** + * Run a subscription write with the account's push types and, when the server + * refuses it while they include device sync's, once more with the mail types + * alone: mail push never depends on device sync (#34). + */ +async function writeWithPushTypes( + types: readonly string[], + write: (types: readonly string[]) => Promise, +): Promise { + try { + return await write(types); + } catch (err) { + if (sameTypes(types, PUSH_TYPES)) throw err; + logPhase('jmap', 'push types refused, retrying with the mail types alone'); + return write(PUSH_TYPES); + } +} + async function readRefusedEmailPushAccounts(accountId: string): Promise { try { const parsed = JSON.parse((await AsyncStorage.getItem(emailPushRefusedKey(accountId))) ?? '[]'); @@ -947,7 +984,7 @@ async function setupPushNotificationsInner( : null; const subKey = subscriptionIdKey(accountId); const storedServerId = await AsyncStorage.getItem(subKey); - const types = await pushTypesFor(accountId); + const types = await pushTypesFor(accountId, jmapClient.currentSession); let jmapAccountId: string | null = null; try { jmapAccountId = jmapClient.accountId; @@ -1006,14 +1043,16 @@ async function setupPushNotificationsInner( let serverAssignedId: string; let serverExpires: string | null; try { - const created = await writeWithEmailPush(emailPush, (filter) => - createPushSubscription({ - deviceClientId, - url: buildRelayUrl(relayBaseUrl, `/api/push/jmap/${encodeURIComponent(deviceClientId)}`), - types: [...types], - expires: expiresFromNow(SUBSCRIPTION_EXPIRES_DAYS), - ...(filter ? { emailPush: filter } : {}), - }), + const created = await writeWithPushTypes(types, (wanted) => + writeWithEmailPush(emailPush, (filter) => + createPushSubscription({ + deviceClientId, + url: buildRelayUrl(relayBaseUrl, `/api/push/jmap/${encodeURIComponent(deviceClientId)}`), + types: [...wanted], + expires: expiresFromNow(SUBSCRIPTION_EXPIRES_DAYS), + ...(filter ? { emailPush: filter } : {}), + }), + ), ); serverAssignedId = created.result.id; serverExpires = created.result.expires; @@ -1063,8 +1102,9 @@ async function addPushAccountId(accountId: string): Promise { // from what this client wants (a subscription created by an older build still // listens to `Email`/`Mailbox`; device sync was turned on or off; a Junk // mailbox id can change under us). Returns false if the server rejects the -// update, which the caller treats as "replace"; otherwise the emailPush map -// the update installed (null when it left the filter alone). +// update, also with the mail types alone, which the caller treats as +// "replace"; otherwise the emailPush map the update installed (null when it +// left the filter alone). async function refreshSubscriptionExpires( sub: { id: string; @@ -1088,15 +1128,17 @@ async function refreshSubscriptionExpires( } } try { - const patch: { expires?: string; types?: string[] } = { - expires: expiresFromNow(SUBSCRIPTION_EXPIRES_DAYS), - }; - if (typesNeedUpdate) patch.types = [...types]; - const { emailPush } = await writeWithEmailPush( - emailPushNeedsUpdate ? desiredEmailPush : null, - (filter) => updatePushSubscription(sub.id, filter ? { ...patch, emailPush: filter } : patch), - ); - return { emailPush }; + return await writeWithPushTypes(types, async (wanted) => { + const patch: { expires?: string; types?: string[] } = { + expires: expiresFromNow(SUBSCRIPTION_EXPIRES_DAYS), + }; + if (!sameTypes(sub.types, wanted)) patch.types = [...wanted]; + const { emailPush } = await writeWithEmailPush( + emailPushNeedsUpdate ? desiredEmailPush : null, + (filter) => updatePushSubscription(sub.id, filter ? { ...patch, emailPush: filter } : patch), + ); + return { emailPush }; + }); } catch { return false; } @@ -1231,12 +1273,12 @@ export async function refreshPushSubscriptionTypes(accountId: string): Promise s.id === subscriptionId); if (!current || sameTypes(current.types, types)) return; await updatePushSubscription(subscriptionId, { types: [...types] }); @@ -1244,6 +1286,7 @@ export async function refreshPushSubscriptionTypes(accountId: string): Promise Date: Sun, 27 Sep 2026 09:33:32 +0200 Subject: [PATCH 73/87] fix: notify new mail again when a push names Email but not EmailDelivery #34 On main every push ran the mail task, whose legacy path notifies the newest unread inbox mail. Device sync narrowed that to pushes naming EmailDelivery, so contact and calendar changes never notify. But a subscription made by an older build listens to Email and Mailbox until the app is opened again, and a server without EmailDelivery reports new mail only as those changes. Their new mail no longer showed a notification while the app was closed. Email and Mailbox now count as possible mail again, in the native push router and in the push task alike. A push that names only contact or calendar types still never takes the mail path. Checked with the push router and push task tests: an Email or Mailbox change starts the task and the legacy path notifies the message, next to the syncs a mixed map asks for. --- .../bulwarkmobile/sync/PushRouting.kt | 14 ++++++-- .../bulwarkmobile/sync/PushRoutingTest.kt | 20 +++++++++++ .../__tests__/push-background-task.test.ts | 36 +++++++++++++++++++ src/lib/push-background-task.ts | 15 +++++--- 4 files changed, 78 insertions(+), 7 deletions(-) diff --git a/android/app/src/main/java/com/anonymous/bulwarkmobile/sync/PushRouting.kt b/android/app/src/main/java/com/anonymous/bulwarkmobile/sync/PushRouting.kt index 3bdcca1b..70c0b97d 100644 --- a/android/app/src/main/java/com/anonymous/bulwarkmobile/sync/PushRouting.kt +++ b/android/app/src/main/java/com/anonymous/bulwarkmobile/sync/PushRouting.kt @@ -13,7 +13,7 @@ import org.json.JSONObject * The relay forwards the JMAP StateChange's `changed` map verbatim (FCM data * key `changed`: JMAP account id → type → state; only its first account is * repeated as `accountId`), so every account key is looked at. The mail task - * runs only for mail: when some account's map has `EmailDelivery`, for an + * runs only for mail: when some account's map has a mail type, for an * EmailPush, or when `changed` is missing or unreadable (older relays), so a * contact change never ends in a mail notification. */ @@ -28,7 +28,15 @@ object PushRouting { ) private const val EMAIL_PUSH_KIND = "jmap-email-push" - private const val EMAIL_DELIVERY = "EmailDelivery" + + /** + * Types that may mean new mail: `EmailDelivery`, and `Email`/`Mailbox`, + * which new mail changes too. A subscription made by an older build + * listens to them until the launch-time resync replaces its types, and a + * server without `EmailDelivery` has only them. Keep in sync with + * MAIL_TYPES in src/lib/push-background-task.ts. + */ + private val MAIL_TYPES = setOf("EmailDelivery", "Email", "Mailbox") private val CONTACT_TYPES = setOf("ContactCard", "AddressBook") private val CALENDAR_TYPES = setOf("CalendarEvent", "Calendar") @@ -42,7 +50,7 @@ object PushRouting { mail = true continue } - if (EMAIL_DELIVERY in types) mail = true + if (types.any { it in MAIL_TYPES }) mail = true val contacts = types.any { it in CONTACT_TYPES } val calendar = types.any { it in CALENDAR_TYPES } for (route in routes[jmapAccountId].orEmpty()) { diff --git a/android/app/src/test/java/com/anonymous/bulwarkmobile/sync/PushRoutingTest.kt b/android/app/src/test/java/com/anonymous/bulwarkmobile/sync/PushRoutingTest.kt index 1a13656d..8e608022 100644 --- a/android/app/src/test/java/com/anonymous/bulwarkmobile/sync/PushRoutingTest.kt +++ b/android/app/src/test/java/com/anonymous/bulwarkmobile/sync/PushRoutingTest.kt @@ -64,6 +64,26 @@ class PushRoutingTest { assertEquals(setOf("alice-work" to calendar), decision.syncs) } + @Test + fun `an Email or Mailbox change is possible mail, as a subscription from an older build sends it`() { + for (changed in listOf( + """{"c":{"Email":"s1","Mailbox":"s2"}}""", + """{"c":{"Email":"s1"}}""", + """{"c":{"Mailbox":"s1"}}""", + )) { + val decision = PushRouting.decide(stateChange(changed), routes) + assertTrue("changed=$changed", decision.startMailTask) + assertTrue("changed=$changed", decision.syncs.isEmpty()) + } + val mixed = PushRouting.decide(stateChange("""{"team":{"CalendarEvent":"e1"},"c":{"Email":"s1"}}"""), routes) + assertTrue(mixed.startMailTask) + assertEquals(setOf("alice-work" to calendar), mixed.syncs) + // Types that are neither mail nor device sync say nothing about mail. + val other = PushRouting.decide(stateChange("""{"c":{"Thread":"t1","ContactCard":"s1"}}"""), routes) + assertFalse(other.startMailTask) + assertEquals(setOf("alice@example.org" to contacts), other.syncs) + } + @Test fun `an email push always starts the mail task`() { val data = mapOf("kind" to "jmap-email-push", "accountId" to "c", "changed" to """{"c":{"EmailDelivery":"s"}}""") diff --git a/src/lib/__tests__/push-background-task.test.ts b/src/lib/__tests__/push-background-task.test.ts index c8984ad9..8f1f0dae 100644 --- a/src/lib/__tests__/push-background-task.test.ts +++ b/src/lib/__tests__/push-background-task.test.ts @@ -237,6 +237,42 @@ describe('pushes for device sync (#34)', () => { await pushBackgroundTask(stateChange({ 'jmap-primary': { EmailDelivery: 'm1', ContactCard: 's1' } })); expect(jmapClient.getStoredCredentials).toHaveBeenCalledWith(LOCAL); }); + + it('takes an Email or Mailbox change for possible mail, as a subscription from an older build sends it', () => { + expect(carriesNoMail(parseRelayPushData(stateChange({ a: { Email: 's1', Mailbox: 's2' } })))).toBe(false); + expect(carriesNoMail(parseRelayPushData(stateChange({ a: { Email: 's1' } })))).toBe(false); + expect(carriesNoMail(parseRelayPushData(stateChange({ a: { Mailbox: 's1' } })))).toBe(false); + expect(carriesNoMail(parseRelayPushData(stateChange({ a: { CalendarEvent: 'e1' }, b: { Email: 's1' } })))).toBe(false); + // Types that are neither mail nor device sync say nothing about mail. + expect(carriesNoMail(parseRelayPushData(stateChange({ a: { Thread: 't1', ContactCard: 's1' } })))).toBe(true); + }); + + it('notifies new mail from a subscription that listens to Email and Mailbox', async () => { + (secureFetch as ReturnType).mockImplementation(async (url: string, init?: { body?: string }) => { + if (url.endsWith('/.well-known/jmap')) { + return { + ok: true, + json: async () => ({ + apiUrl: 'https://mail.example.com/jmap/', + primaryAccounts: { 'urn:ietf:params:jmap:mail': 'jmap-primary' }, + accounts: { 'jmap-primary': {} }, + }), + }; + } + // The legacy path: the inbox, its newest unread ids, then the messages. + const [[name]] = JSON.parse(init?.body ?? '{}').methodCalls; + const body = name === 'Mailbox/get' + ? { list: [{ id: 'inbox', role: 'inbox' }] } + : name === 'Email/query' + ? { ids: ['m1'] } + : { list: [{ id: 'm1', threadId: 't1', keywords: {}, subject: 'Hi', from: [{ email: 'bob@example.com' }] }] }; + return { ok: true, json: async () => ({ methodResponses: [[name, body, '0']] }) }; + }); + + await pushBackgroundTask(stateChange({ 'jmap-primary': { Email: 's1', Mailbox: 's2' } })); + + expect(showNotification).toHaveBeenCalledWith(expect.objectContaining({ emailId: 'm1', accountId: LOCAL })); + }); }); describe('selectNotifiableEmails', () => { diff --git a/src/lib/push-background-task.ts b/src/lib/push-background-task.ts index 776b39c4..2c703f8e 100644 --- a/src/lib/push-background-task.ts +++ b/src/lib/push-background-task.ts @@ -378,18 +378,25 @@ async function detachedNewestUnreadInboxIds( return (qBody.ids as string[]) ?? []; } +// Types that may mean new mail: `EmailDelivery`, and `Email`/`Mailbox`, which +// new mail changes too. A subscription made by an older build listens to them +// until the launch-time resync replaces its types, and a server without +// `EmailDelivery` has only them. Keep in sync with MAIL_TYPES in PushRouting.kt. +const MAIL_TYPES = ['EmailDelivery', 'Email', 'Mailbox']; + /** * A StateChange push in which no account got new mail: only contact or * calendar changes, which the native push router already turned into device - * syncs (#34, docs/device-sync.md "Triggers"). Without `EmailDelivery` there - * is nothing to notify, and the legacy path (the newest unread mail) must - * not run for it. An empty or missing `changed` map keeps the old behaviour. + * syncs (#34, docs/device-sync.md "Triggers"). Without a mail type there is + * nothing to notify, and the legacy path (the newest unread mail) must not + * run for it. An empty or missing `changed` map keeps the old behaviour. */ export function carriesNoMail(payload: RelayPushData): boolean { if (payload.kind !== 'jmap-state-change' || !payload.changed) return false; const accounts = Object.values(payload.changed); if (accounts.length === 0) return false; - return !accounts.some((types) => !!types && typeof types === 'object' && 'EmailDelivery' in types); + return !accounts.some((types) => !!types && typeof types === 'object' + && MAIL_TYPES.some((type) => type in types)); } // Fired by BulwarkPushTaskService when a data FCM message arrives. Runs in a From eebb00238c5f207a39d89d6d291a5718e14d2f9c Mon Sep 17 00:00:00 2001 From: Linus Rath <139418639+rathlinus@users.noreply.github.com> Date: Sun, 27 Sep 2026 09:34:43 +0200 Subject: [PATCH 74/87] fix: look for new mail when a push names changes it can't read #34 The native push router and the push task decided separately whether a push may carry mail, and disagreed. For an account entry that is not an object, or a map that names no type, the router started the mail task as possible mail, but the task's guard read the same payload as "no mail" and returned, so new mail it announced was never looked for. A push without a kind was judged by its map in the router but taken for mail by the task. Both now apply the same rules: anything but a StateChange, a map that is missing, unreadable or names no type, an unreadable account entry, or a mail type means the mail task runs. Only a readable StateChange naming contact, calendar or other non-mail types skips it. Checked with one table of payloads run against the router's decision and the task's guard alike. --- .../bulwarkmobile/sync/PushRouting.kt | 13 +++-- .../bulwarkmobile/sync/PushRoutingTest.kt | 50 ++++++++++++++++ .../__tests__/push-background-task.test.ts | 57 +++++++++++++++++++ src/lib/__tests__/push-notifications.test.ts | 8 +-- src/lib/push-background-task.ts | 15 +++-- 5 files changed, 128 insertions(+), 15 deletions(-) diff --git a/android/app/src/main/java/com/anonymous/bulwarkmobile/sync/PushRouting.kt b/android/app/src/main/java/com/anonymous/bulwarkmobile/sync/PushRouting.kt index 70c0b97d..fd5d938a 100644 --- a/android/app/src/main/java/com/anonymous/bulwarkmobile/sync/PushRouting.kt +++ b/android/app/src/main/java/com/anonymous/bulwarkmobile/sync/PushRouting.kt @@ -13,9 +13,11 @@ import org.json.JSONObject * The relay forwards the JMAP StateChange's `changed` map verbatim (FCM data * key `changed`: JMAP account id → type → state; only its first account is * repeated as `accountId`), so every account key is looked at. The mail task - * runs only for mail: when some account's map has a mail type, for an - * EmailPush, or when `changed` is missing or unreadable (older relays), so a - * contact change never ends in a mail notification. + * runs only for mail: when some account's map has a mail type, for anything + * but a StateChange (an EmailPush), when `changed` is missing, unreadable or + * names no type (older relays), or when an account's entry is unreadable, so + * a contact change never ends in a mail notification. The push task's guard + * (`carriesNoMail` in src/lib/push-background-task.ts) applies the same rules. */ object PushRouting { /** An Android account and the authorities it syncs, fed by one JMAP account. */ @@ -27,7 +29,7 @@ object PushRouting { val startMailTask: Boolean, ) - private const val EMAIL_PUSH_KIND = "jmap-email-push" + private const val STATE_CHANGE_KIND = "jmap-state-change" /** * Types that may mean new mail: `EmailDelivery`, and `Email`/`Mailbox`, @@ -42,7 +44,8 @@ object PushRouting { fun decide(data: Map, routes: Map>): Decision { val changed = data["changed"]?.let(::parseChanged) - var mail = data["kind"] == EMAIL_PUSH_KIND || changed == null + // Only a StateChange can say it carries no mail. + var mail = data["kind"] != STATE_CHANGE_KIND || changed == null val syncs = LinkedHashSet>() for ((jmapAccountId, types) in changed.orEmpty()) { if (types == null) { diff --git a/android/app/src/test/java/com/anonymous/bulwarkmobile/sync/PushRoutingTest.kt b/android/app/src/test/java/com/anonymous/bulwarkmobile/sync/PushRoutingTest.kt index 8e608022..db2d5b53 100644 --- a/android/app/src/test/java/com/anonymous/bulwarkmobile/sync/PushRoutingTest.kt +++ b/android/app/src/test/java/com/anonymous/bulwarkmobile/sync/PushRoutingTest.kt @@ -108,6 +108,56 @@ class PushRoutingTest { assertEquals(setOf("alice@example.org" to contacts), decision.syncs) } + @Test + fun `reads a push the same way as the push task`() { + // kind (null: none), changed (null: none) → whether the mail task starts. The + // same table as MAIL_CASES in src/lib/__tests__/push-background-task.test.ts, + // whose carriesNoMail must agree. + val state = "jmap-state-change" + val emailPush = "jmap-email-push" + val cases: List> = listOf( + // No readable map: as before device sync. + Triple(state, null, true), + Triple(state, "", true), + Triple(state, "not json", true), + Triple(state, "[1]", true), + Triple(state, "{}", true), + // A map that names no type is read like a missing one. + Triple(state, """{"c":{}}""", true), + Triple(state, """{"c":{},"team":{}}""", true), + // An account entry that can't be read may be mail. + Triple(state, """{"c":null}""", true), + Triple(state, """{"c":"garbled","team":{"ContactCard":"s"}}""", true), + Triple(state, """{"c":[],"team":{"ContactCard":"s"}}""", true), + Triple(state, """{"c":5}""", true), + // Mail types anywhere in the map. + Triple(state, """{"c":{"EmailDelivery":"s"}}""", true), + Triple(state, """{"c":{"Email":"s"}}""", true), + Triple(state, """{"c":{"Mailbox":"s"}}""", true), + Triple(state, """{"c":{"ContactCard":"s1"},"team":{"EmailDelivery":"s2"}}""", true), + // Only a StateChange can say it carries no mail. + Triple(null, """{"c":{"ContactCard":"s"}}""", true), + Triple(emailPush, """{"c":{"EmailDelivery":"s"}}""", true), + Triple(emailPush, null, true), + // Contact and calendar changes, and types that say nothing about mail. + Triple(state, """{"c":{"ContactCard":"s1","AddressBook":"s2"}}""", false), + Triple(state, """{"c":{"CalendarEvent":"e1"},"team":{"Calendar":"c1"}}""", false), + Triple(state, """{"c":{"ContactCard":"s"},"team":{}}""", false), + Triple(state, """{"elsewhere":{"ContactCard":"s"}}""", false), + Triple(state, """{"c":{"Thread":"t1","ContactCard":"s1"}}""", false), + ) + for ((kind, changed, mail) in cases) { + val data = buildMap { + if (kind != null) put("kind", kind) + put("accountLabel", "alice") + put("accountId", "c") + put("emailIds", "[]") + if (changed != null) put("changed", changed) + } + assertEquals("kind=$kind changed=$changed", mail, PushRouting.decide(data, routes).startMailTask) + } + } + @Test fun `accounts without routes are ignored`() { val decision = PushRouting.decide(stateChange("""{"elsewhere":{"ContactCard":"s"}}"""), routes) diff --git a/src/lib/__tests__/push-background-task.test.ts b/src/lib/__tests__/push-background-task.test.ts index 8f1f0dae..b8298866 100644 --- a/src/lib/__tests__/push-background-task.test.ts +++ b/src/lib/__tests__/push-background-task.test.ts @@ -238,6 +238,63 @@ describe('pushes for device sync (#34)', () => { expect(jmapClient.getStoredCredentials).toHaveBeenCalledWith(LOCAL); }); + // kind (undefined: none), changed (undefined: none) → whether the push may + // carry mail. The same table as "reads a push the same way as the push task" + // in PushRoutingTest.kt, whose startMailTask must agree. + const STATE = 'jmap-state-change'; + const EMAIL_PUSH = 'jmap-email-push'; + const MAIL_CASES: Array<[string | undefined, string | undefined, boolean]> = [ + // No readable map: as before device sync. + [STATE, undefined, true], + [STATE, '', true], + [STATE, 'not json', true], + [STATE, '[1]', true], + [STATE, '{}', true], + // A map that names no type is read like a missing one. + [STATE, '{"c":{}}', true], + [STATE, '{"c":{},"team":{}}', true], + // An account entry that can't be read may be mail. + [STATE, '{"c":null}', true], + [STATE, '{"c":"garbled","team":{"ContactCard":"s"}}', true], + [STATE, '{"c":[],"team":{"ContactCard":"s"}}', true], + [STATE, '{"c":5}', true], + // Mail types anywhere in the map. + [STATE, '{"c":{"EmailDelivery":"s"}}', true], + [STATE, '{"c":{"Email":"s"}}', true], + [STATE, '{"c":{"Mailbox":"s"}}', true], + [STATE, '{"c":{"ContactCard":"s1"},"team":{"EmailDelivery":"s2"}}', true], + // Only a StateChange can say it carries no mail. + [undefined, '{"c":{"ContactCard":"s"}}', true], + [EMAIL_PUSH, '{"c":{"EmailDelivery":"s"}}', true], + [EMAIL_PUSH, undefined, true], + // Contact and calendar changes, and types that say nothing about mail. + [STATE, '{"c":{"ContactCard":"s1","AddressBook":"s2"}}', false], + [STATE, '{"c":{"CalendarEvent":"e1"},"team":{"Calendar":"c1"}}', false], + [STATE, '{"c":{"ContactCard":"s"},"team":{}}', false], + [STATE, '{"elsewhere":{"ContactCard":"s"}}', false], + [STATE, '{"c":{"Thread":"t1","ContactCard":"s1"}}', false], + ]; + + it('reads a push the same way as the native router', () => { + for (const [kind, changed, mail] of MAIL_CASES) { + const data: Record = { accountLabel: 'alice', accountId: 'c', emailIds: '[]' }; + if (kind !== undefined) data.kind = kind; + if (changed !== undefined) data.changed = changed; + expect(carriesNoMail(parseRelayPushData(data)), `kind=${kind} changed=${changed}`).toBe(!mail); + } + // Native code hands `changed` over as a string; a map that is not an + // object is unreadable in any form. + expect(carriesNoMail(parseRelayPushData({ kind: STATE, changed: [{ ContactCard: 's' }] }))).toBe(false); + }); + + it("looks for mail when it can't read an account's entry, as the native router does", async () => { + for (const changed of ['{"jmap-primary":null}', '{"jmap-primary":{}}']) { + vi.clearAllMocks(); + await pushBackgroundTask({ kind: STATE, accountLabel: 'alice', accountId: 'jmap-primary', emailIds: '[]', changed }); + expect(jmapClient.getStoredCredentials, changed).toHaveBeenCalledWith(LOCAL); + } + }); + it('takes an Email or Mailbox change for possible mail, as a subscription from an older build sends it', () => { expect(carriesNoMail(parseRelayPushData(stateChange({ a: { Email: 's1', Mailbox: 's2' } })))).toBe(false); expect(carriesNoMail(parseRelayPushData(stateChange({ a: { Email: 's1' } })))).toBe(false); diff --git a/src/lib/__tests__/push-notifications.test.ts b/src/lib/__tests__/push-notifications.test.ts index 11db452c..44f1c2b2 100644 --- a/src/lib/__tests__/push-notifications.test.ts +++ b/src/lib/__tests__/push-notifications.test.ts @@ -409,7 +409,7 @@ describe('push types for device sync (#34)', () => { } // A server that does not know a type refuses the whole write. - const refusedTypes = (types: string[] | undefined) => { + const refuseUnknownTypes = (types: string[] | undefined) => { const unknown = (types ?? []).filter((type) => type !== 'EmailDelivery'); if (unknown.length > 0) throw new JMAPMethodError('invalidProperties', `Unknown types: ${unknown.join(', ')}`); }; @@ -458,7 +458,7 @@ describe('push types for device sync (#34)', () => { capabilities: { 'urn:ietf:params:jmap:core': {} }, }; createMock.mockImplementation(async (params: { types: string[] }) => { - refusedTypes(params.types); + refuseUnknownTypes(params.types); return CREATED; }); const result = await setupPushNotifications({ relayBaseUrl: RELAY }); @@ -483,10 +483,10 @@ describe('push types for device sync (#34)', () => { it('keeps mail push on a server that refuses the device sync types', async () => { await syncing({ [CONTACTS]: true }); createMock.mockImplementation(async (params: { types: string[] }) => { - refusedTypes(params.types); + refuseUnknownTypes(params.types); return CREATED; }); - updateMock.mockImplementation(async (_id: string, patch: { types?: string[] }) => refusedTypes(patch.types)); + updateMock.mockImplementation(async (_id: string, patch: { types?: string[] }) => refuseUnknownTypes(patch.types)); // A new subscription: mail-only rather than none. const created = await setupPushNotifications({ relayBaseUrl: RELAY }); diff --git a/src/lib/push-background-task.ts b/src/lib/push-background-task.ts index 2c703f8e..1f513bc9 100644 --- a/src/lib/push-background-task.ts +++ b/src/lib/push-background-task.ts @@ -154,7 +154,7 @@ export function parseRelayPushData(data: unknown): RelayPushData { } catch { // ignore } - } else if (d.changed && typeof d.changed === 'object') { + } else if (d.changed && typeof d.changed === 'object' && !Array.isArray(d.changed)) { out.changed = d.changed as Record>; } // Older relays only sent `changed`; the account id is its first key. @@ -389,14 +389,17 @@ const MAIL_TYPES = ['EmailDelivery', 'Email', 'Mailbox']; * calendar changes, which the native push router already turned into device * syncs (#34, docs/device-sync.md "Triggers"). Without a mail type there is * nothing to notify, and the legacy path (the newest unread mail) must not - * run for it. An empty or missing `changed` map keeps the old behaviour. + * run for it. A missing map, or one that names no type, keeps the old + * behaviour, and an account entry it can't read may be mail. The same rules + * as PushRouting.decide, so the two never disagree about a push. */ export function carriesNoMail(payload: RelayPushData): boolean { if (payload.kind !== 'jmap-state-change' || !payload.changed) return false; - const accounts = Object.values(payload.changed); - if (accounts.length === 0) return false; - return !accounts.some((types) => !!types && typeof types === 'object' - && MAIL_TYPES.some((type) => type in types)); + const entries: unknown[] = Object.values(payload.changed); + if (entries.some((types) => !types || typeof types !== 'object' || Array.isArray(types))) return false; + const types = entries.flatMap((entry) => Object.keys(entry as Record)); + if (types.length === 0) return false; + return !types.some((type) => MAIL_TYPES.includes(type)); } // Fired by BulwarkPushTaskService when a data FCM message arrives. Runs in a From 217bb59d573af6961035fabc7113fa4f5041d6ba Mon Sep 17 00:00:00 2001 From: Linus Rath <139418639+rathlinus@users.noreply.github.com> Date: Sun, 27 Sep 2026 10:01:23 +0200 Subject: [PATCH 75/87] fix: forget a contact created and deleted on the phone between two syncs #34 ContactsProvider soft-deletes every raw contact and group of a sync account, also one that never reached the server (DELETED=1, SOURCE_ID NULL). When a contact or group was created and deleted on the phone before a run claimed it, no upload phase took the row: without an identity or a pending create it belongs to no JMAP account, and the claim step skipped deleted rows. The row stayed for good, and the upload-only check and the teardown counted it as a waiting change: every upload sync the providers schedule became a full run, and turning sync off or signing out always asked about a change that would be lost. The claim step now hands such a row to the planner, whose plan for it is a purge, and `hasLocalWork` and `countPending` leave out deleted rows that have neither an identity nor a pending create. Nothing is uploaded for them. Calendar has no such gap: CalendarProvider hard-deletes an app-deleted event without `_SYNC_ID`, and a claimed event carries `~pending/`, so it is soft-deleted with its pending create and destroyed by uid as before. --- .../engine/deleted-before-claim.test.ts | 97 +++++++++++++++++++ src/device-sync/engine/item-sync.ts | 6 +- src/device-sync/engine/run.ts | 20 +++- src/device-sync/engine/teardown.ts | 10 +- 4 files changed, 121 insertions(+), 12 deletions(-) create mode 100644 src/device-sync/__tests__/engine/deleted-before-claim.test.ts diff --git a/src/device-sync/__tests__/engine/deleted-before-claim.test.ts b/src/device-sync/__tests__/engine/deleted-before-claim.test.ts new file mode 100644 index 00000000..c31d2995 --- /dev/null +++ b/src/device-sync/__tests__/engine/deleted-before-claim.test.ts @@ -0,0 +1,97 @@ +// Items created on the device and deleted before any run claimed them. +// ContactsProvider soft-deletes every raw contact and group of a sync account +// (DELETED=1, SOURCE_ID NULL), CalendarProvider hard-deletes an event without +// `_SYNC_ID` at once. Such an item never reached the server: it is purged, +// nothing uploads, and it never counts as a change waiting for an upload. + +import { describe, expect, it } from 'vitest'; +import { calendarPlanner } from '../../calendar/planner'; +import { contactsPlanner } from '../../contacts/planner'; +import { + addDeviceContact, + addServerCards, + ANDROID_ACCOUNT, + CALENDAR_AUTHORITY, + CONTACTS_AUTHORITY, + createHarness, + type Harness, +} from './harness'; + +function real(): Harness { + const h = createHarness(); + h.deps.planners = { contacts: contactsPlanner, calendar: calendarPlanner }; + return h; +} + +/** Synced once, then a contact and a group created on the device and deleted again. */ +async function createdAndDeleted(): Promise<{ h: Harness; contact: number; group: number }> { + const h = real(); + addServerCards(h, ['Ada Lovelace']); + expect((await h.run()).outcome).toBe('ok'); + const contact = addDeviceContact(h, 'Hedy Lamarr'); + h.device.user.deleteContact(contact); + const group = h.device.user.insertGroup(ANDROID_ACCOUNT, 'Friends'); + h.device.user.deleteGroup(group); + return { h, contact, group }; +} + +function offline(h: Harness): void { + h.deps.jmap = async () => { + const error = new Error('Network request failed'); + error.name = 'NetworkError'; + throw error; + }; +} + +describe('device sync engine: an item deleted on the device before it was claimed', () => { + it('purges a contact and a group, and uploads nothing', async () => { + const { h, contact, group } = await createdAndDeleted(); + + const report = await h.run(); + + expect(report).toMatchObject({ outcome: 'ok', itemErrors: [], stats: { uploaded: { created: 0, updated: 0, deleted: 0 } } }); + expect(h.device.row('raw_contacts', contact)).toBeUndefined(); + expect(h.device.rows('data', 'raw_contact_id = ?', [contact])).toEqual([]); + expect(h.device.row('groups', group)).toBeUndefined(); + expect(h.server.all('ContactCard', 'a').map((c) => c.name)).toEqual([{ full: 'Ada Lovelace' }]); + }); + + it('does not turn an upload-only sync into a full one', async () => { + const { h } = await createdAndDeleted(); + const requests = h.server.requests.length; + + expect((await h.run(CONTACTS_AUTHORITY, { upload: true })).outcome).toBe('ok'); + + expect(h.server.requests.length).toBe(requests); + }); + + it('is no change that turning sync off would lose, also while offline', async () => { + const { h } = await createdAndDeleted(); + offline(h); + + expect(await h.teardown(CONTACTS_AUTHORITY)).toEqual({ pending: 0 }); + + expect(h.device.rows('raw_contacts')).toEqual([]); + expect(h.device.rows('groups')).toEqual([]); + }); + + it('leaves nothing behind for an event: CalendarProvider removes it at once', async () => { + const h = real(); + expect((await h.run(CALENDAR_AUTHORITY)).outcome).toBe('ok'); + const calendarRow = Number(h.device.rows('calendars')[0]._id); + const event = h.device.user.insertEvent(calendarRow, { + title: 'Dentist', + dtstart: Date.UTC(2026, 9, 1, 8), + dtend: Date.UTC(2026, 9, 1, 9), + eventTimezone: 'Europe/Berlin', + }); + h.device.user.deleteEvent(event); + expect(h.events()).toEqual([]); + const requests = h.server.requests.length; + + expect((await h.run(CALENDAR_AUTHORITY, { upload: true })).outcome).toBe('ok'); + expect(h.server.requests.length).toBe(requests); + offline(h); + expect(await h.teardown(CALENDAR_AUTHORITY)).toEqual({ pending: 0 }); + }); +}); diff --git a/src/device-sync/engine/item-sync.ts b/src/device-sync/engine/item-sync.ts index ce7947f5..c747ddae 100644 --- a/src/device-sync/engine/item-sync.ts +++ b/src/device-sync/engine/item-sync.ts @@ -886,13 +886,15 @@ export abstract class ItemSync { /** * New items get their uid and target written before anything is sent. A * claim whose collection is gone from the server is made again where the - * kind allows it (contacts: nothing of it can exist in a deleted book). + * kind allows it (contacts: nothing of it can exist in a deleted book). A + * new item deleted before it was claimed never reached the server, and no + * upload phase takes it (it has no account): its plan is a purge. */ private async claimNewItems(): Promise { const works: Work[] = []; for (const held of await this.loadUploadItems()) { const meta = held.kind.meta(held.local); - if (!meta.isNew || meta.deleted || !this.claimable(held)) continue; + if (!meta.isNew || (meta.deleted && meta.pending) || !this.claimable(held)) continue; const reclaim = !!meta.pending && !!held.kind.unclaimed && this.collectionGone(meta.pending.target); if (meta.pending && !reclaim) continue; const ref = `row:${meta.rowId}`; diff --git a/src/device-sync/engine/run.ts b/src/device-sync/engine/run.ts index 238d0aaa..d478abcb 100644 --- a/src/device-sync/engine/run.ts +++ b/src/device-sync/engine/run.ts @@ -17,6 +17,7 @@ import { type RunPayload, type RunReport, type RunStatus, + type Row, } from '../types'; import { BatchWriter } from './batch'; import { CalendarSync } from './calendar-sync'; @@ -26,7 +27,7 @@ import type { RunEnv } from './context'; import { DEFAULT_TUNING, type EngineDeps, type Tuning } from './deps'; import { classifyFailure, RunAbort } from './errors'; import { acquireLock, lockKey, stopRequested } from './mutex'; -import { ProviderReader, str } from './provider'; +import { flag, ProviderReader, str } from './provider'; import { isQuietRun, ReportBuilder, statusOf, statusToRecord } from './report'; import { emptySyncState, parseSyncState, StateStore, type SyncState } from './sync-state'; import type { ItemSync } from './item-sync'; @@ -46,6 +47,15 @@ function itemTypeOf(authority: Authority): string { return authority === CONTACTS_AUTHORITY ? 'ContactCard' : 'CalendarEvent'; } +/** + * Whether a dirty, deleted or new raw contact or group has anything to + * upload. One created and deleted on the device before a run claimed it (no + * identity, no pending create) never reached the server: a run purges it. + */ +export function waitsForUpload(row: Row): boolean { + return !!str(row.sourceid) || !!str(row.sync3) || !flag(row.deleted); +} + /** Rows waiting for an upload (dirty, deleted, new), or state work a run must do whatever its extras. */ export async function hasLocalWork(reader: ProviderReader, authority: Authority, state: SyncState): Promise { for (const account of Object.values(state.accounts)) { @@ -55,11 +65,11 @@ export async function hasLocalWork(reader: ProviderReader, authority: Authority, const rows = await reader.rows('events', ['_id'], "dirty = 1 OR deleted = 1 OR _sync_id IS NULL OR _sync_id LIKE '~pending/%'"); return rows.length > 0; } - const contacts = await reader.rows('raw_contacts', ['_id'], 'dirty = 1 OR deleted = 1 OR sourceid IS NULL'); - if (contacts.length) return true; - const groups = await reader.rows('groups', ['_id', 'sourceid', 'dirty', 'deleted']); + const contacts = await reader.rows('raw_contacts', ['_id', 'sourceid', 'deleted', 'sync3'], 'dirty = 1 OR deleted = 1 OR sourceid IS NULL'); + if (contacts.some(waitsForUpload)) return true; + const groups = await reader.rows('groups', ['_id', 'sourceid', 'dirty', 'deleted', 'sync3']); const present = new Set(groups.map((g) => str(g.sourceid)).filter(Boolean)); - if (groups.some((g) => !str(g.sourceid) || Number(g.dirty) === 1 || Number(g.deleted) === 1)) return true; + if (groups.some((g) => (!str(g.sourceid) || Number(g.dirty) === 1 || Number(g.deleted) === 1) && waitsForUpload(g))) return true; // A group an app hard-deleted is only found by its absence. return Object.values(state.accounts).some((a) => (a.groups ?? []).some((ref) => !present.has(ref))); } diff --git a/src/device-sync/engine/teardown.ts b/src/device-sync/engine/teardown.ts index 8e0bbc20..abc80dcc 100644 --- a/src/device-sync/engine/teardown.ts +++ b/src/device-sync/engine/teardown.ts @@ -12,7 +12,7 @@ import type { EngineDeps } from './deps'; import { acquireLock, clearStopRequest, lockKey, requestStop } from './mutex'; import { ProviderReader, str } from './provider'; import { ReportBuilder } from './report'; -import { createEnv, tuningOf } from './run'; +import { createEnv, tuningOf, waitsForUpload } from './run'; import { parseSyncState } from './sync-state'; export interface TeardownOptions { @@ -36,12 +36,12 @@ export async function countPending(reader: ProviderReader, authority: Authority) const masters = new Set(rows.map((r) => str(r.original_id) ?? str(r.original_sync_id) ?? `row:${String(r._id)}`)); return masters.size; } - const contacts = await reader.rows('raw_contacts', ['_id'], 'dirty = 1 OR deleted = 1 OR sourceid IS NULL'); - const groups = await reader.rows('groups', ['_id', 'sourceid', 'dirty', 'deleted']); + const contacts = await reader.rows('raw_contacts', ['_id', 'sourceid', 'deleted', 'sync3'], 'dirty = 1 OR deleted = 1 OR sourceid IS NULL'); + const groups = await reader.rows('groups', ['_id', 'sourceid', 'dirty', 'deleted', 'sync3']); const present = new Set(groups.map((g) => str(g.sourceid)).filter(Boolean)); - const waitingGroups = groups.filter((g) => !str(g.sourceid) || Number(g.dirty) === 1 || Number(g.deleted) === 1).length; + const waitingGroups = groups.filter((g) => (!str(g.sourceid) || Number(g.dirty) === 1 || Number(g.deleted) === 1) && waitsForUpload(g)).length; const absent = Object.values(state.accounts).reduce((n, a) => n + (a.groups ?? []).filter((ref) => !present.has(ref)).length, 0); - return contacts.length + waitingGroups + absent; + return contacts.filter(waitsForUpload).length + waitingGroups + absent; } function deleteEverything(authority: Authority): ProviderOp[] { From 0664a31f8b2aa90d33eda5d10fb7074ac64ddf76 Mon Sep 17 00:00:00 2001 From: Linus Rath <139418639+rathlinus@users.noreply.github.com> Date: Sun, 27 Sep 2026 10:02:22 +0200 Subject: [PATCH 76/87] fix: keep reminding of synced events while the phone's calendar can't sync #34 With calendar sync on and the calendar app reminding, Bulwark skipped its own reminders for the synced calendars whenever sync was on in the app, even when the phone's calendar received nothing: its calendar sync paused in Android's account settings, Android's auto-sync off, or the calendar permission revoked. Events created on the server meanwhile never reached the phone, and nobody reminded of them. The reminder filter now keeps what Android last said about each account (the permission, `getMasterSyncAutomatically` and the calendar's `getSyncAutomatically`) and stays quiet only while the phone's calendar receives events. Auto-sync off counts as not receiving: Android then runs manual syncs only, and the periodic, pushed and app-triggered syncs are not manual. Android's side is read when the scheduler subscribes and again whenever the filter is asked for; a changed answer makes the scheduler reschedule. Until it was read, Bulwark reminds: a double reminder is better than a missed one. --- .../__tests__/app/reminders.test.ts | 52 +++++++++++++- src/device-sync/app/reminders.ts | 71 +++++++++++++++++-- 2 files changed, 117 insertions(+), 6 deletions(-) diff --git a/src/device-sync/__tests__/app/reminders.test.ts b/src/device-sync/__tests__/app/reminders.test.ts index 744081c4..ed1b5915 100644 --- a/src/device-sync/__tests__/app/reminders.test.ts +++ b/src/device-sync/__tests__/app/reminders.test.ts @@ -1,9 +1,21 @@ import { describe, it, expect, vi, beforeEach } from 'vitest'; -const h = vi.hoisted(() => ({ available: true })); +// Android's side: the device's auto-sync, the account's calendar sync and the calendar permission. +const h = vi.hoisted(() => ({ available: true, master: true, automatic: true, permitted: true })); vi.mock('../../native', () => ({ isDeviceSyncAvailable: () => h.available, + getSyncSettings: vi.fn(async () => ({ + masterAutomatic: h.master, + authorities: { + 'com.android.contacts': { syncable: 1, automatic: false, periodicSeconds: 0, active: false, pending: false }, + 'com.android.calendar': { syncable: 1, automatic: h.automatic, periodicSeconds: 3600, active: false, pending: false }, + }, + })), +})); + +vi.mock('../../app/permissions', () => ({ + hasSyncPermissions: vi.fn(async () => h.permitted), })); import { CALENDAR_AUTHORITY, CONTACTS_AUTHORITY } from '../../types'; @@ -34,10 +46,16 @@ function calendarSync(extra: Record = {}): void { }); } +/** Waits until the calendar app reminds of Alice's synced events (Android's side is read when the filter is asked for). */ +async function remindedByCalendarApp(): Promise { + await vi.waitFor(() => expect(remindedOnDevice(ALICE, 'c')).not.toBeNull()); +} + beforeEach(async () => { await waitForDeviceSyncHydration(); - h.available = true; + Object.assign(h, { available: true, master: true, automatic: true, permitted: true }); calendarSync(); + await remindedByCalendarApp(); }); describe('remindedOnDevice', () => { @@ -82,6 +100,36 @@ describe('remindedOnDevice', () => { }); }); +describe("remindedOnDevice while the device calendar can't receive events", () => { + it('leaves the reminders to Bulwark while calendar sync is paused in Android settings', async () => { + h.automatic = false; + await vi.waitFor(() => expect(remindedOnDevice(ALICE, 'c')).toBeNull()); + }); + + it("leaves them to Bulwark while Android's auto-sync is off: Android then runs manual syncs only", async () => { + h.master = false; + await vi.waitFor(() => expect(remindedOnDevice(ALICE, 'c')).toBeNull()); + }); + + it('leaves them to Bulwark while the calendar permission is revoked', async () => { + h.permitted = false; + await vi.waitFor(() => expect(remindedOnDevice(ALICE, 'c')).toBeNull()); + }); + + it('hands them back to the calendar app once it syncs again, and tells the scheduler both times', async () => { + const listener = vi.fn(); + const off = onDeviceRemindersChange(listener); + h.automatic = false; + await vi.waitFor(() => expect(listener).toHaveBeenCalledTimes(1)); + expect(remindedOnDevice(ALICE, 'c')).toBeNull(); + + h.automatic = true; + await vi.waitFor(() => expect(listener).toHaveBeenCalledTimes(2)); + expect(remindedOnDevice(ALICE, 'c')).not.toBeNull(); + off(); + }); +}); + describe('onDeviceRemindersChange', () => { it('fires for the owner and the calendar selection, not for sync bookkeeping', () => { const listener = vi.fn(); diff --git a/src/device-sync/app/reminders.ts b/src/device-sync/app/reminders.ts index 023bac16..bb45c209 100644 --- a/src/device-sync/app/reminders.ts +++ b/src/device-sync/app/reminders.ts @@ -4,18 +4,28 @@ * app owns the reminders, the device fires them from its Reminders rows, so * Bulwark's own scheduler skips the events of those calendars. Tasks are not * synced and keep Bulwark's reminders. + * + * Only while the device calendar receives the server's events, as Android + * last said: not while the account's calendar sync is paused in Android's + * settings, the device's auto-sync is off (Android then runs manual syncs + * only, and the periodic, pushed and app-triggered ones are not manual), or + * the calendar permission is revoked. Bulwark reminds then, and until + * Android's side was read: a double reminder is better than a missed one. */ import type { CalendarEvent } from '../../api/types'; +import { getSyncSettings } from '../native'; import { isCollectionSelected, selectionFor, syncOnInApp, useDeviceSyncStore, + waitForDeviceSyncHydration, type AccountDeviceSync, } from '../../stores/device-sync-store'; import { collectionKey } from '../common/ids'; import { CALENDAR_AUTHORITY } from '../types'; import { deviceSyncAvailable } from './available'; +import { hasSyncPermissions } from './permissions'; /** Whether the device's calendar app fires the reminders of an account's synced events. */ function deviceOwnsReminders(entry: AccountDeviceSync | undefined): entry is AccountDeviceSync { @@ -23,11 +33,53 @@ function deviceOwnsReminders(entry: AccountDeviceSync | undefined): entry is Acc return !!entry && syncOnInApp(entry, CALENDAR_AUTHORITY) && (entry.reminderOwner ?? 'device') === 'device'; } +// ─── What Android says ───────────────────────────────── + +/** App accounts whose device calendar receives the server's events, as last read from Android. */ +let receiving = new Set(); +const androidListeners = new Set<() => void>(); +let reads = 0; + +async function calendarReceives(accountName: string | undefined): Promise { + if (!accountName) return false; + try { + const settings = await getSyncSettings(accountName); + return settings.masterAutomatic !== false && !!settings.authorities?.[CALENDAR_AUTHORITY]?.automatic; + } catch { + return false; + } +} + +/** + * Reads from Android whose device calendars receive the server's events; + * a change calls the `onDeviceRemindersChange` listeners. Only the latest + * read lands. + */ +async function readAndroid(): Promise { + const read = ++reads; + await waitForDeviceSyncHydration(); + const owners = Object.entries(useDeviceSyncStore.getState().accounts).filter(([, entry]) => deviceOwnsReminders(entry)); + const next = new Set(); + if (owners.length > 0 && (await hasSyncPermissions(CALENDAR_AUTHORITY))) { + for (const [registryId, entry] of owners) { + if (await calendarReceives(entry.androidAccountName)) next.add(registryId); + } + } + if (read !== reads) return; + if (next.size === receiving.size && [...next].every((id) => receiving.has(id))) return; + receiving = next; + for (const listener of [...androidListeners]) listener(); +} + +// ─── The filter ──────────────────────────────────────── + /** * A test for the events whose reminders the device's calendar app fires for * this app account, so Bulwark schedules none; null when there are none. * `primaryJmapAccountId`: the JMAP account the app's calendar store loads - * events from when they carry no `accountId` (the user's own). + * events from when they carry no `accountId` (the user's own). Android's side + * is read again each time, for the next time: a change calls the + * `onDeviceRemindersChange` listeners. */ export function remindedOnDevice( registryId: string | null | undefined, @@ -36,6 +88,8 @@ export function remindedOnDevice( if (!registryId || !deviceSyncAvailable()) return null; const entry = useDeviceSyncStore.getState().accounts[registryId]; if (!deviceOwnsReminders(entry)) return null; + void readAndroid(); + if (!receiving.has(registryId)) return null; const selection = selectionFor(entry, CALENDAR_AUTHORITY); const personal = entry.primaryJmapAccounts?.[CALENDAR_AUTHORITY] ?? primaryJmapAccountId ?? null; return (event) => { @@ -65,15 +119,24 @@ function reminderSignature(accounts: Record): string /** * Calls `listener` whenever the set of events the device reminds of may have - * changed (reminder owner, calendar sync on or off, the calendar selection). - * Returns the unsubscribe. + * changed (reminder owner, calendar sync on or off, the calendar selection, + * whether Android lets the device calendar sync). Android's side is read + * right away, so the first reminders are scheduled with it known. Returns + * the unsubscribe. */ export function onDeviceRemindersChange(listener: () => void): () => void { let last = reminderSignature(useDeviceSyncStore.getState().accounts); - return useDeviceSyncStore.subscribe((state) => { + const unsubscribe = useDeviceSyncStore.subscribe((state) => { const next = reminderSignature(state.accounts); if (next === last) return; last = next; listener(); }); + const android = () => listener(); + androidListeners.add(android); + if (deviceSyncAvailable()) void readAndroid(); + return () => { + unsubscribe(); + androidListeners.delete(android); + }; } From ef96acd05f68c135e19b34e58b08d13a5fde2f56 Mon Sep 17 00:00:00 2001 From: Linus Rath <139418639+rathlinus@users.noreply.github.com> Date: Sun, 27 Sep 2026 10:03:20 +0200 Subject: [PATCH 77/87] fix: finish turning sync on or off before signing an account out #34 Turning device sync on or off runs one step after the other per account, but signing out did not wait its turn. A sign-out that started while a turn-on was still setting up (or queued behind a slow turn-off) found no Android account yet and went ahead; the turn-on then created the Android account, stored it and requested a sync for the signed-out account, and the next check posted "Sign in to keep syncing". A turn-off still tearing down could likewise store its preferences after the sign-out had forgotten the account. The release before a sign-out is now a step in the queue of every account it releases, taken in a fixed order: it waits for the steps under way and releases what they set up. A turn-on or turn-off asked for while the sign-out runs comes after it, and does nothing once the account was released; if the user stays signed in it goes ahead. --- .../__tests__/app/lifecycle.test.ts | 59 +++++++++++++++++++ src/device-sync/app/lifecycle.ts | 58 +++++++++++++----- 2 files changed, 103 insertions(+), 14 deletions(-) diff --git a/src/device-sync/__tests__/app/lifecycle.test.ts b/src/device-sync/__tests__/app/lifecycle.test.ts index c61d4a85..b95b0bb1 100644 --- a/src/device-sync/__tests__/app/lifecycle.test.ts +++ b/src/device-sync/__tests__/app/lifecycle.test.ts @@ -375,6 +375,65 @@ describe('sign-out', () => { expect(confirm).toHaveBeenCalledWith(4); expect(h.state.accounts).toEqual([]); }); + + it('waits for a turn-on in progress, then releases what it set up', async () => { + let finishEnsure: () => void = () => undefined; + vi.mocked(native.ensureAndroidAccount).mockImplementationOnce(async (name: string, registryId: string) => { + h.state.log.push(`ensure ${name}`); + await new Promise((resolve) => { finishEnsure = resolve; }); + h.state.accounts.push({ name, registryId, sync: {} }); + return true; + }); + const turningOn = enableDeviceSync(ALICE, CONTACTS_AUTHORITY); + await vi.waitFor(() => expect(h.state.log).toContain('ensure alice')); + + const signingOut = releaseDeviceSyncBeforeSignOut([ALICE], vi.fn(async () => true)); + finishEnsure(); + + expect(await signingOut).toBe(true); + await turningOn; + expect(teardown).toHaveBeenCalledWith(ALICE, 'alice', CONTACTS_AUTHORITY, { uploadFirst: true, timeoutMs: TEARDOWN_TIMEOUT_MS }); + expect(h.state.accounts).toEqual([]); + expect(useDeviceSyncStore.getState().accounts[ALICE]).toBeUndefined(); + }); + + it('lets a turn-on or turn-off asked for during the sign-out set nothing up for the account', async () => { + await enableDeviceSync(ALICE, CONTACTS_AUTHORITY); + let finishTeardown: () => void = () => undefined; + teardown.mockImplementationOnce(async () => { + await new Promise((resolve) => { finishTeardown = resolve; }); + return { pending: 0 }; + }); + const signingOut = releaseDeviceSyncBeforeSignOut([ALICE], vi.fn(async () => true)); + await vi.waitFor(() => expect(teardown).toHaveBeenCalled()); + + const turningOn = enableDeviceSync(ALICE, CALENDAR_AUTHORITY); + const turningOff = disableDeviceSync(ALICE, CONTACTS_AUTHORITY); + finishTeardown(); + + expect(await signingOut).toBe(true); + expect((await turningOn).kind).toBe('failed'); + expect(await turningOff).toEqual({ done: true, pending: 0 }); + expect(h.state.accounts).toEqual([]); + expect(h.state.log.filter((l) => l.startsWith('ensure') || l.startsWith('sync'))).toEqual(['ensure alice', `sync alice ${CONTACTS_AUTHORITY} {"manual":true}`]); + expect(useDeviceSyncStore.getState().accounts[ALICE]).toBeUndefined(); + }); + + it('lets a turn-on asked for during the sign-out go ahead once the user chose to stay signed in', async () => { + await enableDeviceSync(ALICE, CONTACTS_AUTHORITY); + teardown.mockImplementation(async (_r: string, _n: string, authority: string) => ({ pending: authority === CONTACTS_AUTHORITY ? 1 : 0 })); + let answer: (signOut: boolean) => void = () => undefined; + const confirm = vi.fn(() => new Promise((resolve) => { answer = resolve; })); + const signingOut = releaseDeviceSyncBeforeSignOut([ALICE], confirm); + await vi.waitFor(() => expect(confirm).toHaveBeenCalled()); + + const turningOn = enableDeviceSync(ALICE, CALENDAR_AUTHORITY); + answer(false); + + expect(await signingOut).toBe(false); + expect(await turningOn).toEqual({ kind: 'enabled', accountName: 'alice' }); + expect(h.find('alice')?.sync[CALENDAR_AUTHORITY].automatic).toBe(true); + }); }); describe('reconcile', () => { diff --git a/src/device-sync/app/lifecycle.ts b/src/device-sync/app/lifecycle.ts index e873595e..4daba392 100644 --- a/src/device-sync/app/lifecycle.ts +++ b/src/device-sync/app/lifecycle.ts @@ -180,6 +180,25 @@ function oneAtATime(registryId: string, step: () => Promise): Promise { return next; } +/** One step in the queue of every one of the accounts, taken in a fixed order so two such steps never wait for each other. */ +function oneAtATimeForAll(registryIds: readonly string[], step: () => Promise): Promise { + const [first, ...rest] = registryIds; + return first === undefined ? step() : oneAtATime(first, () => oneAtATimeForAll(rest, step)); +} + +// Sign-outs under way, per app account: true once they released it. +const signOuts = new Map>(); + +/** + * A turn-on or turn-off of one app account, in its queue. One asked for + * while the account signs out runs after the sign-out, and does nothing + * (`released`) when that released the account. + */ +function accountStep(registryId: string, step: () => Promise, released: T): Promise { + const signOut = signOuts.get(registryId); + return oneAtATime(registryId, async () => ((await signOut) ? released : step())); +} + // ─── Enable ──────────────────────────────────────────── export type EnableOutcome = @@ -193,7 +212,7 @@ export type EnableOutcome = * asked for the permissions (and, for calendars, who reminds the user). */ export function enableDeviceSync(registryId: string, authority: Authority): Promise { - return oneAtATime(registryId, () => enable(registryId, authority)); + return accountStep(registryId, () => enable(registryId, authority), { kind: 'failed', message: 'The account was signed out' }); } async function enable(registryId: string, authority: Authority): Promise { @@ -230,7 +249,7 @@ async function enable(registryId: string, authority: Authority): Promise { - return oneAtATime(registryId, () => resume(registryId, authority)); + return accountStep(registryId, () => resume(registryId, authority), undefined); } async function resume(registryId: string, authority: Authority): Promise { @@ -316,7 +335,8 @@ export function disableDeviceSync( authority: Authority, options: { force?: boolean } = {}, ): Promise { - return oneAtATime(registryId, () => disable(registryId, authority, options)); + // Released for signing out meanwhile: nothing of it is left on the device. + return accountStep(registryId, () => disable(registryId, authority, options), { done: true, pending: 0 }); } async function disable(registryId: string, authority: Authority, options: { force?: boolean }): Promise { @@ -363,21 +383,31 @@ async function disable(registryId: string, authority: Authority, options: { forc * rows and removes their Android accounts, while the credentials are still * there. When changes could not be uploaded, `confirm` asks whether to sign * out anyway. Resolves false when the user chose to stay signed in; their - * accounts then keep syncing. + * accounts then keep syncing. It is a step of the accounts' own turn-ons and + * turn-offs: it waits for those under way, and those asked for meanwhile find + * the accounts released and do nothing. */ -export async function releaseDeviceSyncBeforeSignOut( +export function releaseDeviceSyncBeforeSignOut( registryIds: readonly string[], confirm: (pending: number) => Promise = confirmSignOutLosingChanges, ): Promise { - if (!deviceSyncAvailable() || registryIds.length === 0) return true; - try { - return await release(registryIds, confirm); - } catch (err) { - // Never keep the user from signing out over a bug here; an Android - // account left behind is suspended by the next reconcile. - console.warn('[device-sync] release before sign-out failed', errorMessage(err)); - return true; - } + if (!deviceSyncAvailable() || registryIds.length === 0) return Promise.resolve(true); + const ids = [...new Set(registryIds)].sort(); + const signOut = oneAtATimeForAll(ids, async () => { + try { + return await release(registryIds, confirm); + } catch (err) { + // Never keep the user from signing out over a bug here; an Android + // account left behind is suspended by the next reconcile. + console.warn('[device-sync] release before sign-out failed', errorMessage(err)); + return true; + } + }); + for (const id of ids) signOuts.set(id, signOut); + void signOut.then(() => { + for (const id of ids) if (signOuts.get(id) === signOut) signOuts.delete(id); + }); + return signOut; } async function release( From 5e62cf71f4c8f96c8b6f84b50de5b981ee67b0b5 Mon Sep 17 00:00:00 2001 From: Linus Rath <139418639+rathlinus@users.noreply.github.com> Date: Sun, 27 Sep 2026 10:04:18 +0200 Subject: [PATCH 78/87] fix: add no extra account named after its id when Android refuses one #34 Turning device sync on retried with the app account's registry id as the Android account name whenever adding the account failed. That fallback is for one case only: the login name turned out to belong to another app account's Android account, which the native module reports with the `conflict` code. A passing AccountManager failure or a permission error instead created a second account named after the opaque id, which then showed in the Contacts and Calendar apps. Only a `conflict` rejection takes the fallback now; any other failure fails the turn-on as before. --- src/device-sync/__tests__/app/lifecycle.test.ts | 15 ++++++++++++++- src/device-sync/app/lifecycle.ts | 5 +++-- 2 files changed, 17 insertions(+), 3 deletions(-) diff --git a/src/device-sync/__tests__/app/lifecycle.test.ts b/src/device-sync/__tests__/app/lifecycle.test.ts index b95b0bb1..18729359 100644 --- a/src/device-sync/__tests__/app/lifecycle.test.ts +++ b/src/device-sync/__tests__/app/lifecycle.test.ts @@ -43,7 +43,8 @@ vi.mock('../../native', () => ({ ensureAndroidAccount: vi.fn(async (name: string, registryId: string) => { h.state.log.push(`ensure ${name}`); const existing = h.find(name); - if (existing && existing.registryId !== registryId) throw new Error('conflict'); + // As the native module rejects (DeviceSyncAccounts.ensure → AccountConflictException). + if (existing && existing.registryId !== registryId) throw Object.assign(new Error(`${name} belongs to another app account`), { code: 'conflict' }); if (existing) return false; h.state.accounts.push({ name, registryId, sync: {} }); return true; @@ -212,6 +213,18 @@ describe('enable', () => { expect(outcome).toEqual({ kind: 'enabled', accountName: ALICE }); }); + it('adds no second account named after the registry id when adding the account fails for another reason', async () => { + for (const code of ['failed', 'permission']) { + vi.mocked(native.ensureAndroidAccount).mockClear(); + vi.mocked(native.ensureAndroidAccount).mockRejectedValueOnce(Object.assign(new Error(`AccountManager said no (${code})`), { code })); + const outcome = await enableDeviceSync(ALICE, CONTACTS_AUTHORITY); + expect(outcome).toEqual({ kind: 'failed', message: `AccountManager said no (${code})` }); + expect(native.ensureAndroidAccount).toHaveBeenCalledTimes(1); + expect(h.state.accounts).toEqual([]); + expect(accountDeviceSync(ALICE).androidAccountName).toBeUndefined(); + } + }); + it('leaves no account behind when turning sync on fails', async () => { h.state.failSetSyncEnabled = true; const outcome = await enableDeviceSync(ALICE, CONTACTS_AUTHORITY); diff --git a/src/device-sync/app/lifecycle.ts b/src/device-sync/app/lifecycle.ts index 4daba392..6f058c71 100644 --- a/src/device-sync/app/lifecycle.ts +++ b/src/device-sync/app/lifecycle.ts @@ -122,8 +122,9 @@ async function ensureAccountFor(registryId: string): Promise { await ensureAndroidAccount(name, registryId); return name; } catch (err) { - // The name belongs to another app account after all (the list was stale). - if (name === registryId) throw err; + // The name belongs to another app account after all (the list was stale). Any + // other failure fails the turn-on: a second account would show in the apps. + if (name === registryId || (err as { code?: unknown } | null)?.code !== 'conflict') throw err; await ensureAndroidAccount(registryId, registryId); return registryId; } From 800c4e144203a54cb684082d084ff0385acd8ec5 Mon Sep 17 00:00:00 2001 From: Linus Rath <139418639+rathlinus@users.noreply.github.com> Date: Sun, 27 Sep 2026 10:05:16 +0200 Subject: [PATCH 79/87] fix: stop waiting for React Native once a sync gave up on it #34 A sync that had to start React Native left a listener on the React host until a context came up. When the start failed (a debug build without Metro) or never finished, every retried sync added another one, each holding its run, account and extras, for the life of the process, and they all fired at once when a context finally came up. The wait now ends once, and removes its listener, for whatever reason: a ready context, a start task that faulted, or 10 minutes (no sync or push job waits longer; JobScheduler stops them). A context that comes up later runs nothing of it. The adapter's own check of its run in the callback still covers a run it gave up sooner. --- .../com/anonymous/bulwarkmobile/HeadlessJs.kt | 86 ++++++++++++++----- 1 file changed, 63 insertions(+), 23 deletions(-) diff --git a/android/app/src/main/java/com/anonymous/bulwarkmobile/HeadlessJs.kt b/android/app/src/main/java/com/anonymous/bulwarkmobile/HeadlessJs.kt index e8521c38..b9722584 100644 --- a/android/app/src/main/java/com/anonymous/bulwarkmobile/HeadlessJs.kt +++ b/android/app/src/main/java/com/anonymous/bulwarkmobile/HeadlessJs.kt @@ -1,6 +1,10 @@ package com.anonymous.bulwarkmobile import android.app.Application +import android.os.Handler +import android.os.Looper +import android.os.SystemClock +import android.util.Log import androidx.annotation.MainThread import com.facebook.react.ReactApplication import com.facebook.react.ReactInstanceEventListener @@ -19,6 +23,12 @@ import java.util.concurrent.atomic.AtomicBoolean * sync adapters. */ object HeadlessJs { + private const val TAG = "HeadlessJs" + + /** No caller waits longer for React Native: JobScheduler stops sync and push jobs after 10 minutes. */ + private const val MAX_WAIT_MS = 600_000L + private const val WATCH_MS = 1_000L + /** * Calls [onReady] once with a React context whose JS instance is up, * starting React Native if the process was woken for this work. @@ -30,6 +40,13 @@ object HeadlessJs { * first and the ready check after it; whichever sees the ready instance * first runs the task. * + * The wait ends once, and its listener goes with it: with a ready context, + * or without one when React fails to start (the start task faults) or + * after [MAX_WAIT_MS], when no caller can still be waiting. A context that + * comes up later runs nothing of it, so waits given up on don't pile up on + * the host with everything their [onReady] holds. (A caller that gives up + * sooner, like the sync adapter at its deadline, checks that in [onReady].) + * * Call it on the main thread; [onReady] runs there too. MainApplication * creates the ReactHost on first use without a lock (ExpoReactHostFactory), * so a background caller racing MainActivity could create a second one, @@ -41,45 +58,68 @@ object HeadlessJs { */ @MainThread fun withReadyReactContext(app: Application, onReady: (ReactContext) -> Unit): TaskInterface? { - val done = AtomicBoolean(false) - val runOnce = { context: ReactContext -> if (done.compareAndSet(false, true)) onReady(context) } val reactApp = app as ReactApplication if (ReactNativeNewArchitectureFeatureFlags.enableBridgelessArchitecture()) { val host = checkNotNull(reactApp.reactHost) { "ReactHost is not initialized in New Architecture" } - val listener = object : ReactInstanceEventListener { - override fun onReactContextInitialized(context: ReactContext) { - host.removeReactInstanceEventListener(this) - runOnce(context) - } - } - host.addReactInstanceEventListener(listener) + val wait = ContextWait(onReady) { host.removeReactInstanceEventListener(it) } + host.addReactInstanceEventListener(wait) val current = host.currentReactContext if (current != null && current.hasActiveReactInstance()) { - host.removeReactInstanceEventListener(listener) - runOnce(current) + wait.onReactContextInitialized(current) return null } - return host.start() + return host.start().also { wait.watch(it) } } @Suppress("DEPRECATION") val manager = reactApp.reactNativeHost.reactInstanceManager - val listener = object : ReactInstanceEventListener { - override fun onReactContextInitialized(context: ReactContext) { - manager.removeReactInstanceEventListener(this) - runOnce(context) - } - } - manager.addReactInstanceEventListener(listener) + val wait = ContextWait(onReady) { manager.removeReactInstanceEventListener(it) } + manager.addReactInstanceEventListener(wait) val current = manager.currentReactContext if (current != null && current.hasActiveReactInstance()) { - manager.removeReactInstanceEventListener(listener) - runOnce(current) - } else if (!manager.hasStartedCreatingInitialContext()) { - manager.createReactContextInBackground() + wait.onReactContextInitialized(current) + return null } + if (!manager.hasStartedCreatingInitialContext()) manager.createReactContextInBackground() + wait.watch(null) return null } + /** One caller's wait for a ready context: a listener on the host that ends once and then removes itself. */ + private class ContextWait( + private val onReady: (ReactContext) -> Unit, + private val removeListener: (ReactInstanceEventListener) -> Unit, + ) : ReactInstanceEventListener { + private val ended = AtomicBoolean(false) + + override fun onReactContextInitialized(context: ReactContext) { + if (end()) onReady(context) + } + + /** Ends the wait without a context once [boot] faulted or [MAX_WAIT_MS] passed; checked on the main thread. */ + fun watch(boot: TaskInterface?) { + val main = Handler(Looper.getMainLooper()) + val giveUpAt = SystemClock.elapsedRealtime() + MAX_WAIT_MS + main.postDelayed(object : Runnable { + override fun run() { + if (ended.get()) return + val faulted = boot?.isFaulted() == true + if (!faulted && SystemClock.elapsedRealtime() < giveUpAt) { + main.postDelayed(this, WATCH_MS) + } else if (end()) { + val why = if (faulted) "React Native did not start" else "no React context after $MAX_WAIT_MS ms" + Log.w(TAG, "$why; no longer waiting") + } + } + }, WATCH_MS) + } + + private fun end(): Boolean { + if (!ended.compareAndSet(false, true)) return false + removeListener(this) + return true + } + } + /** * Starts [config] on the UI thread, where HeadlessJsTaskContext wants it. * [onStarted] gets the task id in that same UI-thread turn, so before any From df3175ff72b43d6777cfc4b1dd5ffbaa0238ef16 Mon Sep 17 00:00:00 2001 From: Linus Rath <139418639+rathlinus@users.noreply.github.com> Date: Sun, 27 Sep 2026 10:06:14 +0200 Subject: [PATCH 80/87] refactor: decide in one place whether a card is in read-only books only #34 --- .../__tests__/contacts/real-engine.test.ts | 18 +++++++++++++++++- src/device-sync/engine/contacts-sync.ts | 14 ++++++++------ 2 files changed, 25 insertions(+), 7 deletions(-) diff --git a/src/device-sync/__tests__/contacts/real-engine.test.ts b/src/device-sync/__tests__/contacts/real-engine.test.ts index 3e1ee1c3..e4cba093 100644 --- a/src/device-sync/__tests__/contacts/real-engine.test.ts +++ b/src/device-sync/__tests__/contacts/real-engine.test.ts @@ -6,7 +6,7 @@ import { describe, expect, it } from 'vitest'; import { Data, Groups, MimeType } from '../../android-columns'; import { calendarPlanner } from '../../calendar/planner'; import { contactsPlanner } from '../../contacts/planner'; -import { addServerCards, ANDROID_ACCOUNT, CONTACTS_AUTHORITY, createHarness, rowWrites, type Harness } from '../engine/harness'; +import { addServerCards, ANDROID_ACCOUNT, CONTACTS_AUTHORITY, createHarness, renameDeviceContact, rowWrites, type Harness } from '../engine/harness'; import type { SetTarget } from '../fakes/fake-jmap-server'; import { JPEG } from './fixtures'; @@ -101,6 +101,22 @@ describe('contacts through the engine', () => { expect(await h.teardown(CONTACTS_AUTHORITY)).toEqual({ pending: 0 }); }); + it('puts back an edit of a card that is only in a read-only book, without asking the server', async () => { + const h = real(); + const shared = h.server.addAddressBook('a', { name: 'Shared', myRights: { mayRead: true, mayWrite: false, mayShare: false, mayDelete: false } }); + h.prefs.contactsSelection[`a/${shared}`] = true; + const [ada] = addServerCards(h, ['Ada Lovelace'], 'a', shared); + expect((await h.run()).outcome).toBe('ok'); + + renameDeviceContact(h, h.contactNamed('Ada Lovelace')!.id, 'Ada King'); + const report = await h.run(); + + expect(report).toMatchObject({ outcome: 'ok', stats: { skipped: 1, uploaded: { updated: 0 } } }); + expect(report.itemErrors).toEqual([expect.objectContaining({ side: 'upload', type: 'readOnly' })]); + expect(h.contactNamed('Ada Lovelace')).toMatchObject({ dirty: false }); + expect(h.server.get('ContactCard', 'a', ada)).toMatchObject({ name: { full: 'Ada Lovelace' } }); + }); + it('puts a contact created on the device into the group it was added to', async () => { const h = real(); const group = h.server.addCard('a', { uid: 'g1', kind: 'group', name: { full: 'Friends' }, members: { other: true }, addressBookIds: { [h.book]: true } }); diff --git a/src/device-sync/engine/contacts-sync.ts b/src/device-sync/engine/contacts-sync.ts index 2e3469ba..9a0afa0e 100644 --- a/src/device-sync/engine/contacts-sync.ts +++ b/src/device-sync/engine/contacts-sync.ts @@ -265,8 +265,7 @@ export class ContactsSync extends ItemSync { for (const group of await this.loadGroupsWhere()) { if (!group.sourceId || group.deleted) continue; this.groupRows.set(group.sourceId, group.groupId); - const keys = this.cardKeys(accountOfRef(group.sourceId), group.shadow); - if (keys.length > 0 && keys.every((k) => this.book(k)?.myRights?.mayWrite === false)) this.readOnlyGroups.add(group.sourceId); + if (this.inReadOnlyBooksOnly(accountOfRef(group.sourceId), group.shadow)) this.readOnlyGroups.add(group.sourceId); for (const uid of onIds(group.shadow?.members as Record | undefined)) { let set = this.membership.get(uid); if (!set) this.membership.set(uid, (set = new Set())); @@ -293,6 +292,12 @@ export class ContactsSync extends ItemSync { return !!book && book.myRights?.mayWrite !== false; } + /** Whether the card is in address books, and only in ones that take no writes. */ + private inReadOnlyBooksOnly(acct: string | null, card: Pick | null | undefined): boolean { + const keys = this.cardKeys(acct, card); + return keys.length > 0 && keys.every((k) => this.book(k)?.myRights?.mayWrite === false); + } + protected collectionGone(key: string): boolean { const parsed = parseCollectionKey(key); const books = parsed ? this.books.get(parsed.accountId) : undefined; @@ -373,10 +378,7 @@ export class ContactsSync extends ItemSync { mintKey: (taken) => env.mintKey(taken), mintUid: () => env.mintUid(), selectedCollections: (card) => this.cardKeys(acct, card).filter((k) => this.syncedKeys.has(k)), - isReadOnly: (card) => { - const keys = this.cardKeys(acct, card); - return keys.length > 0 && keys.every((k) => this.book(k)?.myRights?.mayWrite === false); - }, + isReadOnly: (card) => this.inReadOnlyBooksOnly(acct, card), isSelected: (key) => this.syncedKeys.has(key), createTarget: () => this.createTarget(), photoBytes: (entry) => this.photoBytes(entry), From 156e79b348833367cecfcba009b2ed822f4843a2 Mon Sep 17 00:00:00 2001 From: Linus Rath <139418639+rathlinus@users.noreply.github.com> Date: Sun, 27 Sep 2026 10:07:14 +0200 Subject: [PATCH 81/87] fix: keep a card or event another app moved away when it is deleted on the phone #34 A contact or event deleted on the phone that another client meanwhile filed in an address book or calendar the phone doesn't sync was destroyed on the server by the phone's next sync, and the other client's move with it. 0faee56 made the sync decide such a deletion on the server's version, but the planners still destroyed it: the contacts planner saw the card in no synced book and destroyed it, and the calendar planner could not take the new version at all (it placed the event before looking at the deletion), so the old one decided. The deletion is now moot when the server's version holds none of the collections the rows were written for and none the phone syncs: only the rows go, nothing is uploaded, and the object stays where the other client put it. A deletion still goes up when the book or calendar was deselected on the phone before the upload, and when the object was moved into another collection the phone syncs, since the phone would otherwise miss an object it should show. The calendar planner now lets a deleted row's shadow follow the server before it places the event. --- .../__tests__/calendar/lifecycle.test.ts | 28 ++++++++++++ .../__tests__/contacts/upload-editors.test.ts | 30 +++++++++++++ .../__tests__/engine/real-planners.test.ts | 45 ++++++++++++++++++- src/device-sync/calendar/create.ts | 14 ++++++ src/device-sync/calendar/planner.ts | 30 +++++++------ src/device-sync/contacts/upload.ts | 14 ++++++ src/device-sync/engine/item-sync.ts | 2 +- 7 files changed, 148 insertions(+), 15 deletions(-) diff --git a/src/device-sync/__tests__/calendar/lifecycle.test.ts b/src/device-sync/__tests__/calendar/lifecycle.test.ts index 038d2d00..9c9e8a4b 100644 --- a/src/device-sync/__tests__/calendar/lifecycle.test.ts +++ b/src/device-sync/__tests__/calendar/lifecycle.test.ts @@ -336,6 +336,34 @@ describe('calendar planner: accepted uploads, merges and deletions', () => { expect(await h.events()).toEqual([]); }); + it('leaves an event the server moved out of every synced calendar before the deletion reached it, and only purges the rows', async () => { + const { h, id } = await synced(single()); + const isSelected = h.ctx.isSelected; + h.ctx = { ...h.ctx, isSelected: (key) => key !== `${ACCT}/x` && isSelected(key) }; + h.fake.user.deleteEvent(id); + // The shadow follows the server although no synced calendar holds the event now. + expect(await h.download(single({ calendarIds: { x: true } }))).toMatchObject({ stillDirty: true, effect: 'update' }); + expect(h.row(id)).toMatchObject({ [Events.DELETED]: 1 }); + + const plan = calendarPlanner.planUpload((await h.local('e1'))!, h.ctx); + expect(plan.kind).toBe('purge'); + if (plan.kind === 'purge') await h.apply(plan.ops); + expect(await h.events()).toEqual([]); + }); + + it('still destroys an event the server moved to another synced calendar, or whose calendar was deselected before the upload', async () => { + const destroy = [{ kind: 'destroy', id: 'e1', uid: 'single-1@example.org' }]; + const moved = await synced(single()); + moved.h.fake.user.deleteEvent(moved.id); + await moved.h.download(single({ calendarIds: { w: true } })); + expect(actionsOf(calendarPlanner.planUpload((await moved.h.local('e1'))!, moved.h.ctx))).toEqual(destroy); + + const deselected = await synced(single()); + deselected.h.fake.user.deleteEvent(deselected.id); + deselected.h.ctx = { ...deselected.h.ctx, isSelected: () => false }; + expect(actionsOf(calendarPlanner.planUpload((await deselected.h.local('e1'))!, deselected.h.ctx))).toEqual(destroy); + }); + it('destroys a deleted event whose create had an unknown outcome by its uid', async () => { const h = await new Harness().setup(); const id = h.fake.user.insertEvent(h.rowIds.get('b')!, { [Events.TITLE]: 'x', [Events.DTSTART]: 0, [Events.DTEND]: 1000, [Events.EVENT_TIMEZONE]: 'UTC', [Events.STATUS]: 1 }); diff --git a/src/device-sync/__tests__/contacts/upload-editors.test.ts b/src/device-sync/__tests__/contacts/upload-editors.test.ts index d4783bef..eb2dd661 100644 --- a/src/device-sync/__tests__/contacts/upload-editors.test.ts +++ b/src/device-sync/__tests__/contacts/upload-editors.test.ts @@ -326,6 +326,36 @@ describe('contacts upload: deleted contacts', () => { expect(h.device.row('raw_contacts', rawId)).toBeUndefined(); }); + it('leaves a card the server moved out of every synced book before the deletion reached it, and only purges the rows', async () => { + const h = new Harness(); + const archive = h.server.addAddressBook(JMAP, { name: 'Archive' }); + h.unselected.add(`${JMAP}/${archive}`); + const { id, rawId } = await h.seed(appleCard()); + h.device.user.deleteContact(rawId); + h.server.serverUpdate('ContactCard', JMAP, id, { addressBookIds: { [archive]: true } }); + await h.applyOk(h.planner.planDownload(h.card(id), await h.contact(rawId), h.ctx).ops); + expect(await uploadPlan(h, rawId)).toMatchObject({ kind: 'purge' }); + await h.upload(rawId); + expect(h.device.row('raw_contacts', rawId)).toBeUndefined(); + expect(h.server.get('ContactCard', JMAP, id)).toMatchObject({ addressBookIds: { [archive]: true } }); + }); + + it('still destroys a card the server moved to another synced book, or whose book was deselected before the upload', async () => { + const moved = new Harness(); + const work = moved.server.addAddressBook(JMAP, { name: 'Work' }); + const a = await moved.seed(appleCard()); + moved.device.user.deleteContact(a.rawId); + moved.server.serverUpdate('ContactCard', JMAP, a.id, { addressBookIds: { [work]: true } }); + await moved.applyOk(moved.planner.planDownload(moved.card(a.id), await moved.contact(a.rawId), moved.ctx).ops); + expect(await uploadPlan(moved, a.rawId)).toEqual({ kind: 'upload', actions: [{ kind: 'destroy', id: a.id, uid: appleCard().uid }] }); + + const deselected = new Harness(); + const b = await deselected.seed(appleCard()); + deselected.device.user.deleteContact(b.rawId); + deselected.unselected.add(`${JMAP}/${deselected.book}`); + expect(await uploadPlan(deselected, b.rawId)).toEqual({ kind: 'upload', actions: [{ kind: 'destroy', id: b.id, uid: appleCard().uid }] }); + }); + it('deletes its own create by id once the download found it, not by a uid lookup', async () => { const h = new Harness(); const rawId = h.device.user.insertContact(ACCOUNT, [{ [Data.MIMETYPE]: MimeType.NOTE, data1: 'y' }]); diff --git a/src/device-sync/__tests__/engine/real-planners.test.ts b/src/device-sync/__tests__/engine/real-planners.test.ts index 0652f345..68596146 100644 --- a/src/device-sync/__tests__/engine/real-planners.test.ts +++ b/src/device-sync/__tests__/engine/real-planners.test.ts @@ -358,7 +358,25 @@ describe('device sync engine with the real planners', () => { expect(planned).toEqual([[archive]]); }); - it('still uploads the deletion of an event whose calendar the planner cannot place', async () => { + it('leaves a contact deleted on the device that another client moved out of every synced book', async () => { + const h = real(); + const archive = h.server.addAddressBook('a', { name: 'Archive' }); + h.prefs.contactsSelection[`a/${archive}`] = false; + const [ada] = addServerCards(h, ['Ada Lovelace', 'Grace Hopper']); + await h.run(); + h.device.user.deleteContact(h.contactNamed('Ada Lovelace')!.id); + // Another client files Ada in Archive only before the deletion reached the server. + h.server.serverUpdate('ContactCard', 'a', ada, { addressBookIds: { [archive]: true } }); + + const report = await h.run(); + + expect(report).toMatchObject({ outcome: 'ok', itemErrors: [], stats: { uploaded: { created: 0, updated: 0, deleted: 0 } } }); + expect(h.server.get('ContactCard', 'a', ada)).toMatchObject({ addressBookIds: { [archive]: true } }); + expect(h.contacts().map((c) => c.name)).toEqual(['Grace Hopper']); + }); + + /** Lunch synced, deleted on the device, then filed by another client in Archive, which this device does not sync. */ + async function lunchDeletedAndFiledAway() { const h = real(); const archive = h.server.addCalendar('a', { name: 'Archive' }); h.prefs.calendarSelection[`a/${archive}`] = false; @@ -366,6 +384,31 @@ describe('device sync engine with the real planners', () => { await h.run(CALENDAR_AUTHORITY); h.device.user.deleteEvent(Number(h.events()[0]._id)); h.server.serverUpdate('CalendarEvent', 'a', lunch, { calendarIds: { [archive]: true } }); + return { h, archive, lunch }; + } + + it('leaves an event deleted on the device that another client moved out of every synced calendar', async () => { + const { h, archive, lunch } = await lunchDeletedAndFiledAway(); + + const report = await h.run(CALENDAR_AUTHORITY); + + expect(report).toMatchObject({ outcome: 'ok', itemErrors: [], stats: { uploaded: { created: 0, updated: 0, deleted: 0 } } }); + expect(h.server.get('CalendarEvent', 'a', lunch)).toMatchObject({ calendarIds: { [archive]: true } }); + expect(h.events()).toEqual([]); + }); + + it('still uploads the deletion as it was planned when the planner cannot take the server version', async () => { + const { h, lunch } = await lunchDeletedAndFiledAway(); + h.deps.planners = { + ...h.deps.planners, + calendar: { + ...calendarPlanner, + planDownload: (event, local, ctx) => { + if (local?.deleted) throw new Error('cannot take it'); + return calendarPlanner.planDownload(event, local, ctx); + }, + }, + }; const report = await h.run(CALENDAR_AUTHORITY); diff --git a/src/device-sync/calendar/create.ts b/src/device-sync/calendar/create.ts index e1ed3fa3..a2dadf9e 100644 --- a/src/device-sync/calendar/create.ts +++ b/src/device-sync/calendar/create.ts @@ -261,6 +261,20 @@ export function createTarget(local: LocalEvent, ctx: CalendarContext): { key: st // ─── Deletions ────────────────────────────────────────── +/** + * Whether the server moved a deleted event out of the calendar its row is in + * (`own`), and into none this device syncs, before the device's deletion + * reached it: the deletion is moot then, and the event stays where it is now. + * A calendar deselected since is still the row's own, so that deletion goes + * up; so does one of an event moved into another synced calendar. + */ +export function movedAway(local: LocalEvent, own: { jmapAccountId: string; calendarId: string }, ctx: CalendarContext): boolean { + if (!local.shadow) return false; + const ownKey = collectionKey(own.jmapAccountId, own.calendarId); + const keys = Object.entries(local.shadow.calendarIds ?? {}).filter(([, on]) => on).map(([id]) => collectionKey(ctx.jmapAccountId, id)); + return !keys.some((key) => key === ownKey || ctx.isSelected(key)); +} + /** * A deleted event with identity: an object in several calendars loses only * the selected memberships; otherwise it is destroyed, with scheduling diff --git a/src/device-sync/calendar/planner.ts b/src/device-sync/calendar/planner.ts index 7a01bbfd..540f4190 100644 --- a/src/device-sync/calendar/planner.ts +++ b/src/device-sync/calendar/planner.ts @@ -27,7 +27,7 @@ import { isTaskLikeObject } from '../../lib/calendar-component-detection'; import { keepDirtyGroup, type UploadedUnits } from './accepted'; import { ATTENDEE_COLUMNS, CALENDAR_COLUMNS, EVENT_COLUMNS, EXCEPTION_CELLS, MASTER_CELLS, REMINDER_COLUMNS, isTruncatedBaseline, makeBaseline } from './columns'; import { planCalendars } from './calendars'; -import { claimOps, claimUid, claimedUid, computeCreate, createTarget, deleteActions } from './create'; +import { claimOps, claimUid, claimedUid, computeCreate, createTarget, deleteActions, movedAway } from './create'; import { decodeCalendar, decodeEvents, isExceptionRow, isNewMaster } from './decode'; import { isExcluded, overridesOf } from './exceptions'; import { eventImage } from './image'; @@ -65,6 +65,20 @@ function planDownload(event: CalendarEventWire, local: LocalEvent | null, ctx: C refuseTask(local?.shadow); const ref = refOf(local, event, ctx.jmapAccountId); const none = (stillDirty: boolean): DownloadPlan => ({ ops: { ref, ops: [] }, conflicts: 0, stillDirty, effect: 'none', writes: 0 }); + // A different object under a reused id: its rows are replaced, never patched. + const uidChanged = !!local?.shadow && typeof local.shadow.uid === 'string' && typeof event.uid === 'string' && local.shadow.uid !== event.uid; + + if (local?.deleted && local.split !== 'clone' && !uidChanged) { + // Deleted on the device: that wins over a server edit, so only the shadow follows, for the deletion. + // Also when no synced calendar holds the event any more: the deletion is then moot (see planUpload). + if (!local.shadow || deepEqual(local.shadow, event)) return none(true); + const group = new GroupBuilder(ref); + group.assert({ op: 'assert', table: 'events', id: local.eventId, values: { [Events.DELETED]: 1, [Events._SYNC_ID]: local.syncId }, expectCount: 1 }); + group.write({ op: 'update', table: 'events', id: local.eventId, values: { [Events.SYNC_DATA1]: JSON.stringify(event) }, expectCount: 1 }); + const ops = group.build(); + return { ops, conflicts: 0, stillDirty: true, effect: 'update', writes: writesOf(ops) }; + } + // An event another client moved out of every synced calendar still merges into its rows where they are // (they wait for an upload; the accepted write then removes them). const target = pickCalendar(event, local, ctx) ?? (local ? rowCalendar(local, ctx) : null); @@ -80,8 +94,6 @@ function planDownload(event: CalendarEventWire, local: LocalEvent | null, ctx: C // A split clone only looks like this object: it is a new event and uploads as one. The source row is the object's. if (local.split === 'clone') return none(true); - // A different object under a reused id: its rows are replaced, never patched. - const uidChanged = !!local.shadow && typeof local.shadow.uid === 'string' && typeof event.uid === 'string' && local.shadow.uid !== event.uid; if (uidChanged) { const group = new GroupBuilder(ref); for (const x of local.exceptions) deleteRow(group, x.eventId); @@ -98,16 +110,6 @@ function planDownload(event: CalendarEventWire, local: LocalEvent | null, ctx: C return { ops, conflicts: 0, stillDirty: false, effect: 'update', writes: writesOf(ops) }; } - if (local.deleted) { - // Deleted on the device: that wins over a server edit, so only the shadow follows, for the destroy. - if (!local.shadow || deepEqual(local.shadow, event)) return none(true); - const group = new GroupBuilder(ref); - group.assert({ op: 'assert', table: 'events', id: local.eventId, values: { [Events.DELETED]: 1, [Events._SYNC_ID]: local.syncId }, expectCount: 1 }); - group.write({ op: 'update', table: 'events', id: local.eventId, values: { [Events.SYNC_DATA1]: JSON.stringify(event) }, expectCount: 1 }); - const ops = group.build(); - return { ops, conflicts: 0, stillDirty: true, effect: 'update', writes: writesOf(ops) }; - } - const itemDirty = local.dirty || local.split === 'source' || local.exceptions.some((x) => x.dirty || x.deleted); const result = writeEvent(event, image, target, local, { mode: 'merge', @@ -266,6 +268,8 @@ function planUpload(local: LocalEvent, ctx: CalendarContext): Plan { return { kind: 'upload', actions: [{ kind: 'destroy', id: null, uid }] }; } if (!parseObjectRef(local.syncId)) return { kind: 'purge', ops: purgeGroup(local) }; + // Moved away on the server before the deletion reached it: nothing goes up, only the rows go. + if (movedAway(local, calendar, ctx)) return { kind: 'purge', ops: purgeGroup(local) }; if (ctx.isReadOnly(calendar.calendarId)) return { kind: 'revert', ops: purgeGroup(local), refetch: true, reason: SKIP.readOnly }; return { kind: 'upload', actions: deleteActions(local, ctx) }; } diff --git a/src/device-sync/contacts/upload.ts b/src/device-sync/contacts/upload.ts index 98845503..cceb0ed6 100644 --- a/src/device-sync/contacts/upload.ts +++ b/src/device-sync/contacts/upload.ts @@ -215,10 +215,24 @@ function readOnlyOnServer(local: LocalContact, ctx: Ctx): boolean { return !!local.shadow && ctx.isReadOnly(withoutMemberOf(local.shadow)); } +/** + * Whether the server moved the card out of every book its rows were written + * for (SYNC1), and into none this device syncs, before the device's deletion + * reached it: the deletion is moot then, and the card stays where it is now. + * A book deselected since still counts as the rows' own, so that deletion + * goes up; so does one of a card moved into another synced book. + */ +function movedAway(local: LocalContact, accountId: string, ctx: Ctx): boolean { + if (!local.shadow || !local.collections.length) return false; + const books = Object.entries(local.shadow.addressBookIds ?? {}).filter(([, on]) => on).map(([id]) => collectionKey(accountId, id)); + return !books.some((key) => local.collections.includes(key) || ctx.isSelected(key)); +} + function deletedPlan(local: LocalContact, ctx: Ctx): UploadPlan { const purge = (): OpGroup => group(local, [deleteWhereId('raw_contacts', local.rawContactId)]); const ref = parseObjectRef(local.sourceId); if (ref) { + if (movedAway(local, ref.accountId, ctx)) return { kind: 'purge', ops: purge() }; if (readOnlyOnServer(local, ctx)) return { kind: 'revert', ops: purge(), refetch: true, reason: 'readOnly' }; const books = Object.entries(local.shadow?.addressBookIds ?? {}).filter(([, on]) => on).map(([id]) => id); const selected = books.filter((id) => ctx.isSelected(collectionKey(ref.accountId, id))); diff --git a/src/device-sync/engine/item-sync.ts b/src/device-sync/engine/item-sync.ts index c747ddae..1f72ec28 100644 --- a/src/device-sync/engine/item-sync.ts +++ b/src/device-sync/engine/item-sync.ts @@ -659,7 +659,7 @@ export abstract class ItemSync { try { return this.downloadWork(kind, acct, object, found.local, written); } catch (error) { - // A planner that cannot place it (no synced calendar) leaves the deletion as it was planned. + // A planner that cannot take the server version leaves the deletion as it was planned. this.env.log(`deleted ${refOf(acct, object.id)} outside the selection keeps its shadow`, error); return null; } From ae679a7d3d67ff5a6d047bcf8a3cde8981293ee9 Mon Sep 17 00:00:00 2001 From: Linus Rath <139418639+rathlinus@users.noreply.github.com> Date: Sun, 27 Sep 2026 10:24:06 +0200 Subject: [PATCH 82/87] docs: list contacts and calendar sync with the phone as done on Android #34 --- PARITY_CHECKLIST.md | 2 +- README.md | 3 ++- docs/audit-2026-09.md | 6 +++--- docs/parity/06-contacts.md | 2 +- 4 files changed, 7 insertions(+), 6 deletions(-) diff --git a/PARITY_CHECKLIST.md b/PARITY_CHECKLIST.md index c6542d1e..9166f36e 100644 --- a/PARITY_CHECKLIST.md +++ b/PARITY_CHECKLIST.md @@ -153,7 +153,7 @@ Open split them by tick; the P columns count the priority tags on those items | #47 Account security not accessible | [01](docs/parity/01-auth-accounts.md) capability map bug (P1) | | #46 email preview shows css | [03](docs/parity/03-email-viewer.md) HTML-only body detection (P1) | | #45 error when re-registering to relay | [08](docs/parity/08-settings-push-i18n-ui.md) push section | -| #34 Native contacts/calendar sync adapters | feature request, out of parity scope; noted in [06](docs/parity/06-contacts.md) | +| #34 Native contacts/calendar sync adapters | done on Android: Settings → Contacts / Calendar → "Sync to this device" syncs both ways with the device's Contacts and Calendar apps ([design](docs/device-sync.md)); iOS has no API for it | | #5 Sort ascending/descending | in progress in the working tree; ordering presets in [02](docs/parity/02-mail-list-folders.md) | | #3 TLS client auth | Android-only today; gaps in [01](docs/parity/01-auth-accounts.md) and [09](docs/parity/09-jmap-core-sync-security.md) | | #1 Shared settings between webmail and native | [08](docs/parity/08-settings-push-i18n-ui.md) — webmail sync is server-side/cookie-bound; a JMAP-blob design is sketched there | diff --git a/README.md b/README.md index 87861346..6ca38d01 100644 --- a/README.md +++ b/README.md @@ -22,11 +22,12 @@ React Native (Expo SDK 54) client for [Bulwark Webmail](https://github.com/bulwa - Calendar (basic) - Contacts (basic) - Push notifications via the Bulwark relay - FCM by default, or [UnifiedPush](https://unifiedpush.org) (e.g. ntfy) for devices without Google Play services +- Android: contacts and calendars sync both ways with the phone's Contacts and Calendar apps (Settings → Contacts / Calendar → Sync to this device) - In-app sideload updates from GitHub Releases ## What's missing or rough -- iOS builds, but push notifications and client certificates are Android-only so far +- iOS builds, but push notifications, client certificates and syncing with the phone's contacts and calendars are Android-only so far - S/MIME, plugins, themes - UI stubs only (filters & rules, the vacation responder and file storage are real implementations) - Calendar editing is partial; contacts editing is basic - No Play Store distribution yet (sideload APK from Releases); iOS ships via TestFlight diff --git a/docs/audit-2026-09.md b/docs/audit-2026-09.md index 0cfcdc48..2c0ccc98 100644 --- a/docs/audit-2026-09.md +++ b/docs/audit-2026-09.md @@ -1020,7 +1020,8 @@ ones moved to the N/A lists, and the wrong text corrected. | #46 email preview shows CSS | Not reproduced: HTML-only mail with `