From 5a097e7739fc600b5c90c87e0e5b349d0b8dbac4 Mon Sep 17 00:00:00 2001 From: Jinwoo Hong <73622457+Jinwoo-H@users.noreply.github.com> Date: Thu, 22 Oct 2026 20:55:22 -0400 Subject: [PATCH] feat(mobile): decode page-bundle download windows natively (#31817) * feat(mobile): decode page-bundle download windows natively Each 384 KiB window of the page-bundle download cost about 200-300 ms of Hermes JS time (tweetnacl secretbox open, fflate gunzip, a base64 re-encode check and the TextDecoder polyfill), which kept the JS thread busy for the whole download. The phone now opens v2 frames with react-native-libsodium (secretbox open and strict base64) and decodes ranges with a local Expo module, orca-bundle-window-codec (bounded gunzip via system zlib on iOS and java.util.zip.Inflater on Android, plus strict UTF-8). The shared v2 session takes these as optional primitives; the desktop keeps tweetnacl. If any native piece is missing or fails a known-answer probe, the phone keeps today's JS path, with the base64 re-encode replaced by an O(1) canonical check. * refactor(mobile): name the native decode for every inbound frame, not just bundle windows The native secretbox, base64 and UTF-8 primitives decode every inbound v2 frame on the phone, so the files and the Expo module are renamed for what they cover (paired-e2ee-native-*, orca-gzip-utf8-codec). A throw from the native UTF-8 decode now falls back to TextDecoder, since the frame is already open and its counter advanced. * test(mobile): pin the TextDecoder fallback when native UTF-8 throws * fix(mobile): hand the native decode to the transport from the app entry The transport modules are reached by shared, desktop and e2e code. Importing the native bindings there pulled expo-modules-core into the e2e typecheck, whose global types make NODE_ENV required and broke ~20 e2e files. The app entry now provides the bindings; without them the transport runs the JS path. * fix(mobile): a native-codec lookup before the app provides bindings does not pin the JS path --- mobile/index.js | 5 + .../orca-gzip-utf8-codec/android/.gitignore | 1 + .../orca-gzip-utf8-codec/android/build.gradle | 21 ++ .../android/src/main/AndroidManifest.xml | 1 + .../orcagziputf8codec/BoundedGunzip.kt | 111 +++++++ .../OrcaGzipUtf8CodecModule.kt | 20 ++ .../modules/orcagziputf8codec/StrictUtf8.kt | 30 ++ .../orcagziputf8codec/BoundedGunzipTest.kt | 30 ++ .../orcagziputf8codec/StrictUtf8Test.kt | 29 ++ .../expo-module.config.json | 9 + .../ios/BoundedGunzip.swift | 57 ++++ .../ios/OrcaGzipUtf8Codec.podspec | 16 + .../ios/OrcaGzipUtf8CodecModule.swift | 17 + .../orca-gzip-utf8-codec/ios/StrictUtf8.swift | 11 + .../modules/orca-gzip-utf8-codec/package.json | 5 + .../modules/orca-gzip-utf8-codec/src/index.ts | 16 + .../tests/GzipUtf8CodecChecks.swift | 54 ++++ .../tests/gunzip-vectors.json | 121 +++++++ mobile/package.json | 1 + mobile/pnpm-lock.yaml | 42 +++ mobile/pnpm-workspace.yaml | 2 + mobile/src/transport/base64-byte-codec.ts | 29 ++ .../mobile-paired-e2ee-primitives.ts | 18 +- .../mobile-web-bundle-range-decode.test.ts | 22 ++ .../mobile-web-bundle-range-decode.ts | 9 +- .../mobile-web-bundle-window-reader.ts | 16 +- .../transport/paired-e2ee-native-bindings.ts | 46 +++ .../paired-e2ee-native-codec.test.ts | 302 ++++++++++++++++++ .../src/transport/paired-e2ee-native-codec.ts | 141 ++++++++ src/shared/mobile-e2ee-v2-framing.ts | 12 +- src/shared/paired-e2ee-primitives.ts | 8 + src/shared/paired-e2ee-v2-client-session.ts | 13 +- 32 files changed, 1205 insertions(+), 10 deletions(-) create mode 100644 mobile/modules/orca-gzip-utf8-codec/android/.gitignore create mode 100644 mobile/modules/orca-gzip-utf8-codec/android/build.gradle create mode 100644 mobile/modules/orca-gzip-utf8-codec/android/src/main/AndroidManifest.xml create mode 100644 mobile/modules/orca-gzip-utf8-codec/android/src/main/java/expo/modules/orcagziputf8codec/BoundedGunzip.kt create mode 100644 mobile/modules/orca-gzip-utf8-codec/android/src/main/java/expo/modules/orcagziputf8codec/OrcaGzipUtf8CodecModule.kt create mode 100644 mobile/modules/orca-gzip-utf8-codec/android/src/main/java/expo/modules/orcagziputf8codec/StrictUtf8.kt create mode 100644 mobile/modules/orca-gzip-utf8-codec/android/src/test/java/expo/modules/orcagziputf8codec/BoundedGunzipTest.kt create mode 100644 mobile/modules/orca-gzip-utf8-codec/android/src/test/java/expo/modules/orcagziputf8codec/StrictUtf8Test.kt create mode 100644 mobile/modules/orca-gzip-utf8-codec/expo-module.config.json create mode 100644 mobile/modules/orca-gzip-utf8-codec/ios/BoundedGunzip.swift create mode 100644 mobile/modules/orca-gzip-utf8-codec/ios/OrcaGzipUtf8Codec.podspec create mode 100644 mobile/modules/orca-gzip-utf8-codec/ios/OrcaGzipUtf8CodecModule.swift create mode 100644 mobile/modules/orca-gzip-utf8-codec/ios/StrictUtf8.swift create mode 100644 mobile/modules/orca-gzip-utf8-codec/package.json create mode 100644 mobile/modules/orca-gzip-utf8-codec/src/index.ts create mode 100644 mobile/modules/orca-gzip-utf8-codec/tests/GzipUtf8CodecChecks.swift create mode 100644 mobile/modules/orca-gzip-utf8-codec/tests/gunzip-vectors.json create mode 100644 mobile/src/transport/paired-e2ee-native-bindings.ts create mode 100644 mobile/src/transport/paired-e2ee-native-codec.test.ts create mode 100644 mobile/src/transport/paired-e2ee-native-codec.ts diff --git a/mobile/index.js b/mobile/index.js index 489f950c650..86c95f2717f 100644 --- a/mobile/index.js +++ b/mobile/index.js @@ -1,3 +1,8 @@ // Headless notification launches do not mount the router layout. import './src/notifications/push-background-dismissal' +import { loadPairedE2EENativeBindings } from './src/transport/paired-e2ee-native-bindings' +import { providePairedE2EENativeBindings } from './src/transport/paired-e2ee-native-codec' import 'expo-router/entry' + +// Why here: only the app entry may load native modules; tests and desktop code reach the transport. +providePairedE2EENativeBindings(loadPairedE2EENativeBindings) diff --git a/mobile/modules/orca-gzip-utf8-codec/android/.gitignore b/mobile/modules/orca-gzip-utf8-codec/android/.gitignore new file mode 100644 index 00000000000..84c048a73cc --- /dev/null +++ b/mobile/modules/orca-gzip-utf8-codec/android/.gitignore @@ -0,0 +1 @@ +/build/ diff --git a/mobile/modules/orca-gzip-utf8-codec/android/build.gradle b/mobile/modules/orca-gzip-utf8-codec/android/build.gradle new file mode 100644 index 00000000000..35667d48346 --- /dev/null +++ b/mobile/modules/orca-gzip-utf8-codec/android/build.gradle @@ -0,0 +1,21 @@ +apply plugin: 'com.android.library' + +group = 'expo.modules.orcagziputf8codec' +version = '0.0.1' + +def expoModulesCorePlugin = new File(project(':expo-modules-core').projectDir.absolutePath, 'ExpoModulesCorePlugin.gradle') +apply from: expoModulesCorePlugin +applyKotlinExpoModulesCorePlugin() +useCoreDependencies() +useExpoPublishing() +useDefaultAndroidSdkVersions() + +android { + namespace 'expo.modules.orcagziputf8codec' +} + +dependencies { + testImplementation 'junit:junit:4.13.2' + // The android.jar used by JVM unit tests stubs org.json; the shared vectors are JSON. + testImplementation 'org.json:json:20240303' +} diff --git a/mobile/modules/orca-gzip-utf8-codec/android/src/main/AndroidManifest.xml b/mobile/modules/orca-gzip-utf8-codec/android/src/main/AndroidManifest.xml new file mode 100644 index 00000000000..94cbbcfc396 --- /dev/null +++ b/mobile/modules/orca-gzip-utf8-codec/android/src/main/AndroidManifest.xml @@ -0,0 +1 @@ + diff --git a/mobile/modules/orca-gzip-utf8-codec/android/src/main/java/expo/modules/orcagziputf8codec/BoundedGunzip.kt b/mobile/modules/orca-gzip-utf8-codec/android/src/main/java/expo/modules/orcagziputf8codec/BoundedGunzip.kt new file mode 100644 index 00000000000..676d52f35a0 --- /dev/null +++ b/mobile/modules/orca-gzip-utf8-codec/android/src/main/java/expo/modules/orcagziputf8codec/BoundedGunzip.kt @@ -0,0 +1,111 @@ +package expo.modules.orcagziputf8codec + +import java.util.zip.CRC32 +import java.util.zip.DataFormatException +import java.util.zip.Inflater + +class BoundedGunzipException(message: String) : Exception(message) + +private const val FLAG_HCRC = 0x02 +private const val FLAG_EXTRA = 0x04 +private const val FLAG_NAME = 0x08 +private const val FLAG_COMMENT = 0x10 +private const val FLAG_RESERVED = 0xe0 +private const val TRAILER_LENGTH = 8 + +/** + * Inflates exactly one gzip member, as zlib's gzip mode does on iOS: the header, CRC32 and ISIZE + * are checked, and any byte after the member (trailing data or a second member) is refused. + * + * Never produces more than [outputLimit] bytes. A body that reaches the limit comes back at the + * limit with the rest of the stream unread: the caller sets the limit one past anything it accepts. + */ +fun gunzipBounded(input: ByteArray, outputLimit: Int): ByteArray { + if (outputLimit < 0) { + throw BoundedGunzipException("negative output limit") + } + val bodyStart = gzipBodyStart(input) + val output = ByteArray(outputLimit) + val inflater = Inflater(true) + try { + inflater.setInput(input, bodyStart, input.size - bodyStart) + var produced = 0 + while (!inflater.finished()) { + if (produced == outputLimit) { + return output + } + val count = try { + inflater.inflate(output, produced, outputLimit - produced) + } catch (error: DataFormatException) { + throw BoundedGunzipException(error.message ?: "invalid deflate data") + } + if (count == 0 && inflater.needsInput()) { + throw BoundedGunzipException("truncated deflate data") + } + produced += count + } + val trailerStart = input.size - inflater.remaining + if (inflater.remaining < TRAILER_LENGTH) { + throw BoundedGunzipException("truncated gzip trailer") + } + if (inflater.remaining > TRAILER_LENGTH) { + throw BoundedGunzipException("data after the gzip member") + } + val crc = CRC32().apply { update(output, 0, produced) }.value + if (readUint32(input, trailerStart) != crc) { + throw BoundedGunzipException("gzip CRC mismatch") + } + if (readUint32(input, trailerStart + 4) != produced.toLong()) { + throw BoundedGunzipException("gzip length mismatch") + } + return if (produced == outputLimit) output else output.copyOf(produced) + } finally { + inflater.end() + } +} + +/** RFC 1952 header, checked as strictly as zlib checks it: reserved flags and FHCRC included. */ +private fun gzipBodyStart(input: ByteArray): Int { + fun byteAt(index: Int): Int { + if (index >= input.size) { + throw BoundedGunzipException("truncated gzip header") + } + return input[index].toInt() and 0xff + } + if (byteAt(0) != 0x1f || byteAt(1) != 0x8b) { + throw BoundedGunzipException("not gzip") + } + if (byteAt(2) != 8) { + throw BoundedGunzipException("unknown gzip compression method") + } + val flags = byteAt(3) + if (flags and FLAG_RESERVED != 0) { + throw BoundedGunzipException("reserved gzip flags set") + } + // MTIME, XFL and OS are not read, but a header without them is truncated. + byteAt(9) + var cursor = 10 + if (flags and FLAG_EXTRA != 0) { + cursor += 2 + (byteAt(cursor) or (byteAt(cursor + 1) shl 8)) + } + if (flags and FLAG_NAME != 0) { + while (byteAt(cursor++) != 0) {} + } + if (flags and FLAG_COMMENT != 0) { + while (byteAt(cursor++) != 0) {} + } + if (flags and FLAG_HCRC != 0) { + val expected = byteAt(cursor) or (byteAt(cursor + 1) shl 8) + val crc = CRC32().apply { update(input, 0, cursor) }.value and 0xffff + if (crc != expected.toLong()) { + throw BoundedGunzipException("gzip header CRC mismatch") + } + cursor += 2 + } + return cursor +} + +private fun readUint32(input: ByteArray, offset: Int): Long = + (0 until 4).fold(0L) { value, index -> + value or ((input[offset + index].toLong() and 0xff) shl (8 * index)) + } diff --git a/mobile/modules/orca-gzip-utf8-codec/android/src/main/java/expo/modules/orcagziputf8codec/OrcaGzipUtf8CodecModule.kt b/mobile/modules/orca-gzip-utf8-codec/android/src/main/java/expo/modules/orcagziputf8codec/OrcaGzipUtf8CodecModule.kt new file mode 100644 index 00000000000..3cc999f7f16 --- /dev/null +++ b/mobile/modules/orca-gzip-utf8-codec/android/src/main/java/expo/modules/orcagziputf8codec/OrcaGzipUtf8CodecModule.kt @@ -0,0 +1,20 @@ +package expo.modules.orcagziputf8codec + +import expo.modules.kotlin.modules.Module +import expo.modules.kotlin.modules.ModuleDefinition + +// Synchronous on purpose: each call is a few ms per window, and only a sync call reads its +// Uint8Array argument without a thread hop. +class OrcaGzipUtf8CodecModule : Module() { + override fun definition() = ModuleDefinition { + Name("OrcaGzipUtf8Codec") + + Function("gunzip") { input: ByteArray, outputLimit: Int -> + gunzipBounded(input, outputLimit) + } + + Function("decodeUtf8") { input: ByteArray -> + decodeStrictUtf8(input) + } + } +} diff --git a/mobile/modules/orca-gzip-utf8-codec/android/src/main/java/expo/modules/orcagziputf8codec/StrictUtf8.kt b/mobile/modules/orca-gzip-utf8-codec/android/src/main/java/expo/modules/orcagziputf8codec/StrictUtf8.kt new file mode 100644 index 00000000000..f7cf5510744 --- /dev/null +++ b/mobile/modules/orca-gzip-utf8-codec/android/src/main/java/expo/modules/orcagziputf8codec/StrictUtf8.kt @@ -0,0 +1,30 @@ +package expo.modules.orcagziputf8codec + +import java.nio.ByteBuffer +import java.nio.charset.CharacterCodingException +import java.nio.charset.CodingErrorAction +import java.nio.charset.StandardCharsets + +/** + * Well-formed UTF-8 with no leading BOM, decoded; null otherwise. Those are the only inputs where + * every decoder agrees, so the caller hands anything else to the JS TextDecoder for its + * replacement and BOM rules. + */ +fun decodeStrictUtf8(input: ByteArray): String? { + if (input.size >= 3 && + input[0] == 0xef.toByte() && + input[1] == 0xbb.toByte() && + input[2] == 0xbf.toByte() + ) { + return null + } + return try { + StandardCharsets.UTF_8.newDecoder() + .onMalformedInput(CodingErrorAction.REPORT) + .onUnmappableCharacter(CodingErrorAction.REPORT) + .decode(ByteBuffer.wrap(input)) + .toString() + } catch (_: CharacterCodingException) { + null + } +} diff --git a/mobile/modules/orca-gzip-utf8-codec/android/src/test/java/expo/modules/orcagziputf8codec/BoundedGunzipTest.kt b/mobile/modules/orca-gzip-utf8-codec/android/src/test/java/expo/modules/orcagziputf8codec/BoundedGunzipTest.kt new file mode 100644 index 00000000000..a55caf16d6f --- /dev/null +++ b/mobile/modules/orca-gzip-utf8-codec/android/src/test/java/expo/modules/orcagziputf8codec/BoundedGunzipTest.kt @@ -0,0 +1,30 @@ +package expo.modules.orcagziputf8codec + +import java.io.File +import org.json.JSONArray +import org.junit.Assert.assertEquals +import org.junit.Test + +class BoundedGunzipTest { + // Shared with the Swift checks and the JS tests, which pin fflate's answer to the same inputs. + private val vectors = JSONArray(File("../tests/gunzip-vectors.json").readText()) + + @Test + fun `answers every shared vector as zlib's gzip mode does`() { + for (index in 0 until vectors.length()) { + val vector = vectors.getJSONObject(index) + val expected = if (vector.isNull("native")) null else vector.getString("native") + val actual = try { + gunzipBounded(hexBytes(vector.getString("gzipHex")), vector.getInt("outputLimit")).toHex() + } catch (_: BoundedGunzipException) { + null + } + assertEquals(vector.getString("name"), expected, actual) + } + } + + private fun hexBytes(hex: String) = + ByteArray(hex.length / 2) { index -> hex.substring(index * 2, index * 2 + 2).toInt(16).toByte() } + + private fun ByteArray.toHex() = joinToString("") { "%02x".format(it) } +} diff --git a/mobile/modules/orca-gzip-utf8-codec/android/src/test/java/expo/modules/orcagziputf8codec/StrictUtf8Test.kt b/mobile/modules/orca-gzip-utf8-codec/android/src/test/java/expo/modules/orcagziputf8codec/StrictUtf8Test.kt new file mode 100644 index 00000000000..f6c1162898d --- /dev/null +++ b/mobile/modules/orca-gzip-utf8-codec/android/src/test/java/expo/modules/orcagziputf8codec/StrictUtf8Test.kt @@ -0,0 +1,29 @@ +package expo.modules.orcagziputf8codec + +import org.junit.Assert.assertEquals +import org.junit.Assert.assertNull +import org.junit.Test + +class StrictUtf8Test { + @Test + fun `decodes well-formed UTF-8 of every sequence length`() { + val text = "{\"a\":\"xé✓🙂\"}" + assertEquals(text, decodeStrictUtf8(text.toByteArray(Charsets.UTF_8))) + assertEquals("", decodeStrictUtf8(ByteArray(0))) + } + + @Test + fun `hands back to JS what TextDecoder would repair or strip`() { + val refused = listOf( + byteArrayOf(0xef.toByte(), 0xbb.toByte(), 0xbf.toByte(), 0x41), + byteArrayOf(0xc0.toByte(), 0x80.toByte()), + byteArrayOf(0xed.toByte(), 0xa0.toByte(), 0x80.toByte()), + byteArrayOf(0x41, 0xe2.toByte(), 0x82.toByte()), + byteArrayOf(0x80.toByte()), + byteArrayOf(0xf5.toByte(), 0x80.toByte(), 0x80.toByte(), 0x80.toByte()) + ) + for (bytes in refused) { + assertNull(decodeStrictUtf8(bytes)) + } + } +} diff --git a/mobile/modules/orca-gzip-utf8-codec/expo-module.config.json b/mobile/modules/orca-gzip-utf8-codec/expo-module.config.json new file mode 100644 index 00000000000..3ef9f6ada38 --- /dev/null +++ b/mobile/modules/orca-gzip-utf8-codec/expo-module.config.json @@ -0,0 +1,9 @@ +{ + "platforms": ["ios", "android"], + "ios": { + "modules": ["OrcaGzipUtf8CodecModule"] + }, + "android": { + "modules": ["expo.modules.orcagziputf8codec.OrcaGzipUtf8CodecModule"] + } +} diff --git a/mobile/modules/orca-gzip-utf8-codec/ios/BoundedGunzip.swift b/mobile/modules/orca-gzip-utf8-codec/ios/BoundedGunzip.swift new file mode 100644 index 00000000000..cd72613fef2 --- /dev/null +++ b/mobile/modules/orca-gzip-utf8-codec/ios/BoundedGunzip.swift @@ -0,0 +1,57 @@ +import Foundation +import zlib + +struct BoundedGunzipError: Error, CustomStringConvertible { + let description: String +} + +/// Inflates exactly one gzip member with zlib's gzip mode, which checks the header, CRC32 and ISIZE. +/// Any byte after the member (trailing data or a second member) is refused. +/// +/// Never produces more than `outputLimit` bytes. A body that reaches the limit comes back at the +/// limit with the rest of the stream unread: the caller sets the limit one past anything it accepts. +func gunzipBounded(_ input: Data, outputLimit: Int) throws -> Data { + guard outputLimit >= 0 else { + throw BoundedGunzipError(description: "negative output limit") + } + var output = Data(count: outputLimit) + if outputLimit == 0 { + return output + } + var stream = z_stream() + // 16 + MAX_WBITS: a gzip wrapper only, never zlib or raw deflate. + guard inflateInit2_(&stream, 16 + MAX_WBITS, ZLIB_VERSION, Int32(MemoryLayout.size)) == Z_OK + else { + throw BoundedGunzipError(description: "inflateInit2 failed") + } + defer { inflateEnd(&stream) } + let produced = try input.withUnsafeBytes { (inBytes: UnsafeRawBufferPointer) -> Int in + try output.withUnsafeMutableBytes { (outBytes: UnsafeMutableRawBufferPointer) -> Int in + stream.next_in = UnsafeMutablePointer( + mutating: inBytes.bindMemory(to: Bytef.self).baseAddress) + stream.avail_in = uInt(inBytes.count) + stream.next_out = outBytes.bindMemory(to: Bytef.self).baseAddress + stream.avail_out = uInt(outBytes.count) + while true { + let status = inflate(&stream, Z_NO_FLUSH) + if status == Z_STREAM_END { + if stream.avail_in != 0 { + throw BoundedGunzipError(description: "data after the gzip member") + } + return outputLimit - Int(stream.avail_out) + } + if stream.avail_out == 0 && (status == Z_OK || status == Z_BUF_ERROR) { + return outputLimit + } + if status == Z_BUF_ERROR { + throw BoundedGunzipError(description: "truncated gzip stream") + } + if status != Z_OK { + let message = stream.msg.map { String(cString: $0) } ?? "inflate failed (\(status))" + throw BoundedGunzipError(description: message) + } + } + } + } + return produced == outputLimit ? output : output.prefix(produced) +} diff --git a/mobile/modules/orca-gzip-utf8-codec/ios/OrcaGzipUtf8Codec.podspec b/mobile/modules/orca-gzip-utf8-codec/ios/OrcaGzipUtf8Codec.podspec new file mode 100644 index 00000000000..747a17b46d5 --- /dev/null +++ b/mobile/modules/orca-gzip-utf8-codec/ios/OrcaGzipUtf8Codec.podspec @@ -0,0 +1,16 @@ +Pod::Spec.new do |s| + s.name = 'OrcaGzipUtf8Codec' + s.version = '0.0.1' + s.summary = 'Bounded gunzip and strict UTF-8 decode' + s.description = s.summary + s.license = { :type => 'MIT' } + s.author = 'Orca' + s.homepage = 'https://onorca.dev' + s.source = { :git => 'https://github.com/stablyai/orca.git' } + s.platforms = { :ios => '15.1' } + s.swift_version = '5.9' + s.static_framework = true + s.dependency 'ExpoModulesCore' + s.libraries = 'z' + s.source_files = '**/*.swift' +end diff --git a/mobile/modules/orca-gzip-utf8-codec/ios/OrcaGzipUtf8CodecModule.swift b/mobile/modules/orca-gzip-utf8-codec/ios/OrcaGzipUtf8CodecModule.swift new file mode 100644 index 00000000000..fc506534971 --- /dev/null +++ b/mobile/modules/orca-gzip-utf8-codec/ios/OrcaGzipUtf8CodecModule.swift @@ -0,0 +1,17 @@ +import ExpoModulesCore + +// Synchronous on purpose: each call is a few ms per window, and only a sync call reads its +// Uint8Array argument without a thread hop. +public class OrcaGzipUtf8CodecModule: Module { + public func definition() -> ModuleDefinition { + Name("OrcaGzipUtf8Codec") + + Function("gunzip") { (input: Data, outputLimit: Int) throws -> Data in + try gunzipBounded(input, outputLimit: outputLimit) + } + + Function("decodeUtf8") { (input: Data) -> String? in + decodeStrictUtf8(input) + } + } +} diff --git a/mobile/modules/orca-gzip-utf8-codec/ios/StrictUtf8.swift b/mobile/modules/orca-gzip-utf8-codec/ios/StrictUtf8.swift new file mode 100644 index 00000000000..58293b4bfea --- /dev/null +++ b/mobile/modules/orca-gzip-utf8-codec/ios/StrictUtf8.swift @@ -0,0 +1,11 @@ +import Foundation + +/// Well-formed UTF-8 with no leading BOM, decoded; nil otherwise. Those are the only inputs where +/// every decoder agrees, so the caller hands anything else to the JS TextDecoder for its +/// replacement and BOM rules. +func decodeStrictUtf8(_ input: Data) -> String? { + if input.starts(with: [0xef, 0xbb, 0xbf]) { + return nil + } + return String(data: input, encoding: .utf8) +} diff --git a/mobile/modules/orca-gzip-utf8-codec/package.json b/mobile/modules/orca-gzip-utf8-codec/package.json new file mode 100644 index 00000000000..1c4402de8ea --- /dev/null +++ b/mobile/modules/orca-gzip-utf8-codec/package.json @@ -0,0 +1,5 @@ +{ + "name": "orca-gzip-utf8-codec", + "version": "0.0.1", + "private": true +} diff --git a/mobile/modules/orca-gzip-utf8-codec/src/index.ts b/mobile/modules/orca-gzip-utf8-codec/src/index.ts new file mode 100644 index 00000000000..6af07847c07 --- /dev/null +++ b/mobile/modules/orca-gzip-utf8-codec/src/index.ts @@ -0,0 +1,16 @@ +import { requireOptionalNativeModule } from 'expo-modules-core' + +export type OrcaGzipUtf8CodecModule = { + /** + * Inflates exactly one gzip member, checking its header, CRC32 and length, and refusing any byte + * after it. Never produces more than `outputLimit` bytes: a body that reaches the limit comes + * back at the limit with the rest unread. Throws on anything else. + */ + gunzip(input: Uint8Array, outputLimit: number): Uint8Array + /** Well-formed UTF-8 with no leading BOM, decoded; null otherwise. */ + decodeUtf8(input: Uint8Array): string | null +} + +/** Null in a binary built before the module existed. */ +export const OrcaGzipUtf8Codec = + requireOptionalNativeModule('OrcaGzipUtf8Codec') diff --git a/mobile/modules/orca-gzip-utf8-codec/tests/GzipUtf8CodecChecks.swift b/mobile/modules/orca-gzip-utf8-codec/tests/GzipUtf8CodecChecks.swift new file mode 100644 index 00000000000..53263b7f34c --- /dev/null +++ b/mobile/modules/orca-gzip-utf8-codec/tests/GzipUtf8CodecChecks.swift @@ -0,0 +1,54 @@ +import Foundation + +// The codec's decisions, compiled and run without a device, from the module directory: +// +// swiftc -O -o /tmp/gzip-utf8-codec-checks ios/BoundedGunzip.swift \ +// ios/StrictUtf8.swift tests/GzipUtf8CodecChecks.swift && /tmp/gzip-utf8-codec-checks +@main struct GzipUtf8CodecChecks { + static var failures = 0 + + static func check(_ condition: Bool, _ name: String) { + if !condition { + failures += 1 + print("FAIL: \(name)") + } + } + + static func bytes(hex: String) -> Data { + var data = Data(capacity: hex.count / 2) + var index = hex.startIndex + while index < hex.endIndex { + let next = hex.index(index, offsetBy: 2) + data.append(UInt8(hex[index..= 0.8.0'} + libsodium-sumo@0.8.4: + resolution: {integrity: sha512-TMtHShQfVVsaxDygyapvUC3o7YsPgXa/hRWeIgzyFz6w5k/1hirGptCxp1U7XwW3rCskaTTYKgV10v86UiGgNw==} + + libsodium-wrappers-sumo@0.8.4: + resolution: {integrity: sha512-ql7hcgulKZ3ekfa2DGAogcCKsWU0diA/0nArz1CFzh93WQdb46/Kj18ka/Hifq6uA3Ush34Pc6vU/6HXeRwUkg==} + + libsodium-wrappers@0.8.4: + resolution: {integrity: sha512-mu8aAWucZjTB5O/BtGXtW4e1agy7uHxNYG7zPthmmD1jU43LCDmSWZLN4JhflbdPXj3yDO4lxM1O9hLDgIOXDw==} + + libsodium@0.8.4: + resolution: {integrity: sha512-lMcYaRi0zcs7tarATsQUYC7rstliIXZuoq0c6zXSgNtSNtdvBgkSegjWhpMJAXzKX3SUSwIp7+zEsob+j3LuRw==} + lighthouse-logger@1.4.2: resolution: {integrity: sha512-gPWxznF6TKmUHrOQjlVo2UbaL2EJ71mb2CCeRs/2qBpi4L/g4LUVc9+3lKQ6DTUZwJswfM7ainGrLO1+fOqa2g==} @@ -6181,6 +6196,13 @@ packages: react: '*' react-native: '*' + react-native-libsodium@1.7.0: + resolution: {integrity: sha512-0I2FcCeyR2HUiUOIvoukteZFMY2WIhPCQ6OOOYy33nQiyGFpwW6I4ky/f18SdSN84n/GLvgC9DOF0fJaKTYYJg==} + engines: {node: '>= 20.20.0'} + peerDependencies: + react: '*' + react-native: '*' + react-native-reanimated@4.3.4: resolution: {integrity: sha512-kQJMiDROed3eO+asdXfodDxgsDDIGM/ifnHFAiPsKWxmrcPu+Bz4vO0fg79eyuEmKCeQuDczmn4tkTb4RbSVYA==} peerDependencies: @@ -13338,6 +13360,18 @@ snapshots: prelude-ls: 1.2.1 type-check: 0.4.0 + libsodium-sumo@0.8.4: {} + + libsodium-wrappers-sumo@0.8.4: + dependencies: + libsodium-sumo: 0.8.4 + + libsodium-wrappers@0.8.4: + dependencies: + libsodium: 0.8.4 + + libsodium@0.8.4: {} + lighthouse-logger@1.4.2(supports-color@8.1.1): dependencies: debug: 2.6.9(supports-color@8.1.1) @@ -14373,6 +14407,14 @@ snapshots: react: 19.2.8 react-native: 0.83.10(patch_hash=22bc31f0a057b1f96d9949b38a7ccde81225f617c0f243820dc8a455c91debfb)(@babel/core@7.29.7(supports-color@8.1.1))(@react-native/metro-config@0.85.2(@babel/core@7.29.7(supports-color@8.1.1))(supports-color@8.1.1))(@types/react@19.2.18)(react@19.2.8)(supports-color@8.1.1) + react-native-libsodium@1.7.0(react-native@0.83.10(patch_hash=22bc31f0a057b1f96d9949b38a7ccde81225f617c0f243820dc8a455c91debfb)(@babel/core@7.29.7(supports-color@8.1.1))(@react-native/metro-config@0.85.2(@babel/core@7.29.7(supports-color@8.1.1))(supports-color@8.1.1))(@types/react@19.2.18)(react@19.2.8)(supports-color@8.1.1))(react@19.2.8): + dependencies: + '@noble/hashes': 1.8.0 + libsodium-wrappers: 0.8.4 + libsodium-wrappers-sumo: 0.8.4 + react: 19.2.8 + react-native: 0.83.10(patch_hash=22bc31f0a057b1f96d9949b38a7ccde81225f617c0f243820dc8a455c91debfb)(@babel/core@7.29.7(supports-color@8.1.1))(@react-native/metro-config@0.85.2(@babel/core@7.29.7(supports-color@8.1.1))(supports-color@8.1.1))(@types/react@19.2.18)(react@19.2.8)(supports-color@8.1.1) + react-native-reanimated@4.3.4(react-native-worklets@0.8.3(@babel/core@7.29.7(supports-color@8.1.1))(@react-native/metro-config@0.85.2(@babel/core@7.29.7(supports-color@8.1.1))(supports-color@8.1.1))(react-native@0.83.10(patch_hash=22bc31f0a057b1f96d9949b38a7ccde81225f617c0f243820dc8a455c91debfb)(@babel/core@7.29.7(supports-color@8.1.1))(@react-native/metro-config@0.85.2(@babel/core@7.29.7(supports-color@8.1.1))(supports-color@8.1.1))(@types/react@19.2.18)(react@19.2.8)(supports-color@8.1.1))(react@19.2.8)(supports-color@8.1.1))(react-native@0.83.10(patch_hash=22bc31f0a057b1f96d9949b38a7ccde81225f617c0f243820dc8a455c91debfb)(@babel/core@7.29.7(supports-color@8.1.1))(@react-native/metro-config@0.85.2(@babel/core@7.29.7(supports-color@8.1.1))(supports-color@8.1.1))(@types/react@19.2.18)(react@19.2.8)(supports-color@8.1.1))(react@19.2.8): dependencies: react: 19.2.8 diff --git a/mobile/pnpm-workspace.yaml b/mobile/pnpm-workspace.yaml index fb9815de283..52b3791aa9a 100644 --- a/mobile/pnpm-workspace.yaml +++ b/mobile/pnpm-workspace.yaml @@ -1,5 +1,7 @@ allowBuilds: esbuild: true + # Its postinstall unpacks the prebuilt libsodium the native builds link. + react-native-libsodium: true minimumReleaseAge: 4320 # Latest requested releases are still within the three-day delay; exempt only these versions. diff --git a/mobile/src/transport/base64-byte-codec.ts b/mobile/src/transport/base64-byte-codec.ts index 9958e0cf8ea..ad7a006162a 100644 --- a/mobile/src/transport/base64-byte-codec.ts +++ b/mobile/src/transport/base64-byte-codec.ts @@ -28,3 +28,32 @@ export function decodeBase64Bytes(value: string): Uint8Array { } return bytes } + +const BASE64_ALPHABET = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/' + +/** + * Null for anything `btoa` would not have produced, which `atob` forgives: missing padding, + * whitespace and stray bits in the last character. Checked in O(1) after the decode instead of + * re-encoding, which costs ~20 ms per 200 KB frame on Hermes. + */ +export function decodeCanonicalBase64Bytes(value: string): Uint8Array | null { + let bytes: Uint8Array + try { + bytes = decodeBase64Bytes(value) + } catch { + return null + } + if (value.length % 4 !== 0) { + return null + } + const padding = value.endsWith('==') ? 2 : value.endsWith('=') ? 1 : 0 + // Why: the length equation rules out whitespace, which atob strips. + if (bytes.length !== (value.length / 4) * 3 - padding) { + return null + } + if (padding === 0) { + return bytes + } + const lastSextet = BASE64_ALPHABET.indexOf(value[value.length - padding - 1]!) + return (lastSextet & (padding === 2 ? 0x0f : 0x03)) === 0 ? bytes : null +} diff --git a/mobile/src/transport/mobile-paired-e2ee-primitives.ts b/mobile/src/transport/mobile-paired-e2ee-primitives.ts index 822bd4da9b0..d48e5f4061a 100644 --- a/mobile/src/transport/mobile-paired-e2ee-primitives.ts +++ b/mobile/src/transport/mobile-paired-e2ee-primitives.ts @@ -1,6 +1,12 @@ +import nacl from 'tweetnacl' import type { PairedE2EEPrimitives } from '../../../src/shared/paired-e2ee-primitives' import { derivePairedE2EEV2KeySchedule } from '../../../src/shared/paired-e2ee-v2-key-schedule' -import { decodeBase64Bytes, encodeBase64Bytes } from './base64-byte-codec' +import { + decodeBase64Bytes, + decodeCanonicalBase64Bytes, + encodeBase64Bytes +} from './base64-byte-codec' +import { pairedE2EENativeCodec } from './paired-e2ee-native-codec' import * as e2ee from './e2ee' // Why arrows: resolved per call, so transport tests that mock ./e2ee partially still load. @@ -17,5 +23,15 @@ export const MOBILE_PAIRED_E2EE_PRIMITIVES: PairedE2EEPrimitives = { openLegacyText: (frame, sharedKey) => e2ee.decrypt(frame, sharedKey), sealLegacyBinary: (plaintext, sharedKey) => e2ee.encryptBytes(plaintext, sharedKey), openLegacyBinary: (frame, sharedKey) => e2ee.decryptBytes(frame, sharedKey), + openSecretbox: (box, nonce, key) => { + const native = pairedE2EENativeCodec() + return native ? native.openSecretbox(box, nonce, key) : nacl.secretbox.open(box, nonce, key) + }, + decodeCanonicalBase64: (value) => { + const native = pairedE2EENativeCodec() + return native ? native.decodeBase64(value) : decodeCanonicalBase64Bytes(value) + }, + decodeUtf8: (bytes) => + pairedE2EENativeCodec()?.decodeUtf8(bytes) ?? new TextDecoder().decode(bytes), deriveV2KeySchedule: derivePairedE2EEV2KeySchedule } diff --git a/mobile/src/transport/mobile-web-bundle-range-decode.test.ts b/mobile/src/transport/mobile-web-bundle-range-decode.test.ts index 729474ef1d1..59680a265e0 100644 --- a/mobile/src/transport/mobile-web-bundle-range-decode.test.ts +++ b/mobile/src/transport/mobile-web-bundle-range-decode.test.ts @@ -1,5 +1,6 @@ import { gzipSync } from 'fflate' import { describe, expect, it } from 'vitest' +import gunzipVectors from '../../modules/orca-gzip-utf8-codec/tests/gunzip-vectors.json' import { MobileWebBundleFetchError } from './mobile-web-bundle-fetch-refusal' import { decodeMobileWebBundleRange } from './mobile-web-bundle-range-decode' @@ -62,4 +63,25 @@ describe('decoding a bundle range', () => { RAW.byteLength ) }) + + // The vectors the native codec's Kotlin and Swift tests run. Where their `native` answer is a + // refusal and `js` is not (bad CRC or length, trailing bytes), this pins fflate's leniency on + // the fallback path. + it('answers the shared gunzip vectors as fflate does', () => { + for (const vector of gunzipVectors) { + let actual: string | null + try { + actual = Buffer.from( + decodeMobileWebBundleRange( + RANGE, + Buffer.from(vector.gzipHex, 'hex'), + vector.outputLimit - 1 + ) + ).toString('hex') + } catch { + actual = null + } + expect(actual, vector.name).toBe(vector.js) + } + }) }) diff --git a/mobile/src/transport/mobile-web-bundle-range-decode.ts b/mobile/src/transport/mobile-web-bundle-range-decode.ts index b36eca0f524..6efc7761a1e 100644 --- a/mobile/src/transport/mobile-web-bundle-range-decode.ts +++ b/mobile/src/transport/mobile-web-bundle-range-decode.ts @@ -1,4 +1,5 @@ import { gunzipSync } from 'fflate' +import { pairedE2EENativeCodec } from './paired-e2ee-native-codec' import { MobileWebBundleFetchError } from './mobile-web-bundle-fetch-refusal' /** @@ -7,6 +8,9 @@ import { MobileWebBundleFetchError } from './mobile-web-bundle-fetch-refusal' * Inflated into a buffer one byte past the window: fflate fills a supplied `out` and never grows it, * so a gzip bomb is bounded in memory, not in the time spent inflating. A wrong length is left to the * fetch's slot checks, where a filled spare byte reads as overlong. + * + * The native codec is stricter than fflate: it also refuses a bad CRC or length, and bytes after + * the member. No host sends those, since every host gzips with node:zlib. */ export function decodeMobileWebBundleRange( range: { readonly path: string; readonly offset: number; readonly encoding: string }, @@ -23,7 +27,10 @@ export function decodeMobileWebBundleRange( ) } try { - return gunzipSync(wire, { out: new Uint8Array(expectedLength + 1) }) + const native = pairedE2EENativeCodec() + return native + ? native.gunzip(wire, expectedLength + 1) + : gunzipSync(wire, { out: new Uint8Array(expectedLength + 1) }) } catch (error) { throw new MobileWebBundleFetchError( 'range-undecodable', diff --git a/mobile/src/transport/mobile-web-bundle-window-reader.ts b/mobile/src/transport/mobile-web-bundle-window-reader.ts index 7f1a6cbdd12..fe2620d4ad9 100644 --- a/mobile/src/transport/mobile-web-bundle-window-reader.ts +++ b/mobile/src/transport/mobile-web-bundle-window-reader.ts @@ -1,3 +1,5 @@ +import { decodeBase64Bytes } from './base64-byte-codec' +import { pairedE2EENativeCodec } from './paired-e2ee-native-codec' import { mobileWebBundleChunkRead, mobileWebBundleRangeRead } from './mobile-web-bundle-operations' import { decodeMobileWebBundleRange } from './mobile-web-bundle-range-decode' import type { MobileWebBundleManifestReply } from './mobile-web-bundle-reply-schemas' @@ -78,12 +80,16 @@ export function mobileWebBundleWindowReader( } } -/** Metro ships no Buffer; `atob` is the decoder the pairing and E2EE paths already run on Hermes. */ +/** Every host encodes with `Buffer#toString('base64')`, so the strict native decoder refuses nothing a + * host sends; `atob` is the fallback the pairing and E2EE paths already run on Hermes. */ function decodeBase64(value: string): Uint8Array { - const binary = atob(value) - const bytes = new Uint8Array(binary.length) - for (let index = 0; index < binary.length; index += 1) { - bytes[index] = binary.charCodeAt(index) + const native = pairedE2EENativeCodec() + if (!native) { + return decodeBase64Bytes(value) + } + const bytes = native.decodeBase64(value) + if (!bytes) { + throw new Error('bundle window is not canonical base64') } return bytes } diff --git a/mobile/src/transport/paired-e2ee-native-bindings.ts b/mobile/src/transport/paired-e2ee-native-bindings.ts new file mode 100644 index 00000000000..ddc73bb53e1 --- /dev/null +++ b/mobile/src/transport/paired-e2ee-native-bindings.ts @@ -0,0 +1,46 @@ +import { NativeModules } from 'react-native' +import { OrcaGzipUtf8Codec } from '../../modules/orca-gzip-utf8-codec/src' +import type { LibsodiumBindings, PairedE2EENativeBindings } from './paired-e2ee-native-codec' + +declare global { + // Installed by react-native-libsodium's native install; unknown until checked. + var jsi_from_base64_to_arraybuffer: unknown + var jsi_crypto_secretbox_open_easy: unknown +} + +/** + * Null when either native piece is missing from this binary. Only the app entry imports this file: + * shared, desktop and e2e code reach the transport modules and must not load native-module types. + */ +export function loadPairedE2EENativeBindings(): PairedE2EENativeBindings | null { + const sodium = installLibsodium() + return sodium && OrcaGzipUtf8Codec ? { codec: OrcaGzipUtf8Codec, sodium } : null +} + +/** + * react-native-libsodium's JS entry installs its JSI functions on import, logs as it does, and + * wraps every call; only its native install and two of the functions are needed here. + */ +function installLibsodium(): LibsodiumBindings | null { + const libsodium: { install?: unknown } | undefined = NativeModules.Libsodium + if (typeof libsodium?.install !== 'function') { + return null + } + libsodium.install() + const fromBase64 = globalThis.jsi_from_base64_to_arraybuffer + const secretboxOpen = globalThis.jsi_crypto_secretbox_open_easy + if (typeof fromBase64 !== 'function' || typeof secretboxOpen !== 'function') { + return null + } + return { + fromBase64: (value, variant) => arrayBufferOf(fromBase64(value, variant)), + secretboxOpen: (box, nonce, key) => arrayBufferOf(secretboxOpen(box, nonce, key)) + } +} + +function arrayBufferOf(value: unknown): ArrayBuffer { + if (!(value instanceof ArrayBuffer)) { + throw new TypeError('libsodium returned something other than an ArrayBuffer') + } + return value +} diff --git a/mobile/src/transport/paired-e2ee-native-codec.test.ts b/mobile/src/transport/paired-e2ee-native-codec.test.ts new file mode 100644 index 00000000000..f8b21b662fd --- /dev/null +++ b/mobile/src/transport/paired-e2ee-native-codec.test.ts @@ -0,0 +1,302 @@ +import { gunzipSync } from 'node:zlib' +import sodium from 'react-native-libsodium' +import nacl from 'tweetnacl' +import { beforeAll, describe, expect, it, vi } from 'vitest' +import { + encodeMobileE2EEV2Transcript, + validateMobileE2EEV2Handshake, + type MobileE2EEV2Ready +} from '../../../src/shared/mobile-e2ee-v2-contract' +import { sealMobileE2EEV2Frame } from '../../../src/shared/mobile-e2ee-v2-framing' +import type { PairedE2EEPrimitives } from '../../../src/shared/paired-e2ee-primitives' +import { PairedE2EEV2ClientSession } from '../../../src/shared/paired-e2ee-v2-client-session' +import { derivePairedE2EEV2KeySchedule } from '../../../src/shared/paired-e2ee-v2-key-schedule' +import type { PairedE2EENativeBindings } from './paired-e2ee-native-codec' + +// Under Node, react-native-libsodium resolves to its web build: the same libsodium C, as wasm. +// The codec module's half is a stand-in here; its own Kotlin and Swift tests pin it. +const secretboxOpenCalls = vi.fn() +let bindings: PairedE2EENativeBindings | null = null + +function libsodiumBindings(): PairedE2EENativeBindings { + return { + sodium: { + fromBase64: (value, variant) => sodium.from_base64(value, variant).slice().buffer, + secretboxOpen: (box, nonce, key) => { + secretboxOpenCalls(box.byteLength) + return sodium + .crypto_secretbox_open_easy( + new Uint8Array(box), + new Uint8Array(nonce), + new Uint8Array(key) + ) + .slice().buffer + } + }, + codec: { + gunzip: (input, outputLimit) => new Uint8Array(gunzipSync(input)).slice(0, outputLimit), + decodeUtf8: (input) => { + if (input[0] === 0xef && input[1] === 0xbb && input[2] === 0xbf) { + return null + } + try { + return new TextDecoder('utf-8', { fatal: true }).decode(input) + } catch { + return null + } + } + } + } +} + +/** A fresh module graph, so the once-only detection runs again with or without native. */ +async function load(withNative: boolean) { + bindings = withNative ? libsodiumBindings() : null + vi.resetModules() + const codec = await import('./paired-e2ee-native-codec') + if (withNative) { + codec.providePairedE2EENativeBindings(() => bindings) + } + const primitives = await import('./mobile-paired-e2ee-primitives') + return { + primitives: primitives.MOBILE_PAIRED_E2EE_PRIMITIVES, + native: codec.pairedE2EENativeCodec() + } +} + +const desktop = nacl.box.keyPair.fromSecretKey(new Uint8Array(32).fill(1)) +const client = nacl.box.keyPair.fromSecretKey(new Uint8Array(32).fill(2)) +const b64 = (bytes: Uint8Array) => Buffer.from(bytes).toString('base64') + +function pairedSession(primitives: PairedE2EEPrimitives) { + const session = PairedE2EEV2ClientSession.create({ + primitives, + desktopPublicKeyB64: b64(desktop.publicKey), + transport: 'relay', + relayHostId: 'AbCdEf0123_-xyZ9', + clientNonce: new Uint8Array(32).fill(3), + clientKeyPair: client + }) + const ready: MobileE2EEV2Ready = { + type: 'e2ee_ready', + v: 2, + desktopPublicKeyB64: b64(desktop.publicKey), + clientNonceB64: session.hello.clientNonceB64, + desktopNonceB64: b64(new Uint8Array(32).fill(4)), + selection: { framing: 2, payloadKinds: ['text', 'binary'] }, + context: session.hello.context + } + expect(session.acceptReady(ready)).toBe(true) + const handshake = validateMobileE2EEV2Handshake(session.hello, ready)! + const schedule = derivePairedE2EEV2KeySchedule({ + sharedSecret: nacl.box.before(client.publicKey, desktop.secretKey), + transcript: encodeMobileE2EEV2Transcript(handshake), + clientNonce: handshake.clientNonce, + desktopNonce: handshake.desktopNonce + }) + return { session, schedule } +} + +let seed = 7 +function random(below: number): number { + seed = (seed * 1103515245 + 12345) % 2 ** 31 + // Why the shift: an LCG's low bits cycle with short periods. + return Math.floor(seed / 256) % below +} +function randomBytes(length: number): Uint8Array { + return Uint8Array.from({ length }, () => random(256)) +} + +const PAYLOADS = [ + '{"id":1,"result":{"ok":true}}', + '', + '\uFEFF{"bom":"stripped by TextDecoder"}', + 'é✓🙂 μ', + JSON.stringify({ dataBase64: b64(randomBytes(150_000)) }) +] + +/** Every desktop frame for counters 0..n, each preceded by the ways a frame can be wrong. */ +function inboundWire(schedule: ReturnType) { + const wire: { frame: string; plaintext: string | null }[] = [] + const seal = (payload: string, counter: bigint, key = schedule.desktopToMobileKey) => + sealMobileE2EEV2Frame({ + payload: new TextEncoder().encode(payload), + key, + sessionId: schedule.sessionId, + direction: 'desktop-to-mobile', + payloadKind: 'text', + counter + }) + const flipped = (frame: Uint8Array, index: number) => { + const copy = frame.slice() + copy[index]! ^= 1 << random(8) + return copy + } + PAYLOADS.forEach((payload, index) => { + const counter = BigInt(index) + const good = seal(payload, counter) + const goodB64 = b64(good) + const bad = [ + flipped(good, random(24)), + flipped(good, 24 + random(16)), + flipped(good, 40 + random(good.length - 40)), + good.subarray(0, -1), + good.subarray(0, 81), + good.subarray(0, 30), + seal(payload, counter, randomBytes(32)), + seal(payload, counter + 1n) + ].map(b64) + const nonCanonical = [ + goodB64.replace(/=+$/, ''), + `${goodB64.slice(0, 8)}\n${goodB64.slice(8)}`, + `${goodB64} `, + goodB64.replaceAll('+', '-').replaceAll('/', '_'), + `${goodB64}=` + ].filter((variant) => variant !== goodB64) + for (const frame of [...bad, ...nonCanonical]) { + wire.push({ frame, plaintext: null }) + } + wire.push({ + frame: goodB64, + plaintext: new TextDecoder().decode(new TextEncoder().encode(payload)) + }) + }) + return wire +} + +describe('the native paired E2EE codec', () => { + beforeAll(async () => { + await sodium.ready + }) + + it('opens and refuses exactly the frames the JS paths do, never advancing on a refusal', async () => { + const { primitives: nativePrimitives, native } = await load(true) + const { primitives: fallbackPrimitives, native: absent } = await load(false) + expect(native).not.toBeNull() + expect(absent).toBeNull() + // What the desktop's own client session runs: tweetnacl, re-encode, TextDecoder. + const originalPrimitives: PairedE2EEPrimitives = { + ...fallbackPrimitives, + openSecretbox: undefined, + decodeCanonicalBase64: undefined, + decodeUtf8: undefined + } + const sessions = [nativePrimitives, fallbackPrimitives, originalPrimitives].map((primitives) => + pairedSession(primitives) + ) + const wire = inboundWire(sessions[0]!.schedule) + expect(wire.filter((entry) => entry.plaintext === null).length).toBeGreaterThan(50) + for (const { session } of sessions) { + expect(wire.map(({ frame }) => session.openText(frame))).toEqual( + wire.map(({ plaintext }) => plaintext) + ) + } + }) + + it('uses native once provided, even after a lookup with no provider', async () => { + const { native: before } = await load(false) + expect(before).toBeNull() + const codec = await import('./paired-e2ee-native-codec') + bindings = libsodiumBindings() + codec.providePairedE2EENativeBindings(() => bindings) + expect(codec.pairedE2EENativeCodec()).not.toBeNull() + }) + + it('agrees with tweetnacl on boxes either library sealed', async () => { + const { native } = await load(true) + for (let index = 0; index < 300; index++) { + const key = randomBytes(32) + const nonce = randomBytes(24) + const message = randomBytes(random(600)) + const boxes = [ + sodium.crypto_secretbox_easy(message, nonce, key), + nacl.secretbox(message, nonce, key) + ] + expect(boxes[0]).toEqual(boxes[1]) + for (const box of boxes) { + expect(native!.openSecretbox(box, nonce, key)).toEqual(message) + const tampered = box.slice() + tampered[random(box.length)]! ^= 1 << random(8) + expect(native!.openSecretbox(tampered, nonce, key)).toBeNull() + expect(nacl.secretbox.open(tampered, nonce, key)).toBeNull() + } + } + }) + + // The frame is open and the counter advanced before UTF-8, so a native throw must not drop it. + it('falls back to TextDecoder when native UTF-8 throws', async () => { + const { primitives } = await load(true) + bindings!.codec.decodeUtf8 = () => { + throw new Error('native UTF-8 failed') + } + const { session, schedule } = pairedSession(primitives) + const frames = ['{"a":1}', 'é✓🙂'].map((payload, counter) => + b64( + sealMobileE2EEV2Frame({ + payload: new TextEncoder().encode(payload), + key: schedule.desktopToMobileKey, + sessionId: schedule.sessionId, + direction: 'desktop-to-mobile', + payloadKind: 'text', + counter: BigInt(counter) + }) + ) + ) + expect(frames.map((frame) => session.openText(frame))).toEqual(['{"a":1}', 'é✓🙂']) + }) + + // libsodium sizes its output as the box length minus the MAC, which underflows and crashes. + it('never hands libsodium a box shorter than its MAC', async () => { + const { native } = await load(true) + secretboxOpenCalls.mockClear() + for (let length = 0; length < 16; length++) { + expect( + native!.openSecretbox(new Uint8Array(length), randomBytes(24), randomBytes(32)) + ).toBeNull() + } + expect(secretboxOpenCalls).not.toHaveBeenCalled() + }) + + it('accepts exactly the base64 that re-encodes to itself, on both paths', async () => { + const { native, primitives } = await load(true) + const { primitives: fallback } = await load(false) + const reencodes = (value: string) => { + try { + const bytes = Uint8Array.from(atob(value), (char) => char.charCodeAt(0)) + return b64(bytes) === value ? bytes : null + } catch { + return null + } + } + const alphabet = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/' + const noise = `${alphabet}=-_ \n\t.` + const cases = [ + '', + 'QR==', + 'QQ==', + 'QQ', + 'QQ=', + 'Q===', + '====', + 'QUJD', + 'QUJ D', + 'Q=Q=', + '-_8=', + 'AB+/' + ] + for (let index = 0; index < 5_000; index++) { + let value = b64(randomBytes(random(10))) + if (random(2) === 0) { + const at = random(value.length + 1) + value = value.slice(0, at) + noise[random(noise.length)] + value.slice(at + 1) + } + cases.push(value) + } + for (const value of cases) { + const expected = reencodes(value) + expect(native!.decodeBase64(value), value).toEqual(expected) + expect(primitives.decodeCanonicalBase64!(value), value).toEqual(expected) + expect(fallback.decodeCanonicalBase64!(value), value).toEqual(expected) + } + }) +}) diff --git a/mobile/src/transport/paired-e2ee-native-codec.ts b/mobile/src/transport/paired-e2ee-native-codec.ts new file mode 100644 index 00000000000..6406a91a64a --- /dev/null +++ b/mobile/src/transport/paired-e2ee-native-codec.ts @@ -0,0 +1,141 @@ +/** + * Inbound decode steps done natively instead of in Hermes JS: every v2 text frame the phone opens + * (base64, secretbox, UTF-8), plus the gunzip of page-bundle ranges. + */ +export type PairedE2EENativeCodec = { + /** Canonical padded base64 only; null for anything else, including what `atob` forgives. */ + decodeBase64(value: string): Uint8Array | null + /** XSalsa20-Poly1305 secretbox open; null for a box that fails authentication. */ + openSecretbox(box: Uint8Array, nonce: Uint8Array, key: Uint8Array): Uint8Array | null + /** Throws on a body that is not one well-formed gzip member. See orca-gzip-utf8-codec. */ + gunzip(input: Uint8Array, outputLimit: number): Uint8Array + /** Null where only TextDecoder's replacement or BOM rules give the answer, or native failed. */ + decodeUtf8(input: Uint8Array): string | null +} + +/** The two react-native-libsodium JSI functions the phone needs; both throw on failure. */ +export type LibsodiumBindings = { + fromBase64: (value: string, variant: number) => ArrayBuffer + secretboxOpen: (box: ArrayBuffer, nonce: ArrayBuffer, key: ArrayBuffer) => ArrayBuffer +} + +/** What the app entry hands over: libsodium plus the orca-gzip-utf8-codec module. */ +export type PairedE2EENativeBindings = { + codec: { + gunzip(input: Uint8Array, outputLimit: number): Uint8Array + decodeUtf8(input: Uint8Array): string | null + } + sodium: LibsodiumBindings +} + +// libsodium's sodium_base64_VARIANT_ORIGINAL: the standard alphabet, padded. +const SODIUM_BASE64_ORIGINAL = 1 +const SECRETBOX_MAC_BYTES = 16 + +// A box sealed by tweetnacl under key 0..31 and nonce 0..23, and a gzip member, both of "ok". +const PROBE_BOX_BASE64 = 'sqQBX9c2fQaZ4F6j8gHDpjGU' +const PROBE_GZIP_BASE64 = 'H4sIAAAAAAAAE8vPBgBH3dx5AgAAAA==' + +let loadBindings: (() => PairedE2EENativeBindings | null) | null = null +let detected: PairedE2EENativeCodec | null | undefined + +/** + * Called once by the app entry. Injected rather than imported so that this module, which shared, + * desktop and e2e code reach, never loads the native modules or their global types. + */ +export function providePairedE2EENativeBindings(load: () => PairedE2EENativeBindings | null): void { + loadBindings = load + // Why: a lookup that ran before this must not pin the JS path for the whole run. + detected = undefined +} + +/** + * The native codec, or null when none was provided or any part of it is missing or answers the + * probe wrongly; the caller then runs the JS path. Detected once, on first use. + */ +export function pairedE2EENativeCodec(): PairedE2EENativeCodec | null { + if (detected === undefined) { + detected = detectPairedE2EENativeCodec() + } + return detected +} + +function detectPairedE2EENativeCodec(): PairedE2EENativeCodec | null { + try { + const bindings = loadBindings?.() ?? null + if (!bindings) { + return null + } + const { codec: module, sodium } = bindings + const codec: PairedE2EENativeCodec = { + decodeBase64: (value) => { + // Why: libsodium sizes its output from the input, and an empty one has nothing to size. + if (value.length === 0) { + return new Uint8Array(0) + } + try { + return new Uint8Array(sodium.fromBase64(value, SODIUM_BASE64_ORIGINAL)) + } catch { + return null + } + }, + openSecretbox: (box, nonce, key) => { + // Why: libsodium's output length underflows below the MAC length and crashes the app. + if (box.length < SECRETBOX_MAC_BYTES) { + return null + } + try { + return new Uint8Array( + sodium.secretboxOpen(wholeBuffer(box), wholeBuffer(nonce), wholeBuffer(key)) + ) + } catch { + // libsodium throws where tweetnacl returns null: a forged or corrupted box. + return null + } + }, + gunzip: (input, outputLimit) => module.gunzip(input, outputLimit), + decodeUtf8: (input) => { + // Why: the frame is already open and the counter advanced, so a throw here must not drop it. + try { + return module.decodeUtf8(input) + } catch { + return null + } + } + } + return answersProbe(codec) ? codec : null + } catch { + return null + } +} + +function answersProbe(codec: PairedE2EENativeCodec): boolean { + const box = codec.decodeBase64(PROBE_BOX_BASE64) + const gzip = codec.decodeBase64(PROBE_GZIP_BASE64) + if (!box || !gzip) { + return false + } + const key = Uint8Array.from({ length: 32 }, (_, index) => index) + const nonce = Uint8Array.from({ length: 24 }, (_, index) => index) + const opened = codec.openSecretbox(box, nonce, key) + return ( + opened !== null && + codec.decodeUtf8(opened) === 'ok' && + codec.decodeUtf8(codec.gunzip(gzip, 3)) === 'ok' && + codec.decodeBase64('b2s=') !== null && + codec.decodeBase64('b2t=') === null + ) +} + +/** libsodium reads whole ArrayBuffers, so a view into a larger one is copied out first. */ +function wholeBuffer(view: Uint8Array): ArrayBuffer { + const { buffer } = view + if ( + buffer instanceof ArrayBuffer && + view.byteOffset === 0 && + view.byteLength === buffer.byteLength + ) { + return buffer + } + return view.slice().buffer +} diff --git a/src/shared/mobile-e2ee-v2-framing.ts b/src/shared/mobile-e2ee-v2-framing.ts index fae0733a06d..cf47ab3557c 100644 --- a/src/shared/mobile-e2ee-v2-framing.ts +++ b/src/shared/mobile-e2ee-v2-framing.ts @@ -3,6 +3,13 @@ import type { MobileE2EEPayloadKind } from './mobile-e2ee-v2-contract' export type MobileE2EEDirection = 'mobile-to-desktop' | 'desktop-to-mobile' +/** NaCl secretbox open: null for a box that fails authentication. */ +export type SecretboxOpen = ( + box: Uint8Array, + nonce: Uint8Array, + key: Uint8Array +) => Uint8Array | null + const NONCE_LENGTH = 24 const SESSION_ID_LENGTH = 32 const HEADER_LENGTH = SESSION_ID_LENGTH + 1 + 1 + 8 @@ -32,6 +39,7 @@ export function openMobileE2EEV2Frame(args: { direction: MobileE2EEDirection payloadKind: MobileE2EEPayloadKind expectedCounter: bigint + openSecretbox?: SecretboxOpen }): Uint8Array | null { validateFrameInputs(args.key, args.sessionId, args.expectedCounter) if (args.frame.length < NONCE_LENGTH + nacl.secretbox.overheadLength + HEADER_LENGTH) { @@ -48,7 +56,9 @@ export function openMobileE2EEV2Frame(args: { return null } - const plaintext = nacl.secretbox.open(args.frame.subarray(NONCE_LENGTH), nonce, args.key) + // The length check above must stay ahead of this: native libsodium crashes on a box shorter than its MAC. + const openSecretbox = args.openSecretbox ?? nacl.secretbox.open + const plaintext = openSecretbox(args.frame.subarray(NONCE_LENGTH), nonce, args.key) if (!plaintext) { return null } diff --git a/src/shared/paired-e2ee-primitives.ts b/src/shared/paired-e2ee-primitives.ts index 3636944083e..6e96a38dd13 100644 --- a/src/shared/paired-e2ee-primitives.ts +++ b/src/shared/paired-e2ee-primitives.ts @@ -1,3 +1,5 @@ +import type { SecretboxOpen } from './mobile-e2ee-v2-framing' + // Why: desktop/CLI (Node), web and mobile each keep their own NaCl box and // base64 adapters (size limits, Hermes typed-array quirks, no Buffer in a // browser), so the shared handshake core takes them as a parameter. @@ -23,6 +25,12 @@ export type PairedE2EEPrimitives = { openLegacyText: (frame: string, sharedKey: Uint8Array) => string | null sealLegacyBinary: (plaintext: Uint8Array, sharedKey: Uint8Array) => Uint8Array openLegacyBinary: (frame: Uint8Array, sharedKey: Uint8Array) => Uint8Array | null + // Why optional: only the phone swaps these, for native code on every inbound v2 frame, since + // Hermes JS is ~100x slower than V8. Absent, the session uses tweetnacl, re-encode and TextDecoder. + openSecretbox?: SecretboxOpen + // Null for anything but canonical padded base64. + decodeCanonicalBase64?: (value: string) => Uint8Array | null + decodeUtf8?: (bytes: Uint8Array) => string // Why injected: Node uses node:crypto, so the CLI's unbundled runtime needs no extra package. deriveV2KeySchedule: (args: { sharedSecret: Uint8Array diff --git a/src/shared/paired-e2ee-v2-client-session.ts b/src/shared/paired-e2ee-v2-client-session.ts index f872256c48f..d989dd39570 100644 --- a/src/shared/paired-e2ee-v2-client-session.ts +++ b/src/shared/paired-e2ee-v2-client-session.ts @@ -99,7 +99,12 @@ export class PairedE2EEV2ClientSession { return null } const plaintext = this.open(frame, 'text') - return plaintext ? new TextDecoder().decode(plaintext) : null + if (!plaintext) { + return null + } + return this.primitives.decodeUtf8 + ? this.primitives.decodeUtf8(plaintext) + : new TextDecoder().decode(plaintext) } openBinary(frame: Uint8Array): Uint8Array | null { @@ -124,7 +129,8 @@ export class PairedE2EEV2ClientSession { sessionId: this.schedule.sessionId, direction: 'desktop-to-mobile', payloadKind, - expectedCounter: this.inboundCounter + expectedCounter: this.inboundCounter, + openSecretbox: this.primitives.openSecretbox }) if (plaintext) { this.inboundCounter++ @@ -149,6 +155,9 @@ export class PairedE2EEV2ClientSession { } private decodeCanonicalBase64(value: string): Uint8Array | null { + if (this.primitives.decodeCanonicalBase64) { + return this.primitives.decodeCanonicalBase64(value) + } try { const bytes = this.primitives.decodeBase64(value) return this.primitives.encodeBase64(bytes) === value ? bytes : null