diff --git a/.gitignore b/.gitignore index 984fa217d6..4812025436 100644 --- a/.gitignore +++ b/.gitignore @@ -14,6 +14,10 @@ !/vendor/deno_v8-0.3.0/** !/vendor/xml5ever-0.39.0/ !/vendor/xml5ever-0.39.0/** +!/vendor/web-audio-api-1.7.0/ +!/vendor/web-audio-api-1.7.0/** +/vendor/web-audio-api-1.7.0/target/ +/vendor/web-audio-api-1.7.0/Cargo.lock /.moli __pycache__ *.pyc diff --git a/Cargo.lock b/Cargo.lock index 9571b102fe..051f301bd5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -38,6 +38,12 @@ version = "0.2.21" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923" +[[package]] +name = "almost" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3aa2999eb46af81abb65c2d30d446778d7e613b60bbf4e174a027e80f90a3c14" + [[package]] name = "anstream" version = "1.0.0" @@ -140,6 +146,15 @@ dependencies = [ "serde", ] +[[package]] +name = "arc-swap" +version = "1.9.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c049c0be4daef0b145cb3555416b3b8ef5b7888a38aea1a3a155801fe7b0810b" +dependencies = [ + "rustversion", +] + [[package]] name = "arrayref" version = "0.3.9" @@ -164,6 +179,43 @@ version = "0.1.13" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "41e67cd8309bbd06cd603a9e693a784ac2e5d1e955f11286e355089fcab3047c" +[[package]] +name = "audio-core" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f93ebbf82d06013f4c41fe71303feb980cddd78496d904d06be627972de51a24" + +[[package]] +name = "audioadapter" +version = "3.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91f87b70b051c5866680ad79f6743a42ccab264c009d1a71f4d33a3872ae60c8" +dependencies = [ + "audio-core", + "num-traits", +] + +[[package]] +name = "audioadapter-buffers" +version = "3.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9097d67933fb083d382ce980430afdb758aada60846010aee6be068c06cef0ca" +dependencies = [ + "audioadapter", + "audioadapter-sample", + "num-traits", +] + +[[package]] +name = "audioadapter-sample" +version = "3.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34ab94f2bc04a14e1f49ee5f222f66460e8a1b51627bdfedf34eed394d747938" +dependencies = [ + "audio-core", + "num-traits", +] + [[package]] name = "autocfg" version = "1.5.0" @@ -278,7 +330,7 @@ version = "0.70.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f49d8fed880d473ea71efb9bf597651e77201bdd4893efe54c9e5d65ae04ce6f" dependencies = [ - "bitflags", + "bitflags 2.11.0", "cexpr", "clang-sys", "itertools 0.13.0", @@ -298,7 +350,7 @@ version = "0.72.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "993776b509cfb49c750f11b8f07a46fa23e0a1386ffc01fb1e7d343efc387895" dependencies = [ - "bitflags", + "bitflags 2.11.0", "cexpr", "clang-sys", "itertools 0.13.0", @@ -312,6 +364,12 @@ dependencies = [ "syn 2.0.117", ] +[[package]] +name = "bitflags" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" + [[package]] name = "bitflags" version = "2.11.0" @@ -399,6 +457,12 @@ dependencies = [ "cc", ] +[[package]] +name = "byte-slice-cast" +version = "1.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7575182f7272186991736b70173b0ea045398f984bf5ebbb3804736ce1330c9d" + [[package]] name = "bytemuck" version = "1.25.2" @@ -740,6 +804,47 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "creek" +version = "1.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fb3106d374fa70e8a7a8cf3abf6afbb7f3016e923a1e8941a3eee2b3a21583ee" +dependencies = [ + "creek-core", + "creek-decode-symphonia", + "creek-encode-wav", +] + +[[package]] +name = "creek-core" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b8d299b2001167b9a51ac46458cbdcfafc42d6a59125a265d8c1de2e16b28a3" +dependencies = [ + "rtrb", +] + +[[package]] +name = "creek-decode-symphonia" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "416063cb8f815ee27036075e544a778770b577e82348f017d145a704df90f246" +dependencies = [ + "creek-core", + "log", + "symphonia 0.5.5", +] + +[[package]] +name = "creek-encode-wav" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37cb9ea02c12a68eb40a0576086394b3129b297970612592222284ff060bbf31" +dependencies = [ + "byte-slice-cast", + "creek-core", +] + [[package]] name = "crossbeam-channel" version = "0.5.15" @@ -970,6 +1075,12 @@ dependencies = [ "syn 2.0.117", ] +[[package]] +name = "dasp_sample" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c87e182de0887fd5361989c677c4e8f5000cd9491d6d563161a8f3a5519fc7f" + [[package]] name = "data-encoding" version = "2.10.0" @@ -1251,6 +1362,12 @@ dependencies = [ "serde", ] +[[package]] +name = "extended" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af9673d8203fcb076b19dfd17e38b3d4ae9f44959416ea532ce72415a6020365" + [[package]] name = "fastrand" version = "2.3.0" @@ -1282,6 +1399,17 @@ dependencies = [ "subtle", ] +[[package]] +name = "fft-convolver" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ecb4fbed063c755ecaa4cd1356cd82cc48e9d5c7edd5c30cd144656833fc0661" +dependencies = [ + "realfft", + "rtsan-standalone", + "thiserror", +] + [[package]] name = "find-msvc-tools" version = "0.1.9" @@ -1305,6 +1433,12 @@ version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "98de4bbd547a563b716d8dfa9aad1cb19bfab00f4fa09a6a4ed21dbcf44ce9c4" +[[package]] +name = "float_eq" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "28a80e3145d8ad11ba0995949bbcf48b9df2be62772b3d351ef017dff6ecb853" + [[package]] name = "fnv" version = "1.0.7" @@ -1417,6 +1551,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "07bbe89c50d7a535e539b8c17bc0b49bdb77747034daa8087407d655f3f7cc1d" dependencies = [ "futures-core", + "futures-sink", ] [[package]] @@ -1582,7 +1717,7 @@ version = "0.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c03d949a14aa089bbb282f7dd76a498a7f684428e4257202efc119ec010376f9" dependencies = [ - "bitflags", + "bitflags 2.11.0", "bytemuck", "read-fonts", "smallvec", @@ -1635,6 +1770,23 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "hound" +version = "3.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "62adaabb884c94955b19907d60019f4e145d091c75345379e70d1ee696f7854f" + +[[package]] +name = "hrtf" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1344e2123ddd443aceddc30d60a7fa78b9099121d57726e96611fb43f6d6b191" +dependencies = [ + "byteorder", + "rubato 3.0.0", + "rustfft", +] + [[package]] name = "html-escape" version = "0.2.13" @@ -2164,6 +2316,12 @@ version = "0.4.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d26c52dbd32dccf2d10cac7725f8eae5296885fb5703b261f7d0a0739ec807ab" +[[package]] +name = "linux-raw-sys" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53" + [[package]] name = "litemap" version = "0.8.1" @@ -2176,6 +2334,12 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "11d3d7f243d5c5a8b9bb5d6dd2b1602c0cb0b9db1621bafc7ed66e35ff9fe092" +[[package]] +name = "llq" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22edcb39a6fc7e511be7ba578a719af09125391ecde44e0bd61ad3c8e003a4a3" + [[package]] name = "lock_api" version = "0.4.14" @@ -2696,7 +2860,7 @@ dependencies = [ name = "moli-page-types" version = "0.1.0" dependencies = [ - "bitflags", + "bitflags 2.11.0", "http-auth", "moli-cookie-jar", "moli-dom", @@ -2936,6 +3100,7 @@ dependencies = [ "percent-encoding", "png", "precomputed-hash", + "realfft", "selectors", "serde", "serde_json", @@ -2952,6 +3117,7 @@ dependencies = [ "uuid", "v8", "wasm_dep_analyzer", + "web-audio-api", "widestring", "wuff", "xml5ever", @@ -3302,6 +3468,12 @@ version = "1.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "650eef8c711430f1a879fdd01d4745a7deea475becfb90269c06775983bbf086" +[[package]] +name = "no_denormals" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6bcfe410abc339c9f8c226aceebf946bf26e3e2eca738be3fec9e9ee97d54aa" + [[package]] name = "nom" version = "7.1.3" @@ -3337,6 +3509,15 @@ dependencies = [ "num-traits", ] +[[package]] +name = "num-complex" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73f88a1307638156682bada9d7604135552957b7818057dcef22705b4d509495" +dependencies = [ + "num-traits", +] + [[package]] name = "num-conv" version = "0.2.2" @@ -3406,7 +3587,7 @@ version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2a180dd8642fa45cdb7dd721cd4c11b1cadd4929ce112ebd8b9f5803cc79d536" dependencies = [ - "bitflags", + "bitflags 2.11.0", ] [[package]] @@ -3415,7 +3596,7 @@ version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0cde0dfb48d25d2b4862161a4d5fcc0e3c24367869ad306b0c9ec0073bfed92d" dependencies = [ - "bitflags", + "bitflags 2.11.0", "objc2-core-foundation", ] @@ -3431,7 +3612,7 @@ version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e3e0adef53c21f888deb4fa59fc59f7eb17404926ee8a6f59f5df0fd7f9f3272" dependencies = [ - "bitflags", + "bitflags 2.11.0", "objc2", ] @@ -3453,7 +3634,7 @@ version = "0.10.80" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a45fa2aa886c42762255da344f0a0d313e254066c46aad76f300c3d3da62d967" dependencies = [ - "bitflags", + "bitflags 2.11.0", "cfg-if", "foreign-types", "libc", @@ -3541,7 +3722,7 @@ version = "0.127.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "381ae8356082431bd7e217dd78c7179bfc379dbbe7a32494e28be4fc678812c7" dependencies = [ - "bitflags", + "bitflags 2.11.0", "oxc_allocator", "oxc_ast_macros", "oxc_data_structures", @@ -3632,7 +3813,7 @@ version = "0.127.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c71acdb67749ff68bfbbd346da7dd2fe4947964be49ac9ec34d73d10a2396dcd" dependencies = [ - "bitflags", + "bitflags 2.11.0", "cow-utils", "memchr", "num-bigint", @@ -3656,7 +3837,7 @@ version = "0.127.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "08a1273168ec6d8083e161565d264847249b9aad51c430d92d344303ede058b2" dependencies = [ - "bitflags", + "bitflags 2.11.0", "oxc_allocator", "oxc_ast_macros", "oxc_diagnostics", @@ -3699,7 +3880,7 @@ version = "0.127.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b448a086623714675f66b79271e25fa2b51708255fa6af7dad83be88cc6e8726" dependencies = [ - "bitflags", + "bitflags 2.11.0", "cow-utils", "dragonbox_ecma", "nonmax", @@ -3875,6 +4056,12 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" +[[package]] +name = "piston-float" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ad78bf43dcf80e8f950c92b84f938a0fc7590b7f6866fbcbeca781609c115590" + [[package]] name = "pkcs8" version = "0.11.0" @@ -3897,7 +4084,7 @@ version = "0.18.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "60769b8b31b2a9f263dae2776c37b1b28ae246943cf719eb6946a1db05128a61" dependencies = [ - "bitflags", + "bitflags 2.11.0", "crc32fast", "fdeflate", "flate2", @@ -3955,6 +4142,15 @@ dependencies = [ "syn 2.0.117", ] +[[package]] +name = "primal-check" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc0d895b311e3af9902528fbb8f928688abbd95872819320517cc24ca6b2bd08" +dependencies = [ + "num-integer", +] + [[package]] name = "proc-macro2" version = "1.0.106" @@ -4082,13 +4278,22 @@ dependencies = [ "once_cell", ] +[[package]] +name = "realfft" +version = "3.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f821338fddb99d089116342c46e9f1fbf3828dba077674613e734e01d6ea8677" +dependencies = [ + "rustfft", +] + [[package]] name = "redox_syscall" version = "0.5.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d" dependencies = [ - "bitflags", + "bitflags 2.11.0", ] [[package]] @@ -4114,6 +4319,12 @@ dependencies = [ "regex-syntax", ] +[[package]] +name = "regex-lite" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cab834c73d247e67f4fae452806d17d3c7501756d98c8808d7c9c7aa7d18f973" + [[package]] name = "regex-syntax" version = "0.8.10" @@ -4159,6 +4370,70 @@ dependencies = [ "memchr", ] +[[package]] +name = "rtrb" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fae8ee26b0371a29a77d2b2d6b3ae13aa81def6f9bf1b1b92a32d279a5e709b7" + +[[package]] +name = "rtsan-standalone" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fd1b6d61d69481a68e555916d3a52213846f5c2d2140bdc74ebc308e2338fc3" +dependencies = [ + "rtsan-standalone-macros", + "rtsan-standalone-sys", +] + +[[package]] +name = "rtsan-standalone-macros" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a8320af894782374141c8e0d4521ee613a8f24d7ab08b6ad359881311e37b9ef" +dependencies = [ + "quote", + "syn 2.0.117", +] + +[[package]] +name = "rtsan-standalone-sys" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "362a9c531f731e574a870cdb28ee7d81178ba7a87ed54380eb9b8d8f0c3b5301" +dependencies = [ + "num_cpus", + "tempfile", +] + +[[package]] +name = "rubato" +version = "0.16.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5258099699851cfd0082aeb645feb9c084d9a5e1f1b8d5372086b989fc5e56a1" +dependencies = [ + "num-complex", + "num-integer", + "num-traits", + "realfft", +] + +[[package]] +name = "rubato" +version = "3.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d4be9c88e3d722d3d36939e41941f6b6f52810c1235bf19998a7d4535b402892" +dependencies = [ + "audioadapter", + "audioadapter-buffers", + "num-complex", + "num-integer", + "num-traits", + "realfft", + "visibility", + "windowfunctions", +] + [[package]] name = "rustc-hash" version = "1.1.0" @@ -4180,19 +4455,46 @@ dependencies = [ "semver", ] +[[package]] +name = "rustfft" +version = "6.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "21db5f9893e91f41798c88680037dba611ca6674703c1a18601b01a72c8adb89" +dependencies = [ + "num-complex", + "num-integer", + "num-traits", + "primal-check", + "strength_reduce", + "transpose", +] + [[package]] name = "rustix" version = "0.38.44" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fdb5bc1ae2baa591800df16c9ca78619bf65c0488b41b96ccec5d11220d8c154" dependencies = [ - "bitflags", + "bitflags 2.11.0", "errno", "libc", - "linux-raw-sys", + "linux-raw-sys 0.4.15", "windows-sys 0.59.0", ] +[[package]] +name = "rustix" +version = "1.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6fe4565b9518b83ef4f91bb47ce29620ca828bd32cb7e408f0062e9930ba190" +dependencies = [ + "bitflags 2.11.0", + "errno", + "libc", + "linux-raw-sys 0.12.1", + "windows-sys 0.61.2", +] + [[package]] name = "rustls" version = "0.23.38" @@ -4299,7 +4601,7 @@ name = "selectors" version = "0.40.0" source = "git+https://github.com/ldm0/stylo?rev=671d13d31b3b00e7b1e0088ba2b36a4e35c0858c#671d13d31b3b00e7b1e0088ba2b36a4e35c0858c" dependencies = [ - "bitflags", + "bitflags 2.11.0", "cssparser", "derive_more", "log", @@ -4603,6 +4905,12 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "strength_reduce" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fe895eb47f22e2ddd4dabc02bce419d2e643c8e3b585c78158b349195bc24d82" + [[package]] name = "strict-num" version = "0.1.1" @@ -4681,7 +4989,7 @@ dependencies = [ "app_units", "arrayvec", "atomic_refcell", - "bitflags", + "bitflags 2.11.0", "byteorder", "cssparser", "derive_more", @@ -4755,7 +5063,7 @@ name = "stylo_dom" version = "0.20.0" source = "git+https://github.com/ldm0/stylo?rev=671d13d31b3b00e7b1e0088ba2b36a4e35c0858c#671d13d31b3b00e7b1e0088ba2b36a4e35c0858c" dependencies = [ - "bitflags", + "bitflags 2.11.0", "stylo_malloc_size_of", ] @@ -4800,7 +5108,7 @@ version = "0.20.0" source = "git+https://github.com/ldm0/stylo?rev=671d13d31b3b00e7b1e0088ba2b36a4e35c0858c#671d13d31b3b00e7b1e0088ba2b36a4e35c0858c" dependencies = [ "app_units", - "bitflags", + "bitflags 2.11.0", "cssparser", "euclid", "malloc_size_of_derive", @@ -4831,6 +5139,387 @@ dependencies = [ "siphasher", ] +[[package]] +name = "symphonia" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5773a4c030a19d9bfaa090f49746ff35c75dfddfa700df7a5939d5e076a57039" +dependencies = [ + "lazy_static", + "symphonia-bundle-flac 0.5.5", + "symphonia-bundle-mp3 0.5.5", + "symphonia-codec-aac 0.5.5", + "symphonia-codec-adpcm 0.5.5", + "symphonia-codec-alac 0.5.5", + "symphonia-codec-pcm 0.5.5", + "symphonia-codec-vorbis 0.5.5", + "symphonia-core 0.5.5", + "symphonia-format-isomp4 0.5.5", + "symphonia-format-mkv 0.5.5", + "symphonia-format-ogg 0.5.5", + "symphonia-format-riff 0.5.5", + "symphonia-metadata 0.5.5", +] + +[[package]] +name = "symphonia" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7edef6a96b696d4e0cab5ee9ebb7ca155ed95f30a6b45bbb8b97d2727f02424" +dependencies = [ + "lazy_static", + "symphonia-bundle-flac 0.6.1", + "symphonia-bundle-mp3 0.6.1", + "symphonia-codec-aac 0.6.1", + "symphonia-codec-adpcm 0.6.1", + "symphonia-codec-alac 0.6.1", + "symphonia-codec-pcm 0.6.1", + "symphonia-codec-vorbis 0.6.1", + "symphonia-core 0.6.1", + "symphonia-format-caf", + "symphonia-format-isomp4 0.6.1", + "symphonia-format-mkv 0.6.1", + "symphonia-format-ogg 0.6.1", + "symphonia-format-riff 0.6.1", + "symphonia-metadata 0.6.1", +] + +[[package]] +name = "symphonia-bundle-flac" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c91565e180aea25d9b80a910c546802526ffd0072d0b8974e3ebe59b686c9976" +dependencies = [ + "log", + "symphonia-core 0.5.5", + "symphonia-metadata 0.5.5", + "symphonia-utils-xiph", +] + +[[package]] +name = "symphonia-bundle-flac" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6405d5c34ff6f8f7ca08a4101efe66d21e180f7322ef9359900a0e55698a7892" +dependencies = [ + "log", + "symphonia-common", + "symphonia-core 0.6.1", + "symphonia-metadata 0.6.1", +] + +[[package]] +name = "symphonia-bundle-mp3" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4872dd6bb56bf5eac799e3e957aa1981086c3e613b27e0ac23b176054f7c57ed" +dependencies = [ + "lazy_static", + "log", + "symphonia-core 0.5.5", + "symphonia-metadata 0.5.5", +] + +[[package]] +name = "symphonia-bundle-mp3" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "98ea5ffc8716bff677dfb3b01b420c7b758de901a72b8c330bf2040ab74b4add" +dependencies = [ + "lazy_static", + "log", + "symphonia-core 0.6.1", +] + +[[package]] +name = "symphonia-codec-aac" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c263845aa86881416849c1729a54c7f55164f8b96111dba59de46849e73a790" +dependencies = [ + "lazy_static", + "log", + "symphonia-core 0.5.5", +] + +[[package]] +name = "symphonia-codec-aac" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f5bf8e39552d34a3c4c98333370e62f48c92456d2e814f273b5c3ad7c4a5f45c" +dependencies = [ + "lazy_static", + "log", + "symphonia-common", + "symphonia-core 0.6.1", +] + +[[package]] +name = "symphonia-codec-adpcm" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2dddc50e2bbea4cfe027441eece77c46b9f319748605ab8f3443350129ddd07f" +dependencies = [ + "log", + "symphonia-core 0.5.5", +] + +[[package]] +name = "symphonia-codec-adpcm" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "445932ecb0c59362fde9c082dd63a75380650c5077fdb8b80b8cac850442a74c" +dependencies = [ + "log", + "symphonia-core 0.6.1", +] + +[[package]] +name = "symphonia-codec-alac" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8413fa754942ac16a73634c9dfd1500ed5c61430956b33728567f667fdd393ab" +dependencies = [ + "log", + "symphonia-core 0.5.5", +] + +[[package]] +name = "symphonia-codec-alac" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "920a78f96f3cf62932d0c497959c0cc2100ef36b03c7ca1417b260f432a482d8" +dependencies = [ + "log", + "symphonia-common", + "symphonia-core 0.6.1", +] + +[[package]] +name = "symphonia-codec-pcm" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4e89d716c01541ad3ebe7c91ce4c8d38a7cf266a3f7b2f090b108fb0cb031d95" +dependencies = [ + "log", + "symphonia-core 0.5.5", +] + +[[package]] +name = "symphonia-codec-pcm" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e04ba75686acbe43542fdd374571195f0530c0b7785ca25cc6840e9c6c4b6eea" +dependencies = [ + "log", + "symphonia-core 0.6.1", +] + +[[package]] +name = "symphonia-codec-vorbis" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f025837c309cd69ffef572750b4a2257b59552c5399a5e49707cc5b1b85d1c73" +dependencies = [ + "log", + "symphonia-core 0.5.5", + "symphonia-utils-xiph", +] + +[[package]] +name = "symphonia-codec-vorbis" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73d90b4fcf796137cc683c538282804ff9629f8ad9dbfd881fcbba331ac4e986" +dependencies = [ + "log", + "symphonia-common", + "symphonia-core 0.6.1", +] + +[[package]] +name = "symphonia-common" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2acc3fcc18ec9b8cdd48614e259c4cf0d27b71d41e5d9b120b42c5adab12d7c4" +dependencies = [ + "log", + "symphonia-core 0.6.1", + "symphonia-metadata 0.6.1", +] + +[[package]] +name = "symphonia-core" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ea00cc4f79b7f6bb7ff87eddc065a1066f3a43fe1875979056672c9ef948c2af" +dependencies = [ + "arrayvec", + "bitflags 1.3.2", + "bytemuck", + "lazy_static", + "log", +] + +[[package]] +name = "symphonia-core" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "01c412864d599d4750d0c3d684d7e093ec05e5309681ef5252cc1096a437f6e0" +dependencies = [ + "bitflags 2.11.0", + "bytemuck", + "lazy_static", + "log", + "num-complex", + "rustfft", + "smallvec", +] + +[[package]] +name = "symphonia-format-caf" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab64327ee1920531c5bf86cf7e9cd1a599d57013ddc30691da62683cefc65191" +dependencies = [ + "log", + "symphonia-common", + "symphonia-core 0.6.1", +] + +[[package]] +name = "symphonia-format-isomp4" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "243739585d11f81daf8dac8d9f3d18cc7898f6c09a259675fc364b382c30e0a5" +dependencies = [ + "encoding_rs", + "log", + "symphonia-core 0.5.5", + "symphonia-metadata 0.5.5", + "symphonia-utils-xiph", +] + +[[package]] +name = "symphonia-format-isomp4" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e681a70e1870d34e02abf1dbc51e4267c3f1827801474e8870be8c689fc4dc3" +dependencies = [ + "log", + "symphonia-common", + "symphonia-core 0.6.1", + "symphonia-metadata 0.6.1", +] + +[[package]] +name = "symphonia-format-mkv" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "122d786d2c43a49beb6f397551b4a050d8229eaa54c7ddf9ee4b98899b8742d0" +dependencies = [ + "lazy_static", + "log", + "symphonia-core 0.5.5", + "symphonia-metadata 0.5.5", + "symphonia-utils-xiph", +] + +[[package]] +name = "symphonia-format-mkv" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d015c5c0558864665894b3f4cbd95e10abb01b9c868e751c72670f326a56360e" +dependencies = [ + "lazy_static", + "log", + "symphonia-common", + "symphonia-core 0.6.1", +] + +[[package]] +name = "symphonia-format-ogg" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b4955c67c1ed3aa8ae8428d04ca8397fbef6a19b2b051e73b5da8b1435639cb" +dependencies = [ + "log", + "symphonia-core 0.5.5", + "symphonia-metadata 0.5.5", + "symphonia-utils-xiph", +] + +[[package]] +name = "symphonia-format-ogg" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b5495e7f7e3c7035328d82b6d6e377eef289bb0c4105bdeb557fc93a833f994" +dependencies = [ + "log", + "symphonia-common", + "symphonia-core 0.6.1", + "symphonia-metadata 0.6.1", +] + +[[package]] +name = "symphonia-format-riff" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2d7c3df0e7d94efb68401d81906eae73c02b40d5ec1a141962c592d0f11a96f" +dependencies = [ + "extended", + "log", + "symphonia-core 0.5.5", + "symphonia-metadata 0.5.5", +] + +[[package]] +name = "symphonia-format-riff" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ff70929083a8c1a5f6cd7c904b6071c7914ad04739b510c2f7239dfc9b7dabe" +dependencies = [ + "extended", + "log", + "symphonia-core 0.6.1", + "symphonia-metadata 0.6.1", +] + +[[package]] +name = "symphonia-metadata" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "36306ff42b9ffe6e5afc99d49e121e0bd62fe79b9db7b9681d48e29fa19e6b16" +dependencies = [ + "encoding_rs", + "lazy_static", + "log", + "symphonia-core 0.5.5", +] + +[[package]] +name = "symphonia-metadata" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "83713a97705d77bdef7cdbc0768fd6e5a54e4cd7e48d60a806ae85639e2c87c6" +dependencies = [ + "lazy_static", + "log", + "regex-lite", + "smallvec", + "symphonia-core 0.6.1", +] + +[[package]] +name = "symphonia-utils-xiph" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee27c85ab799a338446b68eec77abf42e1a6f1bb490656e121c6e27bfbab9f16" +dependencies = [ + "symphonia-core 0.5.5", + "symphonia-metadata 0.5.5", +] + [[package]] name = "syn" version = "2.0.117" @@ -4881,6 +5570,19 @@ dependencies = [ "smallvec", ] +[[package]] +name = "tempfile" +version = "3.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" +dependencies = [ + "fastrand", + "getrandom 0.3.4", + "once_cell", + "rustix 1.1.4", + "windows-sys 0.61.2", +] + [[package]] name = "temporal_capi" version = "0.2.6" @@ -5311,6 +6013,16 @@ dependencies = [ "tracing-log", ] +[[package]] +name = "transpose" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ad61aed86bc3faea4300c7aee358b4c6d0c8d6ccc36524c96e4c92ccf26e77e" +dependencies = [ + "num-integer", + "strength_reduce", +] + [[package]] name = "tungstenite" version = "0.28.0" @@ -5484,7 +6196,7 @@ name = "v8" version = "152.2.0" dependencies = [ "bindgen 0.72.1", - "bitflags", + "bitflags 2.11.0", "cc", "fslock", "gzip-header", @@ -5502,7 +6214,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3701ffb3692a24808503d17f758b0a3f688d2c451779247982566b5941033e63" dependencies = [ "bindgen 0.70.1", - "bitflags", + "bitflags 2.11.0", "cc", "paste", "temporal_capi", @@ -5522,6 +6234,15 @@ version = "0.2.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" +[[package]] +name = "vecmath" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "956ae1e0d85bca567dee1dcf87fb1ca2e792792f66f87dced8381f99cd91156a" +dependencies = [ + "piston-float", +] + [[package]] name = "vello_common" version = "0.2.0" @@ -5594,6 +6315,17 @@ version = "0.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" +[[package]] +name = "visibility" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d674d135b4a8c1d7e813e2f8d1c9a58308aee4a680323066025e53132218bd91" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + [[package]] name = "void" version = "1.0.2" @@ -5643,6 +6375,34 @@ dependencies = [ "thiserror", ] +[[package]] +name = "web-audio-api" +version = "1.7.0" +dependencies = [ + "almost", + "arc-swap", + "arrayvec", + "creek", + "crossbeam-channel", + "dasp_sample", + "fft-convolver", + "float_eq", + "futures-channel", + "futures-core", + "futures-util", + "hound", + "hrtf", + "llq", + "log", + "no_denormals", + "num-complex", + "realfft", + "rubato 0.16.2", + "smallvec", + "symphonia 0.6.1", + "vecmath", +] + [[package]] name = "web_atoms" version = "0.2.3" @@ -5678,7 +6438,7 @@ checksum = "b4ee928febd44d98f2f459a4a79bd4d928591333a494a10a868418ac1b39cf1f" dependencies = [ "either", "home", - "rustix", + "rustix 0.38.44", "winsafe", ] @@ -5728,6 +6488,15 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" +[[package]] +name = "windowfunctions" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "90628d739333b7c5d2ee0b70210b97b8cddc38440c682c96fd9e2c24c2db5f3a" +dependencies = [ + "num-traits", +] + [[package]] name = "windows" version = "0.62.2" diff --git a/Cargo.toml b/Cargo.toml index 314dcfbccc..934fd5917d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -82,12 +82,13 @@ members = [ "moli-http-cache", "moli-wpt-compat" ] -exclude = ["vendor/xml5ever-0.39.0"] +exclude = ["vendor/xml5ever-0.39.0", "vendor/web-audio-api-1.7.0"] [workspace.package] license = "MIT OR Apache-2.0" [patch.crates-io] +web-audio-api = { path = "vendor/web-audio-api-1.7.0" } xml5ever = { path = "vendor/xml5ever-0.39.0" } curl = { git = "https://github.com/ldm0/curl-rust", branch = "moli" } curl-sys = { git = "https://github.com/ldm0/curl-rust", branch = "moli" } diff --git a/licenses/Blink-WebAudio-NOTICE.txt b/licenses/Blink-WebAudio-NOTICE.txt new file mode 100644 index 0000000000..e2bcfbd37d --- /dev/null +++ b/licenses/Blink-WebAudio-NOTICE.txt @@ -0,0 +1,33 @@ +Blink Web Audio algorithms used by Moli: + third_party/blink/renderer/platform/audio/dynamics_compressor.cc + third_party/blink/renderer/modules/webaudio/periodic_wave.cc + +Source: https://chromium.googlesource.com/chromium/src/ +The Rust ports are in moli-renderer-v8/src/context_bootstrap/web_audio_runtime/. + +Copyright (C) 2011, 2012 Google Inc. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: +1. Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. +2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. +3. Neither the name of Apple Computer, Inc. ("Apple") nor the names of its + contributors may be used to endorse or promote products derived from this + software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY APPLE AND ITS CONTRIBUTORS "AS IS" AND ANY +EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL APPLE OR ITS CONTRIBUTORS BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +! Stateful look-ahead compressor, ported from Blink's +! `third_party/blink/renderer/platform/audio/dynamics_compressor.cc`. diff --git a/licenses/web-audio-api-MIT.txt b/licenses/web-audio-api-MIT.txt new file mode 100644 index 0000000000..d6e5087db2 --- /dev/null +++ b/licenses/web-audio-api-MIT.txt @@ -0,0 +1,24 @@ +web-audio-api 1.7.0 +https://github.com/orottier/web-audio-api-rs + +MIT License + +Copyright (c) 2021 Otto Rottier + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/moli-core/tests/fixtures/runtime/audio_worklet_wasm_source_phase.html b/moli-core/tests/fixtures/runtime/audio_worklet_wasm_source_phase.html index 2850296e00..a68ea936d8 100644 --- a/moli-core/tests/fixtures/runtime/audio_worklet_wasm_source_phase.html +++ b/moli-core/tests/fixtures/runtime/audio_worklet_wasm_source_phase.html @@ -5,6 +5,7 @@ globalThis.audioWorkletWasmDone = false; globalThis.audioWorkletWasmError = ""; globalThis.audioWorkletWasmResult = ""; + globalThis.audioWorkletWasmConnections = ""; try { const audioContext = new AudioContext(); @@ -25,6 +26,18 @@ ); }; audioNode.port.postMessage(""); + const connected = audioNode.connect(audioContext.destination); + audioNode.disconnect(audioContext.destination); + let disconnectedError = ""; + try { + audioNode.disconnect(audioContext.destination); + } catch (error) { + disconnectedError = error.name; + } + globalThis.audioWorkletWasmConnections = [ + connected === audioContext.destination, + disconnectedError + ].join("|"); audioNode.connect(audioContext.destination); } catch (error) { globalThis.audioWorkletWasmDone = true; diff --git a/moli-core/tests/scripts.rs b/moli-core/tests/scripts.rs index e03d6af4b0..6c88b71f01 100644 --- a/moli-core/tests/scripts.rs +++ b/moli-core/tests/scripts.rs @@ -5062,6 +5062,12 @@ async fn audio_worklet_module_imports_wasm_source_phase() -> Result<()> { diagnostic_global(&page, "audioWorkletWasmError"), Some(&JsValueSnapshot::String(String::new())) ); + assert_eq!( + diagnostic_global(&page, "audioWorkletWasmConnections"), + Some(&JsValueSnapshot::String( + "true|InvalidAccessError".to_owned() + )) + ); assert_eq!( diagnostic_global(&page, "audioWorkletWasmResult"), Some(&JsValueSnapshot::String( diff --git a/moli-renderer-v8/Cargo.toml b/moli-renderer-v8/Cargo.toml index d1def308a4..2372c2dbf6 100644 --- a/moli-renderer-v8/Cargo.toml +++ b/moli-renderer-v8/Cargo.toml @@ -28,6 +28,8 @@ flate2 = { version = "1.1.9", default-features = false, features = ["zlib"] } euclid = "0.22" kurbo = "0.13.1" futures-util = "0.3.31" +web-audio-api = { version = "=1.7.0", default-features = false, features = ["spec-compliant-worklet-inputs"] } +realfft = "3.5.0" precomputed-hash = "0.1" percent-encoding = "2.3" selectors = "0.40" diff --git a/moli-renderer-v8/src/context_bootstrap/web_audio_runtime.rs b/moli-renderer-v8/src/context_bootstrap/web_audio_runtime.rs index 991f6a4d7f..8f1c0a4859 100644 --- a/moli-renderer-v8/src/context_bootstrap/web_audio_runtime.rs +++ b/moli-renderer-v8/src/context_bootstrap/web_audio_runtime.rs @@ -12,13 +12,19 @@ use crate::util::{ use crate::webidl; use moli_webapi_declare::{WebApiFunctionTemplate, WebApiObject}; +mod analyser; mod audio_buffer; mod audio_param; +mod backend; mod biquad; +mod compressor; mod graph; +mod oscillator; +mod oscillator_dsp; +mod wavetable; pub(in crate::context_bootstrap) use audio_buffer::build_constructor_template as build_audio_buffer_constructor_template; -use audio_param::{audio_param, detune_param}; +use backend::{Node, NodeKind, State}; const AUDIO_CONTEXT_LISTENERS_SLOT: &str = "__moliAudioContextListeners"; const AUDIO_CONTEXT_MODULES_SLOT: &str = "__moliAudioContextModules"; @@ -32,20 +38,12 @@ const AUDIO_WORKLET_MODULE_RESOLVER_SLOT: &str = "__moliAudioWorkletModuleResolv const AUDIO_WORKLET_MODULE_LOADED_SLOT: &str = "__moliAudioWorkletModuleLoaded"; const AUDIO_WORKLET_MODULE_SETTLED_SLOT: &str = "__moliAudioWorkletModuleSettled"; const AUDIO_WORKLET_CALLBACK_MODULE_SLOT: &str = "__moliAudioWorkletCallbackModule"; -const OFFLINE_AUDIO_LENGTH_SLOT: &str = "__moliOfflineAudioLength"; -const OFFLINE_AUDIO_SAMPLE_RATE_SLOT: &str = "__moliOfflineAudioSampleRate"; -const OFFLINE_AUDIO_CHANNEL_COUNT_SLOT: &str = "__moliOfflineAudioChannelCount"; const OFFLINE_AUDIO_COMPLETE_CONTEXT_SLOT: &str = "__moliOfflineAudioCompleteContext"; -const OFFLINE_AUDIO_COMPLETE_BUFFER_SLOT: &str = "__moliOfflineAudioCompleteBuffer"; -const OFFLINE_AUDIO_COMPRESSORS_SLOT: &str = "__moliOfflineAudioCompressors"; -const DYNAMICS_COMPRESSOR_REDUCTION_SLOT: &str = "__moliDynamicsCompressorReduction"; -const ANALYSER_FFT_SIZE_SLOT: &str = "__moliAnalyserFftSize"; +const OFFLINE_AUDIO_COMPLETE_RESOLVER_SLOT: &str = "__moliOfflineAudioCompleteResolver"; #[derive(WebApiObject)] #[webapi(interface = "AudioContext")] struct AudioContextObjectDeclaration<'scope> { - #[webapi(data_property = "currentTime")] - current_time: f64, #[webapi(data_property = "sampleRate")] sample_rate: f64, #[webapi(data_property)] @@ -132,20 +130,10 @@ struct AudioWorkletProcessorConstructMessageDeclaration<'scope> { #[derive(WebApiObject)] #[webapi(interface = "OfflineAudioContext")] struct OfflineAudioContextObjectDeclaration<'scope> { - #[webapi(data_property = "currentTime")] - current_time: f64, #[webapi(data_property)] length: f64, #[webapi(data_property = "sampleRate")] sample_rate: f64, - #[webapi(slot = OFFLINE_AUDIO_LENGTH_SLOT)] - internal_length: f64, - #[webapi(slot = OFFLINE_AUDIO_SAMPLE_RATE_SLOT)] - internal_sample_rate: f64, - #[webapi(slot = OFFLINE_AUDIO_CHANNEL_COUNT_SLOT)] - channel_count: f64, - #[webapi(slot = OFFLINE_AUDIO_COMPRESSORS_SLOT)] - compressors: v8::Local<'scope, v8::Array>, #[webapi(slot = SIMPLE_EVENT_TARGET_SLOT)] event_target_slot: &'static str, #[webapi(data_property)] @@ -169,9 +157,7 @@ struct OfflineAudioContextObjectDeclaration<'scope> { #[derive(WebApiObject)] #[webapi(interface = "OscillatorNode")] struct OscillatorNodeObjectDeclaration<'scope> { - #[webapi(data_property = "type")] - kind: &'static str, - #[webapi(data_property)] + #[webapi(data_property, readonly)] frequency: v8::Local<'scope, v8::Object>, #[webapi(data_property, readonly)] detune: v8::Local<'scope, v8::Object>, @@ -186,18 +172,16 @@ struct OscillatorNodeObjectDeclaration<'scope> { #[derive(WebApiObject)] #[webapi(interface = "DynamicsCompressorNode")] struct DynamicsCompressorNodeObjectDeclaration<'scope> { - #[webapi(data_property)] + #[webapi(data_property, readonly)] threshold: v8::Local<'scope, v8::Object>, - #[webapi(data_property)] + #[webapi(data_property, readonly)] knee: v8::Local<'scope, v8::Object>, - #[webapi(data_property)] + #[webapi(data_property, readonly)] ratio: v8::Local<'scope, v8::Object>, - #[webapi(data_property)] + #[webapi(data_property, readonly)] attack: v8::Local<'scope, v8::Object>, - #[webapi(data_property)] + #[webapi(data_property, readonly)] release: v8::Local<'scope, v8::Object>, - #[webapi(slot = DYNAMICS_COMPRESSOR_REDUCTION_SLOT)] - reduction: f64, #[webapi(method, length = 1, callback = audio_node_connect_callback)] connect: (), #[webapi(method, length = 0, callback = audio_node_disconnect_callback)] @@ -211,47 +195,13 @@ struct DynamicsCompressorNodePrototypeDeclaration { reduction: (), } -#[derive(WebApiObject)] -#[webapi(interface = "AnalyserNode")] -struct AnalyserNodeObjectDeclaration { - #[webapi(slot = ANALYSER_FFT_SIZE_SLOT)] - fft_size: f64, - #[webapi(data_property = "minDecibels")] - min_decibels: f64, - #[webapi(data_property = "maxDecibels")] - max_decibels: f64, - #[webapi(data_property = "smoothingTimeConstant")] - smoothing_time_constant: f64, - #[webapi(method, length = 1, callback = audio_node_connect_callback)] - connect: (), - #[webapi(method, length = 0, callback = audio_node_disconnect_callback)] - disconnect: (), -} - -#[derive(WebApiFunctionTemplate)] -#[webapi(name = "AnalyserNode", enumerable)] -struct AnalyserNodePrototypeDeclaration { - #[webapi(accessor_property = "fftSize", getter = analyser_fft_size_getter_callback, setter = analyser_fft_size_setter_callback)] - fft_size: (), - #[webapi(accessor_property = "frequencyBinCount", getter = analyser_frequency_bin_count_getter_callback)] - frequency_bin_count: (), - #[webapi(method, length = 1, callback = analyser_get_float_frequency_data_callback)] - get_float_frequency_data: (), - #[webapi(method, length = 1, callback = analyser_get_float_time_domain_data_callback)] - get_float_time_domain_data: (), - #[webapi(method, length = 1, callback = analyser_get_byte_frequency_data_callback)] - get_byte_frequency_data: (), - #[webapi(method, length = 1, callback = analyser_get_byte_time_domain_data_callback)] - get_byte_time_domain_data: (), -} - #[derive(WebApiObject)] #[webapi(interface = "Object")] struct OfflineAudioCompletePayloadDeclaration<'scope> { #[webapi(slot = OFFLINE_AUDIO_COMPLETE_CONTEXT_SLOT)] context: v8::Local<'scope, v8::Object>, - #[webapi(slot = OFFLINE_AUDIO_COMPLETE_BUFFER_SLOT)] - rendered_buffer: v8::Local<'scope, v8::Object>, + #[webapi(slot = OFFLINE_AUDIO_COMPLETE_RESOLVER_SLOT)] + resolver: v8::Local<'scope, v8::Object>, } #[derive(WebApiObject)] @@ -267,85 +217,13 @@ struct OfflineAudioCompletionEventDeclaration<'scope> { #[webapi(interface = "AudioDestinationNode", allow_empty)] struct AudioDestinationNodeObjectDeclaration {} -// Captured from the Chromium-on-Linux baseline we use for Zhihu probe parity. -// If that browser profile changes, update this together with the audio probe -// assertions in `script_vm/tests.rs`. -const TARGET_AUDIO_FINGERPRINT_SUM: f64 = 124.04347527516074; -const SYNTHETIC_COMPRESSOR_REDUCTION: f64 = -82.26815795898438; -const SYNTHETIC_ANALYSER_FREQUENCY_BINS: &[f64] = &[ - -90.25955200195312, - -90.22233581542969, - -90.11856842041016, - -89.96821594238281, - -89.79446411132812, - -89.61327362060547, - -89.42459106445312, - -89.20698547363281, - -88.91907501220703, - -88.51146697998047, - -87.94953918457031, - -87.2354736328125, - -86.41204833984375, - -85.54572296142578, - -84.70391082763672, - -83.94123077392578, - -83.29612731933594, - -82.79357147216797, - -82.4489974975586, - -82.27188110351562, - -82.26815795898438, - -82.4417953491211, - -82.79552459716797, - -83.33106231689453, - -84.04853820800781, - -84.94512939453125, - -86.0123062133789, - -87.2312240600586, - -88.56591796875, - -89.95575714111328, - -91.31378936767578, - -92.54295349121094, - -93.5753173828125, - -94.40994262695312, - -95.10946655273438, - -95.75737762451172, - -96.4164810180664, - -97.1112060546875, - -97.82959747314453, - -98.53618621826172, - -99.19161987304688, - -99.77345275878906, - -100.28733825683594, - -100.76123046875, - -101.2282485961914, - -101.71035766601562, - -102.21070098876953, - -102.71531677246094, - -103.20209503173828, - -103.65292358398438, - -104.06330108642578, - -104.44380950927734, - -104.81304168701172, - -105.1873779296875, - -105.57337951660156, - -105.9659423828125, - -106.35223388671875, - -106.71916961669922, - -107.06056213378906, - -107.38026428222656, - -107.68936920166016, - -107.99988555908203, - -108.31855773925781, - -108.6440658569336, -]; - #[derive(webidl::WebIdlArgs)] #[webidl(prefix = "OfflineAudioContext")] struct OfflineAudioContextConstructorArgs { #[webidl(required)] - channel_count: f64, + channel_count: u32, #[webidl(required)] - length: f64, + length: u32, #[webidl(required)] sample_rate: f64, } @@ -359,13 +237,6 @@ struct AudioParamSetValueAtTimeArgs { start_time: f64, } -#[derive(webidl::WebIdlArgs)] -#[webidl(prefix = "AnalyserNode.fftSize")] -struct AnalyserFftSizeArgs { - #[webidl(required)] - value: u32, -} - #[derive(WebApiFunctionTemplate)] #[webapi( name = "AudioContext", @@ -395,6 +266,9 @@ struct AudioWorkletNodeTemplateDeclaration { #[derive(WebApiFunctionTemplate)] #[webapi(name = "BaseAudioContext", enumerable)] struct BaseAudioContextPrototypeDeclaration { + #[webapi(accessor_property, getter = audio_context_current_time)] + current_time: (), + #[webapi(method = "createBuffer", length = 3, callback = audio_buffer::create_buffer)] create_buffer: (), @@ -463,6 +337,7 @@ pub(in crate::context_bootstrap) fn install_web_audio_template_bindings<'s>( } match interface_name { "AudioParam" => audio_param::install(scope, template), + "OscillatorNode" => oscillator::install(scope, template), "BiquadFilterNode" => biquad::install(scope, template), "BaseAudioContext" => { BaseAudioContextPrototypeDeclaration::initialize_prototype_template( @@ -476,12 +351,8 @@ pub(in crate::context_bootstrap) fn install_web_audio_template_bindings<'s>( template.prototype_template(scope), ); } - "AnalyserNode" => { - AnalyserNodePrototypeDeclaration::initialize_prototype_template( - scope, - template.prototype_template(scope), - ); - } + "AnalyserNode" => analyser::install(scope, template), + "DynamicsCompressorNode" => { DynamicsCompressorNodePrototypeDeclaration::initialize_prototype_template( scope, @@ -506,6 +377,7 @@ fn audio_context_constructor_callback<'s>( } let context = args.this(); + backend::initialize(scope, context, State::Context(backend::Context::realtime())); let destination = audio_destination_node(scope, context); let modules = new_web_audio_map_object(scope); let module_list = v8::Array::new(scope, 0); @@ -516,7 +388,6 @@ fn audio_context_constructor_callback<'s>( set_symbol_to_string_tag(scope, audio_worklet, "AudioWorklet"); AudioContextObjectDeclaration::new( - 0.0, 44_100.0, "running", destination, @@ -537,12 +408,35 @@ pub(in crate::context_bootstrap) fn is_audio_context_object<'s>( get_private_value(scope, object, AUDIO_CONTEXT_MODULES_SLOT).is_some() } +fn audio_context_current_time<'s>( + scope: &mut v8::PinScope<'s, '_>, + args: v8::FunctionCallbackArguments<'s>, + mut rv: v8::ReturnValue<'s, v8::Value>, +) { + if let Some(state) = backend::get(scope, args.this()) + && let State::Context(context) = &*state.borrow() + { + rv.set(v8::Number::new(scope, context.current_time()).into()); + return; + } + throw_type_error(scope, "Illegal invocation: expected a BaseAudioContext."); +} + fn audio_context_close_callback<'s>( scope: &mut v8::PinScope<'s, '_>, args: v8::FunctionCallbackArguments<'s>, mut rv: v8::ReturnValue<'s, v8::Value>, ) { let context = args.this(); + if !is_audio_context_object(scope, context) { + throw_type_error(scope, "Illegal invocation: expected an AudioContext."); + return; + } + if let Some(state) = backend::get(scope, context) + && let State::Context(backend::Context::Realtime(native)) = &*state.borrow() + { + native.close_sync(); + } if let Some(module_list) = web_audio_array_slot(scope, context, AUDIO_CONTEXT_MODULE_LIST_SLOT) { let length = module_list @@ -841,6 +735,7 @@ fn audio_worklet_node_constructor_callback<'s>( AudioWorkletNodeObjectDeclaration::new(context, port1) .initialize(scope, node) .expect("AudioWorkletNode declaration should initialize object"); + backend::initialize(scope, node, State::ModuleWorklet); graph::initialize_node(scope, node, context); if let Some(worker) = web_audio_object_slot(scope, module_state, AUDIO_WORKLET_MODULE_WORKER_SLOT) @@ -1207,9 +1102,17 @@ fn dynamics_compressor_reduction_getter_callback<'s>( args: v8::FunctionCallbackArguments<'s>, mut rv: v8::ReturnValue<'s, v8::Value>, ) { - let value = web_audio_number_slot(scope, args.this(), DYNAMICS_COMPRESSOR_REDUCTION_SLOT) - .unwrap_or(0.0); - rv.set(v8::Number::new(scope, value).into()); + if let Some(state) = backend::get(scope, args.this()) + && let State::Node(Node::Compressor { reduction, .. }) = &*state.borrow() + { + let value = f32::from_bits(reduction.load(std::sync::atomic::Ordering::Relaxed)); + rv.set(v8::Number::new(scope, f64::from(value)).into()); + return; + } + throw_type_error( + scope, + "Illegal invocation: expected a DynamicsCompressorNode.", + ); } pub(in crate::context_bootstrap) fn offline_audio_context_constructor_callback<'s>( @@ -1230,34 +1133,43 @@ pub(in crate::context_bootstrap) fn offline_audio_context_constructor_callback<' return; }; - let channel_count = parsed.channel_count.trunc(); - let length = parsed.length.trunc(); - let sample_rate = parsed.sample_rate; - if !channel_count.is_finite() - || !length.is_finite() - || !sample_rate.is_finite() - || channel_count <= 0.0 - || length <= 0.0 - || sample_rate <= 0.0 - { + let channel_count = parsed.channel_count; + let length = parsed.length; + let sample_rate = f64::from(parsed.sample_rate as f32); + if !sample_rate.is_finite() { throw_type_error( scope, + "OfflineAudioContext sampleRate must be a finite float.", + ); + return; + } + if !(1..=32).contains(&channel_count) + || !(1..=i32::MAX as u32).contains(&length) + || !(3000.0..=768000.0).contains(&sample_rate) + { + throw_dom_exception( + scope, + "NotSupportedError", + 9, "Failed to construct 'OfflineAudioContext': invalid channel count, length, or sample rate.", ); return; } let context = args.this(); + backend::initialize( + scope, + context, + State::Context(backend::Context::offline( + channel_count as usize, + length as usize, + sample_rate as f32, + )), + ); let destination = audio_destination_node(scope, context); - let compressors = v8::Array::new(scope, 0); OfflineAudioContextObjectDeclaration::new( - 0.0, - length, + f64::from(length), sample_rate, - length, - sample_rate, - channel_count, - compressors, OFFLINE_AUDIO_LISTENERS_SLOT, "suspended", destination, @@ -1271,8 +1183,8 @@ fn require_base_audio_context<'s>( scope: &mut v8::PinScope<'s, '_>, object: v8::Local<'s, v8::Object>, ) -> bool { - if is_audio_context_object(scope, object) - || get_private_value(scope, object, OFFLINE_AUDIO_LENGTH_SLOT).is_some() + if backend::get(scope, object) + .is_some_and(|state| matches!(&*state.borrow(), State::Context(_))) { return true; } @@ -1288,14 +1200,15 @@ fn audio_context_create_oscillator_callback<'s>( if !require_base_audio_context(scope, args.this()) { return; } - let nyquist = audio_context_sample_rate(scope, args.this()) / 2.0; - let frequency = audio_param(scope, 440.0, -nyquist, nyquist); - let detune = detune_param(scope); - let node = OscillatorNodeObjectDeclaration::new("sine", frequency, detune) + let native = backend::create_node(scope, args.this(), NodeKind::Oscillator); + let frequency = audio_param::wrap(scope, native.param("frequency")); + let detune = audio_param::wrap(scope, native.param("detune")); + let node = OscillatorNodeObjectDeclaration::new(frequency, detune) .bind(scope) .expect("OscillatorNode declaration should bind"); graph::initialize_node(scope, node, args.this()); graph::initialize_source(scope, node); + backend::initialize(scope, node, State::Node(native)); rv.set(node.into()); } @@ -1307,17 +1220,18 @@ fn audio_context_create_dynamics_compressor_callback<'s>( if !require_base_audio_context(scope, args.this()) { return; } - let threshold = audio_param(scope, -24.0, -100.0, 0.0); - let knee = audio_param(scope, 30.0, 0.0, 40.0); - let ratio = audio_param(scope, 12.0, 1.0, 20.0); - let attack = audio_param(scope, 0.003, 0.0, 1.0); - let release = audio_param(scope, 0.25, 0.0, 1.0); + let native = backend::create_node(scope, args.this(), NodeKind::Compressor); + let threshold = audio_param::wrap(scope, native.param("threshold")); + let knee = audio_param::wrap(scope, native.param("knee")); + let ratio = audio_param::wrap(scope, native.param("ratio")); + let attack = audio_param::wrap(scope, native.param("attack")); + let release = audio_param::wrap(scope, native.param("release")); let node = - DynamicsCompressorNodeObjectDeclaration::new(threshold, knee, ratio, attack, release, 0.0) + DynamicsCompressorNodeObjectDeclaration::new(threshold, knee, ratio, attack, release) .bind(scope) .expect("DynamicsCompressorNode declaration should bind"); graph::initialize_node(scope, node, args.this()); - remember_context_compressor(scope, args.this(), node); + backend::initialize(scope, node, State::Node(native)); rv.set(node.into()); } @@ -1329,10 +1243,7 @@ fn audio_context_create_analyser_callback<'s>( if !require_base_audio_context(scope, args.this()) { return; } - let node = AnalyserNodeObjectDeclaration::new(2048.0, -100.0, -30.0, 0.8) - .bind(scope) - .expect("AnalyserNode declaration should bind"); - graph::initialize_node(scope, node, args.this()); + let node = analyser::create(scope, args.this()); rv.set(node.into()); } @@ -1342,38 +1253,50 @@ fn offline_audio_context_start_rendering_callback<'s>( mut rv: v8::ReturnValue<'s, v8::Value>, ) { let context = args.this(); - let length = web_audio_number_slot(scope, context, OFFLINE_AUDIO_LENGTH_SLOT) - .and_then(float_to_usize) - .unwrap_or(44_100); - let sample_rate = - web_audio_number_slot(scope, context, OFFLINE_AUDIO_SAMPLE_RATE_SLOT).unwrap_or(44_100.0); - let has_input = graph::prepare_offline_render(scope, context, length as f64 / sample_rate); - let channel_count = web_audio_number_slot(scope, context, OFFLINE_AUDIO_CHANNEL_COUNT_SLOT) - .unwrap_or(1.0) as u32; - let Some(rendered_buffer) = - build_audio_buffer(scope, channel_count, length, sample_rate, has_input) + let Some(state) = backend::get(scope, context) else { + throw_type_error( + scope, + "Illegal invocation: expected an OfflineAudioContext.", + ); + return; + }; + { + let mut state = state.borrow_mut(); + let State::Context(backend::Context::Offline { rendered, .. }) = &mut *state else { + throw_type_error( + scope, + "Illegal invocation: expected an OfflineAudioContext.", + ); + return; + }; + if *rendered { + let error = new_dom_exception_value( + scope, + "Offline rendering has already started.", + "InvalidStateError", + ); + set_rejected_promise_return(scope, &mut rv, error); + return; + } + *rendered = true; + } + let Some(resolver) = v8::PromiseResolver::new(scope) else { + return; + }; + let payload = OfflineAudioCompletePayloadDeclaration::new(context, resolver.into()) + .bind(scope) + .expect("OfflineAudio payload should bind"); + let Some(callback) = v8::Function::builder(offline_audio_context_complete_microtask_callback) + .data(payload.into()) + .build(scope) else { return; }; - define_non_enumerable_string_property(scope, context, "state", "closed"); - - let payload = OfflineAudioCompletePayloadDeclaration::new(context, rendered_buffer) - .bind(scope) - .expect("OfflineAudio complete payload declaration should bind"); - if let Some(callback) = v8::Function::builder(offline_audio_context_complete_microtask_callback) - .data(payload.into()) - .build(scope) - { - scope.enqueue_microtask(callback); - } - - if let Some(resolver) = v8::PromiseResolver::new(scope) { - let promise = resolver.get_promise(scope); - let _ = resolver.resolve(scope, rendered_buffer.into()); - rv.set(promise.into()); - return; - } - rv.set(v8::undefined(scope).into()); + // Defer work and observable results until after the initiating JS call. + // Both PCM and analyser history are produced by the same native graph. + scope.enqueue_microtask(callback); + define_non_enumerable_string_property(scope, context, "state", "running"); + rv.set(resolver.get_promise(scope).into()); } fn offline_audio_context_complete_microtask_callback<'s>( @@ -1388,16 +1311,45 @@ fn offline_audio_context_complete_microtask_callback<'s>( else { return; }; - let Some(rendered_buffer) = - web_audio_object_slot(scope, payload, OFFLINE_AUDIO_COMPLETE_BUFFER_SLOT) + let Some(resolver) = + web_audio_object_slot(scope, payload, OFFLINE_AUDIO_COMPLETE_RESOLVER_SLOT) else { return; }; - mark_context_compressors_rendered(scope, context); - - let event = OfflineAudioCompletionEventDeclaration::new("complete", rendered_buffer) + // Only this callback's native payload can provide this private slot. + let resolver = unsafe { v8::Local::::cast_unchecked(resolver) }; + let state = backend::get(scope, context).expect("rendering context must retain its backend"); + let rendered = { + let mut state = state.borrow_mut(); + let State::Context(backend::Context::Offline { context, .. }) = &mut *state else { + unreachable!() + }; + context.start_rendering_sync() + }; + let Some(buffer) = audio_buffer::new_buffer( + scope, + rendered.number_of_channels() as u32, + rendered.length() as u32, + f64::from(rendered.sample_rate()), + ) else { + return; + }; + for channel in 0..rendered.number_of_channels() { + if audio_buffer::write_channel( + scope, + buffer, + channel as u32, + rendered.get_channel_data(channel), + ) + .is_none() + { + return; + } + } + define_non_enumerable_string_property(scope, context, "state", "closed"); + let event = OfflineAudioCompletionEventDeclaration::new("complete", buffer) .bind(scope) - .expect("OfflineAudio completion event declaration should bind"); + .expect("OfflineAudio completion event should bind"); let _ = dispatch_simple_event_target_event( scope, context, @@ -1405,50 +1357,7 @@ fn offline_audio_context_complete_microtask_callback<'s>( "complete", event, ); -} - -fn remember_context_compressor<'s>( - scope: &mut v8::PinScope<'s, '_>, - context: v8::Local<'s, v8::Object>, - node: v8::Local<'s, v8::Object>, -) { - let Some(compressors) = web_audio_object_slot(scope, context, OFFLINE_AUDIO_COMPRESSORS_SLOT) - else { - return; - }; - let length = compressors - .get(scope, v8str(scope, "length").into()) - .and_then(|value| value.uint32_value(scope)) - .unwrap_or(0); - let _ = compressors.set_index(scope, length, node.into()); -} - -fn mark_context_compressors_rendered<'s>( - scope: &mut v8::PinScope<'s, '_>, - context: v8::Local<'s, v8::Object>, -) { - let Some(compressors) = web_audio_object_slot(scope, context, OFFLINE_AUDIO_COMPRESSORS_SLOT) - else { - return; - }; - let length = compressors - .get(scope, v8str(scope, "length").into()) - .and_then(|value| value.uint32_value(scope)) - .unwrap_or(0); - for index in 0..length { - let Some(value) = compressors.get_index(scope, index) else { - continue; - }; - let Ok(node) = v8::Local::::try_from(value) else { - continue; - }; - let reduction = if graph::rendered_with_input(scope, node) { - SYNTHETIC_COMPRESSOR_REDUCTION - } else { - 0.0 - }; - set_web_audio_number_slot(scope, node, DYNAMICS_COMPRESSOR_REDUCTION_SLOT, reduction); - } + let _ = resolver.resolve(scope, buffer.into()); } fn audio_node_connect_callback<'s>( @@ -1469,173 +1378,6 @@ fn audio_node_disconnect_callback<'s>( graph::disconnect(scope, &args); } -fn analyser_get_float_frequency_data_callback<'s>( - scope: &mut v8::PinScope<'s, '_>, - args: v8::FunctionCallbackArguments<'s>, - _rv: v8::ReturnValue<'s, v8::Value>, -) { - copy_analyser_data(scope, &args, AnalyserData::FloatFrequency); -} - -fn analyser_get_float_time_domain_data_callback<'s>( - scope: &mut v8::PinScope<'s, '_>, - args: v8::FunctionCallbackArguments<'s>, - _rv: v8::ReturnValue<'s, v8::Value>, -) { - copy_analyser_data(scope, &args, AnalyserData::FloatTimeDomain); -} - -fn analyser_get_byte_frequency_data_callback<'s>( - scope: &mut v8::PinScope<'s, '_>, - args: v8::FunctionCallbackArguments<'s>, - _rv: v8::ReturnValue<'s, v8::Value>, -) { - copy_analyser_data(scope, &args, AnalyserData::ByteFrequency); -} - -fn analyser_get_byte_time_domain_data_callback<'s>( - scope: &mut v8::PinScope<'s, '_>, - args: v8::FunctionCallbackArguments<'s>, - _rv: v8::ReturnValue<'s, v8::Value>, -) { - copy_analyser_data(scope, &args, AnalyserData::ByteTimeDomain); -} - -fn analyser_fft_size<'s>( - scope: &mut v8::PinScope<'s, '_>, - object: v8::Local<'s, v8::Object>, -) -> Option { - let Some(size) = web_audio_number_slot(scope, object, ANALYSER_FFT_SIZE_SLOT) else { - throw_type_error(scope, "Illegal invocation: expected an AnalyserNode."); - return None; - }; - Some(size as u32) -} - -fn analyser_fft_size_getter_callback<'s>( - scope: &mut v8::PinScope<'s, '_>, - args: v8::FunctionCallbackArguments<'s>, - mut rv: v8::ReturnValue<'s, v8::Value>, -) { - if let Some(size) = analyser_fft_size(scope, args.this()) { - rv.set_uint32(size); - } -} - -fn analyser_fft_size_setter_callback<'s>( - scope: &mut v8::PinScope<'s, '_>, - args: v8::FunctionCallbackArguments<'s>, - _rv: v8::ReturnValue<'s, v8::Value>, -) { - if analyser_fft_size(scope, args.this()).is_none() { - return; - } - let Some(parsed) = webidl::parse_args::(scope, &args) else { - return; - }; - if !(32..=32768).contains(&parsed.value) || !parsed.value.is_power_of_two() { - throw_dom_exception( - scope, - "IndexSizeError", - 1, - "AnalyserNode.fftSize must be a power of two between 32 and 32768.", - ); - return; - } - set_web_audio_number_slot( - scope, - args.this(), - ANALYSER_FFT_SIZE_SLOT, - parsed.value as f64, - ); -} - -fn analyser_frequency_bin_count_getter_callback<'s>( - scope: &mut v8::PinScope<'s, '_>, - args: v8::FunctionCallbackArguments<'s>, - mut rv: v8::ReturnValue<'s, v8::Value>, -) { - if let Some(size) = analyser_fft_size(scope, args.this()) { - rv.set_uint32(size / 2); - } -} - -enum AnalyserData { - FloatFrequency, - FloatTimeDomain, - ByteFrequency, - ByteTimeDomain, -} - -fn copy_analyser_data<'s>( - scope: &mut v8::PinScope<'s, '_>, - args: &v8::FunctionCallbackArguments<'s>, - kind: AnalyserData, -) { - let Some(fft_size) = analyser_fft_size(scope, args.this()) else { - return; - }; - let array: Option> = match kind { - AnalyserData::FloatFrequency | AnalyserData::FloatTimeDomain => { - v8::Local::::try_from(args.get(0)) - .ok() - .map(Into::into) - } - AnalyserData::ByteFrequency | AnalyserData::ByteTimeDomain => { - v8::Local::::try_from(args.get(0)) - .ok() - .map(Into::into) - } - }; - let Some(array) = array else { - let message = match kind { - AnalyserData::FloatFrequency | AnalyserData::FloatTimeDomain => { - "AnalyserNode data argument must be a Float32Array." - } - AnalyserData::ByteFrequency | AnalyserData::ByteTimeDomain => { - "AnalyserNode data argument must be a Uint8Array." - } - }; - throw_type_error(scope, message); - return; - }; - if let Some(store) = array.get_backing_store() - && (store.is_shared() || store.is_resizable_by_user_javascript()) - { - throw_type_error( - scope, - "AnalyserNode data requires a non-shared, fixed-length buffer.", - ); - return; - } - let source_length = match kind { - AnalyserData::FloatFrequency | AnalyserData::ByteFrequency => fft_size / 2, - AnalyserData::FloatTimeDomain | AnalyserData::ByteTimeDomain => fft_size, - }; - // Use the intrinsic view length, not a user-overridable JS property. The - // validated fftSize bounds the work, and excess destination entries stay intact. - let length = array.length().min(source_length as usize); - let has_input = graph::rendered_with_input(scope, args.this()); - for index in 0..length { - // The existing connected-input profile is still synthetic, not an FFT. - // Never expose it before rendering or for a graph with no started source. - let fill = match kind { - AnalyserData::FloatFrequency if !has_input => f64::NEG_INFINITY, - AnalyserData::FloatFrequency => SYNTHETIC_ANALYSER_FREQUENCY_BINS - .get(index) - .or_else(|| SYNTHETIC_ANALYSER_FREQUENCY_BINS.last()) - .copied() - .unwrap_or(-100.0), - AnalyserData::FloatTimeDomain | AnalyserData::ByteFrequency => 0.0, - AnalyserData::ByteTimeDomain => 128.0, - }; - let value = v8::Number::new(scope, fill); - if array.set_index(scope, index as u32, value.into()).is_none() { - return; - } - } -} - fn oscillator_start_callback<'s>( scope: &mut v8::PinScope<'s, '_>, args: v8::FunctionCallbackArguments<'s>, @@ -1644,29 +1386,6 @@ fn oscillator_start_callback<'s>( graph::start_source(scope, &args); } -fn audio_param_set_value_at_time_callback<'s>( - scope: &mut v8::PinScope<'s, '_>, - args: v8::FunctionCallbackArguments<'s>, - mut rv: v8::ReturnValue<'s, v8::Value>, -) { - let param = args.this(); - let Some(parsed) = webidl::parse_args::(scope, &args) else { - return; - }; - let _ = parsed.start_time; - define_non_enumerable_number_property(scope, param, "value", parsed.value); - rv.set(param.into()); -} - -fn audio_context_sample_rate<'s>( - scope: &mut v8::PinScope<'s, '_>, - context: v8::Local<'s, v8::Object>, -) -> f64 { - // Realtime contexts currently have a fixed 44.1 kHz sample rate. Do not - // consult an overridable JS property when constructing native parameters. - web_audio_number_slot(scope, context, OFFLINE_AUDIO_SAMPLE_RATE_SLOT).unwrap_or(44_100.0) -} - fn web_audio_number_slot<'s>( scope: &mut v8::PinScope<'s, '_>, object: v8::Local<'s, v8::Object>, @@ -1712,68 +1431,7 @@ fn audio_destination_node<'s>( .expect("AudioDestinationNode declaration should bind"); graph::initialize_node(scope, node, context); graph::set_destination(scope, context, node); + let native = backend::create_node(scope, context, NodeKind::Destination); + backend::initialize(scope, node, State::Node(native)); node } - -fn build_audio_buffer<'s>( - scope: &mut v8::PinScope<'s, '_>, - channel_count: u32, - length: usize, - sample_rate: f64, - has_input: bool, -) -> Option> { - let Ok(length) = u32::try_from(length) else { - throw_dom_exception( - scope, - "NotSupportedError", - 9, - "AudioBuffer length is too large.", - ); - return None; - }; - let buffer = audio_buffer::new_buffer(scope, channel_count, length, sample_rate)?; - if has_input { - // The existing input renderer remains synthetic. Buffer ownership and - // channel copies are independent of that backend and use actual PCM. - let samples = synthetic_audio_samples(length as usize); - for channel in 0..channel_count { - audio_buffer::write_channel(scope, buffer, channel, &samples)?; - } - } - Some(buffer) -} - -fn synthetic_audio_samples(length: usize) -> Vec { - let mut samples = vec![0.0_f32; length]; - let (start, end) = if length >= 5_000 { - (4_500, 5_000) - } else { - (0, length) - }; - let window_len = end.saturating_sub(start); - if window_len == 0 { - return samples; - } - - let mut weights = Vec::with_capacity(window_len); - for index in 0..window_len { - let x = index as f64; - let weight = (x * 0.137).sin().abs() + 0.31 * (x * 0.053).cos().abs() + 0.07; - weights.push(weight); - } - let total = weights.iter().sum::().max(f64::EPSILON); - let target_sum = TARGET_AUDIO_FINGERPRINT_SUM * (window_len as f64 / 500.0).min(1.0); - let scale = target_sum / total; - for (index, weight) in weights.into_iter().enumerate() { - let signed = if index % 2 == 0 { weight } else { -weight }; - samples[start + index] = (signed * scale) as f32; - } - samples -} - -fn float_to_usize(value: f64) -> Option { - if !value.is_finite() || value < 0.0 || value.fract() != 0.0 { - return None; - } - usize::try_from(value as u64).ok() -} diff --git a/moli-renderer-v8/src/context_bootstrap/web_audio_runtime/analyser.rs b/moli-renderer-v8/src/context_bootstrap/web_audio_runtime/analyser.rs new file mode 100644 index 0000000000..ed0c26f3be --- /dev/null +++ b/moli-renderer-v8/src/context_bootstrap/web_audio_runtime/analyser.rs @@ -0,0 +1,263 @@ +//! Analyser views are copies of native graph history, never fabricated samples. +use super::*; +use web_audio_api::node::AnalyserNode; + +#[derive(WebApiObject)] +#[webapi(interface = "AnalyserNode")] +struct AnalyserObject { + #[webapi(method, length = 1, callback = audio_node_connect_callback)] + connect: (), + #[webapi(method, length = 0, callback = audio_node_disconnect_callback)] + disconnect: (), +} + +#[derive(WebApiFunctionTemplate)] +#[webapi(name = "AnalyserNode", enumerable)] +struct AnalyserPrototype { + #[webapi(accessor_property = "fftSize", getter = fft_size, setter = set_fft_size)] + fft_size: (), + #[webapi(accessor_property = "frequencyBinCount", getter = frequency_bin_count)] + frequency_bin_count: (), + #[webapi(accessor_property, getter = min_decibels, setter = set_min_decibels)] + min_decibels: (), + #[webapi(accessor_property, getter = max_decibels, setter = set_max_decibels)] + max_decibels: (), + #[webapi(accessor_property, getter = smoothing_time_constant, setter = set_smoothing_time_constant)] + smoothing_time_constant: (), + #[webapi(method, length = 1, callback = get_float_frequency_data)] + get_float_frequency_data: (), + #[webapi(method, length = 1, callback = get_float_time_domain_data)] + get_float_time_domain_data: (), + #[webapi(method, length = 1, callback = get_byte_frequency_data)] + get_byte_frequency_data: (), + #[webapi(method, length = 1, callback = get_byte_time_domain_data)] + get_byte_time_domain_data: (), +} + +pub(super) fn install<'s>( + scope: &mut v8::PinScope<'s, '_, ()>, + template: v8::Local<'s, v8::FunctionTemplate>, +) { + AnalyserPrototype::initialize_prototype_template(scope, template.prototype_template(scope)); +} + +pub(super) fn create<'s>( + scope: &mut v8::PinScope<'s, '_>, + context: v8::Local<'s, v8::Object>, +) -> v8::Local<'s, v8::Object> { + let native = backend::create_node(scope, context, NodeKind::Analyser); + let object = AnalyserObject::new() + .bind(scope) + .expect("AnalyserNode should bind"); + graph::initialize_node(scope, object, context); + backend::initialize(scope, object, State::Node(native)); + object +} + +fn with_analyser<'s, T>( + scope: &mut v8::PinScope<'s, '_>, + object: v8::Local<'s, v8::Object>, + operation: impl FnOnce(&mut AnalyserNode) -> T, +) -> Option { + if let Some(state) = backend::get(scope, object) + && let State::Node(Node::Analyser(node)) = &mut *state.borrow_mut() + { + return Some(operation(node)); + } + throw_type_error(scope, "Illegal invocation: expected an AnalyserNode."); + None +} + +fn fft_size<'s>( + scope: &mut v8::PinScope<'s, '_>, + args: v8::FunctionCallbackArguments<'s>, + mut rv: v8::ReturnValue<'s, v8::Value>, +) { + if let Some(size) = with_analyser(scope, args.this(), |node| node.fft_size()) { + rv.set_uint32(size as u32); + } +} + +fn frequency_bin_count<'s>( + scope: &mut v8::PinScope<'s, '_>, + args: v8::FunctionCallbackArguments<'s>, + mut rv: v8::ReturnValue<'s, v8::Value>, +) { + if let Some(size) = with_analyser(scope, args.this(), |node| node.frequency_bin_count()) { + rv.set_uint32(size as u32); + } +} + +fn set_fft_size<'s>( + scope: &mut v8::PinScope<'s, '_>, + args: v8::FunctionCallbackArguments<'s>, + _rv: v8::ReturnValue<'s, v8::Value>, +) { + if with_analyser(scope, args.this(), |_| ()).is_none() { + return; + } + let Some(size) = args.get(0).uint32_value(scope) else { + return; + }; + if !(32..=32768).contains(&size) || !size.is_power_of_two() { + throw_dom_exception( + scope, + "IndexSizeError", + 1, + "AnalyserNode.fftSize must be a power of two between 32 and 32768.", + ); + return; + } + with_analyser(scope, args.this(), |node| node.set_fft_size(size as usize)); +} + +macro_rules! configuration { + ($get:ident, $set:ident, $valid:expr) => { + fn $get<'s>( + scope: &mut v8::PinScope<'s, '_>, + args: v8::FunctionCallbackArguments<'s>, + mut rv: v8::ReturnValue<'s, v8::Value>, + ) { + if let Some(value) = with_analyser(scope, args.this(), |node| node.$get()) { + rv.set(v8::Number::new(scope, value).into()); + } + } + fn $set<'s>( + scope: &mut v8::PinScope<'s, '_>, + args: v8::FunctionCallbackArguments<'s>, + _rv: v8::ReturnValue<'s, v8::Value>, + ) { + if with_analyser(scope, args.this(), |_| ()).is_none() { + return; + } + let Some(value) = args.get(0).number_value(scope) else { + return; + }; + if !value.is_finite() { + throw_type_error(scope, "AnalyserNode configuration must be finite."); + return; + } + let valid = with_analyser(scope, args.this(), |node| { + if !($valid)(node, value) { + return false; + } + node.$set(value); + true + }); + if valid == Some(false) { + throw_dom_exception( + scope, + "IndexSizeError", + 1, + "AnalyserNode configuration is out of range.", + ); + } + } + }; +} +configuration!( + min_decibels, + set_min_decibels, + |node: &AnalyserNode, value| value < node.max_decibels() +); +configuration!( + max_decibels, + set_max_decibels, + |node: &AnalyserNode, value| value > node.min_decibels() +); +configuration!( + smoothing_time_constant, + set_smoothing_time_constant, + |_: &AnalyserNode, value| (0.0..=1.0).contains(&value) +); + +enum Data { + FloatFrequency, + FloatTime, + ByteFrequency, + ByteTime, +} +macro_rules! data_method { + ($name:ident, $kind:ident) => { + fn $name<'s>( + scope: &mut v8::PinScope<'s, '_>, + args: v8::FunctionCallbackArguments<'s>, + _rv: v8::ReturnValue<'s, v8::Value>, + ) { + copy_data(scope, &args, Data::$kind); + } + }; +} +data_method!(get_float_frequency_data, FloatFrequency); +data_method!(get_float_time_domain_data, FloatTime); +data_method!(get_byte_frequency_data, ByteFrequency); +data_method!(get_byte_time_domain_data, ByteTime); + +fn copy_data<'s>( + scope: &mut v8::PinScope<'s, '_>, + args: &v8::FunctionCallbackArguments<'s>, + kind: Data, +) { + let Some(size) = with_analyser(scope, args.this(), |node| node.fft_size()) else { + return; + }; + let array: Option> = match kind { + Data::FloatFrequency | Data::FloatTime => { + v8::Local::::try_from(args.get(0)) + .ok() + .map(Into::into) + } + Data::ByteFrequency | Data::ByteTime => v8::Local::::try_from(args.get(0)) + .ok() + .map(Into::into), + }; + let Some(array) = array else { + throw_type_error( + scope, + "AnalyserNode requires the matching Float32Array or Uint8Array.", + ); + return; + }; + if let Some(store) = array.get_backing_store() + && (store.is_shared() || store.is_resizable_by_user_javascript()) + { + throw_type_error( + scope, + "AnalyserNode data requires a non-shared, fixed-length buffer.", + ); + return; + } + let count = match kind { + Data::FloatFrequency | Data::ByteFrequency => size / 2, + _ => size, + }; + let count = count.min(array.length()); + let Some(values) = with_analyser(scope, args.this(), |node| match kind { + Data::FloatFrequency | Data::FloatTime => { + let mut values = vec![0.0_f32; count]; + match kind { + Data::FloatFrequency => node.get_float_frequency_data(&mut values), + _ => node.get_float_time_domain_data(&mut values), + } + values.into_iter().map(f64::from).collect::>() + } + Data::ByteFrequency | Data::ByteTime => { + let mut values = vec![0_u8; count]; + match kind { + Data::ByteFrequency => node.get_byte_frequency_data(&mut values), + _ => node.get_byte_time_domain_data(&mut values), + } + values.into_iter().map(f64::from).collect::>() + } + }) else { + return; + }; + // Numeric indexed writes to a validated fixed typed array cannot invoke + // page getters. The intrinsic length respects subarrays and detached views. + for (index, value) in values.into_iter().enumerate() { + let value = v8::Number::new(scope, value); + if array.set_index(scope, index as u32, value.into()).is_none() { + return; + } + } +} diff --git a/moli-renderer-v8/src/context_bootstrap/web_audio_runtime/audio_param.rs b/moli-renderer-v8/src/context_bootstrap/web_audio_runtime/audio_param.rs index 5e860b9d3f..6cdd1572a7 100644 --- a/moli-renderer-v8/src/context_bootstrap/web_audio_runtime/audio_param.rs +++ b/moli-renderer-v8/src/context_bootstrap/web_audio_runtime/audio_param.rs @@ -1,27 +1,21 @@ use super::*; -const DEFAULT_VALUE: &str = "__moliAudioParamDefaultValue"; -const MIN_VALUE: &str = "__moliAudioParamMinValue"; -const MAX_VALUE: &str = "__moliAudioParamMaxValue"; - +// The wrapper retains the actual graph parameter. Public property overrides do +// not replace the value consumed by the audio render thread. #[derive(WebApiObject)] #[webapi(interface = "AudioParam")] struct AudioParamObjectDeclaration { - #[webapi(data_property)] - value: f64, - #[webapi(slot = DEFAULT_VALUE)] - default_value: f64, - #[webapi(slot = MIN_VALUE)] - min_value: f64, - #[webapi(slot = MAX_VALUE)] - max_value: f64, - #[webapi(method, length = 2, callback = audio_param_set_value_at_time_callback)] + #[webapi(method, length = 2, callback = set_value_at_time)] set_value_at_time: (), } #[derive(WebApiFunctionTemplate)] #[webapi(name = "AudioParam", enumerable)] struct AudioParamPrototypeDeclaration { + #[webapi(accessor_property, getter = automation_rate, setter = set_automation_rate)] + automation_rate: (), + #[webapi(accessor_property, getter = value, setter = set_value)] + value: (), #[webapi(accessor_property, getter = default_value)] default_value: (), #[webapi(accessor_property, getter = min_value)] @@ -40,56 +34,146 @@ pub(super) fn install<'s>( ); } -pub(super) fn audio_param<'s>( +pub(super) fn wrap<'s>( scope: &mut v8::PinScope<'s, '_>, - value: f64, - min: f64, - max: f64, + param: backend::Parameter, ) -> v8::Local<'s, v8::Object> { - let value = value as f32 as f64; - AudioParamObjectDeclaration::new(value, value, min as f32 as f64, max as f32 as f64) + let object = AudioParamObjectDeclaration::new() .bind(scope) - .expect("AudioParam declaration should bind") + .expect("AudioParam declaration should bind"); + backend::initialize(scope, object, backend::State::Param(param)); + object } -pub(super) fn detune_param<'s>(scope: &mut v8::PinScope<'s, '_>) -> v8::Local<'s, v8::Object> { - let limit = (1200.0_f32 * f32::MAX.log2()) as f64; - audio_param(scope, 0.0, -limit, limit) -} - -fn read_metadata<'s>( +fn automation_rate<'s>( scope: &mut v8::PinScope<'s, '_>, args: v8::FunctionCallbackArguments<'s>, mut rv: v8::ReturnValue<'s, v8::Value>, - slot: &'static str, ) { - let Some(value) = web_audio_number_slot(scope, args.this(), slot) else { - throw_type_error(scope, "Illegal invocation: expected an AudioParam."); - return; - }; - rv.set(v8::Number::new(scope, value).into()); + if let Some(param) = backend::param(scope, args.this()) { + let value = match param.automation_rate() { + web_audio_api::AutomationRate::A => "a-rate", + web_audio_api::AutomationRate::K => "k-rate", + }; + rv.set(v8str(scope, value).into()); + } } +fn set_automation_rate<'s>( + scope: &mut v8::PinScope<'s, '_>, + args: v8::FunctionCallbackArguments<'s>, + _rv: v8::ReturnValue<'s, v8::Value>, +) { + let Some(param) = backend::param(scope, args.this()) else { + return; + }; + let Some(value) = args.get(0).to_string(scope) else { + return; + }; + let value = value.to_rust_string_lossy(scope); + let rate = match value.as_str() { + "a-rate" => web_audio_api::AutomationRate::A, + "k-rate" => web_audio_api::AutomationRate::K, + _ => { + throw_type_error(scope, "Invalid AutomationRate."); + return; + } + }; + if !param.set_automation_rate(rate) { + throw_dom_exception( + scope, + "InvalidStateError", + 11, + "This AudioParam has a fixed automation rate.", + ); + } +} + +fn read<'s>( + scope: &mut v8::PinScope<'s, '_>, + args: v8::FunctionCallbackArguments<'s>, + mut rv: v8::ReturnValue<'s, v8::Value>, + get: fn(&backend::Parameter) -> f32, +) { + if let Some(param) = backend::param(scope, args.this()) { + rv.set(v8::Number::new(scope, f64::from(get(¶m))).into()); + } +} + +fn value<'s>( + scope: &mut v8::PinScope<'s, '_>, + args: v8::FunctionCallbackArguments<'s>, + rv: v8::ReturnValue<'s, v8::Value>, +) { + read(scope, args, rv, backend::Parameter::value); +} fn default_value<'s>( scope: &mut v8::PinScope<'s, '_>, args: v8::FunctionCallbackArguments<'s>, rv: v8::ReturnValue<'s, v8::Value>, ) { - read_metadata(scope, args, rv, DEFAULT_VALUE); + read(scope, args, rv, backend::Parameter::default_value); } - fn min_value<'s>( scope: &mut v8::PinScope<'s, '_>, args: v8::FunctionCallbackArguments<'s>, rv: v8::ReturnValue<'s, v8::Value>, ) { - read_metadata(scope, args, rv, MIN_VALUE); + read(scope, args, rv, backend::Parameter::min_value); } - fn max_value<'s>( scope: &mut v8::PinScope<'s, '_>, args: v8::FunctionCallbackArguments<'s>, rv: v8::ReturnValue<'s, v8::Value>, ) { - read_metadata(scope, args, rv, MAX_VALUE); + read(scope, args, rv, backend::Parameter::max_value); +} + +fn finite_float<'s>( + scope: &mut v8::PinScope<'s, '_>, + value: v8::Local<'s, v8::Value>, +) -> Option { + let value = value.number_value(scope)? as f32; + if !value.is_finite() { + throw_type_error(scope, "AudioParam value must be a finite float."); + return None; + } + Some(value) +} + +fn set_value<'s>( + scope: &mut v8::PinScope<'s, '_>, + args: v8::FunctionCallbackArguments<'s>, + _rv: v8::ReturnValue<'s, v8::Value>, +) { + let Some(param) = backend::param(scope, args.this()) else { + return; + }; + let Some(value) = finite_float(scope, args.get(0)) else { + return; + }; + param.set_value(value); +} + +fn set_value_at_time<'s>( + scope: &mut v8::PinScope<'s, '_>, + args: v8::FunctionCallbackArguments<'s>, + mut rv: v8::ReturnValue<'s, v8::Value>, +) { + let Some(param) = backend::param(scope, args.this()) else { + return; + }; + let Some(parsed) = webidl::parse_args::(scope, &args) else { + return; + }; + if !(parsed.value as f32).is_finite() || !parsed.start_time.is_finite() { + throw_type_error(scope, "AudioParam value and time must be finite."); + return; + } + if parsed.start_time < 0.0 { + throw_range_error(scope, "AudioParam start time must not be negative."); + return; + } + param.set_value_at_time(parsed.value as f32, parsed.start_time); + rv.set(args.this().into()); } diff --git a/moli-renderer-v8/src/context_bootstrap/web_audio_runtime/backend.rs b/moli-renderer-v8/src/context_bootstrap/web_audio_runtime/backend.rs new file mode 100644 index 0000000000..a5f66e9e48 --- /dev/null +++ b/moli-renderer-v8/src/context_bootstrap/web_audio_runtime/backend.rs @@ -0,0 +1,467 @@ +//! Native Web Audio state. The JS wrapper, not a page-global strong root, owns +//! its lifetime. GC reclaims abandoned nodes/contexts; isolate drop also drains +//! the store. No raw pointers or JavaScript-visible state IDs are exposed. + +use crate::util::{get_private_value, set_private_value}; +use std::{ + cell::RefCell, + collections::HashMap, + rc::Rc, + sync::{ + Arc, + atomic::{AtomicBool, AtomicU32, Ordering}, + }, +}; +use web_audio_api::context::{ + AudioContext, AudioContextOptions, BaseAudioContext, OfflineAudioContext, +}; +use web_audio_api::node::{AnalyserNode, AudioDestinationNode, AudioNode, BiquadFilterNode}; +use web_audio_api::worklet::{ + AudioParamValues, AudioWorkletGlobalScope, AudioWorkletNode, AudioWorkletNodeOptions, + AudioWorkletProcessor, +}; +use web_audio_api::{AudioParam, AudioParamDescriptor, AutomationRate}; + +const SLOT: &str = "__moliWebAudioBackend"; +type Store = Rc>; +pub(super) type StateRef = Rc>; + +#[derive(Default)] +struct States { + next_id: u64, + entries: HashMap, StateRef)>, +} + +pub(super) enum State { + Context(Context), + Node(Node), + Param(Parameter), + // Existing JS worklets load modules and exchange messages, but do not yet + // run process() on the audio thread. Preserve their control graph without + // inventing a silent or pass-through native processor. + ModuleWorklet, +} + +/// The nominal range can depend on the context's sample rate, whereas an +/// AudioWorklet's descriptors are static. Keep that immutable range alongside +/// its actual timeline; the oscillator also applies it at a-rate on the DSP side. +#[derive(Clone)] +pub(super) struct Parameter { + native: AudioParam, + minimum: f32, + maximum: f32, + fixed_rate: Option, +} + +impl Parameter { + pub(super) fn new(native: AudioParam) -> Self { + let minimum = native.min_value(); + let maximum = native.max_value(); + Self { + native, + minimum, + maximum, + fixed_rate: None, + } + } + pub(super) fn with_range(mut self, minimum: f32, maximum: f32) -> Self { + self.minimum = minimum; + self.maximum = maximum; + self + } + pub(super) fn automation_rate(&self) -> AutomationRate { + self.fixed_rate + .unwrap_or_else(|| self.native.automation_rate()) + } + pub(super) fn set_automation_rate(&self, rate: AutomationRate) -> bool { + if self.fixed_rate.is_some() { + return false; + } + self.native.set_automation_rate(rate); + true + } + pub(super) fn value(&self) -> f32 { + self.native.value().clamp(self.minimum, self.maximum) + } + pub(super) fn default_value(&self) -> f32 { + self.native.default_value() + } + pub(super) fn min_value(&self) -> f32 { + self.minimum + } + pub(super) fn max_value(&self) -> f32 { + self.maximum + } + pub(super) fn set_value(&self, value: f32) { + self.native.set_value(value); + } + pub(super) fn set_value_at_time(&self, value: f32, when: f64) { + self.native.set_value_at_time(value, when); + } +} + +pub(super) enum Context { + Realtime(Box), + Offline { + context: Box, + rendered: bool, + }, +} + +impl Drop for Context { + fn drop(&mut self) { + if let Self::Realtime(context) = self { + // The upstream AudioContext intentionally keeps playback alive on + // drop. A collected wrapper or disposed document must instead stop + // its headless audio thread, including contexts never explicitly + // closed by page code. + context.close_sync(); + } + } +} + +#[derive(Clone, Copy)] +pub(super) enum NodeKind { + Destination, + Oscillator, + Compressor, + Biquad, + Analyser, +} + +pub(super) enum Node { + Destination(AudioDestinationNode), + Oscillator(super::oscillator_dsp::Oscillator), + Compressor { + node: AudioWorkletNode, + reduction: Arc, + demanded: Arc, + }, + Biquad(BiquadFilterNode), + Analyser(Box), +} + +impl Context { + pub(super) fn realtime() -> Self { + // Headless processing must not open the host's speakers or microphone. + Self::Realtime(Box::new(AudioContext::new(AudioContextOptions { + sample_rate: Some(44_100.0), + sink_id: "none".into(), + ..Default::default() + }))) + } + + pub(super) fn offline(channels: usize, length: usize, sample_rate: f32) -> Self { + Self::Offline { + context: Box::new(OfflineAudioContext::new(channels, length, sample_rate)), + rendered: false, + } + } + + pub(super) fn create_node(&self, kind: NodeKind) -> Node { + match self { + Self::Realtime(context) => Node::create(context.as_ref(), kind), + Self::Offline { context, .. } => Node::create(context.as_ref(), kind), + } + } + + pub(super) fn current_time(&self) -> f64 { + match self { + Self::Realtime(context) => context.current_time(), + Self::Offline { context, .. } => context.current_time(), + } + } +} + +impl Node { + fn create(context: &C, kind: NodeKind) -> Self { + match kind { + NodeKind::Destination => Self::Destination(context.destination()), + NodeKind::Oscillator => { + Self::Oscillator(super::oscillator_dsp::Oscillator::new(context)) + } + NodeKind::Biquad => Self::Biquad(context.create_biquad_filter()), + NodeKind::Analyser => Self::Analyser(Box::new(context.create_analyser())), + NodeKind::Compressor => { + let reduction = Arc::new(AtomicU32::new(0)); + let demanded = Arc::new(AtomicBool::new(false)); + let node = AudioWorkletNode::new::( + context, + AudioWorkletNodeOptions { + processor_options: ( + context.sample_rate(), + Arc::clone(&reduction), + Arc::clone(&demanded), + ), + output_channel_count: vec![2], + ..Default::default() + }, + ); + Self::Compressor { + node, + reduction, + demanded, + } + } + } + } + + pub(super) fn audio_node(&self) -> &dyn AudioNode { + match self { + Self::Destination(node) => node, + Self::Oscillator(node) => &node.node, + Self::Compressor { node, .. } => node, + Self::Biquad(node) => node, + Self::Analyser(node) => node.as_ref(), + } + } + + pub(super) fn param(&self, name: &str) -> Parameter { + let native = match self { + Self::Oscillator(node) => match name { + "frequency" => node.frequency().clone(), + "detune" => node.detune().clone(), + _ => unreachable!("unknown oscillator parameter"), + }, + Self::Compressor { node, .. } => node.parameters()[name].clone(), + Self::Biquad(node) => match name { + "frequency" => node.frequency().clone(), + "detune" => node.detune().clone(), + "Q" => node.q().clone(), + "gain" => node.gain().clone(), + _ => unreachable!("unknown biquad parameter"), + }, + _ => unreachable!("node has no parameters"), + }; + let mut parameter = Parameter::new(native); + match self { + Self::Oscillator(node) if name == "frequency" => { + let nyquist = node.node.context().sample_rate() / 2.0; + parameter = parameter.with_range(-nyquist, nyquist); + } + Self::Compressor { .. } => parameter.fixed_rate = Some(AutomationRate::K), + _ => {} + } + parameter + } +} + +struct CompressorProcessor { + dsp: super::compressor::Compressor, + reduction: Arc, + demanded: Arc, + sample_rate: f32, + tail_frames: usize, +} + +impl AudioWorkletProcessor for CompressorProcessor { + type ProcessorOptions = (f32, Arc, Arc); + + fn constructor((sample_rate, reduction, demanded): Self::ProcessorOptions) -> Self { + Self { + dsp: super::compressor::Compressor::new(sample_rate), + reduction, + demanded, + sample_rate, + tail_frames: 0, + } + } + + fn parameter_descriptors() -> Vec { + [ + ("threshold", -24.0, -100.0, 0.0), + ("knee", 30.0, 0.0, 40.0), + ("ratio", 12.0, 1.0, 20.0), + ("attack", 0.003, 0.0, 1.0), + ("release", 0.25, 0.0, 1.0), + ] + .into_iter() + .map( + |(name, default_value, min_value, max_value)| AudioParamDescriptor { + name: name.into(), + default_value, + min_value, + max_value, + automation_rate: AutomationRate::K, + }, + ) + .collect() + } + + fn process<'a, 'b>( + &mut self, + inputs: &'b [&'a [&'a [f32]]], + outputs: &'b mut [&'a mut [&'a mut [f32]]], + params: AudioParamValues<'b>, + _scope: &'b AudioWorkletGlobalScope, + ) -> bool { + // The backend processes orphan nodes too. Built-in compressors follow + // Blink's pull demand; they are not automatically pulled worklets. + if !self.demanded.load(Ordering::Acquire) { + for channel in outputs[0].iter_mut() { + channel.fill(0.0); + } + return false; + } + let input = inputs[0]; + if input.is_empty() { + self.tail_frames += outputs[0][0].len(); + } else { + self.tail_frames = 0; + } + let values = + ["threshold", "knee", "ratio", "attack", "release"].map(|name| params.get(name)[0]); + self.dsp.process(input, outputs[0], values); + self.reduction + .store(self.dsp.reduction().to_bits(), Ordering::Relaxed); + self.tail_frames < (1.625 * self.sample_rate) as usize + } +} + +pub(super) fn initialize<'s>( + scope: &mut v8::PinScope<'s, '_>, + owner: v8::Local<'s, v8::Object>, + state: State, +) { + let store = scope.get_slot::().cloned().unwrap_or_else(|| { + let store = Store::default(); + scope.set_slot(store.clone()); + store + }); + let id = { + let mut store = store.borrow_mut(); + store.next_id = store + .next_id + .checked_add(1) + .expect("audio identity exhausted"); + store.next_id + }; + let weak_store = Rc::downgrade(&store); + let weak = v8::Weak::with_finalizer( + scope, + owner, + Box::new(move |_| { + if let Some(store) = weak_store.upgrade() { + store.borrow_mut().entries.remove(&id); + } + }), + ); + store + .borrow_mut() + .entries + .insert(id, (weak, Rc::new(RefCell::new(state)))); + set_private_value( + scope, + owner, + SLOT, + v8::BigInt::new_from_u64(scope, id).into(), + ); +} + +pub(super) fn get<'s>( + scope: &mut v8::PinScope<'s, '_>, + owner: v8::Local<'s, v8::Object>, +) -> Option { + let id = v8::Local::::try_from(get_private_value(scope, owner, SLOT)?) + .ok()? + .u64_value() + .0; + scope + .get_slot::()? + .borrow() + .entries + .get(&id) + .map(|entry| entry.1.clone()) +} + +pub(super) fn create_node<'s>( + scope: &mut v8::PinScope<'s, '_>, + context: v8::Local<'s, v8::Object>, + kind: NodeKind, +) -> Node { + let state = get(scope, context).expect("validated audio context must retain its backend"); + let state = state.borrow(); + let State::Context(context) = &*state else { + unreachable!("audio context brand mismatch") + }; + context.create_node(kind) +} + +pub(super) fn param<'s>( + scope: &mut v8::PinScope<'s, '_>, + object: v8::Local<'s, v8::Object>, +) -> Option { + if let Some(state) = get(scope, object) + && let State::Param(param) = &*state.borrow() + { + return Some(param.clone()); + } + super::throw_type_error(scope, "Illegal invocation: expected an AudioParam."); + None +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn dropping_realtime_context_closes_its_render_graph() { + let context = Context::realtime(); + let destination = context.create_node(NodeKind::Destination); + drop(context); + assert_eq!( + destination.audio_node().context().state(), + web_audio_api::context::AudioContextState::Closed + ); + } + + #[test] + fn audio_backend_wrappers_are_reclaimed_on_gc_and_isolate_drop() { + moli_v8_test_util::ensure_v8(); + let remaining = { + let mut isolate = v8::Isolate::new(Default::default()); + let abandoned = { + let scope = std::pin::pin!(v8::HandleScope::new(&mut isolate)); + let scope = &mut scope.init(); + let context = v8::Context::new(scope, Default::default()); + let scope = &mut v8::ContextScope::new(scope, context); + let context = v8::Object::new(scope); + initialize( + scope, + context, + State::Context(Context::offline(1, 128, 8000.0)), + ); + let node = create_node(scope, context, NodeKind::Oscillator); + let param = node.param("frequency"); + let owner = v8::Object::new(scope); + initialize(scope, owner, State::Node(node)); + let parameter = v8::Object::new(scope); + initialize(scope, parameter, State::Param(param)); + [context, owner, parameter] + .map(|object| Rc::downgrade(&get(scope, object).unwrap())) + }; + isolate.low_memory_notification(); + assert!(abandoned.iter().all(|state| state.upgrade().is_none())); + assert!( + isolate + .get_slot::() + .unwrap() + .borrow() + .entries + .is_empty() + ); + let scope = std::pin::pin!(v8::HandleScope::new(&mut isolate)); + let scope = &mut scope.init(); + let context = v8::Context::new(scope, Default::default()); + let scope = &mut v8::ContextScope::new(scope, context); + let object = v8::Object::new(scope); + initialize( + scope, + object, + State::Context(Context::offline(1, 128, 8000.0)), + ); + Rc::downgrade(&get(scope, object).unwrap()) + }; + assert!(remaining.upgrade().is_none()); + } +} diff --git a/moli-renderer-v8/src/context_bootstrap/web_audio_runtime/biquad.rs b/moli-renderer-v8/src/context_bootstrap/web_audio_runtime/biquad.rs index 1897ed7dd2..7458fef8e1 100644 --- a/moli-renderer-v8/src/context_bootstrap/web_audio_runtime/biquad.rs +++ b/moli-renderer-v8/src/context_bootstrap/web_audio_runtime/biquad.rs @@ -1,16 +1,11 @@ use super::*; - -const BIQUAD_FREQUENCY_SLOT: &str = "__moliBiquadFrequency"; +use web_audio_api::node::BiquadFilterType; #[derive(WebApiObject)] #[webapi(interface = "BiquadFilterNode")] struct BiquadFilterNodeObjectDeclaration<'scope> { - #[webapi(data_property = "type")] - kind: &'static str, #[webapi(data_property, readonly)] frequency: v8::Local<'scope, v8::Object>, - #[webapi(slot = BIQUAD_FREQUENCY_SLOT)] - internal_frequency: v8::Local<'scope, v8::Object>, #[webapi(data_property, readonly)] detune: v8::Local<'scope, v8::Object>, #[webapi(data_property = "Q", readonly)] @@ -26,6 +21,8 @@ struct BiquadFilterNodeObjectDeclaration<'scope> { #[derive(WebApiFunctionTemplate)] #[webapi(name = "BiquadFilterNode", enumerable)] struct BiquadFilterNodePrototypeDeclaration { + #[webapi(accessor_property = "type", getter = kind, setter = set_kind)] + kind: (), #[webapi(method, length = 3, callback = get_frequency_response)] get_frequency_response: (), } @@ -48,36 +45,156 @@ pub(super) fn create_biquad_filter<'s>( if !require_base_audio_context(scope, args.this()) { return; } - // Match Blink's BiquadFilterNode parameter defaults and float bounds. - let nyquist = audio_context_sample_rate(scope, args.this()) / 2.0; - let frequency = audio_param(scope, 350.0, 0.0, nyquist); - let detune = detune_param(scope); - let q = audio_param(scope, 1.0, f32::MIN as f64, f32::MAX as f64); - let gain_max = (40.0_f32 * f32::MAX.log10()) as f64; - let gain = audio_param(scope, 0.0, f32::MIN as f64, gain_max); - let node = - BiquadFilterNodeObjectDeclaration::new("lowpass", frequency, frequency, detune, q, gain) - .bind(scope) - .expect("BiquadFilterNode declaration should bind"); + let native = backend::create_node(scope, args.this(), NodeKind::Biquad); + let frequency = audio_param::wrap(scope, native.param("frequency")); + let detune = audio_param::wrap(scope, native.param("detune")); + let q = audio_param::wrap(scope, native.param("Q")); + let gain = audio_param::wrap(scope, native.param("gain")); + let node = BiquadFilterNodeObjectDeclaration::new(frequency, detune, q, gain) + .bind(scope) + .expect("BiquadFilterNode declaration should bind"); graph::initialize_node(scope, node, args.this()); + backend::initialize(scope, node, State::Node(native)); rv.set(node.into()); } +fn state<'s>( + scope: &mut v8::PinScope<'s, '_>, + object: v8::Local<'s, v8::Object>, +) -> Option { + if let Some(state) = backend::get(scope, object) + && matches!(&*state.borrow(), State::Node(Node::Biquad(_))) + { + return Some(state); + } + throw_type_error(scope, "Illegal invocation: expected a BiquadFilterNode."); + None +} + +const TYPES: [(&str, BiquadFilterType); 8] = [ + ("lowpass", BiquadFilterType::Lowpass), + ("highpass", BiquadFilterType::Highpass), + ("bandpass", BiquadFilterType::Bandpass), + ("lowshelf", BiquadFilterType::Lowshelf), + ("highshelf", BiquadFilterType::Highshelf), + ("peaking", BiquadFilterType::Peaking), + ("notch", BiquadFilterType::Notch), + ("allpass", BiquadFilterType::Allpass), +]; + +fn kind<'s>( + scope: &mut v8::PinScope<'s, '_>, + args: v8::FunctionCallbackArguments<'s>, + mut rv: v8::ReturnValue<'s, v8::Value>, +) { + let Some(state) = state(scope, args.this()) else { + return; + }; + if let State::Node(Node::Biquad(node)) = &*state.borrow() { + let name = TYPES + .iter() + .find(|(_, kind)| *kind == node.type_()) + .unwrap() + .0; + rv.set(v8str(scope, name).into()); + } +} + +fn set_kind<'s>( + scope: &mut v8::PinScope<'s, '_>, + args: v8::FunctionCallbackArguments<'s>, + _rv: v8::ReturnValue<'s, v8::Value>, +) { + let Some(state) = state(scope, args.this()) else { + return; + }; + let Some(value) = args.get(0).to_string(scope) else { + return; + }; + let value = value.to_rust_string_lossy(scope); + let Some((_, kind)) = TYPES.iter().find(|(name, _)| *name == value) else { + throw_type_error(scope, "Invalid BiquadFilterType."); + return; + }; + if let State::Node(Node::Biquad(node)) = &mut *state.borrow_mut() { + node.set_type(*kind); + } +} + +fn float_array<'s>( + scope: &mut v8::PinScope<'s, '_>, + value: v8::Local<'s, v8::Value>, +) -> Option> { + if let Ok(array) = v8::Local::::try_from(value) + && array + .get_backing_store() + .is_none_or(|store| !store.is_shared() && !store.is_resizable_by_user_javascript()) + { + return Some(array); + } + throw_type_error( + scope, + "BiquadFilterNode requires non-shared, fixed-length Float32Arrays.", + ); + None +} + fn get_frequency_response<'s>( scope: &mut v8::PinScope<'s, '_>, args: v8::FunctionCallbackArguments<'s>, _rv: v8::ReturnValue<'s, v8::Value>, ) { - if get_private_value(scope, args.this(), BIQUAD_FREQUENCY_SLOT).is_none() { - throw_type_error(scope, "Illegal invocation: expected a BiquadFilterNode."); + let Some(state) = state(scope, args.this()) else { + return; + }; + let Some(frequencies) = float_array(scope, args.get(0)) else { + return; + }; + let Some(magnitudes) = float_array(scope, args.get(1)) else { + return; + }; + let Some(phases) = float_array(scope, args.get(2)) else { + return; + }; + let length = frequencies.length(); + if magnitudes.length() != length || phases.length() != length { + throw_dom_exception( + scope, + "InvalidAccessError", + 15, + "Frequency response arrays must have equal lengths.", + ); return; } - // Node/AudioParam metadata is available independently of the DSP backend. - // Do not silently return fabricated response curves for an unimplemented filter. - throw_dom_exception( - scope, - "NotSupportedError", - 9, - "Biquad frequency response calculation is not implemented.", - ); + let mut frequency_values = Vec::with_capacity(length); + for index in 0..length { + let Some(value) = frequencies + .get_index(scope, index as u32) + .and_then(|value| value.number_value(scope)) + else { + return; + }; + frequency_values.push(value as f32); + } + let mut magnitude_values = vec![0.0; length]; + let mut phase_values = vec![0.0; length]; + if let State::Node(Node::Biquad(node)) = &*state.borrow() { + node.get_frequency_response(&frequency_values, &mut magnitude_values, &mut phase_values); + } + for index in 0..length { + let magnitude = v8::Number::new(scope, f64::from(magnitude_values[index])); + let phase = v8::Number::new(scope, f64::from(phase_values[index])); + if magnitudes + .set_index(scope, index as u32, magnitude.into()) + .is_none() + { + return; + } + if phases + .set_index(scope, index as u32, phase.into()) + .is_none() + { + return; + } + } } diff --git a/moli-renderer-v8/src/context_bootstrap/web_audio_runtime/compressor.rs b/moli-renderer-v8/src/context_bootstrap/web_audio_runtime/compressor.rs new file mode 100644 index 0000000000..f31f07c81d --- /dev/null +++ b/moli-renderer-v8/src/context_bootstrap/web_audio_runtime/compressor.rs @@ -0,0 +1,237 @@ +// Copyright (C) 2011 Google Inc. All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are met: +// 1. Redistributions of source code must retain the above copyright notice, +// this list of conditions and the following disclaimer. +// 2. Redistributions in binary form must reproduce the above copyright notice, +// this list of conditions and the following disclaimer in the documentation +// and/or other materials provided with the distribution. +// 3. Neither the name of Apple Computer, Inc. ("Apple") nor the names of its +// contributors may be used to endorse or promote products derived from this +// software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY APPLE AND ITS CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +// DISCLAIMED. IN NO EVENT SHALL APPLE OR ITS CONTRIBUTORS BE LIABLE FOR ANY +// DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +// (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +// LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +// ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +//! Stateful look-ahead compressor, ported from Blink's +//! `third_party/blink/renderer/platform/audio/dynamics_compressor.cc`. +//! +//! Keep the detector, 32-frame envelope updates, stereo link and metering on +//! the same samples that reach the output. No browser fingerprint constants. + +const DELAY_CAPACITY: usize = 1024; + +pub(super) struct Compressor { + sample_rate: f32, + delay: [Vec; 2], + read: usize, + write: usize, + detector: f32, + gain: f32, + meter: f32, + meter_release: f32, + attack_difference: f32, + curve: Curve, +} + +struct Curve { + parameters: [f32; 3], + threshold: f32, + knee_threshold: f32, + knee_output_db: f32, + knee: f32, + slope: f32, +} + +fn to_db(value: f32) -> f32 { + 20.0 * value.log10() +} + +fn from_db(value: f32) -> f32 { + 10.0_f32.powf(0.05 * value) +} + +fn finite(value: f32, fallback: f32) -> f32 { + if value.is_finite() { value } else { fallback } +} + +impl Curve { + fn new(parameters: [f32; 3]) -> Self { + let [threshold_db, knee_db, ratio] = parameters; + let threshold = from_db(threshold_db); + let knee_threshold = from_db(threshold_db + knee_db); + let mut curve = Self { + parameters, + threshold, + knee_threshold, + knee_output_db: 0.0, + knee: 5.0, + slope: 1.0 / ratio, + }; + let mut min = 0.1_f32; + let mut max = 10_000.0_f32; + let x2 = (knee_threshold as f64 * 1.001) as f32; + for _ in 0..15 { + let slope = (to_db(curve.knee_curve(x2)) - to_db(curve.knee_curve(knee_threshold))) + / (to_db(x2) - (threshold_db + knee_db)); + if slope < curve.slope { + max = curve.knee; + } else { + min = curve.knee; + } + curve.knee = (min * max).sqrt(); + } + curve.knee_output_db = to_db(curve.knee_curve(knee_threshold)); + curve + } + + fn knee_curve(&self, input: f32) -> f32 { + if input < self.threshold { + return input; + } + self.threshold + + (1.0 - f64::from(-self.knee * (input - self.threshold)).exp() as f32) / self.knee + } + + fn saturate(&self, input: f32) -> f32 { + if input < self.knee_threshold { + self.knee_curve(input) + } else { + from_db( + self.knee_output_db + + self.slope * (to_db(input) - (self.parameters[0] + self.parameters[1])), + ) + } + } +} + +impl Compressor { + pub(super) fn new(sample_rate: f32) -> Self { + Self { + sample_rate, + delay: std::array::from_fn(|_| vec![0.0; DELAY_CAPACITY]), + read: 0, + write: ((0.006_f32 * sample_rate) as usize).min(DELAY_CAPACITY - 1), + detector: 0.0, + gain: 1.0, + meter: 1.0, + meter_release: (1.0 - (-1.0 / (0.325_f32 as f64 * sample_rate as f64)).exp()) as f32, + attack_difference: -1.0, + curve: Curve::new([-24.0, 30.0, 12.0]), + } + } + + pub(super) fn reduction(&self) -> f32 { + self.meter + } + + /// Process a render quantum (a multiple of 32 samples), in mono or stereo. + /// `parameters` are threshold, knee, ratio, attack and release, sampled at + /// the start of the quantum as required by k-rate AudioParams. + pub(super) fn process( + &mut self, + input: &[&[f32]], + output: &mut [&mut [f32]], + parameters: [f32; 5], + ) { + let [threshold, knee, ratio, attack, release] = parameters; + if self.curve.parameters != [threshold, knee, ratio] { + self.curve = Curve::new([threshold, knee, ratio]); + } + let post_gain = (1.0 / self.curve.saturate(1.0)).powf(0.6); + let attack_frames = attack.max(0.001) * self.sample_rate; + let release_frames = release * self.sample_rate; + let sat_release_frames = 0.0025 * self.sample_rate; + let zones = [0.09_f32, 0.16, 0.42, 0.98]; + let [z1, z2, z3, z4] = zones; + let a = release_frames * z1; + let b = release_frames + * (-1.578_832 * z1 + 2.330_583_8 * z2 - 0.914_119_4 * z3 + 0.162_367_75 * z4); + let c = release_frames + * (0.533_414_3 * z1 - 1.272_736_8 * z2 + 0.925_885_6 * z3 - 0.186_563_1 * z4); + let d = release_frames + * (0.087_834_634 * z1 - 0.169_416_3 * z2 + 0.085_880_58 * z3 - 0.004_298_914 * z4); + let e = release_frames + * (-0.042_416_885 * z1 + 0.111_569_38 * z2 - 0.097_646_765 * z3 + 0.028_494_263 * z4); + let frames = output.first().map_or(0, |channel| channel.len()); + debug_assert_eq!(frames % 32, 0); + debug_assert!(output.len() <= 2); + for base in (0..frames).step_by(32) { + self.detector = finite(self.detector, 1.0); + let desired = self.detector.asin() / std::f32::consts::FRAC_PI_2; + let releasing = desired > self.gain; + let difference = if desired == 0.0 { + if releasing { -1.0 } else { 1.0 } + } else { + to_db(self.gain / desired) + }; + let rate = if releasing { + self.attack_difference = -1.0; + let x = 0.25 * (finite(difference, -1.0).clamp(-12.0, 0.0) + 12.0); + let x2 = x * x; + let x3 = x2 * x; + let x4 = x2 * x2; + from_db(5.0 / (a + b * x + c * x2 + d * x3 + e * x4)) + } else { + self.attack_difference = self.attack_difference.max(finite(difference, 1.0)); + 1.0 - (0.25 / self.attack_difference.max(0.5)).powf(1.0 / attack_frames) + }; + for frame in base..base + 32 { + let mut peak = 0.0_f32; + for channel in 0..output.len() { + let sample = input + .get(channel) + .or_else(|| input.first()) + .map_or(0.0, |samples| samples[frame]); + self.delay[channel][self.write] = sample; + peak = peak.max(sample.abs()); + } + let shaped = self.curve.saturate(peak); + let attenuation = if peak <= 0.0001 { 1.0 } else { shaped / peak }; + let sat_rate = from_db((-to_db(attenuation)).max(2.0) / sat_release_frames) - 1.0; + let detector_rate = if attenuation > self.detector { + sat_rate + } else { + 1.0 + }; + self.detector = finite( + (self.detector + (attenuation - self.detector) * detector_rate).min(1.0), + 1.0, + ); + if rate < 1.0 { + self.gain += (desired - self.gain) * rate; + } else { + self.gain = (self.gain * rate).min(1.0); + } + let warped = f64::from(std::f32::consts::FRAC_PI_2 * self.gain).sin() as f32; + let total_gain = post_gain * warped; + let gain_db = to_db(warped); + if gain_db < self.meter { + self.meter = gain_db; + } else { + self.meter += (gain_db - self.meter) * self.meter_release; + } + for (channel, samples) in output.iter_mut().enumerate() { + samples[frame] = self.delay[channel][self.read] * total_gain; + } + self.read = (self.read + 1) & (DELAY_CAPACITY - 1); + self.write = (self.write + 1) & (DELAY_CAPACITY - 1); + } + if self.detector.is_subnormal() { + self.detector = 0.0; + } + if self.gain.is_subnormal() { + self.gain = 0.0; + } + } + } +} diff --git a/moli-renderer-v8/src/context_bootstrap/web_audio_runtime/graph.rs b/moli-renderer-v8/src/context_bootstrap/web_audio_runtime/graph.rs index 9093e469bd..4e2fc6625a 100644 --- a/moli-renderer-v8/src/context_bootstrap/web_audio_runtime/graph.rs +++ b/moli-renderer-v8/src/context_bootstrap/web_audio_runtime/graph.rs @@ -1,18 +1,15 @@ -//! Minimal connection/start bookkeeping for the existing audio backend. -//! -//! Reachability can establish silence without pretending to implement DSP. Keep -//! connections in native slots, handle cycles, and snapshot input availability -//! before completion callbacks can disconnect or reconnect nodes. +//! WebIDL connection validation and GC-traced wrapper relationships. +//! Signal processing and scheduling belong to the native audio graph. use super::*; const CONTEXT: &str = "__moliAudioNodeContext"; -const ANALYSERS: &str = "__moliAudioContextActiveAnalysers"; +const ANALYSERS: &str = "__moliAudioContextPullAnalysers"; +const DEMANDED: &str = "__moliAudioContextDemandedCompressors"; const DESTINATION: &str = "__moliAudioContextDestination"; const INPUTS: &str = "__moliAudioNodeInputs"; const OUTPUTS: &str = "__moliAudioNodeOutputs"; const START_TIME: &str = "__moliAudioSourceStartTime"; -const RENDERED_INPUT: &str = "__moliAudioNodeRenderedInput"; #[derive(WebApiFunctionTemplate)] #[webapi(name = "AudioNode", enumerable)] @@ -102,15 +99,15 @@ fn add_edge<'s>( slot: &'static str, other: v8::Local<'s, v8::Object>, ) { - if objects(scope, node, slot).contains(&other) { + let mut values = objects(scope, node, slot); + if values.contains(&other) { return; } - let array = web_audio_array_slot(scope, node, slot).unwrap_or_else(|| { - let array = v8::Array::new(scope, 0); - set_private_value(scope, node, slot, array.into()); - array - }); - let _ = array.set_index(scope, array.length(), other.into()); + values.push(other); + let values: Vec> = values.into_iter().map(Into::into).collect(); + // Define dense own elements without invoking Array.prototype setters. + let array = v8::Array::new_with_elements(scope, &values); + set_private_value(scope, node, slot, array.into()); } fn remove_edge<'s>( @@ -174,13 +171,17 @@ pub(super) fn connect<'s>( return None; } } + if !with_connection(scope, source, destination, |source, destination| { + source.audio_node().connect(destination.audio_node()); + }) { + return None; + } add_edge(scope, source, OUTPUTS, destination); add_edge(scope, destination, INPUTS, source); - if get_private_value(scope, destination, ANALYSER_FFT_SIZE_SLOT).is_some() { - // Track candidates for offline automatic pull: analysers with input - // can process even without an output connection. + if is_analyser(scope, destination) { add_edge(scope, context, ANALYSERS, destination); } + update_pull_demand(scope, context); Some(destination) } @@ -192,7 +193,6 @@ pub(super) fn disconnect<'s>( let Some(context) = require_node_context(scope, source) else { return; }; - let outputs = objects(scope, source, OUTPUTS); let selected = if args.length() == 0 { None } else if args.get(0).is_object() { @@ -200,15 +200,6 @@ pub(super) fn disconnect<'s>( if require_node_context(scope, destination).is_none() { return; } - if !outputs.contains(&destination) { - throw_dom_exception( - scope, - "InvalidAccessError", - 15, - "The audio nodes are not connected.", - ); - return; - } Some(destination) } else { let Some(port) = args.get(0).uint32_value(scope) else { @@ -239,17 +230,35 @@ pub(super) fn disconnect<'s>( return; } } + // Port conversion can reenter connect/disconnect. Never pass a stale edge + // to the backend, whose targeted-disconnect contract requires it to exist. + let outputs = objects(scope, source, OUTPUTS); + if selected.is_some_and(|destination| !outputs.contains(&destination)) { + throw_dom_exception( + scope, + "InvalidAccessError", + 15, + "The audio nodes are not connected.", + ); + return; + } for destination in outputs { if selected.is_none_or(|selected| selected == destination) { + if !with_connection(scope, source, destination, |source, destination| { + source + .audio_node() + .disconnect_dest(destination.audio_node()); + }) { + return; + } remove_edge(scope, source, OUTPUTS, destination); remove_edge(scope, destination, INPUTS, source); - if get_private_value(scope, destination, ANALYSER_FFT_SIZE_SLOT).is_some() - && objects(scope, destination, INPUTS).is_empty() - { + if is_analyser(scope, destination) && objects(scope, destination, INPUTS).is_empty() { remove_edge(scope, context, ANALYSERS, destination); } } } + update_pull_demand(scope, context); } pub(super) fn start_source<'s>( @@ -257,10 +266,10 @@ pub(super) fn start_source<'s>( args: &v8::FunctionCallbackArguments<'s>, ) { let node = args.this(); - let Some(start) = web_audio_number_slot(scope, node, START_TIME) else { + if web_audio_number_slot(scope, node, START_TIME).is_none() { throw_type_error(scope, "Illegal invocation: expected an OscillatorNode."); return; - }; + } let when = if args.get(0).is_undefined() { 0.0 } else { @@ -273,11 +282,9 @@ pub(super) fn start_source<'s>( throw_type_error(scope, "Audio source start time must be finite."); return; } - if when < 0.0 { - throw_range_error(scope, "Audio source start time must not be negative."); - return; - } - if start.is_finite() { + // WebIDL conversion above may run page code and start this same source. + // Check the live state afterwards, before the algorithm's range check. + if web_audio_number_slot(scope, node, START_TIME).is_some_and(f64::is_finite) { throw_dom_exception( scope, "InvalidStateError", @@ -286,57 +293,104 @@ pub(super) fn start_source<'s>( ); return; } + if when < 0.0 { + throw_range_error(scope, "Audio source start time must not be negative."); + return; + } + let Some(state) = backend::get(scope, node) else { + return; + }; + if let State::Node(Node::Oscillator(native)) = &mut *state.borrow_mut() { + native.start_at(when); + } set_web_audio_number_slot(scope, node, START_TIME, when); } -fn has_started_source<'s>( +fn is_analyser<'s>(scope: &mut v8::PinScope<'s, '_>, node: v8::Local<'s, v8::Object>) -> bool { + backend::get(scope, node) + .is_some_and(|state| matches!(&*state.borrow(), State::Node(Node::Analyser(_)))) +} + +fn set_demand<'s>( scope: &mut v8::PinScope<'s, '_>, node: v8::Local<'s, v8::Object>, - end_time: f64, + active: bool, ) -> bool { - let mut pending = vec![node]; - let mut visited = Vec::new(); - while let Some(node) = pending.pop() { - if visited.contains(&node) { - continue; - } - visited.push(node); - if web_audio_number_slot(scope, node, START_TIME).is_some_and(|start| start < end_time) { - return true; - } - pending.extend(objects(scope, node, INPUTS)); + if let Some(state) = backend::get(scope, node) + && let State::Node(Node::Compressor { demanded, .. }) = &*state.borrow() + { + demanded.store(active, std::sync::atomic::Ordering::Release); + return true; } false } -pub(super) fn prepare_offline_render<'s>( - scope: &mut v8::PinScope<'s, '_>, - context: v8::Local<'s, v8::Object>, - end_time: f64, -) -> bool { - let destination = web_audio_object_slot(scope, context, DESTINATION); - let mut pending: Vec<_> = objects(scope, context, ANALYSERS) - .into_iter() - .filter(|node| objects(scope, *node, OUTPUTS).is_empty()) - .collect(); - pending.extend(destination); +// Demand and input availability are different: a pulled compressor processes +// silent input too. This only selects processors; it never substitutes PCM or +// decides whether the output is silent from graph reachability. +fn update_pull_demand<'s>(scope: &mut v8::PinScope<'s, '_>, context: v8::Local<'s, v8::Object>) { + let previous = objects(scope, context, DEMANDED); + let mut pending = objects(scope, context, ANALYSERS); + pending.extend(web_audio_object_slot(scope, context, DESTINATION)); let mut visited = Vec::new(); + let mut demanded = Vec::new(); while let Some(node) = pending.pop() { if visited.contains(&node) { continue; } visited.push(node); + if backend::get(scope, node) + .is_some_and(|state| matches!(*state.borrow(), State::ModuleWorklet)) + { + // Module-only worklets have no DSP pull edge yet. Do not propagate + // native processor demand across a control-only connection. + continue; + } pending.extend(objects(scope, node, INPUTS)); - let has_input = has_started_source(scope, node, end_time); - let flag = v8::Boolean::new(scope, has_input); - set_private_value(scope, node, RENDERED_INPUT, flag.into()); + if set_demand(scope, node, true) { + demanded.push(node.into()); + } } - destination.is_some_and(|node| rendered_with_input(scope, node)) + // Retained processors must not briefly observe false on the audio thread + // just because an unrelated edge changed in the control thread. + for node in previous { + if !visited.contains(&node) { + set_demand(scope, node, false); + } + } + let nodes = v8::Array::new_with_elements(scope, &demanded); + set_private_value(scope, context, DEMANDED, nodes.into()); } -pub(super) fn rendered_with_input<'s>( +fn with_connection<'s>( scope: &mut v8::PinScope<'s, '_>, - node: v8::Local<'s, v8::Object>, + source: v8::Local<'s, v8::Object>, + destination: v8::Local<'s, v8::Object>, + operation: impl FnOnce(&Node, &Node), ) -> bool { - get_private_value(scope, node, RENDERED_INPUT).is_some_and(|value| value.boolean_value(scope)) + if let Some(source) = backend::get(scope, source) + && let Some(destination) = backend::get(scope, destination) + { + match (&*source.borrow(), &*destination.borrow()) { + (State::Node(source), State::Node(destination)) => { + operation(source, destination); + return true; + } + (State::ModuleWorklet, State::Node(_) | State::ModuleWorklet) + | (State::Node(_), State::ModuleWorklet) => { + // Keep the pre-existing module/MessagePort surface usable. + // The caller records the logical edge; unlike native-to-native + // edges, it does not imply that JS worklet DSP is implemented. + return true; + } + _ => {} + } + } + throw_dom_exception( + scope, + "NotSupportedError", + 9, + "This AudioNode has no native audio processor.", + ); + false } diff --git a/moli-renderer-v8/src/context_bootstrap/web_audio_runtime/oscillator.rs b/moli-renderer-v8/src/context_bootstrap/web_audio_runtime/oscillator.rs new file mode 100644 index 0000000000..7dd15ac00f --- /dev/null +++ b/moli-renderer-v8/src/context_bootstrap/web_audio_runtime/oscillator.rs @@ -0,0 +1,78 @@ +use super::*; +use web_audio_api::node::OscillatorType; + +#[derive(WebApiFunctionTemplate)] +#[webapi(name = "OscillatorNode", enumerable)] +struct OscillatorPrototype { + #[webapi(accessor_property = "type", getter = kind, setter = set_kind)] + kind: (), +} + +pub(super) fn install<'s>( + scope: &mut v8::PinScope<'s, '_, ()>, + template: v8::Local<'s, v8::FunctionTemplate>, +) { + OscillatorPrototype::initialize_prototype_template(scope, template.prototype_template(scope)); +} + +fn kind<'s>( + scope: &mut v8::PinScope<'s, '_>, + args: v8::FunctionCallbackArguments<'s>, + mut rv: v8::ReturnValue<'s, v8::Value>, +) { + if let Some(state) = backend::get(scope, args.this()) + && let State::Node(Node::Oscillator(node)) = &*state.borrow() + { + let name = match node.type_() { + OscillatorType::Sine => "sine", + OscillatorType::Square => "square", + OscillatorType::Sawtooth => "sawtooth", + OscillatorType::Triangle => "triangle", + OscillatorType::Custom => "custom", + }; + rv.set(v8str(scope, name).into()); + return; + } + throw_type_error(scope, "Illegal invocation: expected an OscillatorNode."); +} + +fn set_kind<'s>( + scope: &mut v8::PinScope<'s, '_>, + args: v8::FunctionCallbackArguments<'s>, + _rv: v8::ReturnValue<'s, v8::Value>, +) { + let Some(state) = backend::get(scope, args.this()) else { + throw_type_error(scope, "Illegal invocation: expected an OscillatorNode."); + return; + }; + if !matches!(&*state.borrow(), State::Node(Node::Oscillator(_))) { + throw_type_error(scope, "Illegal invocation: expected an OscillatorNode."); + return; + } + let Some(value) = args.get(0).to_string(scope) else { + return; + }; + let value = value.to_rust_string_lossy(scope); + let kind = match value.as_str() { + "sine" => OscillatorType::Sine, + "square" => OscillatorType::Square, + "sawtooth" => OscillatorType::Sawtooth, + "triangle" => OscillatorType::Triangle, + "custom" => { + throw_dom_exception( + scope, + "InvalidStateError", + 11, + "Use setPeriodicWave to select a custom waveform.", + ); + return; + } + _ => { + throw_type_error(scope, "Invalid OscillatorType."); + return; + } + }; + if let State::Node(Node::Oscillator(node)) = &mut *state.borrow_mut() { + node.set_type(kind); + } +} diff --git a/moli-renderer-v8/src/context_bootstrap/web_audio_runtime/oscillator_dsp.rs b/moli-renderer-v8/src/context_bootstrap/web_audio_runtime/oscillator_dsp.rs new file mode 100644 index 0000000000..69522b5493 --- /dev/null +++ b/moli-renderer-v8/src/context_bootstrap/web_audio_runtime/oscillator_dsp.rs @@ -0,0 +1,153 @@ +use super::wavetable::{self, Tables}; +use std::sync::{ + Arc, + atomic::{AtomicU8, AtomicU64, Ordering}, +}; +use web_audio_api::context::BaseAudioContext; +use web_audio_api::node::OscillatorType; +use web_audio_api::worklet::{ + AudioParamValues, AudioWorkletGlobalScope, AudioWorkletNode, AudioWorkletNodeOptions, + AudioWorkletProcessor, +}; +use web_audio_api::{AudioParam, AudioParamDescriptor, AutomationRate}; + +struct Controls { + shape: AtomicU8, + start: AtomicU64, +} + +pub(super) struct Oscillator { + pub(super) node: AudioWorkletNode, + controls: Arc, +} + +impl Oscillator { + pub(super) fn new(context: &C) -> Self { + let controls = Arc::new(Controls { + shape: AtomicU8::new(0), + start: AtomicU64::new(f64::INFINITY.to_bits()), + }); + let tables = wavetable::for_sample_rate(context.sample_rate()); + let node = AudioWorkletNode::new::( + context, + AudioWorkletNodeOptions { + number_of_inputs: 0, + number_of_outputs: 1, + output_channel_count: vec![1], + processor_options: (Arc::clone(&controls), tables), + parameter_data: Default::default(), + audio_node_options: Default::default(), + }, + ); + Self { node, controls } + } + pub(super) fn frequency(&self) -> &AudioParam { + &self.node.parameters()["frequency"] + } + pub(super) fn detune(&self) -> &AudioParam { + &self.node.parameters()["detune"] + } + pub(super) fn start_at(&mut self, when: f64) { + self.controls.start.store( + when.max(self.node.context().current_time()).to_bits(), + Ordering::Release, + ); + } + pub(super) fn type_(&self) -> OscillatorType { + match self.controls.shape.load(Ordering::Relaxed) { + 0 => OscillatorType::Sine, + 1 => OscillatorType::Square, + 2 => OscillatorType::Sawtooth, + 3 => OscillatorType::Triangle, + _ => unreachable!(), + } + } + pub(super) fn set_type(&mut self, kind: OscillatorType) { + let value = match kind { + OscillatorType::Sine => 0, + OscillatorType::Square => 1, + OscillatorType::Sawtooth => 2, + OscillatorType::Triangle => 3, + OscillatorType::Custom => unreachable!("custom waves require their own table"), + }; + self.controls.shape.store(value, Ordering::Release); + } +} + +struct Processor { + controls: Arc, + tables: [Arc; 4], + position: f64, + started: bool, +} + +impl AudioWorkletProcessor for Processor { + type ProcessorOptions = (Arc, [Arc; 4]); + fn constructor((controls, tables): Self::ProcessorOptions) -> Self { + Self { + controls, + tables, + position: 0.0, + started: false, + } + } + fn parameter_descriptors() -> Vec { + [ + ("frequency", 440.0, f32::MIN, f32::MAX), + ("detune", 0.0, -153600.0, 153600.0), + ] + .into_iter() + .map( + |(name, default_value, min_value, max_value)| AudioParamDescriptor { + name: name.into(), + default_value, + min_value, + max_value, + automation_rate: AutomationRate::A, + }, + ) + .collect() + } + fn process<'a, 'b>( + &mut self, + _inputs: &'b [&'a [&'a [f32]]], + outputs: &'b mut [&'a mut [&'a mut [f32]]], + params: AudioParamValues<'b>, + scope: &'b AudioWorkletGlobalScope, + ) -> bool { + let start = f64::from_bits(self.controls.start.load(Ordering::Acquire)); + let output = &mut outputs[0][0]; + if !start.is_finite() { + output.fill(0.0); + return false; + } + let table = &self.tables[self.controls.shape.load(Ordering::Acquire) as usize]; + let frequency = params.get("frequency"); + let detune = params.get("detune"); + let sample_rate = scope.sample_rate; + let nyquist = sample_rate / 2.0; + let rate_scale = table.size as f32 / sample_rate; + for (index, sample) in output.iter_mut().enumerate() { + let time = scope.current_time + index as f64 / f64::from(sample_rate); + if time < start { + *sample = 0.0; + continue; + } + let hz = frequency[index.min(frequency.len() - 1)].clamp(-nyquist, nyquist); + let detune = detune[index.min(detune.len() - 1)]; + let hz = ((f64::from(hz) * 2.0_f64.powf(f64::from(detune) / 1200.0)) as f32) + .clamp(-nyquist, nyquist); + let increment = f64::from(hz * rate_scale); + if !self.started { + self.position = (increment * ((time - start) * f64::from(sample_rate))) + .rem_euclid(table.size as f64); + self.started = true; + } + *sample = table.sample(self.position, hz, sample_rate); + self.position = (self.position + increment).rem_euclid(table.size as f64); + } + true + } +} + +use web_audio_api::node::AudioNode; diff --git a/moli-renderer-v8/src/context_bootstrap/web_audio_runtime/wavetable.rs b/moli-renderer-v8/src/context_bootstrap/web_audio_runtime/wavetable.rs new file mode 100644 index 0000000000..53e8fa83e3 --- /dev/null +++ b/moli-renderer-v8/src/context_bootstrap/web_audio_runtime/wavetable.rs @@ -0,0 +1,141 @@ +// Copyright (C) 2012 Google Inc. All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are met: +// 1. Redistributions of source code must retain the above copyright notice, +// this list of conditions and the following disclaimer. +// 2. Redistributions in binary form must reproduce the above copyright notice, +// this list of conditions and the following disclaimer in the documentation +// and/or other materials provided with the distribution. +// 3. Neither the name of Apple Computer, Inc. ("Apple") nor the names of its +// contributors may be used to endorse or promote products derived from this +// software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY APPLE AND ITS CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +// DISCLAIMED. IN NO EVENT SHALL APPLE OR ITS CONTRIBUTORS BE LIABLE FOR ANY +// DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +// (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +// LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +// ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +//! Octave-banded Fourier tables, ported from Blink's +//! `third_party/blink/renderer/modules/webaudio/periodic_wave.cc`. +//! Band limiting is essential: a raw +//! triangle or square aliases differently at high frequencies and changes both +//! the compressor's detector input and the output spectrum. + +use realfft::{RealFftPlanner, num_complex::Complex32}; +use std::sync::{Arc, OnceLock}; + +pub(super) struct Tables { + pub(super) size: usize, + bands: Vec>, +} + +pub(super) fn for_sample_rate(sample_rate: f32) -> [Arc; 4] { + static LOW: [OnceLock>; 4] = [const { OnceLock::new() }; 4]; + static MID: [OnceLock>; 4] = [const { OnceLock::new() }; 4]; + static HIGH: [OnceLock>; 4] = [const { OnceLock::new() }; 4]; + let (size, cache) = if sample_rate <= 24000.0 { + (2048, &LOW) + } else if sample_rate <= 88200.0 { + (4096, &MID) + } else { + (16384, &HIGH) + }; + std::array::from_fn(|shape| { + Arc::clone(cache[shape].get_or_init(|| Arc::new(Tables::new(size, shape)))) + }) +} + +impl Tables { + fn new(size: usize, shape: usize) -> Self { + let half = size / 2; + let band_count = 3 * size.ilog2() as usize; + let inverse = RealFftPlanner::::new().plan_fft_inverse(size); + let mut harmonics = inverse.make_input_vec(); + for (n, harmonic) in harmonics.iter_mut().enumerate().take(half).skip(1) { + let pi_factor = 2.0 / (n as f32 * std::f32::consts::PI); + let coefficient = match shape { + 0 => { + if n == 1 { + 1.0 + } else { + 0.0 + } + } + 1 => { + if n & 1 != 0 { + 2.0 * pi_factor + } else { + 0.0 + } + } + 2 => pi_factor * if n & 1 != 0 { 1.0 } else { -1.0 }, + 3 => { + if n & 1 != 0 { + 2.0 * (pi_factor * pi_factor) + * if ((n - 1) >> 1) & 1 != 0 { -1.0 } else { 1.0 } + } else { + 0.0 + } + } + _ => unreachable!(), + }; + *harmonic = Complex32::new(0.0, -coefficient); + } + let mut bands = Vec::with_capacity(band_count); + let mut normalization = 0.5; + let mut scratch = inverse.make_scratch_vec(); + for band in 0..band_count { + let scale = 2.0_f64.powf(f64::from(-(band as f32 * 400.0) / 1200.0)) as f32; + let partials = (scale * half as f32) as usize; + let mut spectrum = harmonics.clone(); + spectrum[half.min(partials + 1)..].fill(Complex32::new(0.0, 0.0)); + let mut samples = inverse.make_output_vec(); + inverse + .process_with_scratch(&mut spectrum, &mut samples, &mut scratch) + .expect("valid Fourier table"); + if band == 0 { + let peak = samples + .iter() + .map(|sample| sample.abs()) + .fold(0.0_f32, f32::max); + if peak != 0.0 { + normalization = 1.0 / peak; + } + } + for sample in &mut samples { + *sample *= normalization; + } + bands.push(samples); + } + Self { size, bands } + } + + pub(super) fn sample(&self, position: f64, frequency: f32, sample_rate: f32) -> f32 { + let ratio = if frequency != 0.0 { + frequency.abs() / (sample_rate / self.size as f32) + } else { + 0.5 + }; + let pitch = (1.0 + ratio.log2() * 1200.0 / 400.0).clamp(0.0, (self.bands.len() - 1) as f32); + let upper = pitch as usize; + let lower = (upper + 1).min(self.bands.len() - 1); + let blend = pitch - upper as f32; + let integer = position as usize; + let index = integer & (self.size - 1); + let next = (index + 1) & (self.size - 1); + let fraction = position as f32 - integer as f32; + let interpolate = |band: usize| { + let first = self.bands[band][index]; + first + fraction * (self.bands[band][next] - first) + }; + let high = interpolate(upper); + high + blend * (interpolate(lower) - high) + } +} diff --git a/moli-renderer-v8/src/script_vm/tests/browser_api/media.rs b/moli-renderer-v8/src/script_vm/tests/browser_api/media.rs index 1378297529..4ad16b949c 100644 --- a/moli-renderer-v8/src/script_vm/tests/browser_api/media.rs +++ b/moli-renderer-v8/src/script_vm/tests/browser_api/media.rs @@ -2438,7 +2438,9 @@ fn web_audio_declared_objects_keep_brands_and_hidden_properties() { }; }; const rendering = ctx.startRendering(); - osc.frequency.setValueAtTime(123, 0); + osc.frequency.value = 123; + // Scheduling does not synchronously replace the intrinsic value. + osc.frequency.setValueAtTime(246, 1); Object.assign(globalThis.__webAudioDeclaredProbe, { ctxTag: Object.prototype.toString.call(ctx), ctxCtor: ctx.constructor && ctx.constructor.name, @@ -2478,7 +2480,7 @@ fn web_audio_declared_objects_keep_brands_and_hidden_properties() { assert_eq!( result, - r#"{"completion":{"tag":"[object Object]","ctor":"Object","type":"complete","renderedBufferTag":"[object AudioBuffer]","renderedBufferCtor":"AudioBuffer","renderedBufferLength":32,"renderedBufferSampleRate":8000,"renderedBufferDuration":0.004,"renderedBufferEnumerable":false,"targetIsContext":true,"currentTargetIsContext":true},"ctxTag":"[object OfflineAudioContext]","ctxCtor":"OfflineAudioContext","ctxLength":32,"ctxSampleRate":8000,"ctxState":"closed","lengthEnumerable":false,"destinationTag":"[object AudioDestinationNode]","destinationCtor":"AudioDestinationNode","destinationKeys":"","oscTag":"[object OscillatorNode]","oscCtor":"OscillatorNode","oscKeys":"","oscType":"sine","frequencyTag":"[object AudioParam]","frequencyCtor":"AudioParam","frequencyValue":123,"frequencyEnumerable":false,"setValueAtTimeType":"function","compTag":"[object DynamicsCompressorNode]","compCtor":"DynamicsCompressorNode","compKeys":"","thresholdTag":"[object AudioParam]","reduction":0,"reductionOwn":false,"promiseTag":"[object Promise]"}"# + r#"{"completion":{"tag":"[object Object]","ctor":"Object","type":"complete","renderedBufferTag":"[object AudioBuffer]","renderedBufferCtor":"AudioBuffer","renderedBufferLength":32,"renderedBufferSampleRate":8000,"renderedBufferDuration":0.004,"renderedBufferEnumerable":false,"targetIsContext":true,"currentTargetIsContext":true},"ctxTag":"[object OfflineAudioContext]","ctxCtor":"OfflineAudioContext","ctxLength":32,"ctxSampleRate":8000,"ctxState":"running","lengthEnumerable":false,"destinationTag":"[object AudioDestinationNode]","destinationCtor":"AudioDestinationNode","destinationKeys":"","oscTag":"[object OscillatorNode]","oscCtor":"OscillatorNode","oscKeys":"","oscType":"sine","frequencyTag":"[object AudioParam]","frequencyCtor":"AudioParam","frequencyValue":123,"frequencyEnumerable":false,"setValueAtTimeType":"function","compTag":"[object DynamicsCompressorNode]","compCtor":"DynamicsCompressorNode","compKeys":"","thresholdTag":"[object AudioParam]","reduction":0,"reductionOwn":false,"promiseTag":"[object Promise]"}"# ); } @@ -2853,6 +2855,8 @@ fn web_audio_numeric_entrypoints_parse_webidl_arguments() { record("setSymbol", () => osc.frequency.setValueAtTime(Symbol("value"), 0)); osc.frequency.setValueAtTime("123.5", ctx.currentTime); results.frequency = osc.frequency.value; + osc.connect(ctx.destination); + osc.start(); ctx.oncomplete = (event) => { const buffer = event.renderedBuffer; @@ -2860,6 +2864,7 @@ fn web_audio_numeric_entrypoints_parse_webidl_arguments() { record("channelSymbol", () => buffer.getChannelData(Symbol("channel"))); record("channelRange", () => buffer.getChannelData(1)); results.channel0 = Object.prototype.toString.call(buffer.getChannelData("0.9")); + results.renderedFrequency = osc.frequency.value; }; ctx.startRendering(); globalThis.__audioWebIdlProbe = results; @@ -2875,7 +2880,7 @@ fn web_audio_numeric_entrypoints_parse_webidl_arguments() { assert_eq!( result, - r#"{"ctorMissing":"TypeError","ctorSymbol":"TypeError","length":16,"sampleRate":44100,"setMissingTime":"TypeError","setSymbol":"TypeError","frequency":123.5,"channelMissing":"TypeError","channelSymbol":"TypeError","channelRange":"IndexSizeError","channel0":"[object Float32Array]"}"# + r#"{"ctorMissing":"TypeError","ctorSymbol":"TypeError","length":16,"sampleRate":44100,"setMissingTime":"TypeError","setSymbol":"TypeError","frequency":440,"channelMissing":"TypeError","channelSymbol":"TypeError","channelRange":"IndexSizeError","channel0":"[object Float32Array]","renderedFrequency":123.5}"# ); } #[test] diff --git a/moli-renderer-v8/src/script_vm/tests/browser_api/web_audio.rs b/moli-renderer-v8/src/script_vm/tests/browser_api/web_audio.rs index da7e9e7c50..c9c97e4b00 100644 --- a/moli-renderer-v8/src/script_vm/tests/browser_api/web_audio.rs +++ b/moli-renderer-v8/src/script_vm/tests/browser_api/web_audio.rs @@ -1,5 +1,54 @@ use super::*; +#[test] +fn web_audio_connections_revalidate_after_coercion_without_page_hooks() { + let mut vm = new_storage_test_vm("https://audio-connection-reentrancy.test/"); + let source = include_str!("../../../../tests/fixtures/audio-graph-reentrancy-check.js"); + assert_eq!(vm.eval(&format!("JSON.stringify({source})")).unwrap(), "[]"); +} + +#[test] +fn web_audio_native_state_boundaries_match_chromium() { + let mut vm = new_storage_test_vm("https://audio-native-boundaries.test/"); + let source = include_str!("../../../../tests/fixtures/offline-audio-boundaries-check.js"); + vm.exec( + &format!("({source}).then(value => globalThis.__audioBoundaries = value)"), + None, + ) + .expect("audio boundary fixture should execute"); + assert_eq!(vm.eval("JSON.stringify(__audioBoundaries)").unwrap(), "[]"); +} + +#[test] +fn offline_audio_analyser_uses_rendered_pcm_and_a_real_fft() { + let mut vm = new_storage_test_vm("https://audio-rendered-pcm.test/"); + let source = include_str!("../../../../tests/fixtures/offline-audio-analyser-check.js"); + vm.exec( + &format!("({source}).then(value => globalThis.__renderedPcm = value)"), + None, + ) + .expect("offline audio fixture should execute"); + let result = vm + .eval("JSON.stringify(globalThis.__renderedPcm)") + .expect("offline rendering should settle its promise"); + let rows: serde_json::Value = serde_json::from_str(&result).unwrap(); + assert_eq!(rows.as_array().unwrap().len(), 3); + for row in rows.as_array().unwrap() { + for key in [ + "waveformCorrect", + "timeMatchesPcm", + "retainsAfterDisconnect", + "peakCorrect", + "blackmanAmplitudeCorrect", + "frequencyReadStable", + "byteFrequencyCorrect", + "byteTimeCorrect", + ] { + assert_eq!(row[key], true, "{key}: {row}"); + } + } +} + #[test] fn web_audio_factories_are_shared_by_realtime_and_offline_contexts() { let mut vm = new_storage_test_vm("https://audio-factories.test/"); @@ -85,7 +134,10 @@ fn web_audio_factories_reject_forged_contexts_and_keep_nodes_independent() { const a = context.createOscillator(), b = context.createOscillator(); a.type = 'triangle'; if (a.frequency.setValueAtTime(880, 0) !== a.frequency) errors.push('AudioParam return value'); - if (a.frequency.value !== 880 || b.frequency.value !== 440 || b.type !== 'sine') errors.push('shared oscillator state'); + // Scheduling does not synchronously change .value (also true in Chromium). + if (a.frequency.value !== 440 || b.frequency.value !== 440 || b.type !== 'sine') errors.push('premature automation'); + a.frequency.value = 880; + if (a.frequency.value !== 880 || b.frequency.value !== 440) errors.push('shared oscillator state'); const analyser = context.createAnalyser(); if (a.connect(analyser) !== analyser || analyser.connect(context.destination) !== context.destination) errors.push('connect return value'); if (a.start(0) !== undefined || a.disconnect() !== undefined) errors.push('source return value'); @@ -254,8 +306,10 @@ fn web_audio_biquad_and_existing_params_expose_native_readonly_metadata() { } } if (filter.gain.defaultValue !== 0 || filter.gain.minValue !== Math.fround(-3.4028234663852886e38) || !(filter.gain.maxValue > 1541 && filter.gain.maxValue < 1542)) errors.push('gain range'); - try { filter.getFrequencyResponse(new Float32Array(1), new Float32Array(1), new Float32Array(1)); errors.push('fabricated frequency response'); } - catch (e) { if (e.name !== 'NotSupportedError') errors.push('response ' + e.name); } + filter.type = 'allpass'; + const magnitudes = new Float32Array(3), phases = new Float32Array(3); + filter.getFrequencyResponse(new Float32Array([0, 350, rate / 4]), magnitudes, phases); + if (!magnitudes.every(value => Math.abs(value - 1) < 1e-6) || !phases.every(Number.isFinite)) errors.push('allpass response'); } return JSON.stringify(errors); })() @@ -268,8 +322,9 @@ fn offline_audio_silence_depends_on_reachable_started_sources() { let mut vm = new_storage_test_vm("https://audio-graph-silence.test/"); vm.exec(r#" globalThis.__audioSilenceResults = []; +for (const length of [100, 1024]) { for (const mode of ['empty', 'unconnected', 'not-started', 'future', 'disconnected-source', 'disconnected-sink', 'cycle', 'connected', 'duplicate', 'selected-disconnect']) { - const ctx = new OfflineAudioContext(1, 100, 44100); + const ctx = new OfflineAudioContext(1, length, 44100); const osc = ctx.createOscillator(), comp = ctx.createDynamicsCompressor(); if (mode !== 'empty' && mode !== 'unconnected') { osc.connect(comp); @@ -282,14 +337,16 @@ for (const mode of ['empty', 'unconnected', 'not-started', 'future', 'disconnect if (mode === 'duplicate') { osc.connect(comp); osc.connect(comp); } if (mode === 'selected-disconnect') { const a = ctx.createAnalyser(); osc.connect(a); osc.disconnect(a); } ctx.startRendering().then(buffer => { - __audioSilenceResults.push([mode, buffer.getChannelData(0).some(x => x !== 0), comp.reduction < 0]); + const pcm = buffer.getChannelData(0); + __audioSilenceResults.push([length, mode, pcm.some(x => x !== 0), pcm.subarray(0, Math.min(264, length)).every(x => x === 0)]); }); } +} "#, None).expect("offline graph scenarios should render without hanging on a cycle"); let result = vm.eval("JSON.stringify(__audioSilenceResults)").unwrap(); assert_eq!( result, - r#"[["empty",false,false],["unconnected",false,false],["not-started",false,false],["future",false,false],["disconnected-source",false,false],["disconnected-sink",false,false],["cycle",false,false],["connected",true,true],["duplicate",true,true],["selected-disconnect",true,true]]"# + r#"[[100,"empty",false,true],[100,"unconnected",false,true],[100,"not-started",false,true],[100,"future",false,true],[100,"disconnected-source",false,true],[100,"disconnected-sink",false,true],[100,"cycle",false,true],[100,"connected",false,true],[100,"duplicate",false,true],[100,"selected-disconnect",false,true],[1024,"empty",false,true],[1024,"unconnected",false,true],[1024,"not-started",false,true],[1024,"future",false,true],[1024,"disconnected-source",false,true],[1024,"disconnected-sink",false,true],[1024,"cycle",false,true],[1024,"connected",true,true],[1024,"duplicate",true,true],[1024,"selected-disconnect",true,true]]"# ); } @@ -316,6 +373,64 @@ ctx.startRendering(); ); } +#[test] +fn offline_compressor_meter_follows_pull_demand_not_input_reachability() { + let mut vm = new_storage_test_vm("https://audio-compressor-demand.test/"); + vm.exec(r#" +globalThis.__meter = []; +for (const mode of ['orphan', 'silent-destination', 'source-without-destination', 'source-to-destination']) { + const context = new OfflineAudioContext(1, 1024, 44100); + const compressor = context.createDynamicsCompressor(), oscillator = context.createOscillator(); + if (mode.startsWith('source')) { oscillator.connect(compressor); oscillator.start(); } + if (mode.endsWith('destination') && mode !== 'source-without-destination') compressor.connect(context.destination); + const before = compressor.reduction; + context.startRendering().then(buffer => __meter.push([mode, before, compressor.reduction < 0, buffer.getChannelData(0).some(x => x !== 0)])); +} +"#, None).unwrap(); + // Blink can have negative metering while processing silence. The old + // reachability-based "no source => zero reduction" assertion was incorrect. + assert_eq!( + vm.eval("JSON.stringify(__meter)").unwrap(), + r#"[["orphan",0,false,false],["silent-destination",0,true,false],["source-without-destination",0,false,false],["source-to-destination",0,true,true]]"# + ); +} + +#[test] +fn offline_audio_automation_and_delayed_start_use_the_sample_clock() { + let mut vm = new_storage_test_vm("https://audio-automation.test/"); + vm.exec( + r#" +const context = new OfflineAudioContext(1, 4096, 8192), oscillator = context.createOscillator(); +oscillator.frequency.value = 256; +oscillator.frequency.setValueAtTime(512, .25); +oscillator.detune.setValueAtTime(1200, .375); +oscillator.connect(context.destination); +oscillator.start(.125); +const before = oscillator.frequency.value; +context.startRendering().then(buffer => { + const pcm = buffer.getChannelData(0); + const expected = i => i < 1024 ? 0 : + Math.sin(2 * Math.PI * (i < 2048 ? 256 : i < 3072 ? 512 : 1024) * i / 8192); + globalThis.__automation = { + before, + error: Math.max(...pcm.map((value, i) => Math.abs(value - expected(i)))), + silentPrefix: pcm.subarray(0, 1024).every(value => value === 0), + state: context.state, time: context.currentTime + }; +}); +"#, + None, + ) + .unwrap(); + let result: serde_json::Value = + serde_json::from_str(&vm.eval("JSON.stringify(__automation)").unwrap()).unwrap(); + assert_eq!(result["before"], 256); + assert!(result["error"].as_f64().unwrap() < 1e-4, "{result}"); + assert_eq!(result["silentPrefix"], true); + assert_eq!(result["state"], "closed"); + assert_eq!(result["time"], 0.5); +} + #[test] fn web_audio_connections_validate_context_and_source_receivers() { let mut vm = new_storage_test_vm("https://audio-connection-receivers.test/"); diff --git a/moli-renderer-v8/tests/fixtures/audio-graph-reentrancy-check.js b/moli-renderer-v8/tests/fixtures/audio-graph-reentrancy-check.js new file mode 100644 index 0000000000..29bc01e510 --- /dev/null +++ b/moli-renderer-v8/tests/fixtures/audio-graph-reentrancy-check.js @@ -0,0 +1,35 @@ +(() => { + const failures = []; + const context = new OfflineAudioContext(1, 128, 8192); + const source = context.createOscillator(); + const sink = context.createAnalyser(); + source.connect(sink); + try { + source.disconnect(sink, {valueOf() { source.disconnect(sink); return 0; }}); + failures.push('accepted an edge removed during conversion'); + } catch (error) { + if (error.name !== 'InvalidAccessError') failures.push('removed edge: ' + error.name); + } + try { + source.disconnect(sink, {valueOf() { source.connect(sink); return 0; }}); + } catch (error) { + failures.push('new edge during conversion: ' + error.name); + } + let traps = 0; + const previous = Object.getOwnPropertyDescriptor(Array.prototype, '0'); + Object.defineProperty(Array.prototype, '0', { + configurable: true, + set() { traps++; } + }); + try { + source.connect(sink); + sink.connect(context.destination); + source.disconnect(sink); + sink.disconnect(); + } finally { + if (previous) Object.defineProperty(Array.prototype, '0', previous); + else delete Array.prototype[0]; + } + if (traps) failures.push('graph storage called page setters: ' + traps); + return failures; +})() diff --git a/moli-renderer-v8/tests/fixtures/offline-audio-analyser-check.js b/moli-renderer-v8/tests/fixtures/offline-audio-analyser-check.js new file mode 100644 index 0000000000..28c33938b0 --- /dev/null +++ b/moli-renderer-v8/tests/fixtures/offline-audio-analyser-check.js @@ -0,0 +1,51 @@ +(async () => { + const results = []; + for (const frequency of [256, 512, 1024]) { + const rate = 8192, length = 4096, fftSize = 256; + const context = new OfflineAudioContext(1, length, rate); + const oscillator = context.createOscillator(); + oscillator.frequency.value = frequency; + const analyser = context.createAnalyser(); + analyser.fftSize = fftSize; + analyser.smoothingTimeConstant = 0; + analyser.minDecibels = -60; + analyser.maxDecibels = 0; + oscillator.connect(analyser); + analyser.connect(context.destination); + oscillator.start(); + const buffer = await context.startRendering(); + const pcm = buffer.getChannelData(0); + const time = new Float32Array(fftSize); + analyser.getFloatTimeDomainData(time); + const bins = new Float32Array(fftSize / 2); + analyser.getFloatFrequencyData(bins); + const again = new Float32Array(bins.length); + analyser.getFloatFrequencyData(again); + const bytes = new Uint8Array(bins.length); + analyser.getByteFrequencyData(bytes); + const byteTime = new Uint8Array(fftSize); + analyser.getByteTimeDomainData(byteTime); + let peak = 0; + for (let i = 1; i < bins.length; ++i) if (bins[i] > bins[peak]) peak = i; + const waveformError = Math.max(...pcm.map((value, i) => Math.abs(value - Math.sin(2 * Math.PI * frequency * i / rate)))); + const retained = Array.from(time); + oscillator.disconnect(); + analyser.disconnect(); + analyser.getFloatTimeDomainData(time); + results.push({ + frequency, + waveformError, + waveformCorrect: waveformError < 0.0001, + timeMatchesPcm: time.every((value, i) => value === pcm[length - fftSize + i]), + retainsAfterDisconnect: time.every((value, i) => value === retained[i]), + peak, + peakCorrect: peak === frequency * fftSize / rate, + peakDecibels: bins[peak], + blackmanAmplitudeCorrect: Math.abs(bins[peak] - 20 * Math.log10(0.21)) < 0.001, + frequencyReadStable: bins.every((value, i) => value === again[i]), + byteFrequencyCorrect: bytes.every((value, i) => value === Math.max(0, Math.min(255, Math.floor(255 * (bins[i] + 60) / 60)))), + byteTimeCorrect: byteTime.every((value, i) => value === Math.max(0, Math.min(255, Math.floor(128 * (time[i] + 1))))), + }); + } + return results; +})() diff --git a/moli-renderer-v8/tests/fixtures/offline-audio-boundaries-check.js b/moli-renderer-v8/tests/fixtures/offline-audio-boundaries-check.js new file mode 100644 index 0000000000..0593309242 --- /dev/null +++ b/moli-renderer-v8/tests/fixtures/offline-audio-boundaries-check.js @@ -0,0 +1,102 @@ +(async () => { + const failures = []; + const check = (name, value) => { if (!value) failures.push(name); }; + const throws = (name, action, expected) => { + try { action(); failures.push(name + ': did not throw'); } + catch (error) { if (error.name !== expected) failures.push(name + ': ' + error.name); } + }; + for (const [channels, length, rate, error] of [ + [0, 128, 8000, 'NotSupportedError'], [33, 128, 8000, 'NotSupportedError'], + [1, 0, 8000, 'NotSupportedError'], [NaN, 128, 8000, 'NotSupportedError'], + [1, 128, 2999, 'NotSupportedError'], [1, 128, 768001, 'NotSupportedError'], + [1, 128, NaN, 'TypeError'], [1, 128, Infinity, 'TypeError'] + ]) throws('constructor ' + [channels, length, rate], () => new OfflineAudioContext(channels, length, rate), error); + const converted = new OfflineAudioContext(2 ** 32 + 1, 2 ** 32 + 128, 8192); + check('unsigned long conversion', converted.length === 128 && converted.sampleRate === 8192); + const context = new OfflineAudioContext(1, 128, 8192); + const clock = Object.getOwnPropertyDescriptor(BaseAudioContext.prototype, 'currentTime'); + check('currentTime prototype getter', typeof clock?.get === 'function' && !clock.set && clock.enumerable && clock.configurable); + check('currentTime readonly', !Object.hasOwn(context, 'currentTime') && !Reflect.set(context, 'currentTime', 100) && context.currentTime === 0); + if (clock?.get) { + for (const receiver of [{}, BaseAudioContext.prototype, Object.create(context), new Proxy(context, {})]) + throws('currentTime receiver', () => clock.get.call(receiver), 'TypeError'); + } + const oscillator = context.createOscillator(); + for (const parameter of [oscillator.frequency, oscillator.detune, context.createBiquadFilter().frequency]) { + check('default a-rate', parameter.automationRate === 'a-rate'); + parameter.automationRate = 'k-rate'; + check('set k-rate', parameter.automationRate === 'k-rate'); + parameter.automationRate = 'a-rate'; + check('set a-rate', parameter.automationRate === 'a-rate'); + } + const compressor = context.createDynamicsCompressor(); + for (const name of ['threshold', 'knee', 'ratio', 'attack', 'release']) { + const parameter = compressor[name]; + check('fixed k-rate ' + name, parameter.automationRate === 'k-rate'); + for (const rate of ['k-rate', 'a-rate']) + throws('fixed setter ' + name + ' ' + rate, () => { parameter.automationRate = rate; }, 'InvalidStateError'); + check('fixed rate unchanged', parameter.automationRate === 'k-rate'); + } + let completionCount = 0; + context.oncomplete = () => { completionCount++; }; + const rendering = context.startRendering(); + check('render returns promise', rendering instanceof Promise); + check('render immediate state', context.state === 'running'); + await context.startRendering().then(() => failures.push('second rendering resolved'), error => check('second rendering rejects', error.name === 'InvalidStateError')); + const buffer = await rendering; + check('render clock', context.currentTime === 128 / 8192 && context.state === 'closed'); + check('completion once', completionCount === 1 && buffer.length === 128); + await context.startRendering().then(() => failures.push('closed rendering resolved'), error => check('closed rendering rejects', error.name === 'InvalidStateError')); + + const reentrant = new OfflineAudioContext(1, 4096, 8192); + const source = reentrant.createOscillator(); + source.frequency.value = 512; + source.connect(reentrant.destination); + throws('reentrant start', () => source.start({valueOf() { source.start(.25); return 0; }}), 'InvalidStateError'); + throws('repeat start before range validation', () => source.start(-1), 'InvalidStateError'); + throws('restricted time before repeat check', () => source.start(NaN), 'TypeError'); + const samples = (await reentrant.startRendering()).getChannelData(0); + check('reentrant start keeps inner schedule', samples.subarray(0, 2048).every(x => x === 0) && samples.subarray(2048).some(x => x !== 0)); + + for (const frequency of [-512, -1e-20, -Number.MIN_VALUE, 0, 512]) { + const context = new OfflineAudioContext(1, 512, 8192); + const source = context.createOscillator(); + source.frequency.value = frequency; + source.connect(context.destination); + source.start(); + const pcm = (await context.startRendering()).getChannelData(0); + check('signed frequency ' + frequency, pcm.every((x, i) => Number.isFinite(x) && Math.abs(x - Math.sin(2 * Math.PI * frequency * i / 8192)) < .0001)); + } + for (const rate of ['a-rate', 'k-rate']) { + const context = new OfflineAudioContext(1, 128, 8192); + const source = context.createOscillator(); + source.frequency.automationRate = rate; + source.frequency.setValueAtTime(256, 0); + source.connect(context.destination); + source.start(); + check('scheduled intrinsic before rendering ' + rate, source.frequency.value === 440); + const pcm = (await context.startRendering()).getChannelData(0); + check('quantum boundary PCM ' + rate, pcm.every((x, i) => Math.abs(x - Math.sin(2 * Math.PI * 256 * i / 8192)) < .0001)); + check('first quantum intrinsic ' + rate, source.frequency.value === 256); + } + { + const context = new OfflineAudioContext(1, 128, 8192); + const source = context.createOscillator(); + const filter = context.createBiquadFilter(); + const compressor = context.createDynamicsCompressor(); + filter.frequency.setValueAtTime(1024, 0); + compressor.threshold.setValueAtTime(-50, 0); + source.connect(filter).connect(compressor).connect(context.destination); + source.start(); + await context.startRendering(); + check('biquad first quantum intrinsic', filter.frequency.value === 1024); + check('compressor first quantum intrinsic', compressor.threshold.value === -50); + } + const realtime = new AudioContext(); + check('realtime shared clock', !Object.hasOwn(realtime, 'currentTime') && realtime.currentTime >= 0); + await realtime.close(); + const closedClock = realtime.currentTime; + await Promise.resolve(); + check('closed clock stable', realtime.currentTime === closedClock); + return failures; +})() diff --git a/moli/tests/fetch_cli/eval.rs b/moli/tests/fetch_cli/eval.rs index f14289cc3f..686b8b5a52 100644 --- a/moli/tests/fetch_cli/eval.rs +++ b/moli/tests/fetch_cli/eval.rs @@ -66,7 +66,7 @@ fn run_eval_source( } #[test] -fn eval_can_use_realtime_audio_oscillator_and_analyser_shims() -> Result<()> { +fn eval_can_use_realtime_audio_oscillator_and_analyser_nodes() -> Result<()> { let output = run_eval( "data:text/html,Web Audio", r#"(async () => { @@ -74,7 +74,9 @@ fn eval_can_use_realtime_audio_oscillator_and_analyser_shims() -> Result<()> { const oscillator = context.createOscillator(); const analyser = context.createAnalyser(); oscillator.type = 'triangle'; - oscillator.frequency.setValueAtTime(880, context.currentTime); + oscillator.frequency.value = 880; + const frequency = oscillator.frequency.value; + const scheduled = oscillator.frequency.setValueAtTime(880, context.currentTime); analyser.fftSize = 32; const connected = oscillator.connect(analyser).connect(context.destination); // Before starting the source, the analyser must be silent. Reading @@ -89,7 +91,8 @@ fn eval_can_use_realtime_audio_oscillator_and_analyser_shims() -> Result<()> { await context.close(); return { oscillator: oscillator instanceof OscillatorNode, - frequency: oscillator.frequency.value, + frequency, + scheduledSameParameter: scheduled === oscillator.frequency, analyser: analyser instanceof AnalyserNode, connected: connected === context.destination, readReturnedUndefined: readResult === undefined, @@ -110,6 +113,7 @@ fn eval_can_use_realtime_audio_oscillator_and_analyser_shims() -> Result<()> { serde_json::json!({ "oscillator": true, "frequency": 880, + "scheduledSameParameter": true, "analyser": true, "connected": true, "readReturnedUndefined": true, diff --git a/vendor/web-audio-api-1.7.0/Cargo.toml b/vendor/web-audio-api-1.7.0/Cargo.toml new file mode 100644 index 0000000000..84d51c9cd2 --- /dev/null +++ b/vendor/web-audio-api-1.7.0/Cargo.toml @@ -0,0 +1,179 @@ +# THIS FILE IS AUTOMATICALLY GENERATED BY CARGO +# +# When uploading crates to the registry Cargo will automatically +# "normalize" Cargo.toml files for maximal compatibility +# with all versions of Cargo and also rewrite `path` dependencies +# to registry (e.g., crates.io) dependencies. +# +# If you are reading this file be aware that the original Cargo.toml +# will likely look very different (and much more reasonable). +# See Cargo.toml.orig for the original contents. + +[package] +edition = "2021" +rust-version = "1.86" +name = "web-audio-api" +version = "1.7.0" +authors = ["Otto "] +build = false +include = [ + "/resources", + "/src", + "LICENSE", + "README.md", +] +autolib = false +autobins = false +autoexamples = false +autotests = false +autobenches = false +description = "A pure Rust implementation of the Web Audio API, for use in non-browser contexts" +readme = "README.md" +keywords = [ + "web-audio-api", + "audio", + "sound", + "dsp", +] +categories = ["multimedia::audio"] +license = "MIT" +repository = "https://github.com/orottier/web-audio-api-rs" + +[features] +aac = [] +alac = [] +cpal = ["dep:cpal"] +cpal-asio = [ + "cpal", + "cpal/asio", +] +cpal-jack = [ + "cpal", + "cpal/jack", +] +cpal-pipewire = [ + "cpal", + "cpal/pipewire", +] +cubeb = ["dep:cubeb"] +default = ["cpal"] +diagnostics = [] +flac = [] +iai = [] +m4a = [] +mp3 = [] +ogg = [] +spec-compliant-worklet-inputs = [] +wav = [] + +[lib] +name = "web_audio_api" +path = "src/lib.rs" + +[dependencies.almost] +version = "0.2.0" + +[dependencies.arc-swap] +version = "1.6" + +[dependencies.arrayvec] +version = "0.7" + +[dependencies.cpal] +version = "0.18" +optional = true + +[dependencies.creek] +version = "1.2" +features = [ + "decode", + "decode-all", + "encode-wav", +] +default-features = false + +[dependencies.crossbeam-channel] +version = "0.5" + +[dependencies.cubeb] +version = "0.34" +optional = true + +[dependencies.dasp_sample] +version = "0.11" + +[dependencies.fft-convolver] +version = "0.3" + +[dependencies.float_eq] +version = "1.0" + +[dependencies.futures-channel] +version = "0.3.30" +features = ["sink"] + +[dependencies.futures-core] +version = "0.3.30" +default-features = false + +[dependencies.futures-util] +version = "0.3.30" +features = ["sink"] +default-features = false + +[dependencies.hound] +version = "3.5" + +[dependencies.hrtf] +version = "0.9" + +[dependencies.llq] +version = "0.1.1" + +[dependencies.log] +version = "0.4" + +[dependencies.num-complex] +version = "0.4" + +[dependencies.realfft] +version = "3.3" + +[dependencies.rubato] +version = "0.16" + +[dependencies.smallvec] +version = "1.11" + +[dependencies.symphonia] +version = "0.6" +features = [ + "all", + "opt-simd", +] +default-features = false + +[dependencies.vecmath] +version = "1.0" + +[dev-dependencies.alloc_counter] +version = "0.0.4" + +[dev-dependencies.criterion] +version = "0.8" + +[dev-dependencies.env_logger] +version = "0.11" + +[dev-dependencies.futures] +version = "0.3.30" +features = ["executor"] + +[dev-dependencies.paste] +version = "1.0.14" + +[dev-dependencies.rand] +version = "0.10" + +[target.'cfg(any(target_arch = "x86", target_arch = "x86_64", target_arch = "aarch64"))'.dependencies.no_denormals] +version = "0.3" diff --git a/vendor/web-audio-api-1.7.0/Cargo.toml.orig b/vendor/web-audio-api-1.7.0/Cargo.toml.orig new file mode 100644 index 0000000000..02f07b44e9 --- /dev/null +++ b/vendor/web-audio-api-1.7.0/Cargo.toml.orig @@ -0,0 +1,105 @@ +[package] +name = "web-audio-api" +version = "1.7.0" +authors = ["Otto "] +edition = "2021" +description = "A pure Rust implementation of the Web Audio API, for use in non-browser contexts" +readme = "README.md" +repository = "https://github.com/orottier/web-audio-api-rs" +keywords = ["web-audio-api", "audio", "sound", "dsp"] +license = "MIT" +categories = ["multimedia::audio"] +include = [ + "/resources", + "/src", + "LICENSE", + "README.md", +] +rust-version = "1.86" + +[dependencies] +almost = "0.2.0" +arc-swap = "1.6" +arrayvec = "0.7" +fft-convolver = "0.3" +cpal = { version = "0.18", optional = true } +creek = { version = "1.2", default-features = false, features = [ + "decode", + "decode-all", + "encode-wav", +] } +crossbeam-channel = "0.5" +cubeb = { version = "0.34", optional = true } +dasp_sample = "0.11" +float_eq = "1.0" +futures-channel = { version = "0.3.30", features = ["sink"] } +futures-core = { version = "0.3.30", default-features = false } +futures-util = { version = "0.3.30", default-features = false, features = [ + "sink", +] } +hound = "3.5" +hrtf = "0.9" +llq = "0.1.1" +log = "0.4" +num-complex = "0.4" +realfft = "3.3" +rubato = "0.16" +smallvec = "1.11" +symphonia = { version = "0.6", default-features = false, features = [ + "all", + "opt-simd", +] } +vecmath = "1.0" + +[target.'cfg(any(target_arch = "x86", target_arch = "x86_64", target_arch = "aarch64"))'.dependencies] +no_denormals = "0.3" + +[dev-dependencies] +futures = { version = "0.3.30", features = ["executor"] } +alloc_counter = "0.0.4" +criterion = "0.8" +env_logger = "0.11" +iai = { git = "https://github.com/sigaloid/iai", rev = "d56a597" } +rand = "0.10" +paste = "1.0.14" + +# Uncomment the following lines to enable debug symbols +# during CPU profiling +# [profile.release] +# debug = true + +[[bench]] +name = "my_benchmark" +# The path is required for excluding /benches from the package. +# See also: https://users.rust-lang.org/t/cargo-publish-with-excluded-benchmark-fails-validation/53444/2 +path = "benches/my_benchmark.rs" +harness = false + +[features] +default = ["cpal"] + +# Expose structured runtime diagnostics for AudioContext and the render graph. +diagnostics = [] + +# Make AudioWorklet inputs follow the Web Audio spec for inactive inputs: +# inputs[n] is an empty slice when no connected source is actively processing. +# This is opt-in for v1 because existing processors may assume a stable channel count. +spec-compliant-worklet-inputs = [] + +cpal = ["dep:cpal"] +cubeb = ["dep:cubeb"] +cpal-jack = ["cpal", "cpal/jack"] +cpal-pipewire = ["cpal", "cpal/pipewire"] +cpal-asio = ["cpal", "cpal/asio"] +iai = [] + +# Legacy codec feature names kept for backwards compatibility. +# Decoding support is no longer split by file format; all supported Symphonia +# and Creek decoders are enabled unconditionally. +mp3 = [] +ogg = [] +flac = [] +wav = [] +aac = [] +m4a = [] +alac = [] diff --git a/vendor/web-audio-api-1.7.0/LICENSE b/vendor/web-audio-api-1.7.0/LICENSE new file mode 100644 index 0000000000..616dc850ee --- /dev/null +++ b/vendor/web-audio-api-1.7.0/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2021 Otto Rottier + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/vendor/web-audio-api-1.7.0/MOLI_PATCHES.md b/vendor/web-audio-api-1.7.0/MOLI_PATCHES.md new file mode 100644 index 0000000000..aacf65ede4 --- /dev/null +++ b/vendor/web-audio-api-1.7.0/MOLI_PATCHES.md @@ -0,0 +1,26 @@ +# Moli's web-audio-api 1.7.0 patch set + +Source: the published `web-audio-api` 1.7.0 crate, upstream +, commit +`2af21af6ff2bbee17201b2dd2a5ebc56f55c3511`. + +Crate archive SHA-256: +`013903031fb735e7fd9aee7c159f3440a19c6969dc81365aec8c64caddcf39bc`. +The upstream MIT license is retained in `LICENSE`. Source, resources, README +and normalized/original manifests are otherwise copied from that archive. + +Local changes: + +1. `src/context/concrete_base.rs`: defer the HRTF database to the existing + control-thread `PannerNode::set_panning_model(HRTF)` path. Context construction + otherwise eagerly resamples the full database even with the `none` backend + and no panner. This can exhaust a renderer script deadline in debug builds. +2. `src/param.rs`: apply automation events at the quantum boundary before + publishing `AudioParam.value` and sampling k-rate. A k-rate boundary step + updates the scalar buffer; a step later in the quantum remains deferred. + The existing scalar fast path and k-rate computation are retained. + A regression exercises a boundary step and another step within the quantum. + +No host-audio backend is enabled by Moli. The native integration tests cover +PCM, FFT, automation, descriptors, JS reentrancy and document/context cleanup. +The upstream parameter unit tests should also be run when editing this patch. diff --git a/vendor/web-audio-api-1.7.0/README.md b/vendor/web-audio-api-1.7.0/README.md new file mode 100644 index 0000000000..c003aea761 --- /dev/null +++ b/vendor/web-audio-api-1.7.0/README.md @@ -0,0 +1,170 @@ +# Rust Web Audio API + +[![crates.io](https://img.shields.io/crates/v/web-audio-api.svg)](https://crates.io/crates/web-audio-api) +[![docs.rs](https://img.shields.io/docsrs/web-audio-api)](https://docs.rs/web-audio-api) + +A pure Rust implementation of the Web Audio API, for use in non-browser contexts + +## About the Web Audio API + +The [Web Audio API](https://www.w3.org/TR/webaudio/) +([MDN docs](https://developer.mozilla.org/en-US/docs/Web/API/Web_Audio_API)) +provides a powerful and versatile system for controlling audio on the Web, +allowing developers to choose audio sources, add effects to audio, create audio +visualizations, apply spatial effects (such as panning) and much more. + +Our Rust implementation decouples the Web Audio API from the Web. You can now +use it in desktop apps, command line utilities, headless execution, etc. + +## Example usage + +```rust,no_run +use web_audio_api::context::{AudioContext, BaseAudioContext}; +use web_audio_api::node::{AudioNode, AudioScheduledSourceNode}; + +// set up the audio context with optimized settings for your hardware +let context = AudioContext::default(); + +// for background music, read from local file +let file = std::fs::File::open("samples/major-scale.ogg").unwrap(); +let buffer = context.decode_audio_data_sync(file).unwrap(); + +// setup an AudioBufferSourceNode +let mut src = context.create_buffer_source(); +src.set_buffer(buffer); +src.set_loop(true); + +// create a biquad filter +let biquad = context.create_biquad_filter(); +biquad.frequency().set_value(125.); + +// connect the audio nodes +src.connect(&biquad); +biquad.connect(&context.destination()); + +// play the buffer +src.start(); + +// enjoy listening +std::thread::sleep(std::time::Duration::from_secs(4)); +``` + +Check out the [docs](https://docs.rs/web-audio-api) for more info. + +## Spec compliance + +We have tried to stick to the official W3C spec as close as possible, but some +deviations could not be avoided: + +- naming: snake_case instead of CamelCase +- getters/setters methods instead of exposed attributes +- introduced some namespacing +- inheritance is modelled with traits + +## Bindings + +We provide NodeJS bindings to this library over at + so you can use this library +by simply writing native NodeJS code. + +This enables us to run the official [WebAudioAPI test +harness](https://github.com/web-platform-tests/wpt/tree/master/webaudio) and +[track our spec compliance +score](https://github.com/ircam-ismm/node-web-audio-api/issues/57). + +## Audio backends + +By default, the [`cpal`](https://github.com/rustaudio/cpal) library is used for +cross platform audio I/O. + +We offer [experimental support](https://github.com/orottier/web-audio-api-rs/issues/187) for the +[`cubeb`](https://github.com/mozilla/cubeb-rs) backend via the `cubeb` feature +flag. Please note that `cmake` must be installed locally in order to run +`cubeb`. + +| Feature flag | Backends | +| -------------- | -------------------------------------------------------------- | +| cpal (default) | ALSA, WASAPI, CoreAudio, Oboe (Android) | +| cpal-jack | JACK | +| cpal-pipewire | PipeWire | +| cpal-asio | ASIO see | +| cubeb | PulseAudio, AudioUnit, WASAPI, OpenSL, AAudio, sndio, Sun, OSS | + +### Notes for Linux users + +Using the library on Linux with the ALSA backend might lead to unexpected +cranky sound with the default render size (i.e. 128 frames). In such cases, a +simple workaround is to pass the `AudioContextLatencyCategory::Playback` +latency hint when creating the audio context, which will increase the render +size to 1024 frames: + +```rs +let audio_context = AudioContext::new(AudioContextOptions { + latency_hint: AudioContextLatencyCategory::Playback, + ..AudioContextOptions::default() +}); +``` + +For real-time and interactive applications where low latency is crucial, you +should instead rely on the JACK backend provided by `cpal`. To that end you +will need a running JACK server and build your application with the `cpal-jack` +feature, e.g. `cargo run --release --features "cpal-jack" --example +microphone`. + +### Targeting the browser + +We can go full circle and pipe the Rust WebAudio output back into the browser +via `cpal`'s `wasm-bindgen` backend. Check out [an example WASM +project](https://github.com/orottier/wasm-web-audio-rs). +Warning: experimental! + +## Audio decoding support + +Audio decoding is powered by Symphonia. The current implementation enables all +Symphonia-supported audio formats and codecs, including AIFF, CAF, ISO/MP4, +MKV/WebM, Ogg, WAV, AAC, ADPCM, ALAC, FLAC, MP1/MP2/MP3, PCM, and Vorbis. + +## Contributing + +web-audio-api-rs welcomes contribution from everyone in the form of +suggestions, bug reports, pull requests, and feedback. 💛 + +To mirror the CI check locally, run: + +```sh +cargo fmt +cargo build --all-targets +cargo test --all-targets +cargo clippy --all-targets +``` + +The formatting and lint checks remain separate and can be run locally with: + +```sh +pre-commit run --all-files +``` + +If you need ideas for contribution, there are several ways to get started: + +- Try out some of our examples (located in the `examples/` directory) and start + building your own audio graphs +- Found a bug or have a feature request? + [Submit an issue](https://github.com/orottier/web-audio-api-rs/issues/new)! +- Issues labeled with + [good first issue](https://github.com/orottier/web-audio-api-rs/issues?q=is%3Aissue+is%3Aopen+sort%3Aupdated-desc+label%3A%22good+first+issue%22) + are relatively easy starter issues. + +Unless you explicitly state otherwise, any contribution intentionally submitted +for inclusion in web-audio-api-rs by you, shall be licensed as MIT, without any +additional terms or conditions. + +## License + +This project is licensed under the [MIT license]. + +[mit license]: https://github.com/orottier/web-audio-api-rs/blob/main/LICENSE + +## Acknowledgements + +The IR files used for HRTF spatialization are part of the LISTEN database +created by the EAC team from Ircam. diff --git a/vendor/web-audio-api-1.7.0/resources/IRC_1003_C.bin b/vendor/web-audio-api-1.7.0/resources/IRC_1003_C.bin new file mode 100644 index 0000000000..fd6ceae477 Binary files /dev/null and b/vendor/web-audio-api-1.7.0/resources/IRC_1003_C.bin differ diff --git a/vendor/web-audio-api-1.7.0/src/analysis.rs b/vendor/web-audio-api-1.7.0/src/analysis.rs new file mode 100644 index 0000000000..f68d7d4e68 --- /dev/null +++ b/vendor/web-audio-api-1.7.0/src/analysis.rs @@ -0,0 +1,842 @@ +//! Helpers for time domain and frequency analysis +//! +//! These are used in the [`AnalyserNode`](crate::node::AnalyserNode) + +use std::f32::consts::PI; +use std::sync::atomic::{AtomicUsize, Ordering}; +use std::sync::{Arc, Mutex}; + +use realfft::{num_complex::Complex, RealFftPlanner}; + +use crate::{AtomicF32, RENDER_QUANTUM_SIZE}; + +/// Blackman window values iterator with alpha = 0.16 +fn generate_blackman(size: usize) -> impl Iterator { + let alpha = 0.16; + let a0 = (1. - alpha) / 2.; + let a1 = 1. / 2.; + let a2 = alpha / 2.; + + (0..size).map(move |i| { + a0 - a1 * (2. * PI * i as f32 / size as f32).cos() + + a2 * (4. * PI * i as f32 / size as f32).cos() + }) +} + +pub(crate) const DEFAULT_SMOOTHING_TIME_CONSTANT: f64 = 0.8; +pub(crate) const DEFAULT_MIN_DECIBELS: f64 = -100.; +pub(crate) const DEFAULT_MAX_DECIBELS: f64 = -30.; +pub(crate) const DEFAULT_FFT_SIZE: usize = 2048; + +const MIN_FFT_SIZE: usize = 32; +const MAX_FFT_SIZE: usize = 32768; + +// [spec] This MUST be a power of two in the range 32 to 32768, otherwise an +// IndexSizeError exception MUST be thrown. +#[allow(clippy::manual_range_contains)] +fn assert_valid_fft_size(fft_size: usize) { + assert!( + fft_size.is_power_of_two(), + "IndexSizeError - Invalid fft size: {:?} is not a power of two", + fft_size + ); + + assert!( + fft_size >= MIN_FFT_SIZE && fft_size <= MAX_FFT_SIZE, + "IndexSizeError - Invalid fft size: {:?} is outside range [{:?}, {:?}]", + fft_size, + MIN_FFT_SIZE, + MAX_FFT_SIZE + ); +} + +// [spec] If the value of this attribute is set to a value less than 0 or more +// than 1, an IndexSizeError exception MUST be thrown. +#[allow(clippy::manual_range_contains)] +fn assert_valid_smoothing_time_constant(smoothing_time_constant: f64) { + assert!( + smoothing_time_constant >= 0. && smoothing_time_constant <= 1., + "IndexSizeError - Invalid smoothing time constant: {:?} is outside range [0, 1]", + smoothing_time_constant + ); +} + +// [spec] If the value of minDecibels is set to a value more than or equal to maxDecibels, an +// IndexSizeError exception MUST be thrown. +fn assert_valid_decibels(min_decibels: f64, max_decibels: f64) { + assert!( + min_decibels < max_decibels, + "IndexSizeError - Invalid min decibels: {:?} is greater than or equals to max decibels {:?}", + min_decibels, max_decibels + ); +} + +// as the queue is composed of AtomicF32 having only 1 render quantum of extra +// room should be enough +const RING_BUFFER_SIZE: usize = MAX_FFT_SIZE + RENDER_QUANTUM_SIZE; + +// single producer / multiple consumer ring buffer +#[derive(Clone)] +pub(crate) struct AnalyserRingBuffer { + buffer: Arc<[AtomicF32]>, + write_index: Arc, +} + +impl AnalyserRingBuffer { + pub fn new() -> Self { + let mut buffer = Vec::with_capacity(RING_BUFFER_SIZE); + buffer.resize_with(RING_BUFFER_SIZE, || AtomicF32::new(0.)); + + Self { + buffer: buffer.into(), + write_index: Arc::new(AtomicUsize::new(0)), + } + } + + pub fn write(&self, src: &[f32]) { + let mut write_index = self.write_index.load(Ordering::SeqCst); + let len = src.len(); + + src.iter().enumerate().for_each(|(index, value)| { + let position = (write_index + index) % RING_BUFFER_SIZE; + self.buffer[position].store(*value, Ordering::Relaxed); + }); + + write_index += len; + + if write_index >= RING_BUFFER_SIZE { + write_index -= RING_BUFFER_SIZE; + } + + self.write_index.store(write_index, Ordering::SeqCst); + } + + pub fn read(&self, dst: &mut [f32], max_len: usize) { + let write_index = self.write_index.load(Ordering::SeqCst); + // let fft_size = self.fft_size.load(Ordering::SeqCst); + let len = dst.len().min(max_len); + + dst.iter_mut() + .take(len) + .enumerate() + .for_each(|(index, value)| { + // offset calculation by RING_BUFFER_SIZE so we can't negative values + let position = (RING_BUFFER_SIZE + write_index - len + index) % RING_BUFFER_SIZE; + *value = self.buffer[position].load(Ordering::Relaxed); + }); + } + + // to simply share tests with the unsafe version + #[cfg(test)] + fn raw(&self) -> Vec { + let mut slice = vec![0.; RING_BUFFER_SIZE]; + + self.buffer.iter().zip(slice.iter_mut()).for_each(|(a, b)| { + *b = a.load(Ordering::SeqCst); + }); + + slice + } +} + +// As the analyser is wrapped into a Arc> by the analyser node to get interior +// mutability and expose an immutable public API, we should be ok with thread safety. +pub(crate) struct Analyser { + ring_buffer: AnalyserRingBuffer, + fft_size: usize, + smoothing_time_constant: f64, + min_decibels: f64, + max_decibels: f64, + fft_planner: Mutex>, // RealFftPlanner is not `Sync` on all platforms + fft_input: Vec, + fft_scratch: Vec>, + fft_output: Vec>, + last_fft_output: Vec, + last_fft_time: f64, + blackman: Vec, +} + +impl std::fmt::Debug for Analyser { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("Analyser") + .field("fft_size", &self.fft_size()) + .field("smoothing_time_constant", &self.smoothing_time_constant()) + .field("min_decibels", &self.min_decibels()) + .field("max_decibels", &self.max_decibels()) + .finish_non_exhaustive() + } +} + +impl Analyser { + pub fn new() -> Self { + let ring_buffer = AnalyserRingBuffer::new(); + // FFT utils + let mut fft_planner = RealFftPlanner::::new(); + let max_fft = fft_planner.plan_fft_forward(MAX_FFT_SIZE); + + let fft_input = max_fft.make_input_vec(); + let fft_scratch = max_fft.make_scratch_vec(); + let fft_output = max_fft.make_output_vec(); + let mut last_fft_output = Vec::with_capacity(fft_output.len()); + last_fft_output.resize_with(fft_output.len(), || 0.); + + // precalculate Blackman window values, reserve enough space for all input sizes + let mut blackman = Vec::with_capacity(fft_input.len()); + generate_blackman(DEFAULT_FFT_SIZE).for_each(|v| blackman.push(v)); + + Self { + ring_buffer, + fft_size: DEFAULT_FFT_SIZE, + smoothing_time_constant: DEFAULT_SMOOTHING_TIME_CONSTANT, + min_decibels: DEFAULT_MIN_DECIBELS, + max_decibels: DEFAULT_MAX_DECIBELS, + fft_planner: Mutex::new(fft_planner), + fft_input, + fft_scratch, + fft_output, + last_fft_output, + last_fft_time: f64::NEG_INFINITY, + blackman, + } + } + + pub fn get_ring_buffer_clone(&self) -> AnalyserRingBuffer { + self.ring_buffer.clone() + } + + pub fn fft_size(&self) -> usize { + self.fft_size + } + + pub fn set_fft_size(&mut self, fft_size: usize) { + assert_valid_fft_size(fft_size); + + let current_fft_size = self.fft_size; + + if current_fft_size != fft_size { + // reset last fft buffer + self.last_fft_output.iter_mut().for_each(|v| *v = 0.); + // generate blackman window + self.blackman.clear(); + generate_blackman(fft_size).for_each(|v| self.blackman.push(v)); + + self.fft_size = fft_size; + } + } + + pub fn smoothing_time_constant(&self) -> f64 { + self.smoothing_time_constant + } + + pub fn set_smoothing_time_constant(&mut self, value: f64) { + assert_valid_smoothing_time_constant(value); + self.smoothing_time_constant = value; + } + + pub fn min_decibels(&self) -> f64 { + self.min_decibels + } + + pub fn max_decibels(&self) -> f64 { + self.max_decibels + } + + pub fn set_decibels(&mut self, min: f64, max: f64) { + // set them together to avoid invalid intermediate min/max combinations + assert_valid_decibels(min, max); + + self.min_decibels = min; + self.max_decibels = max; + } + + pub fn frequency_bin_count(&self) -> usize { + self.fft_size() / 2 + } + + // [spec] Write the current time-domain data (waveform data) into array. + // If array has fewer elements than the value of fftSize, the excess elements + // will be dropped. If array has more elements than the value of fftSize, + // the excess elements will be ignored. The most recent fftSize frames are + // written (after downmixing) + pub fn get_float_time_domain_data(&self, dst: &mut [f32]) { + let fft_size = self.fft_size(); + self.ring_buffer.read(dst, fft_size); + } + + pub fn get_byte_time_domain_data(&self, dst: &mut [u8]) { + let fft_size = self.fft_size(); + let mut tmp = vec![0.; dst.len()]; + self.ring_buffer.read(&mut tmp, fft_size); + + dst.iter_mut().zip(tmp.iter()).for_each(|(o, i)| { + let scaled = 128. * (1. + i); + let clamped = scaled.clamp(0., 255.); + *o = clamped as u8; + }); + } + + fn compute_fft(&mut self) { + let fft_size = self.fft_size(); + let smoothing_time_constant = self.smoothing_time_constant() as f32; + // setup FFT planner and properly sized buffers + let r2c = self.fft_planner.lock().unwrap().plan_fft_forward(fft_size); + let input = &mut self.fft_input[..fft_size]; + let output = &mut self.fft_output[..fft_size / 2 + 1]; + let scratch = &mut self.fft_scratch[..r2c.get_scratch_len()]; + // we ignore the Nyquist bin in output, see comment below + let last_fft_output = &mut self.last_fft_output[..fft_size / 2]; + + // Compute the current time-domain data. + // The most recent fftSize frames are used in computing the frequency data. + self.ring_buffer.read(input, fft_size); + + // Apply a Blackman window to the time domain input data. + input + .iter_mut() + .zip(self.blackman.iter()) + .for_each(|(i, b)| *i *= *b); + + // Apply a Fourier transform to the windowed time domain input data to + // get real and imaginary frequency data. + r2c.process_with_scratch(input, output, scratch).unwrap(); + + // Notes from chromium source code (tbc) + // + // cf. third_party/blink/renderer/platform/audio/fft_frame.h" + // ``` + // Since x[n] is assumed to be real, X[k] has complex conjugate symmetry with + // X[N-k] = conj(X[k]). Thus, we only need to keep X[k] for k = 0 to N/2. + // But since X[0] is purely real and X[N/2] is also purely real, so we could + // place the real part of X[N/2] in the imaginary part of X[0]. Thus + // for k = 1 to N/2: + // + // real_data[k] = Re(X[k]) + // imag_data[k] = Im(X[k]) + // + // and + // + // real_data[0] = Re(X[0]); + // imag_data[0] = Re(X[N/2]) + // ``` + // + // It seems to be why their FFT return only `fft_size / 2` components + // instead `fft_size * 2 + 1`, they pack DC and Nyquist bins together. + // + // However in their `realtime_analyser` they then remove the packed nyquist + // imaginary component: + // cf. third_party/blink/renderer/modules/webaudio/realtime_analyser.h + // ``` + // // Blow away the packed nyquist component. + // imag[0] = 0; + // ``` + // In our case, it seems we can thus just ignore the Nyquist information + // and take the DC bin as it is + + let normalize_factor = 1. / fft_size as f32; + + last_fft_output + .iter_mut() + .zip(output.iter()) + .for_each(|(o, c)| { + let norm = c.norm() * normalize_factor; + let value = smoothing_time_constant * *o + (1. - smoothing_time_constant) * norm; + *o = if value.is_finite() { value } else { 0. }; + }); + } + + pub fn get_float_frequency_data(&mut self, dst: &mut [f32], current_time: f64) { + let frequency_bin_count = self.frequency_bin_count(); + + // [spec] If another call to getByteFrequencyData() or getFloatFrequencyData() + // occurs within the same render quantum as a previous call, the current + // frequency data is not updated with the same data. Instead, the previously + // computed data is returned. + if current_time != self.last_fft_time { + self.compute_fft(); + self.last_fft_time = current_time; + } + + // [spec] Write the current frequency data into array. If array’s byte + // length is less than frequencyBinCount, the excess elements will be + // dropped. If array’s byte length is greater than the frequencyBinCount + let len = dst.len().min(frequency_bin_count); + + // Convert to dB. + dst.iter_mut() + .take(len) + .zip(self.last_fft_output.iter()) + .for_each(|(v, b)| *v = 20. * b.log10()); + } + + pub fn get_byte_frequency_data(&mut self, dst: &mut [u8], current_time: f64) { + let frequency_bin_count = self.frequency_bin_count(); + let min_decibels = self.min_decibels() as f32; + let max_decibels = self.max_decibels() as f32; + + // [spec] If another call to getByteFrequencyData() or getFloatFrequencyData() + // occurs within the same render quantum as a previous call, the current + // frequency data is not updated with the same data. Instead, the previously + // computed data is returned. + if current_time != self.last_fft_time { + self.compute_fft(); + self.last_fft_time = current_time; + } + + // [spec] Write the current frequency data into array. If array’s byte + // length is less than frequencyBinCount, the excess elements will be + // dropped. If array’s byte length is greater than the frequencyBinCount + let len = dst.len().min(frequency_bin_count); + + // Convert to dB and convert / scale to u8 + dst.iter_mut() + .take(len) + .zip(self.last_fft_output.iter()) + .for_each(|(v, b)| { + let db = 20. * b.log10(); + // 𝑏[𝑘] = ⌊255 / dB𝑚𝑎𝑥−dB𝑚𝑖𝑛 * (𝑌[𝑘]−dB𝑚𝑖𝑛)⌋ + let scaled = 255. / (max_decibels - min_decibels) * (db - min_decibels); + let clamped = scaled.clamp(0., 255.); + *v = clamped as u8; + }); + } +} + +#[cfg(test)] +mod tests { + use std::sync::RwLock; + use std::thread; + + use float_eq::{assert_float_eq, float_eq}; + use rand::RngExt; + + use super::*; + + #[test] + fn test_blackman() { + let values: Vec = generate_blackman(2048).collect(); + + let min = values + .iter() + .fold(1000., |min, &val| if val < min { val } else { min }); + let max = values + .iter() + .fold(0., |max, &val| if val > max { val } else { max }); + assert!(min < 0.01 && min > 0.); + assert!(max > 0.99 && max <= 1.); + + let min_pos = values + .iter() + .position(|&v| float_eq!(v, min, abs_all <= 0.)) + .unwrap(); + let max_pos = values + .iter() + .position(|&v| float_eq!(v, max, abs_all <= 0.)) + .unwrap(); + assert_eq!(min_pos, 0); + assert_eq!(max_pos, 1024); + } + + #[test] + fn test_ring_buffer_write_simple() { + let ring_buffer = AnalyserRingBuffer::new(); + + // check index update + { + // fill the buffer twice so we check the buffer wrap + for i in 1..3 { + for j in 0..(RING_BUFFER_SIZE / RENDER_QUANTUM_SIZE) { + let data = [i as f32; RENDER_QUANTUM_SIZE]; + ring_buffer.write(&data); + + // check write index is properly updated + let write_index = ring_buffer.write_index.load(Ordering::SeqCst); + let expected = + (j * RENDER_QUANTUM_SIZE + RENDER_QUANTUM_SIZE) % RING_BUFFER_SIZE; + + assert_eq!(write_index, expected); + } + + // for each loop check the ring buffer is properly filled + let expected = [i as f32; RING_BUFFER_SIZE]; + + assert_float_eq!(&ring_buffer.raw()[..], &expected[..], abs_all <= 1e-12); + } + } + } + + #[test] + fn test_ring_buffer_write_wrap() { + // check values are written in right place + { + let ring_buffer = AnalyserRingBuffer::new(); + + let offset = 10; + ring_buffer + .write_index + .store(RING_BUFFER_SIZE - offset, Ordering::SeqCst); + + let data = [1.; RENDER_QUANTUM_SIZE]; + ring_buffer.write(&data); + + let mut expected = [0.; RING_BUFFER_SIZE]; + + expected.iter_mut().enumerate().for_each(|(index, v)| { + if index < RENDER_QUANTUM_SIZE - offset || index >= RING_BUFFER_SIZE - offset { + *v = 1. + } else { + *v = 0. + } + }); + + assert_float_eq!(&ring_buffer.raw()[..], &expected[..], abs_all <= 1e-12); + } + + // check values are written in right order + { + let ring_buffer = AnalyserRingBuffer::new(); + let offset = 2; + ring_buffer + .write_index + .store(RING_BUFFER_SIZE - offset, Ordering::SeqCst); + + let data = [1., 2., 3., 4.]; + ring_buffer.write(&data); + + let mut expected = [0.; RING_BUFFER_SIZE]; + expected[RING_BUFFER_SIZE - 2] = 1.; + expected[RING_BUFFER_SIZE - 1] = 2.; + expected[0] = 3.; + expected[1] = 4.; + + assert_float_eq!(&ring_buffer.raw()[..], &expected[..], abs_all <= 1e-12); + } + } + + #[test] + fn test_ring_buffer_read_simple() { + let ring_buffer = Arc::new(AnalyserRingBuffer::new()); + + // first pass + let data = [1.; RENDER_QUANTUM_SIZE]; + ring_buffer.write(&data); + + // index is where it should be + let index = ring_buffer.write_index.load(Ordering::SeqCst); + assert_eq!(index, RENDER_QUANTUM_SIZE); + + let mut read_buffer = [0.; RENDER_QUANTUM_SIZE]; + ring_buffer.read(&mut read_buffer, RENDER_QUANTUM_SIZE); + // data is good + let expected = [1.; RENDER_QUANTUM_SIZE]; + assert_float_eq!(&expected, &read_buffer, abs_all <= 1e-12); + + // second pass + let data = [2.; RENDER_QUANTUM_SIZE]; + ring_buffer.write(&data); + + // index is where it should be + let index = ring_buffer.write_index.load(Ordering::SeqCst); + assert_eq!(index, RENDER_QUANTUM_SIZE * 2); + + let mut read_buffer = [0.; RENDER_QUANTUM_SIZE]; + ring_buffer.read(&mut read_buffer, RENDER_QUANTUM_SIZE); + + let expected = [2.; RENDER_QUANTUM_SIZE]; + assert_float_eq!(&expected, &read_buffer, abs_all <= 1e-12); + + let mut full_buffer_expected = [0.; RING_BUFFER_SIZE]; + full_buffer_expected[0..RENDER_QUANTUM_SIZE].copy_from_slice(&[1.; RENDER_QUANTUM_SIZE]); + + full_buffer_expected[RENDER_QUANTUM_SIZE..(RENDER_QUANTUM_SIZE * 2)] + .copy_from_slice(&[2.; RENDER_QUANTUM_SIZE]); + + assert_float_eq!( + &ring_buffer.raw()[..], + &full_buffer_expected[..], + abs_all <= 1e-12 + ); + } + + #[test] + fn test_ring_buffer_read_unwrap() { + // check values are read from right place + { + let ring_buffer = AnalyserRingBuffer::new(); + + let offset = 10; + ring_buffer + .write_index + .store(RING_BUFFER_SIZE - offset, Ordering::SeqCst); + + let data = [1.; RENDER_QUANTUM_SIZE]; + ring_buffer.write(&data); + + let mut read_buffer = [0.; RENDER_QUANTUM_SIZE]; + ring_buffer.read(&mut read_buffer, RENDER_QUANTUM_SIZE); + + assert_float_eq!(&read_buffer, &data, abs_all <= 1e-12); + } + + // check values are read from right place and written in right order + { + let ring_buffer = AnalyserRingBuffer::new(); + let offset = 2; + ring_buffer + .write_index + .store(RING_BUFFER_SIZE - offset, Ordering::SeqCst); + + let data = [1., 2., 3., 4.]; + ring_buffer.write(&data); + + let mut read_buffer = [0.; 4]; + ring_buffer.read(&mut read_buffer, RENDER_QUANTUM_SIZE); + + assert_float_eq!(&read_buffer, &[1., 2., 3., 4.], abs_all <= 1e-12); + } + } + + #[test] + fn test_set_decibels() { + let mut analyser = Analyser::new(); + analyser.set_decibels(-20., 10.); + assert_eq!(analyser.min_decibels(), -20.); + assert_eq!(analyser.max_decibels(), 10.); + } + + #[test] + #[should_panic] + fn test_fft_size_constraints_power_of_two() { + let mut analyser = Analyser::new(); + analyser.set_fft_size(13); + } + + #[test] + #[should_panic] + fn test_fft_size_constraints_ge_min_fft_size() { + let mut analyser = Analyser::new(); + analyser.set_fft_size(MIN_FFT_SIZE / 2); + } + + #[test] + #[should_panic] + fn test_fft_size_constraints_le_max_fft_size() { + let mut analyser = Analyser::new(); + analyser.set_fft_size(MAX_FFT_SIZE * 2); + } + + #[test] + #[should_panic] + fn test_smoothing_time_constant_constraints_lt_zero() { + let mut analyser = Analyser::new(); + analyser.set_smoothing_time_constant(-1.); + } + + #[test] + #[should_panic] + fn test_smoothing_time_constant_constraints_gt_one() { + let mut analyser = Analyser::new(); + analyser.set_smoothing_time_constant(2.); + } + + #[test] + #[should_panic] + fn test_min_decibels_constraints_lt_max_decibels() { + let mut analyser = Analyser::new(); + analyser.set_decibels(DEFAULT_MAX_DECIBELS, analyser.max_decibels()); + } + + #[test] + #[should_panic] + fn test_max_decibels_constraints_lt_min_decibels() { + let mut analyser = Analyser::new(); + analyser.set_decibels(analyser.min_decibels(), DEFAULT_MIN_DECIBELS); + } + + #[test] + fn test_get_float_time_domain_data_vs_fft_size() { + // dst is bigger than fft_size + { + let mut analyser = Analyser::new(); + analyser.set_fft_size(32); + + let data = [1.; RENDER_QUANTUM_SIZE]; + let buffer = analyser.get_ring_buffer_clone(); + buffer.write(&data); + + let mut dst = [0.; RENDER_QUANTUM_SIZE]; + analyser.get_float_time_domain_data(&mut dst); + + let mut expected = [0.; RENDER_QUANTUM_SIZE]; + expected.iter_mut().take(32).for_each(|v| *v = 1.); + + assert_float_eq!(&dst, &expected, abs_all <= 0.); + } + + // dst is smaller than fft_size + { + let mut analyser = Analyser::new(); + analyser.set_fft_size(128); + + let data = [1.; RENDER_QUANTUM_SIZE]; + let buffer = analyser.get_ring_buffer_clone(); + buffer.write(&data); + + let mut dst = [0.; 16]; + analyser.get_float_time_domain_data(&mut dst); + + let expected = [1.; 16]; + + assert_float_eq!(&dst, &expected, abs_all <= 0.); + } + } + + #[test] + fn get_byte_time_domain_data() { + let analyser = Analyser::new(); + + let data = [1.; RENDER_QUANTUM_SIZE]; + let buffer = analyser.get_ring_buffer_clone(); + buffer.write(&data); + + let mut dst = [0; RENDER_QUANTUM_SIZE]; + analyser.get_byte_time_domain_data(&mut dst); + + let expected = [255; RENDER_QUANTUM_SIZE]; + + assert_eq!(&dst, &expected); + + let data = [-1.; RENDER_QUANTUM_SIZE]; + let buffer = analyser.get_ring_buffer_clone(); + buffer.write(&data); + + let mut dst = [0; RENDER_QUANTUM_SIZE]; + analyser.get_byte_time_domain_data(&mut dst); + + let expected = [0; RENDER_QUANTUM_SIZE]; + + assert_eq!(&dst, &expected); + } + + #[test] + fn test_get_float_frequency_data() { + // from https://support.ircam.fr/docs/AudioSculpt/3.0/co/Window%20Size.html + // Let's take a 44100 sampling rate. SR=44100 Hz, F(max) = 22050 Hz. + // With a 1024 window size (512 bins), we get . + // Freq Resolution = 44100/1024 = 43.066 + let sample_rate = 44100.; + let fft_size = 1024; + let freq_resolution = 43.066; + + // note: we don't check all the bin range to keep low tests time + for num_bin in 1..(fft_size / 8) { + // create sines whose frequency centered on `num_bin` bin, we should + // the have highest value in `num_bin` bin + // @note (tbc): bin 0 seems to represent freq_resolution / 2 + let freq = freq_resolution * num_bin as f32; + + let mut analyser = Analyser::new(); + analyser.set_fft_size(fft_size); + + let mut signal = Vec::::with_capacity(fft_size); + + for i in 0..fft_size { + let phase = freq * i as f32 / sample_rate; + let sample = (phase * 2. * PI).sin(); + signal.push(sample); + } + + let ring_buffer = analyser.get_ring_buffer_clone(); + ring_buffer.write(&signal); + + let mut bins = vec![0.; analyser.frequency_bin_count()]; + analyser.get_float_frequency_data(&mut bins[..], 0.); + + let highest = bins[num_bin]; + + bins.iter().enumerate().for_each(|(index, db)| { + if index != num_bin { + assert!(db < &highest); + } + }); + } + } + + #[test] + fn test_get_float_frequency_data_vs_frequenc_bin_count() { + let mut analyser = Analyser::new(); + analyser.set_fft_size(RENDER_QUANTUM_SIZE); + + // get data, should be zero (negative infinity decibel) + let mut bins = vec![-1.; RENDER_QUANTUM_SIZE]; + analyser.get_float_frequency_data(&mut bins[..], 0.); + + // only N / 2 values should contain frequency data, rest is unaltered + assert!( + bins[0..(RENDER_QUANTUM_SIZE / 2)] == [f32::NEG_INFINITY; (RENDER_QUANTUM_SIZE / 2)] + ); + assert_float_eq!( + &bins[(RENDER_QUANTUM_SIZE / 2)..], + &[-1.; (RENDER_QUANTUM_SIZE / 2)][..], + abs_all <= 0. + ); + } + + #[test] + fn test_get_byte_frequency_data_vs_frequenc_bin_count() { + let mut analyser = Analyser::new(); + analyser.set_fft_size(RENDER_QUANTUM_SIZE); + + // get data, should be zero (negative infinity decibel) + let mut bins = [255; RENDER_QUANTUM_SIZE]; + analyser.get_byte_frequency_data(&mut bins[..], 0.); + + // only N / 2 values should contain frequency data, rest is unaltered + assert!(bins[0..(RENDER_QUANTUM_SIZE / 2)] == [0; (RENDER_QUANTUM_SIZE / 2)]); + assert!(bins[(RENDER_QUANTUM_SIZE / 2)..] == [255; (RENDER_QUANTUM_SIZE / 2)][..],); + } + + // this mostly tries to show that it works concurrently and we don't fall into + // SEGFAULT traps or something, but this is difficult to really test something + // in an accurante way, other tests are there for such thing + #[test] + fn test_ring_buffer_concurrency() { + let analyser = Arc::new(Analyser::new()); + let ring_buffer = analyser.get_ring_buffer_clone(); + let num_loops = 10_000; + let (sender, receiver) = crossbeam_channel::bounded(1); + + thread::spawn(move || { + let mut rng = rand::rng(); + sender.send(()).unwrap(); // signal ready + + for _ in 0..num_loops { + let rand = rng.random::(); + let data = [rand; RENDER_QUANTUM_SIZE]; + ring_buffer.write(&data); + + std::thread::sleep(std::time::Duration::from_nanos(30)); + } + }); + + // wait for thread to boot + receiver.recv().unwrap(); + + for _ in 0..num_loops { + let mut read_buffer = [0.; RENDER_QUANTUM_SIZE]; + analyser.get_float_time_domain_data(&mut read_buffer); + std::thread::sleep(std::time::Duration::from_nanos(25)); + } + } + + #[test] + fn test_thread_safety() { + let analyser = Arc::new(RwLock::new(Analyser::new())); + + let handle = thread::spawn(move || { + analyser.write().unwrap().set_fft_size(MIN_FFT_SIZE); + assert_eq!(analyser.write().unwrap().fft_size(), MIN_FFT_SIZE); + }); + + handle.join().unwrap(); + } +} diff --git a/vendor/web-audio-api-1.7.0/src/buffer.rs b/vendor/web-audio-api-1.7.0/src/buffer.rs new file mode 100644 index 0000000000..60561b1dde --- /dev/null +++ b/vendor/web-audio-api-1.7.0/src/buffer.rs @@ -0,0 +1,817 @@ +//! General purpose audio signal data structures +use std::sync::Arc; + +use crate::{ + assert_valid_buffer_length, assert_valid_channel_number, assert_valid_number_of_channels, + assert_valid_sample_rate, +}; + +/// Options for constructing an [`AudioBuffer`] +// dictionary AudioBufferOptions { +// unsigned long numberOfChannels = 1; +// required unsigned long length; +// required float sampleRate; +// }; +#[derive(Clone, Debug)] +pub struct AudioBufferOptions { + /// The number of channels for the buffer + pub number_of_channels: usize, + /// The length in sample frames of the buffer + pub length: usize, + /// The sample rate in Hz for the buffer + pub sample_rate: f32, +} + +/// Memory-resident audio asset, basically a matrix of channels * samples +/// +/// An AudioBuffer has copy-on-write semantics, so it is cheap to clone. +/// +/// - MDN documentation: +/// - specification: +/// - see also: [`BaseAudioContext::create_buffer`](crate::context::BaseAudioContext::create_buffer) +/// +/// # Usage +/// +/// ```no_run +/// use std::f32::consts::PI; +/// use web_audio_api::context::{AudioContext, BaseAudioContext}; +/// use web_audio_api::node::{AudioNode, AudioScheduledSourceNode}; +/// +/// let context = AudioContext::default(); +/// +/// let length = context.sample_rate() as usize; +/// let sample_rate = context.sample_rate(); +/// let mut buffer = context.create_buffer(1, length, sample_rate); +/// +/// // fill buffer with a sine wave +/// let mut sine = vec![]; +/// +/// for i in 0..length { +/// let phase = i as f32 / length as f32 * 2. * PI * 200.; +/// sine.push(phase.sin()); +/// } +/// +/// buffer.copy_to_channel(&sine, 0); +/// +/// // play the buffer in a loop +/// let mut src = context.create_buffer_source(); +/// src.set_buffer(buffer.clone()); +/// src.set_loop(true); +/// src.connect(&context.destination()); +/// src.start(); +/// ``` +/// +/// # Example +/// +/// - `cargo run --release --example audio_buffer` +/// +#[derive(Clone, Debug)] +pub struct AudioBuffer { + pub(crate) channels: Vec, + pub(crate) sample_rate: f32, +} + +impl AudioBuffer { + /// Allocate a silent audiobuffer with [`AudioBufferOptions`] + /// + /// # Panics + /// + /// This function will panic if: + /// - the given sample rate is zero + /// - the given number of channels is outside the [1, 32] range, + /// 32 being defined by the MAX_CHANNELS constant. + pub fn new(options: AudioBufferOptions) -> Self { + assert_valid_sample_rate(options.sample_rate); + assert_valid_buffer_length(options.length); + assert_valid_number_of_channels(options.number_of_channels); + + let silence = ChannelData::new(options.length); + + Self { + channels: vec![silence; options.number_of_channels], + sample_rate: options.sample_rate, + } + } + + /// Convert raw samples to an AudioBuffer + /// + /// The outer Vec determine the channels. The inner Vecs should have the same length. + /// + /// # Panics + /// + /// This function will panic if: + /// - the given sample rate is zero + /// - the given number of channels defined by `samples.len()`is outside the + /// [1, 32] range, 32 being defined by the MAX_CHANNELS constant. + /// - any of its items have different lengths + pub fn from(samples: Vec>, sample_rate: f32) -> Self { + assert_valid_sample_rate(sample_rate); + assert_valid_number_of_channels(samples.len()); + + let channels: Vec<_> = samples.into_iter().map(ChannelData::from).collect(); + if !channels.iter().all(|c| c.len() == channels[0].len()) { + panic!("Trying to create AudioBuffer from channel data with unequal length"); + } + + Self { + channels, + sample_rate, + } + } + + /// Number of channels in this `AudioBuffer` + pub fn number_of_channels(&self) -> usize { + self.channels.len() + } + + /// Number of samples per channel in this `AudioBuffer` + pub fn length(&self) -> usize { + self.channels.first().map(ChannelData::len).unwrap_or(0) + } + + /// Sample rate of this `AudioBuffer` in Hertz + pub fn sample_rate(&self) -> f32 { + self.sample_rate + } + + /// Duration in seconds of the `AudioBuffer` + pub fn duration(&self) -> f64 { + self.length() as f64 / self.sample_rate as f64 + } + + /// Copy data from a given channel to the given `Vec` + /// + /// # Panics + /// + /// This function will panic if `channel_number` is greater or equal than + /// `AudioBuffer::number_of_channels()` + pub fn copy_from_channel(&self, destination: &mut [f32], channel_number: usize) { + assert_valid_channel_number(channel_number, self.number_of_channels()); + + self.copy_from_channel_with_offset(destination, channel_number, 0); + } + + /// Copy data from a given channel to the given `Vec` starting at `offset` + /// + /// # Panics + /// + /// This function will panic if: + /// - the given channel number is greater than or equal to the given number of channels. + pub fn copy_from_channel_with_offset( + &self, + destination: &mut [f32], + channel_number: usize, + offset: usize, + ) { + assert_valid_channel_number(channel_number, self.number_of_channels()); + let offset = offset.min(self.length()); + // [spec] Let buffer be the AudioBuffer with 𝑁𝑏 frames, let 𝑁𝑓 be the number + // of elements in the destination array, and 𝑘 be the value of bufferOffset. + // Then the number of frames copied from buffer to destination is max(0,min(𝑁𝑏−𝑘,𝑁𝑓)). + // If this is less than 𝑁𝑓, then the remaining elements of destination are not modified. + let dest_length = destination.len(); + let max_frame = (self.length() - offset).clamp(0, dest_length); + let channel = self.channel_data(channel_number).as_slice(); + + destination[..max_frame].copy_from_slice(&channel[offset..(max_frame + offset)]); + } + + /// Copy data from a given source to the given channel. + /// + /// # Panics + /// + /// This function will panic if: + /// - the given channel number is greater than or equal to the given number of channels. + pub fn copy_to_channel(&mut self, source: &[f32], channel_number: usize) { + assert_valid_channel_number(channel_number, self.number_of_channels()); + + self.copy_to_channel_with_offset(source, channel_number, 0); + } + + /// Copy data from a given source to the given channel starting at `offset`. + /// + /// # Panics + /// + /// This function will panic if: + /// - the given channel number is greater than or equal to the given number of channels. + pub fn copy_to_channel_with_offset( + &mut self, + source: &[f32], + channel_number: usize, + offset: usize, + ) { + assert_valid_channel_number(channel_number, self.number_of_channels()); + let offset = offset.min(self.length()); + // [spec] Let buffer be the AudioBuffer with 𝑁𝑏 frames, let 𝑁𝑓 be the number + // of elements in the source array, and 𝑘 be the value of bufferOffset. Then + // the number of frames copied from source to the buffer is max(0,min(𝑁𝑏−𝑘,𝑁𝑓)). + // If this is less than 𝑁𝑓, then the remaining elements of buffer are not modified. + let src_len = source.len(); + let max_frame = (self.length() - offset).clamp(0, src_len); + let channel = self.channel_data_mut(channel_number).as_mut_slice(); + + channel[offset..(max_frame + offset)].copy_from_slice(&source[..max_frame]); + } + + /// Return a read-only copy of the underlying data of the channel + /// + /// # Panics + /// + /// This function will panic if: + /// - the given channel number is greater than or equal to the given number of channels. + pub fn get_channel_data(&self, channel_number: usize) -> &[f32] { + assert_valid_channel_number(channel_number, self.number_of_channels()); + // [spec] According to the rules described in acquire the content either allow writing + // into or getting a copy of the bytes stored in [[internal data]] in a new Float32Array + self.channel_data(channel_number).as_slice() + } + + /// Return a mutable slice of the underlying data of the channel + /// + /// # Panics + /// + /// This function will panic if: + /// - the given channel number is greater than or equal to the given number of channels. + pub fn get_channel_data_mut(&mut self, channel_number: usize) -> &mut [f32] { + assert_valid_channel_number(channel_number, self.number_of_channels()); + self.channel_data_mut(channel_number).as_mut_slice() + } + + /// Create a multi-channel audiobuffer directly from `ChannelData`s. + // @todo - remove in favor of `AudioBuffer::from` + pub(crate) fn from_channels(channels: Vec, sample_rate: f32) -> Self { + Self { + channels, + sample_rate, + } + } + + /// Channel data as slice + pub(crate) fn channels(&self) -> &[ChannelData] { + &self.channels + } + + /// Channel data as slice (mutable) + pub(crate) fn channels_mut(&mut self) -> &mut [ChannelData] { + &mut self.channels + } + + /// Get the samples from this specific channel. + /// + /// Panics if the index is greater than the available number of channels + // @note - this one is used in + pub(crate) fn channel_data(&self, index: usize) -> &ChannelData { + &self.channels[index] + } + + /// Get the samples (mutable) from this specific channel. + /// + /// Panics if the index is greater than the available number of channels + pub(crate) fn channel_data_mut(&mut self, index: usize) -> &mut ChannelData { + &mut self.channels[index] + } + + /// Extends an AudioBuffer with the contents of another. + /// + /// This function will panic if the sample_rate and channel_count are not equal + pub(crate) fn extend(&mut self, other: &Self) { + assert_eq!(self.sample_rate, other.sample_rate); + assert_eq!(self.number_of_channels(), other.number_of_channels()); + + let data = self.channels_mut(); + data.iter_mut() + .zip(other.channels.iter()) + .for_each(|(channel, other_channel)| { + let cur_channel_data = Arc::make_mut(&mut channel.data); + cur_channel_data.extend(other_channel.as_slice()); + }) + } + + /// Split an AudioBuffer in two at the given index. + pub(crate) fn split_off(&mut self, index: usize) -> Self { + let channels: Vec<_> = self + .channels_mut() + .iter_mut() + .map(|channel_data| Arc::make_mut(&mut channel_data.data).split_off(index)) + .map(ChannelData::from) + .collect(); + + AudioBuffer::from_channels(channels, self.sample_rate) + } + + /// Resample to the desired sample rate. The method performs a simple linear + /// interpolation an keep the first and last sample intact. The new number + /// of samples is always ceiled according the ratio defined by old and new + /// sample rates. + /// + /// # Panics + /// + /// This function will panic if: + /// - the given sample rate is zero + pub(crate) fn resample(&mut self, sample_rate: f32) { + assert_valid_sample_rate(sample_rate); + + // if requested sample rate is very similar, do not resample + if float_eq::float_eq!(self.sample_rate, sample_rate, abs <= 0.1) { + self.sample_rate = sample_rate; + return; + } + + // handle zero length case + if self.length() == 0 { + self.sample_rate = sample_rate; + return; + } + + let source_sr = self.sample_rate as f64; + let target_sr = sample_rate as f64; + let ratio = target_sr / source_sr; + let source_length = self.length(); + let target_length = (self.length() as f64 * ratio).ceil() as usize; + + let num_channels = self.number_of_channels(); + let mut resampled = Vec::>::with_capacity(num_channels); + resampled.resize_with(num_channels, || Vec::::with_capacity(target_length)); + + for i in 0..target_length { + let position = i as f64 / (target_length - 1) as f64; // [0., 1.] + let playhead = position * (source_length - 1) as f64; + let playhead_floored = playhead.floor(); + let prev_index = playhead_floored as usize; + let next_index = (prev_index + 1).min(source_length - 1); + + let k = (playhead - playhead_floored) as f32; + let k_inv = 1. - k; + + for (channel, resampled_data) in resampled.iter_mut().enumerate() { + let prev_sample = self.channels[channel].data[prev_index]; + let next_sample = self.channels[channel].data[next_index]; + + let value = k_inv * prev_sample + k * next_sample; + resampled_data.push(value); + } + } + + self.channels + .iter_mut() + .zip(resampled) + .for_each(|(channel_data, resampled_data)| { + channel_data.data = Arc::new(resampled_data); + }); + + self.sample_rate = sample_rate; + } +} + +/// Single channel audio samples, basically wraps a `Arc>` +/// +/// ChannelData has copy-on-write semantics, so it is cheap to clone. +#[derive(Clone, PartialEq)] +pub(crate) struct ChannelData { + data: Arc>, +} + +impl std::fmt::Debug for ChannelData { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("ChannelData") + .field("len", &self.len()) + .finish_non_exhaustive() + } +} + +impl ChannelData { + pub fn new(length: usize) -> Self { + let buffer = vec![0.; length]; + let data = Arc::new(buffer); + + Self { data } + } + + pub fn from(data: Vec) -> Self { + Self { + data: Arc::new(data), + } + } + + pub fn len(&self) -> usize { + self.data.len() + } + + // clippy wants to keep it, so keep it :) + #[allow(dead_code)] + pub fn is_empty(&self) -> bool { + self.data.is_empty() + } + + pub fn as_slice(&self) -> &[f32] { + &self.data[..] + } + + pub fn as_mut_slice(&mut self) -> &mut [f32] { + &mut Arc::make_mut(&mut self.data)[..] + } +} + +#[cfg(test)] +mod tests { + use float_eq::assert_float_eq; + use std::f32::consts::PI; + + use super::*; + + #[test] + fn test_constructor() { + let options = AudioBufferOptions { + number_of_channels: 1, + length: 96000, + sample_rate: 48000., + }; + + let audio_buffer = AudioBuffer::new(options); + + assert_eq!(audio_buffer.number_of_channels(), 1); + assert_eq!(audio_buffer.length(), 96000); + assert_float_eq!(audio_buffer.sample_rate(), 48000., abs <= 0.); + assert_float_eq!(audio_buffer.duration(), 2., abs <= 0.); + } + + #[test] + #[should_panic] + fn test_zero_channels() { + let options = AudioBufferOptions { + number_of_channels: 0, + length: 10, + sample_rate: 48000., + }; + + AudioBuffer::new(options); // should panic + } + + #[test] + #[should_panic] + fn test_zero_channels_from() { + let samples = vec![]; + let sample_rate = 48000.; + + AudioBuffer::from(samples, sample_rate); // should panic + } + + #[test] + #[should_panic] + fn test_invalid_sample_rate() { + let options = AudioBufferOptions { + number_of_channels: 1, + length: 10, + sample_rate: 0., + }; + + AudioBuffer::new(options); // should panic + } + + #[test] + #[should_panic] + fn test_invalid_sample_rate_from() { + let samples = vec![vec![0.]]; + let sample_rate = 0.; + + AudioBuffer::from(samples, sample_rate); // should panic + } + + #[test] + #[should_panic] + fn test_invalid_length() { + let options = AudioBufferOptions { + number_of_channels: 1, + length: 0, + sample_rate: 48000., + }; + + AudioBuffer::new(options); // should panic + } + + #[test] + fn test_channel_data_get_set() { + let options = AudioBufferOptions { + number_of_channels: 1, + length: 10, + sample_rate: 48000., + }; + + let mut audio_buffer = AudioBuffer::new(options); + audio_buffer.channel_data_mut(0).as_mut_slice().fill(1.); + assert_float_eq!( + audio_buffer.channel_data(0).as_slice()[..], + [1.; 10][..], + abs_all <= 0. + ); + } + + #[test] + #[should_panic] + fn test_invalid_copy_from_channel() { + let options = AudioBufferOptions { + number_of_channels: 1, + length: 10, + sample_rate: 48000., + }; + + let audio_buffer = AudioBuffer::new(options); + + // same size + let mut dest = vec![1.; 10]; + audio_buffer.copy_from_channel(&mut dest, 1); + } + + #[test] + fn test_copy_from_channel() { + let options = AudioBufferOptions { + number_of_channels: 1, + length: 10, + sample_rate: 48000., + }; + + let audio_buffer = AudioBuffer::new(options); + + // same size + let mut dest = vec![1.; 10]; + audio_buffer.copy_from_channel(&mut dest, 0); + assert_float_eq!(dest[..], vec![0.; 10][..], abs_all <= 0.); + + // smaller destination + let mut dest = vec![1.; 5]; + audio_buffer.copy_from_channel(&mut dest, 0); + assert_float_eq!(dest[..], [0., 0., 0., 0., 0.][..], abs_all <= 0.); + + // larger destination + let mut dest = vec![1.; 11]; + audio_buffer.copy_from_channel(&mut dest, 0); + assert_float_eq!( + dest[..], + [0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 1.][..], + abs_all <= 0. + ); + + // with offset + let mut dest = vec![1.; 10]; + audio_buffer.copy_from_channel_with_offset(&mut dest, 0, 5); + assert_float_eq!( + dest[..], + [0., 0., 0., 0., 0., 1., 1., 1., 1., 1.][..], + abs_all <= 0. + ); + + // w/ offset outside range + let mut dest = vec![1.; 10]; + audio_buffer.copy_from_channel_with_offset(&mut dest, 0, usize::MAX); + + assert_float_eq!(dest[..], vec![1.; 10][..], abs_all <= 0.); + } + + #[test] + #[should_panic] + fn test_invalid_copy_to_channel() { + let options = AudioBufferOptions { + number_of_channels: 1, + length: 10, + sample_rate: 48000., + }; + + let mut audio_buffer = AudioBuffer::new(options); + + // same size + let src = vec![1.; 10]; + audio_buffer.copy_to_channel(&src, 1); + } + + #[test] + fn test_copy_to_channel() { + let options = AudioBufferOptions { + number_of_channels: 1, + length: 10, + sample_rate: 48000., + }; + + { + // same size + let mut audio_buffer = AudioBuffer::new(options.clone()); + let src = vec![1.; 10]; + audio_buffer.copy_to_channel(&src, 0); + assert_float_eq!( + audio_buffer.channel_data(0).as_slice()[..], + [1.; 10][..], + abs_all <= 0. + ); + } + + { + // smaller source + let mut audio_buffer = AudioBuffer::new(options.clone()); + let src = vec![1.; 5]; + audio_buffer.copy_to_channel(&src, 0); + assert_float_eq!( + audio_buffer.channel_data(0).as_slice()[..], + [1., 1., 1., 1., 1., 0., 0., 0., 0., 0.][..], + abs_all <= 0. + ); + } + + { + // larger source + let mut audio_buffer = AudioBuffer::new(options.clone()); + let src = vec![1.; 12]; + audio_buffer.copy_to_channel(&src, 0); + assert_float_eq!( + audio_buffer.channel_data(0).as_slice()[..], + [1., 1., 1., 1., 1., 1., 1., 1., 1., 1.][..], + abs_all <= 0. + ); + } + + { + // w/ offset + let mut audio_buffer = AudioBuffer::new(options.clone()); + let src = vec![1.; 10]; + audio_buffer.copy_to_channel_with_offset(&src, 0, 5); + assert_float_eq!( + audio_buffer.channel_data(0).as_slice()[..], + [0., 0., 0., 0., 0., 1., 1., 1., 1., 1.][..], + abs_all <= 0. + ); + } + + { + // w/ offset outside range + let mut audio_buffer = AudioBuffer::new(options); + let src = vec![1.; 10]; + audio_buffer.copy_to_channel_with_offset(&src, 0, usize::MAX); + assert_float_eq!( + audio_buffer.channel_data(0).as_slice()[..], + [0.; 10][..], + abs_all <= 0. + ); + } + } + + #[test] + #[should_panic] + fn test_invalid_get_channel_data() { + let options = AudioBufferOptions { + number_of_channels: 1, + length: 10, + sample_rate: 48000., + }; + + let audio_buffer = AudioBuffer::new(options); + + audio_buffer.get_channel_data(1); + } + + // internal API + #[test] + fn test_silent() { + let options = AudioBufferOptions { + number_of_channels: 2, + length: 10, + sample_rate: 44100., + }; + let b = AudioBuffer::new(options); + + assert_eq!(b.length(), 10); + assert_eq!(b.number_of_channels(), 2); + assert_float_eq!(b.sample_rate(), 44100., abs_all <= 0.); + assert_float_eq!(b.channel_data(0).as_slice(), &[0.; 10][..], abs_all <= 0.); + assert_float_eq!(b.channel_data(1).as_slice(), &[0.; 10][..], abs_all <= 0.); + assert_eq!(b.channels().get(2), None); + } + + #[test] + fn test_concat() { + let options = AudioBufferOptions { + number_of_channels: 2, + length: 5, + sample_rate: 44100., + }; + let mut b1 = AudioBuffer::new(options.clone()); + let b2 = AudioBuffer::new(options); + b1.extend(&b2); + + assert_eq!(b1.length(), 10); + assert_eq!(b1.number_of_channels(), 2); + assert_float_eq!(b1.sample_rate(), 44100., abs_all <= 0.); + + let channel_data = ChannelData::from(vec![1.; 5]); + let b3 = AudioBuffer::from_channels(vec![channel_data; 2], 44100.); + + b1.extend(&b3); + + assert_eq!(b1.length(), 15); + assert_eq!(b1.number_of_channels(), 2); + assert_float_eq!(b1.sample_rate(), 44100., abs_all <= 0.); + assert_float_eq!( + b1.channel_data(0).as_slice(), + &[0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 1., 1., 1., 1., 1.][..], + abs_all <= 0. + ); + } + + #[test] + #[should_panic] + fn test_resample_to_zero_hertz() { + let channel = ChannelData::from(vec![1., 2., 3., 4., 5.]); + let mut buffer = AudioBuffer::from_channels(vec![channel], 48000.); + buffer.resample(0.); + } + + #[test] + fn test_resample_from_empty() { + let channel = ChannelData::from(vec![]); + let mut buffer = AudioBuffer::from_channels(vec![channel], 48000.); + buffer.resample(48000.); + + assert_eq!(buffer.length(), 0); + assert_float_eq!(buffer.sample_rate, 48000., abs_all <= 0.); + } + + #[test] + fn test_upsample() { + let channel = ChannelData::from(vec![1., 2., 3., 4., 5.]); + let mut buffer = AudioBuffer::from_channels(vec![channel], 48000.); + buffer.resample(96000.); // double + + let mut expected = [0.; 10]; + let incr = 4. / 9.; // (5 - 1) / (10 - 1) + + for (i, value) in expected.iter_mut().enumerate() { + *value = 1. + incr * i as f32; + } + + assert_float_eq!( + buffer.channel_data(0).as_slice(), + &expected[..], + abs_all <= 1e-6 + ); + + assert_float_eq!(buffer.sample_rate, 96000., abs_all <= 0.); + } + + #[test] + fn test_downsample() { + let channel = ChannelData::from(vec![1., 2., 3., 4., 5.]); + let mut buffer = AudioBuffer::from_channels(vec![channel], 96000.); + buffer.resample(48000.); // half + + assert_float_eq!( + buffer.channel_data(0).as_slice(), + &[1., 3., 5.][..], + abs_all <= 0. + ); + + assert_float_eq!(buffer.sample_rate, 48000., abs_all <= 0.); + } + + #[test] + fn test_resample_stereo() { + [22500, 38000, 48000, 96000].iter().for_each(|sr| { + let source_sr = *sr; + let target_sr = 44_100; + + let mut left = Vec::::with_capacity(source_sr); + let mut right = Vec::::with_capacity(source_sr); + + for i in 0..source_sr { + let phase = i as f32 / source_sr as f32 * 2. * PI; + left.push(phase.sin()); + right.push(phase.cos()); + } + + let left_chan = ChannelData::from(left); + let right_chan = ChannelData::from(right); + let mut buffer = + AudioBuffer::from_channels(vec![left_chan, right_chan], source_sr as f32); + buffer.resample(target_sr as f32); + + let mut expected_left = vec![]; + let mut expected_right = vec![]; + + for i in 0..target_sr { + let phase = i as f32 / target_sr as f32 * 2. * PI; + expected_left.push(phase.sin()); + expected_right.push(phase.cos()); + } + + assert_float_eq!( + buffer.get_channel_data(0)[..], + &expected_left[..], + abs_all <= 1e-3 + ); + + assert_float_eq!( + buffer.get_channel_data(1)[..], + &expected_right[..], + abs_all <= 1e-3 + ); + + assert_float_eq!(buffer.sample_rate, target_sr as f32, abs_all <= 0.); + }); + } +} diff --git a/vendor/web-audio-api-1.7.0/src/capacity.rs b/vendor/web-audio-api-1.7.0/src/capacity.rs new file mode 100644 index 0000000000..bdc9726955 --- /dev/null +++ b/vendor/web-audio-api-1.7.0/src/capacity.rs @@ -0,0 +1,283 @@ +use crossbeam_channel::Sender; +use std::sync::{Arc, Mutex}; +use std::time::Duration; + +use crate::context::{BaseAudioContext, ConcreteBaseAudioContext}; +use crate::events::{EventDispatch, EventHandler, EventPayload, EventType}; +use crate::stats::{AudioStats, AudioStatsSnapshot}; +use crate::Event; + +/// Options for constructing an `AudioRenderCapacity` +#[derive(Clone, Debug)] +pub struct AudioRenderCapacityOptions { + /// An update interval (in seconds) for dispatching [`AudioRenderCapacityEvent`]s + pub update_interval: f64, +} + +impl Default for AudioRenderCapacityOptions { + fn default() -> Self { + Self { + update_interval: 1., + } + } +} + +/// Performance metrics of the rendering thread +#[derive(Clone, Debug)] +pub struct AudioRenderCapacityEvent { + /// The start time of the data collection period in terms of the associated AudioContext's currentTime + pub timestamp: f64, + /// An average of collected load values over the given update interval + pub average_load: f64, + /// A maximum value from collected load values over the given update interval. + pub peak_load: f64, + /// A ratio between the number of buffer underruns and the total number of system-level audio callbacks over the given update interval. + pub underrun_ratio: f64, + /// Inherits from this base Event + pub event: Event, +} + +impl AudioRenderCapacityEvent { + fn new(timestamp: f64, average_load: f64, peak_load: f64, underrun_ratio: f64) -> Self { + // We are limiting the precision here conform + // https://webaudio.github.io/web-audio-api/#dom-audiorendercapacityevent-averageload + Self { + timestamp, + average_load: (average_load * 100.).round() / 100., + peak_load: (peak_load * 100.).round() / 100., + underrun_ratio: (underrun_ratio * 100.).ceil() / 100., + event: Event { + type_: "AudioRenderCapacityEvent", + }, + } + } +} + +/// Provider for rendering performance metrics +/// +/// A load value is computed for each system-level audio callback, by dividing its execution +/// duration by the system-level audio callback buffer size divided by the sample rate. +/// +/// Ideally the load value is below 1.0, meaning that it took less time to render the audio than it +/// took to play it out. An audio buffer underrun happens when this load value is greater than 1.0: the +/// system could not render audio fast enough for real-time. +#[derive(Clone)] +pub struct AudioRenderCapacity { + context: ConcreteBaseAudioContext, + stats: AudioStats, + stop_send: Arc>>>, +} + +impl std::fmt::Debug for AudioRenderCapacity { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("AudioRenderCapacity") + .field( + "context", + &format!("BaseAudioContext@{}", self.context.address()), + ) + .finish_non_exhaustive() + } +} + +impl AudioRenderCapacity { + pub(crate) fn new(context: ConcreteBaseAudioContext, stats: AudioStats) -> Self { + let stop_send = Arc::new(Mutex::new(None)); + + Self { + context, + stats, + stop_send, + } + } + + /// Start metric collection and analysis + #[allow(clippy::missing_panics_doc)] + pub fn start(&self, options: AudioRenderCapacityOptions) { + // stop current metric collection, if any + self.stop(); + + let (stop_send, stop_recv) = crossbeam_channel::bounded(0); + *self.stop_send.lock().unwrap() = Some(stop_send); + + let mut timestamp: f64 = self.context.current_time(); + let update_interval = Duration::from_secs_f64(options.update_interval.max(0.001)); + let base_context = self.context.clone(); + let stats = self.stats.clone(); + let mut previous = stats.snapshot(); + stats.take_peak_load(); + std::thread::spawn(move || loop { + if stop_recv.recv_timeout(update_interval).is_ok() { + return; + } + + let next = stats.snapshot(); + if next.callback_count == previous.callback_count { + continue; + } + + let peak_load = stats.take_peak_load(); + let event = render_capacity_event(timestamp, previous, next, peak_load); + + let send_result = base_context.send_event(EventDispatch::render_capacity(event)); + if send_result.is_err() { + break; + }; + + previous = next; + timestamp = base_context.current_time(); + }); + } + + /// Stop metric collection and analysis + #[allow(clippy::missing_panics_doc)] + pub fn stop(&self) { + // halt callback thread + if let Some(stop_send) = self.stop_send.lock().unwrap().take() { + let _ = stop_send.send(()); + } + } + + /// The EventHandler for [`AudioRenderCapacityEvent`]. + /// + /// Only a single event handler is active at any time. Calling this method multiple times will + /// override the previous event handler. + pub fn set_onupdate( + &self, + mut callback: F, + ) { + let callback = move |v| match v { + EventPayload::RenderCapacity(v) => callback(v), + _ => unreachable!(), + }; + + self.context.set_event_handler( + EventType::RenderCapacity, + EventHandler::Multiple(Box::new(callback)), + ); + } + + /// Unset the EventHandler for [`AudioRenderCapacityEvent`]. + pub fn clear_onupdate(&self) { + self.context.clear_event_handler(EventType::RenderCapacity); + } +} + +fn render_capacity_event( + timestamp: f64, + previous: AudioStatsSnapshot, + next: AudioStatsSnapshot, + peak_load: f64, +) -> AudioRenderCapacityEvent { + let callback_count = next + .callback_count + .saturating_sub(previous.callback_count) + .max(1); + let render_duration = next + .render_duration_ns_total + .saturating_sub(previous.render_duration_ns_total); + let callback_budget = next + .callback_budget_ns_total + .saturating_sub(previous.callback_budget_ns_total); + let underruns = next.underrun_count.saturating_sub(previous.underrun_count); + + let average_load = if callback_budget == 0 { + 0. + } else { + render_duration as f64 / callback_budget as f64 + }; + + AudioRenderCapacityEvent::new( + timestamp, + average_load, + peak_load, + underruns as f64 / callback_count as f64, + ) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::context::{AudioContext, AudioContextOptions}; + + #[test] + fn test_same_instance() { + let options = AudioContextOptions { + sink_id: "none".into(), + ..AudioContextOptions::default() + }; + let context = AudioContext::new(options); + + let rc1 = context.render_capacity(); + let rc2 = context.render_capacity(); + let rc3 = rc2.clone(); + + // assert all items are actually the same instance + assert!(Arc::ptr_eq(&rc1.stop_send, &rc2.stop_send)); + assert!(Arc::ptr_eq(&rc1.stop_send, &rc3.stop_send)); + } + + #[test] + fn test_stop_when_not_running() { + let options = AudioContextOptions { + sink_id: "none".into(), + ..AudioContextOptions::default() + }; + let context = AudioContext::new(options); + + let rc = context.render_capacity(); + rc.stop(); + } + + #[test] + fn test_render_capacity() { + let options = AudioContextOptions { + sink_id: "none".into(), + ..AudioContextOptions::default() + }; + let context = AudioContext::new(options); + + let rc = context.render_capacity(); + let (send, recv) = crossbeam_channel::bounded(1); + rc.set_onupdate(move |e| send.send(e).unwrap()); + rc.start(AudioRenderCapacityOptions { + update_interval: 0.05, + }); + let event = recv.recv().unwrap(); + + assert!(event.timestamp >= 0.); + assert!(event.average_load >= 0.); + assert!(event.peak_load >= 0.); + assert!(event.underrun_ratio >= 0.); + + assert!(event.timestamp.is_finite()); + assert!(event.average_load.is_finite()); + assert!(event.peak_load.is_finite()); + assert!(event.underrun_ratio.is_finite()); + + assert_eq!(event.event.type_, "AudioRenderCapacityEvent"); + } + + #[test] + fn test_render_capacity_stops_on_close() { + let options = AudioContextOptions { + sink_id: "none".into(), + ..AudioContextOptions::default() + }; + let context = AudioContext::new(options); + + let rc = context.render_capacity(); + let (send, recv) = crossbeam_channel::unbounded(); + rc.set_onupdate(move |e| send.send(e).unwrap()); + rc.start(AudioRenderCapacityOptions { + update_interval: 0.01, + }); + + recv.recv().unwrap(); + while recv.try_recv().is_ok() {} + + context.close_sync(); + std::thread::sleep(std::time::Duration::from_millis(100)); + + assert_eq!(recv.try_iter().count(), 0); + } +} diff --git a/vendor/web-audio-api-1.7.0/src/context/base.rs b/vendor/web-audio-api-1.7.0/src/context/base.rs new file mode 100644 index 0000000000..63875b3bb1 --- /dev/null +++ b/vendor/web-audio-api-1.7.0/src/context/base.rs @@ -0,0 +1,452 @@ +//! The `BaseAudioContext` interface + +use crate::buffer::{AudioBuffer, AudioBufferOptions}; +use crate::context::{ + AudioContextRegistration, AudioContextState, AudioParamId, ConcreteBaseAudioContext, + DESTINATION_NODE_ID, +}; +use crate::decoding::decode_media_data; +use crate::events::{Event, EventHandler, EventType}; +use crate::node::{AudioNode, AudioNodeOptions}; +use crate::param::AudioParamDescriptor; +use crate::periodic_wave::{PeriodicWave, PeriodicWaveOptions}; +use crate::{node, AudioListener, RENDER_QUANTUM_SIZE}; + +use std::future::Future; + +/// The interface representing an audio-processing graph built from audio modules linked together, +/// each represented by an `AudioNode`. +/// +/// An audio context controls both the creation of the nodes it contains and the execution of the +/// audio processing, or decoding. +#[allow(clippy::module_name_repetitions)] +pub trait BaseAudioContext { + /// Returns the [`BaseAudioContext`] concrete type associated with this `AudioContext` + #[doc(hidden)] // we'd rather not expose the ConcreteBaseAudioContext + fn base(&self) -> &ConcreteBaseAudioContext; + + /// Decode an [`AudioBuffer`] from a given input stream. + /// + /// The current implementation supports Symphonia's audio formats and codecs, + /// including AIFF, CAF, ISO/MP4, MKV/WebM, Ogg, WAV, AAC, ADPCM, ALAC, + /// FLAC, MP1/MP2/MP3, PCM, and Vorbis. + /// + /// In addition to the official spec, the input parameter can be any byte stream (not just an + /// array). This means you can decode audio data from a file, network stream, or in memory + /// buffer, and any other [`std::io::Read`] implementer. The data is buffered internally so you + /// should not wrap the source in a `BufReader`. + /// + /// This function operates synchronously, which may be undesirable on the control thread. The + /// example shows how to avoid this. See also the async method [`Self::decode_audio_data`]. + /// + /// # Errors + /// + /// This method returns an Error in various cases (IO, mime sniffing, decoding). + /// + /// # Usage + /// + /// ```no_run + /// use std::io::Cursor; + /// use web_audio_api::context::{BaseAudioContext, OfflineAudioContext}; + /// + /// let input = Cursor::new(vec![0; 32]); // or a File, TcpStream, ... + /// + /// let context = OfflineAudioContext::new(2, 44_100, 44_100.); + /// let handle = std::thread::spawn(move || context.decode_audio_data_sync(input)); + /// + /// // do other things + /// + /// // await result from the decoder thread + /// let decode_buffer_result = handle.join(); + /// ``` + /// + /// # Examples + /// + /// The following example shows how to use a thread pool for audio buffer decoding: + /// + /// `cargo run --release --example decode_multithreaded` + fn decode_audio_data_sync( + &self, + input: R, + ) -> Result> { + decode_media_data(input, self.sample_rate()) + } + + /// Decode an [`AudioBuffer`] from a given input stream. + /// + /// The current implementation supports Symphonia's audio formats and codecs, + /// including AIFF, CAF, ISO/MP4, MKV/WebM, Ogg, WAV, AAC, ADPCM, ALAC, + /// FLAC, MP1/MP2/MP3, PCM, and Vorbis. + /// + /// In addition to the official spec, the input parameter can be any byte stream (not just an + /// array). This means you can decode audio data from a file, network stream, or in memory + /// buffer, and any other [`std::io::Read`] implementer. The data is buffered internally so you + /// should not wrap the source in a `BufReader`. + /// + /// Warning, the current implementation still uses blocking IO so it's best to use Tokio's + /// `spawn_blocking` to run the decoding on a thread dedicated to blocking operations. See also + /// the async method [`Self::decode_audio_data_sync`]. + /// + /// # Errors + /// + /// This method returns an Error in various cases (IO, mime sniffing, decoding). + // Use of `async fn` in public traits is discouraged as auto trait bounds cannot be specified, + // hence we use `-> impl Future + ..` instead. + fn decode_audio_data( + &self, + input: R, + ) -> impl Future>> + + Send + + 'static { + let sample_rate = self.sample_rate(); + async move { decode_media_data(input, sample_rate) } + } + + /// Create an new "in-memory" `AudioBuffer` with the given number of channels, + /// length (i.e. number of samples per channel) and sample rate. + /// + /// Note: In most cases you will want the sample rate to match the current + /// audio context sample rate. + #[must_use] + fn create_buffer( + &self, + number_of_channels: usize, + length: usize, + sample_rate: f32, + ) -> AudioBuffer { + let options = AudioBufferOptions { + number_of_channels, + length, + sample_rate, + }; + + AudioBuffer::new(options) + } + + /// Creates a `AnalyserNode` + #[must_use] + fn create_analyser(&self) -> node::AnalyserNode { + node::AnalyserNode::new(self.base(), node::AnalyserOptions::default()) + } + + /// Creates an `BiquadFilterNode` which implements a second order filter + #[must_use] + fn create_biquad_filter(&self) -> node::BiquadFilterNode { + node::BiquadFilterNode::new(self.base(), node::BiquadFilterOptions::default()) + } + + /// Creates an `AudioBufferSourceNode` + #[must_use] + fn create_buffer_source(&self) -> node::AudioBufferSourceNode { + node::AudioBufferSourceNode::new(self.base(), node::AudioBufferSourceOptions::default()) + } + + /// Creates an `ConstantSourceNode`, a source representing a constant value + #[must_use] + fn create_constant_source(&self) -> node::ConstantSourceNode { + node::ConstantSourceNode::new(self.base(), node::ConstantSourceOptions::default()) + } + + /// Creates an `ConvolverNode`, a processing node which applies linear convolution + #[must_use] + fn create_convolver(&self) -> node::ConvolverNode { + node::ConvolverNode::new(self.base(), node::ConvolverOptions::default()) + } + + /// Creates a `ChannelMergerNode` + #[must_use] + fn create_channel_merger(&self, number_of_inputs: usize) -> node::ChannelMergerNode { + let opts = node::ChannelMergerOptions { + number_of_inputs, + ..node::ChannelMergerOptions::default() + }; + node::ChannelMergerNode::new(self.base(), opts) + } + + /// Creates a `ChannelSplitterNode` + #[must_use] + fn create_channel_splitter(&self, number_of_outputs: usize) -> node::ChannelSplitterNode { + let opts = node::ChannelSplitterOptions { + number_of_outputs, + ..node::ChannelSplitterOptions::default() + }; + node::ChannelSplitterNode::new(self.base(), opts) + } + + /// Creates a `DelayNode`, delaying the audio signal + #[must_use] + fn create_delay(&self, max_delay_time: f64) -> node::DelayNode { + let opts = node::DelayOptions { + max_delay_time, + ..node::DelayOptions::default() + }; + node::DelayNode::new(self.base(), opts) + } + + /// Creates a `DynamicsCompressorNode`, compressing the audio signal + #[must_use] + fn create_dynamics_compressor(&self) -> node::DynamicsCompressorNode { + node::DynamicsCompressorNode::new(self.base(), node::DynamicsCompressorOptions::default()) + } + + /// Creates an `GainNode`, to control audio volume + #[must_use] + fn create_gain(&self) -> node::GainNode { + node::GainNode::new(self.base(), node::GainOptions::default()) + } + + /// Creates an `IirFilterNode` + /// + /// # Arguments + /// + /// * `feedforward` - An array of the feedforward (numerator) coefficients for the transfer function of the IIR filter. + /// The maximum length of this array is 20 + /// * `feedback` - An array of the feedback (denominator) coefficients for the transfer function of the IIR filter. + /// The maximum length of this array is 20 + #[must_use] + fn create_iir_filter(&self, feedforward: Vec, feedback: Vec) -> node::IIRFilterNode { + let options = node::IIRFilterOptions { + audio_node_options: AudioNodeOptions::default(), + feedforward, + feedback, + }; + node::IIRFilterNode::new(self.base(), options) + } + + /// Creates an `OscillatorNode`, a source representing a periodic waveform. + #[must_use] + fn create_oscillator(&self) -> node::OscillatorNode { + node::OscillatorNode::new(self.base(), node::OscillatorOptions::default()) + } + + /// Creates a `PannerNode` + #[must_use] + fn create_panner(&self) -> node::PannerNode { + node::PannerNode::new(self.base(), node::PannerOptions::default()) + } + + /// Creates a periodic wave + /// + /// Please note that this constructor deviates slightly from the spec by requiring a single + /// argument with the periodic wave options. + #[must_use] + fn create_periodic_wave(&self, options: PeriodicWaveOptions) -> PeriodicWave { + PeriodicWave::new(self.base(), options) + } + + /// Creates an `ScriptProcessorNode` for custom audio processing (deprecated); + /// + /// # Panics + /// + /// This function panics if: + /// - `buffer_size` is not 256, 512, 1024, 2048, 4096, 8192, or 16384 + /// - the number of input and output channels are both zero + /// - either of the channel counts exceed [`crate::MAX_CHANNELS`] + #[must_use] + fn create_script_processor( + &self, + buffer_size: usize, + number_of_input_channels: usize, + number_of_output_channels: usize, + ) -> node::ScriptProcessorNode { + let options = node::ScriptProcessorOptions { + buffer_size, + number_of_input_channels, + number_of_output_channels, + }; + + node::ScriptProcessorNode::new(self.base(), options) + } + + /// Creates an `StereoPannerNode` to pan a stereo output + #[must_use] + fn create_stereo_panner(&self) -> node::StereoPannerNode { + node::StereoPannerNode::new(self.base(), node::StereoPannerOptions::default()) + } + + /// Creates a `WaveShaperNode` + #[must_use] + fn create_wave_shaper(&self) -> node::WaveShaperNode { + node::WaveShaperNode::new(self.base(), node::WaveShaperOptions::default()) + } + + /// Returns an `AudioDestinationNode` representing the final destination of all audio in the + /// context. It can be thought of as the audio-rendering device. + #[must_use] + fn destination(&self) -> node::AudioDestinationNode { + let registration = AudioContextRegistration { + id: DESTINATION_NODE_ID, + context: self.base().clone(), + }; + let channel_config = self.base().destination_channel_config(); + node::AudioDestinationNode::from_raw_parts(registration, channel_config) + } + + /// Returns the `AudioListener` which is used for 3D spatialization + #[must_use] + fn listener(&self) -> AudioListener { + self.base().listener() + } + + /// The sample rate (in sample-frames per second) at which the `AudioContext` handles audio. + #[must_use] + fn sample_rate(&self) -> f32 { + self.base().sample_rate() + } + + /// Returns state of current context + #[must_use] + fn state(&self) -> AudioContextState { + self.base().state() + } + + /// This is the time in seconds of the sample frame immediately following the last sample-frame + /// in the block of audio most recently processed by the context’s rendering graph. + #[must_use] + fn current_time(&self) -> f64 { + self.base().current_time() + } + + // Number of frames computed on each render block + #[must_use] + fn render_quantum_size(&self) -> usize { + RENDER_QUANTUM_SIZE + } + + /// Create an `AudioParam`. + /// + /// Call this inside the `register` closure when setting up your `AudioNode` + #[must_use] + fn create_audio_param( + &self, + opts: AudioParamDescriptor, + dest: &AudioContextRegistration, + ) -> (crate::param::AudioParam, AudioParamId) { + let param = self.base().register(move |registration| { + let (node, proc) = crate::param::audio_param_pair(opts, registration); + + (node, Box::new(proc)) + }); + + // Connect the param to the node, once the node is registered inside the audio graph. + self.base().queue_audio_param_connect(¶m, dest.id()); + + let proc_id = AudioParamId(param.registration().id().0); + (param, proc_id) + } + + /// Register callback to run when the state of the AudioContext has changed + /// + /// Only a single event handler is active at any time. Calling this method multiple times will + /// override the previous event handler. + fn set_onstatechange(&self, mut callback: F) { + let callback = move |_| { + callback(Event { + type_: "statechange", + }) + }; + + self.base().set_event_handler( + EventType::StateChange, + EventHandler::Multiple(Box::new(callback)), + ); + } + + /// Unset the callback to run when the state of the AudioContext has changed + fn clear_onstatechange(&self) { + self.base().clear_event_handler(EventType::StateChange); + } + + #[cfg(test)] + fn mock_registration(&self) -> AudioContextRegistration { + AudioContextRegistration { + id: crate::context::AudioNodeId(0), + context: self.base().clone(), + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::context::OfflineAudioContext; + + use float_eq::assert_float_eq; + + fn require_send_sync_static(_: T) {} + + #[test] + fn test_decode_audio_data_sync() { + let context = OfflineAudioContext::new(1, 1, 44100.); + let file = std::fs::File::open("samples/sample.wav").unwrap(); + let audio_buffer = context.decode_audio_data_sync(file).unwrap(); + + assert_eq!(audio_buffer.sample_rate(), 44100.); + assert_eq!(audio_buffer.length(), 142_187); + assert_eq!(audio_buffer.number_of_channels(), 2); + assert_float_eq!(audio_buffer.duration(), 3.224, abs_all <= 0.001); + + let left_start = &audio_buffer.get_channel_data(0)[0..100]; + let right_start = &audio_buffer.get_channel_data(1)[0..100]; + // assert distinct two channel data + assert!(left_start != right_start); + } + + #[test] + fn test_decode_audio_data_future_send_static() { + let context = OfflineAudioContext::new(1, 1, 44100.); + let file = std::fs::File::open("samples/sample.wav").unwrap(); + let future = context.decode_audio_data(file); + require_send_sync_static(future); + } + + #[test] + fn test_decode_audio_data_async() { + use futures::executor; + let context = OfflineAudioContext::new(1, 1, 44100.); + let file = std::fs::File::open("samples/sample.wav").unwrap(); + let future = context.decode_audio_data(file); + let audio_buffer = executor::block_on(future).unwrap(); + + assert_eq!(audio_buffer.sample_rate(), 44100.); + assert_eq!(audio_buffer.length(), 142_187); + assert_eq!(audio_buffer.number_of_channels(), 2); + assert_float_eq!(audio_buffer.duration(), 3.224, abs_all <= 0.001); + + let left_start = &audio_buffer.get_channel_data(0)[0..100]; + let right_start = &audio_buffer.get_channel_data(1)[0..100]; + // assert distinct two channel data + assert!(left_start != right_start); + } + + // #[test] + // disabled: symphonia cannot handle empty WAV-files + #[allow(dead_code)] + fn test_decode_audio_data_empty() { + let context = OfflineAudioContext::new(1, 1, 44100.); + let file = std::fs::File::open("samples/empty_2c.wav").unwrap(); + let audio_buffer = context.decode_audio_data_sync(file).unwrap(); + assert_eq!(audio_buffer.length(), 0); + } + + #[test] + fn test_decode_audio_data_decoding_error() { + let context = OfflineAudioContext::new(1, 1, 44100.); + let file = std::fs::File::open("samples/corrupt.wav").unwrap(); + assert!(context.decode_audio_data_sync(file).is_err()); + } + + #[test] + fn test_create_buffer() { + let number_of_channels = 3; + let length = 2000; + let sample_rate = 96_000.; + + let context = OfflineAudioContext::new(1, 1, 44100.); + let buffer = context.create_buffer(number_of_channels, length, sample_rate); + + assert_eq!(buffer.number_of_channels(), 3); + assert_eq!(buffer.length(), 2000); + assert_float_eq!(buffer.sample_rate(), 96000., abs_all <= 0.); + } +} diff --git a/vendor/web-audio-api-1.7.0/src/context/concrete_base.rs b/vendor/web-audio-api-1.7.0/src/context/concrete_base.rs new file mode 100644 index 0000000000..f6d75e202c --- /dev/null +++ b/vendor/web-audio-api-1.7.0/src/context/concrete_base.rs @@ -0,0 +1,620 @@ +//! The `ConcreteBaseAudioContext` type + +use crate::context::{ + AudioContextRegistration, AudioContextState, AudioNodeId, BaseAudioContext, + DESTINATION_NODE_ID, LISTENER_NODE_ID, LISTENER_PARAM_IDS, +}; +use crate::events::{EventDispatch, EventHandler, EventLoop, EventType}; +use crate::message::ControlMessage; +use crate::node::{AudioDestinationNode, AudioNode, AudioNodeOptions, ChannelConfig}; +use crate::param::AudioParam; +use crate::render::AudioProcessor; +use crate::spatial::AudioListenerParams; + +use crate::AudioListener; + +use crossbeam_channel::{SendError, Sender}; +use std::collections::HashSet; +use std::sync::atomic::{AtomicU64, AtomicU8, Ordering}; +use std::sync::{Arc, Mutex, RwLock, RwLockWriteGuard}; + +/// This struct assigns new [`AudioNodeId`]s for [`AudioNode`]s +/// +/// It reuses the ids of decommissioned nodes to prevent unbounded growth of the audio graphs node +/// list (which is stored in a Vec indexed by the AudioNodeId). +struct AudioNodeIdProvider { + /// incrementing id + id_inc: AtomicU64, + /// receiver for decommissioned AudioNodeIds, which can be reused + id_consumer: Mutex>, +} + +impl AudioNodeIdProvider { + fn new(id_consumer: llq::Consumer) -> Self { + Self { + id_inc: AtomicU64::new(0), + id_consumer: Mutex::new(id_consumer), + } + } + + fn get(&self) -> AudioNodeId { + if let Some(available_id) = self.id_consumer.lock().unwrap().pop() { + llq::Node::into_inner(available_id) + } else { + AudioNodeId(self.id_inc.fetch_add(1, Ordering::Relaxed)) + } + } +} + +/// The struct that corresponds to the Javascript `BaseAudioContext` object. +/// +/// This object is returned from the `base()` method on +/// [`AudioContext`](crate::context::AudioContext) and +/// [`OfflineAudioContext`](crate::context::OfflineAudioContext), and the `context()` method on +/// `AudioNode`s. +/// +/// The `ConcreteBaseAudioContext` allows for shallow cloning (using an `Arc` internally). +#[allow(clippy::module_name_repetitions)] +#[derive(Clone)] +#[doc(hidden)] +pub struct ConcreteBaseAudioContext { + inner: Arc, +} + +impl PartialEq for ConcreteBaseAudioContext { + fn eq(&self, other: &Self) -> bool { + Arc::ptr_eq(&self.inner, &other.inner) + } +} + +impl std::fmt::Debug for ConcreteBaseAudioContext { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("BaseAudioContext") + .field("id", &self.address()) + .field("state", &self.state()) + .field("sample_rate", &self.sample_rate()) + .field("current_time", &self.current_time()) + .field("max_channel_count", &self.max_channel_count()) + .field("offline", &self.offline()) + .finish_non_exhaustive() + } +} + +/// Inner representation of the `ConcreteBaseAudioContext` +/// +/// These fields are wrapped inside an `Arc` in the actual `ConcreteBaseAudioContext`. +struct ConcreteBaseAudioContextInner { + /// sample rate in Hertz + sample_rate: f32, + /// max number of speaker output channels + max_channel_count: usize, + /// provider for new AudioNodeIds + audio_node_id_provider: AudioNodeIdProvider, + /// destination node's current channel count + destination_channel_config: ChannelConfig, + /// message channel from control to render thread + render_channel: RwLock>, + /// control messages staged while the render thread is suspended + suspended_messages: Mutex>>, + /// control messages that cannot be sent immediately + queued_messages: Mutex>, + /// number of frames played + frames_played: Arc, + /// control msg to add the AudioListener, to be sent when the first panner is created + queued_audio_listener_msgs: Mutex>, + /// AudioListener fields + listener_params: Option, + /// Denotes if this AudioContext is offline or not + offline: bool, + /// Current state of the `ConcreteBaseAudioContext`, shared with the RenderThread + state: Arc, + /// Stores the event handlers + event_loop: EventLoop, + /// Sender for events that will be handled by the EventLoop + event_send: Sender, + /// Current audio graph connections (from node, output port, to node, input port) + connections: Mutex>, +} + +impl BaseAudioContext for ConcreteBaseAudioContext { + fn base(&self) -> &ConcreteBaseAudioContext { + self + } +} + +impl ConcreteBaseAudioContext { + /// Creates a `BaseAudioContext` instance + #[allow(clippy::too_many_arguments)] // TODO refactor with builder pattern + pub(super) fn new( + sample_rate: f32, + max_channel_count: usize, + state: Arc, + frames_played: Arc, + render_channel: Sender, + event_send: Sender, + event_loop: EventLoop, + offline: bool, + node_id_consumer: llq::Consumer, + ) -> Self { + let audio_node_id_provider = AudioNodeIdProvider::new(node_id_consumer); + + let base_inner = ConcreteBaseAudioContextInner { + sample_rate, + max_channel_count, + render_channel: RwLock::new(render_channel), + suspended_messages: Mutex::new(None), + queued_messages: Mutex::new(Vec::new()), + audio_node_id_provider, + destination_channel_config: AudioNodeOptions::default().into(), + frames_played, + queued_audio_listener_msgs: Mutex::new(Vec::new()), + listener_params: None, + offline, + state, + event_loop, + event_send, + connections: Mutex::new(HashSet::new()), + }; + let base = Self { + inner: Arc::new(base_inner), + }; + + // Online AudioContext should start with stereo channels by default + let initial_channel_count = if offline { + max_channel_count + } else { + 2.min(max_channel_count) + }; + + let (listener_params, destination_channel_config) = { + // Register magical nodes. We should not store the nodes inside our context since that + // will create a cyclic reference, but we can reconstruct a new instance on the fly + // when requested + let dest = AudioDestinationNode::new(&base, initial_channel_count); + let destination_channel_config = dest.into_channel_config(); + let listener = crate::spatial::AudioListenerNode::new(&base); + + let listener_params = listener.into_fields(); + let AudioListener { + position_x, + position_y, + position_z, + forward_x, + forward_y, + forward_z, + up_x, + up_y, + up_z, + } = listener_params; + + let listener_params = AudioListenerParams { + position_x: position_x.into_raw_parts(), + position_y: position_y.into_raw_parts(), + position_z: position_z.into_raw_parts(), + forward_x: forward_x.into_raw_parts(), + forward_y: forward_y.into_raw_parts(), + forward_z: forward_z.into_raw_parts(), + up_x: up_x.into_raw_parts(), + up_y: up_y.into_raw_parts(), + up_z: up_z.into_raw_parts(), + }; + + (listener_params, destination_channel_config) + }; // Nodes will drop now, so base.inner has no copies anymore + + let mut base = base; + let inner_mut = Arc::get_mut(&mut base.inner).unwrap(); + inner_mut.listener_params = Some(listener_params); + inner_mut.destination_channel_config = destination_channel_config; + + // Validate if the hardcoded node IDs line up + debug_assert_eq!( + base.inner + .audio_node_id_provider + .id_inc + .load(Ordering::Relaxed), + LISTENER_PARAM_IDS.end, + ); + + // HRTF is loaded on the control thread when a PannerNode selects that + // model. Do not resample its entire database for contexts that never + // create a spatial panner (including headless oscillator/analyser use). + + base + } + + pub(crate) fn address(&self) -> usize { + Arc::as_ptr(&self.inner) as usize + } + + /// Construct a new pair of [`AudioNode`] and [`AudioProcessor`] + pub(crate) fn register< + T: AudioNode, + F: FnOnce(AudioContextRegistration) -> (T, Box), + >( + &self, + f: F, + ) -> T { + // create a unique id for this node + let id = self.inner.audio_node_id_provider.get(); + let registration = AudioContextRegistration { + id, + context: self.clone(), + }; + + // create the node and its renderer + let (node, render) = (f)(registration); + + // pass the renderer to the audio graph + let message = ControlMessage::RegisterNode { + id, + reclaim_id: llq::Node::new(id), + node: render, + inputs: node.number_of_inputs(), + outputs: node.number_of_outputs(), + channel_config: node.channel_config().inner(), + }; + + // if this is the AudioListener or its params, do not add it to the graph just yet + if id == LISTENER_NODE_ID || LISTENER_PARAM_IDS.contains(&id.0) { + let mut queued_audio_listener_msgs = + self.inner.queued_audio_listener_msgs.lock().unwrap(); + queued_audio_listener_msgs.push(message); + } else { + self.send_control_msg(message); + self.resolve_queued_control_msgs(id); + } + + node + } + + /// Send a control message to the render thread + /// + /// When the render thread is closed or crashed, the message is discarded and a log warning is + /// emitted. + pub(crate) fn send_control_msg(&self, msg: ControlMessage) { + if self.state() != AudioContextState::Closed { + let sender = self.inner.render_channel.read().unwrap(); + // if the context is suspended, buffer the message and don't send it + if let Some(queued) = self.inner.suspended_messages.lock().unwrap().as_mut() { + queued.push(msg); + return; + } + + let result = sender.send(msg); + if result.is_err() { + log::warn!("Discarding control message - render thread is closed"); + } + } + } + + pub(crate) fn suspend_control_msgs(&self, msg: ControlMessage) { + let sender = self.inner.render_channel.read().unwrap(); + *self.inner.suspended_messages.lock().unwrap() = Some(Vec::new()); + if sender.send(msg).is_err() { + log::warn!("Discarding control message - render thread is closed"); + } + } + + pub(crate) fn resume_control_msgs(&self, msg: ControlMessage) { + let sender = self.inner.render_channel.read().unwrap(); + let messages = self + .inner + .suspended_messages + .lock() + .unwrap() + .take() + .unwrap_or_default(); + + for msg in messages { + if sender.send(msg).is_err() { + log::warn!("Discarding control message - render thread is closed"); + return; + } + } + + if sender.send(msg).is_err() { + log::warn!("Discarding control message - render thread is closed"); + } + } + + pub(crate) fn send_event(&self, msg: EventDispatch) -> Result<(), SendError> { + self.inner.event_send.send(msg) + } + + pub(crate) fn lock_control_msg_sender(&self) -> RwLockWriteGuard<'_, Sender> { + self.inner.render_channel.write().unwrap() + } + + pub(super) fn mark_node_dropped(&self, id: AudioNodeId) { + // Ignore magic nodes + if id == DESTINATION_NODE_ID || id == LISTENER_NODE_ID || LISTENER_PARAM_IDS.contains(&id.0) + { + return; + } + + // Inform render thread that the control thread AudioNode no longer has any handles + let message = ControlMessage::ControlHandleDropped { id }; + self.send_control_msg(message); + + // Clear the connection administration for this node, the node id may be recycled later + self.inner + .connections + .lock() + .unwrap() + .retain(|&(from, _output, to, _input)| from != id && to != id); + } + + /// Inform render thread that this node can act as a cycle breaker + #[doc(hidden)] + pub fn mark_cycle_breaker(&self, reg: &AudioContextRegistration) { + let id = reg.id(); + let message = ControlMessage::MarkCycleBreaker { id }; + self.send_control_msg(message); + } + + /// `ChannelConfig` of the `AudioDestinationNode` + pub(super) fn destination_channel_config(&self) -> ChannelConfig { + self.inner.destination_channel_config.clone() + } + + /// Returns the `AudioListener` which is used for 3D spatialization + pub(super) fn listener(&self) -> AudioListener { + // instruct to BaseContext to add the AudioListener if it has not already + self.base().ensure_audio_listener_present(); + + let mut ids = LISTENER_PARAM_IDS.map(|i| AudioContextRegistration { + id: AudioNodeId(i), + context: self.clone(), + }); + let params = self.inner.listener_params.as_ref().unwrap(); + + AudioListener { + position_x: AudioParam::from_raw_parts(ids.next().unwrap(), params.position_x.clone()), + position_y: AudioParam::from_raw_parts(ids.next().unwrap(), params.position_y.clone()), + position_z: AudioParam::from_raw_parts(ids.next().unwrap(), params.position_z.clone()), + forward_x: AudioParam::from_raw_parts(ids.next().unwrap(), params.forward_x.clone()), + forward_y: AudioParam::from_raw_parts(ids.next().unwrap(), params.forward_y.clone()), + forward_z: AudioParam::from_raw_parts(ids.next().unwrap(), params.forward_z.clone()), + up_x: AudioParam::from_raw_parts(ids.next().unwrap(), params.up_x.clone()), + up_y: AudioParam::from_raw_parts(ids.next().unwrap(), params.up_y.clone()), + up_z: AudioParam::from_raw_parts(ids.next().unwrap(), params.up_z.clone()), + } + } + + /// Returns state of current context + #[must_use] + pub(super) fn state(&self) -> AudioContextState { + self.inner.state.load(Ordering::Acquire).into() + } + + /// Updates state of current context + pub(super) fn set_state(&self, state: AudioContextState) { + // Only used from OfflineAudioContext or suspended AudioContext, otherwise the state + // changed are spawned from the render thread + let current_state = self.state(); + if current_state != state { + self.inner.state.store(state as u8, Ordering::Release); + let _ = self.send_event(EventDispatch::state_change(state)); + } + } + + /// The sample rate (in sample-frames per second) at which the `AudioContext` handles audio. + #[must_use] + pub(super) fn sample_rate(&self) -> f32 { + self.inner.sample_rate + } + + /// This is the time in seconds of the sample frame immediately following the last sample-frame + /// in the block of audio most recently processed by the context’s rendering graph. + #[must_use] + // web audio api specification requires that `current_time` returns an f64 + // std::sync::AtomicsF64 is not currently implemented in the standard library + // Currently, we have no other choice than casting an u64 into f64, with possible loss of precision + #[allow(clippy::cast_precision_loss)] + pub(super) fn current_time(&self) -> f64 { + self.inner.frames_played.load(Ordering::SeqCst) as f64 / self.inner.sample_rate as f64 + } + + /// Maximum available channels for the audio destination + #[must_use] + pub(crate) fn max_channel_count(&self) -> usize { + self.inner.max_channel_count + } + + /// Release queued control messages to the render thread that were blocking on the availability + /// of the Node with the given `id` + fn resolve_queued_control_msgs(&self, id: AudioNodeId) { + // resolve control messages that depend on this registration + let mut queued = self.inner.queued_messages.lock().unwrap(); + let mut i = 0; // waiting for Vec::drain_filter to stabilize + while i < queued.len() { + if matches!(&queued[i], ControlMessage::ConnectNode {to, ..} if *to == id) { + let m = queued.remove(i); + self.send_control_msg(m); + } else { + i += 1; + } + } + } + + /// Connects the output of the `from` audio node to the input of the `to` audio node + pub(crate) fn connect(&self, from: AudioNodeId, to: AudioNodeId, output: usize, input: usize) { + let inserted = self + .inner + .connections + .lock() + .unwrap() + .insert((from, output, to, input)); + + if !inserted { + return; // do not allow duplicated edges + } + + let message = ControlMessage::ConnectNode { + from, + to, + output, + input, + }; + self.send_control_msg(message); + } + + /// Schedule a connection of an `AudioParam` to the `AudioNode` it belongs to + /// + /// It is not performed immediately as the `AudioNode` is not registered at this point. + pub(super) fn queue_audio_param_connect(&self, param: &AudioParam, audio_node: AudioNodeId) { + // no need to store these type of connections in self.inner.connections + + let message = ControlMessage::ConnectNode { + from: param.registration().id(), + to: audio_node, + output: 0, + input: usize::MAX, // audio params connect to the 'hidden' input port + }; + self.inner.queued_messages.lock().unwrap().push(message); + } + + /// Disconnects outputs of the audio node, possibly filtered by output node, input, output. + pub(crate) fn disconnect( + &self, + from: AudioNodeId, + output: Option, + to: Option, + input: Option, + ) { + // check if the node was connected, otherwise panic + let mut has_disconnected = false; + let mut connections = self.inner.connections.lock().unwrap(); + connections.retain(|&(c_from, c_output, c_to, c_input)| { + let retain = c_from != from + || c_output != output.unwrap_or(c_output) + || c_to != to.unwrap_or(c_to) + || c_input != input.unwrap_or(c_input); + if !retain { + has_disconnected = true; + let message = ControlMessage::DisconnectNode { + from, + to: c_to, + input: c_input, + output: c_output, + }; + self.send_control_msg(message); + } + retain + }); + + // make sure to drop the MutexGuard before the panic to avoid poisoning + drop(connections); + + if !has_disconnected && to.is_some() { + panic!("InvalidAccessError - attempting to disconnect unconnected nodes"); + } + } + + /// Connect the `AudioListener` to a `PannerNode` + pub(crate) fn connect_listener_to_panner(&self, panner: AudioNodeId) { + self.connect(LISTENER_NODE_ID, panner, 0, usize::MAX); + } + + /// Add the [`AudioListener`] to the audio graph (if not already) + pub(crate) fn ensure_audio_listener_present(&self) { + let mut queued_audio_listener_msgs = self.inner.queued_audio_listener_msgs.lock().unwrap(); + let mut released = false; + while let Some(message) = queued_audio_listener_msgs.pop() { + // add the AudioListenerRenderer to the graph + self.send_control_msg(message); + released = true; + } + + if released { + // connect the AudioParamRenderers to the Listener + self.resolve_queued_control_msgs(LISTENER_NODE_ID); + + // hack: Connect the listener to the destination node to force it to render at each + // quantum. Abuse the magical usize::MAX port so it acts as an AudioParam and has no side + // effects + self.connect(LISTENER_NODE_ID, DESTINATION_NODE_ID, 0, usize::MAX); + } + } + + /// Returns true if this is `OfflineAudioContext` (false when it is an `AudioContext`) + pub(crate) fn offline(&self) -> bool { + self.inner.offline + } + + pub(crate) fn set_event_handler(&self, event: EventType, callback: EventHandler) { + self.inner.event_loop.set_handler(event, callback); + } + + pub(crate) fn clear_event_handler(&self, event: EventType) { + self.inner.event_loop.clear_handler(event); + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::context::OfflineAudioContext; + + #[test] + fn test_provide_node_id() { + let (mut id_producer, id_consumer) = llq::Queue::new().split(); + let provider = AudioNodeIdProvider::new(id_consumer); + assert_eq!(provider.get().0, 0); // newly assigned + assert_eq!(provider.get().0, 1); // newly assigned + id_producer.push(llq::Node::new(AudioNodeId(0))); + assert_eq!(provider.get().0, 0); // reused + assert_eq!(provider.get().0, 2); // newly assigned + } + + #[test] + fn test_connect_disconnect() { + let context = OfflineAudioContext::new(1, 128, 48000.); + let node1 = context.create_constant_source(); + let node2 = context.create_gain(); + + // connection list starts empty + assert!(context.base().inner.connections.lock().unwrap().is_empty()); + + node1.disconnect(); // never panic for plain disconnect calls + + node1.connect(&node2); + + // connection should be registered + assert_eq!(context.base().inner.connections.lock().unwrap().len(), 1); + + node1.disconnect(); + assert!(context.base().inner.connections.lock().unwrap().is_empty()); + + node1.connect(&node2); + assert_eq!(context.base().inner.connections.lock().unwrap().len(), 1); + + node1.disconnect_dest(&node2); + assert!(context.base().inner.connections.lock().unwrap().is_empty()); + } + + #[test] + #[should_panic] + fn test_disconnect_not_existing() { + let context = OfflineAudioContext::new(1, 128, 48000.); + let node1 = context.create_constant_source(); + let node2 = context.create_gain(); + + node1.disconnect_dest(&node2); + } + + #[test] + fn test_mark_node_dropped() { + let context = OfflineAudioContext::new(1, 128, 48000.); + + let node1 = context.create_constant_source(); + let node2 = context.create_gain(); + + node1.connect(&node2); + context.base().mark_node_dropped(node1.registration().id()); + + // dropping should clear connections administration + assert!(context.base().inner.connections.lock().unwrap().is_empty()); + } +} diff --git a/vendor/web-audio-api-1.7.0/src/context/diagnostics.rs b/vendor/web-audio-api-1.7.0/src/context/diagnostics.rs new file mode 100644 index 0000000000..ad99706d1f --- /dev/null +++ b/vendor/web-audio-api-1.7.0/src/context/diagnostics.rs @@ -0,0 +1,104 @@ +//! Structured diagnostics for realtime audio contexts. + +/// Snapshot of an [`AudioContext`](crate::context::AudioContext), its backend, render thread, and +/// audio graph. +#[derive(Clone, Debug)] +#[non_exhaustive] +pub struct AudioContextDiagnostics { + /// Audio backend details collected on the control thread. + pub backend: AudioBackendDiagnostics, + /// Render thread details collected on the render thread. + pub render_thread: AudioRenderThreadDiagnostics, + /// Audio graph details collected on the render thread. + pub graph: AudioGraphDiagnostics, +} + +/// Snapshot of the active audio backend. +#[derive(Clone, Debug)] +#[non_exhaustive] +pub struct AudioBackendDiagnostics { + /// Backend implementation name. + pub name: String, + /// Current audio output device id. + pub sink_id: String, + /// Current output latency in seconds, if the backend can report it. + pub output_latency: Option, +} + +/// Snapshot of the active render thread. +#[derive(Clone, Debug)] +#[non_exhaustive] +pub struct AudioRenderThreadDiagnostics { + /// Render thread sample rate in Hz. + pub sample_rate: f32, + /// Backend callback buffer size in frames. + pub buffer_size: usize, + /// Number of frames played by the backend. + pub frames_played: u64, + /// Number of output channels used by the backend stream. + pub number_of_channels: usize, + /// Whether rendering is currently suspended. + pub suspended: bool, +} + +/// Snapshot of the audio graph. +#[derive(Clone, Debug)] +#[non_exhaustive] +pub struct AudioGraphDiagnostics { + /// Whether the graph is loaded + pub active: bool, + /// Number of registered nodes. + pub node_count: usize, + /// Number of outgoing graph connections. + pub edge_count: usize, + /// Node ids in the current render ordering. + pub ordered: Vec, + /// Node ids currently excluded from rendering because they are in an unbreakable cycle. + pub in_cycle: Vec, + /// Node ids marked as eligible cycle breakers. + pub cycle_breakers: Vec, + /// Registered audio nodes. + pub nodes: Vec, +} + +/// Snapshot of a single audio graph node. +#[derive(Clone, Debug)] +#[non_exhaustive] +pub struct AudioNodeDiagnostics { + /// Internal audio node id. + pub id: u64, + /// Renderer type name. + pub processor: String, + /// Number of input ports. + pub inputs: usize, + /// Number of output ports. + pub outputs: usize, + /// Current channel count for each input port. + pub input_channels: Vec, + /// Current channel count for each output port. + pub output_channels: Vec, + /// Channel configuration formatted for inspection. + pub channel_config: String, + /// Outgoing graph connections. + pub outgoing_edges: Vec, + /// Whether the control-side handle for this node has been dropped. + pub control_handle_dropped: bool, + /// Whether this node had incoming connections during the latest render quantum. + pub has_inputs_connected: bool, + /// Whether this node can break render cycles. + pub cycle_breaker: bool, + /// Whether this node reports side effects. + pub has_side_effects: bool, +} + +/// Snapshot of a graph connection from one node output to another node input. +#[derive(Clone, Debug)] +#[non_exhaustive] +pub struct AudioGraphEdgeDiagnostics { + /// Source output port index. + pub output: usize, + /// Destination node id. + pub destination: u64, + /// Destination input port index, or `None` for hidden internal connections. + pub input: Option, +} diff --git a/vendor/web-audio-api-1.7.0/src/context/mod.rs b/vendor/web-audio-api-1.7.0/src/context/mod.rs new file mode 100644 index 0000000000..c5ccaf160b --- /dev/null +++ b/vendor/web-audio-api-1.7.0/src/context/mod.rs @@ -0,0 +1,188 @@ +//! The `BaseAudioContext` interface and the `AudioContext` and `OfflineAudioContext` types + +use std::{any::Any, ops::Range}; + +mod base; +pub use base::*; + +mod concrete_base; +pub use concrete_base::*; + +#[cfg(feature = "diagnostics")] +mod diagnostics; +#[cfg(feature = "diagnostics")] +pub use diagnostics::*; + +mod offline; +pub use offline::*; + +mod online; +pub use online::*; + +// magic node values +/// Destination node id is always at index 0 +pub(crate) const DESTINATION_NODE_ID: AudioNodeId = AudioNodeId(0); +/// listener node id is always at index 1 +const LISTENER_NODE_ID: AudioNodeId = AudioNodeId(1); +/// listener audio parameters ids are always at index 2 through 10 +const LISTENER_PARAM_IDS: Range = 2..11; +/// listener audio parameters ids are always at index 2 through 10 +pub(crate) const LISTENER_AUDIO_PARAM_IDS: [AudioParamId; 9] = [ + AudioParamId(2), + AudioParamId(3), + AudioParamId(4), + AudioParamId(5), + AudioParamId(6), + AudioParamId(7), + AudioParamId(8), + AudioParamId(9), + AudioParamId(10), +]; + +/// Unique identifier for audio nodes. +/// +/// Used for internal bookkeeping. +#[derive(Hash, PartialEq, Eq, Clone, Copy)] +pub(crate) struct AudioNodeId(pub u64); + +impl std::fmt::Debug for AudioNodeId { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "AudioNodeId({})", self.0) + } +} + +/// Unique identifier for audio params. +/// +/// Store these in your `AudioProcessor` to get access to `AudioParam` values. +#[derive(Debug)] +pub struct AudioParamId(u64); + +// bit contrived, but for type safety only the context mod can access the inner u64 +impl From<&AudioParamId> for AudioNodeId { + fn from(i: &AudioParamId) -> Self { + Self(i.0) + } +} + +/// Describes the current state of the `AudioContext` +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub enum AudioContextState { + /// This context is currently suspended (context time is not proceeding, + /// audio hardware may be powered down/released). + Suspended, + /// Audio is being processed. + Running, + /// This context has been released, and can no longer be used to process audio. + /// All system audio resources have been released. + Closed, +} + +impl From for AudioContextState { + fn from(value: u8) -> Self { + match value { + 0 => Self::Suspended, + 1 => Self::Running, + 2 => Self::Closed, + _ => unreachable!(), + } + } +} + +/// Handle of the [`AudioNode`](crate::node::AudioNode) to its associated [`BaseAudioContext`]. +/// +/// Only when implementing the AudioNode trait manually, this struct is of any concern. +/// +/// This object allows for communication with the render thread and dynamic lifetime management. +// The only way to construct this object is by calling [`BaseAudioContext::register`]. +// This struct should not derive Clone because of the Drop handler. +pub struct AudioContextRegistration { + /// the audio context in which nodes and connections lives + context: ConcreteBaseAudioContext, + /// identify a specific `AudioNode` + id: AudioNodeId, +} + +impl std::fmt::Debug for AudioContextRegistration { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("AudioContextRegistration") + .field("id", &self.id) + .field( + "context", + &format!("BaseAudioContext@{}", self.context.address()), + ) + .finish() + } +} + +impl AudioContextRegistration { + /// Get the audio node id of the registration + #[must_use] + pub(crate) fn id(&self) -> AudioNodeId { + self.id + } + + /// Get the [`BaseAudioContext`] concrete type associated with this `AudioContext` + #[must_use] + pub(crate) fn context(&self) -> &ConcreteBaseAudioContext { + &self.context + } + + /// Send a message to the corresponding audio processor of this node + /// + /// The message will be handled by + /// [`AudioProcessor::onmessage`](crate::render::AudioProcessor::onmessage). + pub(crate) fn post_message(&self, msg: M) { + let wrapped = crate::message::ControlMessage::NodeMessage { + id: self.id, + msg: llq::Node::new(Box::new(msg)), + }; + self.context.send_control_msg(wrapped); + } +} + +impl Drop for AudioContextRegistration { + fn drop(&mut self) { + self.context.mark_node_dropped(self.id); + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::node::AudioNode; + + fn require_send_sync_static(_: T) {} + + #[test] + fn test_audio_context_registration_traits() { + let context = OfflineAudioContext::new(1, 1, 44100.); + let registration = context.mock_registration(); + + // we want to be able to ship AudioNodes to another thread, so the Registration should be + // Send, Sync and 'static + require_send_sync_static(registration); + } + + #[test] + fn test_offline_audio_context_send_sync() { + let context = OfflineAudioContext::new(1, 1, 44100.); + require_send_sync_static(context); + } + + #[test] + fn test_online_audio_context_send_sync() { + let options = AudioContextOptions { + sink_id: "none".into(), + ..AudioContextOptions::default() + }; + let context = AudioContext::new(options); + require_send_sync_static(context); + } + + #[test] + fn test_context_equals() { + let context = OfflineAudioContext::new(1, 48000, 96000.); + let dest = context.destination(); + assert!(dest.context() == context.base()); + } +} diff --git a/vendor/web-audio-api-1.7.0/src/context/offline.rs b/vendor/web-audio-api-1.7.0/src/context/offline.rs new file mode 100644 index 0000000000..6cc9c0ede2 --- /dev/null +++ b/vendor/web-audio-api-1.7.0/src/context/offline.rs @@ -0,0 +1,653 @@ +//! The `OfflineAudioContext` type + +use std::sync::atomic::{AtomicU64, AtomicU8}; +use std::sync::{Arc, Mutex}; + +use crate::buffer::AudioBuffer; +use crate::context::{AudioContextState, BaseAudioContext, ConcreteBaseAudioContext}; +use crate::events::{ + Event, EventDispatch, EventHandler, EventPayload, EventType, OfflineAudioCompletionEvent, +}; +use crate::render::RenderThread; +use crate::stats::AudioStats; +use crate::{ + assert_valid_buffer_length, assert_valid_number_of_channels, assert_valid_sample_rate, + RENDER_QUANTUM_SIZE, +}; + +use crate::events::EventLoop; +use futures_channel::{mpsc, oneshot}; +use futures_util::SinkExt as _; + +pub(crate) type OfflineAudioContextCallback = + dyn FnOnce(&mut OfflineAudioContext) + Send + Sync + 'static; + +/// The `OfflineAudioContext` doesn't render the audio to the device hardware; instead, it generates +/// it, as fast as it can, and outputs the result to an `AudioBuffer`. +// the naming comes from the web audio specification +#[allow(clippy::module_name_repetitions)] +pub struct OfflineAudioContext { + /// represents the underlying `BaseAudioContext` + base: ConcreteBaseAudioContext, + /// the size of the buffer in sample-frames + length: usize, + /// actual renderer of the audio graph, can only be called once + renderer: Mutex>, + /// channel to notify resume actions on the rendering + resume_sender: mpsc::Sender<()>, +} + +impl std::fmt::Debug for OfflineAudioContext { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("OfflineAudioContext") + .field("length", &self.length()) + .field("base", &self.base()) + .finish_non_exhaustive() + } +} + +struct OfflineAudioContextRenderer { + /// the rendering 'thread', fully controlled by the offline context + renderer: RenderThread, + /// sorted list of promises to resolve at certain render quanta (via `suspend`) + suspend_promises: Vec<(usize, oneshot::Sender<()>)>, + /// sorted list of callbacks to run at certain render quanta (via `suspend_sync`) + suspend_callbacks: Vec<(usize, Box)>, + /// channel to listen for `resume` calls on a suspended context + resume_receiver: mpsc::Receiver<()>, + /// event loop to run after each render quantum + event_loop: EventLoop, +} + +impl BaseAudioContext for OfflineAudioContext { + fn base(&self) -> &ConcreteBaseAudioContext { + &self.base + } +} + +impl OfflineAudioContext { + /// Creates an `OfflineAudioContext` instance + /// + /// # Arguments + /// + /// * `channels` - number of output channels to render + /// * `length` - length of the rendering audio buffer + /// * `sample_rate` - output sample rate + #[must_use] + #[allow(clippy::missing_panics_doc)] + pub fn new(number_of_channels: usize, length: usize, sample_rate: f32) -> Self { + assert_valid_number_of_channels(number_of_channels); + assert_valid_buffer_length(length); + assert_valid_sample_rate(sample_rate); + + // communication channel to the render thread, + // unbounded is fine because it does not need to be realtime safe + let (sender, receiver) = crossbeam_channel::unbounded(); + + let (node_id_producer, node_id_consumer) = llq::Queue::new().split(); + let graph = crate::render::graph::Graph::new(node_id_producer); + let message = crate::message::ControlMessage::Startup { graph }; + sender.send(message).unwrap(); + + // track number of frames - synced from render thread to control thread + let frames_played = Arc::new(AtomicU64::new(0)); + let frames_played_clone = Arc::clone(&frames_played); + let state = Arc::new(AtomicU8::new(AudioContextState::Suspended as u8)); + let state_clone = Arc::clone(&state); + + // Communication channel for events from the render thread to the control thread. + // Use an unbounded channel because we do not require real-time safety. + let (event_send, event_recv) = crossbeam_channel::unbounded(); + let event_loop = EventLoop::new(event_recv); + + // setup the render 'thread', which will run inside the control thread + let renderer = RenderThread::new( + sample_rate, + number_of_channels, + receiver, + state_clone, + frames_played_clone, + AudioStats::new(), + event_send.clone(), + ); + + // first, setup the base audio context + let base = ConcreteBaseAudioContext::new( + sample_rate, + number_of_channels, + state, + frames_played, + sender, + event_send, + event_loop.clone(), + true, + node_id_consumer, + ); + + let (resume_sender, resume_receiver) = mpsc::channel(0); + + let renderer = OfflineAudioContextRenderer { + renderer, + suspend_promises: Vec::new(), + suspend_callbacks: Vec::new(), + resume_receiver, + event_loop, + }; + + Self { + base, + length, + renderer: Mutex::new(Some(renderer)), + resume_sender, + } + } + + /// Given the current connections and scheduled changes, starts rendering audio. + /// + /// This function will block the current thread and returns the rendered `AudioBuffer` + /// synchronously. + /// + /// This method will only adhere to scheduled suspensions via [`Self::suspend_sync`] and + /// will ignore those provided via [`Self::suspend`]. + /// + /// # Panics + /// + /// Panics if this method is called multiple times + #[must_use] + pub fn start_rendering_sync(&mut self) -> AudioBuffer { + let renderer = self + .renderer + .lock() + .unwrap() + .take() + .expect("InvalidStateError - Cannot call `startRendering` twice"); + + let OfflineAudioContextRenderer { + renderer, + suspend_callbacks, + event_loop, + .. + } = renderer; + + self.base.set_state(AudioContextState::Running); + + let result = renderer.render_audiobuffer_sync(self, suspend_callbacks, &event_loop); + + self.base.set_state(AudioContextState::Closed); + let _ = self + .base + .send_event(EventDispatch::complete(result.clone())); + + // spin the event loop once more to handle the statechange/complete events + event_loop.handle_pending_events(); + + result + } + + /// Given the current connections and scheduled changes, starts rendering audio. + /// + /// Rendering is purely CPU bound and contains no `await` points, so calling this method will + /// block the executor until completion or until the context is suspended. + /// + /// This method will only adhere to scheduled suspensions via [`Self::suspend`] and will + /// ignore those provided via [`Self::suspend_sync`]. + /// + /// # Panics + /// + /// Panics if this method is called multiple times. + pub async fn start_rendering(&self) -> AudioBuffer { + // We are mixing async with a std Mutex, so be sure not to `await` while the lock is held + let renderer = self + .renderer + .lock() + .unwrap() + .take() + .expect("InvalidStateError - Cannot call `startRendering` twice"); + + let OfflineAudioContextRenderer { + renderer, + suspend_promises, + resume_receiver, + event_loop, + .. + } = renderer; + + self.base.set_state(AudioContextState::Running); + + let result = renderer + .render_audiobuffer(self.length, suspend_promises, resume_receiver, &event_loop) + .await; + + self.base.set_state(AudioContextState::Closed); + let _ = self + .base + .send_event(EventDispatch::complete(result.clone())); + + // spin the event loop once more to handle the statechange/complete events + event_loop.handle_pending_events(); + + result + } + + /// get the length of rendering audio buffer + // false positive: OfflineAudioContext is not const + #[allow(clippy::missing_const_for_fn, clippy::unused_self)] + #[must_use] + pub fn length(&self) -> usize { + self.length + } + + #[track_caller] + fn calculate_suspend_frame(&self, suspend_time: f64) -> usize { + assert!( + suspend_time >= 0., + "InvalidStateError: suspendTime cannot be negative" + ); + assert!( + suspend_time < self.length as f64 / self.sample_rate() as f64, + "InvalidStateError: suspendTime cannot be greater than or equal to the total render duration" + ); + (suspend_time * self.base.sample_rate() as f64 / RENDER_QUANTUM_SIZE as f64).ceil() as usize + } + + /// Schedules a suspension of the time progression in the audio context at the specified time + /// and returns a promise + /// + /// The specified time is quantized and rounded up to the render quantum size. + /// + /// # Panics + /// + /// Panics if the quantized frame number + /// + /// - is negative or + /// - is less than or equal to the current time or + /// - is greater than or equal to the total render duration or + /// - is scheduled by another suspend for the same time + /// + /// # Example usage + /// + /// ```rust + /// use futures::{executor, join}; + /// use futures::FutureExt as _; + /// use std::sync::Arc; + /// + /// use web_audio_api::context::BaseAudioContext; + /// use web_audio_api::context::OfflineAudioContext; + /// use web_audio_api::node::{AudioNode, AudioScheduledSourceNode}; + /// + /// let context = Arc::new(OfflineAudioContext::new(1, 512, 44_100.)); + /// let context_clone = Arc::clone(&context); + /// + /// let suspend_promise = context.suspend(128. / 44_100.).then(|_| async move { + /// let mut src = context_clone.create_constant_source(); + /// src.connect(&context_clone.destination()); + /// src.start(); + /// context_clone.resume().await; + /// }); + /// + /// let render_promise = context.start_rendering(); + /// + /// let buffer = executor::block_on(async move { join!(suspend_promise, render_promise).1 }); + /// assert_eq!(buffer.number_of_channels(), 1); + /// assert_eq!(buffer.length(), 512); + /// ``` + pub async fn suspend(&self, suspend_time: f64) { + let quantum = self.calculate_suspend_frame(suspend_time); + + let (sender, receiver) = oneshot::channel(); + + // We are mixing async with a std Mutex, so be sure not to `await` while the lock is held + { + let mut lock = self.renderer.lock().unwrap(); + let renderer = lock + .as_mut() + .expect("InvalidStateError - cannot suspend when rendering has already started"); + + let insert_pos = renderer + .suspend_promises + .binary_search_by_key(&quantum, |&(q, _)| q) + .expect_err( + "InvalidStateError - cannot suspend multiple times at the same render quantum", + ); + + renderer + .suspend_promises + .insert(insert_pos, (quantum, sender)); + } // lock is dropped + + receiver.await.unwrap(); + self.base().set_state(AudioContextState::Suspended); + } + + /// Schedules a suspension of the time progression in the audio context at the specified time + /// and runs a callback. + /// + /// This is a synchronous version of [`Self::suspend`] that runs the provided callback at + /// the `suspendTime`. The rendering resumes automatically after the callback has run, so there + /// is no `resume_sync` method. + /// + /// The specified time is quantized and rounded up to the render quantum size. + /// + /// # Panics + /// + /// Panics if the quantized frame number + /// + /// - is negative or + /// - is less than or equal to the current time or + /// - is greater than or equal to the total render duration or + /// - is scheduled by another suspend for the same time + /// + /// # Example usage + /// + /// ```rust + /// use web_audio_api::context::BaseAudioContext; + /// use web_audio_api::context::OfflineAudioContext; + /// use web_audio_api::node::{AudioNode, AudioScheduledSourceNode}; + /// + /// let mut context = OfflineAudioContext::new(1, 512, 44_100.); + /// + /// context.suspend_sync(128. / 44_100., |context| { + /// let mut src = context.create_constant_source(); + /// src.connect(&context.destination()); + /// src.start(); + /// }); + /// + /// let buffer = context.start_rendering_sync(); + /// assert_eq!(buffer.number_of_channels(), 1); + /// assert_eq!(buffer.length(), 512); + /// ``` + pub fn suspend_sync( + &mut self, + suspend_time: f64, + callback: F, + ) { + let quantum = self.calculate_suspend_frame(suspend_time); + + let mut lock = self.renderer.lock().unwrap(); + let renderer = lock + .as_mut() + .expect("InvalidStateError - cannot suspend when rendering has already started"); + + let insert_pos = renderer + .suspend_callbacks + .binary_search_by_key(&quantum, |(q, _c)| *q) + .expect_err( + "InvalidStateError - cannot suspend multiple times at the same render quantum", + ); + + let boxed_callback = Box::new(|ctx: &mut OfflineAudioContext| { + ctx.base().set_state(AudioContextState::Suspended); + (callback)(ctx); + ctx.base().set_state(AudioContextState::Running); + }); + + renderer + .suspend_callbacks + .insert(insert_pos, (quantum, boxed_callback)); + } + + /// Resumes the progression of the OfflineAudioContext's currentTime when it has been suspended + /// + /// # Panics + /// + /// Panics when the context is closed or rendering has not started + pub async fn resume(&self) { + self.base().set_state(AudioContextState::Running); + self.resume_sender.clone().send(()).await.unwrap() + } + + /// Register callback to run when the rendering has completed + /// + /// Only a single event handler is active at any time. Calling this method multiple times will + /// override the previous event handler. + #[allow(clippy::missing_panics_doc)] + pub fn set_oncomplete( + &self, + callback: F, + ) { + let callback = move |v| match v { + EventPayload::Complete(v) => { + let event = OfflineAudioCompletionEvent { + rendered_buffer: v, + event: Event { type_: "complete" }, + }; + callback(event) + } + _ => unreachable!(), + }; + + self.base() + .set_event_handler(EventType::Complete, EventHandler::Once(Box::new(callback))); + } + + /// Unset the callback to run when the rendering has completed + pub fn clear_oncomplete(&self) { + self.base().clear_event_handler(EventType::Complete); + } +} + +#[cfg(test)] +mod tests { + use super::*; + use float_eq::assert_float_eq; + use std::sync::atomic::{AtomicBool, Ordering}; + + use crate::node::AudioNode; + use crate::node::AudioScheduledSourceNode; + + #[test] + fn test_sample_rate_length() { + let context = OfflineAudioContext::new(1, 48000, 96000.); + assert_float_eq!(context.sample_rate(), 96000., abs_all <= 0.); + assert_eq!(context.length(), 48000); + } + + #[test] + fn render_empty_graph() { + let mut context = OfflineAudioContext::new(2, 555, 44_100.); + assert_eq!(context.state(), AudioContextState::Suspended); + let buffer = context.start_rendering_sync(); + + assert_eq!(context.length(), 555); + + assert_eq!(buffer.number_of_channels(), 2); + assert_eq!(buffer.length(), 555); + assert_float_eq!(buffer.get_channel_data(0), &[0.; 555][..], abs_all <= 0.); + assert_float_eq!(buffer.get_channel_data(1), &[0.; 555][..], abs_all <= 0.); + + assert_eq!(context.state(), AudioContextState::Closed); + } + + #[test] + #[should_panic] + fn render_twice_panics() { + let mut context = OfflineAudioContext::new(2, 555, 44_100.); + let _ = context.start_rendering_sync(); + let _ = context.start_rendering_sync(); + } + + #[test] + fn test_suspend_sync() { + use crate::node::ConstantSourceNode; + use std::sync::OnceLock; + + let len = RENDER_QUANTUM_SIZE * 4; + let sample_rate = 48000_f64; + + let mut context = OfflineAudioContext::new(1, len, sample_rate as f32); + static SOURCE: OnceLock = OnceLock::new(); + + context.suspend_sync(RENDER_QUANTUM_SIZE as f64 / sample_rate, |context| { + assert_eq!(context.state(), AudioContextState::Suspended); + let mut src = context.create_constant_source(); + src.connect(&context.destination()); + src.start(); + SOURCE.set(src).unwrap(); + }); + + context.suspend_sync((3 * RENDER_QUANTUM_SIZE) as f64 / sample_rate, |context| { + assert_eq!(context.state(), AudioContextState::Suspended); + SOURCE.get().unwrap().disconnect(); + }); + + let output = context.start_rendering_sync(); + + assert_float_eq!( + output.get_channel_data(0)[..RENDER_QUANTUM_SIZE], + &[0.; RENDER_QUANTUM_SIZE][..], + abs_all <= 0. + ); + assert_float_eq!( + output.get_channel_data(0)[RENDER_QUANTUM_SIZE..3 * RENDER_QUANTUM_SIZE], + &[1.; 2 * RENDER_QUANTUM_SIZE][..], + abs_all <= 0. + ); + assert_float_eq!( + output.get_channel_data(0)[3 * RENDER_QUANTUM_SIZE..4 * RENDER_QUANTUM_SIZE], + &[0.; RENDER_QUANTUM_SIZE][..], + abs_all <= 0. + ); + } + + #[test] + fn render_suspend_resume_async() { + use futures::executor; + use futures::join; + use futures::FutureExt as _; + + let context = Arc::new(OfflineAudioContext::new(1, 512, 44_100.)); + let context_clone = Arc::clone(&context); + + let suspend_promise = context.suspend(128. / 44_100.).then(|_| async move { + let mut src = context_clone.create_constant_source(); + src.connect(&context_clone.destination()); + src.start(); + context_clone.resume().await; + }); + + let render_promise = context.start_rendering(); + + let buffer = executor::block_on(async move { join!(suspend_promise, render_promise).1 }); + + assert_eq!(buffer.number_of_channels(), 1); + assert_eq!(buffer.length(), 512); + + assert_float_eq!( + buffer.get_channel_data(0)[..128], + &[0.; 128][..], + abs_all <= 0. + ); + assert_float_eq!( + buffer.get_channel_data(0)[128..], + &[1.; 384][..], + abs_all <= 0. + ); + } + + #[test] + #[should_panic] + fn test_suspend_negative_panics() { + let mut context = OfflineAudioContext::new(2, 128, 44_100.); + context.suspend_sync(-1.0, |_| ()); + } + + #[test] + #[should_panic] + fn test_suspend_after_duration_panics() { + let mut context = OfflineAudioContext::new(2, 128, 44_100.); + context.suspend_sync(1.0, |_| ()); + } + + #[test] + #[should_panic] + fn test_suspend_after_render_panics() { + let mut context = OfflineAudioContext::new(2, 128, 44_100.); + let _ = context.start_rendering_sync(); + context.suspend_sync(0.0, |_| ()); + } + + #[test] + #[should_panic] + fn test_suspend_identical_frame_panics() { + let mut context = OfflineAudioContext::new(2, 128, 44_100.); + context.suspend_sync(0.0, |_| ()); + context.suspend_sync(0.0, |_| ()); + } + + #[test] + fn test_onstatechange() { + let mut context = OfflineAudioContext::new(2, 555, 44_100.); + + let changed = Arc::new(AtomicBool::new(false)); + let changed_clone = Arc::clone(&changed); + context.set_onstatechange(move |_event| { + changed_clone.store(true, Ordering::Relaxed); + }); + + let _ = context.start_rendering_sync(); + + assert!(changed.load(Ordering::Relaxed)); + } + + #[test] + fn test_onstatechange_async() { + use futures::executor; + + let context = OfflineAudioContext::new(2, 555, 44_100.); + + let changed = Arc::new(AtomicBool::new(false)); + let changed_clone = Arc::clone(&changed); + context.set_onstatechange(move |_event| { + changed_clone.store(true, Ordering::Relaxed); + }); + + let _ = executor::block_on(context.start_rendering()); + + assert!(changed.load(Ordering::Relaxed)); + } + + #[test] + fn test_oncomplete() { + let mut context = OfflineAudioContext::new(2, 555, 44_100.); + + let complete = Arc::new(AtomicBool::new(false)); + let complete_clone = Arc::clone(&complete); + context.set_oncomplete(move |event| { + assert_eq!(event.rendered_buffer.length(), 555); + complete_clone.store(true, Ordering::Relaxed); + }); + + let _ = context.start_rendering_sync(); + + assert!(complete.load(Ordering::Relaxed)); + } + + #[test] + fn test_oncomplete_async() { + use futures::executor; + + let context = OfflineAudioContext::new(2, 555, 44_100.); + + let complete = Arc::new(AtomicBool::new(false)); + let complete_clone = Arc::clone(&complete); + context.set_oncomplete(move |event| { + assert_eq!(event.rendered_buffer.length(), 555); + complete_clone.store(true, Ordering::Relaxed); + }); + + let _ = executor::block_on(context.start_rendering()); + + assert!(complete.load(Ordering::Relaxed)); + } + + fn require_send_sync(_: T) {} + + #[test] + fn test_all_futures_thread_safe() { + let context = OfflineAudioContext::new(2, 555, 44_100.); + + require_send_sync(context.start_rendering()); + require_send_sync(context.suspend(1.)); + require_send_sync(context.resume()); + } +} diff --git a/vendor/web-audio-api-1.7.0/src/context/online.rs b/vendor/web-audio-api-1.7.0/src/context/online.rs new file mode 100644 index 0000000000..a461db904b --- /dev/null +++ b/vendor/web-audio-api-1.7.0/src/context/online.rs @@ -0,0 +1,1011 @@ +//! The `AudioContext` type and constructor options +use std::error::Error; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::{Arc, Mutex}; + +#[cfg(feature = "diagnostics")] +use crate::context::{AudioBackendDiagnostics, AudioContextDiagnostics}; +use crate::context::{AudioContextState, BaseAudioContext, ConcreteBaseAudioContext}; +#[cfg(feature = "diagnostics")] +use crate::events::EventPayload; +use crate::events::{EventDispatch, EventHandler, EventLoop, EventType}; +use crate::io::{self, AudioBackendManager, ControlThreadInit, NoneBackend, RenderThreadInit}; +use crate::media_devices::{enumerate_devices_sync, MediaDeviceInfoKind}; +use crate::media_streams::{MediaStream, MediaStreamTrack}; +use crate::message::{ControlMessage, OneshotNotify}; +use crate::node::{self, AudioNodeOptions}; +use crate::render::graph::Graph; +use crate::MediaElement; +use crate::{is_valid_sample_rate, AudioPlaybackStats, AudioRenderCapacity, Event}; + +use futures_channel::oneshot; + +/// Check if the provided sink_id is available for playback +/// +/// It should be "", "none" or a valid output `sinkId` returned from [`enumerate_devices_sync`] +fn is_valid_sink_id(sink_id: &str) -> bool { + if sink_id.is_empty() || sink_id == "none" { + true + } else { + enumerate_devices_sync() + .into_iter() + .filter(|d| d.kind() == MediaDeviceInfoKind::AudioOutput) + .any(|d| d.device_id() == sink_id) + } +} + +#[derive(Debug)] +enum AudioContextError { + SinkNotFound { sink_id: String }, + InvalidSampleRate { sample_rate: f32 }, + Backend { error: io::AudioBackendError }, +} + +impl std::fmt::Display for AudioContextError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::SinkNotFound { sink_id } => { + write!(f, "NotFoundError - Invalid sinkId: {sink_id:?}") + } + Self::InvalidSampleRate { sample_rate } => { + write!( + f, + "NotSupportedError - Invalid sample rate: {sample_rate}, should be in the range [3000.0, 768000.0]" + ) + } + Self::Backend { error } => write!(f, "InvalidStateError - {error}"), + } + } +} + +impl Error for AudioContextError {} + +impl From for AudioContextError { + fn from(error: io::AudioBackendError) -> Self { + Self::Backend { error } + } +} + +/// Identify the type of playback, which affects tradeoffs +/// between audio output latency and power consumption +#[derive(Copy, Clone, Debug, Default)] +pub enum AudioContextLatencyCategory { + /// Balance audio output latency and power consumption. + Balanced, + /// Provide the lowest audio output latency possible without glitching. This is the default. + #[default] + Interactive, + /// Prioritize sustained playback without interruption over audio output latency. + /// + /// Lowest power consumption. + Playback, + /// Specify the number of seconds of latency + /// + /// This latency is not guaranteed to be applied, it depends on the audio hardware capabilities + Custom(f64), +} + +#[derive(Copy, Clone, Debug)] +#[non_exhaustive] +/// This allows users to ask for a particular render quantum size. +/// +/// Currently, only the default value is available +#[derive(Default)] +pub enum AudioContextRenderSizeCategory { + /// The default value of 128 frames + #[default] + Default, +} + +/// Specify the playback configuration for the [`AudioContext`] constructor. +/// +/// All fields are optional and will default to the value best suited for interactive playback on +/// your hardware configuration. +/// +/// For future compatibility, it is best to construct a default implementation of this struct and +/// set the fields you would like to override: +/// ``` +/// use web_audio_api::context::AudioContextOptions; +/// +/// // Request a sample rate of 44.1 kHz, leave other fields to their default values +/// let opts = AudioContextOptions { +/// sample_rate: Some(44100.), +/// ..AudioContextOptions::default() +/// }; +#[derive(Clone, Debug, Default)] +pub struct AudioContextOptions { + /// Identify the type of playback, which affects tradeoffs between audio output latency and + /// power consumption. + pub latency_hint: AudioContextLatencyCategory, + + /// Sample rate of the audio context and audio output hardware. Use `None` for a default value. + pub sample_rate: Option, + + /// The audio output device + /// - use `""` for the default audio output device + /// - use `"none"` to process the audio graph without playing through an audio output device. + /// - use `"sinkId"` to use the specified audio sink id, obtained with [`enumerate_devices_sync`] + pub sink_id: String, + + /// Option to request a default, optimized or specific render quantum size. It is a hint that might not be honored. + pub render_size_hint: AudioContextRenderSizeCategory, +} + +/// This interface represents an audio graph whose `AudioDestinationNode` is routed to a real-time +/// output device that produces a signal directed at the user. +// the naming comes from the web audio specification +#[allow(clippy::module_name_repetitions)] +pub struct AudioContext { + /// represents the underlying `BaseAudioContext` + base: ConcreteBaseAudioContext, + /// audio backend (play/pause functionality) + backend_manager: Mutex>, + /// Provider for rendering performance metrics + render_capacity: AudioRenderCapacity, + /// Provider for playback statistics + playback_stats: AudioPlaybackStats, + /// true while the render thread has not yet processed its initial Startup message + startup_pending: std::sync::Arc, + /// Initializer for the render thread (when restart is required) + render_thread_init: RenderThreadInit, +} + +impl std::fmt::Debug for AudioContext { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("AudioContext") + .field("sink_id", &self.sink_id()) + .field("base_latency", &self.base_latency()) + .field("output_latency", &self.output_latency()) + .field("base", &self.base()) + .finish_non_exhaustive() + } +} + +impl Drop for AudioContext { + fn drop(&mut self) { + // Continue playing the stream if the AudioContext goes out of scope + if self.state() == AudioContextState::Running { + let tombstone = Box::new(NoneBackend::void()); + let original = std::mem::replace(self.backend_manager.get_mut().unwrap(), tombstone); + Box::leak(original); + } + } +} + +impl BaseAudioContext for AudioContext { + fn base(&self) -> &ConcreteBaseAudioContext { + &self.base + } +} + +impl Default for AudioContext { + fn default() -> Self { + Self::new(AudioContextOptions::default()) + } +} + +impl AudioContext { + /// Creates and returns a new `AudioContext` object. + /// + /// This will play live audio on the default output device. + /// + /// ```no_run + /// use web_audio_api::context::{AudioContext, AudioContextOptions}; + /// + /// // Request a sample rate of 44.1 kHz and default latency (buffer size 128, if available) + /// let opts = AudioContextOptions { + /// sample_rate: Some(44100.), + /// ..AudioContextOptions::default() + /// }; + /// + /// // Setup the audio context that will emit to your speakers + /// let context = AudioContext::new(opts); + /// + /// // Alternatively, use the default constructor to get the best settings for your hardware + /// // let context = AudioContext::default(); + /// ``` + /// + /// # Panics + /// + /// The `AudioContext` constructor will panic when an invalid `sinkId` is provided in the + /// `AudioContextOptions`, when the sample rate is outside the valid range [3000.0, 768000.0], + /// or when the selected audio backend cannot create or start the output stream. Use + /// [`Self::try_new`] to handle these errors without panicking. + #[must_use] + pub fn new(options: AudioContextOptions) -> Self { + Self::try_new_inner(options).unwrap_or_else(|e| panic!("{e}")) + } + + /// Creates and returns a new `AudioContext` object. + /// + /// This will play live audio on the requested output device and returns backend errors instead + /// of panicking when the stream cannot be created. + /// + /// # Errors + /// + /// Returns an error when the sink id is invalid, the sample rate is outside the valid range + /// [3000.0, 768000.0], or when the selected audio backend cannot create or start the output + /// stream. + pub fn try_new(options: AudioContextOptions) -> Result> { + Self::try_new_inner(options).map_err(Into::into) + } + + fn try_new_inner(options: AudioContextOptions) -> Result { + // https://webaudio.github.io/web-audio-api/#validating-sink-identifier + if !is_valid_sink_id(&options.sink_id) { + return Err(AudioContextError::SinkNotFound { + sink_id: options.sink_id, + }); + } + + // Validate sample_rate if provided + // https://webaudio.github.io/web-audio-api/#sample-rates + if let Some(sample_rate) = options.sample_rate { + if !is_valid_sample_rate(sample_rate) { + return Err(AudioContextError::InvalidSampleRate { sample_rate }); + } + } + + // Set up the audio output thread + let (control_thread_init, render_thread_init) = io::thread_init(); + let startup_pending = Arc::clone(&render_thread_init.startup_pending); + let backend = io::build_output(options, render_thread_init.clone())?; + + let ControlThreadInit { + state, + frames_played, + stats, + ctrl_msg_send, + event_send, + event_recv, + } = control_thread_init; + + // Construct the audio Graph and hand it to the render thread + let (node_id_producer, node_id_consumer) = llq::Queue::new().split(); + let graph = Graph::new(node_id_producer); + let message = ControlMessage::Startup { graph }; + ctrl_msg_send.send(message).unwrap(); + + // Set up the event loop thread that handles the events spawned by the render thread + let event_loop = EventLoop::new(event_recv); + + // Put everything together in the BaseAudioContext (shared with offline context) + let base = ConcreteBaseAudioContext::new( + backend.sample_rate(), + backend.number_of_channels(), + state, + frames_played, + ctrl_msg_send, + event_send, + event_loop.clone(), + false, + node_id_consumer, + ); + + // Setup AudioRenderCapacity for this context + let render_capacity = AudioRenderCapacity::new(base.clone(), stats.clone()); + let playback_stats = AudioPlaybackStats::new(base.clone(), stats); + + // As the final step, spawn a thread for the event loop. If we do this earlier we may miss + // event handling of the initial events that are emitted right after render thread + // construction. + event_loop.run_in_thread(); + + Ok(Self { + base, + backend_manager: Mutex::new(backend), + render_capacity, + playback_stats, + startup_pending, + render_thread_init, + }) + } + + /// This represents the number of seconds of processing latency incurred by + /// the `AudioContext` passing the audio from the `AudioDestinationNode` + /// to the audio subsystem. + // We don't do any buffering between rendering the audio and sending + // it to the audio subsystem, so this value is zero. (see Gecko) + #[allow(clippy::unused_self)] + #[must_use] + pub fn base_latency(&self) -> f64 { + 0. + } + + /// The estimation in seconds of audio output latency, i.e., the interval + /// between the time the UA requests the host system to play a buffer and + /// the time at which the first sample in the buffer is actually processed + /// by the audio output device. + #[must_use] + #[allow(clippy::missing_panics_doc)] + pub fn output_latency(&self) -> f64 { + self.try_output_latency() + .unwrap_or_else(|e| panic!("InvalidStateError - {e}")) + } + + /// The estimation in seconds of audio output latency. + /// + /// # Errors + /// + /// Returns an error when the selected audio backend cannot query the output latency. + fn try_output_latency(&self) -> Result> { + Ok(self.backend_manager.lock().unwrap().output_latency()?) + } + + /// Identifier or the information of the current audio output device. + /// + /// The initial value is `""`, which means the default audio output device. + #[allow(clippy::missing_panics_doc)] + pub fn sink_id(&self) -> String { + self.backend_manager.lock().unwrap().sink_id().to_owned() + } + + /// Returns an [`AudioRenderCapacity`] instance associated with an AudioContext. + #[must_use] + pub fn render_capacity(&self) -> AudioRenderCapacity { + self.render_capacity.clone() + } + + /// Returns an [`AudioPlaybackStats`] instance associated with this `AudioContext`. + #[must_use] + pub fn playback_stats(&self) -> AudioPlaybackStats { + self.playback_stats.clone() + } + + /// Update the current audio output device. + /// + /// The provided `sink_id` string must match a device name `enumerate_devices_sync`. + /// + /// Supplying `"none"` for the `sink_id` will process the audio graph without playing through an + /// audio output device. + /// + /// This function operates synchronously and might block the current thread. An async version + /// is currently not implemented. + #[allow(clippy::needless_collect, clippy::missing_panics_doc)] + pub fn set_sink_id_sync(&self, sink_id: String) -> Result<(), Box> { + log::debug!("SinkChange requested"); + if self.sink_id() == sink_id { + log::debug!("SinkChange: no-op"); + return Ok(()); // sink is already active + } + + if !is_valid_sink_id(&sink_id) { + Err(format!("NotFoundError: invalid sinkId {sink_id}"))?; + }; + + log::debug!("SinkChange: locking backend manager"); + let mut backend_manager_guard = self.backend_manager.lock().unwrap(); + let original_state = self.state(); + if original_state == AudioContextState::Closed { + log::debug!("SinkChange: context is closed"); + return Ok(()); + } + + // Acquire exclusive lock on ctrl msg sender + log::debug!("SinkChange: locking message channel"); + let ctrl_msg_send = self.base.lock_control_msg_sender(); + + // Flush out the ctrl msg receiver, cache + let mut pending_msgs: Vec<_> = self.render_thread_init.ctrl_msg_recv.try_iter().collect(); + + // Acquire the active audio graph from the current render thread, shutting it down + let graph = if matches!(pending_msgs.first(), Some(ControlMessage::Startup { .. })) { + // Handle the edge case where the previous backend was suspended for its entire lifetime. + // In this case, the `Startup` control message was never processed. + log::debug!("SinkChange: recover unstarted graph"); + + let msg = pending_msgs.remove(0); + match msg { + ControlMessage::Startup { graph } => graph, + _ => unreachable!(), + } + } else { + // Acquire the audio graph from the current render thread, shutting it down + log::debug!("SinkChange: recover graph from render thread"); + + let (graph_send, graph_recv) = crossbeam_channel::bounded(1); + let message = ControlMessage::CloseAndRecycle { sender: graph_send }; + ctrl_msg_send.send(message).unwrap(); + if original_state == AudioContextState::Suspended { + // We must wake up the render thread to be able to handle the shutdown. + // No new audio will be produced because it will receive the shutdown command first. + backend_manager_guard.resume()?; + } + graph_recv.recv().unwrap() + }; + + log::debug!("SinkChange: closing audio stream"); + backend_manager_guard.close()?; + + // hotswap the backend + let options = AudioContextOptions { + sample_rate: Some(self.sample_rate()), + latency_hint: AudioContextLatencyCategory::default(), // todo reuse existing setting + sink_id, + render_size_hint: AudioContextRenderSizeCategory::default(), // todo reuse existing setting + }; + log::debug!("SinkChange: starting audio stream"); + *backend_manager_guard = io::build_output(options, self.render_thread_init.clone())?; + + // if the previous backend state was suspend, suspend the new one before shipping the graph + if original_state == AudioContextState::Suspended { + log::debug!("SinkChange: suspending audio stream"); + backend_manager_guard.suspend()?; + } + + // send the audio graph to the new render thread + let message = ControlMessage::Startup { graph }; + ctrl_msg_send.send(message).unwrap(); + + // flush the cached msgs + pending_msgs + .into_iter() + .for_each(|m| self.base().send_control_msg(m)); + + // explicitly release the lock to prevent concurrent render threads + drop(backend_manager_guard); + + // trigger event when all the work is done + let _ = self.base.send_event(EventDispatch::sink_change()); + + log::debug!("SinkChange: done"); + Ok(()) + } + + /// Register callback to run when the audio sink has changed + /// + /// Only a single event handler is active at any time. Calling this method multiple times will + /// override the previous event handler. + pub fn set_onsinkchange(&self, mut callback: F) { + let callback = move |_| { + callback(Event { + type_: "sinkchange", + }) + }; + + self.base().set_event_handler( + EventType::SinkChange, + EventHandler::Multiple(Box::new(callback)), + ); + } + + /// Unset the callback to run when the audio sink has changed + pub fn clear_onsinkchange(&self) { + self.base().clear_event_handler(EventType::SinkChange); + } + + /// Request a structured diagnostic report of the audio context. + /// + /// The report is collected asynchronously: backend details are captured on the control thread, + /// while render thread and graph details are captured in the realtime render thread. The + /// callback is invoked once on the event loop thread. + /// + /// This API is available with the `diagnostics` crate feature. + #[cfg(feature = "diagnostics")] + #[allow(clippy::missing_panics_doc)] + pub fn run_diagnostics(&self, callback: F) { + let backend = { + let backend = self.backend_manager.lock().unwrap(); + AudioBackendDiagnostics { + name: backend.name().to_string(), + sink_id: backend.sink_id().to_string(), + output_latency: backend.output_latency().ok(), + } + }; + + let callback = move |v| match v { + EventPayload::Diagnostics(v) => { + callback(v); + } + _ => unreachable!(), + }; + + self.base().set_event_handler( + EventType::Diagnostics, + EventHandler::Once(Box::new(callback)), + ); + + self.base() + .send_control_msg(ControlMessage::RunDiagnostics { backend }); + } + + /// Suspends the progression of time in the audio context. + /// + /// This will temporarily halt audio hardware access and reducing CPU/battery usage in the + /// process. + /// + /// # Panics + /// + /// Will panic if: + /// + /// * The audio device is not available + /// * For a `BackendSpecificError` + pub async fn suspend(&self) { + // Don't lock the backend manager because we can't hold is across the await point + log::debug!("Suspend called"); + + let state = self.state(); + if state == AudioContextState::Closed { + log::debug!("Suspend no-op - context is closed"); + return; + } + + if state != AudioContextState::Running && !self.startup_pending.load(Ordering::Acquire) { + log::debug!("Suspend no-op - context is not running"); + return; + } + + // Pause rendering via a control message + let (sender, receiver) = oneshot::channel(); + let notify = OneshotNotify::Async(sender); + self.base + .suspend_control_msgs(ControlMessage::Suspend { notify }); + + // Wait for the render thread to have processed the suspend message. + // The AudioContextState will be updated by the render thread. + log::debug!("Suspending audio graph, waiting for signal.."); + receiver.await.unwrap(); + + // Then ask the audio host to suspend the stream + log::debug!("Suspended audio graph. Suspending audio stream.."); + self.backend_manager + .lock() + .unwrap() + .suspend() + .unwrap_or_else(|e| panic!("InvalidStateError - {e}")); + + log::debug!("Suspended audio stream"); + } + + /// Resumes the progression of time in an audio context that has previously been + /// suspended/paused. + /// + /// # Panics + /// + /// Will panic if: + /// + /// * The audio device is not available + /// * For a `BackendSpecificError` + pub async fn resume(&self) { + let (sender, receiver) = oneshot::channel(); + + { + // Lock the backend manager mutex to avoid concurrent calls + log::debug!("Resume called, locking backend manager"); + let backend_manager_guard = self.backend_manager.lock().unwrap(); + + if self.state() != AudioContextState::Suspended { + log::debug!("Resume no-op - context is not suspended"); + return; + } + + // Ask the audio host to resume the stream + backend_manager_guard + .resume() + .unwrap_or_else(|e| panic!("InvalidStateError - {e}")); + + // Then, ask to resume rendering via a control message + log::debug!("Resumed audio stream, waking audio graph"); + let notify = OneshotNotify::Async(sender); + self.base + .resume_control_msgs(ControlMessage::Resume { notify }); + + // Drop the Mutex guard so we won't hold it across an await point + } + + // Wait for the render thread to have processed the resume message + // The AudioContextState will be updated by the render thread. + receiver.await.unwrap(); + log::debug!("Resumed audio graph"); + } + + /// Closes the `AudioContext`, releasing the system resources being used. + /// + /// This will not automatically release all `AudioContext`-created objects, but will suspend + /// the progression of the currentTime, and stop processing audio data. + /// + /// # Panics + /// + /// Will panic when this function is called multiple times + pub async fn close(&self) { + // Don't lock the backend manager because we can't hold is across the await point + log::debug!("Close called"); + + if self.state() == AudioContextState::Closed { + log::debug!("Close no-op - context is already closed"); + return; + } + + // Stop AudioRenderCapacity before closing so no capacity events are queued during shutdown. + self.render_capacity.stop(); + + if self.state() == AudioContextState::Running { + // First, stop rendering via a control message + let (sender, receiver) = oneshot::channel(); + let notify = OneshotNotify::Async(sender); + self.base.send_control_msg(ControlMessage::Close { notify }); + + // Wait for the render thread to have processed the suspend message. + // The AudioContextState will be updated by the render thread. + log::debug!("Suspending audio graph, waiting for signal.."); + receiver.await.unwrap(); + } else { + // if the context is not running, change the state manually + self.base.set_state(AudioContextState::Closed); + } + + // Then ask the audio host to close the stream + log::debug!("Suspended audio graph. Closing audio stream.."); + self.backend_manager + .lock() + .unwrap() + .close() + .unwrap_or_else(|e| panic!("InvalidStateError - {e}")); + + log::debug!("Closed audio stream"); + } + + /// Suspends the progression of time in the audio context. + /// + /// This will temporarily halt audio hardware access and reducing CPU/battery usage in the + /// process. + /// + /// This function operates synchronously and blocks the current thread until the audio thread + /// has stopped processing. + /// + /// # Panics + /// + /// Will panic if: + /// + /// * The audio device is not available + /// * For a `BackendSpecificError` + pub fn suspend_sync(&self) { + // Lock the backend manager mutex to avoid concurrent calls + log::debug!("Suspend_sync called, locking backend manager"); + let backend_manager_guard = self.backend_manager.lock().unwrap(); + + let state = self.state(); + if state == AudioContextState::Closed { + log::debug!("Suspend_sync no-op - context is closed"); + return; + } + + if state != AudioContextState::Running && !self.startup_pending.load(Ordering::Acquire) { + log::debug!("Suspend_sync no-op - context is not running"); + return; + } + + // Pause rendering via a control message + let (sender, receiver) = crossbeam_channel::bounded(0); + let notify = OneshotNotify::Sync(sender); + self.base + .suspend_control_msgs(ControlMessage::Suspend { notify }); + + // Wait for the render thread to have processed the suspend message. + // The AudioContextState will be updated by the render thread. + log::debug!("Suspending audio graph, waiting for signal.."); + receiver.recv().ok(); + + // Then ask the audio host to suspend the stream + log::debug!("Suspended audio graph. Suspending audio stream.."); + backend_manager_guard + .suspend() + .unwrap_or_else(|e| panic!("InvalidStateError - {e}")); + + log::debug!("Suspended audio stream"); + } + + /// Resumes the progression of time in an audio context that has previously been + /// suspended/paused. + /// + /// This function operates synchronously and blocks the current thread until the audio thread + /// has started processing again. + /// + /// # Panics + /// + /// Will panic if: + /// + /// * The audio device is not available + /// * For a `BackendSpecificError` + pub fn resume_sync(&self) { + // Lock the backend manager mutex to avoid concurrent calls + log::debug!("Resume_sync called, locking backend manager"); + let backend_manager_guard = self.backend_manager.lock().unwrap(); + + if self.state() != AudioContextState::Suspended { + log::debug!("Resume no-op - context is not suspended"); + return; + } + + // Ask the audio host to resume the stream + backend_manager_guard + .resume() + .unwrap_or_else(|e| panic!("InvalidStateError - {e}")); + + // Then, ask to resume rendering via a control message + log::debug!("Resumed audio stream, waking audio graph"); + let (sender, receiver) = crossbeam_channel::bounded(0); + let notify = OneshotNotify::Sync(sender); + self.base + .resume_control_msgs(ControlMessage::Resume { notify }); + + // Wait for the render thread to have processed the resume message + // The AudioContextState will be updated by the render thread. + receiver.recv().ok(); + log::debug!("Resumed audio graph"); + } + + /// Closes the `AudioContext`, releasing the system resources being used. + /// + /// This will not automatically release all `AudioContext`-created objects, but will suspend + /// the progression of the currentTime, and stop processing audio data. + /// + /// This function operates synchronously and blocks the current thread until the audio thread + /// has stopped processing. + /// + /// # Panics + /// + /// Will panic when this function is called multiple times + pub fn close_sync(&self) { + // Lock the backend manager mutex to avoid concurrent calls + log::debug!("Close_sync called, locking backend manager"); + let backend_manager_guard = self.backend_manager.lock().unwrap(); + + if self.state() == AudioContextState::Closed { + log::debug!("Close no-op - context is already closed"); + return; + } + + // Stop AudioRenderCapacity before closing so no capacity events are queued during shutdown. + self.render_capacity.stop(); + + // First, stop rendering via a control message + if self.state() == AudioContextState::Running { + let (sender, receiver) = crossbeam_channel::bounded(0); + let notify = OneshotNotify::Sync(sender); + self.base.send_control_msg(ControlMessage::Close { notify }); + + // Wait for the render thread to have processed the suspend message. + // The AudioContextState will be updated by the render thread. + log::debug!("Suspending audio graph, waiting for signal.."); + receiver.recv().ok(); + } else { + // if the context is not running, change the state manually + self.base.set_state(AudioContextState::Closed); + } + + // Then ask the audio host to close the stream + log::debug!("Suspended audio graph. Closing audio stream.."); + backend_manager_guard + .close() + .unwrap_or_else(|e| panic!("InvalidStateError - {e}")); + + log::debug!("Closed audio stream"); + } + + /// Creates a [`MediaStreamAudioSourceNode`](node::MediaStreamAudioSourceNode) from a + /// [`MediaStream`] + #[must_use] + pub fn create_media_stream_source( + &self, + media: &MediaStream, + ) -> node::MediaStreamAudioSourceNode { + let opts = node::MediaStreamAudioSourceOptions { + media_stream: media, + }; + node::MediaStreamAudioSourceNode::new(self, opts) + } + + /// Creates a [`MediaStreamAudioDestinationNode`](node::MediaStreamAudioDestinationNode) + #[must_use] + pub fn create_media_stream_destination(&self) -> node::MediaStreamAudioDestinationNode { + let opts = AudioNodeOptions::default(); + node::MediaStreamAudioDestinationNode::new(self, opts) + } + + /// Creates a [`MediaStreamTrackAudioSourceNode`](node::MediaStreamTrackAudioSourceNode) from a + /// [`MediaStreamTrack`] + #[must_use] + pub fn create_media_stream_track_source( + &self, + media: &MediaStreamTrack, + ) -> node::MediaStreamTrackAudioSourceNode { + let opts = node::MediaStreamTrackAudioSourceOptions { + media_stream_track: media, + }; + node::MediaStreamTrackAudioSourceNode::new(self, opts) + } + + /// Creates a [`MediaElementAudioSourceNode`](node::MediaElementAudioSourceNode) from a + /// [`MediaElement`] + #[must_use] + pub fn create_media_element_source( + &self, + media_element: &mut MediaElement, + ) -> node::MediaElementAudioSourceNode { + let opts = node::MediaElementAudioSourceOptions { media_element }; + node::MediaElementAudioSourceNode::new(self, opts) + } +} + +#[cfg(test)] +mod tests { + use super::*; + #[cfg(feature = "diagnostics")] + use crate::context::DESTINATION_NODE_ID; + use futures::executor; + + #[test] + fn test_suspend_resume_close() { + let options = AudioContextOptions { + sink_id: "none".into(), + ..AudioContextOptions::default() + }; + + // construct with 'none' sink_id + let context = AudioContext::new(options); + + // Ensure startup has been processed before testing suspend/resume transitions. + executor::block_on(context.resume()); + assert_eq!(context.state(), AudioContextState::Running); + + executor::block_on(context.suspend()); + assert_eq!(context.state(), AudioContextState::Suspended); + let time1 = context.current_time(); + assert!(time1 >= 0.); + + // allow some time to progress + std::thread::sleep(std::time::Duration::from_millis(1)); + let time2 = context.current_time(); + assert_eq!(time1, time2); // no progression of time + + executor::block_on(context.resume()); + assert_eq!(context.state(), AudioContextState::Running); + + // allow some time to progress + std::thread::sleep(std::time::Duration::from_millis(1)); + + let time3 = context.current_time(); + assert!(time3 > time2); // time is progressing + + executor::block_on(context.close()); + assert_eq!(context.state(), AudioContextState::Closed); + + let time4 = context.current_time(); + + // allow some time to progress + std::thread::sleep(std::time::Duration::from_millis(1)); + + let time5 = context.current_time(); + assert_eq!(time5, time4); // no progression of time + } + + #[test] + fn test_suspend_during_startup() { + let options = AudioContextOptions { + sink_id: "none".into(), + ..AudioContextOptions::default() + }; + + let context = AudioContext::new(options); + + executor::block_on(context.suspend()); + assert_eq!(context.state(), AudioContextState::Suspended); + + let time1 = context.current_time(); + std::thread::sleep(std::time::Duration::from_millis(5)); + let time2 = context.current_time(); + assert_eq!(time1, time2); + } + + #[test] + fn test_suspend_sync_during_startup() { + let options = AudioContextOptions { + sink_id: "none".into(), + ..AudioContextOptions::default() + }; + + let context = AudioContext::new(options); + + context.suspend_sync(); + assert_eq!(context.state(), AudioContextState::Suspended); + + let time1 = context.current_time(); + std::thread::sleep(std::time::Duration::from_millis(5)); + let time2 = context.current_time(); + assert_eq!(time1, time2); + } + + fn require_send_sync(_: T) {} + + #[test] + fn test_all_futures_thread_safe() { + let options = AudioContextOptions { + sink_id: "none".into(), + ..AudioContextOptions::default() + }; + let context = AudioContext::new(options); + + require_send_sync(context.suspend()); + require_send_sync(context.resume()); + require_send_sync(context.close()); + } + + #[test] + fn test_try_new_invalid_sample_rate() { + let options = AudioContextOptions { + sample_rate: Some(0.), + sink_id: "none".into(), + ..AudioContextOptions::default() + }; + + let result = AudioContext::try_new(options); + assert!(result.is_err()); + let error_msg = result.unwrap_err().to_string(); + assert!(error_msg.contains("Invalid sample rate")); + } + + #[test] + #[should_panic] + fn test_invalid_sink_id() { + let options = AudioContextOptions { + sink_id: "invalid".into(), + ..AudioContextOptions::default() + }; + let _ = AudioContext::new(options); + } + + #[test] + fn test_try_new_invalid_sink_id() { + let options = AudioContextOptions { + sink_id: "invalid".into(), + ..AudioContextOptions::default() + }; + + let error = AudioContext::try_new(options).unwrap_err(); + assert_eq!( + error.to_string(), + "NotFoundError - Invalid sinkId: \"invalid\"" + ); + } + + #[cfg(feature = "diagnostics")] + #[test] + fn test_run_diagnostics_returns_structured_output() { + let options = AudioContextOptions { + sink_id: "none".into(), + ..AudioContextOptions::default() + }; + let context = AudioContext::new(options); + let (sender, receiver) = std::sync::mpsc::channel(); + + context.run_diagnostics(move |diagnostics| { + sender.send(diagnostics).unwrap(); + }); + + let diagnostics = receiver + .recv_timeout(std::time::Duration::from_secs(1)) + .unwrap(); + + assert!(diagnostics.backend.name.contains("NoneBackend")); + assert_eq!(diagnostics.backend.sink_id, "none"); + assert_eq!(diagnostics.backend.output_latency, Some(0.)); + assert_eq!(diagnostics.render_thread.sample_rate, context.sample_rate()); + assert_eq!( + diagnostics.render_thread.number_of_channels, + crate::MAX_CHANNELS + ); + assert!(diagnostics.graph.active); + assert_eq!(diagnostics.graph.node_count, diagnostics.graph.nodes.len()); + assert_eq!(diagnostics.graph.edge_count, 0); + assert!(diagnostics.graph.in_cycle.is_empty()); + assert!(diagnostics.graph.cycle_breakers.is_empty()); + assert!(diagnostics + .graph + .nodes + .iter() + .any(|node| node.id == DESTINATION_NODE_ID.0 + && node.inputs == node.input_channels.len() + && node.outputs == node.output_channels.len())); + } +} diff --git a/vendor/web-audio-api-1.7.0/src/decoding.rs b/vendor/web-audio-api-1.7.0/src/decoding.rs new file mode 100644 index 0000000000..7bc4136bbf --- /dev/null +++ b/vendor/web-audio-api-1.7.0/src/decoding.rs @@ -0,0 +1,310 @@ +use std::error::Error; +use std::io::{Read, Seek, SeekFrom}; + +use crate::buffer::{AudioBuffer, ChannelData}; + +use symphonia::core::audio::GenericAudioBufferRef; +use symphonia::core::codecs::audio::{ + AudioCodecId, AudioDecoder, AudioDecoderOptions, FinalizeResult, +}; +use symphonia::core::errors::Error as SymphoniaError; +use symphonia::core::formats::probe::Hint; +use symphonia::core::formats::{FormatOptions, FormatReader, TrackType}; +use symphonia::core::meta::MetadataOptions; + +pub(crate) fn decode_media_data( + input: R, + target_sample_rate: f32, +) -> Result> { + let mut sample_rate = None; + let mut buffer: Option = None; + + for chunk in MediaDecoder::try_new(input)? { + let chunk = chunk?; + + match sample_rate { + Some(rate) if rate != chunk.sample_rate() => { + return Err(Box::new(std::io::Error::new( + std::io::ErrorKind::InvalidData, + "decoded audio sample rate changed midstream", + ))); + } + Some(_) => {} + None => sample_rate = Some(chunk.sample_rate()), + } + + match buffer { + Some(ref mut buffer) if buffer.number_of_channels() != chunk.number_of_channels() => { + return Err(Box::new(std::io::Error::new( + std::io::ErrorKind::InvalidData, + "decoded audio channel count changed midstream", + ))); + } + Some(ref mut buffer) => buffer.extend(&chunk), + None => buffer = Some(chunk), + } + } + + let mut buffer = buffer.unwrap_or_else(|| AudioBuffer::from(vec![vec![]], target_sample_rate)); + + // Resample to desired rate (no-op if already matching). + buffer.resample(target_sample_rate); + + Ok(buffer) +} + +/// Wrapper for `Read` implementers to be used in Symphonia decoding +/// +/// Symphonia requires its input to impl `Seek` - but allows non-seekable sources. Hence we +/// implement Seek but return false for `is_seekable()`. +struct MediaInput { + input: R, +} + +impl MediaInput { + pub fn new(input: R) -> Self { + Self { input } + } +} + +impl Read for MediaInput { + fn read(&mut self, buf: &mut [u8]) -> std::io::Result { + self.input.read(buf) + } +} + +impl Seek for MediaInput { + fn seek(&mut self, _pos: SeekFrom) -> std::io::Result { + Err(std::io::Error::new( + std::io::ErrorKind::Unsupported, + "MediaInput does not support seeking", + )) + } +} + +impl symphonia::core::io::MediaSource for MediaInput { + fn is_seekable(&self) -> bool { + false + } + fn byte_len(&self) -> Option { + None + } +} + +/// Media stream decoder (OGG, WAV, FLAC, ..) +/// +/// The current implementation supports Symphonia's audio formats and codecs. +pub(crate) struct MediaDecoder { + format: Box, + decoder: Box, + track_index: usize, + packet_count: usize, +} + +impl MediaDecoder { + /// Try to construct a new instance from a `Read` implementer + /// + /// # Errors + /// + /// This method returns an Error in various cases (IO, mime sniffing, decoding). + pub fn try_new( + input: R, + ) -> Result> { + // Symphonia lib needs a Box - use our own MediaInput + let input = Box::new(MediaInput::new(input)); + + // Create the media source stream using the boxed media source from above. + let stream = symphonia::core::io::MediaSourceStream::new(input, Default::default()); + + // Create a hint to help the format registry guess what format reader is appropriate. In this + // function we'll leave it empty. + let hint = Hint::new(); + + // TODO: Allow to customize some options. + let format_opts: FormatOptions = Default::default(); + let metadata_opts: MetadataOptions = Default::default(); + // Opt-in to verify the decoded data against the checksums in the container. + let decoder_opts = AudioDecoderOptions::default().verify(true); + + // Probe the media source stream for a format. + let format = + symphonia::default::get_probe().probe(&hint, stream, format_opts, metadata_opts)?; + + // Get the default audio track. + let track = format + .default_track(TrackType::Audio) + .ok_or(SymphoniaError::Unsupported( + "no default media track available", + ))?; + let track_index = format + .tracks() + .iter() + .position(|t| t.id == track.id) + .unwrap(); + + let codec_params = track + .codec_params + .as_ref() + .and_then(|params| params.audio()) + .ok_or(SymphoniaError::Unsupported( + "default media track is not an audio track", + ))?; + + // Create a (stateful) decoder for the track. + let decoder = symphonia::default::get_codecs() + .make_audio_decoder(codec_params, &decoder_opts) + .map_err(|err| { + if matches!( + err, + SymphoniaError::Unsupported("core (codec): unsupported audio codec") + ) { + Box::new(UnsupportedAudioCodecError { + codec: codec_params.codec, + }) as Box + } else { + Box::new(err) as Box + } + })?; + + Ok(Self { + format, + decoder, + track_index, + packet_count: 0, + }) + } +} + +#[derive(Debug)] +struct UnsupportedAudioCodecError { + codec: AudioCodecId, +} + +impl std::fmt::Display for UnsupportedAudioCodecError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "unsupported audio codec: {}", self.codec) + } +} + +impl std::error::Error for UnsupportedAudioCodecError {} + +impl Iterator for MediaDecoder { + type Item = Result>; + + fn next(&mut self) -> Option { + let Self { + format, + decoder, + track_index, + packet_count, + } = self; + + // Get the track. + let track = format.tracks().get(*track_index)?; + let track_id = track.id; + + loop { + // Get the next packet from the format reader. + let packet = match format.next_packet() { + Ok(None) => { + log::debug!("Decoding finished after {packet_count} packet(s)"); + let FinalizeResult { verify_ok } = decoder.finalize(); + if verify_ok == Some(false) { + log::warn!("Verification of decoded data failed"); + } + return None; + } + Err(err) => { + if let SymphoniaError::IoError(err) = &err { + if err.kind() == std::io::ErrorKind::UnexpectedEof { + log::debug!( + "Decoding finished after {packet_count} packet(s) at unexpected EOF" + ); + let FinalizeResult { verify_ok } = decoder.finalize(); + if verify_ok == Some(false) { + log::warn!("Verification of decoded data failed"); + } + return None; + } + } + + log::warn!( + "Failed to fetch next packet following packet #{packet_count}: {err}" + ); + return Some(Err(Box::new(err))); + } + Ok(Some(packet)) => { + *packet_count += 1; + packet + } + }; + + // If the packet does not belong to the selected track, skip it. + let packet_track_id = packet.track_id; + if packet_track_id != track_id { + log::debug!( + "Skipping packet from other track {packet_track_id} while decoding track {track_id}" + ); + continue; + } + + // Decode the packet into audio samples. + match decoder.decode(&packet) { + Ok(input) => { + let output = input.into(); + return Some(Ok(output)); + } + Err(SymphoniaError::DecodeError(err)) => { + // Recoverable error, continue with the next packet. + log::warn!("Failed to decode packet #{packet_count}: {err}"); + } + Err(SymphoniaError::IoError(err)) => { + // Recoverable error, continue with the next packet. + log::warn!("I/O error while decoding packet #{packet_count}: {err}"); + } + Err(err) => { + // All other errors are considered fatal and decoding must be aborted. + return Some(Err(Box::new(err))); + } + }; + } + } +} + +impl From> for AudioBuffer { + fn from(input: GenericAudioBufferRef<'_>) -> Self { + let sample_rate = input.spec().rate() as f32; + + let mut data = Vec::new(); + input.copy_to_vecs_planar::(&mut data); + + let channels = data.into_iter().map(ChannelData::from).collect(); + AudioBuffer::from_channels(channels, sample_rate) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::io::Cursor; + + #[test] + fn test_media_decoder() { + let input = Cursor::new(vec![0; 32]); + let media = MediaDecoder::try_new(input); + + assert!(media.is_err()); // the input was not a valid MIME type + } + + #[test] + fn test_unsupported_audio_codec_error_includes_codec_id() { + let input = std::fs::File::open("samples/sample.webm").unwrap(); + let media = MediaDecoder::try_new(input); + + let err = match media { + Ok(_) => panic!("expected unsupported codec error"), + Err(err) => err.to_string(), + }; + assert_eq!(err, "unsupported audio codec: 0x1001"); + } +} diff --git a/vendor/web-audio-api-1.7.0/src/events.rs b/vendor/web-audio-api-1.7.0/src/events.rs new file mode 100644 index 0000000000..069e9f36af --- /dev/null +++ b/vendor/web-audio-api-1.7.0/src/events.rs @@ -0,0 +1,264 @@ +#[cfg(feature = "diagnostics")] +use crate::context::AudioContextDiagnostics; +use crate::context::ConcreteBaseAudioContext; +use crate::context::{AudioContextState, AudioNodeId}; +use crate::{AudioBuffer, AudioRenderCapacityEvent}; + +use std::any::Any; +use std::collections::HashMap; +use std::hash::Hash; +use std::ops::ControlFlow; +use std::sync::{Arc, Mutex}; + +use crossbeam_channel::Receiver; + +/// The Event interface +#[derive(Debug, Clone)] +#[non_exhaustive] +pub struct Event { + pub type_: &'static str, +} + +#[derive(Hash, Eq, PartialEq, Debug)] +pub(crate) enum EventType { + Ended(AudioNodeId), + SinkChange, + StateChange, + RenderCapacity, + ProcessorError(AudioNodeId), + #[cfg(feature = "diagnostics")] + Diagnostics, + Message(AudioNodeId), + Complete, + AudioProcessing(AudioNodeId), +} + +/// The Error Event interface +#[non_exhaustive] +#[derive(Debug)] +pub struct ErrorEvent { + /// The error message + pub message: String, + /// The object with which panic was originally invoked. + pub error: Box, + /// Inherits from this base Event + pub event: Event, +} + +/// The AudioProcessingEvent interface +#[non_exhaustive] +#[derive(Debug)] +pub struct AudioProcessingEvent { + /// The input buffer + pub input_buffer: AudioBuffer, + /// The output buffer + pub output_buffer: AudioBuffer, + /// The time when the audio will be played in the same time coordinate system as the + /// AudioContext's currentTime. + pub playback_time: f64, + pub(crate) registration: Option<(ConcreteBaseAudioContext, AudioNodeId)>, +} + +impl Drop for AudioProcessingEvent { + fn drop(&mut self) { + if let Some((context, id)) = self.registration.take() { + let wrapped = crate::message::ControlMessage::NodeMessage { + id, + msg: llq::Node::new(Box::new(self.output_buffer.clone())), + }; + context.send_control_msg(wrapped); + } + } +} + +/// The OfflineAudioCompletionEvent Event interface +#[non_exhaustive] +#[derive(Debug)] +pub struct OfflineAudioCompletionEvent { + /// The rendered AudioBuffer + pub rendered_buffer: AudioBuffer, + /// Inherits from this base Event + pub event: Event, +} + +#[derive(Debug)] +pub(crate) enum EventPayload { + None, + RenderCapacity(AudioRenderCapacityEvent), + ProcessorError(ErrorEvent), + #[cfg(feature = "diagnostics")] + Diagnostics(AudioContextDiagnostics), + Message(Box), + AudioContextState(AudioContextState), + Complete(AudioBuffer), + AudioProcessing(AudioProcessingEvent), +} + +#[derive(Debug)] +pub(crate) struct EventDispatch { + type_: EventType, + payload: EventPayload, +} + +impl EventDispatch { + pub fn ended(id: AudioNodeId) -> Self { + EventDispatch { + type_: EventType::Ended(id), + payload: EventPayload::None, + } + } + + pub fn sink_change() -> Self { + EventDispatch { + type_: EventType::SinkChange, + payload: EventPayload::None, + } + } + + pub fn state_change(state: AudioContextState) -> Self { + EventDispatch { + type_: EventType::StateChange, + payload: EventPayload::AudioContextState(state), + } + } + + pub fn render_capacity(value: AudioRenderCapacityEvent) -> Self { + EventDispatch { + type_: EventType::RenderCapacity, + payload: EventPayload::RenderCapacity(value), + } + } + + pub fn processor_error(id: AudioNodeId, value: ErrorEvent) -> Self { + EventDispatch { + type_: EventType::ProcessorError(id), + payload: EventPayload::ProcessorError(value), + } + } + + #[cfg(feature = "diagnostics")] + pub fn diagnostics(value: AudioContextDiagnostics) -> Self { + EventDispatch { + type_: EventType::Diagnostics, + payload: EventPayload::Diagnostics(value), + } + } + + pub fn message(id: AudioNodeId, value: Box) -> Self { + EventDispatch { + type_: EventType::Message(id), + payload: EventPayload::Message(value), + } + } + + pub fn complete(buffer: AudioBuffer) -> Self { + EventDispatch { + type_: EventType::Complete, + payload: EventPayload::Complete(buffer), + } + } + + pub fn audio_processing(id: AudioNodeId, value: AudioProcessingEvent) -> Self { + EventDispatch { + type_: EventType::AudioProcessing(id), + payload: EventPayload::AudioProcessing(value), + } + } +} + +pub(crate) enum EventHandler { + Once(Box), + Multiple(Box), +} + +#[derive(Clone)] +pub(crate) struct EventLoop { + event_recv: Receiver, + event_handlers: Arc>>, +} + +impl EventLoop { + pub fn new(event_recv: Receiver) -> Self { + Self { + event_recv, + event_handlers: Default::default(), + } + } + + fn handle_event(&self, mut event: EventDispatch) -> ControlFlow<()> { + // Terminate the event loop when the audio context is closing + let mut result = ControlFlow::Continue(()); + if matches!( + event.payload, + EventPayload::AudioContextState(AudioContextState::Closed) + ) { + event.payload = EventPayload::None; // the statechange handler takes no argument + result = ControlFlow::Break(()); + } + + // In unit tests, we rethrow panics to avoid missing critical node exceptions + // https://github.com/orottier/web-audio-api-rs/issues/522 + #[cfg(test)] + if let EventPayload::ProcessorError(e) = event.payload { + panic!("Rethrowing exception during tests: {:?}", e); + } + + let mut event_handler_lock = self.event_handlers.lock().unwrap(); + let callback_option = event_handler_lock.remove(&event.type_); + drop(event_handler_lock); // release Mutex while running callback + + if let Some(callback) = callback_option { + match callback { + EventHandler::Once(f) => (f)(event.payload), + EventHandler::Multiple(mut f) => { + (f)(event.payload); + self.event_handlers + .lock() + .unwrap() + .insert(event.type_, EventHandler::Multiple(f)); + } + }; + } + + result + } + + #[inline(always)] + pub fn handle_pending_events(&self) -> bool { + let mut events_were_handled = false; + // try_iter will yield all pending events, but does not block + for event in self.event_recv.try_iter() { + // we can ignore the return value, it is only useful in the event thread + let _ = self.handle_event(event); + events_were_handled = true; + } + events_were_handled + } + + pub fn run_in_thread(&self) { + log::debug!("Entering event thread"); + + // split borrows to help compiler + let self_clone = self.clone(); + + std::thread::spawn(move || { + // This thread is dedicated to event handling, so we can block + for event in self_clone.event_recv.iter() { + let result = self_clone.handle_event(event); + if result.is_break() { + break; + } + } + + log::debug!("Event loop has terminated"); + }); + } + + pub fn set_handler(&self, event: EventType, callback: EventHandler) { + self.event_handlers.lock().unwrap().insert(event, callback); + } + + pub fn clear_handler(&self, event: EventType) { + self.event_handlers.lock().unwrap().remove(&event); + } +} diff --git a/vendor/web-audio-api-1.7.0/src/io/cpal.rs b/vendor/web-audio-api-1.7.0/src/io/cpal.rs new file mode 100644 index 0000000000..c3089ba193 --- /dev/null +++ b/vendor/web-audio-api-1.7.0/src/io/cpal.rs @@ -0,0 +1,831 @@ +//! Audio IO management API +use std::sync::atomic::Ordering; +use std::sync::Arc; +use std::sync::Mutex; + +use cpal::{ + traits::{DeviceTrait, HostTrait, StreamTrait}, + Device, Error as CpalError, ErrorKind as CpalErrorKind, OutputCallbackInfo, SampleFormat, + Stream, StreamConfig, SupportedBufferSize, +}; +use crossbeam_channel::Receiver; + +use super::{ + AudioBackendError, AudioBackendErrorKind, AudioBackendManager, BackendResult, RenderThreadInit, +}; + +use crate::buffer::AudioBuffer; +use crate::context::AudioContextLatencyCategory; +use crate::context::AudioContextOptions; +use crate::io::microphone::MicrophoneRender; +use crate::media_devices::{MediaDeviceInfo, MediaDeviceInfoKind}; +use crate::render::RenderThread; +use crate::stats::AudioStats; +use crate::{AtomicF64, MAX_CHANNELS}; + +fn get_host() -> BackendResult { + #[cfg(all(feature = "cpal-pipewire", target_os = "linux"))] + { + if let Some(host) = preferred_host(cpal::HostId::PipeWire, "PipeWire") { + return Ok(host); + } + } + + #[cfg(feature = "cpal-jack")] + { + if let Some(host) = preferred_host(cpal::HostId::Jack, "JACK") { + return Ok(host); + } + } + + Ok(cpal::default_host()) +} + +#[cfg(any( + feature = "cpal-jack", + all(feature = "cpal-pipewire", target_os = "linux") +))] +fn preferred_host(host_id: cpal::HostId, host_name: &str) -> Option { + let host_id = cpal::available_hosts() + .into_iter() + .find(|id| *id == host_id)?; + + let host = match cpal::host_from_id(host_id) { + Ok(host) => host, + Err(err) => { + log::warn!("{host_name} host is unavailable, falling back to default host: {err}"); + return None; + } + }; + + match host.devices() { + Ok(devices) => { + if devices.count() > 0 { + Some(host) + } else { + log::warn!("No {host_name} devices found, falling back to default host"); + None + } + } + Err(err) => { + log::warn!( + "Could not enumerate {host_name} devices, falling back to default host: {err}" + ); + None + } + } +} + +fn map_cpal_backend_error( + kind: AudioBackendErrorKind, + operation: &'static str, + err: impl std::fmt::Display, +) -> AudioBackendError { + AudioBackendError::new(kind, "cpal", operation, err.to_string()) +} + +fn map_cpal_error(operation: &'static str, err: CpalError) -> AudioBackendError { + let kind = match err.kind() { + CpalErrorKind::DeviceNotAvailable => AudioBackendErrorKind::DeviceUnavailable, + CpalErrorKind::UnsupportedConfig | CpalErrorKind::UnsupportedOperation => { + AudioBackendErrorKind::NotSupported + } + CpalErrorKind::InvalidInput => AudioBackendErrorKind::InvalidArgument, + CpalErrorKind::DeviceBusy + | CpalErrorKind::DeviceChanged + | CpalErrorKind::HostUnavailable + | CpalErrorKind::PermissionDenied + | CpalErrorKind::RealtimeDenied + | CpalErrorKind::ResourceExhausted + | CpalErrorKind::StreamInvalidated + | CpalErrorKind::Xrun + | CpalErrorKind::BackendError + | CpalErrorKind::Other + | _ => AudioBackendErrorKind::BackendSpecific, + }; + map_cpal_backend_error(kind, operation, err) +} + +fn cpal_device_for_id( + host: &cpal::Host, + kind: MediaDeviceInfoKind, + device_id: &str, +) -> BackendResult> { + let devices: Vec = match kind { + MediaDeviceInfoKind::AudioInput => host + .input_devices() + .map_err(|e| map_cpal_error("enumerate_input_devices", e))? + .collect(), + MediaDeviceInfoKind::AudioOutput => host + .output_devices() + .map_err(|e| map_cpal_error("enumerate_output_devices", e))? + .collect(), + MediaDeviceInfoKind::VideoInput => Vec::new(), + }; + let mut seen = Vec::::new(); + + for device in devices { + let Some(num_channels) = cpal_device_channels(&device, kind) else { + continue; + }; + let stable_id = cpal_stable_device_id(&device, kind, num_channels, &seen)?; + if stable_id == device_id { + return Ok(Some(device)); + } + seen.push(stable_id); + } + + Ok(None) +} + +fn cpal_device_channels(device: &Device, kind: MediaDeviceInfoKind) -> Option { + Some(match kind { + MediaDeviceInfoKind::AudioInput => device.default_input_config().ok()?.channels(), + MediaDeviceInfoKind::AudioOutput => device.default_output_config().ok()?.channels(), + MediaDeviceInfoKind::VideoInput => return None, + }) +} + +fn cpal_stable_device_id( + device: &Device, + kind: MediaDeviceInfoKind, + num_channels: u16, + seen: &[String], +) -> BackendResult { + let name = device + .description() + .map_err(|e| map_cpal_error("device_name", e))? + .to_string(); + + Ok(stable_device_id("cpal", kind, name, num_channels, seen)) +} + +fn stable_device_id( + host: &str, + kind: MediaDeviceInfoKind, + friendly_name: String, + num_channels: u16, + seen: &[String], +) -> String { + let mut index = 0; + loop { + let device_id = crate::media_devices::DeviceId::as_string( + kind, + host.to_string(), + friendly_name.clone(), + num_channels, + index, + ); + + if !seen.iter().any(|id| id == &device_id) { + return device_id; + } + + index += 1; + } +} + +/// Audio backend using the `cpal` library +#[derive(Clone)] +#[allow(unused)] +pub(crate) struct CpalBackend { + stream: Arc>>, + output_latency: Arc, + sample_rate: f32, + number_of_channels: usize, + sink_id: String, +} + +impl AudioBackendManager for CpalBackend { + fn build_output( + options: AudioContextOptions, + render_thread_init: RenderThreadInit, + ) -> BackendResult + where + Self: Sized, + { + let host = get_host()?; + + log::info!("Audio Output Host: cpal {:?}", host.id()); + + let RenderThreadInit { + state, + startup_pending, + frames_played, + stats, + ctrl_msg_recv, + event_send, + } = render_thread_init; + + let device = if options.sink_id.is_empty() { + host.default_output_device().ok_or_else(|| { + AudioBackendError::new( + AudioBackendErrorKind::DeviceUnavailable, + "cpal", + "default_output_device", + "No output device available", + ) + })? + } else { + cpal_device_for_id(&host, MediaDeviceInfoKind::AudioOutput, &options.sink_id)? + .or_else(|| host.default_output_device()) + .ok_or_else(|| { + AudioBackendError::new( + AudioBackendErrorKind::DeviceUnavailable, + "cpal", + "select_output_device", + "No output device available", + ) + })? + }; + + log::info!( + "Output device: {:?}", + device + .description() + .map_err(|e| map_cpal_error("output_device_name", e))? + .to_string() + ); + + let default_device_config = device + .default_output_config() + .map_err(|e| map_cpal_error("default_output_config", e))?; + + // we grab the largest number of channels provided by the soundcard + // clamped to MAX_CHANNELS, this value cannot be changed by the user + let number_of_channels = usize::from(default_device_config.channels()).min(MAX_CHANNELS); + + // override default device configuration with the options provided by + // the user when creating the `AudioContext` + let mut preferred_config: StreamConfig = default_device_config.into(); + // make sure the number of channels is clamped to MAX_CHANNELS + preferred_config.channels = number_of_channels as u16; + + // set specific sample rate if requested + if let Some(sample_rate) = options.sample_rate { + crate::assert_valid_sample_rate(sample_rate); + preferred_config.sample_rate = sample_rate as u32; + } + + // always try to set a decent buffer size + let buffer_size = super::buffer_size_for_latency_category( + options.latency_hint, + preferred_config.sample_rate as f32, + ) as u32; + + let clamped_buffer_size: u32 = match default_device_config.buffer_size() { + SupportedBufferSize::Unknown => buffer_size, + SupportedBufferSize::Range { min, max } => buffer_size.clamp(*min, *max), + }; + + preferred_config.buffer_size = cpal::BufferSize::Fixed(clamped_buffer_size); + + // On android detected range for the buffer size seems to be too big, use default buffer size instead + // See https://github.com/orottier/web-audio-api-rs/issues/515 + if cfg!(target_os = "android") { + if let AudioContextLatencyCategory::Balanced + | AudioContextLatencyCategory::Interactive = options.latency_hint + { + preferred_config.buffer_size = cpal::BufferSize::Default; + } + } + + // report the picked sample rate to the render thread, i.e. if the requested + // sample rate is not supported by the hardware, it will fallback to the + // default device sample rate + let mut sample_rate = preferred_config.sample_rate as f32; + + // shared atomic to report output latency to the control thread + let output_latency = Arc::new(AtomicF64::new(0.)); + + let mut renderer = RenderThread::new( + sample_rate, + preferred_config.channels as usize, + ctrl_msg_recv.clone(), + Arc::clone(&state), + Arc::clone(&frames_played), + stats.clone(), + event_send.clone(), + ); + renderer.set_startup_pending(Arc::clone(&startup_pending)); + renderer.spawn_garbage_collector_thread(); + + log::debug!( + "Attempt output stream with preferred config: {:?}", + preferred_config + ); + + let spawned = spawn_output_stream( + &device, + default_device_config.sample_format(), + preferred_config, + renderer, + Arc::clone(&output_latency), + stats.clone(), + ); + + let stream = match spawned { + Ok(stream) => { + log::debug!("Output stream set up successfully"); + stream + } + Err(e) => { + log::warn!("Output stream build failed with preferred config: {}", e); + + let mut supported_config: StreamConfig = default_device_config.into(); + // make sure number of channels is clamped to MAX_CHANNELS + supported_config.channels = number_of_channels as u16; + // fallback to device default sample rate + sample_rate = supported_config.sample_rate as f32; + + log::debug!( + "Attempt output stream with fallback config: {:?}", + supported_config + ); + + let mut renderer = RenderThread::new( + sample_rate, + supported_config.channels as usize, + ctrl_msg_recv, + state, + frames_played, + stats.clone(), + event_send, + ); + renderer.set_startup_pending(startup_pending); + renderer.spawn_garbage_collector_thread(); + + let spawned = spawn_output_stream( + &device, + default_device_config.sample_format(), + supported_config, + renderer, + Arc::clone(&output_latency), + stats.clone(), + ); + + spawned.map_err(|e| map_cpal_error("build_fallback_output_stream", e))? + } + }; + + // Required because some hosts don't play the stream automatically + stream + .play() + .map_err(|e| map_cpal_error("play_output_stream", e))?; + + Ok(CpalBackend { + stream: Arc::new(Mutex::new(Some(stream))), + output_latency, + sample_rate, + number_of_channels, + sink_id: options.sink_id, + }) + } + + fn build_input( + options: AudioContextOptions, + number_of_channels: Option, + ) -> BackendResult<(Self, Receiver)> + where + Self: Sized, + { + let host = get_host()?; + + log::info!("Audio Input Host: cpal {:?}", host.id()); + + let device = if options.sink_id.is_empty() { + host.default_input_device().ok_or_else(|| { + AudioBackendError::new( + AudioBackendErrorKind::DeviceUnavailable, + "cpal", + "default_input_device", + "No input device available", + ) + })? + } else { + cpal_device_for_id(&host, MediaDeviceInfoKind::AudioInput, &options.sink_id)? + .or_else(|| host.default_input_device()) + .ok_or_else(|| { + AudioBackendError::new( + AudioBackendErrorKind::DeviceUnavailable, + "cpal", + "select_input_device", + "No input device available", + ) + })? + }; + + log::info!( + "Input device: {:?}", + device + .description() + .map_err(|e| map_cpal_error("input_device_name", e))? + .to_string() + ); + + let supported = device + .default_input_config() + .map_err(|e| map_cpal_error("default_input_config", e))?; + + // clone the config, we may need to fall back on it later + let mut preferred: StreamConfig = supported.into(); + + if let Some(number_of_channels) = number_of_channels { + // do not ask for more channels than available + preferred.channels = preferred.channels.min(number_of_channels as u16); + } + + // make sure we don't exceed MAX_CHANNELS + preferred.channels = preferred.channels.min(MAX_CHANNELS as u16); + + // set specific sample rate if requested + if let Some(sample_rate) = options.sample_rate { + crate::assert_valid_sample_rate(sample_rate); + preferred.sample_rate = sample_rate as u32; + } + + // always try to set a decent buffer size + let buffer_size = super::buffer_size_for_latency_category( + options.latency_hint, + preferred.sample_rate as f32, + ) as u32; + + let clamped_buffer_size: u32 = match supported.buffer_size() { + SupportedBufferSize::Unknown => buffer_size, + SupportedBufferSize::Range { min, max } => buffer_size.clamp(*min, *max), + }; + + preferred.buffer_size = cpal::BufferSize::Fixed(clamped_buffer_size); + let mut sample_rate = preferred.sample_rate as f32; + let mut number_of_channels = preferred.channels as usize; + + let smoothing = 3; // todo, use buffering to smooth frame drops + let (sender, mut receiver) = crossbeam_channel::bounded(smoothing); + let renderer = MicrophoneRender::new(number_of_channels, sample_rate, sender); + + log::debug!( + "Attempt input stream with preferred config: {:?}", + preferred + ); + + let spawned = spawn_input_stream(&device, supported.sample_format(), preferred, renderer); + + // the required block size preferred config may not be supported, in that + // case, fallback the supported config + let stream = match spawned { + Ok(stream) => { + log::debug!("Input stream set up successfully"); + stream + } + Err(e) => { + log::warn!("Input stream build failed with preferred config: {}", e); + + let supported_config: StreamConfig = supported.into(); + if usize::from(supported_config.channels) > MAX_CHANNELS { + return Err(AudioBackendError::new( + AudioBackendErrorKind::NotSupported, + "cpal", + "build_input_stream", + format!( + "Input device requires {} channels, exceeding the supported maximum of {MAX_CHANNELS}; building a stream with {MAX_CHANNELS} channels failed: {e}", + supported_config.channels + ), + )); + } + + // fallback to device default sample rate and channel count + number_of_channels = usize::from(supported_config.channels); + sample_rate = supported_config.sample_rate as f32; + + log::debug!( + "Attempt input stream with fallback config: {:?}", + supported_config + ); + + // setup a new comms channel + let (sender, receiver2) = crossbeam_channel::bounded(smoothing); + receiver = receiver2; // overwrite earlier + + let renderer = MicrophoneRender::new(number_of_channels, sample_rate, sender); + + let spawned = spawn_input_stream( + &device, + supported.sample_format(), + supported_config, + renderer, + ); + spawned.map_err(|e| map_cpal_error("build_fallback_input_stream", e))? + } + }; + + // Required because some hosts don't play the stream automatically + stream + .play() + .map_err(|e| map_cpal_error("play_input_stream", e))?; + + let backend = CpalBackend { + stream: Arc::new(Mutex::new(Some(stream))), + output_latency: Arc::new(AtomicF64::new(0.)), + sample_rate, + number_of_channels, + sink_id: options.sink_id, + }; + + Ok((backend, receiver)) + } + + fn resume(&self) -> BackendResult { + if let Some(s) = self.stream.lock().unwrap().as_ref() { + s.play() + .map(|_| true) + .map_err(|e| map_cpal_error("resume", e))?; + return Ok(true); + } + + Ok(false) + } + + fn suspend(&self) -> BackendResult { + if let Some(s) = self.stream.lock().unwrap().as_ref() { + s.pause() + .map(|_| true) + .map_err(|e| map_cpal_error("suspend", e))?; + return Ok(true); + } + + Ok(false) + } + + fn close(&self) -> BackendResult<()> { + self.stream.lock().unwrap().take(); // will Drop + Ok(()) + } + + fn sample_rate(&self) -> f32 { + self.sample_rate + } + + fn number_of_channels(&self) -> usize { + self.number_of_channels + } + + fn output_latency(&self) -> BackendResult { + Ok(self.output_latency.load(Ordering::Relaxed)) + } + + fn sink_id(&self) -> &str { + self.sink_id.as_str() + } + + fn enumerate_devices_sync() -> BackendResult> + where + Self: Sized, + { + let host = get_host()?; + + let input_devices = host + .input_devices() + .map_err(|e| map_cpal_error("enumerate_input_devices", e))? + .filter_map(|d| { + let num_channels = d.default_input_config().ok()?.channels(); + Some((d, MediaDeviceInfoKind::AudioInput, num_channels)) + }); + + let output_devices = host + .output_devices() + .map_err(|e| map_cpal_error("enumerate_output_devices", e))? + .filter_map(|d| { + let num_channels = d.default_output_config().ok()?.channels(); + Some((d, MediaDeviceInfoKind::AudioOutput, num_channels)) + }); + + // cf. https://github.com/orottier/web-audio-api-rs/issues/356 + let mut list = Vec::::new(); + + for (device, kind, num_channels) in input_devices.chain(output_devices) { + let mut index = 0; + + loop { + let device_id = crate::media_devices::DeviceId::as_string( + kind, + "cpal".to_string(), + device + .description() + .map_err(|e| map_cpal_error("device_name", e))? + .to_string(), + num_channels, + index, + ); + + if !list.iter().any(|d| d.device_id() == device_id) { + let device = MediaDeviceInfo::new( + device_id, + None, + kind, + device + .description() + .map_err(|e| map_cpal_error("device_name", e))? + .to_string(), + ); + + list.push(device); + break; + } else { + index += 1; + } + } + } + + Ok(list) + } +} + +fn latency_in_seconds(infos: &OutputCallbackInfo) -> f64 { + let timestamp = infos.timestamp(); + let delta = timestamp.playback.duration_since(timestamp.callback); + delta.as_secs() as f64 + delta.subsec_nanos() as f64 * 1e-9 +} + +/// Creates an output stream +/// +/// # Arguments: +/// +/// * `device` - the output audio device on which the stream is created +/// * `sample_format` - audio sample format of the stream +/// * `config` - stream configuration +/// * `render` - the render thread which process the audio data +fn spawn_output_stream( + device: &Device, + sample_format: SampleFormat, + config: StreamConfig, + mut render: RenderThread, + output_latency: Arc, + stats: AudioStats, +) -> Result { + let err_fn = |err| log::error!("an error occurred on the output audio stream: {}", err); + + match sample_format { + SampleFormat::F32 => device.build_output_stream( + config, + move |d: &mut [f32], i: &OutputCallbackInfo| { + render.render(d); + let latency = latency_in_seconds(i); + output_latency.store(latency, Ordering::Relaxed); + stats.record_latency_seconds(latency); + }, + err_fn, + None, + ), + SampleFormat::F64 => device.build_output_stream( + config, + move |d: &mut [f64], i: &OutputCallbackInfo| { + render.render(d); + let latency = latency_in_seconds(i); + output_latency.store(latency, Ordering::Relaxed); + stats.record_latency_seconds(latency); + }, + err_fn, + None, + ), + SampleFormat::U8 => device.build_output_stream( + config, + move |d: &mut [u8], i: &OutputCallbackInfo| { + render.render(d); + let latency = latency_in_seconds(i); + output_latency.store(latency, Ordering::Relaxed); + stats.record_latency_seconds(latency); + }, + err_fn, + None, + ), + SampleFormat::U16 => device.build_output_stream( + config, + move |d: &mut [u16], i: &OutputCallbackInfo| { + render.render(d); + let latency = latency_in_seconds(i); + output_latency.store(latency, Ordering::Relaxed); + stats.record_latency_seconds(latency); + }, + err_fn, + None, + ), + SampleFormat::U32 => device.build_output_stream( + config, + move |d: &mut [u32], i: &OutputCallbackInfo| { + render.render(d); + let latency = latency_in_seconds(i); + output_latency.store(latency, Ordering::Relaxed); + stats.record_latency_seconds(latency); + }, + err_fn, + None, + ), + SampleFormat::U64 => device.build_output_stream( + config, + move |d: &mut [u64], i: &OutputCallbackInfo| { + render.render(d); + let latency = latency_in_seconds(i); + output_latency.store(latency, Ordering::Relaxed); + stats.record_latency_seconds(latency); + }, + err_fn, + None, + ), + SampleFormat::I8 => device.build_output_stream( + config, + move |d: &mut [i8], i: &OutputCallbackInfo| { + render.render(d); + let latency = latency_in_seconds(i); + output_latency.store(latency, Ordering::Relaxed); + stats.record_latency_seconds(latency); + }, + err_fn, + None, + ), + SampleFormat::I16 => device.build_output_stream( + config, + move |d: &mut [i16], i: &OutputCallbackInfo| { + render.render(d); + let latency = latency_in_seconds(i); + output_latency.store(latency, Ordering::Relaxed); + stats.record_latency_seconds(latency); + }, + err_fn, + None, + ), + SampleFormat::I32 => device.build_output_stream( + config, + move |d: &mut [i32], i: &OutputCallbackInfo| { + render.render(d); + let latency = latency_in_seconds(i); + output_latency.store(latency, Ordering::Relaxed); + stats.record_latency_seconds(latency); + }, + err_fn, + None, + ), + SampleFormat::I64 => device.build_output_stream( + config, + move |d: &mut [i64], i: &OutputCallbackInfo| { + render.render(d); + let latency = latency_in_seconds(i); + output_latency.store(latency, Ordering::Relaxed); + stats.record_latency_seconds(latency); + }, + err_fn, + None, + ), + _ => Err(CpalErrorKind::UnsupportedConfig.into()), + } +} + +/// Creates an input stream +/// +/// # Arguments: +/// +/// * `device` - the input audio device on which the stream is created +/// * `sample_format` - audio sample format of the stream +/// * `config` - stream configuration +/// * `render` - the render thread which process the audio data +fn spawn_input_stream( + device: &Device, + sample_format: SampleFormat, + config: StreamConfig, + render: MicrophoneRender, +) -> Result { + let err_fn = |err| log::error!("an error occurred on the input audio stream: {}", err); + + match sample_format { + SampleFormat::F32 => { + device.build_input_stream(config, move |d: &[f32], _c| render.render(d), err_fn, None) + } + SampleFormat::F64 => { + device.build_input_stream(config, move |d: &[f64], _c| render.render(d), err_fn, None) + } + SampleFormat::U8 => { + device.build_input_stream(config, move |d: &[u8], _c| render.render(d), err_fn, None) + } + SampleFormat::U16 => { + device.build_input_stream(config, move |d: &[u16], _c| render.render(d), err_fn, None) + } + SampleFormat::U32 => { + device.build_input_stream(config, move |d: &[u32], _c| render.render(d), err_fn, None) + } + SampleFormat::U64 => { + device.build_input_stream(config, move |d: &[u64], _c| render.render(d), err_fn, None) + } + SampleFormat::I8 => { + device.build_input_stream(config, move |d: &[i8], _c| render.render(d), err_fn, None) + } + SampleFormat::I16 => { + device.build_input_stream(config, move |d: &[i16], _c| render.render(d), err_fn, None) + } + SampleFormat::I32 => { + device.build_input_stream(config, move |d: &[i32], _c| render.render(d), err_fn, None) + } + SampleFormat::I64 => { + device.build_input_stream(config, move |d: &[i64], _c| render.render(d), err_fn, None) + } + _ => Err(CpalErrorKind::UnsupportedConfig.into()), + } +} diff --git a/vendor/web-audio-api-1.7.0/src/io/cubeb.rs b/vendor/web-audio-api-1.7.0/src/io/cubeb.rs new file mode 100644 index 0000000000..fa473213a1 --- /dev/null +++ b/vendor/web-audio-api-1.7.0/src/io/cubeb.rs @@ -0,0 +1,701 @@ +use std::thread; + +use super::{ + AudioBackendError, AudioBackendErrorKind, AudioBackendManager, BackendResult, RenderThreadInit, +}; + +use crate::buffer::AudioBuffer; +use crate::context::AudioContextOptions; +use crate::io::microphone::MicrophoneRender; +use crate::media_devices::{MediaDeviceInfo, MediaDeviceInfoKind}; +use crate::render::RenderThread; +use crate::{MAX_CHANNELS, RENDER_QUANTUM_SIZE}; + +use cubeb::{Context, DeviceId, DeviceType, StereoFrame, Stream, StreamParams}; + +use crossbeam_channel::{Receiver, Sender}; + +// erase type of `Frame` in cubeb `Stream` +struct BoxedStream(Box); + +impl BoxedStream { + fn new(stream: Stream) -> Self { + Self(Box::new(stream)) + } +} + +trait CubebStream { + fn delegate_start(&self) -> Result<(), cubeb::Error>; + fn delegate_stop(&self) -> Result<(), cubeb::Error>; + fn delegate_latency(&self) -> Result; +} + +impl CubebStream for Stream { + fn delegate_start(&self) -> Result<(), cubeb::Error> { + self.start() + } + fn delegate_stop(&self) -> Result<(), cubeb::Error> { + self.stop() + } + fn delegate_latency(&self) -> Result { + self.latency() + } +} + +mod owner_thread { + use super::*; + + pub struct Responder(Sender>); + + impl Responder { + pub fn send(self, result: BackendResult) { + let _ = self.0.send(result); + } + } + + pub enum CubebCommand { + Resume(Responder), + Suspend(Responder), + Close(Responder<()>), + OutputLatency { + sample_rate: f32, + response: Responder, + }, + } + + pub struct CubebStreamInfo { + pub sample_rate: f32, + pub number_of_channels: usize, + } + + pub struct CubebOwner { + stream: BoxedStream, + _ctx: Context, + } + + impl CubebOwner { + pub fn new(ctx: Context, stream: BoxedStream) -> Self { + Self { stream, _ctx: ctx } + } + + pub fn run(self, commands: Receiver) { + while let Ok(command) = commands.recv() { + match command { + CubebCommand::Resume(response) => { + response.send(self.resume()); + } + CubebCommand::Suspend(response) => { + response.send(self.suspend()); + } + CubebCommand::Close(response) => { + let result = self.close(); + response.send(result); + break; + } + CubebCommand::OutputLatency { + sample_rate, + response, + } => { + response.send(self.output_latency(sample_rate)); + } + } + } + } + + pub fn resume(&self) -> BackendResult { + self.stream + .0 + .delegate_start() + .map(|_| true) + .map_err(|e| map_cubeb_error("resume", e)) + } + + fn suspend(&self) -> BackendResult { + self.stream + .0 + .delegate_stop() + .map(|_| true) + .map_err(|e| map_cubeb_error("suspend", e)) + } + + fn close(self) -> BackendResult<()> { + let _ = self.stream.0.delegate_stop(); + drop(self); + Ok(()) + } + + fn output_latency(&self, sample_rate: f32) -> BackendResult { + self.stream + .0 + .delegate_latency() + .map(|frames| frames as f64 / sample_rate as f64) + .map_err(|e| map_cubeb_error("output_latency", e)) + } + } + + #[derive(Clone)] + pub struct CubebThreadHandle { + commands: Sender, + } + + impl CubebThreadHandle { + pub fn new(commands: Sender) -> Self { + Self { commands } + } + + fn request( + &self, + operation: &'static str, + make_command: impl FnOnce(Responder) -> CubebCommand, + ) -> BackendResult { + let (response_send, response_recv) = crossbeam_channel::bounded(1); + self.commands + .send(make_command(Responder(response_send))) + .map_err(|_| cubeb_backend_error(operation, "Cubeb owner thread has stopped"))?; + response_recv + .recv() + .map_err(|_| cubeb_backend_error(operation, "Cubeb owner thread has stopped"))? + } + + pub fn close(&self) -> BackendResult<()> { + self.request("close", CubebCommand::Close) + } + + pub fn resume(&self) -> BackendResult { + self.request("resume", CubebCommand::Resume) + } + + pub fn suspend(&self) -> BackendResult { + self.request("suspend", CubebCommand::Suspend) + } + + pub fn output_latency(&self, sample_rate: f32) -> BackendResult { + self.request("output_latency", |response| CubebCommand::OutputLatency { + sample_rate, + response, + }) + } + } +} +use owner_thread::{CubebOwner, CubebStreamInfo, CubebThreadHandle}; + +fn spawn_cubeb_owner( + name: &'static str, + build: impl FnOnce() -> BackendResult<(CubebOwner, CubebStreamInfo)> + Send + 'static, +) -> BackendResult<(CubebThreadHandle, CubebStreamInfo)> { + let (commands_send, commands_recv) = crossbeam_channel::unbounded(); + let (ready_send, ready_recv) = crossbeam_channel::bounded(1); + thread::Builder::new() + .name(name.to_string()) + .spawn(move || match build() { + Ok((owner, info)) => { + let _ = ready_send.send(Ok(info)); + owner.run(commands_recv); + } + Err(e) => { + let _ = ready_send.send(Err(e)); + } + }) + .map_err(|e| cubeb_backend_error("spawn_thread", e.to_string()))?; + + let info = ready_recv + .recv() + .map_err(|_| cubeb_backend_error("spawn_thread", "Cubeb owner thread stopped"))??; + Ok((CubebThreadHandle::new(commands_send), info)) +} + +fn map_cubeb_error(operation: &'static str, err: cubeb::Error) -> AudioBackendError { + let kind = match err { + cubeb::Error::DeviceUnavailable => AudioBackendErrorKind::DeviceUnavailable, + cubeb::Error::InvalidFormat | cubeb::Error::NotSupported => { + AudioBackendErrorKind::NotSupported + } + cubeb::Error::InvalidParameter => AudioBackendErrorKind::InvalidArgument, + cubeb::Error::Error => AudioBackendErrorKind::BackendSpecific, + }; + + AudioBackendError::new(kind, "cubeb", operation, err.to_string()) +} + +fn cubeb_backend_error(operation: &'static str, message: impl Into) -> AudioBackendError { + AudioBackendError::new( + AudioBackendErrorKind::BackendSpecific, + "cubeb", + operation, + message, + ) +} + +fn cubeb_device_for_id( + context: &Context, + device_type: DeviceType, + kind: MediaDeviceInfoKind, + device_id: &str, +) -> BackendResult> { + let devices = context + .enumerate_devices(device_type) + .map_err(|e| map_cubeb_error(enumerate_operation(kind), e))?; + let mut seen = Vec::::new(); + + for device in devices.iter() { + let stable_id = cubeb_stable_device_id(device, kind, &seen)?; + if stable_id == device_id { + return Ok(Some(device.devid())); + } + seen.push(stable_id); + } + + Ok(None) +} + +fn init_output_backend( + ctx: &Context, + params: StreamParams, + buffer_size: u32, + device: Option, + mut renderer: RenderThread, +) -> BackendResult { + let mut builder = cubeb::StreamBuilder::<[f32; N]>::new(); + + match device { + None => builder.default_output(¶ms), + Some(devid) => builder.output(devid, ¶ms), + }; + + builder + .name("Cubeb web_audio_api") + .latency(buffer_size) + .data_callback(move |_input, output| { + // `output` is `&mut [[f32; N]]`, a slice of slices. + // The renderer just wants a single slice, flatten it. + // Inspired by the unstable feature + { + let output: &mut [f32] = + // SAFETY: `[T]` is layout-identical to `[T; N]` + unsafe { std::slice::from_raw_parts_mut(output.as_mut_ptr().cast(), output.len() * N) }; + renderer.render(output); + } + + output.len() as isize + }) + .state_callback(|state| { + log::debug!("stream state changed: {state:?}"); + }); + + let stream = builder + .init(ctx) + .map_err(|e| map_cubeb_error("init_output_stream", e))?; + Ok(BoxedStream::new(stream)) +} + +/// Audio backend using the `cubeb` library +#[derive(Clone)] +pub(crate) struct CubebBackend { + handle: CubebThreadHandle, + sample_rate: f32, + number_of_channels: usize, + sink_id: String, +} + +impl AudioBackendManager for CubebBackend { + fn build_output( + options: AudioContextOptions, + render_thread_init: RenderThreadInit, + ) -> BackendResult + where + Self: Sized, + { + let RenderThreadInit { + state, + startup_pending, + frames_played, + stats, + ctrl_msg_recv, + event_send, + } = render_thread_init; + + let sink_id = options.sink_id.clone(); + let (handle, info) = spawn_cubeb_owner("web-audio-api cubeb output", move || { + let ctx = Context::init(None, None).map_err(|e| map_cubeb_error("init_context", e))?; + log::info!("Audio Output Host: cubeb {:?}", ctx.backend_id()); + + // Use user requested sample rate, or else the device preferred one + let device_sample_rate = ctx.preferred_sample_rate().map(|v| v as f32).ok(); + let sample_rate = options.sample_rate.or(device_sample_rate).unwrap_or(48000.); + + let number_of_channels = ctx + .max_channel_count() + .map(|v| v as usize) + .ok() + .unwrap_or(2); + + // clamp the requested stream number of channels to MAX_CHANNELS even if + // the soundcard can provide more channels + let number_of_channels = number_of_channels.min(MAX_CHANNELS); + + let layout = match number_of_channels { + 1 => cubeb::ChannelLayout::MONO, + 2 => cubeb::ChannelLayout::STEREO, + 4 => cubeb::ChannelLayout::QUAD, + _ => cubeb::ChannelLayout::UNDEFINED, // TODO, does this work? + }; + + let mut renderer = RenderThread::new( + sample_rate, + number_of_channels, + ctrl_msg_recv, + state, + frames_played, + stats, + event_send, + ); + renderer.set_startup_pending(startup_pending); + renderer.spawn_garbage_collector_thread(); + + let params = cubeb::StreamParamsBuilder::new() + .format(cubeb::SampleFormat::Float32NE) // use float (native endian) + .rate(sample_rate as u32) + .channels(number_of_channels as u32) + .layout(layout) + .take(); + + // Calculate ideal latency + let buffer_size_req = + super::buffer_size_for_latency_category(options.latency_hint, sample_rate) as u32; + let min_latency = ctx + .min_latency(¶ms) + .ok() + .unwrap_or(RENDER_QUANTUM_SIZE as u32); + let buffer_size = buffer_size_req.max(min_latency); + + let device = if options.sink_id.is_empty() { + None + } else { + cubeb_device_for_id( + &ctx, + DeviceType::OUTPUT, + MediaDeviceInfoKind::AudioOutput, + &options.sink_id, + )? + }; + + let stream = match number_of_channels { + // so sorry, but I need to constify the non-const `number_of_channels` + 1 => init_output_backend::<1>(&ctx, params, buffer_size, device, renderer), + 2 => init_output_backend::<2>(&ctx, params, buffer_size, device, renderer), + 3 => init_output_backend::<3>(&ctx, params, buffer_size, device, renderer), + 4 => init_output_backend::<4>(&ctx, params, buffer_size, device, renderer), + 5 => init_output_backend::<5>(&ctx, params, buffer_size, device, renderer), + 6 => init_output_backend::<6>(&ctx, params, buffer_size, device, renderer), + 7 => init_output_backend::<7>(&ctx, params, buffer_size, device, renderer), + 8 => init_output_backend::<8>(&ctx, params, buffer_size, device, renderer), + 9 => init_output_backend::<9>(&ctx, params, buffer_size, device, renderer), + 10 => init_output_backend::<10>(&ctx, params, buffer_size, device, renderer), + 11 => init_output_backend::<11>(&ctx, params, buffer_size, device, renderer), + 12 => init_output_backend::<12>(&ctx, params, buffer_size, device, renderer), + 13 => init_output_backend::<13>(&ctx, params, buffer_size, device, renderer), + 14 => init_output_backend::<14>(&ctx, params, buffer_size, device, renderer), + 15 => init_output_backend::<15>(&ctx, params, buffer_size, device, renderer), + 16 => init_output_backend::<16>(&ctx, params, buffer_size, device, renderer), + 17 => init_output_backend::<17>(&ctx, params, buffer_size, device, renderer), + 18 => init_output_backend::<18>(&ctx, params, buffer_size, device, renderer), + 19 => init_output_backend::<19>(&ctx, params, buffer_size, device, renderer), + 20 => init_output_backend::<20>(&ctx, params, buffer_size, device, renderer), + 21 => init_output_backend::<21>(&ctx, params, buffer_size, device, renderer), + 22 => init_output_backend::<22>(&ctx, params, buffer_size, device, renderer), + 23 => init_output_backend::<23>(&ctx, params, buffer_size, device, renderer), + 24 => init_output_backend::<24>(&ctx, params, buffer_size, device, renderer), + 25 => init_output_backend::<25>(&ctx, params, buffer_size, device, renderer), + 26 => init_output_backend::<26>(&ctx, params, buffer_size, device, renderer), + 27 => init_output_backend::<27>(&ctx, params, buffer_size, device, renderer), + 28 => init_output_backend::<28>(&ctx, params, buffer_size, device, renderer), + 29 => init_output_backend::<29>(&ctx, params, buffer_size, device, renderer), + 30 => init_output_backend::<30>(&ctx, params, buffer_size, device, renderer), + 31 => init_output_backend::<31>(&ctx, params, buffer_size, device, renderer), + 32 => init_output_backend::<32>(&ctx, params, buffer_size, device, renderer), + _ => Err(cubeb_backend_error( + "init_output_stream", + "Unexpected channel count", + )), + }?; + + let owner = CubebOwner::new(ctx, stream); + owner.resume()?; + Ok(( + owner, + CubebStreamInfo { + sample_rate, + number_of_channels, + }, + )) + })?; + + let backend = CubebBackend { + handle, + number_of_channels: info.number_of_channels, + sample_rate: info.sample_rate, + sink_id, + }; + + Ok(backend) + } + + fn build_input( + options: AudioContextOptions, + _number_of_channels: Option, + ) -> BackendResult<(Self, Receiver)> + where + Self: Sized, + { + /* Set up a dedicated stream for input capturing + * + * This is not how it should be, we should link the input stream together + * with the output stream in one go. However, this means that we always + * capture mic input, even if the user does not want it - TODO + */ + + let smoothing = 3; // todo, use buffering to smooth frame drops + let (sender, receiver) = crossbeam_channel::bounded(smoothing); + let sink_id = options.sink_id.clone(); + let (handle, info) = spawn_cubeb_owner("web-audio-api cubeb input", move || { + let ctx = Context::init(None, None).map_err(|e| map_cubeb_error("init_context", e))?; + log::info!("Audio Input Host: cubeb {:?}", ctx.backend_id()); + + // Use user requested sample rate, or else the device preferred one + let device_sample_rate = ctx.preferred_sample_rate().map(|v| v as f32).ok(); + let sample_rate = options.sample_rate.or(device_sample_rate).unwrap_or(48000.); + + // TODO support all channel configs + let _max_channel_count = ctx.max_channel_count().map(|v| v as usize).ok(); + const NUMBER_OF_INPUT_CHANNELS: usize = 2; + let layout = cubeb::ChannelLayout::STEREO; + + let params = cubeb::StreamParamsBuilder::new() + .format(cubeb::SampleFormat::Float32NE) // use float (native endian) + .rate(sample_rate as u32) + .channels(NUMBER_OF_INPUT_CHANNELS as u32) + .layout(layout) + .take(); + + // Calculate ideal latency + let buffer_size_req = + super::buffer_size_for_latency_category(options.latency_hint, sample_rate) as u32; + let min_latency = ctx + .min_latency(¶ms) + .ok() + .unwrap_or(RENDER_QUANTUM_SIZE as u32); + let buffer_size = buffer_size_req.max(min_latency); + + let device = if options.sink_id.is_empty() { + None + } else { + cubeb_device_for_id( + &ctx, + DeviceType::INPUT, + MediaDeviceInfoKind::AudioInput, + &options.sink_id, + )? + }; + + let renderer = MicrophoneRender::new(NUMBER_OF_INPUT_CHANNELS, sample_rate, sender); + + // Microphone input is always assumed STEREO (TODO) + let mut builder = cubeb::StreamBuilder::>::new(); + + match device { + None => builder.default_input(¶ms), + Some(devid) => builder.input(devid, ¶ms), + }; + + builder + .name("Cubeb web_audio_api (input)") + .latency(buffer_size) + .data_callback(move |input, _output| { + let mut tmp = vec![0.; input.len() * NUMBER_OF_INPUT_CHANNELS]; + tmp.chunks_mut(NUMBER_OF_INPUT_CHANNELS) + .zip(input) + .for_each(|(t, i)| { + t[0] = i.l; + t[1] = i.r; + }); + renderer.render(&tmp); + input.len() as isize + }) + .state_callback(|state| { + log::debug!("stream state changed: {state:?}"); + }); + + let stream = builder + .init(&ctx) + .map_err(|e| map_cubeb_error("init_input_stream", e))?; + let owner = CubebOwner::new(ctx, BoxedStream::new(stream)); + owner.resume()?; + Ok(( + owner, + CubebStreamInfo { + sample_rate, + number_of_channels: NUMBER_OF_INPUT_CHANNELS, + }, + )) + })?; + + let backend = CubebBackend { + handle, + number_of_channels: info.number_of_channels, + sample_rate: info.sample_rate, + sink_id, + }; + + Ok((backend, receiver)) + } + + fn resume(&self) -> BackendResult { + self.handle.resume() + } + + fn suspend(&self) -> BackendResult { + self.handle.suspend() + } + + fn close(&self) -> BackendResult<()> { + self.handle.close() + } + + fn sample_rate(&self) -> f32 { + self.sample_rate + } + + fn number_of_channels(&self) -> usize { + self.number_of_channels + } + + fn output_latency(&self) -> BackendResult { + self.handle.output_latency(self.sample_rate) + } + + fn sink_id(&self) -> &str { + self.sink_id.as_str() + } + + fn enumerate_devices_sync() -> BackendResult> + where + Self: Sized, + { + let context = Context::init(None, None).map_err(|e| map_cubeb_error("init_context", e))?; + + let mut list = + enumerate_cubeb_devices(&context, DeviceType::INPUT, MediaDeviceInfoKind::AudioInput)?; + list.extend(enumerate_cubeb_devices( + &context, + DeviceType::OUTPUT, + MediaDeviceInfoKind::AudioOutput, + )?); + + Ok(list) + } +} + +fn enumerate_cubeb_devices( + context: &Context, + device_type: DeviceType, + kind: MediaDeviceInfoKind, +) -> BackendResult> { + let devices = context + .enumerate_devices(device_type) + .map_err(|e| map_cubeb_error(enumerate_operation(kind), e))?; + let mut list = Vec::::new(); + + for device in devices.iter() { + let friendly_name: String = device + .friendly_name() + .ok_or_else(|| { + cubeb_backend_error("device_friendly_name", "Device has no friendly name") + })? + .into(); + let max_channels = device.max_channels().try_into().map_err(|_| { + cubeb_backend_error("device_max_channels", "Device channel count overflows u16") + })?; + let mut index = 0; + + loop { + let device_id = crate::media_devices::DeviceId::as_string( + kind, + "cubeb".to_string(), + friendly_name.clone(), + max_channels, + index, + ); + + if !list.iter().any(|d| d.device_id() == device_id) { + let device = MediaDeviceInfo::new( + device_id, + device.group_id().map(str::to_string), + kind, + friendly_name, + ); + + list.push(device); + break; + } else { + index += 1; + } + } + } + + Ok(list) +} + +fn enumerate_operation(kind: MediaDeviceInfoKind) -> &'static str { + match kind { + MediaDeviceInfoKind::AudioInput => "enumerate_input_devices", + MediaDeviceInfoKind::AudioOutput => "enumerate_output_devices", + MediaDeviceInfoKind::VideoInput => "enumerate_video_devices", + } +} + +fn cubeb_stable_device_id( + device: &cubeb::DeviceInfo, + kind: MediaDeviceInfoKind, + seen: &[String], +) -> BackendResult { + let friendly_name = device + .friendly_name() + .ok_or_else(|| cubeb_backend_error("device_friendly_name", "Device has no friendly name"))? + .into(); + let max_channels = device.max_channels().try_into().map_err(|_| { + cubeb_backend_error("device_max_channels", "Device channel count overflows u16") + })?; + + Ok(stable_device_id( + "cubeb", + kind, + friendly_name, + max_channels, + seen, + )) +} + +fn stable_device_id( + host: &str, + kind: MediaDeviceInfoKind, + friendly_name: String, + max_channels: u16, + seen: &[String], +) -> String { + let mut index = 0; + loop { + let device_id = crate::media_devices::DeviceId::as_string( + kind, + host.to_string(), + friendly_name.clone(), + max_channels, + index, + ); + + if !seen.iter().any(|id| id == &device_id) { + return device_id; + } + + index += 1; + } +} diff --git a/vendor/web-audio-api-1.7.0/src/io/microphone.rs b/vendor/web-audio-api-1.7.0/src/io/microphone.rs new file mode 100644 index 0000000000..2935c42bc3 --- /dev/null +++ b/vendor/web-audio-api-1.7.0/src/io/microphone.rs @@ -0,0 +1,109 @@ +use std::error::Error; + +use crate::buffer::{AudioBuffer, AudioBufferOptions}; +use crate::io::AudioBackendManager; +use crate::RENDER_QUANTUM_SIZE; + +use crossbeam_channel::{Receiver, Sender, TryRecvError}; + +pub(crate) struct MicrophoneStream { + receiver: Receiver, + number_of_channels: usize, + sample_rate: f32, + stream: Box, +} + +impl MicrophoneStream { + pub(crate) fn new( + receiver: Receiver, + backend: Box, + ) -> Self { + Self { + receiver, + number_of_channels: backend.number_of_channels(), + sample_rate: backend.sample_rate(), + stream: backend, + } + } +} + +impl Drop for MicrophoneStream { + fn drop(&mut self) { + log::debug!("Microphone stream has been dropped"); + let _ = self.stream.close(); + } +} + +impl Iterator for MicrophoneStream { + type Item = Result>; + + fn next(&mut self) -> Option { + let next = match self.receiver.try_recv() { + Ok(buffer) => { + // new frame was ready + buffer + } + Err(TryRecvError::Empty) => { + // frame not received in time, emit silence + log::debug!("empty channel: input frame delayed"); + + let options = AudioBufferOptions { + number_of_channels: self.number_of_channels, + length: RENDER_QUANTUM_SIZE, + sample_rate: self.sample_rate, + }; + + AudioBuffer::new(options) + } + Err(TryRecvError::Disconnected) => { + // MicrophoneRender has stopped, close stream + return None; + } + }; + + Some(Ok(next)) + } +} + +pub(crate) struct MicrophoneRender { + number_of_channels: usize, + sample_rate: f32, + sender: Sender, +} + +impl MicrophoneRender { + pub fn new(number_of_channels: usize, sample_rate: f32, sender: Sender) -> Self { + Self { + number_of_channels, + sample_rate, + sender, + } + } + + pub fn render + Copy>(&self, data: &[S]) { + let mut channels = Vec::with_capacity(self.number_of_channels); + // copy rendered audio into output slice + for i in 0..self.number_of_channels { + channels.push( + data.iter() + .skip(i) + .step_by(self.number_of_channels) + .map(|v| v.to_sample_()) + .collect(), + ); + } + + let buffer = AudioBuffer::from(channels, self.sample_rate); + let result = self.sender.try_send(buffer); // can fail (frame dropped) + + if result.is_err() { + log::debug!("input frame dropped"); + } + } +} + +impl Drop for MicrophoneRender { + fn drop(&mut self) { + log::debug!("Microphone input has been dropped"); + } +} diff --git a/vendor/web-audio-api-1.7.0/src/io/mod.rs b/vendor/web-audio-api-1.7.0/src/io/mod.rs new file mode 100644 index 0000000000..a132c48798 --- /dev/null +++ b/vendor/web-audio-api-1.7.0/src/io/mod.rs @@ -0,0 +1,301 @@ +//! Audio input/output interfaces + +use std::error::Error; +use std::fmt; +use std::sync::atomic::{AtomicBool, AtomicU64, AtomicU8}; +use std::sync::Arc; + +use crossbeam_channel::{Receiver, Sender}; + +use crate::buffer::AudioBuffer; +use crate::context::{AudioContextLatencyCategory, AudioContextOptions, AudioContextState}; +use crate::events::EventDispatch; +use crate::media_devices::MediaDeviceInfo; +use crate::media_streams::{MediaStream, MediaStreamTrack}; +use crate::message::ControlMessage; +use crate::stats::AudioStats; +use crate::RENDER_QUANTUM_SIZE; + +mod none; +pub(crate) use none::NoneBackend; + +#[cfg(feature = "cpal")] +mod cpal; + +#[cfg(feature = "cubeb")] +mod cubeb; + +#[cfg(any(feature = "cubeb", feature = "cpal"))] +mod microphone; + +#[allow(dead_code)] +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) enum AudioBackendErrorKind { + DeviceUnavailable, + NotSupported, + InvalidArgument, + BackendSpecific, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub(crate) struct AudioBackendError { + pub(crate) kind: AudioBackendErrorKind, + pub(crate) backend: &'static str, + pub(crate) operation: &'static str, + pub(crate) message: String, +} + +pub(crate) type BackendResult = Result; + +impl AudioBackendError { + pub(crate) fn new( + kind: AudioBackendErrorKind, + backend: &'static str, + operation: &'static str, + message: impl Into, + ) -> Self { + Self { + kind, + backend, + operation, + message: message.into(), + } + } + + #[cfg(all(not(feature = "cubeb"), not(feature = "cpal")))] + pub(crate) fn no_backend(operation: &'static str) -> Self { + Self::new( + AudioBackendErrorKind::NotSupported, + "none", + operation, + "No audio backend available, enable the 'cpal' or 'cubeb' feature", + ) + } +} + +impl fmt::Display for AudioBackendError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!( + f, + "{} backend error during {}: {:?}: {}", + self.backend, self.operation, self.kind, self.message + ) + } +} + +impl Error for AudioBackendError {} + +#[derive(Debug)] +pub(crate) struct ControlThreadInit { + pub state: Arc, + pub frames_played: Arc, + pub stats: AudioStats, + pub ctrl_msg_send: Sender, + pub event_send: Sender, + pub event_recv: Receiver, +} + +#[derive(Clone, Debug)] +pub(crate) struct RenderThreadInit { + pub state: Arc, + pub startup_pending: Arc, + pub frames_played: Arc, + pub stats: AudioStats, + pub ctrl_msg_recv: Receiver, + pub event_send: Sender, +} + +pub(crate) fn thread_init() -> (ControlThreadInit, RenderThreadInit) { + // Track audio context state - synced from render thread to control thread + let state = Arc::new(AtomicU8::new(AudioContextState::Suspended as u8)); + let startup_pending = Arc::new(AtomicBool::new(true)); + + // Track number of frames - synced from render thread to control thread + let frames_played = Arc::new(AtomicU64::new(0)); + + let stats = AudioStats::new(); + + // Communication channel for ctrl msgs from the control thread to the render thread. + // Use a bounded channel for real-time safety. A maximum of 256 control messages (add/remove + // node, settings, ..) will be handled per render quantum. The control thread will block when + // the capacity is reached. + let (ctrl_msg_send, ctrl_msg_recv) = crossbeam_channel::bounded(256); + + // Communication channel for events from the render thread to the control thread. + // Use a bounded channel for real-time safety. A maximum of 256 events (node ended, error, ..) + // will be sent per render quantum. Excess events are dropped when the capacity is reached. + let (event_send, event_recv) = crossbeam_channel::bounded(256); + + let control_thread_init = ControlThreadInit { + state: Arc::clone(&state), + frames_played: Arc::clone(&frames_played), + stats: stats.clone(), + ctrl_msg_send, + event_send: event_send.clone(), + event_recv, + }; + + let render_thread_init = RenderThreadInit { + state, + startup_pending, + frames_played, + stats, + ctrl_msg_recv, + event_send, + }; + + (control_thread_init, render_thread_init) +} + +/// Set up an output stream (speakers) bases on the selected features (cubeb/cpal/none) +pub(crate) fn build_output( + options: AudioContextOptions, + render_thread_init: RenderThreadInit, +) -> BackendResult> { + if options.sink_id == "none" { + let backend = NoneBackend::build_output(options, render_thread_init)?; + return Ok(Box::new(backend)); + } + + #[cfg(feature = "cubeb")] + { + let backend = cubeb::CubebBackend::build_output(options, render_thread_init)?; + Ok(Box::new(backend)) + } + #[cfg(all(not(feature = "cubeb"), feature = "cpal"))] + { + let backend = cpal::CpalBackend::build_output(options, render_thread_init)?; + Ok(Box::new(backend)) + } + #[cfg(all(not(feature = "cubeb"), not(feature = "cpal")))] + { + Err(AudioBackendError::no_backend("build_output")) + } +} + +/// Set up an input stream (microphone) bases on the selected features (cubeb/cpal/none) +pub(crate) fn build_input( + options: AudioContextOptions, + number_of_channels: Option, +) -> BackendResult { + #[cfg(all(not(feature = "cubeb"), not(feature = "cpal")))] + { + Err(AudioBackendError::no_backend("build_input")) + } + + #[cfg(any(feature = "cubeb", feature = "cpal"))] + { + let (backend, receiver) = { + #[cfg(feature = "cubeb")] + { + cubeb::CubebBackend::build_input(options, number_of_channels)? + } + + #[cfg(all(not(feature = "cubeb"), feature = "cpal"))] + { + cpal::CpalBackend::build_input(options, number_of_channels)? + } + }; + + let media_iter = microphone::MicrophoneStream::new(receiver, Box::new(backend)); + let track = MediaStreamTrack::from_iter(media_iter); + Ok(MediaStream::from_tracks(vec![track])) + } +} + +/// Interface for audio backends +pub(crate) trait AudioBackendManager: Send + Sync + 'static { + /// Name of the concrete implementation - for debug purposes + #[cfg_attr(not(feature = "diagnostics"), allow(dead_code))] + fn name(&self) -> &'static str { + std::any::type_name::() + } + + /// Setup a new output stream (speakers) + fn build_output( + options: AudioContextOptions, + render_thread_init: RenderThreadInit, + ) -> BackendResult + where + Self: Sized; + + /// Setup a new input stream (microphone capture) + fn build_input( + options: AudioContextOptions, + number_of_channels: Option, + ) -> BackendResult<(Self, Receiver)> + where + Self: Sized; + + /// Resume or start the stream + fn resume(&self) -> BackendResult; + + /// Suspend the stream + fn suspend(&self) -> BackendResult; + + /// Close the stream, freeing all resources. It cannot be started again after closing. + fn close(&self) -> BackendResult<()>; + + /// Sample rate of the stream + fn sample_rate(&self) -> f32; + + /// Number of channels of the stream + fn number_of_channels(&self) -> usize; + + /// Output latency of the stream in seconds + /// + /// This is the difference between the time the backend acquires the data in the callback and + /// the listener can hear the sound. + fn output_latency(&self) -> BackendResult; + + /// The audio output device - `""` means the default device + fn sink_id(&self) -> &str; + + fn enumerate_devices_sync() -> BackendResult> + where + Self: Sized; +} + +/// Calculate buffer size in frames for a given latency category +fn buffer_size_for_latency_category( + latency_cat: AudioContextLatencyCategory, + sample_rate: f32, +) -> usize { + // at 44100Hz sample rate (this could be even more relaxed): + // Interactive: 128 samples is 2,9ms + // Balanced: 512 samples is 11,6ms + // Playback: 1024 samples is 23,2ms + match latency_cat { + AudioContextLatencyCategory::Interactive => RENDER_QUANTUM_SIZE, + AudioContextLatencyCategory::Balanced => RENDER_QUANTUM_SIZE * 4, + AudioContextLatencyCategory::Playback => RENDER_QUANTUM_SIZE * 8, + // buffer_size is always positive and truncation is the desired behavior + #[allow(clippy::cast_sign_loss)] + #[allow(clippy::cast_possible_truncation)] + AudioContextLatencyCategory::Custom(latency) => { + assert!( + latency > 0., + "RangeError - Invalid custom latency: {:?}, should be strictly positive", + latency + ); + + let buffer_size = (latency * sample_rate as f64) as usize; + buffer_size.next_power_of_two() + } + } +} + +pub(crate) fn enumerate_devices_sync() -> BackendResult> { + #[cfg(feature = "cubeb")] + { + cubeb::CubebBackend::enumerate_devices_sync() + } + + #[cfg(all(not(feature = "cubeb"), feature = "cpal"))] + { + cpal::CpalBackend::enumerate_devices_sync() + } + + #[cfg(all(not(feature = "cubeb"), not(feature = "cpal")))] + Err(AudioBackendError::no_backend("enumerate_devices_sync")) +} diff --git a/vendor/web-audio-api-1.7.0/src/io/none.rs b/vendor/web-audio-api-1.7.0/src/io/none.rs new file mode 100644 index 0000000000..2cda2e8cb6 --- /dev/null +++ b/vendor/web-audio-api-1.7.0/src/io/none.rs @@ -0,0 +1,217 @@ +use std::thread; +use std::time::{Duration, Instant}; + +use super::{ + AudioBackendError, AudioBackendErrorKind, AudioBackendManager, BackendResult, RenderThreadInit, +}; + +use crate::buffer::AudioBuffer; +use crate::context::AudioContextOptions; +use crate::media_devices::MediaDeviceInfo; +use crate::render::RenderThread; +use crate::{MAX_CHANNELS, RENDER_QUANTUM_SIZE}; + +use crossbeam_channel::{Receiver, Sender}; + +enum NoneBackendMessage { + Resume, + Suspend, + Close, +} + +#[derive(Clone)] +pub(crate) struct NoneBackend { + sender: Sender, + sample_rate: f32, +} + +impl NoneBackend { + /// Creates a mock backend to be used as tombstones + pub(crate) fn void() -> Self { + Self { + sample_rate: 0., + sender: crossbeam_channel::bounded(0).0, + } + } +} + +struct Callback { + receiver: Receiver, + render_thread: RenderThread, + sample_rate: f32, + running: bool, +} + +impl Callback { + fn run(mut self) { + let buffer_size = RENDER_QUANTUM_SIZE; // TODO Latency Category + let mut buffer = vec![0.; buffer_size * MAX_CHANNELS]; + let interval = Duration::from_secs_f32(buffer_size as f32 / self.sample_rate); + + // For an isochronous callback we must calculate the deadline every render quantum + let mut deadline = Instant::now().checked_add(interval).unwrap(); + + loop { + // poll the receiver as long as the deadline is in the future + while let Ok(msg) = self.receiver.recv_deadline(deadline) { + match msg { + NoneBackendMessage::Close => return, + NoneBackendMessage::Resume => { + self.running = true; + deadline = Instant::now().checked_add(interval).unwrap(); + break; // start processing right away + } + NoneBackendMessage::Suspend => self.running = false, + } + } + + if self.running { + self.render_thread.render(&mut buffer[..]); + } + + deadline = deadline.checked_add(interval).unwrap(); + } + } +} + +impl AudioBackendManager for NoneBackend { + /// Setup a new output stream (speakers) + fn build_output( + options: AudioContextOptions, + render_thread_init: RenderThreadInit, + ) -> BackendResult + where + Self: Sized, + { + let sample_rate = options.sample_rate.unwrap_or(48000.); + + let RenderThreadInit { + state, + startup_pending, + frames_played, + stats, + ctrl_msg_recv, + event_send, + } = render_thread_init; + + let mut render_thread = RenderThread::new( + sample_rate, + MAX_CHANNELS, + ctrl_msg_recv, + state, + frames_played, + stats, + event_send, + ); + render_thread.set_startup_pending(startup_pending); + render_thread.spawn_garbage_collector_thread(); + + // Use a bounded channel for real-time safety. A maximum of 32 control messages (resume, + // suspend, ..) will be handled per render quantum. The control thread will block when the + // capacity is reached. + let (sender, receiver) = crossbeam_channel::bounded(32); + + // todo: pass buffer size and sample rate + let callback = Callback { + render_thread, + receiver, + sample_rate, + running: true, + }; + + thread::spawn(move || callback.run()); + + Ok(Self { + sender, + sample_rate, + }) + } + + /// Setup a new input stream (microphone capture) + fn build_input( + _options: AudioContextOptions, + _number_of_channels: Option, + ) -> BackendResult<(Self, Receiver)> + where + Self: Sized, + { + Err(AudioBackendError::new( + AudioBackendErrorKind::NotSupported, + "none", + "build_input", + "The none backend does not support audio input", + )) + } + + /// Resume or start the stream + fn resume(&self) -> BackendResult { + self.sender + .send(NoneBackendMessage::Resume) + .map(|_| true) + .map_err(|_| { + AudioBackendError::new( + AudioBackendErrorKind::BackendSpecific, + "none", + "resume", + "The render thread is no longer available", + ) + }) + } + + /// Suspend the stream + fn suspend(&self) -> BackendResult { + self.sender + .send(NoneBackendMessage::Suspend) + .map(|_| true) + .map_err(|_| { + AudioBackendError::new( + AudioBackendErrorKind::BackendSpecific, + "none", + "suspend", + "The render thread is no longer available", + ) + }) + } + + /// Close the stream, freeing all resources. It cannot be started again after closing. + fn close(&self) -> BackendResult<()> { + self.sender.send(NoneBackendMessage::Close).map_err(|_| { + AudioBackendError::new( + AudioBackendErrorKind::BackendSpecific, + "none", + "close", + "The render thread is no longer available", + ) + }) + } + + /// Sample rate of the stream + fn sample_rate(&self) -> f32 { + self.sample_rate + } + + /// Number of channels of the stream + fn number_of_channels(&self) -> usize { + MAX_CHANNELS + } + + /// Output latency of the stream in seconds + /// + /// This is the difference between the time the backend acquires the data in the callback and + /// the listener can hear the sound. + fn output_latency(&self) -> BackendResult { + Ok(0.) + } + + /// The audio output device + fn sink_id(&self) -> &str { + "none" + } + + fn enumerate_devices_sync() -> BackendResult> + where + Self: Sized, + { + Ok(vec![]) + } +} diff --git a/vendor/web-audio-api-1.7.0/src/lib.rs b/vendor/web-audio-api-1.7.0/src/lib.rs new file mode 100644 index 0000000000..70d9a77fc8 --- /dev/null +++ b/vendor/web-audio-api-1.7.0/src/lib.rs @@ -0,0 +1,328 @@ +#![doc = include_str!("../README.md")] +#![warn(rust_2018_idioms)] +#![warn(rust_2021_compatibility)] +#![warn(clippy::missing_panics_doc)] +#![warn(clippy::clone_on_ref_ptr)] +#![warn(clippy::todo)] +#![warn(clippy::dbg_macro)] +#![warn(clippy::unimplemented)] +#![deny(trivial_numeric_casts)] +#![deny(missing_debug_implementations)] +#![deny(unsafe_op_in_unsafe_fn)] + +use std::error::Error; +use std::sync::atomic::{AtomicU32, AtomicU64, Ordering}; + +/// Render quantum size, the audio graph is rendered in blocks of RENDER_QUANTUM_SIZE samples +/// see. +pub(crate) const RENDER_QUANTUM_SIZE: usize = 128; + +/// Maximum number of channels for audio processing +pub const MAX_CHANNELS: usize = 32; + +mod buffer; +pub use buffer::*; + +mod capacity; +pub use capacity::*; + +mod playback_stats; +pub use playback_stats::*; + +pub mod context; + +pub mod media_devices; +pub mod media_recorder; +pub mod media_streams; + +pub mod node; + +mod events; +pub use events::*; + +mod message_port; +pub use message_port::MessagePort; + +mod param; +pub use param::*; + +mod periodic_wave; +pub use periodic_wave::*; + +mod render; + +mod stats; + +mod spatial; +pub use spatial::AudioListener; + +mod io; + +mod analysis; +mod message; + +mod decoding; + +mod media_element; +pub use media_element::MediaElement; + +mod resampling; +pub mod worklet; + +#[repr(transparent)] +pub(crate) struct AtomicF32 { + bits: AtomicU32, +} + +impl std::fmt::Debug for AtomicF32 { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_fmt(format_args!("{}", self.load(Ordering::Relaxed))) + } +} + +impl AtomicF32 { + #[must_use] + pub fn new(value: f32) -> Self { + Self { + bits: AtomicU32::new(value.to_bits()), + } + } + + #[must_use] + pub fn load(&self, ordering: Ordering) -> f32 { + f32::from_bits(self.bits.load(ordering)) + } + + pub fn store(&self, value: f32, ordering: Ordering) { + self.bits.store(value.to_bits(), ordering); + } +} + +/// Atomic float 64, only `load` and `store` are supported, no arithmetic +#[repr(transparent)] +pub(crate) struct AtomicF64 { + bits: AtomicU64, +} + +impl std::fmt::Debug for AtomicF64 { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_fmt(format_args!("{}", self.load(Ordering::Relaxed))) + } +} + +impl AtomicF64 { + #[must_use] + pub fn new(value: f64) -> Self { + Self { + bits: AtomicU64::new(value.to_bits()), + } + } + + #[must_use] + pub fn load(&self, ordering: Ordering) -> f64 { + f64::from_bits(self.bits.load(ordering)) + } + + pub fn store(&self, value: f64, ordering: Ordering) { + self.bits.store(value.to_bits(), ordering); + } +} + +/// Assert that the given sample rate is valid. +/// +/// Note that in practice sample rates should stand between 8000Hz (lower bound for +/// voice based applications, e.g. see phone bandwidth) and 96000Hz (for very high +/// quality audio applications and spectrum manipulation). +/// Most common sample rates for musical applications are 44100 and 48000. +/// +/// The Web Audio API specification requires implementations to support every +/// sample rate in the inclusive range `[3000, 768000]`. +/// +/// - see +/// +/// # Panics +/// +/// This function will panic if the given sample rate is outside the inclusive +/// range `[3000, 768000]`. +/// +/// The minimum sample rate supported by the Web Audio API specification, in Hz. +pub(crate) const MIN_SAMPLE_RATE: f32 = 3_000.; + +/// The maximum sample rate supported by the Web Audio API specification, in Hz. +pub(crate) const MAX_SAMPLE_RATE: f32 = 768_000.; + +/// Returns whether the given sample rate falls in the inclusive range +/// `[MIN_SAMPLE_RATE, MAX_SAMPLE_RATE]` defined by the Web Audio API specification. +/// +/// - see +#[inline(always)] +pub(crate) fn is_valid_sample_rate(sample_rate: f32) -> bool { + (MIN_SAMPLE_RATE..=MAX_SAMPLE_RATE).contains(&sample_rate) +} + +#[track_caller] +#[inline(always)] +pub(crate) fn assert_valid_sample_rate(sample_rate: f32) { + assert!( + is_valid_sample_rate(sample_rate), + "NotSupportedError - Invalid sample rate: {:?}, should be in the range [{:?}, {:?}]", + sample_rate, + MIN_SAMPLE_RATE, + MAX_SAMPLE_RATE, + ); +} + +/// Assert that the given number of channels is valid. +/// +/// # Panics +/// +/// This function will panic if: +/// - the given number of channels is outside the [1, 32] range, +/// 32 being defined by the MAX_CHANNELS constant. +/// +#[track_caller] +#[inline(always)] +pub(crate) fn assert_valid_number_of_channels(number_of_channels: usize) { + assert!( + number_of_channels > 0 && number_of_channels <= MAX_CHANNELS, + "NotSupportedError - Invalid number of channels: {:?} is outside range [1, {:?}]", + number_of_channels, + MAX_CHANNELS + ); +} + +/// Assert that the given channel number is valid according to the number of channels +/// of an Audio asset (e.g. [`AudioBuffer`]). +/// +/// # Panics +/// +/// This function will panic if: +/// - the given channel number is greater than or equal to the given number of channels. +/// +#[track_caller] +#[inline(always)] +pub(crate) fn assert_valid_channel_number(channel_number: usize, number_of_channels: usize) { + assert!( + channel_number < number_of_channels, + "IndexSizeError - Invalid channel number {:?} (number of channels: {:?})", + channel_number, + number_of_channels + ); +} + +/// Assert that the given value number is a valid buffer length, i.e. greater than zero +/// +/// # Panics +/// +/// This function will panic if: +/// - the given value is not lower than or equal to zero +/// +#[track_caller] +#[inline(always)] +pub(crate) fn assert_valid_buffer_length(length: usize) { + assert!( + length > 0, + "NotSupportedError - Invalid length: {:?} is less than or equal to minimum bound (0)", + length, + ); +} + +/// Assert that the given value number is a valid time information, i.e. greater +/// than or equal to zero and finite. +/// +/// # Panics +/// +/// This function will panic if: +/// - the given value is not finite and lower than zero +/// +#[track_caller] +#[inline(always)] +pub(crate) fn assert_valid_time_value(value: f64) { + assert!( + value.is_finite(), + "TypeError - The provided time value is non-finite.", + ); + + assert!( + value >= 0., + "RangeError - The provided time value ({:?}) cannot be negative", + value + ); +} + +pub(crate) trait AudioBufferIter: Iterator + Send + 'static {} + +impl + Send + 'static> AudioBufferIter for M {} + +type FallibleBuffer = Result>; + +#[cfg(test)] +mod tests { + use float_eq::assert_float_eq; + + use super::*; + + #[test] + fn test_atomic_f64() { + let f = AtomicF64::new(2.0); + assert_float_eq!(f.load(Ordering::SeqCst), 2.0, abs <= 0.); + + f.store(3.0, Ordering::SeqCst); + assert_float_eq!(f.load(Ordering::SeqCst), 3.0, abs <= 0.); + } + + #[test] + fn test_valid_sample_rate() { + assert_valid_sample_rate(48000.); + assert_valid_sample_rate(3_000.); + assert_valid_sample_rate(768_000.); + } + + #[test] + #[should_panic] + fn test_invalid_sample_rate_too_small() { + assert_valid_sample_rate(2_999.); + } + + #[test] + #[should_panic] + fn test_invalid_sample_rate_too_big() { + assert_valid_sample_rate(768_001.); + } + + #[test] + #[should_panic] + fn test_invalid_number_of_channels_min() { + assert_valid_number_of_channels(0); + } + + #[test] + #[should_panic] + fn test_invalid_number_of_channels_max() { + assert_valid_number_of_channels(33); + } + + #[test] + fn test_valid_number_of_channels() { + assert_valid_number_of_channels(1); + assert_valid_number_of_channels(32); + } + + #[test] + #[should_panic] + fn test_invalid_time_value_non_finite() { + assert_valid_time_value(f64::NAN); + } + + #[test] + #[should_panic] + fn test_invalid_time_value_negative() { + assert_valid_time_value(-1.); + } + + #[test] + fn test_valid_time_value() { + assert_valid_time_value(0.); + assert_valid_time_value(1.); + } +} diff --git a/vendor/web-audio-api-1.7.0/src/media_devices/mod.rs b/vendor/web-audio-api-1.7.0/src/media_devices/mod.rs new file mode 100644 index 0000000000..c369c6f589 --- /dev/null +++ b/vendor/web-audio-api-1.7.0/src/media_devices/mod.rs @@ -0,0 +1,250 @@ +//! Primitives of the MediaDevices API +//! +//! The MediaDevices interface provides access to connected media input devices like microphones. +//! +//! + +use std::collections::hash_map::DefaultHasher; +use std::hash::{Hash, Hasher}; + +use crate::context::{AudioContextLatencyCategory, AudioContextOptions}; +use crate::io::BackendResult; +use crate::media_streams::MediaStream; + +/// List the available media output devices, such as speakers, headsets, loopbacks, etc +/// +/// The media device_id can be used to specify the [`sink_id` of the `AudioContext`](crate::context::AudioContextOptions::sink_id) +/// +/// ```no_run +/// use web_audio_api::media_devices::{enumerate_devices_sync, MediaDeviceInfoKind}; +/// +/// let devices = enumerate_devices_sync(); +/// assert_eq!(devices[0].device_id(), "1"); +/// assert_eq!(devices[0].group_id(), None); +/// assert_eq!(devices[0].kind(), MediaDeviceInfoKind::AudioOutput); +/// assert_eq!(devices[0].label(), "Macbook Pro Builtin Speakers"); +/// ``` +pub fn enumerate_devices_sync() -> Vec { + try_enumerate_devices_sync().unwrap_or_else(|e| { + log::error!("Unable to enumerate media devices: {e}"); + vec![] + }) +} + +pub(crate) fn try_enumerate_devices_sync() -> BackendResult> { + crate::io::enumerate_devices_sync() +} + +// Internal struct to derive a stable id for a given input / output device +// cf. https://github.com/orottier/web-audio-api-rs/issues/356 +#[derive(Hash)] +pub(crate) struct DeviceId { + kind: MediaDeviceInfoKind, + host: String, + device_name: String, + num_channels: u16, + index: u8, +} + +impl DeviceId { + pub(crate) fn as_string( + kind: MediaDeviceInfoKind, + host: String, + device_name: String, + num_channels: u16, + index: u8, + ) -> String { + let device_info = Self { + kind, + host, + device_name, + num_channels, + index, + }; + + let mut hasher = DefaultHasher::new(); + device_info.hash(&mut hasher); + format!("{}", hasher.finish()) + } +} + +/// Describes input/output type of a media device +#[derive(Copy, Clone, Debug, PartialEq, Eq, Hash)] +pub enum MediaDeviceInfoKind { + VideoInput, + AudioInput, + AudioOutput, +} + +/// Describes a single media input or output device +/// +/// Call [`enumerate_devices_sync`] to obtain a list of devices for your hardware. +#[derive(Debug)] +pub struct MediaDeviceInfo { + device_id: String, + group_id: Option, + kind: MediaDeviceInfoKind, + label: String, +} + +impl MediaDeviceInfo { + pub(crate) fn new( + device_id: String, + group_id: Option, + kind: MediaDeviceInfoKind, + label: String, + ) -> Self { + Self { + device_id, + group_id, + kind, + label, + } + } + + /// Identifier for the represented device + /// + /// The current implementation is not stable across sessions so you should not persist this + /// value + pub fn device_id(&self) -> &str { + &self.device_id + } + + /// Two devices have the same group identifier if they belong to the same physical device + pub fn group_id(&self) -> Option<&str> { + self.group_id.as_deref() + } + + /// Enumerated value that is either "videoinput", "audioinput" or "audiooutput". + pub fn kind(&self) -> MediaDeviceInfoKind { + self.kind + } + + /// Friendly label describing this device + pub fn label(&self) -> &str { + &self.label + } +} + +/// Dictionary used to instruct what sort of tracks to include in the [`MediaStream`] returned by +/// [`get_user_media_sync`] +#[derive(Clone, Debug)] +pub enum MediaStreamConstraints { + Audio, + AudioWithConstraints(MediaTrackConstraints), +} + +/// Desired media stream track settings for [`MediaTrackConstraints`] +#[derive(Default, Debug, Clone)] +#[non_exhaustive] +pub struct MediaTrackConstraints { + // ConstrainULong width; + // ConstrainULong height; + // ConstrainDouble aspectRatio; + // ConstrainDouble frameRate; + // ConstrainDOMString facingMode; + // ConstrainDOMString resizeMode; + pub sample_rate: Option, + // ConstrainULong sampleSize; + // ConstrainBoolean echoCancellation; + // ConstrainBoolean autoGainControl; + // ConstrainBoolean noiseSuppression; + pub latency: Option, + pub channel_count: Option, // TODO model as ConstrainULong; + pub device_id: Option, + // ConstrainDOMString groupId; +} + +impl From for AudioContextOptions { + fn from(value: MediaTrackConstraints) -> Self { + let latency_hint = match value.latency { + Some(v) => AudioContextLatencyCategory::Custom(v), + None => AudioContextLatencyCategory::Interactive, + }; + let sink_id = value.device_id.unwrap_or(String::from("")); + + AudioContextOptions { + latency_hint, + sample_rate: value.sample_rate, + sink_id, + render_size_hint: Default::default(), + } + } +} + +/// Check if the provided device_id is available for playback +/// +/// It should be "" or a valid input `deviceId` returned from [`enumerate_devices_sync`] +fn is_valid_device_id(device_id: &str) -> bool { + if device_id.is_empty() { + true + } else { + enumerate_devices_sync() + .into_iter() + .filter(|d| d.kind == MediaDeviceInfoKind::AudioInput) + .any(|d| d.device_id() == device_id) + } +} + +/// Prompt for permission to use a media input (audio only) +/// +/// This produces a [`MediaStream`] with tracks containing the requested types of media, which can +/// be used inside a [`MediaStreamAudioSourceNode`](crate::node::MediaStreamAudioSourceNode). +/// +/// It is okay for the `MediaStream` struct to go out of scope, any corresponding stream will still be +/// kept alive and emit audio buffers. Call the `close()` method if you want to stop the media +/// input and release all system resources. +/// +/// This function operates synchronously, which may be undesirable on the control thread. An async +/// version is currently not implemented. +/// +/// # Panics +/// +/// This function will panic when the selected audio backend cannot create or start the input +/// stream. A public `try_get_user_media_sync` could be added in the future to handle these errors +/// without panicking, similar to [`AudioContext::try_new`](crate::context::AudioContext::new). +/// +/// Input streams with more than [`MAX_CHANNELS`](crate::MAX_CHANNELS) are not +/// supported. A panic will occur when the hardware is unable to serve a stream +/// with lower channel count. +/// +/// # Example +/// +/// ```no_run +/// use web_audio_api::context::{BaseAudioContext, AudioContext}; +/// use web_audio_api::context::{AudioContextLatencyCategory, AudioContextOptions}; +/// use web_audio_api::media_devices; +/// use web_audio_api::media_devices::MediaStreamConstraints; +/// use web_audio_api::node::AudioNode; +/// +/// let context = AudioContext::default(); +/// let mic = media_devices::get_user_media_sync(MediaStreamConstraints::Audio); +/// +/// // register as media element in the audio context +/// let background = context.create_media_stream_source(&mic); +/// +/// // connect the node directly to the destination node (speakers) +/// background.connect(&context.destination()); +/// +/// // enjoy listening +/// std::thread::sleep(std::time::Duration::from_secs(4)); +/// ``` +pub fn get_user_media_sync(constraints: MediaStreamConstraints) -> MediaStream { + try_get_user_media_sync(constraints).unwrap_or_else(|e| panic!("InvalidStateError - {e}")) +} + +pub(crate) fn try_get_user_media_sync( + constraints: MediaStreamConstraints, +) -> BackendResult { + let (channel_count, mut options) = match constraints { + MediaStreamConstraints::Audio => (None, AudioContextOptions::default()), + MediaStreamConstraints::AudioWithConstraints(cs) => (cs.channel_count, cs.into()), + }; + + if !is_valid_device_id(&options.sink_id) { + log::error!("NotFoundError: invalid deviceId {:?}", options.sink_id); + options.sink_id = String::from(""); + } + + crate::io::build_input(options, channel_count) +} diff --git a/vendor/web-audio-api-1.7.0/src/media_element.rs b/vendor/web-audio-api-1.7.0/src/media_element.rs new file mode 100644 index 0000000000..d78afb98a8 --- /dev/null +++ b/vendor/web-audio-api-1.7.0/src/media_element.rs @@ -0,0 +1,223 @@ +use std::error::Error; +use std::path::PathBuf; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::Arc; + +use creek::{ReadDiskStream, SeekMode, SymphoniaDecoder}; +use crossbeam_channel::{Receiver, Sender}; + +use crate::{AtomicF64, AudioBuffer, RENDER_QUANTUM_SIZE}; + +/// Real time safe audio stream +pub(crate) struct RTSStream { + stream: ReadDiskStream, + number_of_channels: usize, + current_time: Arc, + receiver: Receiver, + loop_: Arc, + paused: Arc, + playback_rate: Arc, +} + +impl std::fmt::Debug for RTSStream { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("RTSStream") + .field("number_of_channels", &self.number_of_channels) + .finish_non_exhaustive() + } +} + +/// Controller actions for a media element +pub(crate) enum MediaElementAction { + /// Seek to the given timestamp + Seek(f64), + /// Enable/disable looping + SetLoop(bool), + /// Start or restart the stream + Play, + /// Pause the stream + Pause, + /// Update the playback rate + SetPlaybackRate(f64), +} + +/// Shim of the `