diff --git a/docs/per-context-patches.md b/docs/per-context-patches.md index e5f267a..8f59613 100644 --- a/docs/per-context-patches.md +++ b/docs/per-context-patches.md @@ -643,7 +643,7 @@ bundles are shipped in the wheel. **`fonts.json`, `font-bases.json`, `font-groups.json`** — OS font lists, the OS-version bases and the addition units with their probabilities (see [FONTS.md](FONTS.md)). -**`voices.json`** — OS-specific speech voice lists (macOS 190, Windows 53, Linux 131). Format: `"Name:locale:type"`. **`voice-manifests.json`** holds the per-OS model the voice draw follows. +**`voice-manifests.json`, `voice-uris.json`** — The per-OS model the voice draw follows (a base, language packs and additions, each entry `"Name:locale:type"`) and the real `voiceURI` a stock browser reports for each voice. **`properties.json`** — Includes `audio:seed` as a `CAMOU_CONFIG` property (uint type), the MaskConfig fallback for the audio patch when using global config without per-context JavaScript. diff --git a/pythonlib/camoufox/coherence.py b/pythonlib/camoufox/coherence.py index 37b1b0e..a0cf0c0 100644 --- a/pythonlib/camoufox/coherence.py +++ b/pythonlib/camoufox/coherence.py @@ -83,9 +83,9 @@ BROWSER_CHROME_HEIGHT = 86 # GPU strings that are not possible on macOS. Firefox on a Mac reports Apple # Silicon as "Apple M1, or similar", and Intel Macs as an Intel Iris/UHD/HD -# 4000-6000 part; ANGLE is Windows-only (Direct3D), and these two rows in -# webgl_data.db are a Braswell Atom IGP and a desktop PC card, neither of which -# shipped in any Mac. +# 4000-6000 part; ANGLE is Windows-only (Direct3D), and the other two, which +# webgl_data.db once offered to macOS, are a Braswell Atom IGP and a desktop PC +# card, neither of which shipped in any Mac. _NOT_A_MAC_GPU = ('ANGLE', 'Intel(R) HD Graphics 400', 'Radeon R9 200 Series', 'llvmpipe') diff --git a/pythonlib/camoufox/utils.py b/pythonlib/camoufox/utils.py index 05a04ab..ba2f437 100644 --- a/pythonlib/camoufox/utils.py +++ b/pythonlib/camoufox/utils.py @@ -1424,7 +1424,7 @@ def launch_options( try: webgl_fp = sample_webgl(target_os, config['webGl:vendor'], config['webGl:renderer'], seed=identity_seed(config, _identity_salt)) except ValueError: - # The pair is not in webgl_data.db, which holds 33 GPUs. 39 of the + # The pair is not in webgl_data.db, which holds 31 GPUs. 39 of the # 435 bundled presets name one it does not have -- including rows # that cannot be the OS they are filed under, e.g. a Windows # preset claiming "ANGLE (Unknown, Adreno (TM) 650 ...)", a phone diff --git a/pythonlib/camoufox/voices.json b/pythonlib/camoufox/voices.json deleted file mode 100644 index beab022..0000000 --- a/pythonlib/camoufox/voices.json +++ /dev/null @@ -1,382 +0,0 @@ -{ - "mac": [ - "Albert:en-US:local", - "Alex:en-US:local", - "Alice:it-IT:local", - "Alva:sv-SE:local", - "Aman:en-IN:local", - "Amira:ms-MY:local", - "Anna:de-DE:local", - "Aru:kk-KZ:local", - "Bad News:en-US:local", - "Bahh:en-US:local", - "Bells:en-US:local", - "Boing:en-US:local", - "Bubbles:en-US:local", - "Carmit:he-IL:local", - "Cellos:en-US:local", - "Damayanti:id-ID:local", - "Daniel:en-GB:local", - "Daria:bg-BG:local", - "Diego:es-AR:local", - "Eddy (English (UK)):en-GB:local", - "Eddy (English (US)):en-US:local", - "Eddy (Finnish (Finland)):fi-FI:local", - "Eddy (French (Canada)):fr-CA:local", - "Eddy (French (France)):fr-FR:local", - "Eddy (German (Germany)):de-DE:local", - "Eddy (Italian (Italy)):it-IT:local", - "Eddy (Portuguese (Brazil)):pt-BR:local", - "Eddy (Spanish (Mexico)):es-MX:local", - "Eddy (Spanish (Spain)):es-ES:local", - "Eddy (Chinese (China mainland)):zh-CN:local", - "Eddy (Chinese (Taiwan)):zh-TW:local", - "Eddy (Japanese (Japan)):ja-JP:local", - "Eddy (Korean (South Korea)):ko-KR:local", - "Ellen:nl-BE:local", - "Fiona:en-scotland:local", - "Flo (English (UK)):en-GB:local", - "Flo (English (US)):en-US:local", - "Flo (Finnish (Finland)):fi-FI:local", - "Flo (French (Canada)):fr-CA:local", - "Flo (French (France)):fr-FR:local", - "Flo (German (Germany)):de-DE:local", - "Flo (Italian (Italy)):it-IT:local", - "Flo (Portuguese (Brazil)):pt-BR:local", - "Flo (Spanish (Mexico)):es-MX:local", - "Flo (Spanish (Spain)):es-ES:local", - "Flo (Chinese (China mainland)):zh-CN:local", - "Flo (Chinese (Taiwan)):zh-TW:local", - "Flo (Japanese (Japan)):ja-JP:local", - "Flo (Korean (South Korea)):ko-KR:local", - "Fred:en-US:local", - "Geeta:te-IN:local", - "Good News:en-US:local", - "Grandma (English (UK)):en-GB:local", - "Grandma (English (US)):en-US:local", - "Grandma (Finnish (Finland)):fi-FI:local", - "Grandma (French (Canada)):fr-CA:local", - "Grandma (French (France)):fr-FR:local", - "Grandma (German (Germany)):de-DE:local", - "Grandma (Italian (Italy)):it-IT:local", - "Grandma (Portuguese (Brazil)):pt-BR:local", - "Grandma (Spanish (Mexico)):es-MX:local", - "Grandma (Spanish (Spain)):es-ES:local", - "Grandma (Chinese (China mainland)):zh-CN:local", - "Grandma (Chinese (Taiwan)):zh-TW:local", - "Grandma (Japanese (Japan)):ja-JP:local", - "Grandma (Korean (South Korea)):ko-KR:local", - "Grandpa (English (UK)):en-GB:local", - "Grandpa (English (US)):en-US:local", - "Grandpa (Finnish (Finland)):fi-FI:local", - "Grandpa (French (Canada)):fr-CA:local", - "Grandpa (French (France)):fr-FR:local", - "Grandpa (German (Germany)):de-DE:local", - "Grandpa (Italian (Italy)):it-IT:local", - "Grandpa (Portuguese (Brazil)):pt-BR:local", - "Grandpa (Spanish (Mexico)):es-MX:local", - "Grandpa (Spanish (Spain)):es-ES:local", - "Grandpa (Chinese (China mainland)):zh-CN:local", - "Grandpa (Chinese (Taiwan)):zh-TW:local", - "Grandpa (Japanese (Japan)):ja-JP:local", - "Grandpa (Korean (South Korea)):ko-KR:local", - "Ioana:ro-RO:local", - "Jacques:fr-FR:local", - "Jester:en-US:local", - "Joana:pt-PT:local", - "Jorge:es-ES:local", - "Juan:es-MX:local", - "Junior:en-US:local", - "Kanya:th-TH:local", - "Karen:en-AU:local", - "Kathy:en-US:local", - "Kyoko:ja-JP:local", - "Lana:hr-HR:local", - "Laura:sk-SK:local", - "Lekha:hi-IN:local", - "Lesya:uk-UA:local", - "Linh:vi-VN:local", - "Luca:it-IT:local", - "Luciana:pt-BR:local", - "Majed:ar-001:local", - "Meijia:zh-TW:local", - "Melina:el-GR:local", - "Milena:ru-RU:local", - "Moira:en-IE:local", - "Montse:ca-ES:local", - "Nora:nb-NO:local", - "Ona:lt-LT:local", - "Organ:en-US:local", - "Paulina:es-MX:local", - "Piya:bn-IN:local", - "Ralph:en-US:local", - "Reed (English (UK)):en-GB:local", - "Reed (English (US)):en-US:local", - "Reed (Finnish (Finland)):fi-FI:local", - "Reed (French (Canada)):fr-CA:local", - "Reed (German (Germany)):de-DE:local", - "Reed (Italian (Italy)):it-IT:local", - "Reed (Portuguese (Brazil)):pt-BR:local", - "Reed (Spanish (Mexico)):es-MX:local", - "Reed (Spanish (Spain)):es-ES:local", - "Reed (Chinese (China mainland)):zh-CN:local", - "Reed (Chinese (Taiwan)):zh-TW:local", - "Reed (Japanese (Japan)):ja-JP:local", - "Reed (Korean (South Korea)):ko-KR:local", - "Rishi:en-IN:local", - "Rocko (English (UK)):en-GB:local", - "Rocko (English (US)):en-US:local", - "Rocko (Finnish (Finland)):fi-FI:local", - "Rocko (French (Canada)):fr-CA:local", - "Rocko (French (France)):fr-FR:local", - "Rocko (German (Germany)):de-DE:local", - "Rocko (Italian (Italy)):it-IT:local", - "Rocko (Portuguese (Brazil)):pt-BR:local", - "Rocko (Spanish (Mexico)):es-MX:local", - "Rocko (Spanish (Spain)):es-ES:local", - "Rocko (Chinese (China mainland)):zh-CN:local", - "Rocko (Chinese (Taiwan)):zh-TW:local", - "Rocko (Japanese (Japan)):ja-JP:local", - "Rocko (Korean (South Korea)):ko-KR:local", - "Samantha:en-US:local", - "Sandy (English (UK)):en-GB:local", - "Sandy (English (US)):en-US:local", - "Sandy (Finnish (Finland)):fi-FI:local", - "Sandy (French (Canada)):fr-CA:local", - "Sandy (French (France)):fr-FR:local", - "Sandy (German (Germany)):de-DE:local", - "Sandy (Italian (Italy)):it-IT:local", - "Sandy (Portuguese (Brazil)):pt-BR:local", - "Sandy (Spanish (Mexico)):es-MX:local", - "Sandy (Spanish (Spain)):es-ES:local", - "Sandy (Chinese (China mainland)):zh-CN:local", - "Sandy (Chinese (Taiwan)):zh-TW:local", - "Sandy (Japanese (Japan)):ja-JP:local", - "Sandy (Korean (South Korea)):ko-KR:local", - "Sara:da-DK:local", - "Satu:fi-FI:local", - "Shelley (English (UK)):en-GB:local", - "Shelley (English (US)):en-US:local", - "Shelley (Finnish (Finland)):fi-FI:local", - "Shelley (French (Canada)):fr-CA:local", - "Shelley (French (France)):fr-FR:local", - "Shelley (German (Germany)):de-DE:local", - "Shelley (Italian (Italy)):it-IT:local", - "Shelley (Portuguese (Brazil)):pt-BR:local", - "Shelley (Spanish (Mexico)):es-MX:local", - "Shelley (Spanish (Spain)):es-ES:local", - "Shelley (Chinese (China mainland)):zh-CN:local", - "Shelley (Chinese (Taiwan)):zh-TW:local", - "Shelley (Japanese (Japan)):ja-JP:local", - "Shelley (Korean (South Korea)):ko-KR:local", - "Sinji:zh-HK:local", - "Soumya:kn-IN:local", - "Superstar:en-US:local", - "Tara:en-IN:local", - "Tessa:en-ZA:local", - "Thomas:fr-FR:local", - "Tina:sl-SI:local", - "Tingting:zh-CN:local", - "Trinoids:en-US:local", - "Vani:ta-IN:local", - "Veena:en-IN:local", - "Victoria:en-US:local", - "Whisper:en-US:local", - "Wobble:en-US:local", - "Xander:nl-NL:local", - "Yelda:tr-TR:local", - "Yuna:ko-KR:local", - "Yuri:ru-RU:local", - "Zarvox:en-US:local", - "Zosia:pl-PL:local", - "Zuzana:cs-CZ:local" - ], - "win": [ - "Microsoft David - English (United States):en-US:local", - "Microsoft Mark - English (United States):en-US:local", - "Microsoft Zira - English (United States):en-US:local", - "Microsoft David Desktop - English (United States):en-US:local", - "Microsoft Zira Desktop - English (United States):en-US:local", - "Microsoft Hazel - English (United Kingdom):en-GB:local", - "Microsoft Hazel Desktop - English (Great Britain):en-GB:local", - "Microsoft George - English (United Kingdom):en-GB:local", - "Microsoft Susan - English (United Kingdom):en-GB:local", - "Microsoft Catherine - English (Australia):en-AU:local", - "Microsoft James - English (Australia):en-AU:local", - "Microsoft Linda - English (Canada):en-CA:local", - "Microsoft Richard - English (Canada):en-CA:local", - "Microsoft Sean - English (Ireland):en-IE:local", - "Microsoft Heera - English (India):en-IN:local", - "Microsoft Ravi - English (India):en-IN:local", - "Microsoft Hedda - German (Germany):de-DE:local", - "Microsoft Hedda Desktop - German:de-DE:local", - "Microsoft Katja - German (Germany):de-DE:local", - "Microsoft Stefan - German (Germany):de-DE:local", - "Microsoft Hortense - French (France):fr-FR:local", - "Microsoft Hortense Desktop - French:fr-FR:local", - "Microsoft Julie - French (France):fr-FR:local", - "Microsoft Paul - French (France):fr-FR:local", - "Microsoft Helena - Spanish (Spain):es-ES:local", - "Microsoft Helena Desktop - Spanish (Spain):es-ES:local", - "Microsoft Laura - Spanish (Spain):es-ES:local", - "Microsoft Pablo - Spanish (Spain):es-ES:local", - "Microsoft Sabina - Spanish (Mexico):es-MX:local", - "Microsoft Sabina Desktop - Spanish (Mexico):es-MX:local", - "Microsoft Raul - Spanish (Mexico):es-MX:local", - "Microsoft Cosimo - Italian (Italy):it-IT:local", - "Microsoft Elsa - Italian (Italy):it-IT:local", - "Microsoft Elsa Desktop - Italian (Italy):it-IT:local", - "Microsoft Daniel - Portuguese (Brazil):pt-BR:local", - "Microsoft Maria - Portuguese (Brazil):pt-BR:local", - "Microsoft Maria Desktop - Portuguese(Brazil):pt-BR:local", - "Microsoft Helia - Portuguese (Portugal):pt-PT:local", - "Microsoft Huihui - Chinese (Simplified, PRC):zh-CN:local", - "Microsoft Huihui Desktop - Chinese (Simplified):zh-CN:local", - "Microsoft Kangkang - Chinese (Simplified, PRC):zh-CN:local", - "Microsoft Yaoyao - Chinese (Simplified, PRC):zh-CN:local", - "Microsoft Anna - English (United States):en-US:local", - "Microsoft Frank - Dutch (Netherlands):nl-NL:local", - "Microsoft Adam - Polish (Poland):pl-PL:local", - "Microsoft Paulina - Polish (Poland):pl-PL:local", - "Microsoft Paulina Desktop - Polish:pl-PL:local", - "Microsoft Jakub - Czech (Czech Republic):cs-CZ:local", - "Microsoft Stefanos - Greek (Greece):el-GR:local", - "Microsoft Asaf - Hebrew (Israel):he-IL:local", - "Microsoft Naayf - Arabic (Saudi):ar-SA:local", - "Microsoft Tolga - Turkish (Turkey):tr-TR:local", - "Microsoft Sarah Mobile - English (Great Britain):en-GB:local" - ], - "lin": [ - "Afrikaans:af:local", - "Amharic:am:local", - "Aragonese:an:local", - "Arabic:ar:local", - "Assamese:as:local", - "Azerbaijani:az:local", - "Bashkir:ba:local", - "Belarusian:be:local", - "Bulgarian:bg:local", - "Bengali:bn:local", - "Bishnupriya Manipuri:bpy:local", - "Bosnian:bs:local", - "Catalan:ca:local", - "Cherokee:chr-US-QAAA-X-WEST:local", - "Chinese (Mandarin, latin as English):cmn:local", - "Chinese (Mandarin, latin as Pinyin):cmn-LATN-PINYIN:local", - "Czech:cs:local", - "Chuvash:cv:local", - "Welsh:cy:local", - "Danish:da:local", - "German:de:local", - "Greek:el:local", - "English (Caribbean):en-029:local", - "English (Great Britain):en-GB:local", - "English (Scotland):en-GB-SCOTLAND:local", - "English (Lancaster):en-GB-X-GBCLAN:local", - "English (West Midlands):en-GB-X-GBCWMD:local", - "English (Received Pronunciation):en-GB-X-RP:local", - "English (America):en-US:local", - "English (America, New York City):en-US-NYC:local", - "Esperanto:eo:local", - "Spanish (Spain):es:local", - "Spanish (Latin America):es-419:local", - "Estonian:et:local", - "Basque:eu:local", - "Persian:fa:local", - "Persian (Pinglish):fa-LATN:local", - "Finnish:fi:local", - "French (Belgium):fr-BE:local", - "French (Switzerland):fr-CH:local", - "French (France):fr-FR:local", - "Gaelic (Irish):ga:local", - "Gaelic (Scottish):gd:local", - "Guarani:gn:local", - "Greek (Ancient):grc:local", - "Gujarati:gu:local", - "Hakka Chinese:hak:local", - "Hawaiian:haw:local", - "Hebrew:he:local", - "Hindi:hi:local", - "Croatian:hr:local", - "Haitian Creole:ht:local", - "Hungarian:hu:local", - "Armenian (East Armenia):hy:local", - "Armenian (West Armenia):hyw:local", - "Interlingua:ia:local", - "Indonesian:id:local", - "Ido:io:local", - "Icelandic:is:local", - "Italian:it:local", - "Japanese:ja:local", - "Lojban:jbo:local", - "Georgian:ka:local", - "Kazakh:kk:local", - "Greenlandic:kl:local", - "Kannada:kn:local", - "Korean:ko:local", - "Konkani:kok:local", - "Kurdish:ku:local", - "Kyrgyz:ky:local", - "Latin:la:local", - "Luxembourgish:lb:local", - "Lingua Franca Nova:lfn:local", - "Lithuanian:lt:local", - "Latgalian:ltg:local", - "Latvian:lv:local", - "Māori:mi:local", - "Macedonian:mk:local", - "Malayalam:ml:local", - "Marathi:mr:local", - "Malay:ms:local", - "Maltese:mt:local", - "Myanmar (Burmese):my:local", - "Norwegian Bokmål:nb:local", - "Nahuatl (Classical):nci:local", - "Nepali:ne:local", - "Dutch:nl:local", - "Nogai:nog:local", - "Oromo:om:local", - "Oriya:or:local", - "Punjabi:pa:local", - "Papiamento:pap:local", - "Klingon:piqd:local", - "Polish:pl:local", - "Portuguese (Portugal):pt:local", - "Portuguese (Brazil):pt-BR:local", - "Pyash:py:local", - "Lang_Belta:qdb:local", - "Quechua:qu:local", - "K'iche':quc:local", - "Quenya:qya:local", - "Romanian:ro:local", - "Russian:ru:local", - "Russian (Latvia):ru-LV:local", - "Sindhi:sd:local", - "Shan (Tai Yai):shn:local", - "Sinhala:si:local", - "Sindarin:sjn:local", - "Slovak:sk:local", - "Slovenian:sl:local", - "Lule Saami:smj:local", - "Albanian:sq:local", - "Serbian:sr:local", - "Swedish:sv:local", - "Swahili:sw:local", - "Tamil:ta:local", - "Telugu:te:local", - "Thai:th:local", - "Turkmen:tk:local", - "Setswana:tn:local", - "Turkish:tr:local", - "Tatar:tt:local", - "Uyghur:ug:local", - "Ukrainian:uk:local", - "Urdu:ur:local", - "Uzbek:uz:local", - "Vietnamese (Northern):vi:local", - "Vietnamese (Central):vi-VN-X-CENTRAL:local", - "Vietnamese (Southern):vi-VN-X-SOUTH:local", - "Chinese (Cantonese):yue:local", - "Chinese (Cantonese, latin as Jyutping):yue:local" - ] -} diff --git a/pythonlib/camoufox/webgl/sample.py b/pythonlib/camoufox/webgl/sample.py index 0227ecf..0f045ad 100644 --- a/pythonlib/camoufox/webgl/sample.py +++ b/pythonlib/camoufox/webgl/sample.py @@ -117,7 +117,7 @@ def sample_webgl( raise ValueError(f'No WebGL data found for OS: {os}') # Drop pairs this OS cannot report before sampling. webgl_data.db weights - # each pair per OS, and its macOS column carries a Braswell Atom IGP + # each pair per OS, and its macOS column once carried a Braswell Atom IGP # ("Intel(R) HD Graphics 400") at 7.4% and a desktop PC card ("Radeon R9 200 # Series") at 3.7% -- neither shipped in any Mac, so ~11% of macOS # identities were drawing a GPU that would contradict the rest of the diff --git a/pythonlib/camoufox/webgl/webgl_data.db b/pythonlib/camoufox/webgl/webgl_data.db index 58b531d..a7fd2a2 100644 Binary files a/pythonlib/camoufox/webgl/webgl_data.db and b/pythonlib/camoufox/webgl/webgl_data.db differ diff --git a/pythonlib/tests/test_coherence.py b/pythonlib/tests/test_coherence.py index c1d2956..dbd7af3 100644 --- a/pythonlib/tests/test_coherence.py +++ b/pythonlib/tests/test_coherence.py @@ -36,7 +36,7 @@ class TestRules: assert config["navigator.hardwareConcurrency"] == 8 def test_a_mac_cannot_report_a_braswell_atom_igp(self): - # webgl_data.db weights this at 7.4% of the macOS pool. + # webgl_data.db once weighted this at 7.4% of the macOS pool. config = {"webGl:renderer": "Intel(R) HD Graphics 400, or similar"} assert [v.rule for v in coherence.validate(config, "mac")] == ["gpu-matches-os"] diff --git a/pythonlib/tests/test_identity_salt.py b/pythonlib/tests/test_identity_salt.py index 24a7cdd..c6f69c2 100644 --- a/pythonlib/tests/test_identity_salt.py +++ b/pythonlib/tests/test_identity_salt.py @@ -85,7 +85,7 @@ class TestPinnedIdentityIsStable: # The test above draws ONE preset at random, so a preset that cannot # launch shows up as a 1-in-11 flake rather than a failure -- which is # how it reached CI. 39 of the 435 bundled presets name a GPU that is - # not among the 33 in webgl_data.db, and sample_webgl raises for those. + # not among the 31 in webgl_data.db, and sample_webgl raises for those. # Every preset has to produce launch options; see the fallback in # utils.launch_options. from camoufox.webgl import sample_webgl diff --git a/pythonlib/tests/test_shipped_data.py b/pythonlib/tests/test_shipped_data.py index 4efef5d..9909307 100644 --- a/pythonlib/tests/test_shipped_data.py +++ b/pythonlib/tests/test_shipped_data.py @@ -74,3 +74,23 @@ def test_each_os_still_has_gpus_to_draw_from(): assert count >= 2, f"{os_key} has {count} GPU(s) left" finally: connection.close() + + +def test_every_gpu_row_can_be_drawn_on_some_os(): + """A row no OS can draw is dead weight a reader would take as real data.""" + connection = sqlite3.connect(DB_PATH) + try: + rows = connection.execute( + "SELECT vendor, renderer, win, mac, lin FROM webgl_fingerprints" + ).fetchall() + finally: + connection.close() + undrawable = [ + (vendor, renderer) + for vendor, renderer, *weights in rows + if not any( + weight > 0 and coherence.gpu_fits_os(renderer, os_key) + for os_key, weight in zip(("win", "mac", "lin"), weights) + ) + ] + assert undrawable == [] diff --git a/pythonlib/tests/test_voices.py b/pythonlib/tests/test_voices.py index a5b7e52..3f55264 100644 --- a/pythonlib/tests/test_voices.py +++ b/pythonlib/tests/test_voices.py @@ -153,7 +153,7 @@ def test_voice_generation_failure_fails_closed(monkeypatch): import camoufox.utils as utils def boom(*_args, **_kwargs): - raise RuntimeError("voices.json unreadable") + raise RuntimeError("voice-manifests.json unreadable") monkeypatch.setattr(utils, "_generate_random_voice_subset", boom) cfg = _launch_config(os="macos")