From b836f5331f45f76141c19cac7bddd63757b89a2b Mon Sep 17 00:00:00 2001 From: ldm0 Date: Tue, 20 Oct 2026 05:15:17 +0800 Subject: [PATCH] fix(media): honor codec color requirements in capability queries Parse VP9 and AV1 color descriptions in the native MIME module, including BT.709 defaults and unspecified CICP fields. Report unsupported for mismatched or malformed descriptions while preserving derived dictionary conversion and snapshots. Cover codec grammar and 1,144 query combinations, including matching configurations that must retain their existing support. --- .../navigator_runtime/media_capabilities.rs | 34 ++ .../media_capabilities_codec_color.js | 297 ++++++++++++++++++ .../misc/extracted/navigator_devices.rs | 12 + moli-web-mime/src/lib.rs | 4 + moli-web-mime/src/video_codec.rs | 228 ++++++++++++++ 5 files changed, 575 insertions(+) create mode 100644 moli-renderer-v8/src/script_vm/tests/browser_api/misc/extracted/media_capabilities_codec_color.js create mode 100644 moli-web-mime/src/video_codec.rs diff --git a/moli-renderer-v8/src/context_bootstrap/navigator_runtime/media_capabilities.rs b/moli-renderer-v8/src/context_bootstrap/navigator_runtime/media_capabilities.rs index 89dbc6119d..7c2370d2f9 100644 --- a/moli-renderer-v8/src/context_bootstrap/navigator_runtime/media_capabilities.rs +++ b/moli-renderer-v8/src/context_bootstrap/navigator_runtime/media_capabilities.rs @@ -529,5 +529,39 @@ fn configuration_supported( }) && configuration.video.as_ref().is_none_or(|video| { moli_web_mime::media_mime_support(&video.content_type) != moli_web_mime::MediaMimeSupport::Unsupported + && video_codec_color_supported(video) + }) +} + +fn video_codec_color_supported(video: &VideoConfiguration) -> bool { + let Some(codec) = moli_web_mime::mime_parameter(&video.content_type, "codecs") else { + return true; + }; + let Ok(color_space) = moli_web_mime::parse_video_codec_color_space( + codec.trim_matches(|value: char| value.is_ascii_whitespace()), + ) else { + return false; + }; + let Some(color_space) = color_space else { + return true; + }; + // Compare each supplied member independently. An unspecified CICP component + // leaves that requirement to the configuration rather than inventing a color. + video.color_gamut.is_none_or(|gamut| { + color_space.primaries.is_none_or(|primaries| match gamut { + ColorGamut::Srgb => primaries == 1, + ColorGamut::P3 => matches!(primaries, 11 | 12), + ColorGamut::Rec2020 => primaries == 9, + }) + }) && video.transfer_function.is_none_or(|transfer| { + color_space + .transfer + .is_none_or(|characteristics| match transfer { + // BT.709 is the codec default for conventional SDR; 13 explicitly + // signals the IEC 61966-2-1 (sRGB) transfer characteristic. + TransferFunction::Srgb => matches!(characteristics, 1 | 13), + TransferFunction::Pq => characteristics == 16, + TransferFunction::Hlg => characteristics == 18, + }) }) } diff --git a/moli-renderer-v8/src/script_vm/tests/browser_api/misc/extracted/media_capabilities_codec_color.js b/moli-renderer-v8/src/script_vm/tests/browser_api/misc/extracted/media_capabilities_codec_color.js new file mode 100644 index 0000000000..6375ea9361 --- /dev/null +++ b/moli-renderer-v8/src/script_vm/tests/browser_api/misc/extracted/media_capabilities_codec_color.js @@ -0,0 +1,297 @@ +globalThis.__mediaCapabilitiesCodecColorResult = 'pending'; +globalThis.__mediaCapabilitiesCodecColorRows = '[]'; +(async () => { + const cases = [ + { + "codec": "vp09.00.10.08", + "primaries": 1, + "transfer": 1, + "malformed": false + }, + { + "codec": "vp09.02.10.10.01.09.16.09.01", + "primaries": 9, + "transfer": 16, + "malformed": false + }, + { + "codec": "vp09.02.10.10.01.09.18.09.00", + "primaries": 9, + "transfer": 18, + "malformed": false + }, + { + "codec": "vp09.01.10.08.03.11.01.01.00", + "primaries": 11, + "transfer": 1, + "malformed": false + }, + { + "codec": "vp09.01.10.08.03.12.01.01.00", + "primaries": 12, + "transfer": 1, + "malformed": false + }, + { + "codec": "vp09.01.10.08.03.01.13.00.01", + "primaries": 1, + "transfer": 13, + "malformed": false + }, + { + "codec": "vp09.00.10.08.01.02.02.02.00", + "primaries": 2, + "transfer": 2, + "malformed": false + }, + { + "codec": "vp09.02.10.10.01.02.16.09.00", + "primaries": 2, + "transfer": 16, + "malformed": false + }, + { + "codec": "vp09.02.10.10.01.09.02.09.00", + "primaries": 9, + "transfer": 2, + "malformed": false + }, + { + "codec": "av01.0.04M.08", + "primaries": 1, + "transfer": 1, + "malformed": false + }, + { + "codec": "av01.0.04M.10.0.112.09.16.09.0", + "primaries": 9, + "transfer": 16, + "malformed": false + }, + { + "codec": "av01.0.04M.10.0.110.09.18.09.0", + "primaries": 9, + "transfer": 18, + "malformed": false + }, + { + "codec": "av01.1.08M.08.0.000.11.01.01.0", + "primaries": 11, + "transfer": 1, + "malformed": false + }, + { + "codec": "av01.1.08M.08.0.000.12.01.01.0", + "primaries": 12, + "transfer": 1, + "malformed": false + }, + { + "codec": "av01.1.08M.08.0.000.01.13.00.1", + "primaries": 1, + "transfer": 13, + "malformed": false + }, + { + "codec": "av01.0.04M.08.0.110.02.02.02.0", + "primaries": 2, + "transfer": 2, + "malformed": false + }, + { + "codec": "av01.0.04M.10.0.110.02.16.09.0", + "primaries": 2, + "transfer": 16, + "malformed": false + }, + { + "codec": "av01.0.04M.10.0.110.09.02.09.0", + "primaries": 9, + "transfer": 2, + "malformed": false + }, + { + "codec": "vp09", + "malformed": true + }, + { + "codec": "vp09.00.10", + "malformed": true + }, + { + "codec": "vp09.0.10.08", + "malformed": true + }, + { + "codec": "vp09.00.1.08", + "malformed": true + }, + { + "codec": "vp09.00.10.8", + "malformed": true + }, + { + "codec": "vp09.04.10.08", + "malformed": true + }, + { + "codec": "vp09.00.99.08", + "malformed": true + }, + { + "codec": "vp09.00.10.09", + "malformed": true + }, + { + "codec": "vp09.00.10.08.01", + "malformed": true + }, + { + "codec": "vp09.00.10.08.01.09.16", + "malformed": true + }, + { + "codec": "vp09.00.10.08.01.09.16.09.02", + "malformed": true + }, + { + "codec": "vp09.00.10.08.01.09.16.09.00.00", + "malformed": true + }, + { + "codec": "vp09.00.10.08.01.09.16.00.00", + "malformed": true + }, + { + "codec": "vp09.-0.10.08", + "malformed": true + }, + { + "codec": "vp09.00.10.08.01.99.16.09.00", + "malformed": true + }, + { + "codec": "vp09.00.10.08.01.09.99.09.00", + "malformed": true + }, + { + "codec": "av01", + "malformed": true + }, + { + "codec": "av01.0.04M", + "malformed": true + }, + { + "codec": "av01.00.04M.08", + "malformed": true + }, + { + "codec": "av01.0.4M.08", + "malformed": true + }, + { + "codec": "av01.0.04m.08", + "malformed": true + }, + { + "codec": "av01.3.04M.08", + "malformed": true + }, + { + "codec": "av01.0.32M.08", + "malformed": true + }, + { + "codec": "av01.0.04M.09", + "malformed": true + }, + { + "codec": "av01.0.04M.08.0", + "malformed": true + }, + { + "codec": "av01.0.04M.08.0.110.09", + "malformed": true + }, + { + "codec": "av01.0.04M.08.0.110.09.16.09.2", + "malformed": true + }, + { + "codec": "av01.0.04M.08.0.110.09.16.09.0.0", + "malformed": true + }, + { + "codec": "av01.0.04M.08.2.110.09.16.09.0", + "malformed": true + }, + { + "codec": "av01.0.04M.08.0.120.09.16.09.0", + "malformed": true + }, + { + "codec": "av01.0.04M.08.0.113.09.16.09.0", + "malformed": true + }, + { + "codec": "av01.0.04M.08.0.100.09.16.09.0", + "malformed": true + }, + { + "codec": "av01.0.04M.08.0.110.99.16.09.0", + "malformed": true + }, + { + "codec": "av01.0.04M.08.0.110.09.99.09.0", + "malformed": true + } +]; + const options = [ + {}, {colorGamut: 'srgb'}, {colorGamut: 'p3'}, {colorGamut: 'rec2020'}, + {transferFunction: 'srgb'}, {transferFunction: 'pq'}, {transferFunction: 'hlg'}, + {colorGamut: 'srgb', transferFunction: 'srgb'}, + {colorGamut: 'p3', transferFunction: 'srgb'}, + {colorGamut: 'rec2020', transferFunction: 'pq'}, + {colorGamut: 'rec2020', transferFunction: 'hlg'} + ]; + const gamuts = {srgb: [1], p3: [11, 12], rec2020: [9]}; + const transfers = {srgb: [1, 13], pq: [16], hlg: [18]}; + const rows = []; + for (const type of ['file', 'media-source']) { + for (let index = 0; index < cases.length; index++) { + const spec = cases[index]; + for (let option = 0; option < options.length; option++) { + const extra = options[option]; + const expected = !spec.malformed && + (!extra.colorGamut || spec.primaries === 2 || gamuts[extra.colorGamut].includes(spec.primaries)) && + (!extra.transferFunction || spec.transfer === 2 || transfers[extra.transferFunction].includes(spec.transfer)); + const video = {contentType: 'video/webm; codecs="' + spec.codec + '"', width: 320, height: 240, bitrate: 200000, framerate: 24, ...extra}; + let promise = false; + let supported = null; + let smooth = null; + let powerEfficient = null; + let error = null; + let snapshot = false; + try { + const pending = navigator.mediaCapabilities.decodingInfo({type, video}); + promise = pending instanceof Promise; + const info = await pending; + ({supported, smooth, powerEfficient} = info); + const copy = info.configuration?.video; + snapshot = copy !== undefined && Object.getPrototypeOf(info) === Object.prototype && copy !== video && + copy.contentType === video.contentType && Object.keys(extra).every(key => copy[key] === extra[key]); + } catch (caught) {error = caught instanceof TypeError ? 'TypeError' : String(caught);} + rows.push({type, index, option, promise, error, supported, smooth, powerEfficient, snapshot, expected, + pass: promise && error === null && supported === expected && snapshot && + typeof smooth === 'boolean' && typeof powerEfficient === 'boolean' && + (supported || (!smooth && !powerEfficient))}); + } + } + } + globalThis.__mediaCapabilitiesCodecColorRows = JSON.stringify(rows); + const failed = rows.filter(row => !row.pass); + if (failed.length) throw new Error(failed.length + ' codec color failures; first: ' + JSON.stringify(failed[0])); + return 'ok'; +})().then(value => globalThis.__mediaCapabilitiesCodecColorResult = value, + error => globalThis.__mediaCapabilitiesCodecColorResult = String(error)); diff --git a/moli-renderer-v8/src/script_vm/tests/browser_api/misc/extracted/navigator_devices.rs b/moli-renderer-v8/src/script_vm/tests/browser_api/misc/extracted/navigator_devices.rs index 225efac878..b84e88a2a8 100644 --- a/moli-renderer-v8/src/script_vm/tests/browser_api/misc/extracted/navigator_devices.rs +++ b/moli-renderer-v8/src/script_vm/tests/browser_api/misc/extracted/navigator_devices.rs @@ -1,5 +1,17 @@ use super::*; +#[test] +fn media_capabilities_codec_colors_constrain_support_without_rejecting_the_dictionary() { + let mut vm = new_storage_test_vm("https://media-capabilities-codec-color.test/"); + vm.eval(include_str!("media_capabilities_codec_color.js")) + .unwrap(); + vm.eval("0").unwrap(); + assert_eq!( + vm.eval("__mediaCapabilitiesCodecColorResult").unwrap(), + "ok" + ); +} + #[test] fn media_capabilities_mime_rules_distinguish_containers_and_rtp_codecs() { let mut vm = new_storage_test_vm("https://media-capabilities-mime.test/"); diff --git a/moli-web-mime/src/lib.rs b/moli-web-mime/src/lib.rs index 9f2547f19d..165d56f06b 100644 --- a/moli-web-mime/src/lib.rs +++ b/moli-web-mime/src/lib.rs @@ -8,6 +8,7 @@ mod parse; mod resource; mod response_policy; mod sniffing; +mod video_codec; pub use classification::{ is_audio_mime, is_audio_mime_essence, is_binary_document_mime_type, is_css_mime, @@ -50,6 +51,9 @@ pub use sniffing::{ sniff_archive_mime_type, sniff_audio_video_mime_type, sniff_font_mime_type, sniff_image_mime_type, sniff_text_or_binary, sniff_unknown_mime_type, }; +pub use video_codec::{ + InvalidVideoCodecColorSpace, VideoCodecColorSpace, parse_video_codec_color_space, +}; #[cfg(test)] mod tests; diff --git a/moli-web-mime/src/video_codec.rs b/moli-web-mime/src/video_codec.rs new file mode 100644 index 0000000000..dda1c01d66 --- /dev/null +++ b/moli-web-mime/src/video_codec.rs @@ -0,0 +1,228 @@ +//! Color requirements carried by RFC 6381 video codec identifiers. + +/// CICP color identifiers. `None` represents the explicit "unspecified" code 2. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub struct VideoCodecColorSpace { + pub primaries: Option, + pub transfer: Option, +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub struct InvalidVideoCodecColorSpace; + +/// Parses VP9 and AV1 color descriptions, including their BT.709 defaults. +/// Other codec families return `Ok(None)`: their identifiers do not signal +/// color in either of these formats. Parsing does not establish decoder support. +pub fn parse_video_codec_color_space( + codec: &str, +) -> Result, InvalidVideoCodecColorSpace> { + // Bound the number of borrowed fields even for arbitrarily long input. + let fields: Vec<_> = codec.splitn(11, '.').collect(); + match fields[0] { + "vp09" => vp9_color_space(&fields).map(Some), + "av01" => av1_color_space(&fields).map(Some), + _ => Ok(None), + } +} + +fn decimal(field: &str, width: usize) -> Result { + if field.len() != width || !field.bytes().all(|byte| byte.is_ascii_digit()) { + return Err(InvalidVideoCodecColorSpace); + } + field.parse().map_err(|_| InvalidVideoCodecColorSpace) +} + +fn color_space( + primaries: u8, + transfer: u8, + matrix: u8, +) -> Result { + // H.273 / CICP assigned values; reserved identifiers cannot describe a + // supported color space. Code 2 deliberately leaves a component unspecified. + if !matches!(primaries, 1 | 2 | 4..=12 | 22) + || !matches!(transfer, 1 | 2 | 4..=18) + || !matches!(matrix, 0..=2 | 4..=14) + { + return Err(InvalidVideoCodecColorSpace); + } + Ok(VideoCodecColorSpace { + primaries: (primaries != 2).then_some(primaries), + transfer: (transfer != 2).then_some(transfer), + }) +} + +fn vp9_color_space(fields: &[&str]) -> Result { + // https://www.webmproject.org/vp9/mp4/#codecs-parameter-string + // The five optional fields are mutually inclusive, not a partial suffix. + if !matches!(fields.len(), 4 | 9) { + return Err(InvalidVideoCodecColorSpace); + } + let profile = decimal(fields[1], 2)?; + let level = decimal(fields[2], 2)?; + let depth = decimal(fields[3], 2)?; + if profile > 3 + || !matches!(level, 0 | 10 | 11 | 20 | 21 | 30 | 31 | 40 | 41 | 50..=52 | 60..=62) + || !matches!((profile, depth), (0 | 1, 8) | (2 | 3, 10 | 12)) + { + return Err(InvalidVideoCodecColorSpace); + } + let (chroma, primaries, transfer, matrix, full_range) = if fields.len() == 4 { + (1, 1, 1, 1, 0) + } else { + ( + decimal(fields[4], 2)?, + decimal(fields[5], 2)?, + decimal(fields[6], 2)?, + decimal(fields[7], 2)?, + decimal(fields[8], 2)?, + ) + }; + if chroma > 3 + || full_range > 1 + || (matches!(profile, 0 | 2) != (chroma <= 1)) + || (matrix == 0 && chroma != 3) + { + return Err(InvalidVideoCodecColorSpace); + } + color_space(primaries, transfer, matrix) +} + +fn av1_color_space(fields: &[&str]) -> Result { + // https://aomediacodec.github.io/av1-isobmff/#codecsparam + if !matches!(fields.len(), 4 | 10) { + return Err(InvalidVideoCodecColorSpace); + } + let profile = decimal(fields[1], 1)?; + let level_tier = fields[2].as_bytes(); + if level_tier.len() != 3 || !matches!(level_tier[2], b'M' | b'H') { + return Err(InvalidVideoCodecColorSpace); + } + let level = decimal(&fields[2][..2], 2)?; + let depth = decimal(fields[3], 2)?; + if profile > 2 + || level > 31 + || (level_tier[2] == b'H' && level < 8) + || !matches!(depth, 8 | 10 | 12) + || (depth == 12 && profile != 2) + { + return Err(InvalidVideoCodecColorSpace); + } + let (monochrome, chroma, primaries, transfer, matrix, full_range) = if fields.len() == 4 { + (0, "110", 1, 1, 1, 0) + } else { + ( + decimal(fields[4], 1)?, + fields[5], + decimal(fields[6], 2)?, + decimal(fields[7], 2)?, + decimal(fields[8], 2)?, + decimal(fields[9], 1)?, + ) + }; + let valid_chroma = matches!(chroma, "000" | "100" | "110" | "111" | "112"); + if monochrome > 1 + || full_range > 1 + || !valid_chroma + || (monochrome == 1 && (profile == 1 || chroma != "110")) + || (monochrome == 0 + && match profile { + 0 => !chroma.starts_with("11"), + 1 => chroma != "000", + 2 if depth < 12 => chroma != "100", + _ => false, + }) + || (matrix == 0 && chroma != "000") + { + return Err(InvalidVideoCodecColorSpace); + } + color_space(primaries, transfer, matrix) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn omitted_codec_color_fields_mean_bt709() { + for codec in ["vp09.00.10.08", "av01.0.04M.08"] { + assert_eq!( + parse_video_codec_color_space(codec), + Ok(Some(VideoCodecColorSpace { + primaries: Some(1), + transfer: Some(1), + })), + "{codec}" + ); + } + } + + #[test] + fn explicit_codec_colors_preserve_hdr_and_unspecified_components() { + for (codec, primaries, transfer) in [ + ("vp09.02.10.10.01.09.16.09.01", Some(9), Some(16)), + ("av01.0.04M.10.0.112.09.18.09.0", Some(9), Some(18)), + ("vp09.00.10.08.01.02.02.02.00", None, None), + ("av01.0.04M.10.0.110.02.16.09.0", None, Some(16)), + ("vp09.02.10.10.01.09.02.09.00", Some(9), None), + ("av01.1.08M.08.0.000.01.13.00.1", Some(1), Some(13)), + ] { + assert_eq!( + parse_video_codec_color_space(codec), + Ok(Some(VideoCodecColorSpace { + primaries, + transfer, + })), + "{codec}" + ); + } + } + + #[test] + fn codec_fields_require_the_registered_widths_ranges_and_complete_suffix() { + for codec in [ + "vp09", + "vp09.00.10", + "vp09.0.10.08", + "vp09.00.10.8", + "vp09.04.10.08", + "vp09.00.99.08", + "vp09.00.10.10", + "vp09.00.10.08.01.09.16", + "vp09.00.10.08.01.09.16.09.02", + "vp09.00.10.08.01.09.16.00.00", + "vp09.00.10.08.01.99.16.09.00", + "vp09.00.10.08.01.09.99.09.00", + "vp09.00.10.08.01.09.16.09.00.00", + "vp09.-0.10.08", + "vp09.00.10.08.01..16.09.00", + "av01", + "av01.00.04M.08", + "av01.0.4M.08", + "av01.0.04m.08", + "av01.0.éM.08", + "av01.3.04M.08", + "av01.0.32M.08", + "av01.0.04H.08", + "av01.0.04M.12", + "av01.0.04M.08.0.110.09.16", + "av01.0.04M.08.0.113.09.16.09.0", + "av01.0.04M.08.0.100.09.16.09.0", + "av01.1.08M.08.1.110.09.16.09.0", + "av01.0.04M.08.0.110.09.16.09.2", + "av01.0.04M.08.0.110.09.16.09.0.0", + ] { + assert_eq!( + parse_video_codec_color_space(codec), + Err(InvalidVideoCodecColorSpace), + "{codec}" + ); + } + } + + #[test] + fn other_codec_identifiers_do_not_acquire_vp9_color_defaults() { + for codec in ["avc1.42e01e", "vp8", "vp9", "hvc1.1.6.L93.B0", ""] { + assert_eq!(parse_video_codec_color_space(codec), Ok(None), "{codec}"); + } + } +}