diff --git a/CHANGELOG.md b/CHANGELOG.md index 529c70e..0a4fdbe 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,13 @@ Each release is tagged in Git and on the Docker repository ## unreleased +* UI: improve video aspect ratio handling. Live streams formerly worked + around a Firefox pixel aspect ratio bug by forcing all videos to 16:9, which + dramatically distorted 9:16 camera views. Playback didn't have the same + workaround, so anamorphic videos looked correct on Chrome but slightly + stretched on Firefox. Now both live streams and playback are fully correct + on all browsers. + ## `v0.6.4` (2021-06-28) * Default to a new pure-Rust RTSP library, `retina`. If you hit problems, you diff --git a/design/api.md b/design/api.md index 95f0162..09164df 100644 --- a/design/api.md +++ b/design/api.md @@ -333,6 +333,10 @@ the following properties: section 12.1.4.3 `PixelAspectRatioBox`. If absent, assumed to be 1. * `pixelVSpacing`: the relative height of a pixel, as in a ISO/IEC 14496-12 section 12.1.4.3 `PixelAspectRatioBox`. If absent, assumed to be 1. +* `aspectWidth`: the width component of the aspect ratio. (The aspect ratio + can be computed from the dimensions and pixel spacing; it's included as a + convenience.) +* `aspectHeight`: the height component of the aspect ratio. The full initialization segment data for a given video sample entry can be retrieved at the URL `/api/init/.mp4`. @@ -584,6 +588,11 @@ Returns a `.mp4` suitable for use as a [HTML5 Media Source Extensions initialization segment][init-segment]. The MIME type will be `video/mp4`, with a `codecs` parameter as specified in [RFC 6381][rfc-6381]. +An `X-Aspect` HTTP header will include the aspect ratio as width:height, +eg `16:9` (most cameras) or `9:16` (rotated 90 degrees). +This is redundant with the returned `.mp4` but is far easier to parse from +Javascript. + ### `GET /api/init/.mp4.txt` Returns a `text/plain` debugging string for the `.mp4` generated by the diff --git a/server/Cargo.lock b/server/Cargo.lock index 3723e8b..d7348fe 100644 --- a/server/Cargo.lock +++ b/server/Cargo.lock @@ -1264,6 +1264,7 @@ dependencies = [ "moonfire-base", "mylog", "nix 0.22.0", + "num-rational 0.4.0", "odds", "parking_lot", "pretty-hex", diff --git a/server/db/Cargo.toml b/server/db/Cargo.toml index 7fbbe11..9d8ae67 100644 --- a/server/db/Cargo.toml +++ b/server/db/Cargo.toml @@ -29,6 +29,7 @@ libpasta = "0.1.2" log = "0.4" mylog = { git = "https://github.com/scottlamb/mylog" } nix = "0.22.0" +num-rational = { version = "0.4.0", default-features = false, features = ["std"] } odds = { version = "0.4.0", features = ["std-vec"] } parking_lot = { version = "0.11.1", features = [] } pretty-hex = "0.2.1" diff --git a/server/db/db.rs b/server/db/db.rs index e87ecd9..b7a18ae 100644 --- a/server/db/db.rs +++ b/server/db/db.rs @@ -135,6 +135,16 @@ pub struct VideoSampleEntry { pub pasp_v_spacing: u16, } +impl VideoSampleEntry { + /// Returns the aspect ratio as a minimized ratio. + pub fn aspect(&self) -> num_rational::Ratio { + num_rational::Ratio::new( + u32::from(self.width) * u32::from(self.pasp_h_spacing), + u32::from(self.height) * u32::from(self.pasp_v_spacing), + ) + } +} + #[derive(PartialEq, Eq)] pub struct VideoSampleEntryToInsert { pub data: Vec, diff --git a/server/src/json.rs b/server/src/json.rs index a128514..fdfc431 100644 --- a/server/src/json.rs +++ b/server/src/json.rs @@ -496,15 +496,20 @@ pub struct VideoSampleEntry { pub height: u16, pub pasp_h_spacing: u16, pub pasp_v_spacing: u16, + pub aspect_width: u32, + pub aspect_height: u32, } impl VideoSampleEntry { fn from(e: &db::VideoSampleEntry) -> Self { + let aspect = e.aspect(); Self { width: e.width, height: e.height, pasp_h_spacing: e.pasp_h_spacing, pasp_v_spacing: e.pasp_v_spacing, + aspect_width: *aspect.numer(), + aspect_height: *aspect.denom(), } } } diff --git a/server/src/mp4.rs b/server/src/mp4.rs index 6a49f49..636d4b7 100644 --- a/server/src/mp4.rs +++ b/server/src/mp4.rs @@ -1928,6 +1928,15 @@ impl http_serve::Entity for File { ); } } + } else if self.0.type_ == Type::InitSegment { + // FileBuilder::build() should have failed if there were no video_sample_entries. + let ent = self.0.video_sample_entries.first().expect("no video_sample_entries"); + let aspect = ent.aspect(); + hdrs.insert("X-Aspect", + HeaderValue::try_from( + format!("{}:{}", aspect.numer(), aspect.denom()) + ).expect("no invalid chars in X-Aspect format") + ); } } fn last_modified(&self) -> Option { @@ -2690,6 +2699,9 @@ mod tests { let mp4 = builder .build(db.db.clone(), db.dirs_by_stream_id.clone()) .unwrap(); + let mut hdrs = http::header::HeaderMap::new(); + mp4.add_headers(&mut hdrs); + assert_eq!(hdrs.get("X-Aspect").unwrap(), "16:9"); traverse(mp4.clone()).await; } diff --git a/ui/src/List/VideoList.tsx b/ui/src/List/VideoList.tsx index 743485e..25bcf68 100644 --- a/ui/src/List/VideoList.tsx +++ b/ui/src/List/VideoList.tsx @@ -17,7 +17,9 @@ interface Props { range90k: [number, number] | null; split90k?: number; trimStartAndEnd: boolean; - setActiveRecording: (recording: [Stream, api.Recording] | null) => void; + setActiveRecording: ( + recording: [Stream, api.Recording, api.VideoSampleEntry] | null + ) => void; formatTime: (time90k: number) => string; } @@ -169,7 +171,7 @@ const VideoList = ({ setActiveRecording([stream, r])} + onClick={() => setActiveRecording([stream, r, vse])} start={formatTime(start)} end={formatTime(end)} resolution={`${vse.width}x${vse.height}`} diff --git a/ui/src/List/index.tsx b/ui/src/List/index.tsx index 9ee30e9..75e7198 100644 --- a/ui/src/List/index.tsx +++ b/ui/src/List/index.tsx @@ -18,6 +18,9 @@ import DisplaySelector, { DEFAULT_DURATION } from "./DisplaySelector"; import StreamMultiSelector from "./StreamMultiSelector"; import TimerangeSelector from "./TimerangeSelector"; import VideoList from "./VideoList"; +import { useLayoutEffect } from "react"; +import { fillAspect } from "../aspect"; +import useResizeObserver from "@react-hook/resize-observer"; const useStyles = makeStyles((theme: Theme) => ({ root: { @@ -64,13 +67,33 @@ const useStyles = makeStyles((theme: Theme) => ({ alignItems: "center", justifyContent: "center", "& video": { - objectFit: "contain", - maxWidth: "100%", - maxHeight: "100%", + objectFit: "fill", }, }, })); +interface FullScreenVideoProps { + src: string; + aspect: [number, number]; +} + +/** + * A video sized for the entire document window constrained to aspect ratio. + * This is particularly helpful for Firefox (89), which doesn't honor the + * pixel aspect ratio specified in .mp4 files. Thus we need to specify it + * out-of-band. + */ +const FullScreenVideo = ({ src, aspect }: FullScreenVideoProps) => { + const ref = React.useRef(null); + useLayoutEffect(() => { + fillAspect(document.body.getBoundingClientRect(), ref, aspect); + }); + useResizeObserver(document.body, (entry: ResizeObserverEntry) => { + fillAspect(entry.contentRect, ref, aspect); + }); + return