From 97678f42e4e7aec85ccd65ee1dfbfb963ef47282 Mon Sep 17 00:00:00 2001 From: Scott Lamb Date: Tue, 16 Feb 2021 22:15:54 -0800 Subject: [PATCH] rustfmt everything I want to make the project more accessible by not expecting folks to match my idiosyncratic style. Now almost [1] everything is written in the "standard" style. CI enforces this. [1] "Almost": I used #[rustfmt::skip] in a few sections where I felt aligning things in columns significantly improves readability. --- .github/workflows/ci.yml | 6 + server/base/clock.rs | 58 +- server/base/error.rs | 19 +- server/base/lib.rs | 4 +- server/base/strutil.rs | 45 +- server/base/time.rs | 174 ++- server/db/auth.rs | 491 ++++++--- server/db/check.rs | 159 ++- server/db/coding.rs | 137 ++- server/db/compare.rs | 122 ++- server/db/db.rs | 1287 ++++++++++++++-------- server/db/dir.rs | 189 +++- server/db/fs.rs | 12 +- server/db/lib.rs | 2 +- server/db/raw.rs | 218 ++-- server/db/recording.rs | 304 ++++-- server/db/signal.rs | 278 +++-- server/db/testutil.rs | 133 ++- server/db/upgrade/mod.rs | 144 ++- server/db/upgrade/v0_to_v1.rs | 72 +- server/db/upgrade/v1_to_v2.rs | 122 ++- server/db/upgrade/v2_to_v3.rs | 59 +- server/db/upgrade/v3_to_v4.rs | 7 +- server/db/upgrade/v4_to_v5.rs | 71 +- server/db/upgrade/v5_to_v6.rs | 117 +- server/db/writer.rs | 812 +++++++++----- server/src/analytics.rs | 48 +- server/src/body.rs | 68 +- server/src/cmds/check.rs | 23 +- server/src/cmds/config/cameras.rs | 415 ++++--- server/src/cmds/config/dirs.rs | 303 +++-- server/src/cmds/config/mod.rs | 34 +- server/src/cmds/config/users.rs | 197 ++-- server/src/cmds/init.rs | 16 +- server/src/cmds/login.rs | 95 +- server/src/cmds/mod.rs | 47 +- server/src/cmds/run.rs | 118 +- server/src/cmds/sql.rs | 22 +- server/src/cmds/upgrade/mod.rs | 49 +- server/src/h264.rs | 110 +- server/src/json.rs | 156 ++- server/src/main.rs | 36 +- server/src/mp4.rs | 1697 ++++++++++++++++++----------- server/src/slices.rs | 227 ++-- server/src/stream.rs | 81 +- server/src/streamer.rs | 185 +++- server/src/web.rs | 1025 ++++++++++------- 47 files changed, 6541 insertions(+), 3453 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 32a3423..81dc380 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -17,6 +17,8 @@ jobs: include: - rust: nightly extra_args: "--features nightly --benches" + - rust: stable + extra_components: rustfmt runs-on: ubuntu-20.04 steps: - name: Checkout @@ -37,8 +39,12 @@ jobs: profile: minimal toolchain: ${{ matrix.rust }} override: true + components: ${{ matrix.extra_components }} - name: Test run: cd server && cargo test ${{ matrix.extra_args }} --all + - name: Check formatting + if: matrix.rust == 'stable' + run: cd server && cargo fmt --all -- --check js: name: Build and lint Javascript frontend runs-on: ubuntu-20.04 diff --git a/server/base/clock.rs b/server/base/clock.rs index f79fa32..606e5ae 100644 --- a/server/base/clock.rs +++ b/server/base/clock.rs @@ -35,13 +35,13 @@ use libc; use log::warn; use parking_lot::Mutex; use std::mem; -use std::sync::{Arc, mpsc}; +use std::sync::{mpsc, Arc}; use std::thread; use std::time::Duration as StdDuration; use time::{Duration, Timespec}; /// Abstract interface to the system clocks. This is for testability. -pub trait Clocks : Send + Sync + 'static { +pub trait Clocks: Send + Sync + 'static { /// Gets the current time from `CLOCK_REALTIME`. fn realtime(&self) -> Timespec; @@ -52,19 +52,29 @@ pub trait Clocks : Send + Sync + 'static { fn sleep(&self, how_long: Duration); /// Calls `rcv.recv_timeout` or substitutes a test implementation. - fn recv_timeout(&self, rcv: &mpsc::Receiver, - timeout: StdDuration) -> Result; + fn recv_timeout( + &self, + rcv: &mpsc::Receiver, + timeout: StdDuration, + ) -> Result; } pub fn retry_forever(clocks: &C, f: &mut dyn FnMut() -> Result) -> T -where C: Clocks, E: Into { +where + C: Clocks, + E: Into, +{ loop { let e = match f() { Ok(t) => return t, Err(e) => e.into(), }; let sleep_time = Duration::seconds(1); - warn!("sleeping for {:?} after error: {}", sleep_time, crate::error::prettify_failure(&e)); + warn!( + "sleeping for {:?} after error: {}", + sleep_time, + crate::error::prettify_failure(&e) + ); clocks.sleep(sleep_time); } } @@ -84,8 +94,12 @@ impl RealClocks { } impl Clocks for RealClocks { - fn realtime(&self) -> Timespec { self.get(libc::CLOCK_REALTIME) } - fn monotonic(&self) -> Timespec { self.get(libc::CLOCK_MONOTONIC) } + fn realtime(&self) -> Timespec { + self.get(libc::CLOCK_REALTIME) + } + fn monotonic(&self) -> Timespec { + self.get(libc::CLOCK_MONOTONIC) + } fn sleep(&self, how_long: Duration) { match how_long.to_std() { @@ -94,8 +108,11 @@ impl Clocks for RealClocks { }; } - fn recv_timeout(&self, rcv: &mpsc::Receiver, - timeout: StdDuration) -> Result { + fn recv_timeout( + &self, + rcv: &mpsc::Receiver, + timeout: StdDuration, + ) -> Result { rcv.recv_timeout(timeout) } } @@ -119,7 +136,11 @@ impl<'a, C: Clocks + ?Sized, S: AsRef, F: FnOnce() -> S + 'a> TimerGuard<'a } impl<'a, C, S, F> Drop for TimerGuard<'a, C, S, F> -where C: Clocks + ?Sized, S: AsRef, F: FnOnce() -> S + 'a { +where + C: Clocks + ?Sized, + S: AsRef, + F: FnOnce() -> S + 'a, +{ fn drop(&mut self) { let elapsed = self.clocks.monotonic() - self.start; if elapsed.num_seconds() >= 1 { @@ -148,8 +169,12 @@ impl SimulatedClocks { } impl Clocks for SimulatedClocks { - fn realtime(&self) -> Timespec { self.0.boot + *self.0.uptime.lock() } - fn monotonic(&self) -> Timespec { Timespec::new(0, 0) + *self.0.uptime.lock() } + fn realtime(&self) -> Timespec { + self.0.boot + *self.0.uptime.lock() + } + fn monotonic(&self) -> Timespec { + Timespec::new(0, 0) + *self.0.uptime.lock() + } /// Advances the clock by the specified amount without actually sleeping. fn sleep(&self, how_long: Duration) { @@ -158,8 +183,11 @@ impl Clocks for SimulatedClocks { } /// Advances the clock by the specified amount if data is not immediately available. - fn recv_timeout(&self, rcv: &mpsc::Receiver, - timeout: StdDuration) -> Result { + fn recv_timeout( + &self, + rcv: &mpsc::Receiver, + timeout: StdDuration, + ) -> Result { let r = rcv.recv_timeout(StdDuration::new(0, 0)); if let Err(_) = r { self.sleep(Duration::from_std(timeout).unwrap()); diff --git a/server/base/error.rs b/server/base/error.rs index 04be8f5..6815805 100644 --- a/server/base/error.rs +++ b/server/base/error.rs @@ -38,8 +38,11 @@ pub fn prettify_failure(e: &failure::Error) -> String { write!(&mut msg, "\ncaused by: {}", cause).unwrap(); } if e.backtrace().is_empty() { - write!(&mut msg, "\n\n(set environment variable RUST_BACKTRACE=1 to see backtraces)") - .unwrap(); + write!( + &mut msg, + "\n\n(set environment variable RUST_BACKTRACE=1 to see backtraces)" + ) + .unwrap(); } else { write!(&mut msg, "\n\nBacktrace:\n{}", e.backtrace()).unwrap(); } @@ -73,7 +76,9 @@ impl Fail for Error { impl From for Error { fn from(kind: ErrorKind) -> Error { - Error { inner: Context::new(kind) } + Error { + inner: Context::new(kind), + } } } @@ -112,6 +117,8 @@ impl fmt::Display for Error { /// which is a nice general-purpose classification of errors. See that link for descriptions of /// each error. #[derive(Copy, Clone, Eq, PartialEq, Debug, Fail)] +#[non_exhaustive] +#[rustfmt::skip] pub enum ErrorKind { #[fail(display = "Cancelled")] Cancelled, #[fail(display = "Unknown")] Unknown, @@ -129,7 +136,6 @@ pub enum ErrorKind { #[fail(display = "Internal")] Internal, #[fail(display = "Unavailable")] Unavailable, #[fail(display = "Data loss")] DataLoss, - #[doc(hidden)] #[fail(display = "__Nonexhaustive")] __Nonexhaustive, } /// Extension methods for `Result`. @@ -146,7 +152,10 @@ pub trait ResultExt { fn err_kind(self, k: ErrorKind) -> Result; } -impl ResultExt for Result where E: Into { +impl ResultExt for Result +where + E: Into, +{ fn err_kind(self, k: ErrorKind) -> Result { self.map_err(|e| e.into().context(k).into()) } diff --git a/server/base/lib.rs b/server/base/lib.rs index 6eeec8a..d04c662 100644 --- a/server/base/lib.rs +++ b/server/base/lib.rs @@ -29,8 +29,8 @@ // along with this program. If not, see . pub mod clock; -pub mod time; mod error; pub mod strutil; +pub mod time; -pub use crate::error::{Error, ErrorKind, ResultExt, prettify_failure}; +pub use crate::error::{prettify_failure, Error, ErrorKind, ResultExt}; diff --git a/server/base/strutil.rs b/server/base/strutil.rs index d1d57e2..7a215c0 100644 --- a/server/base/strutil.rs +++ b/server/base/strutil.rs @@ -28,12 +28,12 @@ // You should have received a copy of the GNU General Public License // along with this program. If not, see . -use nom::IResult; use nom::branch::alt; use nom::bytes::complete::{tag, take_while1}; use nom::character::complete::space0; use nom::combinator::{map, map_res, opt}; use nom::sequence::{delimited, tuple}; +use nom::IResult; use std::fmt::Write as _; static MULTIPLIERS: [(char, u64); 4] = [ @@ -48,7 +48,7 @@ static MULTIPLIERS: [(char, u64); 4] = [ pub fn encode_size(mut raw: i64) -> String { let mut encoded = String::new(); for &(c, n) in &MULTIPLIERS { - if raw >= 1i64<= 1i64 << n { write!(&mut encoded, "{}{} ", raw >> n, c).unwrap(); raw &= (1i64 << n) - 1; } @@ -56,7 +56,7 @@ pub fn encode_size(mut raw: i64) -> String { if raw > 0 || encoded.len() == 0 { write!(&mut encoded, "{}", raw).unwrap(); } else { - encoded.pop(); // remove trailing space. + encoded.pop(); // remove trailing space. } encoded } @@ -64,24 +64,24 @@ pub fn encode_size(mut raw: i64) -> String { fn decode_sizepart(input: &str) -> IResult<&str, i64> { map( tuple(( - map_res(take_while1(|c: char| c.is_ascii_digit()), - |input: &str| i64::from_str_radix(input, 10)), + map_res(take_while1(|c: char| c.is_ascii_digit()), |input: &str| { + i64::from_str_radix(input, 10) + }), opt(alt(( - nom::combinator::value(1<<40, tag("T")), - nom::combinator::value(1<<30, tag("G")), - nom::combinator::value(1<<20, tag("M")), - nom::combinator::value(1<<10, tag("K")) - ))) + nom::combinator::value(1 << 40, tag("T")), + nom::combinator::value(1 << 30, tag("G")), + nom::combinator::value(1 << 20, tag("M")), + nom::combinator::value(1 << 10, tag("K")), + ))), )), - |(n, opt_unit)| n * opt_unit.unwrap_or(1) + |(n, opt_unit)| n * opt_unit.unwrap_or(1), )(input) } fn decode_size_internal(input: &str) -> IResult<&str, i64> { - nom::multi::fold_many1( - delimited(space0, decode_sizepart, space0), - 0, - |sum, i| sum + i)(input) + nom::multi::fold_many1(delimited(space0, decode_sizepart, space0), 0, |sum, i| { + sum + i + })(input) } /// Decodes a human-readable size as output by encode_size. @@ -95,12 +95,15 @@ pub fn decode_size(encoded: &str) -> Result { /// Returns a hex-encoded version of the input. pub fn hex(raw: &[u8]) -> String { - const HEX_CHARS: [u8; 16] = [b'0', b'1', b'2', b'3', b'4', b'5', b'6', b'7', - b'8', b'9', b'a', b'b', b'c', b'd', b'e', b'f']; + #[rustfmt::skip] + const HEX_CHARS: [u8; 16] = [ + b'0', b'1', b'2', b'3', b'4', b'5', b'6', b'7', + b'8', b'9', b'a', b'b', b'c', b'd', b'e', b'f', + ]; let mut hex = Vec::with_capacity(2 * raw.len()); for b in raw { hex.push(HEX_CHARS[((b & 0xf0) >> 4) as usize]); - hex.push(HEX_CHARS[( b & 0x0f ) as usize]); + hex.push(HEX_CHARS[(b & 0x0f) as usize]); } unsafe { String::from_utf8_unchecked(hex) } } @@ -108,8 +111,8 @@ pub fn hex(raw: &[u8]) -> String { /// Returns [0, 16) or error. fn dehex_byte(hex_byte: u8) -> Result { match hex_byte { - b'0' ..= b'9' => Ok(hex_byte - b'0'), - b'a' ..= b'f' => Ok(hex_byte - b'a' + 10), + b'0'..=b'9' => Ok(hex_byte - b'0'), + b'a'..=b'f' => Ok(hex_byte - b'a' + 10), _ => Err(()), } } @@ -122,7 +125,7 @@ pub fn dehex(hexed: &[u8]) -> Result<[u8; 20], ()> { } let mut out = [0; 20]; for i in 0..20 { - out[i] = (dehex_byte(hexed[i<<1])? << 4) + dehex_byte(hexed[(i<<1) + 1])?; + out[i] = (dehex_byte(hexed[i << 1])? << 4) + dehex_byte(hexed[(i << 1) + 1])?; } Ok(out) } diff --git a/server/base/time.rs b/server/base/time.rs index cf6ed44..9a9ddb9 100644 --- a/server/base/time.rs +++ b/server/base/time.rs @@ -30,13 +30,13 @@ //! Time and durations for Moonfire NVR's internal format. -use failure::{Error, bail, format_err}; +use failure::{bail, format_err, Error}; use nom::branch::alt; use nom::bytes::complete::{tag, take_while_m_n}; use nom::combinator::{map, map_res, opt}; use nom::sequence::{preceded, tuple}; -use std::ops; use std::fmt; +use std::ops; use std::str::FromStr; use time; @@ -50,8 +50,10 @@ pub struct Time(pub i64); /// Returns a parser for a `len`-digit non-negative number which fits into an i32. fn fixed_len_num<'a>(len: usize) -> impl FnMut(&'a str) -> IResult<&'a str, i32> { - map_res(take_while_m_n(len, len, |c: char| c.is_ascii_digit()), - |input: &str| i32::from_str_radix(input, 10)) + map_res( + take_while_m_n(len, len, |c: char| c.is_ascii_digit()), + |input: &str| i32::from_str_radix(input, 10), + ) } /// Parses `YYYY-mm-dd` into pieces. @@ -59,7 +61,7 @@ fn parse_datepart(input: &str) -> IResult<&str, (i32, i32, i32)> { tuple(( fixed_len_num(4), preceded(tag("-"), fixed_len_num(2)), - preceded(tag("-"), fixed_len_num(2)) + preceded(tag("-"), fixed_len_num(2)), ))(input) } @@ -67,9 +69,9 @@ fn parse_datepart(input: &str) -> IResult<&str, (i32, i32, i32)> { fn parse_timepart(input: &str) -> IResult<&str, (i32, i32, i32, i32)> { let (input, (hr, _, min)) = tuple((fixed_len_num(2), tag(":"), fixed_len_num(2)))(input)?; let (input, stuff) = opt(tuple(( - preceded(tag(":"), fixed_len_num(2)), - opt(preceded(tag(":"), fixed_len_num(5))) - )))(input)?; + preceded(tag(":"), fixed_len_num(2)), + opt(preceded(tag(":"), fixed_len_num(5))), + )))(input)?; let (sec, opt_subsec) = stuff.unwrap_or((0, None)); Ok((input, (hr, min, sec, opt_subsec.unwrap_or(0)))) } @@ -77,18 +79,23 @@ fn parse_timepart(input: &str) -> IResult<&str, (i32, i32, i32, i32)> { /// Parses `Z` (UTC) or `{+,-,}HH:MM` into a time zone offset in seconds. fn parse_zone(input: &str) -> IResult<&str, i32> { alt(( - nom::combinator::value(0, tag("Z")), - map( - tuple(( - opt(nom::character::complete::one_of(&b"+-"[..])), - fixed_len_num(2), - tag(":"), - fixed_len_num(2) - )), - |(sign, hr, _, min)| { - let off = hr * 3600 + min * 60; - if sign == Some('-') { off } else { -off } - }) + nom::combinator::value(0, tag("Z")), + map( + tuple(( + opt(nom::character::complete::one_of(&b"+-"[..])), + fixed_len_num(2), + tag(":"), + fixed_len_num(2), + )), + |(sign, hr, _, min)| { + let off = hr * 3600 + min * 60; + if sign == Some('-') { + off + } else { + -off + } + }, + ), ))(input) } @@ -97,8 +104,12 @@ impl Time { Time(tm.sec * TIME_UNITS_PER_SEC + tm.nsec as i64 * TIME_UNITS_PER_SEC / 1_000_000_000) } - pub const fn min_value() -> Self { Time(i64::min_value()) } - pub const fn max_value() -> Self { Time(i64::max_value()) } + pub const fn min_value() -> Self { + Time(i64::min_value()) + } + pub const fn max_value() -> Self { + Time(i64::max_value()) + } /// Parses a time as either 90,000ths of a second since epoch or a RFC 3339-like string. /// @@ -112,20 +123,21 @@ impl Time { // First try parsing as 90,000ths of a second since epoch. match i64::from_str(input) { Ok(i) => return Ok(Time(i)), - Err(_) => {}, + Err(_) => {} } // If that failed, parse as a time string or bust. - let (remaining, ((tm_year, tm_mon, tm_mday), opt_time, opt_zone)) = - tuple((parse_datepart, - opt(preceded(tag("T"), parse_timepart)), - opt(parse_zone)))(input) - .map_err(|e| match e { - nom::Err::Incomplete(_) => format_err!("incomplete"), - nom::Err::Error(e) | nom::Err::Failure(e) => { - format_err!("{}", nom::error::convert_error(input, e)) - } - })?; + let (remaining, ((tm_year, tm_mon, tm_mday), opt_time, opt_zone)) = tuple(( + parse_datepart, + opt(preceded(tag("T"), parse_timepart)), + opt(parse_zone), + ))(input) + .map_err(|e| match e { + nom::Err::Incomplete(_) => format_err!("incomplete"), + nom::Err::Error(e) | nom::Err::Failure(e) => { + format_err!("{}", nom::error::convert_error(input, e)) + } + })?; if remaining != "" { bail!("unexpected suffix {:?} following time string", remaining); } @@ -166,32 +178,44 @@ impl Time { } /// Convert to unix seconds by floor method (rounding down). - pub fn unix_seconds(&self) -> i64 { self.0 / TIME_UNITS_PER_SEC } + pub fn unix_seconds(&self) -> i64 { + self.0 / TIME_UNITS_PER_SEC + } } impl std::str::FromStr for Time { type Err = Error; - fn from_str(s: &str) -> Result { Self::parse(s) } + fn from_str(s: &str) -> Result { + Self::parse(s) + } } impl ops::Sub for Time { type Output = Duration; - fn sub(self, rhs: Time) -> Duration { Duration(self.0 - rhs.0) } + fn sub(self, rhs: Time) -> Duration { + Duration(self.0 - rhs.0) + } } impl ops::AddAssign for Time { - fn add_assign(&mut self, rhs: Duration) { self.0 += rhs.0 } + fn add_assign(&mut self, rhs: Duration) { + self.0 += rhs.0 + } } impl ops::Add for Time { type Output = Time; - fn add(self, rhs: Duration) -> Time { Time(self.0 + rhs.0) } + fn add(self, rhs: Duration) -> Time { + Time(self.0 + rhs.0) + } } impl ops::Sub for Time { type Output = Time; - fn sub(self, rhs: Duration) -> Time { Time(self.0 - rhs.0) } + fn sub(self, rhs: Duration) -> Time { + Time(self.0 - rhs.0) + } } impl fmt::Debug for Time { @@ -203,11 +227,20 @@ impl fmt::Debug for Time { impl fmt::Display for Time { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - let tm = time::at(time::Timespec{sec: self.0 / TIME_UNITS_PER_SEC, nsec: 0}); + let tm = time::at(time::Timespec { + sec: self.0 / TIME_UNITS_PER_SEC, + nsec: 0, + }); let zone_minutes = tm.tm_utcoff.abs() / 60; - write!(f, "{}:{:05}{}{:02}:{:02}", tm.strftime("%FT%T").or_else(|_| Err(fmt::Error))?, - self.0 % TIME_UNITS_PER_SEC, - if tm.tm_utcoff > 0 { '+' } else { '-' }, zone_minutes / 60, zone_minutes % 60) + write!( + f, + "{}:{:05}{}{:02}:{:02}", + tm.strftime("%FT%T").or_else(|_| Err(fmt::Error))?, + self.0 % TIME_UNITS_PER_SEC, + if tm.tm_utcoff > 0 { '+' } else { '-' }, + zone_minutes / 60, + zone_minutes % 60 + ) } } @@ -242,18 +275,33 @@ impl fmt::Display for Duration { false }; if hours > 0 { - write!(f, "{}{} hour{}", if have_written { " " } else { "" }, - hours, if hours == 1 { "" } else { "s" })?; + write!( + f, + "{}{} hour{}", + if have_written { " " } else { "" }, + hours, + if hours == 1 { "" } else { "s" } + )?; have_written = true; } if minutes > 0 { - write!(f, "{}{} minute{}", if have_written { " " } else { "" }, - minutes, if minutes == 1 { "" } else { "s" })?; + write!( + f, + "{}{} minute{}", + if have_written { " " } else { "" }, + minutes, + if minutes == 1 { "" } else { "s" } + )?; have_written = true; } if seconds > 0 || !have_written { - write!(f, "{}{} second{}", if have_written { " " } else { "" }, - seconds, if seconds == 1 { "" } else { "s" })?; + write!( + f, + "{}{} second{}", + if have_written { " " } else { "" }, + seconds, + if seconds == 1 { "" } else { "s" } + )?; } Ok(()) } @@ -261,15 +309,21 @@ impl fmt::Display for Duration { impl ops::Add for Duration { type Output = Duration; - fn add(self, rhs: Duration) -> Duration { Duration(self.0 + rhs.0) } + fn add(self, rhs: Duration) -> Duration { + Duration(self.0 + rhs.0) + } } impl ops::AddAssign for Duration { - fn add_assign(&mut self, rhs: Duration) { self.0 += rhs.0 } + fn add_assign(&mut self, rhs: Duration) { + self.0 += rhs.0 + } } impl ops::SubAssign for Duration { - fn sub_assign(&mut self, rhs: Duration) { self.0 -= rhs.0 } + fn sub_assign(&mut self, rhs: Duration) { + self.0 -= rhs.0 + } } #[cfg(test)] @@ -280,17 +334,18 @@ mod tests { fn test_parse_time() { std::env::set_var("TZ", "America/Los_Angeles"); time::tzset(); + #[rustfmt::skip] let tests = &[ ("2006-01-02T15:04:05-07:00", 102261550050000), ("2006-01-02T15:04:05:00001-07:00", 102261550050001), ("2006-01-02T15:04:05-08:00", 102261874050000), - ("2006-01-02T15:04:05", 102261874050000), // implied -08:00 - ("2006-01-02T15:04", 102261873600000), // implied -08:00 - ("2006-01-02T15:04:05:00001", 102261874050001), // implied -08:00 + ("2006-01-02T15:04:05", 102261874050000), // implied -08:00 + ("2006-01-02T15:04", 102261873600000), // implied -08:00 + ("2006-01-02T15:04:05:00001", 102261874050001), // implied -08:00 ("2006-01-02T15:04:05-00:00", 102259282050000), ("2006-01-02T15:04:05Z", 102259282050000), - ("2006-01-02-08:00", 102256992000000), // implied -08:00 - ("2006-01-02", 102256992000000), // implied -08:00 + ("2006-01-02-08:00", 102256992000000), // implied -08:00 + ("2006-01-02", 102256992000000), // implied -08:00 ("2006-01-02Z", 102254400000000), ("102261550050000", 102261550050000), ]; @@ -303,7 +358,10 @@ mod tests { fn test_format_time() { std::env::set_var("TZ", "America/Los_Angeles"); time::tzset(); - assert_eq!("2006-01-02T15:04:05:00000-08:00", format!("{}", Time(102261874050000))); + assert_eq!( + "2006-01-02T15:04:05:00000-08:00", + format!("{}", Time(102261874050000)) + ); } #[test] diff --git a/server/db/auth.rs b/server/db/auth.rs index 2970674..52cce08 100644 --- a/server/db/auth.rs +++ b/server/db/auth.rs @@ -28,17 +28,17 @@ // You should have received a copy of the GNU General Public License // along with this program. If not, see . -use log::info; -use base::strutil; use crate::schema::Permissions; -use failure::{Error, bail, format_err}; +use base::strutil; +use failure::{bail, format_err, Error}; use fnv::FnvHashMap; use lazy_static::lazy_static; use libpasta; +use log::info; use parking_lot::Mutex; use protobuf::Message; use ring::rand::{SecureRandom, SystemRandom}; -use rusqlite::{Connection, Transaction, params}; +use rusqlite::{params, Connection, Transaction}; use std::collections::BTreeMap; use std::fmt; use std::net::IpAddr; @@ -54,8 +54,9 @@ lazy_static! { /// See also . /// Call via `testutil::init()`. pub(crate) fn set_test_config() { - *PASTA_CONFIG.lock() = - Arc::new(libpasta::Config::with_primitive(libpasta::primitives::Bcrypt::new(2))); + *PASTA_CONFIG.lock() = Arc::new(libpasta::Config::with_primitive( + libpasta::primitives::Bcrypt::new(2), + )); } enum UserFlag { @@ -91,8 +92,12 @@ impl User { } } - pub fn has_password(&self) -> bool { self.password_hash.is_some() } - fn disabled(&self) -> bool { (self.flags & UserFlag::Disabled as i32) != 0 } + pub fn has_password(&self) -> bool { + self.password_hash.is_some() + } + fn disabled(&self) -> bool { + (self.flags & UserFlag::Disabled as i32) != 0 + } } /// A change to a user. @@ -175,20 +180,18 @@ impl rusqlite::types::FromSql for FromSqlIpAddr { use rusqlite::types::ValueRef; match value { ValueRef::Null => Ok(FromSqlIpAddr(None)), - ValueRef::Blob(ref b) => { - match b.len() { - 4 => { - let mut buf = [0u8; 4]; - buf.copy_from_slice(b); - Ok(FromSqlIpAddr(Some(buf.into()))) - }, - 16 => { - let mut buf = [0u8; 16]; - buf.copy_from_slice(b); - Ok(FromSqlIpAddr(Some(buf.into()))) - }, - _ => Err(rusqlite::types::FromSqlError::InvalidType), + ValueRef::Blob(ref b) => match b.len() { + 4 => { + let mut buf = [0u8; 4]; + buf.copy_from_slice(b); + Ok(FromSqlIpAddr(Some(buf.into()))) } + 16 => { + let mut buf = [0u8; 16]; + buf.copy_from_slice(b); + Ok(FromSqlIpAddr(Some(buf.into()))) + } + _ => Err(rusqlite::types::FromSqlError::InvalidType), }, _ => Err(rusqlite::types::FromSqlError::InvalidType), } @@ -227,7 +230,7 @@ pub enum RevocationReason { #[derive(Debug, Default)] pub struct Session { user_id: i32, - flags: i32, // bitmask of SessionFlag enum values + flags: i32, // bitmask of SessionFlag enum values domain: Option>, description: Option, seed: Seed, @@ -236,7 +239,7 @@ pub struct Session { creation: Request, revocation: Request, - revocation_reason: Option, // see RevocationReason enum + revocation_reason: Option, // see RevocationReason enum revocation_reason_detail: Option, pub permissions: Permissions, @@ -259,7 +262,9 @@ impl Session { pub struct RawSessionId([u8; 48]); impl RawSessionId { - pub fn new() -> Self { RawSessionId([0u8; 48]) } + pub fn new() -> Self { + RawSessionId([0u8; 48]) + } pub fn decode_base64(input: &[u8]) -> Result { let mut s = RawSessionId::new(); @@ -279,11 +284,15 @@ impl RawSessionId { } impl AsRef<[u8]> for RawSessionId { - fn as_ref(&self) -> &[u8] { &self.0[..] } + fn as_ref(&self) -> &[u8] { + &self.0[..] + } } impl AsMut<[u8]> for RawSessionId { - fn as_mut(&mut self) -> &mut [u8] { &mut self.0[..] } + fn as_mut(&mut self) -> &mut [u8] { + &mut self.0[..] + } } impl fmt::Debug for RawSessionId { @@ -319,7 +328,11 @@ impl fmt::Debug for SessionHash { fn fmt(&self, f: &mut fmt::Formatter) -> Result<(), fmt::Error> { let mut buf = [0; 32]; self.encode_base64(&mut buf); - write!(f, "SessionHash(\"{}\")", ::std::str::from_utf8(&buf[..]).expect("base64 is UTF-8")) + write!( + f, + "SessionHash(\"{}\")", + ::std::str::from_utf8(&buf[..]).expect("base64 is UTF-8") + ) } } @@ -330,8 +343,9 @@ impl rusqlite::types::FromSql for Seed { fn column_result(value: rusqlite::types::ValueRef) -> rusqlite::types::FromSqlResult { let b = value.as_blob()?; if b.len() != 32 { - return Err(rusqlite::types::FromSqlError::Other( - Box::new(format_err!("expected a 32-byte seed").compat()))); + return Err(rusqlite::types::FromSqlError::Other(Box::new( + format_err!("expected a 32-byte seed").compat(), + ))); } let mut s = Seed::default(); s.0.copy_from_slice(b); @@ -363,7 +377,8 @@ impl State { sessions: FnvHashMap::default(), rand: ring::rand::SystemRandom::new(), }; - let mut stmt = conn.prepare(r#" + let mut stmt = conn.prepare( + r#" select id, username, @@ -375,24 +390,28 @@ impl State { permissions from user - "#)?; + "#, + )?; let mut rows = stmt.query(params![])?; while let Some(row) = rows.next()? { let id = row.get(0)?; let name: String = row.get(1)?; let mut permissions = Permissions::new(); permissions.merge_from_bytes(row.get_raw_checked(7)?.as_blob()?)?; - state.users_by_id.insert(id, User { + state.users_by_id.insert( id, - username: name.clone(), - flags: row.get(2)?, - password_hash: row.get(3)?, - password_id: row.get(4)?, - password_failure_count: row.get(5)?, - unix_uid: row.get(6)?, - dirty: false, - permissions, - }); + User { + id, + username: name.clone(), + flags: row.get(2)?, + password_hash: row.get(3)?, + password_id: row.get(4)?, + password_failure_count: row.get(5)?, + unix_uid: row.get(6)?, + dirty: false, + permissions, + }, + ); state.users_by_name.insert(name, id); } Ok(state) @@ -406,11 +425,18 @@ impl State { } } - pub fn users_by_id(&self) -> &BTreeMap { &self.users_by_id } + pub fn users_by_id(&self) -> &BTreeMap { + &self.users_by_id + } - fn update_user(&mut self, conn: &Connection, id: i32, change: UserChange) - -> Result<&User, Error> { - let mut stmt = conn.prepare_cached(r#" + fn update_user( + &mut self, + conn: &Connection, + id: i32, + change: UserChange, + ) -> Result<&User, Error> { + let mut stmt = conn.prepare_cached( + r#" update user set username = :username, @@ -422,7 +448,8 @@ impl State { permissions = :permissions where id = :id - "#)?; + "#, + )?; let e = self.users_by_id.entry(id); let e = match e { ::std::collections::btree_map::Entry::Vacant(_) => panic!("missing uid {}!", id), @@ -433,10 +460,13 @@ impl State { None => { let u = e.get(); (&u.password_hash, u.password_id, u.password_failure_count) - }, + } Some(h) => (h, e.get().password_id + 1, 0), }; - let permissions = change.permissions.write_to_bytes().expect("proto3->vec is infallible"); + let permissions = change + .permissions + .write_to_bytes() + .expect("proto3->vec is infallible"); stmt.execute_named(&[ (":username", &&change.username[..]), (":password_hash", phash), @@ -462,12 +492,17 @@ impl State { } fn add_user(&mut self, conn: &Connection, change: UserChange) -> Result<&User, Error> { - let mut stmt = conn.prepare_cached(r#" + let mut stmt = conn.prepare_cached( + r#" insert into user (username, password_hash, flags, unix_uid, permissions) values (:username, :password_hash, :flags, :unix_uid, :permissions) - "#)?; + "#, + )?; let password_hash = change.set_password_hash.unwrap_or(None); - let permissions = change.permissions.write_to_bytes().expect("proto3->vec is infallible"); + let permissions = change + .permissions + .write_to_bytes() + .expect("proto3->vec is infallible"); stmt.execute_named(&[ (":username", &&change.username[..]), (":password_hash", &password_hash), @@ -512,17 +547,30 @@ impl State { } pub fn get_user(&self, username: &str) -> Option<&User> { - self.users_by_name - .get(username) - .map(|id| self.users_by_id.get(id).expect("users_by_name implies users_by_id")) + self.users_by_name.get(username).map(|id| { + self.users_by_id + .get(id) + .expect("users_by_name implies users_by_id") + }) } - pub fn login_by_password(&mut self, conn: &Connection, req: Request, username: &str, - password: String, domain: Option>, session_flags: i32) - -> Result<(RawSessionId, &Session), Error> { - let id = self.users_by_name.get(username) + pub fn login_by_password( + &mut self, + conn: &Connection, + req: Request, + username: &str, + password: String, + domain: Option>, + session_flags: i32, + ) -> Result<(RawSessionId, &Session), Error> { + let id = self + .users_by_name + .get(username) .ok_or_else(|| format_err!("no such user {:?}", username))?; - let u = self.users_by_id.get_mut(id).expect("users_by_name implies users_by_id"); + let u = self + .users_by_id + .get_mut(id) + .expect("users_by_name implies users_by_id"); if u.disabled() { bail!("user {:?} is disabled", username); } @@ -537,7 +585,7 @@ impl State { u.dirty = true; u.password_failure_count += 1; bail!("incorrect password for user {:?}", username); - }, + } libpasta::HashUpdate::Verified(new_pwd) => new_pwd, } }; @@ -546,34 +594,67 @@ impl State { u.dirty = true; } let password_id = u.password_id; - State::make_session_int(&self.rand, conn, req, u, domain, Some(password_id), session_flags, - &mut self.sessions, u.permissions.clone()) + State::make_session_int( + &self.rand, + conn, + req, + u, + domain, + Some(password_id), + session_flags, + &mut self.sessions, + u.permissions.clone(), + ) } /// Makes a session directly (no password required). - pub fn make_session<'s>(&'s mut self, conn: &Connection, creation: Request, uid: i32, - domain: Option>, flags: i32, permissions: Permissions) - -> Result<(RawSessionId, &'s Session), Error> { - let u = self.users_by_id.get_mut(&uid).ok_or_else(|| format_err!("no such uid {:?}", uid))?; + pub fn make_session<'s>( + &'s mut self, + conn: &Connection, + creation: Request, + uid: i32, + domain: Option>, + flags: i32, + permissions: Permissions, + ) -> Result<(RawSessionId, &'s Session), Error> { + let u = self + .users_by_id + .get_mut(&uid) + .ok_or_else(|| format_err!("no such uid {:?}", uid))?; if u.disabled() { bail!("user is disabled"); } - State::make_session_int(&self.rand, conn, creation, u, domain, None, flags, - &mut self.sessions, permissions) + State::make_session_int( + &self.rand, + conn, + creation, + u, + domain, + None, + flags, + &mut self.sessions, + permissions, + ) } - fn make_session_int<'s>(rand: &SystemRandom, conn: &Connection, creation: Request, - user: &mut User, domain: Option>, - creation_password_id: Option, flags: i32, - sessions: &'s mut FnvHashMap, - permissions: Permissions) - -> Result<(RawSessionId, &'s Session), Error> { + fn make_session_int<'s>( + rand: &SystemRandom, + conn: &Connection, + creation: Request, + user: &mut User, + domain: Option>, + creation_password_id: Option, + flags: i32, + sessions: &'s mut FnvHashMap, + permissions: Permissions, + ) -> Result<(RawSessionId, &'s Session), Error> { let mut session_id = RawSessionId::new(); rand.fill(&mut session_id.0).unwrap(); let mut seed = [0u8; 32]; rand.fill(&mut seed).unwrap(); let hash = session_id.hash(); - let mut stmt = conn.prepare_cached(r#" + let mut stmt = conn.prepare_cached( + r#" insert into user_session (session_id_hash, user_id, seed, flags, domain, creation_password_id, creation_time_sec, creation_user_agent, creation_peer_addr, @@ -582,10 +663,13 @@ impl State { :creation_password_id, :creation_time_sec, :creation_user_agent, :creation_peer_addr, :permissions) - "#)?; + "#, + )?; let addr = creation.addr_buf(); let addr: Option<&[u8]> = addr.as_ref().map(|a| a.as_ref()); - let permissions_blob = permissions.write_to_bytes().expect("proto3->vec is infallible"); + let permissions_blob = permissions + .write_to_bytes() + .expect("proto3->vec is infallible"); stmt.execute_named(&[ (":session_id_hash", &&hash.0[..]), (":user_id", &user.id), @@ -615,8 +699,12 @@ impl State { Ok((session_id, session)) } - pub fn authenticate_session(&mut self, conn: &Connection, req: Request, hash: &SessionHash) - -> Result<(&Session, &User), Error> { + pub fn authenticate_session( + &mut self, + conn: &Connection, + req: Request, + hash: &SessionHash, + ) -> Result<(&Session, &User), Error> { let s = match self.sessions.entry(*hash) { ::std::collections::hash_map::Entry::Occupied(e) => e.into_mut(), ::std::collections::hash_map::Entry::Vacant(e) => e.insert(lookup_session(conn, hash)?), @@ -637,15 +725,21 @@ impl State { Ok((s, u)) } - pub fn revoke_session(&mut self, conn: &Connection, reason: RevocationReason, - detail: Option, req: Request, hash: &SessionHash) - -> Result<(), Error> { + pub fn revoke_session( + &mut self, + conn: &Connection, + reason: RevocationReason, + detail: Option, + req: Request, + hash: &SessionHash, + ) -> Result<(), Error> { let s = match self.sessions.entry(*hash) { ::std::collections::hash_map::Entry::Occupied(e) => e.into_mut(), ::std::collections::hash_map::Entry::Vacant(e) => e.insert(lookup_session(conn, hash)?), }; if s.revocation_reason.is_none() { - let mut stmt = conn.prepare(r#" + let mut stmt = conn.prepare( + r#" update user_session set revocation_time_sec = ?, @@ -655,7 +749,8 @@ impl State { revocation_reason_detail = ? where session_id_hash = ? - "#)?; + "#, + )?; let addr = req.addr_buf(); let addr: Option<&[u8]> = addr.as_ref().map(|a| a.as_ref()); stmt.execute(params![ @@ -677,15 +772,18 @@ impl State { /// The caller is expected to call `post_flush` afterward if the transaction is /// successfully committed. pub fn flush(&self, tx: &Transaction) -> Result<(), Error> { - let mut u_stmt = tx.prepare(r#" + let mut u_stmt = tx.prepare( + r#" update user set password_failure_count = :password_failure_count, password_hash = :password_hash where id = :id - "#)?; - let mut s_stmt = tx.prepare(r#" + "#, + )?; + let mut s_stmt = tx.prepare( + r#" update user_session set last_use_time_sec = :last_use_time_sec, @@ -694,12 +792,16 @@ impl State { use_count = :use_count where session_id_hash = :hash - "#)?; + "#, + )?; for (&id, u) in &self.users_by_id { if !u.dirty { continue; } - info!("flushing user with hash: {}", u.password_hash.as_ref().unwrap()); + info!( + "flushing user with hash: {}", + u.password_hash.as_ref().unwrap() + ); u_stmt.execute_named(&[ (":password_failure_count", &u.password_failure_count), (":password_hash", &u.password_hash), @@ -736,7 +838,8 @@ impl State { } fn lookup_session(conn: &Connection, hash: &SessionHash) -> Result { - let mut stmt = conn.prepare_cached(r#" + let mut stmt = conn.prepare_cached( + r#" select user_id, seed, @@ -761,7 +864,8 @@ fn lookup_session(conn: &Connection, hash: &SessionHash) -> Result Result, - files_to_trash: FnvHashSet<(i32, CompositeId)>, // (dir_id, composite_id) + files_to_trash: FnvHashSet<(i32, CompositeId)>, // (dir_id, composite_id) } pub fn run(conn: &mut rusqlite::Connection, opts: &Options) -> Result { @@ -65,7 +65,9 @@ pub fn run(conn: &mut rusqlite::Connection, opts: &Options) -> Result Result = FnvHashMap::default(); { - let mut dir_stmt = conn.prepare(r#" + let mut dir_stmt = conn.prepare( + r#" select d.id, d.path, d.uuid, d.last_complete_open_id, o.uuid from sample_file_dir d left join open o on (d.last_complete_open_id = o.id) - "#)?; - let mut garbage_stmt = conn.prepare_cached( - "select composite_id from garbage where sample_file_dir_id = ?")?; + "#, + )?; + let mut garbage_stmt = + conn.prepare_cached("select composite_id from garbage where sample_file_dir_id = ?")?; let mut rows = dir_stmt.query(params![])?; while let Some(row) = rows.next()? { let mut meta = schema::DirMeta::default(); @@ -131,8 +135,10 @@ pub fn run(conn: &mut rusqlite::Connection, opts: &Options) -> Result Result Result Result Result Result { video_samples, video_sync_samples, media_duration, - flags: if it.duration_90k == 0 { db::RecordingFlags::TrailingZero as i32 } else { 0 }, + flags: if it.duration_90k == 0 { + db::RecordingFlags::TrailingZero as i32 + } else { + 0 + }, }) } @@ -275,35 +294,53 @@ fn read_dir(d: &dir::SampleFileDir, opts: &Options) -> Result { let f = e.file_name(); match f.to_bytes() { b"." | b".." | b"meta" => continue, - _ => {}, + _ => {} }; let id = match dir::parse_id(f.to_bytes()) { Ok(id) => id, Err(_) => { - error!("sample file directory contains file {:?} which isn't an id", f); + error!( + "sample file directory contains file {:?} which isn't an id", + f + ); continue; } }; let len = if opts.compare_lens { nix::sys::stat::fstatat(fd, f, AtFlags::empty())?.st_size as u64 - } else { 0 }; + } else { + 0 + }; let stream = dir.entry(id.stream()).or_insert_with(Stream::default); - stream.recordings.entry(id.recording()).or_insert_with(Recording::default).file = Some(len); + stream + .recordings + .entry(id.recording()) + .or_insert_with(Recording::default) + .file = Some(len); } Ok(dir) } /// Looks through a known stream for errors. -fn compare_stream(conn: &rusqlite::Connection, dir_id: i32, stream_id: i32, opts: &Options, - mut stream: Stream, ctx: &mut Context) -> Result { +fn compare_stream( + conn: &rusqlite::Connection, + dir_id: i32, + stream_id: i32, + opts: &Options, + mut stream: Stream, + ctx: &mut Context, +) -> Result { let start = CompositeId::new(stream_id, 0); let end = CompositeId::new(stream_id, i32::max_value()); let mut printed_error = false; - let cum_recordings = stream.cum_recordings.expect("cum_recordings must be set on known stream"); + let cum_recordings = stream + .cum_recordings + .expect("cum_recordings must be set on known stream"); // recording row. { - let mut stmt = conn.prepare_cached(r#" + let mut stmt = conn.prepare_cached( + r#" select composite_id, flags, @@ -315,7 +352,8 @@ fn compare_stream(conn: &rusqlite::Connection, dir_id: i32, stream_id: i32, opts recording where composite_id between ? and ? - "#)?; + "#, + )?; let mut rows = stmt.query(params![start.0, end.0])?; while let Some(row) = rows.next()? { let id = CompositeId(row.get(0)?); @@ -326,15 +364,18 @@ fn compare_stream(conn: &rusqlite::Connection, dir_id: i32, stream_id: i32, opts video_samples: row.get(4)?, video_sync_samples: row.get(5)?, }; - stream.recordings.entry(id.recording()) - .or_insert_with(Recording::default) - .recording_row = Some(s); + stream + .recordings + .entry(id.recording()) + .or_insert_with(Recording::default) + .recording_row = Some(s); } } // recording_playback row. { - let mut stmt = conn.prepare_cached(r#" + let mut stmt = conn.prepare_cached( + r#" select composite_id, video_index @@ -342,7 +383,8 @@ fn compare_stream(conn: &rusqlite::Connection, dir_id: i32, stream_id: i32, opts recording_playback where composite_id between ? and ? - "#)?; + "#, + )?; let mut rows = stmt.query(params![start.0, end.0])?; while let Some(row) = rows.next()? { let id = CompositeId(row.get(0)?); @@ -357,30 +399,36 @@ fn compare_stream(conn: &rusqlite::Connection, dir_id: i32, stream_id: i32, opts ctx.files_to_trash.insert((dir_id, id)); } continue; - }, + } }; - stream.recordings.entry(id.recording()) - .or_insert_with(Recording::default) - .playback_row = Some(s); + stream + .recordings + .entry(id.recording()) + .or_insert_with(Recording::default) + .playback_row = Some(s); } } // recording_integrity row. { - let mut stmt = conn.prepare_cached(r#" + let mut stmt = conn.prepare_cached( + r#" select composite_id from recording_integrity where composite_id between ? and ? - "#)?; + "#, + )?; let mut rows = stmt.query(params![start.0, end.0])?; while let Some(row) = rows.next()? { let id = CompositeId(row.get(0)?); - stream.recordings.entry(id.recording()) - .or_insert_with(Recording::default) - .integrity_row = true; + stream + .recordings + .entry(id.recording()) + .or_insert_with(Recording::default) + .integrity_row = true; } } @@ -400,14 +448,15 @@ fn compare_stream(conn: &rusqlite::Connection, dir_id: i32, stream_id: i32, opts continue; } r - }, + } None => { if db_rows_expected { error!("Missing recording row for {}: {:#?}", id, recording); if opts.trash_orphan_sample_files { ctx.files_to_trash.insert((dir_id, id)); } - if opts.delete_orphan_rows { // also delete playback/integrity rows, if any. + if opts.delete_orphan_rows { + // also delete playback/integrity rows, if any. ctx.rows_to_delete.insert(id); } printed_error = true; @@ -419,38 +468,44 @@ fn compare_stream(conn: &rusqlite::Connection, dir_id: i32, stream_id: i32, opts printed_error = true; } continue; - }, + } }; match recording.playback_row { Some(ref p) => { if r != p { - error!("Recording {} summary doesn't match video_index: {:#?}", id, recording); + error!( + "Recording {} summary doesn't match video_index: {:#?}", + id, recording + ); printed_error = true; } - }, + } None => { error!("Recording {} missing playback row: {:#?}", id, recording); printed_error = true; if opts.trash_orphan_sample_files { ctx.files_to_trash.insert((dir_id, id)); } - if opts.delete_orphan_rows { // also delete recording/integrity rows, if any. + if opts.delete_orphan_rows { + // also delete recording/integrity rows, if any. ctx.rows_to_delete.insert(id); } - }, + } } match recording.file { - Some(len) => if opts.compare_lens && r.bytes != len { - error!("Recording {} length mismatch: {:#?}", id, recording); - printed_error = true; - }, + Some(len) => { + if opts.compare_lens && r.bytes != len { + error!("Recording {} length mismatch: {:#?}", id, recording); + printed_error = true; + } + } None => { error!("Recording {} missing file: {:#?}", id, recording); if opts.delete_orphan_rows { ctx.rows_to_delete.insert(id); } printed_error = true; - }, + } } } diff --git a/server/db/coding.rs b/server/db/coding.rs index 9b1fe7e..94f2f92 100644 --- a/server/db/coding.rs +++ b/server/db/coding.rs @@ -34,28 +34,35 @@ /// encoding](https://developers.google.com/protocol-buffers/docs/encoding#types). Uses the low bit /// to indicate signedness (1 = negative, 0 = non-negative). #[inline(always)] -pub fn zigzag32(i: i32) -> u32 { ((i << 1) as u32) ^ ((i >> 31) as u32) } +pub fn zigzag32(i: i32) -> u32 { + ((i << 1) as u32) ^ ((i >> 31) as u32) +} /// Zigzag-decodes to a signed integer. /// See `zigzag`. #[inline(always)] -pub fn unzigzag32(i: u32) -> i32 { ((i >> 1) as i32) ^ -((i & 1) as i32) } +pub fn unzigzag32(i: u32) -> i32 { + ((i >> 1) as i32) ^ -((i & 1) as i32) +} #[inline(always)] pub fn decode_varint32(data: &[u8], i: usize) -> Result<(u32, usize), ()> { // Unroll a few likely possibilities before going into the robust out-of-line loop. // This aids branch prediction. if data.len() > i && (data[i] & 0x80) == 0 { - return Ok((data[i] as u32, i+1)) - } else if data.len() > i + 1 && (data[i+1] & 0x80) == 0 { - return Ok((( (data[i] & 0x7f) as u32) | - (( data[i+1] as u32) << 7), - i+2)) - } else if data.len() > i + 2 && (data[i+2] & 0x80) == 0 { - return Ok((( (data[i] & 0x7f) as u32) | - (((data[i+1] & 0x7f) as u32) << 7) | - (( data[i+2] as u32) << 14), - i+3)) + return Ok((data[i] as u32, i + 1)); + } else if data.len() > i + 1 && (data[i + 1] & 0x80) == 0 { + return Ok(( + ((data[i] & 0x7f) as u32) | ((data[i + 1] as u32) << 7), + i + 2, + )); + } else if data.len() > i + 2 && (data[i + 2] & 0x80) == 0 { + return Ok(( + ((data[i] & 0x7f) as u32) + | (((data[i + 1] & 0x7f) as u32) << 7) + | ((data[i + 2] as u32) << 14), + i + 3, + )); } decode_varint32_slow(data, i) } @@ -67,11 +74,11 @@ fn decode_varint32_slow(data: &[u8], mut i: usize) -> Result<(u32, usize), ()> { let mut shift = 0; loop { if i == l { - return Err(()) + return Err(()); } let b = data[i]; if shift == 28 && (b & 0xf0) != 0 { - return Err(()) + return Err(()); } out |= ((b & 0x7f) as u32) << shift; shift += 7; @@ -87,27 +94,31 @@ pub fn append_varint32(i: u32, data: &mut Vec) { if i < 1u32 << 7 { data.push(i as u8); } else if i < 1u32 << 14 { - data.extend_from_slice(&[(( i & 0x7F) | 0x80) as u8, - (i >> 7) as u8]); + data.extend_from_slice(&[((i & 0x7F) | 0x80) as u8, (i >> 7) as u8]); } else if i < 1u32 << 21 { - data.extend_from_slice(&[(( i & 0x7F) | 0x80) as u8, - (((i >> 7) & 0x7F) | 0x80) as u8, - (i >> 14) as u8]); + data.extend_from_slice(&[ + ((i & 0x7F) | 0x80) as u8, + (((i >> 7) & 0x7F) | 0x80) as u8, + (i >> 14) as u8, + ]); } else if i < 1u32 << 28 { - data.extend_from_slice(&[(( i & 0x7F) | 0x80) as u8, - (((i >> 7) & 0x7F) | 0x80) as u8, - (((i >> 14) & 0x7F) | 0x80) as u8, - (i >> 21) as u8]); + data.extend_from_slice(&[ + ((i & 0x7F) | 0x80) as u8, + (((i >> 7) & 0x7F) | 0x80) as u8, + (((i >> 14) & 0x7F) | 0x80) as u8, + (i >> 21) as u8, + ]); } else { - data.extend_from_slice(&[(( i & 0x7F) | 0x80) as u8, - (((i >> 7) & 0x7F) | 0x80) as u8, - (((i >> 14) & 0x7F) | 0x80) as u8, - (((i >> 21) & 0x7F) | 0x80) as u8, - (i >> 28) as u8]); + data.extend_from_slice(&[ + ((i & 0x7F) | 0x80) as u8, + (((i >> 7) & 0x7F) | 0x80) as u8, + (((i >> 14) & 0x7F) | 0x80) as u8, + (((i >> 21) & 0x7F) | 0x80) as u8, + (i >> 28) as u8, + ]); } } - #[cfg(test)] mod tests { use super::*; @@ -119,12 +130,30 @@ mod tests { encoded: u32, } let tests = [ - Test{decoded: 0, encoded: 0}, - Test{decoded: -1, encoded: 1}, - Test{decoded: 1, encoded: 2}, - Test{decoded: -2, encoded: 3}, - Test{decoded: 2147483647, encoded: 4294967294}, - Test{decoded: -2147483648, encoded: 4294967295}, + Test { + decoded: 0, + encoded: 0, + }, + Test { + decoded: -1, + encoded: 1, + }, + Test { + decoded: 1, + encoded: 2, + }, + Test { + decoded: -2, + encoded: 3, + }, + Test { + decoded: 2147483647, + encoded: 4294967294, + }, + Test { + decoded: -2147483648, + encoded: 4294967295, + }, ]; for test in &tests { assert_eq!(test.encoded, zigzag32(test.decoded)); @@ -139,11 +168,26 @@ mod tests { encoded: &'static [u8], } let tests = [ - Test{decoded: 1, encoded: b"\x01"}, - Test{decoded: 257, encoded: b"\x81\x02"}, - Test{decoded: 49409, encoded: b"\x81\x82\x03"}, - Test{decoded: 8438017, encoded: b"\x81\x82\x83\x04"}, - Test{decoded: 1350615297, encoded: b"\x81\x82\x83\x84\x05"}, + Test { + decoded: 1, + encoded: b"\x01", + }, + Test { + decoded: 257, + encoded: b"\x81\x02", + }, + Test { + decoded: 49409, + encoded: b"\x81\x82\x03", + }, + Test { + decoded: 8438017, + encoded: b"\x81\x82\x83\x04", + }, + Test { + decoded: 1350615297, + encoded: b"\x81\x82\x83\x84\x05", + }, ]; for test in &tests { // Test encoding to an empty buffer. @@ -161,13 +205,17 @@ mod tests { assert_eq!(out, buf); // Test decoding from the beginning of the string. - assert_eq!((test.decoded, test.encoded.len()), - decode_varint32(test.encoded, 0).unwrap()); + assert_eq!( + (test.decoded, test.encoded.len()), + decode_varint32(test.encoded, 0).unwrap() + ); // ...and from the middle of a buffer. buf.push(b'x'); - assert_eq!((test.decoded, test.encoded.len() + 1), - decode_varint32(&buf, 1).unwrap()); + assert_eq!( + (test.decoded, test.encoded.len() + 1), + decode_varint32(&buf, 1).unwrap() + ); } } @@ -180,7 +228,6 @@ mod tests { b"\x80\x80", b"\x80\x80\x80", b"\x80\x80\x80\x80", - // int32 overflows b"\x80\x80\x80\x80\x80", b"\x80\x80\x80\x80\x80\x00", diff --git a/server/db/compare.rs b/server/db/compare.rs index 386d704..3332f23 100644 --- a/server/db/compare.rs +++ b/server/db/compare.rs @@ -79,7 +79,8 @@ impl std::fmt::Display for IndexColumn { /// Returns a sorted vec of table names in the given connection. fn get_tables(c: &rusqlite::Connection) -> Result, rusqlite::Error> { - c.prepare(r#" + c.prepare( + r#" select name from @@ -88,66 +89,86 @@ fn get_tables(c: &rusqlite::Connection) -> Result, rusqlite::Error> type = 'table' and name not like 'sqlite_%' order by name - "#)? - .query_map(params![], |r| r.get(0))? - .collect() + "#, + )? + .query_map(params![], |r| r.get(0))? + .collect() } /// Returns a vec of columns in the given table. -fn get_table_columns(c: &rusqlite::Connection, table: &str) - -> Result, rusqlite::Error> { +fn get_table_columns( + c: &rusqlite::Connection, + table: &str, +) -> Result, rusqlite::Error> { // Note that placeholders aren't allowed for these pragmas. Just assume sane table names // (no escaping). "select * from pragma_..." syntax would be nicer but requires SQLite // 3.16.0 (2017-01-02). Ubuntu 16.04 Xenial (still used on Travis CI) has an older SQLite. c.prepare(&format!("pragma table_info(\"{}\")", table))? - .query_map(params![], |r| Ok(Column { - cid: r.get(0)?, - name: r.get(1)?, - type_: r.get(2)?, - notnull: r.get(3)?, - dflt_value: r.get(4)?, - pk: r.get(5)?, - }))? - .collect() + .query_map(params![], |r| { + Ok(Column { + cid: r.get(0)?, + name: r.get(1)?, + type_: r.get(2)?, + notnull: r.get(3)?, + dflt_value: r.get(4)?, + pk: r.get(5)?, + }) + })? + .collect() } /// Returns a vec of indices associated with the given table. fn get_indices(c: &rusqlite::Connection, table: &str) -> Result, rusqlite::Error> { // See note at get_tables_columns about placeholders. c.prepare(&format!("pragma index_list(\"{}\")", table))? - .query_map(params![], |r| Ok(Index { - seq: r.get(0)?, - name: r.get(1)?, - unique: r.get(2)?, - origin: r.get(3)?, - partial: r.get(4)?, - }))? - .collect() + .query_map(params![], |r| { + Ok(Index { + seq: r.get(0)?, + name: r.get(1)?, + unique: r.get(2)?, + origin: r.get(3)?, + partial: r.get(4)?, + }) + })? + .collect() } /// Returns a vec of all the columns in the given index. -fn get_index_columns(c: &rusqlite::Connection, index: &str) - -> Result, rusqlite::Error> { +fn get_index_columns( + c: &rusqlite::Connection, + index: &str, +) -> Result, rusqlite::Error> { // See note at get_tables_columns about placeholders. c.prepare(&format!("pragma index_info(\"{}\")", index))? - .query_map(params![], |r| Ok(IndexColumn { - seqno: r.get(0)?, - cid: r.get(1)?, - name: r.get(2)?, - }))? - .collect() + .query_map(params![], |r| { + Ok(IndexColumn { + seqno: r.get(0)?, + cid: r.get(1)?, + name: r.get(2)?, + }) + })? + .collect() } -pub fn get_diffs(n1: &str, c1: &rusqlite::Connection, n2: &str, c2: &rusqlite::Connection) - -> Result, Error> { +pub fn get_diffs( + n1: &str, + c1: &rusqlite::Connection, + n2: &str, + c2: &rusqlite::Connection, +) -> Result, Error> { let mut diffs = String::new(); // Compare table list. let tables1 = get_tables(c1)?; let tables2 = get_tables(c2)?; if tables1 != tables2 { - write!(&mut diffs, "table list mismatch, {} vs {}:\n{}", - n1, n2, diff_slice(&tables1, &tables2))?; + write!( + &mut diffs, + "table list mismatch, {} vs {}:\n{}", + n1, + n2, + diff_slice(&tables1, &tables2) + )?; } // Compare columns and indices for each table. @@ -155,8 +176,14 @@ pub fn get_diffs(n1: &str, c1: &rusqlite::Connection, n2: &str, c2: &rusqlite::C let columns1 = get_table_columns(c1, &t)?; let columns2 = get_table_columns(c2, &t)?; if columns1 != columns2 { - write!(&mut diffs, "table {:?} column, {} vs {}:\n{}", - t, n1, n2, diff_slice(&columns1, &columns2))?; + write!( + &mut diffs, + "table {:?} column, {} vs {}:\n{}", + t, + n1, + n2, + diff_slice(&columns1, &columns2) + )?; } let mut indices1 = get_indices(c1, &t)?; @@ -164,16 +191,29 @@ pub fn get_diffs(n1: &str, c1: &rusqlite::Connection, n2: &str, c2: &rusqlite::C indices1.sort_by(|a, b| a.name.cmp(&b.name)); indices2.sort_by(|a, b| a.name.cmp(&b.name)); if indices1 != indices2 { - write!(&mut diffs, "table {:?} indices, {} vs {}:\n{}", - t, n1, n2, diff_slice(&indices1, &indices2))?; + write!( + &mut diffs, + "table {:?} indices, {} vs {}:\n{}", + t, + n1, + n2, + diff_slice(&indices1, &indices2) + )?; } for i in &indices1 { let ic1 = get_index_columns(c1, &i.name)?; let ic2 = get_index_columns(c2, &i.name)?; if ic1 != ic2 { - write!(&mut diffs, "table {:?} index {:?} columns {} vs {}:\n{}", - t, i, n1, n2, diff_slice(&ic1, &ic2))?; + write!( + &mut diffs, + "table {:?} index {:?} columns {} vs {}:\n{}", + t, + i, + n1, + n2, + diff_slice(&ic1, &ic2) + )?; } } } diff --git a/server/db/db.rs b/server/db/db.rs index d9b4122..2fae598 100644 --- a/server/db/db.rs +++ b/server/db/db.rs @@ -52,30 +52,30 @@ //! A list of mutations is built up in-memory and occasionally flushed to reduce SSD write //! cycles. -use base::clock::{self, Clocks}; -use base::strutil::encode_size; use crate::auth; use crate::dir; use crate::raw; use crate::recording::{self, TIME_UNITS_PER_SEC}; use crate::schema; use crate::signal; -use failure::{Error, bail, format_err}; +use base::clock::{self, Clocks}; +use base::strutil::encode_size; +use failure::{bail, format_err, Error}; use fnv::{FnvHashMap, FnvHashSet}; use hashlink::LinkedHashMap; use itertools::Itertools; use log::{error, info, trace}; -use parking_lot::{Mutex,MutexGuard}; +use parking_lot::{Mutex, MutexGuard}; use rusqlite::{named_params, params}; use smallvec::SmallVec; use std::cell::RefCell; +use std::cmp; use std::collections::{BTreeMap, VecDeque}; use std::convert::TryInto; -use std::cmp; use std::fmt::Write as _; use std::io::Write; -use std::ops::Range; use std::mem; +use std::ops::Range; use std::str; use std::string::String; use std::sync::Arc; @@ -154,7 +154,6 @@ pub struct VideoSampleEntry { pub id: i32, // Fields matching VideoSampleEntryToInsert below. - pub data: Vec, pub rfc6381_codec: String, pub width: u16, @@ -213,13 +212,14 @@ pub struct ListAggregatedRecordingsRow { pub growing: bool, } -impl ListAggregatedRecordingsRow { fn from(row: ListRecordingsRow) -> Self { +impl ListAggregatedRecordingsRow { + fn from(row: ListRecordingsRow) -> Self { let recording_id = row.id.recording(); let uncommitted = (row.flags & RecordingFlags::Uncommitted as i32) != 0; let growing = (row.flags & RecordingFlags::Growing as i32) != 0; ListAggregatedRecordingsRow { - time: row.start .. recording::Time(row.start.0 + row.wall_duration_90k as i64), - ids: recording_id .. recording_id+1, + time: row.start..recording::Time(row.start.0 + row.wall_duration_90k as i64), + ids: recording_id..recording_id + 1, video_samples: row.video_samples as i64, video_sync_samples: row.video_sync_samples as i64, sample_file_bytes: row.sample_file_bytes as i64, @@ -227,7 +227,11 @@ impl ListAggregatedRecordingsRow { fn from(row: ListRecordingsRow) -> Self { stream_id: row.id.stream(), run_start_id: recording_id - row.run_offset, open_id: row.open_id, - first_uncommitted: if uncommitted { Some(recording_id) } else { None }, + first_uncommitted: if uncommitted { + Some(recording_id) + } else { + None + }, growing, } } @@ -262,7 +266,7 @@ pub struct RecordingToInsert { /// Filled in by `add_recording`. pub prev_runs: i32, - pub wall_duration_90k: i32, // a recording::Duration, but guaranteed to fit in i32. + pub wall_duration_90k: i32, // a recording::Duration, but guaranteed to fit in i32. pub media_duration_90k: i32, pub local_time_delta: recording::Duration, pub video_samples: i32, @@ -291,7 +295,6 @@ impl RecordingToInsert { } } - /// A row used in `raw::list_oldest_recordings` and `db::delete_oldest_recordings`. #[derive(Copy, Clone, Debug)] pub(crate) struct ListOldestRecordingsRow { @@ -314,7 +317,7 @@ impl StreamDayKey { pub fn bounds(&self) -> Range { let mut my_tm = time::strptime(self.as_ref(), "%Y-%m-%d").expect("days must be parseable"); - my_tm.tm_utcoff = 1; // to the time crate, values != 0 mean local time. + my_tm.tm_utcoff = 1; // to the time crate, values != 0 mean local time. my_tm.tm_isdst = -1; let start = recording::Time(my_tm.to_timespec().sec * recording::TIME_UNITS_PER_SEC); my_tm.tm_hour = 0; @@ -322,12 +325,14 @@ impl StreamDayKey { my_tm.tm_sec = 0; my_tm.tm_mday += 1; let end = recording::Time(my_tm.to_timespec().sec * recording::TIME_UNITS_PER_SEC); - start .. end + start..end } } impl AsRef for StreamDayKey { - fn as_ref(&self) -> &str { str::from_utf8(&self.0[..]).expect("days are always UTF-8") } + fn as_ref(&self) -> &str { + str::from_utf8(&self.0[..]).expect("days are always UTF-8") + } } /// In-memory state about a particular camera on a particular day. @@ -366,10 +371,11 @@ impl SampleFileDir { /// /// Use `LockedDatabase::open_sample_file_dirs` prior to calling this method. pub fn get(&self) -> Result, Error> { - Ok(self.dir - .as_ref() - .ok_or_else(|| format_err!("sample file dir {} is closed", self.id))? - .clone()) + Ok(self + .dir + .as_ref() + .ok_or_else(|| format_err!("sample file dir {} is closed", self.id))? + .clone()) } /// Returns expected existing metadata when opening this directory. @@ -386,8 +392,8 @@ impl SampleFileDir { } } -pub use crate::auth::Request; pub use crate::auth::RawSessionId; +pub use crate::auth::Request; pub use crate::auth::Session; pub use crate::auth::User; pub use crate::auth::UserChange; @@ -406,7 +412,10 @@ pub struct Camera { } #[derive(Copy, Clone, Debug, Eq, PartialEq)] -pub enum StreamType { MAIN, SUB } +pub enum StreamType { + MAIN, + SUB, +} impl StreamType { pub fn from_index(i: usize) -> Option { @@ -557,11 +566,16 @@ pub struct CameraChange { /// Adds non-zero `delta` to the day represented by `day` in the map `m`. /// Inserts a map entry if absent; removes the entry if it has 0 entries on exit. -fn adjust_day(day: StreamDayKey, delta: StreamDayValue, - m: &mut BTreeMap) { +fn adjust_day( + day: StreamDayKey, + delta: StreamDayValue, + m: &mut BTreeMap, +) { use ::std::collections::btree_map::Entry; match m.entry(day) { - Entry::Vacant(e) => { e.insert(delta); }, + Entry::Vacant(e) => { + e.insert(delta); + } Entry::Occupied(mut e) => { let v = e.get_mut(); v.recordings += delta.recordings; @@ -569,7 +583,7 @@ fn adjust_day(day: StreamDayKey, delta: StreamDayValue, if v.recordings == 0 { e.remove_entry(); } - }, + } } } @@ -579,14 +593,23 @@ fn adjust_day(day: StreamDayKey, delta: StreamDayValue, /// /// This function swallows/logs date formatting errors because they shouldn't happen and there's /// not much that can be done about them. (The database operation has already gone through.) -fn adjust_days(r: Range, sign: i64, - m: &mut BTreeMap) { +fn adjust_days( + r: Range, + sign: i64, + m: &mut BTreeMap, +) { // Find first day key. - let mut my_tm = time::at(time::Timespec{sec: r.start.unix_seconds(), nsec: 0}); + let mut my_tm = time::at(time::Timespec { + sec: r.start.unix_seconds(), + nsec: 0, + }); let day = match StreamDayKey::new(my_tm) { Ok(d) => d, Err(ref e) => { - error!("Unable to fill first day key from {:?}: {}; will ignore.", my_tm, e); + error!( + "Unable to fill first day key from {:?}: {}; will ignore.", + my_tm, e + ); return; } }; @@ -618,7 +641,10 @@ fn adjust_days(r: Range, sign: i64, let day = match StreamDayKey::new(my_tm) { Ok(d) => d, Err(ref e) => { - error!("Unable to fill second day key from {:?}: {}; will ignore.", my_tm, e); + error!( + "Unable to fill second day key from {:?}: {}; will ignore.", + my_tm, e + ); return; } }; @@ -633,8 +659,8 @@ impl Stream { /// Adds a single fully committed recording with the given properties to the in-memory state. fn add_recording(&mut self, r: Range, sample_file_bytes: i32) { self.range = Some(match self.range { - Some(ref e) => cmp::min(e.start, r.start) .. cmp::max(e.end, r.end), - None => r.start .. r.end, + Some(ref e) => cmp::min(e.start, r.start)..cmp::max(e.end, r.end), + None => r.start..r.end, }); self.duration += r.end - r.start; self.sample_file_bytes += sample_file_bytes as i64; @@ -647,19 +673,29 @@ impl Stream { let mut days = self.committed_days.clone(); for u in &self.uncommitted { let l = u.lock(); - adjust_days(l.start .. l.start + recording::Duration(i64::from(l.wall_duration_90k)), - 1, &mut days); + adjust_days( + l.start..l.start + recording::Duration(i64::from(l.wall_duration_90k)), + 1, + &mut days, + ); } days } } /// Initializes the recordings associated with the given camera. -fn init_recordings(conn: &mut rusqlite::Connection, stream_id: i32, camera: &Camera, - stream: &mut Stream) - -> Result<(), Error> { - info!("Loading recordings for camera {} stream {:?}", camera.short_name, stream.type_); - let mut stmt = conn.prepare(r#" +fn init_recordings( + conn: &mut rusqlite::Connection, + stream_id: i32, + camera: &Camera, + stream: &mut Stream, +) -> Result<(), Error> { + info!( + "Loading recordings for camera {} stream {:?}", + camera.short_name, stream.type_ + ); + let mut stmt = conn.prepare( + r#" select recording.start_time_90k, recording.wall_duration_90k, @@ -668,17 +704,21 @@ fn init_recordings(conn: &mut rusqlite::Connection, stream_id: i32, camera: &Cam recording where stream_id = :stream_id - "#)?; - let mut rows = stmt.query_named(named_params!{":stream_id": stream_id})?; + "#, + )?; + let mut rows = stmt.query_named(named_params! {":stream_id": stream_id})?; let mut i = 0; while let Some(row) = rows.next()? { let start = recording::Time(row.get(0)?); let duration = recording::Duration(row.get(1)?); let bytes = row.get(2)?; - stream.add_recording(start .. start + duration, bytes); + stream.add_recording(start..start + duration, bytes); i += 1; } - info!("Loaded {} recordings for camera {} stream {:?}", i, camera.short_name, stream.type_); + info!( + "Loaded {} recordings for camera {} stream {:?}", + i, camera.short_name, stream.type_ + ); Ok(()) } @@ -700,7 +740,7 @@ pub struct LockedDatabase { sample_file_dirs_by_id: BTreeMap, cameras_by_id: BTreeMap, streams_by_id: BTreeMap, - cameras_by_uuid: BTreeMap, // values are ids. + cameras_by_uuid: BTreeMap, // values are ids. video_sample_entries_by_id: BTreeMap>, video_index_cache: RefCell, fnv::FnvBuildHasher>>, on_flush: Vec>, @@ -721,8 +761,12 @@ impl CompositeId { CompositeId((stream_id as i64) << 32 | recording_id as i64) } - pub fn stream(self) -> i32 { (self.0 >> 32) as i32 } - pub fn recording(self) -> i32 { self.0 as i32 } + pub fn stream(self) -> i32 { + (self.0 >> 32) as i32 + } + pub fn recording(self) -> i32 { + self.0 as i32 + } } impl ::std::fmt::Display for CompositeId { @@ -741,9 +785,13 @@ struct StreamStateChanger { impl StreamStateChanger { /// Performs the database updates (guarded by the given transaction) and returns the state /// change to be applied on successful commit. - fn new(tx: &rusqlite::Transaction, camera_id: i32, existing: Option<&Camera>, - streams_by_id: &BTreeMap, change: &mut CameraChange) - -> Result { + fn new( + tx: &rusqlite::Transaction, + camera_id: i32, + existing: Option<&Camera>, + streams_by_id: &BTreeMap, + change: &mut CameraChange, + ) -> Result { let mut sids = [None; 2]; let mut streams = Vec::with_capacity(2); let existing_streams = existing.map(|e| e.streams).unwrap_or_default(); @@ -754,25 +802,37 @@ impl StreamStateChanger { let s = streams_by_id.get(&sid).unwrap(); if s.range.is_some() { have_data = true; - if let (Some(d), false) = (s.sample_file_dir_id, - s.sample_file_dir_id == sc.sample_file_dir_id) { - bail!("can't change sample_file_dir_id {:?}->{:?} for non-empty stream {}", - d, sc.sample_file_dir_id, sid); + if let (Some(d), false) = ( + s.sample_file_dir_id, + s.sample_file_dir_id == sc.sample_file_dir_id, + ) { + bail!( + "can't change sample_file_dir_id {:?}->{:?} for non-empty stream {}", + d, + sc.sample_file_dir_id, + sid + ); } } - if !have_data && sc.rtsp_url.is_empty() && sc.sample_file_dir_id.is_none() && - !sc.record { + if !have_data + && sc.rtsp_url.is_empty() + && sc.sample_file_dir_id.is_none() + && !sc.record + { // Delete stream. - let mut stmt = tx.prepare_cached(r#" + let mut stmt = tx.prepare_cached( + r#" delete from stream where id = ? - "#)?; + "#, + )?; if stmt.execute(params![sid])? != 1 { bail!("missing stream {}", sid); } streams.push((sid, None)); } else { // Update stream. - let mut stmt = tx.prepare_cached(r#" + let mut stmt = tx.prepare_cached( + r#" update stream set rtsp_url = :rtsp_url, record = :record, @@ -780,8 +840,9 @@ impl StreamStateChanger { sample_file_dir_id = :sample_file_dir_id where id = :id - "#)?; - let rows = stmt.execute_named(named_params!{ + "#, + )?; + let rows = stmt.execute_named(named_params! { ":rtsp_url": &sc.rtsp_url, ":record": sc.record, ":flush_if_sec": sc.flush_if_sec, @@ -801,15 +862,17 @@ impl StreamStateChanger { continue; } // Insert stream. - let mut stmt = tx.prepare_cached(r#" + let mut stmt = tx.prepare_cached( + r#" insert into stream (camera_id, sample_file_dir_id, type, rtsp_url, record, retain_bytes, flush_if_sec, cum_recordings, cum_media_duration_90k, cum_runs) values (:camera_id, :sample_file_dir_id, :type, :rtsp_url, :record, 0, :flush_if_sec, 0, 0, 0) - "#)?; - stmt.execute_named(named_params!{ + "#, + )?; + stmt.execute_named(named_params! { ":camera_id": camera_id, ":sample_file_dir_id": sc.sample_file_dir_id, ":type": type_.as_str(), @@ -823,10 +886,7 @@ impl StreamStateChanger { streams.push((id, Some((camera_id, type_, sc)))); } } - Ok(StreamStateChanger { - sids, - streams, - }) + Ok(StreamStateChanger { sids, streams }) } /// Applies the change to the given `streams_by_id`. The caller is expected to set @@ -862,16 +922,18 @@ impl StreamStateChanger { synced_recordings: 0, on_live_segment: Vec::new(), }); - }, - (Entry::Vacant(_), None) => {}, + } + (Entry::Vacant(_), None) => {} (Entry::Occupied(e), Some((_, _, sc))) => { let e = e.into_mut(); e.sample_file_dir_id = sc.sample_file_dir_id; e.rtsp_url = sc.rtsp_url; e.record = sc.record; e.flush_if_sec = sc.flush_if_sec; - }, - (Entry::Occupied(e), None) => { e.remove(); }, + } + (Entry::Occupied(e), None) => { + e.remove(); + } }; } self.sids @@ -887,13 +949,17 @@ pub struct RetentionChange { impl LockedDatabase { /// Returns an immutable view of the cameras by id. - pub fn cameras_by_id(&self) -> &BTreeMap { &self.cameras_by_id } + pub fn cameras_by_id(&self) -> &BTreeMap { + &self.cameras_by_id + } pub fn sample_file_dirs_by_id(&self) -> &BTreeMap { &self.sample_file_dirs_by_id } /// Returns the number of completed database flushes since startup. - pub fn flushes(&self) -> usize { self.flush_count } + pub fn flushes(&self) -> usize { + self.flush_count + } /// Adds a placeholder for an uncommitted recording. /// @@ -906,25 +972,30 @@ impl LockedDatabase { /// synced and committed) won't change. /// /// This fills the `prev_media_duration` and `prev_runs` fields. - pub(crate) fn add_recording(&mut self, stream_id: i32, mut r: RecordingToInsert) - -> Result<(CompositeId, Arc>), Error> { + pub(crate) fn add_recording( + &mut self, + stream_id: i32, + mut r: RecordingToInsert, + ) -> Result<(CompositeId, Arc>), Error> { let stream = match self.streams_by_id.get_mut(&stream_id) { None => bail!("no such stream {}", stream_id), Some(s) => s, }; - let id = CompositeId::new(stream_id, - stream.cum_recordings + (stream.uncommitted.len() as i32)); + let id = CompositeId::new( + stream_id, + stream.cum_recordings + (stream.uncommitted.len() as i32), + ); match stream.uncommitted.back() { Some(s) => { let l = s.lock(); r.prev_media_duration = l.prev_media_duration + recording::Duration(l.wall_duration_90k.into()); r.prev_runs = l.prev_runs + if l.run_offset == 0 { 1 } else { 0 }; - }, + } None => { r.prev_media_duration = stream.cum_media_duration; r.prev_runs = stream.cum_runs; - }, + } }; let recording = Arc::new(Mutex::new(r)); stream.uncommitted.push_back(Arc::clone(&recording)); @@ -940,8 +1011,12 @@ impl LockedDatabase { }; let next_unsynced = stream.cum_recordings + (stream.synced_recordings as i32); if id.recording() != next_unsynced { - bail!("can't sync {} when next unsynced recording is {} (next unflushed is {})", - id, next_unsynced, stream.cum_recordings); + bail!( + "can't sync {} when next unsynced recording is {} (next unflushed is {})", + id, + next_unsynced, + stream.cum_recordings + ); } if stream.synced_recordings == stream.uncommitted.len() { bail!("can't sync un-added recording {}", id); @@ -954,8 +1029,11 @@ impl LockedDatabase { Ok(()) } - pub(crate) fn delete_garbage(&mut self, dir_id: i32, ids: &mut Vec) - -> Result<(), Error> { + pub(crate) fn delete_garbage( + &mut self, + dir_id: i32, + ids: &mut Vec, + ) -> Result<(), Error> { let dir = match self.sample_file_dirs_by_id.get_mut(&dir_id) { None => bail!("no such dir {}", dir_id), Some(d) => d, @@ -977,8 +1055,11 @@ impl LockedDatabase { /// Registers a callback to run on every live segment immediately after it's recorded. /// The callback is run with the database lock held, so it must not call back into the database /// or block. The callback should return false to unregister. - pub fn watch_live(&mut self, stream_id: i32, cb: Box bool + Send>) - -> Result<(), Error> { + pub fn watch_live( + &mut self, + stream_id: i32, + cb: Box bool + Send>, + ) -> Result<(), Error> { let s = match self.streams_by_id.get_mut(&stream_id) { None => bail!("no such stream {}", stream_id), Some(s) => s, @@ -1017,8 +1098,8 @@ impl LockedDatabase { Some(o) => o, }; let tx = self.conn.transaction()?; - let mut new_ranges = FnvHashMap::with_capacity_and_hasher(self.streams_by_id.len(), - Default::default()); + let mut new_ranges = + FnvHashMap::with_capacity_and_hasher(self.streams_by_id.len(), Default::default()); { let mut stmt = tx.prepare_cached(UPDATE_STREAM_COUNTERS_SQL)?; for (&stream_id, s) in &self.streams_by_id { @@ -1028,13 +1109,17 @@ impl LockedDatabase { for i in 0..s.synced_recordings { let l = s.uncommitted[i].lock(); raw::insert_recording( - &tx, o, CompositeId::new(stream_id, s.cum_recordings + i as i32), &l)?; + &tx, + o, + CompositeId::new(stream_id, s.cum_recordings + i as i32), + &l, + )?; new_duration += i64::from(l.wall_duration_90k); new_runs += if l.run_offset == 0 { 1 } else { 0 }; } if s.synced_recordings > 0 { new_ranges.entry(stream_id).or_insert(None); - stmt.execute_named(named_params!{ + stmt.execute_named(named_params! { ":stream_id": stream_id, ":cum_recordings": s.cum_recordings + s.synced_recordings as i32, ":cum_media_duration_90k": s.cum_media_duration.0 + new_duration, @@ -1056,10 +1141,16 @@ impl LockedDatabase { // oldest recordings for the stream. let start = CompositeId::new(stream_id, 0); let end = CompositeId(l.id.0 + 1); - let n = raw::delete_recordings(&tx, dir, start .. end)? as usize; + let n = raw::delete_recordings(&tx, dir, start..end)? as usize; if n != s.to_delete.len() { - bail!("Found {} rows in {} .. {}, expected {}: {:?}", - n, start, end, s.to_delete.len(), &s.to_delete); + bail!( + "Found {} rows in {} .. {}, expected {}: {:?}", + n, + start, + end, + s.to_delete.len(), + &s.to_delete + ); } } } @@ -1072,7 +1163,8 @@ impl LockedDatabase { } { let mut stmt = tx.prepare_cached( - r"update open set duration_90k = ?, end_time_90k = ? where id = ?")?; + r"update open set duration_90k = ?, end_time_90k = ? where id = ?", + )?; let rows = stmt.execute(params![ (recording::Time::new(clocks.monotonic()) - self.open_monotonic).0, recording::Time::new(clocks.realtime()).0, @@ -1088,9 +1180,9 @@ impl LockedDatabase { #[derive(Default)] struct DirLog { - added: SmallVec::<[CompositeId; 32]>, - deleted: SmallVec::<[CompositeId; 32]>, - gced: SmallVec::<[CompositeId; 32]>, + added: SmallVec<[CompositeId; 32]>, + deleted: SmallVec<[CompositeId; 32]>, + gced: SmallVec<[CompositeId; 32]>, added_bytes: i64, deleted_bytes: i64, } @@ -1099,7 +1191,11 @@ impl LockedDatabase { // Process delete_garbage. for (&id, dir) in &mut self.sample_file_dirs_by_id { if !dir.garbage_unlinked.is_empty() { - dir_logs.entry(id).or_default().gced.extend(dir.garbage_unlinked.drain(..)); + dir_logs + .entry(id) + .or_default() + .gced + .extend(dir.garbage_unlinked.drain(..)); } } @@ -1121,7 +1217,7 @@ impl LockedDatabase { dir.garbage_needs_unlink.insert(row.id); let d = recording::Duration(i64::from(row.wall_duration_90k)); s.duration -= d; - adjust_days(row.start .. row.start + d, -1, &mut s.committed_days); + adjust_days(row.start..row.start + d, -1, &mut s.committed_days); } // Process add_recordings. @@ -1131,7 +1227,8 @@ impl LockedDatabase { log.added.reserve(s.synced_recordings); for _ in 0..s.synced_recordings { let u = s.uncommitted.pop_front().unwrap(); - log.added.push(CompositeId::new(stream_id, s.cum_recordings)); + log.added + .push(CompositeId::new(stream_id, s.cum_recordings)); let l = u.lock(); s.cum_recordings += 1; let wall_dur = recording::Duration(l.wall_duration_90k.into()); @@ -1139,7 +1236,7 @@ impl LockedDatabase { s.cum_media_duration += media_dur; s.cum_runs += if l.run_offset == 0 { 1 } else { 0 }; let end = l.start + wall_dur; - s.add_recording(l.start .. end, l.sample_file_bytes); + s.add_recording(l.start..end, l.sample_file_bytes); } s.synced_recordings = 0; @@ -1152,13 +1249,21 @@ impl LockedDatabase { let mut log_msg = String::with_capacity(256); for (&dir_id, log) in &dir_logs { let dir = self.sample_file_dirs_by_id.get(&dir_id).unwrap(); - write!(&mut log_msg, - "\n{}: added {}B in {} recordings ({}), deleted {}B in {} ({}), \ + write!( + &mut log_msg, + "\n{}: added {}B in {} recordings ({}), deleted {}B in {} ({}), \ GCed {} recordings ({}).", - &dir.path, &encode_size(log.added_bytes), log.added.len(), - log.added.iter().join(", "), &encode_size(log.deleted_bytes), log.deleted.len(), - log.deleted.iter().join(", "), log.gced.len(), - log.gced.iter().join(", ")).unwrap(); + &dir.path, + &encode_size(log.added_bytes), + log.added.len(), + log.added.iter().join(", "), + &encode_size(log.deleted_bytes), + log.deleted.len(), + log.deleted.iter().join(", "), + log.gced.len(), + log.gced.iter().join(", ") + ) + .unwrap(); } if log_msg.is_empty() { log_msg.push_str(" no recording changes"); @@ -1180,7 +1285,7 @@ impl LockedDatabase { // handlers given that it didn't add them. pub fn clear_on_flush(&mut self) { self.on_flush.clear(); - } + } /// Opens the given sample file directories. /// @@ -1200,13 +1305,16 @@ impl LockedDatabase { let e = in_progress.entry(id); use ::std::collections::hash_map::Entry; let e = match e { - Entry::Occupied(_) => continue, // suppress duplicate. + Entry::Occupied(_) => continue, // suppress duplicate. Entry::Vacant(e) => e, }; - let dir = self.sample_file_dirs_by_id - .get_mut(&id) - .ok_or_else(|| format_err!("no such dir {}", id))?; - if dir.dir.is_some() { continue } + let dir = self + .sample_file_dirs_by_id + .get_mut(&id) + .ok_or_else(|| format_err!("no such dir {}", id))?; + if dir.dir.is_some() { + continue; + } let mut meta = dir.meta(&self.uuid); if let Some(o) = self.open.as_ref() { let open = meta.in_progress_open.set_default(); @@ -1215,23 +1323,27 @@ impl LockedDatabase { } let d = dir::SampleFileDir::open(&dir.path, &meta) .map_err(|e| e.context(format!("Failed to open dir {}", dir.path)))?; - if self.open.is_none() { // read-only mode; it's already fully opened. + if self.open.is_none() { + // read-only mode; it's already fully opened. dir.dir = Some(d); - } else { // read-write mode; there are more steps to do. + } else { + // read-write mode; there are more steps to do. e.insert((meta, d)); } } let o = match self.open.as_ref() { - None => return Ok(()), // read-only mode; all done. + None => return Ok(()), // read-only mode; all done. Some(o) => o, }; let tx = self.conn.transaction()?; { - let mut stmt = tx.prepare_cached(r#" + let mut stmt = tx.prepare_cached( + r#" update sample_file_dir set last_complete_open_id = ? where id = ? - "#)?; + "#, + )?; for &id in in_progress.keys() { if stmt.execute(params![o.id, id])? != 1 { bail!("unable to update dir {}", id); @@ -1251,7 +1363,9 @@ impl LockedDatabase { Ok(()) } - pub fn streams_by_id(&self) -> &BTreeMap { &self.streams_by_id } + pub fn streams_by_id(&self) -> &BTreeMap { + &self.streams_by_id + } /// Returns an immutable view of the video sample entries. pub fn video_sample_entries_by_id(&self) -> &BTreeMap> { @@ -1261,7 +1375,11 @@ impl LockedDatabase { /// Gets a given camera by uuid. pub fn get_camera(&self, uuid: Uuid) -> Option<&Camera> { match self.cameras_by_uuid.get(&uuid) { - Some(id) => Some(self.cameras_by_id.get(id).expect("uuid->id requires id->cam")), + Some(id) => Some( + self.cameras_by_id + .get(id) + .expect("uuid->id requires id->cam"), + ), None => None, } } @@ -1272,8 +1390,11 @@ impl LockedDatabase { /// Note that at present, the returned recordings are _not_ completely ordered by start time. /// Uncommitted recordings are returned id order after the others. pub fn list_recordings_by_time( - &self, stream_id: i32, desired_time: Range, - f: &mut dyn FnMut(ListRecordingsRow) -> Result<(), Error>) -> Result<(), Error> { + &self, + stream_id: i32, + desired_time: Range, + f: &mut dyn FnMut(ListRecordingsRow) -> Result<(), Error>, + ) -> Result<(), Error> { let s = match self.streams_by_id.get(&stream_id) { None => bail!("no such stream {}", stream_id), Some(s) => s, @@ -1285,10 +1406,12 @@ impl LockedDatabase { if l.video_samples > 0 { let end = l.start + recording::Duration(l.wall_duration_90k as i64); if l.start > desired_time.end || end < desired_time.start { - continue; // there's no overlap with the requested range. + continue; // there's no overlap with the requested range. } - l.to_list_row(CompositeId::new(stream_id, s.cum_recordings + i as i32), - self.open.unwrap().id) + l.to_list_row( + CompositeId::new(stream_id, s.cum_recordings + i as i32), + self.open.unwrap().id, + ) } else { continue; } @@ -1300,8 +1423,11 @@ impl LockedDatabase { /// Lists the specified recordings in ascending order by id. pub fn list_recordings_by_id( - &self, stream_id: i32, desired_ids: Range, - f: &mut dyn FnMut(ListRecordingsRow) -> Result<(), Error>) -> Result<(), Error> { + &self, + stream_id: i32, + desired_ids: Range, + f: &mut dyn FnMut(ListRecordingsRow) -> Result<(), Error>, + ) -> Result<(), Error> { let s = match self.streams_by_id.get(&stream_id) { None => bail!("no such stream {}", stream_id), Some(s) => s, @@ -1311,14 +1437,18 @@ impl LockedDatabase { } if desired_ids.end > s.cum_recordings { let start = cmp::max(0, desired_ids.start - s.cum_recordings) as usize; - let end = cmp::min((desired_ids.end - s.cum_recordings) as usize, - s.uncommitted.len()); - for i in start .. end { + let end = cmp::min( + (desired_ids.end - s.cum_recordings) as usize, + s.uncommitted.len(), + ); + for i in start..end { let row = { let l = s.uncommitted[i].lock(); if l.video_samples > 0 { - l.to_list_row(CompositeId::new(stream_id, s.cum_recordings + i as i32), - self.open.unwrap().id) + l.to_list_row( + CompositeId::new(stream_id, s.cum_recordings + i as i32), + self.open.unwrap().id, + ) } else { continue; } @@ -1333,10 +1463,12 @@ impl LockedDatabase { /// Rows are given to the callback in arbitrary order. Callers which care about ordering /// should do their own sorting. pub fn list_aggregated_recordings( - &self, stream_id: i32, desired_time: Range, + &self, + stream_id: i32, + desired_time: Range, forced_split: recording::Duration, - f: &mut dyn FnMut(&ListAggregatedRecordingsRow) -> Result<(), Error>) - -> Result<(), Error> { + f: &mut dyn FnMut(&ListAggregatedRecordingsRow) -> Result<(), Error>, + ) -> Result<(), Error> { // Iterate, maintaining a map from a recording_id to the aggregated row for the latest // batch of recordings from the run starting at that id. Runs can be split into multiple // batches for a few reasons: @@ -1361,23 +1493,36 @@ impl LockedDatabase { match aggs.entry(run_start_id) { Entry::Occupied(mut e) => { let a = e.get_mut(); - let new_dur = a.time.end - a.time.start + - recording::Duration(row.wall_duration_90k as i64); - let needs_flush = - a.ids.end != recording_id || - row.video_sample_entry_id != a.video_sample_entry_id || - new_dur >= forced_split; - if needs_flush { // flush then start a new entry. + let new_dur = a.time.end - a.time.start + + recording::Duration(row.wall_duration_90k as i64); + let needs_flush = a.ids.end != recording_id + || row.video_sample_entry_id != a.video_sample_entry_id + || new_dur >= forced_split; + if needs_flush { + // flush then start a new entry. f(a)?; *a = ListAggregatedRecordingsRow::from(row); - } else { // append. + } else { + // append. if a.time.end != row.start { - bail!("stream {} recording {} ends at {} but {} starts at {}", - stream_id, a.ids.end - 1, a.time.end, row.id, row.start); + bail!( + "stream {} recording {} ends at {} but {} starts at {}", + stream_id, + a.ids.end - 1, + a.time.end, + row.id, + row.start + ); } if a.open_id != row.open_id { - bail!("stream {} recording {} has open id {} but {} has {}", - stream_id, a.ids.end - 1, a.open_id, row.id, row.open_id); + bail!( + "stream {} recording {} has open id {} but {} has {}", + stream_id, + a.ids.end - 1, + a.open_id, + row.id, + row.open_id + ); } a.time.end.0 += row.wall_duration_90k as i64; a.ids.end = recording_id + 1; @@ -1389,8 +1534,10 @@ impl LockedDatabase { } a.growing = growing; } - }, - Entry::Vacant(e) => { e.insert(ListAggregatedRecordingsRow::from(row)); }, + } + Entry::Vacant(e) => { + e.insert(ListAggregatedRecordingsRow::from(row)); + } } Ok(()) })?; @@ -1403,21 +1550,30 @@ impl LockedDatabase { /// Calls `f` with a single `recording_playback` row. /// Note the lock is held for the duration of `f`. /// This uses a LRU cache to reduce the number of retrievals from the database. - pub fn with_recording_playback(&self, id: CompositeId, - f: &mut dyn FnMut(&RecordingPlayback) -> Result) - -> Result { + pub fn with_recording_playback( + &self, + id: CompositeId, + f: &mut dyn FnMut(&RecordingPlayback) -> Result, + ) -> Result { // Check for uncommitted path. - let s = self.streams_by_id - .get(&id.stream()) - .ok_or_else(|| format_err!("no stream for {}", id))?; + let s = self + .streams_by_id + .get(&id.stream()) + .ok_or_else(|| format_err!("no stream for {}", id))?; if s.cum_recordings <= id.recording() { let i = id.recording() - s.cum_recordings; if i as usize >= s.uncommitted.len() { - bail!("no such recording {}; latest committed is {}, latest is {}", - id, s.cum_recordings, s.cum_recordings + s.uncommitted.len() as i32); + bail!( + "no such recording {}; latest committed is {}, latest is {}", + id, + s.cum_recordings, + s.cum_recordings + s.uncommitted.len() as i32 + ); } let l = s.uncommitted[i as usize].lock(); - return f(&RecordingPlayback { video_index: &l.video_index }); + return f(&RecordingPlayback { + video_index: &l.video_index, + }); } // Committed path. @@ -1429,14 +1585,16 @@ impl LockedDatabase { occupied.to_back(); let video_index = occupied.get(); return f(&RecordingPlayback { video_index }); - }, + } RawEntryMut::Vacant(vacant) => { trace!("cache miss for recording {}", id); let mut stmt = self.conn.prepare_cached(GET_RECORDING_PLAYBACK_SQL)?; - let mut rows = stmt.query_named(named_params!{":composite_id": id.0})?; + let mut rows = stmt.query_named(named_params! {":composite_id": id.0})?; if let Some(row) = rows.next()? { let video_index: VideoIndex = row.get(0)?; - let result = f(&RecordingPlayback { video_index: &video_index.0[..] }); + let result = f(&RecordingPlayback { + video_index: &video_index.0[..], + }); vacant.insert(id.0, video_index.0); if cache.len() > VIDEO_INDEX_CACHE_LEN { cache.pop_front(); @@ -1444,15 +1602,17 @@ impl LockedDatabase { return result; } Err(format_err!("no such recording {}", id)) - }, + } } } /// Queues for deletion the oldest recordings that aren't already queued. /// `f` should return true for each row that should be deleted. pub(crate) fn delete_oldest_recordings( - &mut self, stream_id: i32, f: &mut dyn FnMut(&ListOldestRecordingsRow) -> bool) - -> Result<(), Error> { + &mut self, + stream_id: i32, + f: &mut dyn FnMut(&ListOldestRecordingsRow) -> bool, + ) -> Result<(), Error> { let s = match self.streams_by_id.get_mut(&stream_id) { None => bail!("no stream {}", stream_id), Some(s) => s, @@ -1476,7 +1636,8 @@ impl LockedDatabase { /// Initializes the video_sample_entries. To be called during construction. fn init_video_sample_entries(&mut self) -> Result<(), Error> { info!("Loading video sample entries"); - let mut stmt = self.conn.prepare(r#" + let mut stmt = self.conn.prepare( + r#" select id, width, @@ -1487,24 +1648,30 @@ impl LockedDatabase { data from video_sample_entry - "#)?; + "#, + )?; let mut rows = stmt.query(params![])?; while let Some(row) = rows.next()? { let id = row.get(0)?; let data: Vec = row.get(6)?; - self.video_sample_entries_by_id.insert(id, Arc::new(VideoSampleEntry { + self.video_sample_entries_by_id.insert( id, - width: row.get::<_, i32>(1)?.try_into()?, - height: row.get::<_, i32>(2)?.try_into()?, - pasp_h_spacing: row.get::<_, i32>(3)?.try_into()?, - pasp_v_spacing: row.get::<_, i32>(4)?.try_into()?, - data, - rfc6381_codec: row.get(5)?, - })); + Arc::new(VideoSampleEntry { + id, + width: row.get::<_, i32>(1)?.try_into()?, + height: row.get::<_, i32>(2)?.try_into()?, + pasp_h_spacing: row.get::<_, i32>(3)?.try_into()?, + pasp_v_spacing: row.get::<_, i32>(4)?.try_into()?, + data, + rfc6381_codec: row.get(5)?, + }), + ); } - info!("Loaded {} video sample entries", - self.video_sample_entries_by_id.len()); + info!( + "Loaded {} video sample entries", + self.video_sample_entries_by_id.len() + ); Ok(()) } @@ -1512,7 +1679,8 @@ impl LockedDatabase { /// To be called during construction. fn init_sample_file_dirs(&mut self) -> Result<(), Error> { info!("Loading sample file dirs"); - let mut stmt = self.conn.prepare(r#" + let mut stmt = self.conn.prepare( + r#" select d.id, d.path, @@ -1521,7 +1689,8 @@ impl LockedDatabase { o.uuid from sample_file_dir d left join open o on (d.last_complete_open_id = o.id); - "#)?; + "#, + )?; let mut rows = stmt.query(params![])?; while let Some(row) = rows.next()? { let id = row.get(0)?; @@ -1529,21 +1698,27 @@ impl LockedDatabase { let open_id: Option = row.get(3)?; let open_uuid: Option = row.get(4)?; let last_complete_open = match (open_id, open_uuid) { - (Some(id), Some(uuid)) => Some(Open { id, uuid: uuid.0, }), + (Some(id), Some(uuid)) => Some(Open { id, uuid: uuid.0 }), (None, None) => None, _ => bail!("open table missing id {}", id), }; - self.sample_file_dirs_by_id.insert(id, SampleFileDir { + self.sample_file_dirs_by_id.insert( id, - uuid: dir_uuid.0, - path: row.get(1)?, - dir: None, - last_complete_open, - garbage_needs_unlink: raw::list_garbage(&self.conn, id)?, - garbage_unlinked: Vec::new(), - }); + SampleFileDir { + id, + uuid: dir_uuid.0, + path: row.get(1)?, + dir: None, + last_complete_open, + garbage_needs_unlink: raw::list_garbage(&self.conn, id)?, + garbage_unlinked: Vec::new(), + }, + ); } - info!("Loaded {} sample file dirs", self.sample_file_dirs_by_id.len()); + info!( + "Loaded {} sample file dirs", + self.sample_file_dirs_by_id.len() + ); Ok(()) } @@ -1551,7 +1726,8 @@ impl LockedDatabase { /// To be called during construction. fn init_cameras(&mut self) -> Result<(), Error> { info!("Loading cameras"); - let mut stmt = self.conn.prepare(r#" + let mut stmt = self.conn.prepare( + r#" select id, uuid, @@ -1562,21 +1738,25 @@ impl LockedDatabase { password from camera; - "#)?; + "#, + )?; let mut rows = stmt.query(params![])?; while let Some(row) = rows.next()? { let id = row.get(0)?; let uuid: FromSqlUuid = row.get(1)?; - self.cameras_by_id.insert(id, Camera { - id: id, - uuid: uuid.0, - short_name: row.get(2)?, - description: row.get(3)?, - onvif_host: row.get(4)?, - username: row.get(5)?, - password: row.get(6)?, - streams: Default::default(), - }); + self.cameras_by_id.insert( + id, + Camera { + id: id, + uuid: uuid.0, + short_name: row.get(2)?, + description: row.get(3)?, + onvif_host: row.get(4)?, + username: row.get(5)?, + password: row.get(6)?, + streams: Default::default(), + }, + ); self.cameras_by_uuid.insert(uuid.0, id); } info!("Loaded {} cameras", self.cameras_by_id.len()); @@ -1587,7 +1767,8 @@ impl LockedDatabase { /// To be called during construction. fn init_streams(&mut self) -> Result<(), Error> { info!("Loading streams"); - let mut stmt = self.conn.prepare(r#" + let mut stmt = self.conn.prepare( + r#" select id, type, @@ -1602,46 +1783,49 @@ impl LockedDatabase { record from stream; - "#)?; + "#, + )?; let mut rows = stmt.query(params![])?; while let Some(row) = rows.next()? { let id = row.get(0)?; let type_: String = row.get(1)?; - let type_ = StreamType::parse(&type_).ok_or_else( - || format_err!("no such stream type {}", type_))?; + let type_ = StreamType::parse(&type_) + .ok_or_else(|| format_err!("no such stream type {}", type_))?; let camera_id = row.get(2)?; let c = self - .cameras_by_id - .get_mut(&camera_id) - .ok_or_else(|| format_err!("missing camera {} for stream {}", - camera_id, id))?; + .cameras_by_id + .get_mut(&camera_id) + .ok_or_else(|| format_err!("missing camera {} for stream {}", camera_id, id))?; let flush_if_sec = row.get(6)?; - self.streams_by_id.insert(id, Stream { + self.streams_by_id.insert( id, - type_, - camera_id, - sample_file_dir_id: row.get(3)?, - rtsp_url: row.get(4)?, - retain_bytes: row.get(5)?, - flush_if_sec, - range: None, - sample_file_bytes: 0, - fs_bytes: 0, - to_delete: Vec::new(), - bytes_to_delete: 0, - fs_bytes_to_delete: 0, - bytes_to_add: 0, - fs_bytes_to_add: 0, - duration: recording::Duration(0), - committed_days: BTreeMap::new(), - cum_recordings: row.get(7)?, - cum_media_duration: recording::Duration(row.get(8)?), - cum_runs: row.get(9)?, - record: row.get(10)?, - uncommitted: VecDeque::new(), - synced_recordings: 0, - on_live_segment: Vec::new(), - }); + Stream { + id, + type_, + camera_id, + sample_file_dir_id: row.get(3)?, + rtsp_url: row.get(4)?, + retain_bytes: row.get(5)?, + flush_if_sec, + range: None, + sample_file_bytes: 0, + fs_bytes: 0, + to_delete: Vec::new(), + bytes_to_delete: 0, + fs_bytes_to_delete: 0, + bytes_to_add: 0, + fs_bytes_to_add: 0, + duration: recording::Duration(0), + committed_days: BTreeMap::new(), + cum_recordings: row.get(7)?, + cum_media_duration: recording::Duration(row.get(8)?), + cum_runs: row.get(9)?, + record: row.get(10)?, + uncommitted: VecDeque::new(), + synced_recordings: 0, + on_live_segment: Vec::new(), + }, + ); c.streams[type_.index()] = Some(id); } info!("Loaded {} streams", self.streams_by_id.len()); @@ -1650,25 +1834,33 @@ impl LockedDatabase { /// Inserts the specified video sample entry if absent. /// On success, returns the id of a new or existing row. - pub fn insert_video_sample_entry(&mut self, entry: VideoSampleEntryToInsert) - -> Result { - + pub fn insert_video_sample_entry( + &mut self, + entry: VideoSampleEntryToInsert, + ) -> Result { // Check if it already exists. // There shouldn't be too many entries, so it's fine to enumerate everything. for (&id, v) in &self.video_sample_entries_by_id { if v.data == entry.data { // The other fields are derived from data, so differences indicate a bug. - if v.width != entry.width || v.height != entry.height || - v.pasp_h_spacing != entry.pasp_h_spacing || - v.pasp_v_spacing != entry.pasp_v_spacing { - bail!("video_sample_entry id {}: existing entry {:?}, new {:?}", id, v, &entry); + if v.width != entry.width + || v.height != entry.height + || v.pasp_h_spacing != entry.pasp_h_spacing + || v.pasp_v_spacing != entry.pasp_v_spacing + { + bail!( + "video_sample_entry id {}: existing entry {:?}, new {:?}", + id, + v, + &entry + ); } return Ok(id); } } let mut stmt = self.conn.prepare_cached(INSERT_VIDEO_SAMPLE_ENTRY_SQL)?; - stmt.execute_named(named_params!{ + stmt.execute_named(named_params! { ":width": i32::from(entry.width), ":height": i32::from(entry.height), ":pasp_h_spacing": i32::from(entry.pasp_h_spacing), @@ -1678,15 +1870,18 @@ impl LockedDatabase { })?; let id = self.conn.last_insert_rowid() as i32; - self.video_sample_entries_by_id.insert(id, Arc::new(VideoSampleEntry { + self.video_sample_entries_by_id.insert( id, - width: entry.width, - height: entry.height, - pasp_h_spacing: entry.pasp_h_spacing, - pasp_v_spacing: entry.pasp_v_spacing, - data: entry.data, - rfc6381_codec: entry.rfc6381_codec, - })); + Arc::new(VideoSampleEntry { + id, + width: entry.width, + height: entry.height, + pasp_h_spacing: entry.pasp_h_spacing, + pasp_v_spacing: entry.pasp_v_spacing, + data: entry.data, + rfc6381_codec: entry.rfc6381_codec, + }), + ); Ok(id) } @@ -1695,9 +1890,10 @@ impl LockedDatabase { let mut meta = schema::DirMeta::default(); let uuid = Uuid::new_v4(); let uuid_bytes = &uuid.as_bytes()[..]; - let o = self.open - .as_ref() - .ok_or_else(|| format_err!("database is read-only"))?; + let o = self + .open + .as_ref() + .ok_or_else(|| format_err!("database is read-only"))?; // Populate meta. { @@ -1709,10 +1905,13 @@ impl LockedDatabase { } let dir = dir::SampleFileDir::create(&path, &meta)?; - self.conn.execute(r#" + self.conn.execute( + r#" insert into sample_file_dir (path, uuid, last_complete_open_id) values (?, ?, ?) - "#, params![&path, uuid_bytes, o.id])?; + "#, + params![&path, uuid_bytes, o.id], + )?; let id = self.conn.last_insert_rowid() as i32; use ::std::collections::btree_map::Entry; let e = self.sample_file_dirs_by_id.entry(id); @@ -1745,25 +1944,35 @@ impl LockedDatabase { _ => bail!("no such dir {} to remove", dir_id), }; if !d.get().garbage_needs_unlink.is_empty() || !d.get().garbage_unlinked.is_empty() { - bail!("must collect garbage before deleting directory {}", d.get().path); + bail!( + "must collect garbage before deleting directory {}", + d.get().path + ); } let dir = match d.get_mut().dir.take() { None => dir::SampleFileDir::open(&d.get().path, &d.get().meta(&self.uuid))?, Some(arc) => match Arc::strong_count(&arc) { 1 => { - d.get_mut().dir = Some(arc); // put it back. + d.get_mut().dir = Some(arc); // put it back. bail!("can't delete in-use directory {}", dir_id); - }, + } _ => arc, }, }; if !dir.is_empty()? { - bail!("Can't delete sample file directory {} which still has files", &d.get().path); + bail!( + "Can't delete sample file directory {} which still has files", + &d.get().path + ); } let mut meta = d.get().meta(&self.uuid); meta.in_progress_open = meta.last_complete_open.take().into(); dir.write_meta(&meta)?; - if self.conn.execute("delete from sample_file_dir where id = ?", params![dir_id])? != 1 { + if self + .conn + .execute("delete from sample_file_dir where id = ?", params![dir_id])? + != 1 + { bail!("missing database row for dir {}", dir_id); } d.remove_entry(); @@ -1778,13 +1987,15 @@ impl LockedDatabase { let streams; let camera_id; { - let mut stmt = tx.prepare_cached(r#" + let mut stmt = tx.prepare_cached( + r#" insert into camera (uuid, short_name, description, onvif_host, username, password) values (:uuid, :short_name, :description, :onvif_host, :username, :password) - "#)?; - stmt.execute_named(named_params!{ + "#, + )?; + stmt.execute_named(named_params! { ":uuid": uuid_bytes, ":short_name": &camera.short_name, ":description": &camera.description, @@ -1793,21 +2004,24 @@ impl LockedDatabase { ":password": &camera.password, })?; camera_id = tx.last_insert_rowid() as i32; - streams = StreamStateChanger::new(&tx, camera_id, None, &self.streams_by_id, - &mut camera)?; + streams = + StreamStateChanger::new(&tx, camera_id, None, &self.streams_by_id, &mut camera)?; } tx.commit()?; let streams = streams.apply(&mut self.streams_by_id); - self.cameras_by_id.insert(camera_id, Camera { - id: camera_id, - uuid, - short_name: camera.short_name, - description: camera.description, - onvif_host: camera.onvif_host, - username: camera.username, - password: camera.password, - streams, - }); + self.cameras_by_id.insert( + camera_id, + Camera { + id: camera_id, + uuid, + short_name: camera.short_name, + description: camera.description, + onvif_host: camera.onvif_host, + username: camera.username, + password: camera.password, + streams, + }, + ); self.cameras_by_uuid.insert(uuid, camera_id); Ok(camera_id) } @@ -1817,13 +2031,14 @@ impl LockedDatabase { let tx = self.conn.transaction()?; let streams; let c = self - .cameras_by_id - .get_mut(&camera_id) - .ok_or_else(|| format_err!("no such camera {}", camera_id))?; + .cameras_by_id + .get_mut(&camera_id) + .ok_or_else(|| format_err!("no such camera {}", camera_id))?; { - streams = StreamStateChanger::new(&tx, camera_id, Some(c), &self.streams_by_id, - &mut camera)?; - let mut stmt = tx.prepare_cached(r#" + streams = + StreamStateChanger::new(&tx, camera_id, Some(c), &self.streams_by_id, &mut camera)?; + let mut stmt = tx.prepare_cached( + r#" update camera set short_name = :short_name, description = :description, @@ -1832,8 +2047,9 @@ impl LockedDatabase { password = :password where id = :id - "#)?; - let rows = stmt.execute_named(named_params!{ + "#, + )?; + let rows = stmt.execute_named(named_params! { ":id": camera_id, ":short_name": &camera.short_name, ":description": &camera.description, @@ -1857,26 +2073,30 @@ impl LockedDatabase { /// Deletes a camera and its streams. The camera must have no recordings. pub fn delete_camera(&mut self, id: i32) -> Result<(), Error> { - let uuid = self.cameras_by_id.get(&id) - .map(|c| c.uuid) - .ok_or_else(|| format_err!("No such camera {} to remove", id))?; + let uuid = self + .cameras_by_id + .get(&id) + .map(|c| c.uuid) + .ok_or_else(|| format_err!("No such camera {} to remove", id))?; let mut streams_to_delete = Vec::new(); let tx = self.conn.transaction()?; { let mut stream_stmt = tx.prepare_cached(r"delete from stream where id = :id")?; for (stream_id, stream) in &self.streams_by_id { - if stream.camera_id != id { continue }; + if stream.camera_id != id { + continue; + }; if stream.range.is_some() { bail!("Can't remove camera {}; has recordings.", id); } - let rows = stream_stmt.execute_named(named_params!{":id": stream_id})?; + let rows = stream_stmt.execute_named(named_params! {":id": stream_id})?; if rows != 1 { bail!("Stream {} missing from database", id); } streams_to_delete.push(*stream_id); } let mut cam_stmt = tx.prepare_cached(r"delete from camera where id = :id")?; - let rows = cam_stmt.execute_named(named_params!{":id": id})?; + let rows = cam_stmt.execute_named(named_params! {":id": id})?; if rows != 1 { bail!("Camera {} missing from database", id); } @@ -1887,26 +2107,31 @@ impl LockedDatabase { } self.cameras_by_id.remove(&id); self.cameras_by_uuid.remove(&uuid); - return Ok(()) + Ok(()) } pub fn update_retention(&mut self, changes: &[RetentionChange]) -> Result<(), Error> { let tx = self.conn.transaction()?; { - let mut stmt = tx.prepare_cached(r#" + let mut stmt = tx.prepare_cached( + r#" update stream set record = :record, retain_bytes = :retain where id = :id - "#)?; + "#, + )?; for c in changes { if c.new_limit < 0 { - bail!("can't set limit for stream {} to {}; must be >= 0", - c.stream_id, c.new_limit); + bail!( + "can't set limit for stream {} to {}; must be >= 0", + c.stream_id, + c.new_limit + ); } - let rows = stmt.execute_named(named_params!{ + let rows = stmt.execute_named(named_params! { ":record": c.new_record, ":retain": c.new_limit, ":id": c.stream_id, @@ -1918,7 +2143,10 @@ impl LockedDatabase { } tx.commit()?; for c in changes { - let s = self.streams_by_id.get_mut(&c.stream_id).expect("stream in db but not state"); + let s = self + .streams_by_id + .get_mut(&c.stream_id) + .expect("stream in db but not state"); s.record = c.new_record; s.retain_bytes = c.new_limit; } @@ -1927,7 +2155,9 @@ impl LockedDatabase { // ---- auth ---- - pub fn users_by_id(&self) -> &BTreeMap { self.auth.users_by_id() } + pub fn users_by_id(&self) -> &BTreeMap { + self.auth.users_by_id() + } pub fn apply_user_change(&mut self, change: UserChange) -> Result<&User, Error> { self.auth.apply(&self.conn, change) @@ -1941,41 +2171,70 @@ impl LockedDatabase { self.auth.get_user(username) } - pub fn login_by_password(&mut self, req: auth::Request, username: &str, password: String, - domain: Option>, session_flags: i32) - -> Result<(RawSessionId, &Session), Error> { - self.auth.login_by_password(&self.conn, req, username, password, domain, session_flags) + pub fn login_by_password( + &mut self, + req: auth::Request, + username: &str, + password: String, + domain: Option>, + session_flags: i32, + ) -> Result<(RawSessionId, &Session), Error> { + self.auth + .login_by_password(&self.conn, req, username, password, domain, session_flags) } - pub fn make_session(&mut self, creation: Request, uid: i32, - domain: Option>, flags: i32, permissions: schema::Permissions) - -> Result<(RawSessionId, &Session), Error> { - self.auth.make_session(&self.conn, creation, uid, domain, flags, permissions) + pub fn make_session( + &mut self, + creation: Request, + uid: i32, + domain: Option>, + flags: i32, + permissions: schema::Permissions, + ) -> Result<(RawSessionId, &Session), Error> { + self.auth + .make_session(&self.conn, creation, uid, domain, flags, permissions) } - pub fn authenticate_session(&mut self, req: auth::Request, sid: &auth::SessionHash) - -> Result<(&auth::Session, &User), Error> { + pub fn authenticate_session( + &mut self, + req: auth::Request, + sid: &auth::SessionHash, + ) -> Result<(&auth::Session, &User), Error> { self.auth.authenticate_session(&self.conn, req, sid) } - pub fn revoke_session(&mut self, reason: auth::RevocationReason, detail: Option, - req: auth::Request, hash: &auth::SessionHash) -> Result<(), Error> { - self.auth.revoke_session(&self.conn, reason, detail, req, hash) + pub fn revoke_session( + &mut self, + reason: auth::RevocationReason, + detail: Option, + req: auth::Request, + hash: &auth::SessionHash, + ) -> Result<(), Error> { + self.auth + .revoke_session(&self.conn, reason, detail, req, hash) } // ---- signal ---- - pub fn signals_by_id(&self) -> &BTreeMap { self.signal.signals_by_id() } + pub fn signals_by_id(&self) -> &BTreeMap { + self.signal.signals_by_id() + } pub fn signal_types_by_uuid(&self) -> &FnvHashMap { self.signal.types_by_uuid() } pub fn list_changes_by_time( - &self, desired_time: Range, f: &mut dyn FnMut(&signal::ListStateChangesRow)) { + &self, + desired_time: Range, + f: &mut dyn FnMut(&signal::ListStateChangesRow), + ) { self.signal.list_changes_by_time(desired_time, f) } pub fn update_signals( - &mut self, when: Range, signals: &[u32], states: &[u16]) - -> Result<(), base::Error> { + &mut self, + when: Range, + signals: &[u32], + states: &[u16], + ) -> Result<(), base::Error> { self.signal.update_signals(when, signals, states) } } @@ -2017,32 +2276,43 @@ pub fn init(conn: &mut rusqlite::Connection) -> Result<(), Error> { pub fn get_schema_version(conn: &rusqlite::Connection) -> Result, Error> { let ver_tables: i32 = conn.query_row_and_then( "select count(*) from sqlite_master where name = 'version'", - params![], |row| row.get(0))?; + params![], + |row| row.get(0), + )?; if ver_tables == 0 { return Ok(None); } - Ok(Some(conn.query_row_and_then("select max(id) from version", params![], - |row| row.get(0))?)) + Ok(Some(conn.query_row_and_then( + "select max(id) from version", + params![], + |row| row.get(0), + )?)) } /// Checks that the schema version in the given database is as expected. pub(crate) fn check_schema_version(conn: &rusqlite::Connection) -> Result<(), Error> { - let ver = get_schema_version(conn)?.ok_or_else(|| format_err!( - "no such table: version. \ - \ - If you are starting from an \ - empty database, see README.md to complete the \ - installation. If you are starting from a database \ - that predates schema versioning, see guide/schema.md."))?; + let ver = get_schema_version(conn)?.ok_or_else(|| { + format_err!( + "no such table: version. \ + If you are starting from an empty database, see README.md to \ + complete the installation. If you are starting from a database \ + that predates schema versioning, see guide/schema.md." + ) + })?; if ver < EXPECTED_VERSION { - bail!("Database schema version {} is too old (expected {}); \ - see upgrade instructions in guide/upgrade.md.", - ver, EXPECTED_VERSION); + bail!( + "Database schema version {} is too old (expected {}); \ + see upgrade instructions in guide/upgrade.md.", + ver, + EXPECTED_VERSION + ); } else if ver > EXPECTED_VERSION { - bail!("Database schema version {} is too new (expected {}); \ - must use a newer binary to match.", ver, - EXPECTED_VERSION); - + bail!( + "Database schema version {} is too new (expected {}); \ + must use a newer binary to match.", + ver, + EXPECTED_VERSION + ); } Ok(()) } @@ -2063,7 +2333,7 @@ pub struct Database { impl Drop for Database { fn drop(&mut self) { if ::std::thread::panicking() { - return; // don't flush while panicking. + return; // don't flush while panicking. } if let Some(m) = self.db.take() { if let Err(e) = m.into_inner().flush(&self.clocks, "drop") { @@ -2074,13 +2344,20 @@ impl Drop for Database { } // Helpers for Database::lock(). Closures don't implement Fn. -fn acquisition() -> &'static str { "database lock acquisition" } -fn operation() -> &'static str { "database operation" } +fn acquisition() -> &'static str { + "database lock acquisition" +} +fn operation() -> &'static str { + "database operation" +} impl Database { /// Creates the database from a caller-supplied SQLite connection. - pub fn new(clocks: C, mut conn: rusqlite::Connection, - read_write: bool) -> Result, Error> { + pub fn new( + clocks: C, + mut conn: rusqlite::Connection, + read_write: bool, + ) -> Result, Error> { set_integrity_pragmas(&mut conn)?; check_schema_version(&conn)?; @@ -2090,7 +2367,8 @@ impl Database { let open_monotonic = recording::Time::new(clocks.monotonic()); let open = if read_write { let real = recording::Time::new(clocks.realtime()); - let mut stmt = conn.prepare(" insert into open (uuid, start_time_90k) values (?, ?)")?; + let mut stmt = + conn.prepare(" insert into open (uuid, start_time_90k) values (?, ?)")?; let uuid = Uuid::new_v4(); let uuid_bytes = &uuid.as_bytes()[..]; stmt.execute(params![uuid_bytes, real.0])?; @@ -2098,7 +2376,9 @@ impl Database { id: conn.last_insert_rowid() as u32, uuid, }) - } else { None }; + } else { + None + }; let auth = auth::State::init(&conn)?; let signal = signal::State::init(&conn)?; let db = Database { @@ -2116,7 +2396,9 @@ impl Database { streams_by_id: BTreeMap::new(), video_sample_entries_by_id: BTreeMap::new(), video_index_cache: RefCell::new(LinkedHashMap::with_capacity_and_hasher( - VIDEO_INDEX_CACHE_LEN + 1, Default::default())), + VIDEO_INDEX_CACHE_LEN + 1, + Default::default(), + )), on_flush: Vec::new(), })), clocks, @@ -2137,7 +2419,9 @@ impl Database { } #[inline(always)] - pub fn clocks(&self) -> C { self.clocks.clone() } + pub fn clocks(&self) -> C { + self.clocks.clone() + } /// Locks the database; the returned reference is the only way to perform (read or write) /// operations. @@ -2146,7 +2430,9 @@ impl Database { let db = self.db.as_ref().unwrap().lock(); drop(timer); let _timer = clock::TimerGuard:: &'static str>::new( - &self.clocks, operation); + &self.clocks, + operation, + ); DatabaseGuard { clocks: &self.clocks, db, @@ -2185,22 +2471,26 @@ impl<'db, C: Clocks + Clone> DatabaseGuard<'db, C> { impl<'db, C: Clocks + Clone> ::std::ops::Deref for DatabaseGuard<'db, C> { type Target = LockedDatabase; - fn deref(&self) -> &LockedDatabase { &*self.db } + fn deref(&self) -> &LockedDatabase { + &*self.db + } } impl<'db, C: Clocks + Clone> ::std::ops::DerefMut for DatabaseGuard<'db, C> { - fn deref_mut(&mut self) -> &mut LockedDatabase { &mut *self.db } + fn deref_mut(&mut self) -> &mut LockedDatabase { + &mut *self.db + } } #[cfg(test)] mod tests { - use base::clock; + use super::adjust_days; // non-public. + use super::*; use crate::recording::{self, TIME_UNITS_PER_SEC}; + use crate::testutil; + use base::clock; use rusqlite::Connection; use std::collections::BTreeMap; - use crate::testutil; - use super::*; - use super::adjust_days; // non-public. use uuid::Uuid; fn setup_conn() -> Connection { @@ -2231,15 +2521,16 @@ mod tests { } assert_eq!(1, rows); - let stream_id = camera_id; // TODO + let stream_id = camera_id; // TODO rows = 0; { let db = db.lock(); - let all_time = recording::Time(i64::min_value()) .. recording::Time(i64::max_value()); + let all_time = recording::Time(i64::min_value())..recording::Time(i64::max_value()); db.list_recordings_by_time(stream_id, all_time, &mut |_row| { rows += 1; Ok(()) - }).unwrap(); + }) + .unwrap(); } assert_eq!(0, rows); } @@ -2249,7 +2540,7 @@ mod tests { let db = db.lock(); let stream = db.streams_by_id().get(&stream_id).unwrap(); let dur = recording::Duration(r.wall_duration_90k as i64); - assert_eq!(Some(r.start .. r.start + dur), stream.range); + assert_eq!(Some(r.start..r.start + dur), stream.range); assert_eq!(r.sample_file_bytes as i64, stream.sample_file_bytes); assert_eq!(dur, stream.duration); db.cameras_by_id().get(&stream.camera_id).unwrap(); @@ -2261,7 +2552,7 @@ mod tests { let mut recording_id = None; { let db = db.lock(); - let all_time = recording::Time(i64::min_value()) .. recording::Time(i64::max_value()); + let all_time = recording::Time(i64::min_value())..recording::Time(i64::max_value()); db.list_recordings_by_time(stream_id, all_time, &mut |row| { rows += 1; recording_id = Some(row.id); @@ -2270,22 +2561,31 @@ mod tests { assert_eq!(r.video_samples, row.video_samples); assert_eq!(r.video_sync_samples, row.video_sync_samples); assert_eq!(r.sample_file_bytes, row.sample_file_bytes); - let vse = db.video_sample_entries_by_id().get(&row.video_sample_entry_id).unwrap(); + let vse = db + .video_sample_entries_by_id() + .get(&row.video_sample_entry_id) + .unwrap(); assert_eq!(vse.rfc6381_codec, "avc1.4d0029"); Ok(()) - }).unwrap(); + }) + .unwrap(); } assert_eq!(1, rows); rows = 0; - raw::list_oldest_recordings(&db.lock().conn, CompositeId::new(stream_id, 0), &mut |row| { - rows += 1; - assert_eq!(recording_id, Some(row.id)); - assert_eq!(r.start, row.start); - assert_eq!(r.wall_duration_90k, row.wall_duration_90k); - assert_eq!(r.sample_file_bytes, row.sample_file_bytes); - true - }).unwrap(); + raw::list_oldest_recordings( + &db.lock().conn, + CompositeId::new(stream_id, 0), + &mut |row| { + rows += 1; + assert_eq!(recording_id, Some(row.id)); + assert_eq!(r.start, row.start); + assert_eq!(r.wall_duration_90k, row.wall_duration_90k); + assert_eq!(r.sample_file_bytes, row.sample_file_bytes); + true + }, + ) + .unwrap(); assert_eq!(1, rows); // TODO: list_aggregated_recordings. @@ -2298,70 +2598,135 @@ mod tests { let mut m = BTreeMap::new(); // Create a day. - let test_time = recording::Time(130647162600000i64); // 2015-12-31 23:59:00 (Pacific). + let test_time = recording::Time(130647162600000i64); // 2015-12-31 23:59:00 (Pacific). let one_min = recording::Duration(60 * TIME_UNITS_PER_SEC); let two_min = recording::Duration(2 * 60 * TIME_UNITS_PER_SEC); let three_min = recording::Duration(3 * 60 * TIME_UNITS_PER_SEC); let four_min = recording::Duration(4 * 60 * TIME_UNITS_PER_SEC); let test_day1 = &StreamDayKey(*b"2015-12-31"); let test_day2 = &StreamDayKey(*b"2016-01-01"); - adjust_days(test_time .. test_time + one_min, 1, &mut m); + adjust_days(test_time..test_time + one_min, 1, &mut m); assert_eq!(1, m.len()); - assert_eq!(Some(&StreamDayValue{recordings: 1, duration: one_min}), m.get(test_day1)); + assert_eq!( + Some(&StreamDayValue { + recordings: 1, + duration: one_min + }), + m.get(test_day1) + ); // Add to a day. - adjust_days(test_time .. test_time + one_min, 1, &mut m); + adjust_days(test_time..test_time + one_min, 1, &mut m); assert_eq!(1, m.len()); - assert_eq!(Some(&StreamDayValue{recordings: 2, duration: two_min}), m.get(test_day1)); + assert_eq!( + Some(&StreamDayValue { + recordings: 2, + duration: two_min + }), + m.get(test_day1) + ); // Subtract from a day. - adjust_days(test_time .. test_time + one_min, -1, &mut m); + adjust_days(test_time..test_time + one_min, -1, &mut m); assert_eq!(1, m.len()); - assert_eq!(Some(&StreamDayValue{recordings: 1, duration: one_min}), m.get(test_day1)); + assert_eq!( + Some(&StreamDayValue { + recordings: 1, + duration: one_min + }), + m.get(test_day1) + ); // Remove a day. - adjust_days(test_time .. test_time + one_min, -1, &mut m); + adjust_days(test_time..test_time + one_min, -1, &mut m); assert_eq!(0, m.len()); // Create two days. - adjust_days(test_time .. test_time + three_min, 1, &mut m); + adjust_days(test_time..test_time + three_min, 1, &mut m); assert_eq!(2, m.len()); - assert_eq!(Some(&StreamDayValue{recordings: 1, duration: one_min}), m.get(test_day1)); - assert_eq!(Some(&StreamDayValue{recordings: 1, duration: two_min}), m.get(test_day2)); + assert_eq!( + Some(&StreamDayValue { + recordings: 1, + duration: one_min + }), + m.get(test_day1) + ); + assert_eq!( + Some(&StreamDayValue { + recordings: 1, + duration: two_min + }), + m.get(test_day2) + ); // Add to two days. - adjust_days(test_time .. test_time + three_min, 1, &mut m); + adjust_days(test_time..test_time + three_min, 1, &mut m); assert_eq!(2, m.len()); - assert_eq!(Some(&StreamDayValue{recordings: 2, duration: two_min}), m.get(test_day1)); - assert_eq!(Some(&StreamDayValue{recordings: 2, duration: four_min}), m.get(test_day2)); + assert_eq!( + Some(&StreamDayValue { + recordings: 2, + duration: two_min + }), + m.get(test_day1) + ); + assert_eq!( + Some(&StreamDayValue { + recordings: 2, + duration: four_min + }), + m.get(test_day2) + ); // Subtract from two days. - adjust_days(test_time .. test_time + three_min, -1, &mut m); + adjust_days(test_time..test_time + three_min, -1, &mut m); assert_eq!(2, m.len()); - assert_eq!(Some(&StreamDayValue{recordings: 1, duration: one_min}), m.get(test_day1)); - assert_eq!(Some(&StreamDayValue{recordings: 1, duration: two_min}), m.get(test_day2)); + assert_eq!( + Some(&StreamDayValue { + recordings: 1, + duration: one_min + }), + m.get(test_day1) + ); + assert_eq!( + Some(&StreamDayValue { + recordings: 1, + duration: two_min + }), + m.get(test_day2) + ); // Remove two days. - adjust_days(test_time .. test_time + three_min, -1, &mut m); + adjust_days(test_time..test_time + three_min, -1, &mut m); assert_eq!(0, m.len()); } #[test] fn test_day_bounds() { testutil::init(); - assert_eq!(StreamDayKey(*b"2017-10-10").bounds(), // normal day (24 hrs) - recording::Time(135685692000000) .. recording::Time(135693468000000)); - assert_eq!(StreamDayKey(*b"2017-03-12").bounds(), // spring forward (23 hrs) - recording::Time(134037504000000) .. recording::Time(134044956000000)); - assert_eq!(StreamDayKey(*b"2017-11-05").bounds(), // fall back (25 hrs) - recording::Time(135887868000000) .. recording::Time(135895968000000)); + assert_eq!( + StreamDayKey(*b"2017-10-10").bounds(), // normal day (24 hrs) + recording::Time(135685692000000)..recording::Time(135693468000000) + ); + assert_eq!( + StreamDayKey(*b"2017-03-12").bounds(), // spring forward (23 hrs) + recording::Time(134037504000000)..recording::Time(134044956000000) + ); + assert_eq!( + StreamDayKey(*b"2017-11-05").bounds(), // fall back (25 hrs) + recording::Time(135887868000000)..recording::Time(135895968000000) + ); } #[test] fn test_no_meta_or_version() { testutil::init(); - let e = Database::new(clock::RealClocks {}, Connection::open_in_memory().unwrap(), - false).err().unwrap(); + let e = Database::new( + clock::RealClocks {}, + Connection::open_in_memory().unwrap(), + false, + ) + .err() + .unwrap(); assert!(e.to_string().starts_with("no such table"), "{}", e); } @@ -2369,20 +2734,30 @@ mod tests { fn test_version_too_old() { testutil::init(); let c = setup_conn(); - c.execute_batch("delete from version; insert into version values (5, 0, '');").unwrap(); + c.execute_batch("delete from version; insert into version values (5, 0, '');") + .unwrap(); let e = Database::new(clock::RealClocks {}, c, false).err().unwrap(); - assert!(e.to_string().starts_with( - "Database schema version 5 is too old (expected 6)"), "got: {:?}", e); + assert!( + e.to_string() + .starts_with("Database schema version 5 is too old (expected 6)"), + "got: {:?}", + e + ); } #[test] fn test_version_too_new() { testutil::init(); let c = setup_conn(); - c.execute_batch("delete from version; insert into version values (7, 0, '');").unwrap(); + c.execute_batch("delete from version; insert into version values (7, 0, '');") + .unwrap(); let e = Database::new(clock::RealClocks {}, c, false).err().unwrap(); - assert!(e.to_string().starts_with( - "Database schema version 7 is too new (expected 6)"), "got: {:?}", e); + assert!( + e.to_string() + .starts_with("Database schema version 7 is too new (expected 6)"), + "got: {:?}", + e + ); } /// Basic test of running some queries on a fresh database. @@ -2438,7 +2813,8 @@ mod tests { stream_id: main_stream_id, new_record: true, new_limit: 42, - }]).unwrap(); + }]) + .unwrap(); { let main = l.streams_by_id().get(&main_stream_id).unwrap(); assert!(main.record); @@ -2446,32 +2822,62 @@ mod tests { assert_eq!(main.flush_if_sec, 1); } - assert_eq!(l.streams_by_id().get(&sub_stream_id).unwrap().flush_if_sec, 1); + assert_eq!( + l.streams_by_id().get(&sub_stream_id).unwrap().flush_if_sec, + 1 + ); c.streams[1].flush_if_sec = 2; l.update_camera(camera_id, c).unwrap(); - assert_eq!(l.streams_by_id().get(&sub_stream_id).unwrap().flush_if_sec, 2); + assert_eq!( + l.streams_by_id().get(&sub_stream_id).unwrap().flush_if_sec, + 2 + ); } let camera_uuid = { db.lock().cameras_by_id().get(&camera_id).unwrap().uuid }; assert_no_recordings(&db, camera_uuid); - assert_eq!(db.lock().streams_by_id().get(&main_stream_id).unwrap().cum_recordings, 0); + assert_eq!( + db.lock() + .streams_by_id() + .get(&main_stream_id) + .unwrap() + .cum_recordings, + 0 + ); // Closing and reopening the database should present the same contents. let conn = db.close(); let db = Database::new(clock::RealClocks {}, conn, true).unwrap(); - assert_eq!(db.lock().streams_by_id().get(&sub_stream_id).unwrap().flush_if_sec, 2); + assert_eq!( + db.lock() + .streams_by_id() + .get(&sub_stream_id) + .unwrap() + .flush_if_sec, + 2 + ); assert_no_recordings(&db, camera_uuid); - assert_eq!(db.lock().streams_by_id().get(&main_stream_id).unwrap().cum_recordings, 0); + assert_eq!( + db.lock() + .streams_by_id() + .get(&main_stream_id) + .unwrap() + .cum_recordings, + 0 + ); // TODO: assert_eq!(db.lock().list_garbage(sample_file_dir_id).unwrap(), &[]); - let vse_id = db.lock().insert_video_sample_entry(VideoSampleEntryToInsert { - width: 1920, - height: 1080, - pasp_h_spacing: 1, - pasp_v_spacing: 1, - data: include_bytes!("testdata/avc1").to_vec(), - rfc6381_codec: "avc1.4d0029".to_owned(), - }).unwrap(); + let vse_id = db + .lock() + .insert_video_sample_entry(VideoSampleEntryToInsert { + width: 1920, + height: 1080, + pasp_h_spacing: 1, + pasp_v_spacing: 1, + data: include_bytes!("testdata/avc1").to_vec(), + rfc6381_codec: "avc1.4d0029".to_owned(), + }) + .unwrap(); assert!(vse_id > 0, "vse_id = {}", vse_id); // Inserting a recording should succeed and advance the next recording id. @@ -2499,7 +2905,14 @@ mod tests { db.flush("add test").unwrap(); id }; - assert_eq!(db.lock().streams_by_id().get(&main_stream_id).unwrap().cum_recordings, 1); + assert_eq!( + db.lock() + .streams_by_id() + .get(&main_stream_id) + .unwrap() + .cum_recordings, + 1 + ); // Queries should return the correct result (with caches update on insert). assert_single_recording(&db, main_stream_id, &recording); @@ -2514,7 +2927,11 @@ mod tests { { let mut db = db.lock(); let mut n = 0; - db.delete_oldest_recordings(main_stream_id, &mut |_| { n += 1; true }).unwrap(); + db.delete_oldest_recordings(main_stream_id, &mut |_| { + n += 1; + true + }) + .unwrap(); assert_eq!(n, 1); { let s = db.streams_by_id().get(&main_stream_id).unwrap(); @@ -2524,40 +2941,52 @@ mod tests { n = 0; // A second run - db.delete_oldest_recordings(main_stream_id, &mut |_| { n += 1; true }).unwrap(); + db.delete_oldest_recordings(main_stream_id, &mut |_| { + n += 1; + true + }) + .unwrap(); assert_eq!(n, 0); - assert_eq!(db.streams_by_id().get(&main_stream_id).unwrap().bytes_to_delete, 42); + assert_eq!( + db.streams_by_id() + .get(&main_stream_id) + .unwrap() + .bytes_to_delete, + 42 + ); db.flush("delete test").unwrap(); let s = db.streams_by_id().get(&main_stream_id).unwrap(); assert_eq!(s.sample_file_bytes, 0); assert_eq!(s.bytes_to_delete, 0); } assert_no_recordings(&db, camera_uuid); - let g: Vec<_> = db.lock() - .sample_file_dirs_by_id() - .get(&sample_file_dir_id) - .unwrap() - .garbage_needs_unlink - .iter() - .map(|&id| id) - .collect(); + let g: Vec<_> = db + .lock() + .sample_file_dirs_by_id() + .get(&sample_file_dir_id) + .unwrap() + .garbage_needs_unlink + .iter() + .map(|&id| id) + .collect(); assert_eq!(&g, &[id]); - let g: Vec<_> = db.lock() - .sample_file_dirs_by_id() - .get(&sample_file_dir_id) - .unwrap() - .garbage_unlinked - .iter() - .map(|&id| id) - .collect(); + let g: Vec<_> = db + .lock() + .sample_file_dirs_by_id() + .get(&sample_file_dir_id) + .unwrap() + .garbage_unlinked + .iter() + .map(|&id| id) + .collect(); assert_eq!(&g, &[]); } #[test] fn round_up() { assert_eq!(super::round_up(0), 0); - assert_eq!(super::round_up(8_191), 8_192); - assert_eq!(super::round_up(8_192), 8_192); + assert_eq!(super::round_up(8_191), 8_192); + assert_eq!(super::round_up(8_192), 8_192); assert_eq!(super::round_up(8_193), 12_288); } } diff --git a/server/db/dir.rs b/server/db/dir.rs index c0072d6..17c355f 100644 --- a/server/db/dir.rs +++ b/server/db/dir.rs @@ -36,11 +36,15 @@ use crate::coding; use crate::db::CompositeId; use crate::schema; use cstr::cstr; -use failure::{Error, Fail, bail, format_err}; +use failure::{bail, format_err, Error, Fail}; use log::warn; -use protobuf::Message; -use nix::{NixPath, fcntl::{FlockArg, OFlag}, sys::stat::Mode}; use nix::sys::statvfs::Statvfs; +use nix::{ + fcntl::{FlockArg, OFlag}, + sys::stat::Mode, + NixPath, +}; +use protobuf::Message; use std::ffi::CStr; use std::fs; use std::io::{Read, Write}; @@ -76,11 +80,17 @@ impl CompositeIdPath { } impl NixPath for CompositeIdPath { - fn is_empty(&self) -> bool { false } - fn len(&self) -> usize { 16 } + fn is_empty(&self) -> bool { + false + } + fn len(&self) -> usize { + 16 + } fn with_nix_path(&self, f: F) -> Result - where F: FnOnce(&CStr) -> T { + where + F: FnOnce(&CStr) -> T, + { let p = CStr::from_bytes_with_nul(&self.0[..]).expect("no interior nuls"); Ok(f(p)) } @@ -91,7 +101,9 @@ impl NixPath for CompositeIdPath { pub struct Fd(std::os::unix::io::RawFd); impl std::os::unix::io::AsRawFd for Fd { - fn as_raw_fd(&self) -> std::os::unix::io::RawFd { self.0 } + fn as_raw_fd(&self) -> std::os::unix::io::RawFd { + self.0 + } } impl Drop for Fd { @@ -107,7 +119,7 @@ impl Fd { pub fn open(path: &P, mkdir: bool) -> Result { if mkdir { match nix::unistd::mkdir(path, nix::sys::stat::Mode::S_IRWXU) { - Ok(()) | Err(nix::Error::Sys(nix::errno::Errno::EEXIST)) => {}, + Ok(()) | Err(nix::Error::Sys(nix::errno::Errno::EEXIST)) => {} Err(e) => return Err(e), } } @@ -138,7 +150,7 @@ pub(crate) fn read_meta(dir: &Fd) -> Result { return Ok(meta); } return Err(e.into()); - }, + } Ok(f) => f, }; let mut data = Vec::new(); @@ -146,38 +158,63 @@ pub(crate) fn read_meta(dir: &Fd) -> Result { let (len, pos) = coding::decode_varint32(&data, 0) .map_err(|_| format_err!("Unable to decode varint length in meta file"))?; if data.len() != FIXED_DIR_META_LEN || len as usize + pos > FIXED_DIR_META_LEN { - bail!("Expected a {}-byte file with a varint length of a DirMeta message; got \ - a {}-byte file with length {}", FIXED_DIR_META_LEN, data.len(), len); + bail!( + "Expected a {}-byte file with a varint length of a DirMeta message; got \ + a {}-byte file with length {}", + FIXED_DIR_META_LEN, + data.len(), + len + ); } - let data = &data[pos..pos+len as usize]; + let data = &data[pos..pos + len as usize]; let mut s = protobuf::CodedInputStream::from_bytes(&data); - meta.merge_from(&mut s).map_err(|e| e.context("Unable to parse metadata proto"))?; + meta.merge_from(&mut s) + .map_err(|e| e.context("Unable to parse metadata proto"))?; Ok(meta) } /// Write `dir`'s metadata, clobbering existing data. pub(crate) fn write_meta(dirfd: RawFd, meta: &schema::DirMeta) -> Result<(), Error> { - let mut data = meta.write_length_delimited_to_bytes().expect("proto3->vec is infallible"); + let mut data = meta + .write_length_delimited_to_bytes() + .expect("proto3->vec is infallible"); if data.len() > FIXED_DIR_META_LEN { - bail!("Length-delimited DirMeta message requires {} bytes, over limit of {}", - data.len(), FIXED_DIR_META_LEN); + bail!( + "Length-delimited DirMeta message requires {} bytes, over limit of {}", + data.len(), + FIXED_DIR_META_LEN + ); } - data.resize(FIXED_DIR_META_LEN, 0); // pad to required length. - let mut f = crate::fs::openat(dirfd, cstr!("meta"), OFlag::O_CREAT | OFlag::O_WRONLY, - Mode::S_IRUSR | Mode::S_IWUSR) - .map_err(|e| e.context("Unable to open meta file"))?; - let stat = f.metadata().map_err(|e| e.context("Unable to stat meta file"))?; + data.resize(FIXED_DIR_META_LEN, 0); // pad to required length. + let mut f = crate::fs::openat( + dirfd, + cstr!("meta"), + OFlag::O_CREAT | OFlag::O_WRONLY, + Mode::S_IRUSR | Mode::S_IWUSR, + ) + .map_err(|e| e.context("Unable to open meta file"))?; + let stat = f + .metadata() + .map_err(|e| e.context("Unable to stat meta file"))?; if stat.len() == 0 { // Need to sync not only the data but also the file metadata and dirent. - f.write_all(&data).map_err(|e| e.context("Unable to write to meta file"))?; - f.sync_all().map_err(|e| e.context("Unable to sync meta file"))?; + f.write_all(&data) + .map_err(|e| e.context("Unable to write to meta file"))?; + f.sync_all() + .map_err(|e| e.context("Unable to sync meta file"))?; nix::unistd::fsync(dirfd).map_err(|e| e.context("Unable to sync dir"))?; } else if stat.len() == FIXED_DIR_META_LEN as u64 { // Just syncing the data will suffice; existing metadata and dirent are fine. - f.write_all(&data).map_err(|e| e.context("Unable to write to meta file"))?; - f.sync_data().map_err(|e| e.context("Unable to sync meta file"))?; + f.write_all(&data) + .map_err(|e| e.context("Unable to write to meta file"))?; + f.sync_data() + .map_err(|e| e.context("Unable to sync meta file"))?; } else { - bail!("Existing meta file is {}-byte; expected {}", stat.len(), FIXED_DIR_META_LEN); + bail!( + "Existing meta file is {}-byte; expected {}", + stat.len(), + FIXED_DIR_META_LEN + ); } Ok(()) } @@ -187,21 +224,26 @@ impl SampleFileDir { /// /// `db_meta.in_progress_open` should be filled if the directory should be opened in read/write /// mode; absent in read-only mode. - pub fn open(path: &str, db_meta: &schema::DirMeta) - -> Result, Error> { + pub fn open(path: &str, db_meta: &schema::DirMeta) -> Result, Error> { let read_write = db_meta.in_progress_open.is_some(); let s = SampleFileDir::open_self(path, false)?; s.fd.lock(if read_write { - FlockArg::LockExclusiveNonblock - } else { - FlockArg::LockSharedNonblock - }).map_err(|e| e.context(format!("unable to lock dir {}", path)))?; + FlockArg::LockExclusiveNonblock + } else { + FlockArg::LockSharedNonblock + }) + .map_err(|e| e.context(format!("unable to lock dir {}", path)))?; let dir_meta = read_meta(&s.fd).map_err(|e| e.context("unable to read meta file"))?; if !SampleFileDir::consistent(db_meta, &dir_meta) { - let serialized = - db_meta.write_length_delimited_to_bytes().expect("proto3->vec is infallible"); - bail!("metadata mismatch.\ndb: {:#?}\ndir: {:#?}\nserialized db: {:#?}", - db_meta, &dir_meta, &serialized); + let serialized = db_meta + .write_length_delimited_to_bytes() + .expect("proto3->vec is infallible"); + bail!( + "metadata mismatch.\ndb: {:#?}\ndir: {:#?}\nserialized db: {:#?}", + db_meta, + &dir_meta, + &serialized + ); } if db_meta.in_progress_open.is_some() { s.write_meta(db_meta)?; @@ -212,12 +254,17 @@ impl SampleFileDir { /// Returns true if the existing directory and database metadata are consistent; the directory /// is then openable. pub(crate) fn consistent(db_meta: &schema::DirMeta, dir_meta: &schema::DirMeta) -> bool { - if dir_meta.db_uuid != db_meta.db_uuid { return false; } - if dir_meta.dir_uuid != db_meta.dir_uuid { return false; } + if dir_meta.db_uuid != db_meta.db_uuid { + return false; + } + if dir_meta.dir_uuid != db_meta.dir_uuid { + return false; + } - if db_meta.last_complete_open.is_some() && - (db_meta.last_complete_open != dir_meta.last_complete_open && - db_meta.last_complete_open != dir_meta.in_progress_open) { + if db_meta.last_complete_open.is_some() + && (db_meta.last_complete_open != dir_meta.last_complete_open + && db_meta.last_complete_open != dir_meta.in_progress_open) + { return false; } @@ -228,8 +275,10 @@ impl SampleFileDir { true } - pub(crate) fn create(path: &str, db_meta: &schema::DirMeta) - -> Result, Error> { + pub(crate) fn create( + path: &str, + db_meta: &schema::DirMeta, + ) -> Result, Error> { let s = SampleFileDir::open_self(path, true)?; s.fd.lock(FlockArg::LockExclusiveNonblock) .map_err(|e| e.context(format!("unable to lock dir {}", path)))?; @@ -238,7 +287,11 @@ impl SampleFileDir { // Verify metadata. We only care that it hasn't been completely opened. // Partial opening by this or another database is fine; we won't overwrite anything. if old_meta.last_complete_open.is_some() { - bail!("Can't create dir at path {}: is already in use:\n{:?}", path, old_meta); + bail!( + "Can't create dir at path {}: is already in use:\n{:?}", + path, + old_meta + ); } if !s.is_empty()? { bail!("Can't create dir at path {} with existing files", path); @@ -248,8 +301,12 @@ impl SampleFileDir { } pub(crate) fn opendir(&self) -> Result { - nix::dir::Dir::openat(self.fd.as_raw_fd(), ".", OFlag::O_DIRECTORY | OFlag::O_RDONLY, - Mode::empty()) + nix::dir::Dir::openat( + self.fd.as_raw_fd(), + ".", + OFlag::O_DIRECTORY | OFlag::O_RDONLY, + Mode::empty(), + ) } /// Determines if the directory is empty, aside form metadata. @@ -259,7 +316,7 @@ impl SampleFileDir { let e = e?; match e.file_name().to_bytes() { b"." | b".." => continue, - b"meta" => continue, // existing metadata is fine. + b"meta" => continue, // existing metadata is fine. _ => return Ok(false), } } @@ -268,9 +325,7 @@ impl SampleFileDir { fn open_self(path: &str, create: bool) -> Result, Error> { let fd = Fd::open(path, create)?; - Ok(Arc::new(SampleFileDir { - fd, - })) + Ok(Arc::new(SampleFileDir { fd })) } /// Opens the given sample file for reading. @@ -281,15 +336,21 @@ impl SampleFileDir { pub fn create_file(&self, composite_id: CompositeId) -> Result { let p = CompositeIdPath::from(composite_id); - crate::fs::openat(self.fd.0, &p, OFlag::O_WRONLY | OFlag::O_EXCL | OFlag::O_CREAT, - Mode::S_IRUSR | Mode::S_IWUSR) + crate::fs::openat( + self.fd.0, + &p, + OFlag::O_WRONLY | OFlag::O_EXCL | OFlag::O_CREAT, + Mode::S_IRUSR | Mode::S_IWUSR, + ) } pub(crate) fn write_meta(&self, meta: &schema::DirMeta) -> Result<(), Error> { write_meta(self.fd.0, meta) } - pub fn statfs(&self) -> Result { self.fd.statfs() } + pub fn statfs(&self) -> Result { + self.fd.statfs() + } /// Unlinks the given sample file within this directory. pub(crate) fn unlink_file(&self, id: CompositeId) -> Result<(), nix::Error> { @@ -312,11 +373,12 @@ pub(crate) fn parse_id(id: &[u8]) -> Result { } let mut v: u64 = 0; for i in 0..16 { - v = (v << 4) | match id[i] { - b @ b'0'..=b'9' => b - b'0', - b @ b'a'..=b'f' => b - b'a' + 10, - _ => return Err(()), - } as u64; + v = (v << 4) + | match id[i] { + b @ b'0'..=b'9' => b - b'0', + b @ b'a'..=b'f' => b - b'a' + 10, + _ => return Err(()), + } as u64; } Ok(CompositeId(v as i64)) } @@ -353,7 +415,14 @@ mod tests { o.id = u32::max_value(); o.uuid.extend_from_slice(fake_uuid); } - let data = meta.write_length_delimited_to_bytes().expect("proto3->vec is infallible"); - assert!(data.len() <= FIXED_DIR_META_LEN, "{} vs {}", data.len(), FIXED_DIR_META_LEN); + let data = meta + .write_length_delimited_to_bytes() + .expect("proto3->vec is infallible"); + assert!( + data.len() <= FIXED_DIR_META_LEN, + "{} vs {}", + data.len(), + FIXED_DIR_META_LEN + ); } } diff --git a/server/db/fs.rs b/server/db/fs.rs index 13629c1..e03c142 100644 --- a/server/db/fs.rs +++ b/server/db/fs.rs @@ -28,13 +28,17 @@ // You should have received a copy of the GNU General Public License // along with this program. If not, see . -use std::os::unix::io::{FromRawFd, RawFd}; -use nix::NixPath; use nix::fcntl::OFlag; use nix::sys::stat::Mode; +use nix::NixPath; +use std::os::unix::io::{FromRawFd, RawFd}; -pub fn openat(dirfd: RawFd, path: &P, oflag: OFlag, mode: Mode) - -> Result { +pub fn openat( + dirfd: RawFd, + path: &P, + oflag: OFlag, + mode: Mode, +) -> Result { let fd = nix::fcntl::openat(dirfd, path, oflag, mode)?; Ok(unsafe { std::fs::File::from_raw_fd(fd) }) } diff --git a/server/db/lib.rs b/server/db/lib.rs index 4c5bd85..cf05560 100644 --- a/server/db/lib.rs +++ b/server/db/lib.rs @@ -28,7 +28,7 @@ // You should have received a copy of the GNU General Public License // along with this program. If not, see . -#![cfg_attr(all(feature="nightly", test), feature(test))] +#![cfg_attr(all(feature = "nightly", test), feature(test))] pub mod auth; pub mod check; diff --git a/server/db/raw.rs b/server/db/raw.rs index 69587b8..f35a806 100644 --- a/server/db/raw.rs +++ b/server/db/raw.rs @@ -31,9 +31,9 @@ //! Raw database access: SQLite statements which do not touch any cached state. use crate::db::{self, CompositeId, FromSqlUuid}; -use failure::{Error, ResultExt, bail}; -use fnv::FnvHashSet; use crate::recording; +use failure::{bail, Error, ResultExt}; +use fnv::FnvHashSet; use rusqlite::{named_params, params}; use std::ops::Range; use uuid::Uuid; @@ -126,10 +126,13 @@ const LIST_OLDEST_RECORDINGS_SQL: &'static str = r#" /// Lists the specified recordings in ascending order by start time, passing them to a supplied /// function. Given that the function is called with the database lock held, it should be quick. pub(crate) fn list_recordings_by_time( - conn: &rusqlite::Connection, stream_id: i32, desired_time: Range, - f: &mut dyn FnMut(db::ListRecordingsRow) -> Result<(), Error>) -> Result<(), Error> { + conn: &rusqlite::Connection, + stream_id: i32, + desired_time: Range, + f: &mut dyn FnMut(db::ListRecordingsRow) -> Result<(), Error>, +) -> Result<(), Error> { let mut stmt = conn.prepare_cached(LIST_RECORDINGS_BY_TIME_SQL)?; - let rows = stmt.query_named(named_params!{ + let rows = stmt.query_named(named_params! { ":stream_id": stream_id, ":start_time_90k": desired_time.start.0, ":end_time_90k": desired_time.end.0, @@ -139,19 +142,24 @@ pub(crate) fn list_recordings_by_time( /// Lists the specified recordings in ascending order by id. pub(crate) fn list_recordings_by_id( - conn: &rusqlite::Connection, stream_id: i32, desired_ids: Range, - f: &mut dyn FnMut(db::ListRecordingsRow) -> Result<(), Error>) -> Result<(), Error> { + conn: &rusqlite::Connection, + stream_id: i32, + desired_ids: Range, + f: &mut dyn FnMut(db::ListRecordingsRow) -> Result<(), Error>, +) -> Result<(), Error> { let mut stmt = conn.prepare_cached(LIST_RECORDINGS_BY_ID_SQL)?; - let rows = stmt.query_named(named_params!{ + let rows = stmt.query_named(named_params! { ":start": CompositeId::new(stream_id, desired_ids.start).0, ":end": CompositeId::new(stream_id, desired_ids.end).0, })?; list_recordings_inner(rows, true, f) } -fn list_recordings_inner(mut rows: rusqlite::Rows, include_prev: bool, - f: &mut dyn FnMut(db::ListRecordingsRow) -> Result<(), Error>) - -> Result<(), Error> { +fn list_recordings_inner( + mut rows: rusqlite::Rows, + include_prev: bool, + f: &mut dyn FnMut(db::ListRecordingsRow) -> Result<(), Error>, +) -> Result<(), Error> { while let Some(row) = rows.next()? { let wall_duration_90k = row.get(4)?; let media_duration_delta_90k: i32 = row.get(5)?; @@ -177,17 +185,27 @@ fn list_recordings_inner(mut rows: rusqlite::Rows, include_prev: bool, } pub(crate) fn get_db_uuid(conn: &rusqlite::Connection) -> Result { - Ok(conn.query_row("select uuid from meta", params![], |row| -> rusqlite::Result { - let uuid: FromSqlUuid = row.get(0)?; - Ok(uuid.0) - })?) + Ok(conn.query_row( + "select uuid from meta", + params![], + |row| -> rusqlite::Result { + let uuid: FromSqlUuid = row.get(0)?; + Ok(uuid.0) + }, + )?) } /// Inserts the specified recording (for from `try_flush` only). -pub(crate) fn insert_recording(tx: &rusqlite::Transaction, o: &db::Open, id: CompositeId, - r: &db::RecordingToInsert) -> Result<(), Error> { - let mut stmt = tx.prepare_cached(r#" - insert into recording (composite_id, stream_id, open_id, run_offset, flags, +pub(crate) fn insert_recording( + tx: &rusqlite::Transaction, + o: &db::Open, + id: CompositeId, + r: &db::RecordingToInsert, +) -> Result<(), Error> { + let mut stmt = tx + .prepare_cached( + r#" + insert into recording (composite_id, stream_id, open_id, run_offset, flags, sample_file_bytes, start_time_90k, prev_media_duration_90k, prev_runs, wall_duration_90k, media_duration_delta_90k, video_samples, video_sync_samples, video_sample_entry_id) @@ -195,8 +213,10 @@ pub(crate) fn insert_recording(tx: &rusqlite::Transaction, o: &db::Open, id: Com :sample_file_bytes, :start_time_90k, :prev_media_duration_90k, :prev_runs, :wall_duration_90k, :media_duration_delta_90k, :video_samples, :video_sync_samples, :video_sample_entry_id) - "#).with_context(|e| format!("can't prepare recording insert: {}", e))?; - stmt.execute_named(named_params!{ + "#, + ) + .with_context(|e| format!("can't prepare recording insert: {}", e))?; + stmt.execute_named(named_params! { ":composite_id": id.0, ":stream_id": i64::from(id.stream()), ":open_id": o.id, @@ -211,32 +231,49 @@ pub(crate) fn insert_recording(tx: &rusqlite::Transaction, o: &db::Open, id: Com ":video_samples": r.video_samples, ":video_sync_samples": r.video_sync_samples, ":video_sample_entry_id": r.video_sample_entry_id, - }).with_context(|e| format!("unable to insert recording for recording {} {:#?}: {}", - id, r, e))?; + }) + .with_context(|e| { + format!( + "unable to insert recording for recording {} {:#?}: {}", + id, r, e + ) + })?; - let mut stmt = tx.prepare_cached(r#" - insert into recording_integrity (composite_id, local_time_delta_90k, sample_file_blake3) - values (:composite_id, :local_time_delta_90k, :sample_file_blake3) - "#).with_context(|e| format!("can't prepare recording_integrity insert: {}", e))?; + let mut stmt = tx + .prepare_cached( + r#" + insert into recording_integrity (composite_id, local_time_delta_90k, + sample_file_blake3) + values (:composite_id, :local_time_delta_90k, + :sample_file_blake3) + "#, + ) + .with_context(|e| format!("can't prepare recording_integrity insert: {}", e))?; let blake3 = r.sample_file_blake3.as_ref().map(|b| &b[..]); let delta = match r.run_offset { 0 => None, _ => Some(r.local_time_delta.0), }; - stmt.execute_named(named_params!{ + stmt.execute_named(named_params! { ":composite_id": id.0, ":local_time_delta_90k": delta, ":sample_file_blake3": blake3, - }).with_context(|e| format!("unable to insert recording_integrity for {:#?}: {}", r, e))?; + }) + .with_context(|e| format!("unable to insert recording_integrity for {:#?}: {}", r, e))?; - let mut stmt = tx.prepare_cached(r#" - insert into recording_playback (composite_id, video_index) - values (:composite_id, :video_index) - "#).with_context(|e| format!("can't prepare recording_playback insert: {}", e))?; - stmt.execute_named(named_params!{ + let mut stmt = tx + .prepare_cached( + r#" + insert into recording_playback (composite_id, video_index) + values (:composite_id, :video_index) + "#, + ) + .with_context(|e| format!("can't prepare recording_playback insert: {}", e))?; + stmt.execute_named(named_params! { ":composite_id": id.0, ":video_index": &r.video_index, - }).with_context(|e| format!("unable to insert recording_playback for {:#?}: {}", r, e))?; + }) + .with_context(|e| format!("unable to insert recording_playback for {:#?}: {}", r, e))?; Ok(()) } @@ -245,10 +282,13 @@ pub(crate) fn insert_recording(tx: &rusqlite::Transaction, o: &db::Open, id: Com /// table. `sample_file_dir_id` is assumed to be correct. /// /// Returns the number of recordings which were deleted. -pub(crate) fn delete_recordings(tx: &rusqlite::Transaction, sample_file_dir_id: i32, - ids: Range) - -> Result { - let mut insert = tx.prepare_cached(r#" +pub(crate) fn delete_recordings( + tx: &rusqlite::Transaction, + sample_file_dir_id: i32, + ids: Range, +) -> Result { + let mut insert = tx.prepare_cached( + r#" insert into garbage (sample_file_dir_id, composite_id) select :sample_file_dir_id, @@ -258,54 +298,78 @@ pub(crate) fn delete_recordings(tx: &rusqlite::Transaction, sample_file_dir_id: where :start <= composite_id and composite_id < :end - "#)?; - let mut del_playback = tx.prepare_cached(r#" + "#, + )?; + let mut del_playback = tx.prepare_cached( + r#" delete from recording_playback where :start <= composite_id and composite_id < :end - "#)?; - let mut del_integrity = tx.prepare_cached(r#" + "#, + )?; + let mut del_integrity = tx.prepare_cached( + r#" delete from recording_integrity where :start <= composite_id and composite_id < :end - "#)?; - let mut del_main = tx.prepare_cached(r#" + "#, + )?; + let mut del_main = tx.prepare_cached( + r#" delete from recording where :start <= composite_id and composite_id < :end - "#)?; - let n = insert.execute_named(named_params!{ + "#, + )?; + let n = insert.execute_named(named_params! { ":sample_file_dir_id": sample_file_dir_id, ":start": ids.start.0, ":end": ids.end.0, })?; - let p = named_params!{ + let p = named_params! { ":start": ids.start.0, ":end": ids.end.0, }; let n_playback = del_playback.execute_named(p)?; if n_playback != n { - bail!("inserted {} garbage rows but deleted {} recording_playback rows!", n, n_playback); + bail!( + "inserted {} garbage rows but deleted {} recording_playback rows!", + n, + n_playback + ); } let n_integrity = del_integrity.execute_named(p)?; - if n_integrity > n { // fewer is okay; recording_integrity is optional. - bail!("inserted {} garbage rows but deleted {} recording_integrity rows!", n, n_integrity); + if n_integrity > n { + // fewer is okay; recording_integrity is optional. + bail!( + "inserted {} garbage rows but deleted {} recording_integrity rows!", + n, + n_integrity + ); } let n_main = del_main.execute_named(p)?; if n_main != n { - bail!("inserted {} garbage rows but deleted {} recording rows!", n, n_main); + bail!( + "inserted {} garbage rows but deleted {} recording rows!", + n, + n_main + ); } Ok(n) } /// Marks the given sample files as deleted. This shouldn't be called until the files have /// been `unlink()`ed and the parent directory `fsync()`ed. -pub(crate) fn mark_sample_files_deleted(tx: &rusqlite::Transaction, ids: &[CompositeId]) - -> Result<(), Error> { - if ids.is_empty() { return Ok(()); } +pub(crate) fn mark_sample_files_deleted( + tx: &rusqlite::Transaction, + ids: &[CompositeId], +) -> Result<(), Error> { + if ids.is_empty() { + return Ok(()); + } let mut stmt = tx.prepare_cached("delete from garbage where composite_id = ?")?; for &id in ids { let changes = stmt.execute(params![id.0])?; @@ -323,11 +387,13 @@ pub(crate) fn mark_sample_files_deleted(tx: &rusqlite::Transaction, ids: &[Compo } /// Gets the time range of recordings for the given stream. -pub(crate) fn get_range(conn: &rusqlite::Connection, stream_id: i32) - -> Result>, Error> { +pub(crate) fn get_range( + conn: &rusqlite::Connection, + stream_id: i32, +) -> Result>, Error> { // The minimum is straightforward, taking advantage of the start_time_90k index. let mut stmt = conn.prepare_cached(STREAM_MIN_START_SQL)?; - let mut rows = stmt.query_named(named_params!{":stream_id": stream_id})?; + let mut rows = stmt.query_named(named_params! {":stream_id": stream_id})?; let min_start = match rows.next()? { Some(row) => recording::Time(row.get(0)?), None => return Ok(None), @@ -338,15 +404,15 @@ pub(crate) fn get_range(conn: &rusqlite::Connection, stream_id: i32) // last MAX_RECORDING_DURATION must be examined in order to take advantage of the // start_time_90k index. let mut stmt = conn.prepare_cached(STREAM_MAX_START_SQL)?; - let mut rows = stmt.query_named(named_params!{":stream_id": stream_id})?; + let mut rows = stmt.query_named(named_params! {":stream_id": stream_id})?; let mut maxes_opt = None; while let Some(row) = rows.next()? { let row_start = recording::Time(row.get(0)?); let row_duration: i64 = row.get(1)?; let row_end = recording::Time(row_start.0 + row_duration); let maxes = match maxes_opt { - None => row_start .. row_end, - Some(Range{start: s, end: e}) => s .. ::std::cmp::max(e, row_end), + None => row_start..row_end, + Some(Range { start: s, end: e }) => s..::std::cmp::max(e, row_end), }; if row_start.0 <= maxes.start.0 - recording::MAX_RECORDING_WALL_DURATION { break; @@ -354,18 +420,24 @@ pub(crate) fn get_range(conn: &rusqlite::Connection, stream_id: i32) maxes_opt = Some(maxes); } let max_end = match maxes_opt { - Some(Range{start: _, end: e}) => e, - None => bail!("missing max for stream {} which had min {}", stream_id, min_start), + Some(Range { start: _, end: e }) => e, + None => bail!( + "missing max for stream {} which had min {}", + stream_id, + min_start + ), }; - Ok(Some(min_start .. max_end)) + Ok(Some(min_start..max_end)) } /// Lists all garbage ids for the given sample file directory. -pub(crate) fn list_garbage(conn: &rusqlite::Connection, dir_id: i32) - -> Result, Error> { +pub(crate) fn list_garbage( + conn: &rusqlite::Connection, + dir_id: i32, +) -> Result, Error> { let mut garbage = FnvHashSet::default(); - let mut stmt = conn.prepare_cached( - "select composite_id from garbage where sample_file_dir_id = ?")?; + let mut stmt = + conn.prepare_cached("select composite_id from garbage where sample_file_dir_id = ?")?; let mut rows = stmt.query(&[&dir_id])?; while let Some(row) = rows.next()? { garbage.insert(CompositeId(row.get(0)?)); @@ -375,11 +447,13 @@ pub(crate) fn list_garbage(conn: &rusqlite::Connection, dir_id: i32) /// Lists the oldest recordings for a stream, starting with the given id. /// `f` should return true as long as further rows are desired. -pub(crate) fn list_oldest_recordings(conn: &rusqlite::Connection, start: CompositeId, - f: &mut dyn FnMut(db::ListOldestRecordingsRow) -> bool) - -> Result<(), Error> { +pub(crate) fn list_oldest_recordings( + conn: &rusqlite::Connection, + start: CompositeId, + f: &mut dyn FnMut(db::ListOldestRecordingsRow) -> bool, +) -> Result<(), Error> { let mut stmt = conn.prepare_cached(LIST_OLDEST_RECORDINGS_SQL)?; - let mut rows = stmt.query_named(named_params!{ + let mut rows = stmt.query_named(named_params! { ":start": start.0, ":end": CompositeId::new(start.stream() + 1, 0).0, })?; diff --git a/server/db/recording.rs b/server/db/recording.rs index fcb1a2b..f246fc6 100644 --- a/server/db/recording.rs +++ b/server/db/recording.rs @@ -30,7 +30,7 @@ use crate::coding::{append_varint32, decode_varint32, unzigzag32, zigzag32}; use crate::db; -use failure::{Error, bail}; +use failure::{bail, Error}; use log::trace; use std::convert::TryFrom; use std::ops::Range; @@ -40,14 +40,18 @@ pub use base::time::TIME_UNITS_PER_SEC; pub const DESIRED_RECORDING_WALL_DURATION: i64 = 60 * TIME_UNITS_PER_SEC; pub const MAX_RECORDING_WALL_DURATION: i64 = 5 * 60 * TIME_UNITS_PER_SEC; -pub use base::time::Time; pub use base::time::Duration; +pub use base::time::Time; /// Converts from a wall time offset into a recording to a media time offset or vice versa. pub fn rescale(from_off_90k: i32, from_duration_90k: i32, to_duration_90k: i32) -> i32 { - debug_assert!(from_off_90k <= from_duration_90k, - "from_off_90k={} from_duration_90k={} to_duration_90k={}", - from_off_90k, from_duration_90k, to_duration_90k); + debug_assert!( + from_off_90k <= from_duration_90k, + "from_off_90k={} from_duration_90k={} to_duration_90k={}", + from_off_90k, + from_duration_90k, + to_duration_90k + ); if from_duration_90k == 0 { return 0; // avoid a divide by zero. } @@ -56,12 +60,16 @@ pub fn rescale(from_off_90k: i32, from_duration_90k: i32, to_duration_90k: i32) // time is recording::MAX_RECORDING_WALL_DURATION; the max media duration should be // roughly the same (design limit of 500 ppm correction). The final result should fit // within i32. - i32::try_from(i64::from(from_off_90k) * - i64::from(to_duration_90k) / - i64::from(from_duration_90k)) - .map_err(|_| format!("rescale overflow: {} * {} / {} > i32::max_value()", - from_off_90k, to_duration_90k, from_duration_90k)) - .unwrap() + i32::try_from( + i64::from(from_off_90k) * i64::from(to_duration_90k) / i64::from(from_duration_90k), + ) + .map_err(|_| { + format!( + "rescale overflow: {} * {} / {} > i32::max_value()", + from_off_90k, to_duration_90k, from_duration_90k + ) + }) + .unwrap() } /// An iterator through a sample index. @@ -91,12 +99,14 @@ pub struct SampleIndexIterator { impl SampleIndexIterator { pub fn new() -> SampleIndexIterator { - SampleIndexIterator{i_and_is_key: 0, - pos: 0, - start_90k: 0, - duration_90k: 0, - bytes: 0, - bytes_other: 0} + SampleIndexIterator { + i_and_is_key: 0, + pos: 0, + start_90k: 0, + duration_90k: 0, + bytes: 0, + bytes_other: 0, + } } pub fn next(&mut self, data: &[u8]) -> Result { @@ -104,7 +114,7 @@ impl SampleIndexIterator { self.start_90k += self.duration_90k; let i = (self.i_and_is_key & 0x7FFF_FFFF) as usize; if i == data.len() { - return Ok(false) + return Ok(false); } let (raw1, i1) = match decode_varint32(data, i) { Ok(tuple) => tuple, @@ -117,11 +127,17 @@ impl SampleIndexIterator { let duration_90k_delta = unzigzag32(raw1 >> 1); self.duration_90k += duration_90k_delta; if self.duration_90k < 0 { - bail!("negative duration {} after applying delta {}", - self.duration_90k, duration_90k_delta); + bail!( + "negative duration {} after applying delta {}", + self.duration_90k, + duration_90k_delta + ); } if self.duration_90k == 0 && data.len() > i2 { - bail!("zero duration only allowed at end; have {} bytes left", data.len() - i2); + bail!( + "zero duration only allowed at end; have {} bytes left", + data.len() - i2 + ); } let (prev_bytes_key, prev_bytes_nonkey) = match self.is_key() { true => (self.bytes, self.bytes_other), @@ -137,14 +153,21 @@ impl SampleIndexIterator { self.bytes_other = prev_bytes_key; } if self.bytes <= 0 { - bail!("non-positive bytes {} after applying delta {} to key={} frame at ts {}", - self.bytes, bytes_delta, self.is_key(), self.start_90k); + bail!( + "non-positive bytes {} after applying delta {} to key={} frame at ts {}", + self.bytes, + bytes_delta, + self.is_key(), + self.start_90k + ); } Ok(true) } #[inline] - pub fn is_key(&self) -> bool { (self.i_and_is_key & 0x8000_0000) != 0 } + pub fn is_key(&self) -> bool { + (self.i_and_is_key & 0x8000_0000) != 0 + } } #[derive(Debug)] @@ -163,24 +186,33 @@ impl SampleIndexEncoder { } } - pub fn add_sample(&mut self, duration_90k: i32, bytes: i32, is_key: bool, - r: &mut db::RecordingToInsert) { + pub fn add_sample( + &mut self, + duration_90k: i32, + bytes: i32, + is_key: bool, + r: &mut db::RecordingToInsert, + ) { let duration_delta = duration_90k - self.prev_duration_90k; self.prev_duration_90k = duration_90k; r.media_duration_90k += duration_90k; r.sample_file_bytes += bytes; r.video_samples += 1; - let bytes_delta = bytes - if is_key { - let prev = self.prev_bytes_key; - r.video_sync_samples += 1; - self.prev_bytes_key = bytes; - prev - } else { - let prev = self.prev_bytes_nonkey; - self.prev_bytes_nonkey = bytes; - prev - }; - append_varint32((zigzag32(duration_delta) << 1) | (is_key as u32), &mut r.video_index); + let bytes_delta = bytes + - if is_key { + let prev = self.prev_bytes_key; + r.video_sync_samples += 1; + self.prev_bytes_key = bytes; + prev + } else { + let prev = self.prev_bytes_nonkey; + self.prev_bytes_nonkey = bytes; + prev + }; + append_varint32( + (zigzag32(duration_delta) << 1) | (is_key as u32), + &mut r.video_index, + ); append_varint32(zigzag32(bytes_delta), &mut r.video_index); } } @@ -218,10 +250,12 @@ impl Segment { /// The actual range will end at the first frame after the desired range (unless the desired /// range extends beyond the recording). Likewise, the caller is responsible for trimming the /// final frame's duration if desired. - pub fn new(db: &db::LockedDatabase, - recording: &db::ListRecordingsRow, - desired_media_range_90k: Range, - start_at_key: bool) -> Result { + pub fn new( + db: &db::LockedDatabase, + recording: &db::ListRecordingsRow, + desired_media_range_90k: Range, + start_at_key: bool, + ) -> Result { let mut self_ = Segment { id: recording.id, open_id: recording.open_id, @@ -229,28 +263,39 @@ impl Segment { file_end: recording.sample_file_bytes, frames: recording.video_samples as u16, key_frames: recording.video_sync_samples as u16, - video_sample_entry_id_and_trailing_zero: - recording.video_sample_entry_id | - ((((recording.flags & db::RecordingFlags::TrailingZero as i32) != 0) as i32) << 31), + video_sample_entry_id_and_trailing_zero: recording.video_sample_entry_id + | ((((recording.flags & db::RecordingFlags::TrailingZero as i32) != 0) as i32) + << 31), }; - if desired_media_range_90k.start > desired_media_range_90k.end || - desired_media_range_90k.end > recording.media_duration_90k { - bail!("desired media range [{}, {}) invalid for recording of length {}", - desired_media_range_90k.start, desired_media_range_90k.end, - recording.media_duration_90k); + if desired_media_range_90k.start > desired_media_range_90k.end + || desired_media_range_90k.end > recording.media_duration_90k + { + bail!( + "desired media range [{}, {}) invalid for recording of length {}", + desired_media_range_90k.start, + desired_media_range_90k.end, + recording.media_duration_90k + ); } - if desired_media_range_90k.start == 0 && - desired_media_range_90k.end == recording.media_duration_90k { + if desired_media_range_90k.start == 0 + && desired_media_range_90k.end == recording.media_duration_90k + { // Fast path. Existing entry is fine. - trace!("recording::Segment::new fast path, recording={:#?}", recording); - return Ok(self_) + trace!( + "recording::Segment::new fast path, recording={:#?}", + recording + ); + return Ok(self_); } // Slow path. Need to iterate through the index. - trace!("recording::Segment::new slow path, desired_media_range_90k={:?}, recording={:#?}", - desired_media_range_90k, recording); + trace!( + "recording::Segment::new slow path, desired_media_range_90k={:?}, recording={:#?}", + desired_media_range_90k, + recording + ); db.with_recording_playback(self_.id, &mut |playback| { let mut begin = Box::new(SampleIndexIterator::new()); let data = &(&playback).video_index; @@ -274,8 +319,7 @@ impl Segment { }; loop { - if it.start_90k <= desired_media_range_90k.start && - (!start_at_key || it.is_key()) { + if it.start_90k <= desired_media_range_90k.start && (!start_at_key || it.is_key()) { // new start candidate. *begin = it; self_.frames = 0; @@ -293,8 +337,7 @@ impl Segment { self_.begin = Some(begin); self_.file_end = it.pos; self_.video_sample_entry_id_and_trailing_zero = - recording.video_sample_entry_id | - (((it.duration_90k == 0) as i32) << 31); + recording.video_sample_entry_id | (((it.duration_90k == 0) as i32) << 31); Ok(()) })?; Ok(self_) @@ -304,23 +347,33 @@ impl Segment { self.video_sample_entry_id_and_trailing_zero & 0x7FFFFFFF } - pub fn have_trailing_zero(&self) -> bool { self.video_sample_entry_id_and_trailing_zero < 0 } + pub fn have_trailing_zero(&self) -> bool { + self.video_sample_entry_id_and_trailing_zero < 0 + } /// Returns the byte range within the sample file of data associated with this segment. pub fn sample_file_range(&self) -> Range { - self.begin.as_ref().map(|b| b.pos as u64).unwrap_or(0) .. self.file_end as u64 + self.begin.as_ref().map(|b| b.pos as u64).unwrap_or(0)..self.file_end as u64 } /// Returns the actual media start time. As described in `new`, this can be less than the /// desired media start time if there is no key frame at the right position. - pub fn actual_start_90k(&self) -> i32 { self.begin.as_ref().map(|b| b.start_90k).unwrap_or(0) } + pub fn actual_start_90k(&self) -> i32 { + self.begin.as_ref().map(|b| b.start_90k).unwrap_or(0) + } /// Iterates through each frame in the segment. /// Must be called without the database lock held; retrieves video index from the cache. pub fn foreach(&self, playback: &db::RecordingPlayback, mut f: F) -> Result<(), Error> - where F: FnMut(&SampleIndexIterator) -> Result<(), Error> { - trace!("foreach on recording {}: {} frames, actual_start_90k: {}", - self.id, self.frames, self.actual_start_90k()); + where + F: FnMut(&SampleIndexIterator) -> Result<(), Error>, + { + trace!( + "foreach on recording {}: {} frames, actual_start_90k: {}", + self.id, + self.frames, + self.actual_start_90k() + ); let data = &(&playback).video_index; let mut it = match self.begin { Some(ref b) => **b, @@ -338,15 +391,23 @@ impl Segment { let mut have_frame = true; let mut key_frame = 0; - for i in 0 .. self.frames { + for i in 0..self.frames { if !have_frame { - bail!("recording {}: expected {} frames, found only {}", self.id, self.frames, i+1); + bail!( + "recording {}: expected {} frames, found only {}", + self.id, + self.frames, + i + 1 + ); } if it.is_key() { key_frame += 1; if key_frame > self.key_frames { - bail!("recording {}: more than expected {} key frames", - self.id, self.key_frames); + bail!( + "recording {}: more than expected {} key frames", + self.id, + self.key_frames + ); } } @@ -362,8 +423,12 @@ impl Segment { }; } if key_frame < self.key_frames { - bail!("recording {}: expected {} key frames, found only {}", - self.id, self.key_frames, key_frame); + bail!( + "recording {}: expected {} key frames, found only {}", + self.id, + self.key_frames, + key_frame + ); } Ok(()) } @@ -382,9 +447,9 @@ impl Segment { #[cfg(test)] mod tests { - use base::clock::RealClocks; use super::*; use crate::testutil::{self, TestDb}; + use base::clock::RealClocks; /// Tests encoding the example from design/schema.md. #[test] @@ -397,7 +462,10 @@ mod tests { e.add_sample(11, 15, false, &mut r); e.add_sample(10, 12, false, &mut r); e.add_sample(10, 1050, true, &mut r); - assert_eq!(r.video_index, b"\x29\xd0\x0f\x02\x14\x08\x0a\x02\x05\x01\x64"); + assert_eq!( + r.video_index, + b"\x29\xd0\x0f\x02\x14\x08\x0a\x02\x05\x01\x64" + ); assert_eq!(10 + 9 + 11 + 10 + 10, r.media_duration_90k); assert_eq!(5, r.video_samples); assert_eq!(2, r.video_sync_samples); @@ -413,12 +481,13 @@ mod tests { bytes: i32, is_key: bool, } + #[rustfmt::skip] let samples = [ - Sample{duration_90k: 10, bytes: 30000, is_key: true}, - Sample{duration_90k: 9, bytes: 1000, is_key: false}, - Sample{duration_90k: 11, bytes: 1100, is_key: false}, - Sample{duration_90k: 18, bytes: 31000, is_key: true}, - Sample{duration_90k: 0, bytes: 1000, is_key: false}, + Sample { duration_90k: 10, bytes: 30000, is_key: true, }, + Sample { duration_90k: 9, bytes: 1000, is_key: false, }, + Sample { duration_90k: 11, bytes: 1100, is_key: false, }, + Sample { duration_90k: 18, bytes: 31000, is_key: true, }, + Sample { duration_90k: 0, bytes: 1000, is_key: false, }, ]; let mut r = db::RecordingToInsert::default(); let mut e = SampleIndexEncoder::new(); @@ -428,10 +497,14 @@ mod tests { let mut it = SampleIndexIterator::new(); for sample in &samples { assert!(it.next(&r.video_index).unwrap()); - assert_eq!(sample, - &Sample{duration_90k: it.duration_90k, - bytes: it.bytes, - is_key: it.is_key()}); + assert_eq!( + sample, + &Sample { + duration_90k: it.duration_90k, + bytes: it.bytes, + is_key: it.is_key() + } + ); } assert!(!it.next(&r.video_index).unwrap()); } @@ -446,14 +519,26 @@ mod tests { err: &'static str, } let tests = [ - Test{encoded: b"\x80", err: "bad varint 1 at offset 0"}, - Test{encoded: b"\x00\x80", err: "bad varint 2 at offset 1"}, - Test{encoded: b"\x00\x02\x00\x00", - err: "zero duration only allowed at end; have 2 bytes left"}, - Test{encoded: b"\x02\x02", - err: "negative duration -1 after applying delta -1"}, - Test{encoded: b"\x04\x00", - err: "non-positive bytes 0 after applying delta 0 to key=false frame at ts 0"}, + Test { + encoded: b"\x80", + err: "bad varint 1 at offset 0", + }, + Test { + encoded: b"\x00\x80", + err: "bad varint 2 at offset 1", + }, + Test { + encoded: b"\x00\x02\x00\x00", + err: "zero duration only allowed at end; have 2 bytes left", + }, + Test { + encoded: b"\x02\x02", + err: "negative duration -1 after applying delta -1", + }, + Test { + encoded: b"\x04\x00", + err: "non-positive bytes 0 after applying delta 0 to key=false frame at ts 0", + }, ]; for test in &tests { let mut it = SampleIndexIterator::new(); @@ -462,11 +547,18 @@ mod tests { } fn get_frames(db: &db::Database, segment: &Segment, f: F) -> Vec - where F: Fn(&SampleIndexIterator) -> T { + where + F: Fn(&SampleIndexIterator) -> T, + { let mut v = Vec::new(); - db.lock().with_recording_playback(segment.id, &mut |playback| { - segment.foreach(playback, |it| { v.push(f(it)); Ok(()) }) - }).unwrap(); + db.lock() + .with_recording_playback(segment.id, &mut |playback| { + segment.foreach(playback, |it| { + v.push(f(it)); + Ok(()) + }) + }) + .unwrap(); v } @@ -486,8 +578,11 @@ mod tests { let row = db.insert_recording_from_encoder(r); // Time range [2, 2 + 4 + 6 + 8) means the 2nd, 3rd, 4th samples should be // included. - let segment = Segment::new(&db.db.lock(), &row, 2 .. 2+4+6+8, true).unwrap(); - assert_eq!(&get_frames(&db.db, &segment, |it| it.duration_90k), &[4, 6, 8]); + let segment = Segment::new(&db.db.lock(), &row, 2..2 + 4 + 6 + 8, true).unwrap(); + assert_eq!( + &get_frames(&db.db, &segment, |it| it.duration_90k), + &[4, 6, 8] + ); } /// Half sync frames means starting from the last sync frame <= desired point. @@ -505,7 +600,7 @@ mod tests { let row = db.insert_recording_from_encoder(r); // Time range [2 + 4 + 6, 2 + 4 + 6 + 8) means the 4th sample should be included. // The 3rd also gets pulled in because it is a sync frame and the 4th is not. - let segment = Segment::new(&db.db.lock(), &row, 2+4+6 .. 2+4+6+8, true).unwrap(); + let segment = Segment::new(&db.db.lock(), &row, 2 + 4 + 6..2 + 4 + 6 + 8, true).unwrap(); assert_eq!(&get_frames(&db.db, &segment, |it| it.duration_90k), &[6, 8]); } @@ -519,7 +614,7 @@ mod tests { encoder.add_sample(0, 3, true, &mut r); let db = TestDb::new(RealClocks {}); let row = db.insert_recording_from_encoder(r); - let segment = Segment::new(&db.db.lock(), &row, 1 .. 2, true).unwrap(); + let segment = Segment::new(&db.db.lock(), &row, 1..2, true).unwrap(); assert_eq!(&get_frames(&db.db, &segment, |it| it.bytes), &[2, 3]); } @@ -532,7 +627,7 @@ mod tests { encoder.add_sample(1, 1, true, &mut r); let db = TestDb::new(RealClocks {}); let row = db.insert_recording_from_encoder(r); - let segment = Segment::new(&db.db.lock(), &row, 0 .. 0, true).unwrap(); + let segment = Segment::new(&db.db.lock(), &row, 0..0, true).unwrap(); assert_eq!(&get_frames(&db.db, &segment, |it| it.bytes), &[1]); } @@ -550,8 +645,11 @@ mod tests { } let db = TestDb::new(RealClocks {}); let row = db.insert_recording_from_encoder(r); - let segment = Segment::new(&db.db.lock(), &row, 0 .. 2+4+6+8+10, true).unwrap(); - assert_eq!(&get_frames(&db.db, &segment, |it| it.duration_90k), &[2, 4, 6, 8, 10]); + let segment = Segment::new(&db.db.lock(), &row, 0..2 + 4 + 6 + 8 + 10, true).unwrap(); + assert_eq!( + &get_frames(&db.db, &segment, |it| it.duration_90k), + &[2, 4, 6, 8, 10] + ); } #[test] @@ -564,14 +662,14 @@ mod tests { encoder.add_sample(0, 3, true, &mut r); let db = TestDb::new(RealClocks {}); let row = db.insert_recording_from_encoder(r); - let segment = Segment::new(&db.db.lock(), &row, 0 .. 2, true).unwrap(); + let segment = Segment::new(&db.db.lock(), &row, 0..2, true).unwrap(); assert_eq!(&get_frames(&db.db, &segment, |it| it.bytes), &[1, 2, 3]); } // TODO: test segment error cases involving mismatch between row frames/key_frames and index. } -#[cfg(all(test, feature="nightly"))] +#[cfg(all(test, feature = "nightly"))] mod bench { extern crate test; diff --git a/server/db/signal.rs b/server/db/signal.rs index 415dfd4..6267a10 100644 --- a/server/db/signal.rs +++ b/server/db/signal.rs @@ -28,16 +28,16 @@ // You should have received a copy of the GNU General Public License // along with this program. If not, see . -use base::bail_t; use crate::coding; use crate::db::FromSqlUuid; use crate::recording; -use failure::{Error, bail, format_err}; +use base::bail_t; +use failure::{bail, format_err, Error}; use fnv::FnvHashMap; use log::debug; -use rusqlite::{Connection, Transaction, params}; -use std::collections::{BTreeMap, BTreeSet}; +use rusqlite::{params, Connection, Transaction}; use std::collections::btree_map::Entry; +use std::collections::{BTreeMap, BTreeSet}; use std::ops::Range; use uuid::Uuid; @@ -132,7 +132,9 @@ impl Point { /// `from` must be an iterator of `(signal, state)` with signal numbers in monotonically increasing /// order. fn append_serialized<'a, I>(from: I, to: &mut Vec) -where I: IntoIterator { +where + I: IntoIterator, +{ let mut next_allowed = 0; for (&signal, &state) in from.into_iter() { assert!(signal >= next_allowed); @@ -170,15 +172,18 @@ impl<'a> PointDataIterator<'a> { if self.cur_pos == self.data.len() { return Ok(None); } - let (signal_delta, p) = coding::decode_varint32(self.data, self.cur_pos) - .map_err(|()| format_err!("varint32 decode failure; data={:?} pos={}", - self.data, self.cur_pos))?; + let (signal_delta, p) = coding::decode_varint32(self.data, self.cur_pos).map_err(|()| { + format_err!( + "varint32 decode failure; data={:?} pos={}", + self.data, + self.cur_pos + ) + })?; let (state, p) = coding::decode_varint32(self.data, p) - .map_err(|()| format_err!("varint32 decode failure; data={:?} pos={}", - self.data, p))?; - let signal = self.cur_signal.checked_add(signal_delta) - .ok_or_else(|| format_err!("signal overflow: {} + {}", - self.cur_signal, signal_delta))?; + .map_err(|()| format_err!("varint32 decode failure; data={:?} pos={}", self.data, p))?; + let signal = self.cur_signal.checked_add(signal_delta).ok_or_else(|| { + format_err!("signal overflow: {} + {}", self.cur_signal, signal_delta) + })?; if state > u16::max_value() as u32 { bail!("state overflow: {}", state); } @@ -221,7 +226,9 @@ pub struct ListStateChangesRow { impl State { pub fn init(conn: &Connection) -> Result { let max_signal_changes: Option = - conn.query_row("select max_signal_changes from meta", params![], |row| row.get(0))?; + conn.query_row("select max_signal_changes from meta", params![], |row| { + row.get(0) + })?; let mut signals_by_id = State::init_signals(conn)?; State::fill_signal_cameras(conn, &mut signals_by_id)?; Ok(State { @@ -234,8 +241,10 @@ impl State { } pub fn list_changes_by_time( - &self, desired_time: Range, f: &mut dyn FnMut(&ListStateChangesRow)) { - + &self, + desired_time: Range, + f: &mut dyn FnMut(&ListStateChangesRow), + ) { // First find the state immediately before. If it exists, include it. if let Some((&when, p)) = self.points_by_time.range(..desired_time.start).next_back() { for (&signal, &state) in &p.after() { @@ -261,8 +270,11 @@ impl State { } pub fn update_signals( - &mut self, when: Range, signals: &[u32], states: &[u16]) - -> Result<(), base::Error> { + &mut self, + when: Range, + signals: &[u32], + states: &[u16], + ) -> Result<(), base::Error> { // Do input validation before any mutation. self.update_signals_validate(signals, states)?; @@ -294,11 +306,19 @@ impl State { None => return, Some(p) => p, }; - debug!("Performing signal GC: have {} points, want only {}, so removing {}", - self.points_by_time.len(), max, to_remove); + debug!( + "Performing signal GC: have {} points, want only {}, so removing {}", + self.points_by_time.len(), + max, + to_remove + ); - let remove: smallvec::SmallVec<[recording::Time; 4]> = - self.points_by_time.keys().take(to_remove).map(|p| *p).collect(); + let remove: smallvec::SmallVec<[recording::Time; 4]> = self + .points_by_time + .keys() + .take(to_remove) + .map(|p| *p) + .collect(); for p in &remove { self.points_by_time.remove(p); @@ -320,14 +340,20 @@ impl State { None => bail_t!(InvalidArgument, "unknown signal {}", signal), Some(ref s) => { let empty = Vec::new(); - let states = self.types_by_uuid.get(&s.type_) - .map(|t| &t.states) - .unwrap_or(&empty); + let states = self + .types_by_uuid + .get(&s.type_) + .map(|t| &t.states) + .unwrap_or(&empty); if state != 0 && states.binary_search_by_key(&state, |s| s.value).is_err() { - bail_t!(FailedPrecondition, "signal {} specifies unknown state {}", - signal, state); + bail_t!( + FailedPrecondition, + "signal {} specifies unknown state {}", + signal, + state + ); } - }, + } } next_allowed = signal + 1; } @@ -354,7 +380,10 @@ impl State { // Any existing changes should still be applied. They win over reverting to prev. let mut it = p.changes(); while let Some((signal, state)) = it.next().expect("in-mem changes is valid") { - changes.entry(signal).and_modify(|e| *e = state).or_insert(state); + changes + .entry(signal) + .and_modify(|e| *e = state) + .or_insert(state); } self.dirty_by_time.insert(t); p.swap(&mut Point::new(&prev, &serialize(&changes))); @@ -374,20 +403,25 @@ impl State { return; } self.dirty_by_time.insert(end); - self.points_by_time.insert(end, Point::new(&prev, &serialize(&changes))); + self.points_by_time + .insert(end, Point::new(&prev, &serialize(&changes))); } /// Helper for `update_signals_end`. Adjusts `prev` (the state prior to the end point) to /// reflect the desired update (in `signals` and `states`). Adjusts `changes` (changes to /// execute at the end point) to undo the change. - fn update_signals_end_maps(signals: &[u32], states: &[u16], prev: &mut BTreeMap, - changes: &mut BTreeMap) { + fn update_signals_end_maps( + signals: &[u32], + states: &[u16], + prev: &mut BTreeMap, + changes: &mut BTreeMap, + ) { for (&signal, &state) in signals.iter().zip(states) { match prev.entry(signal) { Entry::Vacant(e) => { changes.insert(signal, 0); e.insert(state); - }, + } Entry::Occupied(mut e) => { if state == 0 { changes.insert(signal, *e.get()); @@ -396,7 +430,7 @@ impl State { changes.insert(signal, *e.get()); *e.get_mut() = state; } - }, + } } } } @@ -421,13 +455,13 @@ impl State { *e.get_mut() = state; } } - }, + } Entry::Vacant(e) => { if signal != 0 { dirty = true; e.insert(state); } - }, + } } } if dirty { @@ -456,14 +490,19 @@ impl State { } self.dirty_by_time.insert(start); - self.points_by_time.insert(start, Point::new(&prev, &serialize(&changes))); + self.points_by_time + .insert(start, Point::new(&prev, &serialize(&changes))); } /// Helper for `update_signals` to apply all points in `(when.start, when.end)`. - fn update_signals_middle(&mut self, when: Range, signals: &[u32], - states: &[u16]) { + fn update_signals_middle( + &mut self, + when: Range, + signals: &[u32], + states: &[u16], + ) { let mut to_delete = Vec::new(); - let after_start = recording::Time(when.start.0+1); + let after_start = recording::Time(when.start.0 + 1); for (&t, ref mut p) in self.points_by_time.range_mut(after_start..when.end) { let mut prev = p.prev().to_map().expect("in-mem prev is valid"); @@ -476,7 +515,7 @@ impl State { } else if *e.get() != state { *e.get_mut() = state; } - }, + } Entry::Vacant(e) => { if state != 0 { e.insert(state); @@ -486,14 +525,16 @@ impl State { } // Trim changes to omit any change to signals. - let mut changes = Vec::with_capacity(3*signals.len()); + let mut changes = Vec::with_capacity(3 * signals.len()); let mut it = p.changes(); let mut next_allowed = 0; let mut dirty = false; while let Some((signal, state)) = it.next().expect("in-memory changes is valid") { - if signals.binary_search(&signal).is_ok() { // discard. + if signals.binary_search(&signal).is_ok() { + // discard. dirty = true; - } else { // keep. + } else { + // keep. assert!(signal >= next_allowed); coding::append_varint32(signal - next_allowed, &mut changes); coding::append_varint32(state as u32, &mut changes); @@ -521,24 +562,25 @@ impl State { /// The caller is expected to call `post_flush` afterward if the transaction is /// successfully committed. No mutations should happen between these calls. pub fn flush(&mut self, tx: &Transaction) -> Result<(), Error> { - let mut i_stmt = tx.prepare(r#" + let mut i_stmt = tx.prepare( + r#" insert or replace into signal_change (time_90k, changes) values (?, ?) - "#)?; - let mut d_stmt = tx.prepare(r#" + "#, + )?; + let mut d_stmt = tx.prepare( + r#" delete from signal_change where time_90k = ? - "#)?; + "#, + )?; for &t in &self.dirty_by_time { match self.points_by_time.entry(t) { Entry::Occupied(ref e) => { let p = e.get(); - i_stmt.execute(params![ - t.0, - &p.data[p.changes_off..], - ])?; - }, + i_stmt.execute(params![t.0, &p.data[p.changes_off..],])?; + } Entry::Vacant(_) => { d_stmt.execute(params![t.0])?; - }, + } } } Ok(()) @@ -553,7 +595,8 @@ impl State { fn init_signals(conn: &Connection) -> Result, Error> { let mut signals = BTreeMap::new(); - let mut stmt = conn.prepare(r#" + let mut stmt = conn.prepare( + r#" select id, source_uuid, @@ -561,35 +604,41 @@ impl State { short_name from signal - "#)?; + "#, + )?; let mut rows = stmt.query(params![])?; while let Some(row) = rows.next()? { let id = row.get(0)?; let source: FromSqlUuid = row.get(1)?; let type_: FromSqlUuid = row.get(2)?; - signals.insert(id, Signal { + signals.insert( id, - source: source.0, - type_: type_.0, - short_name: row.get(3)?, - cameras: Vec::new(), - }); + Signal { + id, + source: source.0, + type_: type_.0, + short_name: row.get(3)?, + cameras: Vec::new(), + }, + ); } Ok(signals) } fn init_points(conn: &Connection) -> Result, Error> { - let mut stmt = conn.prepare(r#" + let mut stmt = conn.prepare( + r#" select time_90k, changes from signal_change order by time_90k - "#)?; + "#, + )?; let mut rows = stmt.query(params![])?; let mut points = BTreeMap::new(); - let mut cur = BTreeMap::new(); // latest signal -> state, where state != 0 + let mut cur = BTreeMap::new(); // latest signal -> state, where state != 0 while let Some(row) = rows.next()? { let time_90k = recording::Time(row.get(0)?); let changes = row.get_raw_checked(1)?.as_blob()?; @@ -607,9 +656,12 @@ impl State { } /// Fills the `cameras` field of the `Signal` structs within the supplied `signals`. - fn fill_signal_cameras(conn: &Connection, signals: &mut BTreeMap) - -> Result<(), Error> { - let mut stmt = conn.prepare(r#" + fn fill_signal_cameras( + conn: &Connection, + signals: &mut BTreeMap, + ) -> Result<(), Error> { + let mut stmt = conn.prepare( + r#" select signal_id, camera_id, @@ -617,13 +669,14 @@ impl State { from signal_camera order by signal_id, camera_id - "#)?; + "#, + )?; let mut rows = stmt.query(params![])?; while let Some(row) = rows.next()? { let signal_id = row.get(0)?; - let s = signals.get_mut(&signal_id) - .ok_or_else(|| format_err!("signal_camera row for unknown signal id {}", - signal_id))?; + let s = signals.get_mut(&signal_id).ok_or_else(|| { + format_err!("signal_camera row for unknown signal id {}", signal_id) + })?; let type_ = row.get(2)?; s.cameras.push(SignalCamera { camera_id: row.get(1)?, @@ -639,7 +692,8 @@ impl State { fn init_types(conn: &Connection) -> Result, Error> { let mut types = FnvHashMap::default(); - let mut stmt = conn.prepare(r#" + let mut stmt = conn.prepare( + r#" select type_uuid, value, @@ -649,22 +703,31 @@ impl State { from signal_type_enum order by type_uuid, value - "#)?; + "#, + )?; let mut rows = stmt.query(params![])?; while let Some(row) = rows.next()? { let type_: FromSqlUuid = row.get(0)?; - types.entry(type_.0).or_insert_with(Type::default).states.push(TypeState { - value: row.get(1)?, - name: row.get(2)?, - motion: row.get(3)?, - color: row.get(4)?, - }); + types + .entry(type_.0) + .or_insert_with(Type::default) + .states + .push(TypeState { + value: row.get(1)?, + name: row.get(2)?, + motion: row.get(3)?, + color: row.get(4)?, + }); } Ok(types) } - pub fn signals_by_id(&self) -> &BTreeMap { &self.signals_by_id } - pub fn types_by_uuid(&self) -> &FnvHashMap { & self.types_by_uuid } + pub fn signals_by_id(&self) -> &BTreeMap { + &self.signals_by_id + } + pub fn types_by_uuid(&self) -> &FnvHashMap { + &self.types_by_uuid + } } /// Representation of a `signal` row. @@ -698,9 +761,9 @@ pub struct Type { #[cfg(test)] mod tests { + use super::*; use crate::{db, testutil}; use rusqlite::Connection; - use super::*; #[test] fn test_point_data_it() { @@ -719,8 +782,10 @@ mod tests { let mut conn = Connection::open_in_memory().unwrap(); db::init(&mut conn).unwrap(); let s = State::init(&conn).unwrap(); - s.list_changes_by_time(recording::Time::min_value() .. recording::Time::max_value(), - &mut |_r| panic!("no changes expected")); + s.list_changes_by_time( + recording::Time::min_value()..recording::Time::max_value(), + &mut |_r| panic!("no changes expected"), + ); } #[test] @@ -728,7 +793,8 @@ mod tests { testutil::init(); let mut conn = Connection::open_in_memory().unwrap(); db::init(&mut conn).unwrap(); - conn.execute_batch(r#" + conn.execute_batch( + r#" update meta set max_signal_changes = 2; insert into signal (id, source_uuid, type_uuid, short_name) @@ -740,12 +806,16 @@ mod tests { insert into signal_type_enum (type_uuid, value, name, motion, color) values (x'EE66270FD9C648198B339720D4CBCA6B', 1, 'still', 0, 'black'), (x'EE66270FD9C648198B339720D4CBCA6B', 2, 'moving', 1, 'red'); - "#).unwrap(); + "#, + ) + .unwrap(); let mut s = State::init(&conn).unwrap(); - s.list_changes_by_time(recording::Time::min_value() .. recording::Time::max_value(), - &mut |_r| panic!("no changes expected")); + s.list_changes_by_time( + recording::Time::min_value()..recording::Time::max_value(), + &mut |_r| panic!("no changes expected"), + ); const START: recording::Time = recording::Time(140067462600000); // 2019-04-26T11:59:00 - const NOW: recording::Time = recording::Time(140067468000000); // 2019-04-26T12:00:00 + const NOW: recording::Time = recording::Time(140067468000000); // 2019-04-26T12:00:00 s.update_signals(START..NOW, &[1, 2], &[2, 1]).unwrap(); let mut rows = Vec::new(); @@ -770,10 +840,12 @@ mod tests { signal: 2, state: 0, }, - ]; + ]; - s.list_changes_by_time(recording::Time::min_value() .. recording::Time::max_value(), - &mut |r| rows.push(*r)); + s.list_changes_by_time( + recording::Time::min_value()..recording::Time::max_value(), + &mut |r| rows.push(*r), + ); assert_eq!(&rows[..], EXPECTED); { @@ -785,8 +857,10 @@ mod tests { drop(s); let mut s = State::init(&conn).unwrap(); rows.clear(); - s.list_changes_by_time(recording::Time::min_value() .. recording::Time::max_value(), - &mut |r| rows.push(*r)); + s.list_changes_by_time( + recording::Time::min_value()..recording::Time::max_value(), + &mut |r| rows.push(*r), + ); assert_eq!(&rows[..], EXPECTED); // Go through it again. This time, hit the max number of signals, forcing START to be @@ -815,9 +889,11 @@ mod tests { signal: 2, state: 0, }, - ]; - s.list_changes_by_time(recording::Time::min_value() .. recording::Time::max_value(), - &mut |r| rows.push(*r)); + ]; + s.list_changes_by_time( + recording::Time::min_value()..recording::Time::max_value(), + &mut |r| rows.push(*r), + ); assert_eq!(&rows[..], EXPECTED2); { @@ -828,8 +904,10 @@ mod tests { drop(s); let s = State::init(&conn).unwrap(); rows.clear(); - s.list_changes_by_time(recording::Time::min_value() .. recording::Time::max_value(), - &mut |r| rows.push(*r)); + s.list_changes_by_time( + recording::Time::min_value()..recording::Time::max_value(), + &mut |r| rows.push(*r), + ); assert_eq!(&rows[..], EXPECTED2); } } diff --git a/server/db/testutil.rs b/server/db/testutil.rs index c8df386..4c62da7 100644 --- a/server/db/testutil.rs +++ b/server/db/testutil.rs @@ -28,9 +28,10 @@ // You should have received a copy of the GNU General Public License // along with this program. If not, see . -use base::clock::Clocks; use crate::db; use crate::dir; +use crate::writer; +use base::clock::Clocks; use fnv::FnvHashMap; use mylog; use rusqlite; @@ -40,7 +41,6 @@ use std::thread; use tempdir::TempDir; use time; use uuid::Uuid; -use crate::writer; static INIT: parking_lot::Once = parking_lot::Once::new(); @@ -101,29 +101,39 @@ impl TestDb { { let mut l = db.lock(); sample_file_dir_id = l.add_sample_file_dir(path.to_owned()).unwrap(); - assert_eq!(TEST_CAMERA_ID, l.add_camera(db::CameraChange { - short_name: "test camera".to_owned(), - description: "".to_owned(), - onvif_host: "test-camera".to_owned(), - username: "foo".to_owned(), - password: "bar".to_owned(), - streams: [ - db::StreamChange { - sample_file_dir_id: Some(sample_file_dir_id), - rtsp_url: "rtsp://test-camera/main".to_owned(), - record: true, - flush_if_sec, - }, - Default::default(), - ], - }).unwrap()); + assert_eq!( + TEST_CAMERA_ID, + l.add_camera(db::CameraChange { + short_name: "test camera".to_owned(), + description: "".to_owned(), + onvif_host: "test-camera".to_owned(), + username: "foo".to_owned(), + password: "bar".to_owned(), + streams: [ + db::StreamChange { + sample_file_dir_id: Some(sample_file_dir_id), + rtsp_url: "rtsp://test-camera/main".to_owned(), + record: true, + flush_if_sec, + }, + Default::default(), + ], + }) + .unwrap() + ); test_camera_uuid = l.cameras_by_id().get(&TEST_CAMERA_ID).unwrap().uuid; l.update_retention(&[db::RetentionChange { stream_id: TEST_STREAM_ID, new_record: true, new_limit: 1048576, - }]).unwrap(); - dir = l.sample_file_dirs_by_id().get(&sample_file_dir_id).unwrap().get().unwrap(); + }]) + .unwrap(); + dir = l + .sample_file_dirs_by_id() + .get(&sample_file_dir_id) + .unwrap() + .get() + .unwrap(); } let mut dirs_by_stream_id = FnvHashMap::default(); dirs_by_stream_id.insert(TEST_STREAM_ID, dir.clone()); @@ -142,48 +152,63 @@ impl TestDb { /// Creates a recording with a fresh `RecordingToInsert` row which has been touched only by /// a `SampleIndexEncoder`. Fills in a video sample entry id and such to make it valid. /// There will no backing sample file, so it won't be possible to generate a full `.mp4`. - pub fn insert_recording_from_encoder(&self, r: db::RecordingToInsert) - -> db::ListRecordingsRow { + pub fn insert_recording_from_encoder(&self, r: db::RecordingToInsert) -> db::ListRecordingsRow { use crate::recording::{self, TIME_UNITS_PER_SEC}; let mut db = self.db.lock(); - let video_sample_entry_id = db.insert_video_sample_entry(db::VideoSampleEntryToInsert { + let video_sample_entry_id = db + .insert_video_sample_entry(db::VideoSampleEntryToInsert { + width: 1920, + height: 1080, + pasp_h_spacing: 1, + pasp_v_spacing: 1, + data: [0u8; 100].to_vec(), + rfc6381_codec: "avc1.000000".to_owned(), + }) + .unwrap(); + let (id, _) = db + .add_recording( + TEST_STREAM_ID, + db::RecordingToInsert { + start: recording::Time(1430006400i64 * TIME_UNITS_PER_SEC), + video_sample_entry_id, + wall_duration_90k: r.media_duration_90k, + ..r + }, + ) + .unwrap(); + db.mark_synced(id).unwrap(); + db.flush("create_recording_from_encoder").unwrap(); + let mut row = None; + db.list_recordings_by_id( + TEST_STREAM_ID, + id.recording()..id.recording() + 1, + &mut |r| { + row = Some(r); + Ok(()) + }, + ) + .unwrap(); + row.unwrap() + } +} + +// For benchmarking +#[cfg(feature = "nightly")] +pub fn add_dummy_recordings_to_db(db: &db::Database, num: usize) { + use crate::recording::{self, TIME_UNITS_PER_SEC}; + let mut data = Vec::new(); + data.extend_from_slice(include_bytes!("testdata/video_sample_index.bin")); + let mut db = db.lock(); + let video_sample_entry_id = db + .insert_video_sample_entry(db::VideoSampleEntryToInsert { width: 1920, height: 1080, pasp_h_spacing: 1, pasp_v_spacing: 1, data: [0u8; 100].to_vec(), rfc6381_codec: "avc1.000000".to_owned(), - }).unwrap(); - let (id, _) = db.add_recording(TEST_STREAM_ID, db::RecordingToInsert { - start: recording::Time(1430006400i64 * TIME_UNITS_PER_SEC), - video_sample_entry_id, - wall_duration_90k: r.media_duration_90k, - ..r - }).unwrap(); - db.mark_synced(id).unwrap(); - db.flush("create_recording_from_encoder").unwrap(); - let mut row = None; - db.list_recordings_by_id(TEST_STREAM_ID, id.recording() .. id.recording()+1, - &mut |r| { row = Some(r); Ok(()) }).unwrap(); - row.unwrap() - } -} - -// For benchmarking -#[cfg(feature="nightly")] -pub fn add_dummy_recordings_to_db(db: &db::Database, num: usize) { - use crate::recording::{self, TIME_UNITS_PER_SEC}; - let mut data = Vec::new(); - data.extend_from_slice(include_bytes!("testdata/video_sample_index.bin")); - let mut db = db.lock(); - let video_sample_entry_id = db.insert_video_sample_entry(db::VideoSampleEntryToInsert { - width: 1920, - height: 1080, - pasp_h_spacing: 1, - pasp_v_spacing: 1, - data: [0u8; 100].to_vec(), - rfc6381_codec: "avc1.000000".to_owned(), - }).unwrap(); + }) + .unwrap(); let mut recording = db::RecordingToInsert { sample_file_bytes: 30104460, start: recording::Time(1430006400i64 * TIME_UNITS_PER_SEC), diff --git a/server/db/upgrade/mod.rs b/server/db/upgrade/mod.rs index 78b5d58..c44c370 100644 --- a/server/db/upgrade/mod.rs +++ b/server/db/upgrade/mod.rs @@ -31,14 +31,13 @@ /// Upgrades the database schema. /// /// See `guide/schema.md` for more information. - use crate::db; -use failure::{Error, bail}; +use failure::{bail, Error}; use log::info; -use std::ffi::CStr; -use std::io::Write; use nix::NixPath; use rusqlite::params; +use std::ffi::CStr; +use std::io::Write; use uuid::Uuid; mod v0_to_v1; @@ -59,10 +58,16 @@ pub struct Args<'a> { } fn set_journal_mode(conn: &rusqlite::Connection, requested: &str) -> Result<(), Error> { - assert!(!requested.contains(';')); // quick check for accidental sql injection. - let actual = conn.query_row(&format!("pragma journal_mode = {}", requested), params![], - |row| row.get::<_, String>(0))?; - info!("...database now in journal_mode {} (requested {}).", actual, requested); + assert!(!requested.contains(';')); // quick check for accidental sql injection. + let actual = conn.query_row( + &format!("pragma journal_mode = {}", requested), + params![], + |row| row.get::<_, String>(0), + )?; + info!( + "...database now in journal_mode {} (requested {}).", + actual, requested + ); Ok(()) } @@ -78,24 +83,31 @@ fn upgrade(args: &Args, target_ver: i32, conn: &mut rusqlite::Connection) -> Res { assert_eq!(upgraders.len(), db::EXPECTED_VERSION as usize); - let old_ver = - conn.query_row("select max(id) from version", params![], - |row| row.get(0))?; + let old_ver = conn.query_row("select max(id) from version", params![], |row| row.get(0))?; if old_ver > db::EXPECTED_VERSION { - bail!("Database is at version {}, later than expected {}", - old_ver, db::EXPECTED_VERSION); + bail!( + "Database is at version {}, later than expected {}", + old_ver, + db::EXPECTED_VERSION + ); } else if old_ver < 0 { bail!("Database is at negative version {}!", old_ver); } - info!("Upgrading database from version {} to version {}...", old_ver, target_ver); - for ver in old_ver .. target_ver { + info!( + "Upgrading database from version {} to version {}...", + old_ver, target_ver + ); + for ver in old_ver..target_ver { info!("...from version {} to version {}", ver, ver + 1); let tx = conn.transaction()?; upgraders[ver as usize](&args, &tx)?; - tx.execute(r#" + tx.execute( + r#" insert into version (id, unix_time, notes) values (?, cast(strftime('%s', 'now') as int32), ?) - "#, params![ver + 1, UPGRADE_NOTES])?; + "#, + params![ver + 1, UPGRADE_NOTES], + )?; tx.commit()?; } } @@ -117,10 +129,12 @@ pub fn run(args: &Args, conn: &mut rusqlite::Connection) -> Result<(), Error> { // non-WAL mode. https://www.sqlite.org/wal.html if !args.no_vacuum { info!("...vacuuming database after upgrade."); - conn.execute_batch(r#" + conn.execute_batch( + r#" pragma page_size = 16384; vacuum; - "#)?; + "#, + )?; } set_journal_mode(&conn, "wal")?; @@ -142,11 +156,17 @@ impl UuidPath { } impl NixPath for UuidPath { - fn is_empty(&self) -> bool { false } - fn len(&self) -> usize { 36 } + fn is_empty(&self) -> bool { + false + } + fn len(&self) -> usize { + 36 + } fn with_nix_path(&self, f: F) -> Result - where F: FnOnce(&CStr) -> T { + where + F: FnOnce(&CStr) -> T, + { let p = CStr::from_bytes_with_nul(&self.0[..]).expect("no interior nuls"); Ok(f(p)) } @@ -154,14 +174,13 @@ impl NixPath for UuidPath { #[cfg(test)] mod tests { + use super::*; use crate::compare; use crate::testutil; use failure::ResultExt; use fnv::FnvHashMap; - use super::*; - const BAD_ANAMORPHIC_VIDEO_SAMPLE_ENTRY: &[u8] = - b"\x00\x00\x00\x84\x61\x76\x63\x31\x00\x00\ + const BAD_ANAMORPHIC_VIDEO_SAMPLE_ENTRY: &[u8] = b"\x00\x00\x00\x84\x61\x76\x63\x31\x00\x00\ \x00\x00\x00\x00\x00\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\ \x00\x00\x00\x00\x00\x00\x01\x40\x00\xf0\x00\x48\x00\x00\x00\x48\ \x00\x00\x00\x00\x00\x00\x00\x01\x00\x00\x00\x00\x00\x00\x00\x00\ @@ -209,29 +228,44 @@ mod tests { //let path = tmpdir.path().to_str().ok_or_else(|| format_err!("invalid UTF-8"))?.to_owned(); let mut upgraded = new_conn()?; upgraded.execute_batch(include_str!("v0.sql"))?; - upgraded.execute_batch(r#" + upgraded.execute_batch( + r#" insert into camera (id, uuid, short_name, description, host, username, password, main_rtsp_path, sub_rtsp_path, retain_bytes) values (1, zeroblob(16), 'test camera', 'desc', 'host', 'user', 'pass', 'main', 'sub', 42); - "#)?; - upgraded.execute(r#" + "#, + )?; + upgraded.execute( + r#" insert into video_sample_entry (id, sha1, width, height, data) values (1, X'0000000000000000000000000000000000000000', 1920, 1080, ?); - "#, params![testutil::TEST_VIDEO_SAMPLE_ENTRY_DATA])?; - upgraded.execute(r#" + "#, + params![testutil::TEST_VIDEO_SAMPLE_ENTRY_DATA], + )?; + upgraded.execute( + r#" insert into video_sample_entry (id, sha1, width, height, data) values (2, X'0000000000000000000000000000000000000001', 320, 240, ?); - "#, params![BAD_ANAMORPHIC_VIDEO_SAMPLE_ENTRY])?; - upgraded.execute(r#" + "#, + params![BAD_ANAMORPHIC_VIDEO_SAMPLE_ENTRY], + )?; + upgraded.execute( + r#" insert into video_sample_entry (id, sha1, width, height, data) values (3, X'0000000000000000000000000000000000000002', 704, 480, ?); - "#, params![GOOD_ANAMORPHIC_VIDEO_SAMPLE_ENTRY])?; - upgraded.execute(r#" + "#, + params![GOOD_ANAMORPHIC_VIDEO_SAMPLE_ENTRY], + )?; + upgraded.execute( + r#" insert into video_sample_entry (id, sha1, width, height, data) values (4, X'0000000000000000000000000000000000000003', 704, 480, ?); - "#, params![GOOD_ANAMORPHIC_VIDEO_SAMPLE_ENTRY])?; - upgraded.execute_batch(r#" + "#, + params![GOOD_ANAMORPHIC_VIDEO_SAMPLE_ENTRY], + )?; + upgraded.execute_batch( + r#" insert into recording (id, camera_id, sample_file_bytes, start_time_90k, duration_90k, local_time_delta_90k, video_samples, video_sync_samples, video_sample_entry_id, sample_file_uuid, sample_file_sha1, @@ -244,7 +278,8 @@ mod tests { X'C94D4D0B533746059CD40B29039E641E', zeroblob(20), X'00'); insert into reserved_sample_files values (X'51EF700C933E4197AAE4EE8161E94221', 0), (X'E69D45E8CBA64DC1BA2ECB1585983A10', 1); - "#)?; + "#, + )?; let rec1 = tmpdir.path().join("e69d45e8-cba6-4dc1-ba2e-cb1585983a10"); let rec2 = tmpdir.path().join("94de8484-ff87-4a52-95d4-88c8038a0312"); let rec3 = tmpdir.path().join("c94d4d0b-5337-4605-9cd4-0b29039e641e"); @@ -254,17 +289,24 @@ mod tests { std::fs::File::create(&rec3)?; std::fs::File::create(&garbage)?; - for (ver, fresh_sql) in &[(1, Some(include_str!("v1.sql"))), - (2, None), // transitional; don't compare schemas. - (3, Some(include_str!("v3.sql"))), - (4, None), // transitional; don't compare schemas. - (5, Some(include_str!("v5.sql"))), - (6, Some(include_str!("../schema.sql")))] { - upgrade(&Args { - sample_file_dir: Some(&tmpdir.path()), - preset_journal: "delete", - no_vacuum: false, - }, *ver, &mut upgraded).context(format!("upgrading to version {}", ver))?; + for (ver, fresh_sql) in &[ + (1, Some(include_str!("v1.sql"))), + (2, None), // transitional; don't compare schemas. + (3, Some(include_str!("v3.sql"))), + (4, None), // transitional; don't compare schemas. + (5, Some(include_str!("v5.sql"))), + (6, Some(include_str!("../schema.sql"))), + ] { + upgrade( + &Args { + sample_file_dir: Some(&tmpdir.path()), + preset_journal: "delete", + no_vacuum: false, + }, + *ver, + &mut upgraded, + ) + .context(format!("upgrading to version {}", ver))?; if let Some(f) = fresh_sql { compare(&upgraded, *ver, f)?; } @@ -277,14 +319,16 @@ mod tests { } if *ver == 6 { // Check that the pasp was set properly. - let mut stmt = upgraded.prepare(r#" + let mut stmt = upgraded.prepare( + r#" select id, pasp_h_spacing, pasp_v_spacing from video_sample_entry - "#)?; + "#, + )?; let mut rows = stmt.query(params![])?; let mut pasp_by_id = FnvHashMap::default(); while let Some(row) = rows.next()? { diff --git a/server/db/upgrade/v0_to_v1.rs b/server/db/upgrade/v0_to_v1.rs index e44e735..0ba2b81 100644 --- a/server/db/upgrade/v0_to_v1.rs +++ b/server/db/upgrade/v0_to_v1.rs @@ -29,7 +29,6 @@ // along with this program. If not, see . /// Upgrades a version 0 schema to a version 1 schema. - use crate::db; use crate::recording; use failure::Error; @@ -39,7 +38,8 @@ use std::collections::HashMap; pub fn run(_args: &super::Args, tx: &rusqlite::Transaction) -> Result<(), Error> { // These create statements match the schema.sql when version 1 was the latest. - tx.execute_batch(r#" + tx.execute_batch( + r#" alter table camera rename to old_camera; create table camera ( id integer primary key, @@ -103,13 +103,16 @@ pub fn run(_args: &super::Args, tx: &rusqlite::Transaction) -> Result<(), Error> 1 as next_recording_id from old_camera; - "#)?; + "#, + )?; let camera_state = fill_recording(tx)?; update_camera(tx, camera_state)?; - tx.execute_batch(r#" + tx.execute_batch( + r#" drop table old_recording; drop table old_camera; - "#)?; + "#, + )?; Ok(()) } @@ -130,7 +133,8 @@ fn has_trailing_zero(video_index: &[u8]) -> Result { /// Fills the `recording` and `recording_playback` tables from `old_recording`, returning /// the `camera_state` map for use by a following call to `fill_cameras`. fn fill_recording(tx: &rusqlite::Transaction) -> Result, Error> { - let mut select = tx.prepare(r#" + let mut select = tx.prepare( + r#" select camera_id, sample_file_bytes, @@ -146,27 +150,32 @@ fn fill_recording(tx: &rusqlite::Transaction) -> Result = HashMap::new(); while let Some(row) = rows.next()? { let camera_id: i32 = row.get(0)?; - let camera_state = camera_state.entry(camera_id).or_insert_with(|| { - CameraState{ + let camera_state = camera_state + .entry(camera_id) + .or_insert_with(|| CameraState { current_run: None, next_recording_id: 1, - } - }); + }); let composite_id = ((camera_id as i64) << 32) | (camera_state.next_recording_id as i64); camera_state.next_recording_id += 1; let sample_file_bytes: i32 = row.get(1)?; @@ -181,9 +190,15 @@ fn fill_recording(tx: &rusqlite::Transaction) -> Result = row.get(10)?; let old_id: i32 = row.get(11)?; let trailing_zero = has_trailing_zero(&video_index).unwrap_or_else(|e| { - warn!("recording {}/{} (sample file {}, formerly recording {}) has corrupt \ - video_index: {}", - camera_id, composite_id & 0xFFFF, sample_file_uuid.0, old_id, e); + warn!( + "recording {}/{} (sample file {}, formerly recording {}) has corrupt \ + video_index: {}", + camera_id, + composite_id & 0xFFFF, + sample_file_uuid.0, + old_id, + e + ); false }); let run_id = match camera_state.current_run { @@ -194,7 +209,14 @@ fn fill_recording(tx: &rusqlite::Transaction) -> Result Result) - -> Result<(), Error> { - let mut stmt = tx.prepare(r#" +fn update_camera( + tx: &rusqlite::Transaction, + camera_state: HashMap, +) -> Result<(), Error> { + let mut stmt = tx.prepare( + r#" update camera set next_recording_id = :next_recording_id where id = :id - "#)?; + "#, + )?; for (ref id, ref state) in &camera_state { stmt.execute_named(&[ (":id", &id), diff --git a/server/db/upgrade/v1_to_v2.rs b/server/db/upgrade/v1_to_v2.rs index 160e050..f873187 100644 --- a/server/db/upgrade/v1_to_v2.rs +++ b/server/db/upgrade/v1_to_v2.rs @@ -29,29 +29,31 @@ // along with this program. If not, see . /// Upgrades a version 1 schema to a version 2 schema. - use crate::dir; -use failure::{Error, bail, format_err}; +use crate::schema::DirMeta; +use failure::{bail, format_err, Error}; use nix::fcntl::{FlockArg, OFlag}; use nix::sys::stat::Mode; use rusqlite::params; -use crate::schema::DirMeta; use std::os::unix::io::AsRawFd; use uuid::Uuid; pub fn run(args: &super::Args, tx: &rusqlite::Transaction) -> Result<(), Error> { - let sample_file_path = - args.sample_file_dir - .ok_or_else(|| format_err!("--sample-file-dir required when upgrading from \ - schema version 1 to 2."))?; + let sample_file_path = args.sample_file_dir.ok_or_else(|| { + format_err!("--sample-file-dir required when upgrading from schema version 1 to 2.") + })?; - let mut d = nix::dir::Dir::open(sample_file_path, OFlag::O_DIRECTORY | OFlag::O_RDONLY, - Mode::empty())?; + let mut d = nix::dir::Dir::open( + sample_file_path, + OFlag::O_DIRECTORY | OFlag::O_RDONLY, + Mode::empty(), + )?; nix::fcntl::flock(d.as_raw_fd(), FlockArg::LockExclusiveNonblock)?; verify_dir_contents(sample_file_path, &mut d, tx)?; // These create statements match the schema.sql when version 2 was the latest. - tx.execute_batch(r#" + tx.execute_batch( + r#" create table meta ( uuid blob not null check (length(uuid) = 16) ); @@ -99,13 +101,17 @@ pub fn run(args: &super::Args, tx: &rusqlite::Transaction) -> Result<(), Error> use_count not null default 0 ) without rowid; create index user_session_uid on user_session (user_id); - "#)?; + "#, + )?; let db_uuid = ::uuid::Uuid::new_v4(); let db_uuid_bytes = &db_uuid.as_bytes()[..]; tx.execute("insert into meta (uuid) values (?)", params![db_uuid_bytes])?; let open_uuid = ::uuid::Uuid::new_v4(); let open_uuid_bytes = &open_uuid.as_bytes()[..]; - tx.execute("insert into open (uuid) values (?)", params![open_uuid_bytes])?; + tx.execute( + "insert into open (uuid) values (?)", + params![open_uuid_bytes], + )?; let open_id = tx.last_insert_rowid() as u32; let dir_uuid = ::uuid::Uuid::new_v4(); let dir_uuid_bytes = &dir_uuid.as_bytes()[..]; @@ -121,15 +127,22 @@ pub fn run(args: &super::Args, tx: &rusqlite::Transaction) -> Result<(), Error> } dir::write_meta(d.as_raw_fd(), &meta)?; - let sample_file_path = sample_file_path.to_str() - .ok_or_else(|| format_err!("sample file dir {} is not a valid string", - sample_file_path.display()))?; - tx.execute(r#" + let sample_file_path = sample_file_path.to_str().ok_or_else(|| { + format_err!( + "sample file dir {} is not a valid string", + sample_file_path.display() + ) + })?; + tx.execute( + r#" insert into sample_file_dir (path, uuid, last_complete_open_id) values (?, ?, ?) - "#, params![sample_file_path, dir_uuid_bytes, open_id])?; + "#, + params![sample_file_path, dir_uuid_bytes, open_id], + )?; - tx.execute_batch(r#" + tx.execute_batch( + r#" drop table reserved_sample_files; alter table camera rename to old_camera; alter table recording rename to old_recording; @@ -253,12 +266,14 @@ pub fn run(args: &super::Args, tx: &rusqlite::Transaction) -> Result<(), Error> old_camera cross join sample_file_dir where old_camera.sub_rtsp_path != ''; - "#)?; + "#, + )?; // Add the new video_sample_entry rows, before inserting the recordings referencing them. fix_video_sample_entry(tx)?; - tx.execute_batch(r#" + tx.execute_batch( + r#" insert into recording select r.composite_id, @@ -282,7 +297,8 @@ pub fn run(args: &super::Args, tx: &rusqlite::Transaction) -> Result<(), Error> p.sample_file_sha1 from old_recording r join recording_playback p on (r.composite_id = p.composite_id); - "#)?; + "#, + )?; Ok(()) } @@ -295,16 +311,23 @@ pub fn run(args: &super::Args, tx: &rusqlite::Transaction) -> Result<(), Error> /// * optional: reserved sample file uuids. /// * optional: meta and meta-tmp from half-completed update attempts. /// * forbidden: anything else. -fn verify_dir_contents(sample_file_path: &std::path::Path, dir: &mut nix::dir::Dir, - tx: &rusqlite::Transaction) -> Result<(), Error> { +fn verify_dir_contents( + sample_file_path: &std::path::Path, + dir: &mut nix::dir::Dir, + tx: &rusqlite::Transaction, +) -> Result<(), Error> { // Build a hash of the uuids found in the directory. - let n: i64 = tx.query_row(r#" + let n: i64 = tx.query_row( + r#" select a.c + b.c from (select count(*) as c from recording) a, (select count(*) as c from reserved_sample_files) b; - "#, params![], |r| r.get(0))?; + "#, + params![], + |r| r.get(0), + )?; let mut files = ::fnv::FnvHashSet::with_capacity_and_hasher(n as usize, Default::default()); for e in dir.iter() { let e = e?; @@ -315,8 +338,8 @@ fn verify_dir_contents(sample_file_path: &std::path::Path, dir: &mut nix::dir::D // Ignore metadata files. These might from a half-finished update attempt. // If the directory is actually an in-use >v3 format, other contents won't match. continue; - }, - _ => {}, + } + _ => {} }; let s = match f.to_str() { Ok(s) => s, @@ -326,7 +349,8 @@ fn verify_dir_contents(sample_file_path: &std::path::Path, dir: &mut nix::dir::D Ok(u) => u, Err(_) => bail!("unexpected file {:?} in {:?}", f, sample_file_path), }; - if s != uuid.to_hyphenated_ref().to_string() { // non-canonical form. + if s != uuid.to_hyphenated_ref().to_string() { + // non-canonical form. bail!("unexpected file {:?} in {:?}", f, sample_file_path); } files.insert(uuid); @@ -339,7 +363,11 @@ fn verify_dir_contents(sample_file_path: &std::path::Path, dir: &mut nix::dir::D while let Some(row) = rows.next()? { let uuid: crate::db::FromSqlUuid = row.get(0)?; if !files.remove(&uuid.0) { - bail!("{} is missing from dir {}!", uuid.0, sample_file_path.display()); + bail!( + "{} is missing from dir {}!", + uuid.0, + sample_file_path.display() + ); } } } @@ -355,20 +383,28 @@ fn verify_dir_contents(sample_file_path: &std::path::Path, dir: &mut nix::dir::D // a garbage file so if the upgrade transation fails this is still a valid and complete // version 1 database. let p = super::UuidPath::from(uuid.0); - nix::unistd::unlinkat(Some(dir.as_raw_fd()), &p, - nix::unistd::UnlinkatFlags::NoRemoveDir)?; + nix::unistd::unlinkat( + Some(dir.as_raw_fd()), + &p, + nix::unistd::UnlinkatFlags::NoRemoveDir, + )?; } } if !files.is_empty() { - bail!("{} unexpected sample file uuids in dir {}: {:?}!", - files.len(), sample_file_path.display(), files); + bail!( + "{} unexpected sample file uuids in dir {}: {:?}!", + files.len(), + sample_file_path.display(), + files + ); } Ok(()) } fn fix_video_sample_entry(tx: &rusqlite::Transaction) -> Result<(), Error> { - let mut select = tx.prepare(r#" + let mut select = tx.prepare( + r#" select id, sha1, @@ -377,10 +413,13 @@ fn fix_video_sample_entry(tx: &rusqlite::Transaction) -> Result<(), Error> { data from old_video_sample_entry - "#)?; - let mut insert = tx.prepare(r#" + "#, + )?; + let mut insert = tx.prepare( + r#" insert into video_sample_entry values (:id, :sha1, :width, :height, :rfc6381_codec, :data) - "#)?; + "#, + )?; let mut rows = select.query(params![])?; while let Some(row) = rows.next()? { let data: Vec = row.get(4)?; @@ -398,12 +437,15 @@ fn fix_video_sample_entry(tx: &rusqlite::Transaction) -> Result<(), Error> { // This previously lived in h264.rs. As of version 1, H.264 is the only supported codec. fn rfc6381_codec_from_sample_entry(sample_entry: &[u8]) -> Result { - if sample_entry.len() < 99 || &sample_entry[4..8] != b"avc1" || - &sample_entry[90..94] != b"avcC" { + if sample_entry.len() < 99 || &sample_entry[4..8] != b"avc1" || &sample_entry[90..94] != b"avcC" + { bail!("not a valid AVCSampleEntry"); } let profile_idc = sample_entry[103]; let constraint_flags_byte = sample_entry[104]; let level_idc = sample_entry[105]; - Ok(format!("avc1.{:02x}{:02x}{:02x}", profile_idc, constraint_flags_byte, level_idc)) + Ok(format!( + "avc1.{:02x}{:02x}{:02x}", + profile_idc, constraint_flags_byte, level_idc + )) } diff --git a/server/db/upgrade/v2_to_v3.rs b/server/db/upgrade/v2_to_v3.rs index b8f7276..c1f5372 100644 --- a/server/db/upgrade/v2_to_v3.rs +++ b/server/db/upgrade/v2_to_v3.rs @@ -31,11 +31,10 @@ /// Upgrades a version 2 schema to a version 3 schema. /// Note that a version 2 schema is never actually used; so we know the upgrade from version 1 was /// completed, and possibly an upgrade from 2 to 3 is half-finished. - use crate::db::{self, FromSqlUuid}; use crate::dir; -use failure::Error; use crate::schema; +use failure::Error; use rusqlite::params; use std::os::unix::io::AsRawFd; use std::sync::Arc; @@ -47,20 +46,26 @@ use std::sync::Arc; /// * it has a last completed open. fn open_sample_file_dir(tx: &rusqlite::Transaction) -> Result, Error> { let (p, s_uuid, o_id, o_uuid, db_uuid): (String, FromSqlUuid, i32, FromSqlUuid, FromSqlUuid) = - tx.query_row(r#" - select - s.path, s.uuid, s.last_complete_open_id, o.uuid, m.uuid - from - sample_file_dir s - join open o on (s.last_complete_open_id = o.id) - cross join meta m - "#, params![], |row| { - Ok((row.get(0)?, - row.get(1)?, - row.get(2)?, - row.get(3)?, - row.get(4)?)) - })?; + tx.query_row( + r#" + select + s.path, s.uuid, s.last_complete_open_id, o.uuid, m.uuid + from + sample_file_dir s + join open o on (s.last_complete_open_id = o.id) + cross join meta m + "#, + params![], + |row| { + Ok(( + row.get(0)?, + row.get(1)?, + row.get(2)?, + row.get(3)?, + row.get(4)?, + )) + }, + )?; let mut meta = schema::DirMeta::default(); meta.db_uuid.extend_from_slice(&db_uuid.0.as_bytes()[..]); meta.dir_uuid.extend_from_slice(&s_uuid.0.as_bytes()[..]); @@ -74,30 +79,37 @@ fn open_sample_file_dir(tx: &rusqlite::Transaction) -> Result Result<(), Error> { let d = open_sample_file_dir(&tx)?; - let mut stmt = tx.prepare(r#" + let mut stmt = tx.prepare( + r#" select composite_id, sample_file_uuid from recording_playback - "#)?; + "#, + )?; let mut rows = stmt.query(params![])?; while let Some(row) = rows.next()? { let id = db::CompositeId(row.get(0)?); let sample_file_uuid: FromSqlUuid = row.get(1)?; let from_path = super::UuidPath::from(sample_file_uuid.0); let to_path = crate::dir::CompositeIdPath::from(id); - if let Err(e) = nix::fcntl::renameat(Some(d.fd.as_raw_fd()), &from_path, - Some(d.fd.as_raw_fd()), &to_path) { + if let Err(e) = nix::fcntl::renameat( + Some(d.fd.as_raw_fd()), + &from_path, + Some(d.fd.as_raw_fd()), + &to_path, + ) { if e == nix::Error::Sys(nix::errno::Errno::ENOENT) { - continue; // assume it was already moved. + continue; // assume it was already moved. } Err(e)?; } } // These create statements match the schema.sql when version 3 was the latest. - tx.execute_batch(r#" + tx.execute_batch( + r#" alter table recording_playback rename to old_recording_playback; create table recording_playback ( composite_id integer primary key references recording (composite_id), @@ -113,6 +125,7 @@ pub fn run(_args: &super::Args, tx: &rusqlite::Transaction) -> Result<(), Error> drop table old_recording; drop table old_camera; drop table old_video_sample_entry; - "#)?; + "#, + )?; Ok(()) } diff --git a/server/db/upgrade/v3_to_v4.rs b/server/db/upgrade/v3_to_v4.rs index ccf047f..90ba4e0 100644 --- a/server/db/upgrade/v3_to_v4.rs +++ b/server/db/upgrade/v3_to_v4.rs @@ -29,12 +29,12 @@ // along with this program. If not, see . /// Upgrades a version 3 schema to a version 4 schema. - use failure::Error; pub fn run(_args: &super::Args, tx: &rusqlite::Transaction) -> Result<(), Error> { // These create statements match the schema.sql when version 4 was the latest. - tx.execute_batch(r#" + tx.execute_batch( + r#" alter table meta add column max_signal_changes integer check (max_signal_changes >= 0); create table signal ( @@ -191,6 +191,7 @@ pub fn run(_args: &super::Args, tx: &rusqlite::Transaction) -> Result<(), Error> -- This was supposed to be present in version 2, but the upgrade procedure used to miss it. -- Catch up so we know a version 4 database is right. create index if not exists user_session_uid on user_session (user_id); - "#)?; + "#, + )?; Ok(()) } diff --git a/server/db/upgrade/v4_to_v5.rs b/server/db/upgrade/v4_to_v5.rs index 5ec1c65..892098f 100644 --- a/server/db/upgrade/v4_to_v5.rs +++ b/server/db/upgrade/v4_to_v5.rs @@ -32,11 +32,10 @@ /// /// This just handles the directory meta files. If they're already in the new format, great. /// Otherwise, verify they are consistent with the database then upgrade them. - use crate::db::FromSqlUuid; use crate::{dir, schema}; use cstr::cstr; -use failure::{Error, Fail, bail}; +use failure::{bail, Error, Fail}; use log::info; use nix::fcntl::{FlockArg, OFlag}; use nix::sys::stat::Mode; @@ -61,25 +60,42 @@ fn maybe_upgrade_meta(dir: &dir::Fd, db_meta: &schema::DirMeta) -> Resultvec is infallible"); + let mut f = crate::fs::openat( + dir.as_raw_fd(), + tmp_path, + OFlag::O_CREAT | OFlag::O_TRUNC | OFlag::O_WRONLY, + Mode::S_IRUSR | Mode::S_IWUSR, + )?; + let mut data = dir_meta + .write_length_delimited_to_bytes() + .expect("proto3->vec is infallible"); if data.len() > FIXED_DIR_META_LEN { - bail!("Length-delimited DirMeta message requires {} bytes, over limit of {}", - data.len(), FIXED_DIR_META_LEN); + bail!( + "Length-delimited DirMeta message requires {} bytes, over limit of {}", + data.len(), + FIXED_DIR_META_LEN + ); } - data.resize(FIXED_DIR_META_LEN, 0); // pad to required length. + data.resize(FIXED_DIR_META_LEN, 0); // pad to required length. f.write_all(&data)?; f.sync_all()?; - nix::fcntl::renameat(Some(dir.as_raw_fd()), tmp_path, Some(dir.as_raw_fd()), meta_path)?; + nix::fcntl::renameat( + Some(dir.as_raw_fd()), + tmp_path, + Some(dir.as_raw_fd()), + meta_path, + )?; Ok(true) } @@ -91,8 +107,12 @@ fn maybe_upgrade_meta(dir: &dir::Fd, db_meta: &schema::DirMeta) -> Result Result { let mut need_sync = false; - let mut dir2 = nix::dir::Dir::openat(dir.as_raw_fd(), ".", - OFlag::O_DIRECTORY | OFlag::O_RDONLY, Mode::empty())?; + let mut dir2 = nix::dir::Dir::openat( + dir.as_raw_fd(), + ".", + OFlag::O_DIRECTORY | OFlag::O_RDONLY, + Mode::empty(), + )?; for e in dir2.iter() { let e = e?; let f = e.file_name(); @@ -103,8 +123,11 @@ fn maybe_cleanup_garbage_uuids(dir: &dir::Fd) -> Result { }; if Uuid::parse_str(f_str).is_ok() { info!("removing leftover garbage file {}", f_str); - nix::unistd::unlinkat(Some(dir.as_raw_fd()), f, - nix::unistd::UnlinkatFlags::NoRemoveDir)?; + nix::unistd::unlinkat( + Some(dir.as_raw_fd()), + f, + nix::unistd::UnlinkatFlags::NoRemoveDir, + )?; need_sync = true; } } @@ -115,7 +138,8 @@ fn maybe_cleanup_garbage_uuids(dir: &dir::Fd) -> Result { pub fn run(_args: &super::Args, tx: &rusqlite::Transaction) -> Result<(), Error> { let db_uuid: FromSqlUuid = tx.query_row_and_then(r"select uuid from meta", params![], |row| row.get(0))?; - let mut stmt = tx.prepare(r#" + let mut stmt = tx.prepare( + r#" select d.path, d.uuid, @@ -124,7 +148,8 @@ pub fn run(_args: &super::Args, tx: &rusqlite::Transaction) -> Result<(), Error> from sample_file_dir d left join open o on (d.last_complete_open_id = o.id); - "#)?; + "#, + )?; let mut rows = stmt.query(params![])?; while let Some(row) = rows.next()? { let path = row.get_raw_checked(0)?.as_str()?; @@ -134,14 +159,16 @@ pub fn run(_args: &super::Args, tx: &rusqlite::Transaction) -> Result<(), Error> let open_uuid: Option = row.get(3)?; let mut db_meta = schema::DirMeta::new(); db_meta.db_uuid.extend_from_slice(&db_uuid.0.as_bytes()[..]); - db_meta.dir_uuid.extend_from_slice(&dir_uuid.0.as_bytes()[..]); + db_meta + .dir_uuid + .extend_from_slice(&dir_uuid.0.as_bytes()[..]); match (open_id, open_uuid) { (Some(id), Some(uuid)) => { let mut o = db_meta.last_complete_open.set_default(); o.id = id; o.uuid.extend_from_slice(&uuid.0.as_bytes()[..]); - }, - (None, None) => {}, + } + (None, None) => {} _ => bail!("open table missing id"), } diff --git a/server/db/upgrade/v5_to_v6.rs b/server/db/upgrade/v5_to_v6.rs index 7270ad2..db975e9 100644 --- a/server/db/upgrade/v5_to_v6.rs +++ b/server/db/upgrade/v5_to_v6.rs @@ -29,9 +29,8 @@ // along with this program. If not, see . /// Upgrades a version 4 schema to a version 5 schema. - use byteorder::{BigEndian, ByteOrder, WriteBytesExt}; -use failure::{Error, ResultExt, bail, format_err}; +use failure::{bail, format_err, Error, ResultExt}; use h264_reader::avcc::AvcDecoderConfigurationRecord; use rusqlite::{named_params, params}; use std::convert::{TryFrom, TryInto}; @@ -39,9 +38,9 @@ use std::convert::{TryFrom, TryInto}; // Copied from src/h264.rs. h264 stuff really doesn't belong in the db crate, but we do what we // must for schema upgrades. const PIXEL_ASPECT_RATIOS: [((u16, u16), (u16, u16)); 4] = [ - ((320, 240), ( 4, 3)), + ((320, 240), (4, 3)), ((352, 240), (40, 33)), - ((640, 480), ( 4, 3)), + ((640, 480), (4, 3)), ((704, 480), (40, 33)), ]; fn default_pixel_aspect_ratio(width: u16, height: u16) -> (u16, u16) { @@ -59,13 +58,16 @@ fn parse(data: &[u8]) -> Result { bail!("data of len {} doesn't have an avcC", data.len()); } let avcc_len = BigEndian::read_u32(&data[86..90]); - if avcc_len < 8 { // length and type. + if avcc_len < 8 { + // length and type. bail!("invalid avcc len {}", avcc_len); } let end_pos = 86 + usize::try_from(avcc_len)?; if end_pos != data.len() { - bail!("expected avcC to be end of extradata; there are {} more bytes.", - data.len() - end_pos); + bail!( + "expected avcC to be end of extradata; there are {} more bytes.", + data.len() - end_pos + ); } AvcDecoderConfigurationRecord::try_from(&data[94..end_pos]) .map_err(|e| format_err!("Bad AvcDecoderConfigurationRecord: {:?}", e)) @@ -73,7 +75,8 @@ fn parse(data: &[u8]) -> Result { pub fn run(_args: &super::Args, tx: &rusqlite::Transaction) -> Result<(), Error> { // These create statements match the schema.sql when version 5 was the latest. - tx.execute_batch(r#" + tx.execute_batch( + r#" alter table video_sample_entry rename to old_video_sample_entry; create table video_sample_entry ( @@ -85,19 +88,23 @@ pub fn run(_args: &super::Args, tx: &rusqlite::Transaction) -> Result<(), Error> pasp_h_spacing integer not null default 1 check (pasp_h_spacing > 0), pasp_v_spacing integer not null default 1 check (pasp_v_spacing > 0) ); - "#)?; + "#, + )?; - let mut insert = tx.prepare(r#" + let mut insert = tx.prepare( + r#" insert into video_sample_entry (id, width, height, rfc6381_codec, data, pasp_h_spacing, pasp_v_spacing) values (:id, :width, :height, :rfc6381_codec, :data, :pasp_h_spacing, :pasp_v_spacing) - "#)?; + "#, + )?; // Only insert still-referenced video sample entries. I've had problems with // no-longer-referenced ones (perhaps from some ancient, buggy version of Moonfire NVR) for // which avcc.create_context(()) fails. - let mut stmt = tx.prepare(r#" + let mut stmt = tx.prepare( + r#" select id, width, @@ -114,7 +121,8 @@ pub fn run(_args: &super::Args, tx: &rusqlite::Transaction) -> Result<(), Error> recording r where r.video_sample_entry_id = v.id) - "#)?; + "#, + )?; let mut rows = stmt.query(params![])?; while let Some(row) = rows.next()? { let id: i32 = row.get(0)?; @@ -126,24 +134,31 @@ pub fn run(_args: &super::Args, tx: &rusqlite::Transaction) -> Result<(), Error> if avcc.num_of_sequence_parameter_sets() != 1 { bail!("Multiple SPSs!"); } - let ctx = avcc.create_context(()) - .map_err(|e| format_err!("Can't load SPS+PPS for video_sample_entry_id {}: {:?}", - id, e))?; - let sps = ctx.sps_by_id(h264_reader::nal::pps::ParamSetId::from_u32(0).unwrap()) + let ctx = avcc.create_context(()).map_err(|e| { + format_err!( + "Can't load SPS+PPS for video_sample_entry_id {}: {:?}", + id, + e + ) + })?; + let sps = ctx + .sps_by_id(h264_reader::nal::pps::ParamSetId::from_u32(0).unwrap()) .ok_or_else(|| format_err!("No SPS 0 for video_sample_entry_id {}", id))?; - let pasp = sps.vui_parameters.as_ref() - .and_then(|v| v.aspect_ratio_info.as_ref()) - .and_then(|a| a.clone().get()) - .unwrap_or_else(|| default_pixel_aspect_ratio(width, height)); + let pasp = sps + .vui_parameters + .as_ref() + .and_then(|v| v.aspect_ratio_info.as_ref()) + .and_then(|a| a.clone().get()) + .unwrap_or_else(|| default_pixel_aspect_ratio(width, height)); if pasp != (1, 1) { - data.extend_from_slice(b"\x00\x00\x00\x10pasp"); // length + box name + data.extend_from_slice(b"\x00\x00\x00\x10pasp"); // length + box name data.write_u32::(pasp.0.into())?; data.write_u32::(pasp.1.into())?; let len = data.len(); BigEndian::write_u32(&mut data[0..4], u32::try_from(len)?); } - insert.execute_named(named_params!{ + insert.execute_named(named_params! { ":id": id, ":width": width, ":height": height, @@ -153,7 +168,8 @@ pub fn run(_args: &super::Args, tx: &rusqlite::Transaction) -> Result<(), Error> ":pasp_v_spacing": pasp.1, })?; } - tx.execute_batch(r#" + tx.execute_batch( + r#" alter table stream rename to old_stream; create table stream ( id integer primary key, @@ -205,14 +221,16 @@ pub fn run(_args: &super::Args, tx: &rusqlite::Transaction) -> Result<(), Error> video_sample_entry_id integer references video_sample_entry (id), check (composite_id >> 32 = stream_id) ); - "#)?; + "#, + )?; // SQLite added window functions in 3.25.0. macOS still ships SQLite 3.24.0 (no support). // Compute cumulative columns by hand. let mut cur_stream_id = None; let mut cum_duration_90k = 0; let mut cum_runs = 0; - let mut stmt = tx.prepare(r#" + let mut stmt = tx.prepare( + r#" select composite_id, open_id, @@ -228,8 +246,10 @@ pub fn run(_args: &super::Args, tx: &rusqlite::Transaction) -> Result<(), Error> from old_recording order by composite_id - "#)?; - let mut insert = tx.prepare(r#" + "#, + )?; + let mut insert = tx.prepare( + r#" insert into recording (composite_id, open_id, stream_id, run_offset, flags, sample_file_bytes, start_time_90k, prev_media_duration_90k, prev_runs, wall_duration_90k, media_duration_delta_90k, @@ -238,7 +258,8 @@ pub fn run(_args: &super::Args, tx: &rusqlite::Transaction) -> Result<(), Error> :sample_file_bytes, :start_time_90k, :prev_media_duration_90k, :prev_runs, :wall_duration_90k, 0, :video_samples, :video_sync_samples, :video_sample_entry_id) - "#)?; + "#, + )?; let mut rows = stmt.query(params![])?; while let Some(row) = rows.next()? { let composite_id: i64 = row.get(0)?; @@ -257,25 +278,28 @@ pub fn run(_args: &super::Args, tx: &rusqlite::Transaction) -> Result<(), Error> cum_runs = 0; cur_stream_id = Some(stream_id); } - insert.execute_named(named_params!{ - ":composite_id": composite_id, - ":open_id": open_id, - ":stream_id": stream_id, - ":run_offset": run_offset, - ":flags": flags, - ":sample_file_bytes": sample_file_bytes, - ":start_time_90k": start_time_90k, - ":prev_media_duration_90k": cum_duration_90k, - ":prev_runs": cum_runs, - ":wall_duration_90k": wall_duration_90k, - ":video_samples": video_samples, - ":video_sync_samples": video_sync_samples, - ":video_sample_entry_id": video_sample_entry_id, - }).with_context(|_| format!("Unable to insert composite_id {}", composite_id))?; + insert + .execute_named(named_params! { + ":composite_id": composite_id, + ":open_id": open_id, + ":stream_id": stream_id, + ":run_offset": run_offset, + ":flags": flags, + ":sample_file_bytes": sample_file_bytes, + ":start_time_90k": start_time_90k, + ":prev_media_duration_90k": cum_duration_90k, + ":prev_runs": cum_runs, + ":wall_duration_90k": wall_duration_90k, + ":video_samples": video_samples, + ":video_sync_samples": video_sync_samples, + ":video_sample_entry_id": video_sample_entry_id, + }) + .with_context(|_| format!("Unable to insert composite_id {}", composite_id))?; cum_duration_90k += i64::from(wall_duration_90k); cum_runs += if run_offset == 0 { 1 } else { 0 }; } - tx.execute_batch(r#" + tx.execute_batch( + r#" drop index recording_cover; create index recording_cover on recording ( stream_id, @@ -328,6 +352,7 @@ pub fn run(_args: &super::Args, tx: &rusqlite::Transaction) -> Result<(), Error> revocation_reason_detail = 'Blake2b->Blake3 upgrade' where revocation_reason is null; - "#)?; + "#, + )?; Ok(()) } diff --git a/server/db/writer.rs b/server/db/writer.rs index f970f6d..e51de7e 100644 --- a/server/db/writer.rs +++ b/server/db/writer.rs @@ -32,32 +32,32 @@ //! //! This includes opening files for serving, rotating away old files, and saving new files. -use base::clock::{self, Clocks}; use crate::db::{self, CompositeId}; use crate::dir; use crate::recording::{self, MAX_RECORDING_WALL_DURATION}; -use failure::{Error, bail, format_err}; +use base::clock::{self, Clocks}; +use failure::{bail, format_err, Error}; use fnv::FnvHashMap; -use parking_lot::Mutex; use log::{debug, trace, warn}; -use std::convert::TryFrom; +use parking_lot::Mutex; use std::cmp::{self, Ordering}; +use std::convert::TryFrom; use std::io; use std::mem; -use std::sync::{Arc, mpsc}; +use std::sync::{mpsc, Arc}; use std::thread; use std::time::Duration as StdDuration; use time::{Duration, Timespec}; -pub trait DirWriter : 'static + Send { - type File : FileWriter; +pub trait DirWriter: 'static + Send { + type File: FileWriter; fn create_file(&self, id: CompositeId) -> Result; fn sync(&self) -> Result<(), nix::Error>; fn unlink_file(&self, id: CompositeId) -> Result<(), nix::Error>; } -pub trait FileWriter : 'static { +pub trait FileWriter: 'static { /// As in `std::fs::File::sync_all`. fn sync_all(&self) -> Result<(), io::Error>; @@ -71,15 +71,21 @@ impl DirWriter for Arc { fn create_file(&self, id: CompositeId) -> Result { dir::SampleFileDir::create_file(self, id) } - fn sync(&self) -> Result<(), nix::Error> { dir::SampleFileDir::sync(self) } + fn sync(&self) -> Result<(), nix::Error> { + dir::SampleFileDir::sync(self) + } fn unlink_file(&self, id: CompositeId) -> Result<(), nix::Error> { dir::SampleFileDir::unlink_file(self, id) } } impl FileWriter for ::std::fs::File { - fn sync_all(&self) -> Result<(), io::Error> { self.sync_all() } - fn write(&mut self, buf: &[u8]) -> Result { io::Write::write(self, buf) } + fn sync_all(&self) -> Result<(), io::Error> { + self.sync_all() + } + fn write(&mut self, buf: &[u8]) -> Result { + io::Write::write(self, buf) + } } /// A command sent to the syncer. These correspond to methods in the `SyncerChannel` struct. @@ -94,7 +100,9 @@ enum SyncerCommand { pub struct SyncerChannel(mpsc::Sender>); impl ::std::clone::Clone for SyncerChannel { - fn clone(&self) -> Self { SyncerChannel(self.0.clone()) } + fn clone(&self) -> Self { + SyncerChannel(self.0.clone()) + } } /// State of the worker thread. @@ -160,23 +168,32 @@ impl Eq for PlannedFlush {} /// Note that dropping all `SyncerChannel` clones currently includes calling /// `LockedDatabase::clear_on_flush`, as this function installs a hook to watch database flushes. /// TODO: add a join wrapper which arranges for the on flush hook to be removed automatically. -pub fn start_syncer(db: Arc>, dir_id: i32) - -> Result<(SyncerChannel<::std::fs::File>, thread::JoinHandle<()>), Error> -where C: Clocks + Clone { +pub fn start_syncer( + db: Arc>, + dir_id: i32, +) -> Result<(SyncerChannel<::std::fs::File>, thread::JoinHandle<()>), Error> +where + C: Clocks + Clone, +{ let db2 = db.clone(); let (mut syncer, path) = Syncer::new(&db.lock(), db2, dir_id)?; syncer.initial_rotation()?; let (snd, rcv) = mpsc::channel(); db.lock().on_flush(Box::new({ let snd = snd.clone(); - move || if let Err(e) = snd.send(SyncerCommand::DatabaseFlushed) { - warn!("Unable to notify syncer for dir {} of flush: {}", dir_id, e); + move || { + if let Err(e) = snd.send(SyncerCommand::DatabaseFlushed) { + warn!("Unable to notify syncer for dir {} of flush: {}", dir_id, e); + } } })); - Ok((SyncerChannel(snd), + Ok(( + SyncerChannel(snd), thread::Builder::new() .name(format!("sync-{}", path)) - .spawn(move || { while syncer.iter(&rcv) {} }).unwrap())) + .spawn(move || while syncer.iter(&rcv) {}) + .unwrap(), + )) } pub struct NewLimit { @@ -187,21 +204,28 @@ pub struct NewLimit { /// Deletes recordings if necessary to fit within the given new `retain_bytes` limit. /// Note this doesn't change the limit in the database; it only deletes files. /// Pass a limit of 0 to delete all recordings associated with a camera. -pub fn lower_retention(db: Arc, dir_id: i32, limits: &[NewLimit]) - -> Result<(), Error> { +pub fn lower_retention( + db: Arc, + dir_id: i32, + limits: &[NewLimit], +) -> Result<(), Error> { let db2 = db.clone(); let (mut syncer, _) = Syncer::new(&db.lock(), db2, dir_id)?; syncer.do_rotation(|db| { for l in limits { let (fs_bytes_before, extra); { - let stream = db.streams_by_id().get(&l.stream_id) - .ok_or_else(|| format_err!("no such stream {}", l.stream_id))?; - fs_bytes_before = stream.fs_bytes + stream.fs_bytes_to_add - - stream.fs_bytes_to_delete; + let stream = db + .streams_by_id() + .get(&l.stream_id) + .ok_or_else(|| format_err!("no such stream {}", l.stream_id))?; + fs_bytes_before = + stream.fs_bytes + stream.fs_bytes_to_add - stream.fs_bytes_to_delete; extra = stream.retain_bytes - l.limit; } - if l.limit >= fs_bytes_before { continue } + if l.limit >= fs_bytes_before { + continue; + } delete_recordings(db, l.stream_id, extra)?; } Ok(()) @@ -209,8 +233,11 @@ pub fn lower_retention(db: Arc, dir_id: i32, limits: &[NewLimit]) } /// Deletes recordings to bring a stream's disk usage within bounds. -fn delete_recordings(db: &mut db::LockedDatabase, stream_id: i32, - extra_bytes_needed: i64) -> Result<(), Error> { +fn delete_recordings( + db: &mut db::LockedDatabase, + stream_id: i32, + extra_bytes_needed: i64, +) -> Result<(), Error> { let fs_bytes_needed = { let stream = match db.streams_by_id().get(&stream_id) { None => bail!("no stream {}", stream_id), @@ -221,8 +248,11 @@ fn delete_recordings(db: &mut db::LockedDatabase, stream_id: i32, }; let mut fs_bytes_to_delete = 0; if fs_bytes_needed <= 0 { - debug!("{}: have remaining quota of {}", stream_id, - base::strutil::encode_size(-fs_bytes_needed)); + debug!( + "{}: have remaining quota of {}", + stream_id, + base::strutil::encode_size(-fs_bytes_needed) + ); return Ok(()); } let mut n = 0; @@ -241,7 +271,9 @@ impl SyncerChannel { /// Asynchronously syncs the given writer, closes it, records it into the database, and /// starts rotation. fn async_save_recording(&self, id: CompositeId, wall_duration: recording::Duration, f: F) { - self.0.send(SyncerCommand::AsyncSaveRecording(id, wall_duration, f)).unwrap(); + self.0 + .send(SyncerCommand::AsyncSaveRecording(id, wall_duration, f)) + .unwrap(); } /// For testing: flushes the syncer, waiting for all currently-queued commands to complete, @@ -250,14 +282,16 @@ impl SyncerChannel { pub fn flush(&self) { let (snd, rcv) = mpsc::sync_channel(0); self.0.send(SyncerCommand::Flush(snd)).unwrap(); - rcv.recv().unwrap_err(); // syncer should just drop the channel, closing it. + rcv.recv().unwrap_err(); // syncer should just drop the channel, closing it. } } /// Lists files which should be "abandoned" (deleted without ever recording in the database) /// on opening. -fn list_files_to_abandon(dir: &dir::SampleFileDir, streams_to_next: FnvHashMap) - -> Result, Error> { +fn list_files_to_abandon( + dir: &dir::SampleFileDir, + streams_to_next: FnvHashMap, +) -> Result, Error> { let mut v = Vec::new(); let mut d = dir.opendir()?; for e in d.iter() { @@ -268,7 +302,7 @@ fn list_files_to_abandon(dir: &dir::SampleFileDir, streams_to_next: FnvHashMap *n, - None => continue, // unknown stream. + None => continue, // unknown stream. }; if id.recording() >= next { v.push(id); @@ -278,26 +312,30 @@ fn list_files_to_abandon(dir: &dir::SampleFileDir, streams_to_next: FnvHashMap Syncer> { - fn new(l: &db::LockedDatabase, db: Arc>, dir_id: i32) - -> Result<(Self, String), Error> { - let d = l.sample_file_dirs_by_id() - .get(&dir_id) - .ok_or_else(|| format_err!("no dir {}", dir_id))?; + fn new( + l: &db::LockedDatabase, + db: Arc>, + dir_id: i32, + ) -> Result<(Self, String), Error> { + let d = l + .sample_file_dirs_by_id() + .get(&dir_id) + .ok_or_else(|| format_err!("no dir {}", dir_id))?; let dir = d.get()?; // Abandon files. // First, get a list of the streams in question. - let streams_to_next: FnvHashMap<_, _> = - l.streams_by_id() - .iter() - .filter_map(|(&k, v)| { - if v.sample_file_dir_id == Some(dir_id) { + let streams_to_next: FnvHashMap<_, _> = l + .streams_by_id() + .iter() + .filter_map(|(&k, v)| { + if v.sample_file_dir_id == Some(dir_id) { Some((k, v.cum_recordings)) - } else { - None - } - }) - .collect(); + } else { + None + } + }) + .collect(); let to_abandon = list_files_to_abandon(&dir, streams_to_next)?; let mut undeletable = 0; for &id in &to_abandon { @@ -314,12 +352,15 @@ impl Syncer> { bail!("Unable to delete {} abandoned recordings.", undeletable); } - Ok((Syncer { - dir_id, - dir, - db, - planned_flushes: std::collections::BinaryHeap::new(), - }, d.path.clone())) + Ok(( + Syncer { + dir_id, + dir, + db, + planned_flushes: std::collections::BinaryHeap::new(), + }, + d.path.clone(), + )) } /// Rotates files for all streams and deletes stale files from previous runs. @@ -336,7 +377,9 @@ impl Syncer> { /// Helper to do initial or retention-lowering rotation. Called from main thread. fn do_rotation(&mut self, delete_recordings: F) -> Result<(), Error> - where F: Fn(&mut db::LockedDatabase) -> Result<(), Error> { + where + F: Fn(&mut db::LockedDatabase) -> Result<(), Error>, + { { let mut db = self.db.lock(); delete_recordings(&mut *db)?; @@ -359,8 +402,10 @@ impl Syncer> { } } if errors > 0 { - bail!("Unable to unlink {} files (see earlier warning messages for details)", - errors); + bail!( + "Unable to unlink {} files (see earlier warning messages for details)", + errors + ); } self.dir.sync()?; self.db.lock().delete_garbage(self.dir_id, &mut garbage)?; @@ -379,7 +424,7 @@ impl Syncer { let next_flush = self.planned_flushes.peek().map(|f| f.when); let cmd = match next_flush { None => match cmds.recv() { - Err(_) => return false, // all cmd senders are gone. + Err(_) => return false, // all cmd senders are gone. Ok(cmd) => cmd, }, Some(t) => { @@ -388,14 +433,14 @@ impl Syncer { // Calculate the timeout to use, mapping negative durations to 0. let timeout = (t - now).to_std().unwrap_or(StdDuration::new(0, 0)); match self.db.clocks().recv_timeout(&cmds, timeout) { - Err(mpsc::RecvTimeoutError::Disconnected) => return false, // cmd senders gone. + Err(mpsc::RecvTimeoutError::Disconnected) => return false, // cmd senders gone. Err(mpsc::RecvTimeoutError::Timeout) => { self.flush(); return true; - }, + } Ok(cmd) => cmd, } - }, + } }; // Have a command; handle it. @@ -408,7 +453,7 @@ impl Syncer { if let Some(mut f) = self.planned_flushes.peek_mut() { f.senders.push(flush); } - }, + } }; true @@ -439,7 +484,9 @@ impl Syncer { }); } clock::retry_forever(c, &mut || self.dir.sync()); - clock::retry_forever(c, &mut || self.db.lock().delete_garbage(self.dir_id, &mut garbage)); + clock::retry_forever(c, &mut || { + self.db.lock().delete_garbage(self.dir_id, &mut garbage) + }); } /// Saves the given recording and causes rotation to happen. Called from worker thread. @@ -465,8 +512,14 @@ impl Syncer { let how_soon = Duration::seconds(s.flush_if_sec) - wall_duration.to_tm_duration(); let now = self.db.clocks().monotonic(); let when = now + how_soon; - let reason = format!("{} sec after start of {} {}-{} recording {}", - s.flush_if_sec, wall_duration, c.short_name, s.type_.as_str(), id); + let reason = format!( + "{} sec after start of {} {}-{} recording {}", + s.flush_if_sec, + wall_duration, + c.short_name, + s.type_.as_str(), + id + ); trace!("scheduling flush in {} because {}", how_soon, &reason); self.planned_flushes.push(PlannedFlush { when, @@ -492,13 +545,17 @@ impl Syncer { None => { // Removing streams while running hasn't been implemented yet, so this should // be impossible. - warn!("bug: no stream for {} which was scheduled to be flushed", f.recording); + warn!( + "bug: no stream for {} which was scheduled to be flushed", + f.recording + ); PeekMut::pop(f); continue; } }; - if s.cum_recordings <= f.recording.recording() { // not yet committed. + if s.cum_recordings <= f.recording.recording() { + // not yet committed. break; } @@ -517,10 +574,14 @@ impl Syncer { } if let Err(e) = l.flush(&f.reason) { let d = Duration::minutes(1); - warn!("flush failure on save for reason {}; will retry after {}: {:?}", - f.reason, d, e); - self.planned_flushes.peek_mut().expect("planned_flushes is non-empty").when = - self.db.clocks().monotonic() + Duration::minutes(1); + warn!( + "flush failure on save for reason {}; will retry after {}: {:?}", + f.reason, d, e + ); + self.planned_flushes + .peek_mut() + .expect("planned_flushes is non-empty") + .when = self.db.clocks().monotonic() + Duration::minutes(1); return; } @@ -594,8 +655,13 @@ struct PreviousWriter { impl<'a, C: Clocks + Clone, D: DirWriter> Writer<'a, C, D> { /// `db` must not be locked. - pub fn new(dir: &'a D, db: &'a db::Database, channel: &'a SyncerChannel, - stream_id: i32, video_sample_entry_id: i32) -> Self { + pub fn new( + dir: &'a D, + db: &'a db::Database, + channel: &'a SyncerChannel, + stream_id: i32, + video_sample_entry_id: i32, + ) -> Self { Writer { dir, db, @@ -616,13 +682,18 @@ impl<'a, C: Clocks + Clone, D: DirWriter> Writer<'a, C, D> { WriterState::Open(_) => return Ok(()), WriterState::Closed(prev) => Some(prev), }; - let (id, r) = self.db.lock().add_recording(self.stream_id, db::RecordingToInsert { - run_offset: prev.map(|p| p.run_offset + 1).unwrap_or(0), - start: prev.map(|p| p.end).unwrap_or(recording::Time(i64::max_value())), - video_sample_entry_id: self.video_sample_entry_id, - flags: db::RecordingFlags::Growing as i32, - ..Default::default() - })?; + let (id, r) = self.db.lock().add_recording( + self.stream_id, + db::RecordingToInsert { + run_offset: prev.map(|p| p.run_offset + 1).unwrap_or(0), + start: prev + .map(|p| p.end) + .unwrap_or(recording::Time(i64::max_value())), + video_sample_entry_id: self.video_sample_entry_id, + flags: db::RecordingFlags::Growing as i32, + ..Default::default() + }, + )?; let f = clock::retry_forever(&self.db.clocks(), &mut || self.dir.create_file(id)); self.state = WriterState::Open(InnerWriter { @@ -633,9 +704,9 @@ impl<'a, C: Clocks + Clone, D: DirWriter> Writer<'a, C, D> { hasher: blake3::Hasher::new(), local_start: recording::Time(i64::max_value()), unindexed_sample: None, - }); + }); Ok(()) - } + } pub fn previously_opened(&self) -> Result { Ok(match self.state { @@ -647,8 +718,13 @@ impl<'a, C: Clocks + Clone, D: DirWriter> Writer<'a, C, D> { /// Writes a new frame to this segment. /// `local_time` should be the local clock's time as of when this packet was received. - pub fn write(&mut self, pkt: &[u8], local_time: recording::Time, pts_90k: i64, - is_key: bool) -> Result<(), Error> { + pub fn write( + &mut self, + pkt: &[u8], + local_time: recording::Time, + pts_90k: i64, + is_key: bool, + ) -> Result<(), Error> { self.open()?; let w = match self.state { WriterState::Open(ref mut w) => w, @@ -662,11 +738,20 @@ impl<'a, C: Clocks + Clone, D: DirWriter> Writer<'a, C, D> { let duration = i32::try_from(pts_90k - i64::from(unindexed.pts_90k))?; if duration <= 0 { w.unindexed_sample = Some(unindexed); // restore invariant. - bail!("pts not monotonically increasing; got {} then {}", - unindexed.pts_90k, pts_90k); + bail!( + "pts not monotonically increasing; got {} then {}", + unindexed.pts_90k, + pts_90k + ); } - if let Err(e) = w.add_sample(duration, unindexed.len, unindexed.is_key, - unindexed.local_time, self.db, self.stream_id) { + if let Err(e) = w.add_sample( + duration, + unindexed.len, + unindexed.is_key, + unindexed.local_time, + self.db, + self.stream_id, + ) { w.unindexed_sample = Some(unindexed); // restore invariant. return Err(e); } @@ -694,7 +779,7 @@ impl<'a, C: Clocks + Clone, D: DirWriter> Writer<'a, C, D> { WriterState::Open(w) => { let prev = w.close(self.channel, next_pts, self.db, self.stream_id)?; WriterState::Closed(prev) - }, + } s => s, }; Ok(()) @@ -706,18 +791,24 @@ fn clamp(v: i64, min: i64, max: i64) -> i64 { } impl InnerWriter { - fn add_sample(&mut self, duration_90k: i32, bytes: i32, is_key: bool, - pkt_local_time: recording::Time, db: &db::Database, - stream_id: i32) - -> Result<(), Error> { + fn add_sample( + &mut self, + duration_90k: i32, + bytes: i32, + is_key: bool, + pkt_local_time: recording::Time, + db: &db::Database, + stream_id: i32, + ) -> Result<(), Error> { let mut l = self.r.lock(); // design/time.md explains these time manipulations in detail. let prev_media_duration_90k = l.media_duration_90k; let media_duration_90k = l.media_duration_90k + duration_90k; - let local_start = - cmp::min(self.local_start, - pkt_local_time - recording::Duration(i64::from(media_duration_90k))); + let local_start = cmp::min( + self.local_start, + pkt_local_time - recording::Duration(i64::from(media_duration_90k)), + ); let limit = i64::from(media_duration_90k / 2000); // 1/2000th, aka 500 ppm. let start = if l.run_offset == 0 { // Start time isn't anchored to previous recording's end; adjust. @@ -725,37 +816,58 @@ impl InnerWriter { } else { l.start }; - let wall_duration_90k = - media_duration_90k + - i32::try_from(clamp(local_start.0 - start.0, -limit, limit)).unwrap(); + let wall_duration_90k = media_duration_90k + + i32::try_from(clamp(local_start.0 - start.0, -limit, limit)).unwrap(); if wall_duration_90k > i32::try_from(MAX_RECORDING_WALL_DURATION).unwrap() { - bail!("Duration {} exceeds maximum {}", wall_duration_90k, - MAX_RECORDING_WALL_DURATION); + bail!( + "Duration {} exceeds maximum {}", + wall_duration_90k, + MAX_RECORDING_WALL_DURATION + ); } l.wall_duration_90k = wall_duration_90k; l.start = start; self.local_start = local_start; self.e.add_sample(duration_90k, bytes, is_key, &mut l); drop(l); - db.lock().send_live_segment(stream_id, db::LiveSegment { - recording: self.id.recording(), - is_key, - media_off_90k: prev_media_duration_90k .. media_duration_90k, - }).unwrap(); + db.lock() + .send_live_segment( + stream_id, + db::LiveSegment { + recording: self.id.recording(), + is_key, + media_off_90k: prev_media_duration_90k..media_duration_90k, + }, + ) + .unwrap(); Ok(()) } - fn close(mut self, channel: &SyncerChannel, next_pts: Option, - db: &db::Database, stream_id: i32) -> Result { - let unindexed = self.unindexed_sample.take().expect("should always be an unindexed sample"); + fn close( + mut self, + channel: &SyncerChannel, + next_pts: Option, + db: &db::Database, + stream_id: i32, + ) -> Result { + let unindexed = self + .unindexed_sample + .take() + .expect("should always be an unindexed sample"); let (last_sample_duration, flags) = match next_pts { None => (0, db::RecordingFlags::TrailingZero as i32), Some(p) => (i32::try_from(p - unindexed.pts_90k)?, 0), }; let blake3 = self.hasher.finalize(); let (run_offset, end); - self.add_sample(last_sample_duration, unindexed.len, unindexed.is_key, - unindexed.local_time, db, stream_id)?; + self.add_sample( + last_sample_duration, + unindexed.len, + unindexed.is_key, + unindexed.local_time, + db, + stream_id, + )?; // This always ends a live segment. let wall_duration; @@ -770,10 +882,7 @@ impl InnerWriter { } drop(self.r); channel.async_save_recording(self.id, wall_duration, self.f); - Ok(PreviousWriter { - end, - run_offset, - }) + Ok(PreviousWriter { end, run_offset }) } } @@ -794,57 +903,84 @@ impl<'a, C: Clocks + Clone, D: DirWriter> Drop for Writer<'a, C, D> { #[cfg(test)] mod tests { - use base::clock::{Clocks, SimulatedClocks}; + use super::Writer; use crate::db::{self, CompositeId, VideoSampleEntryToInsert}; use crate::recording; - use parking_lot::Mutex; + use crate::testutil; + use base::clock::{Clocks, SimulatedClocks}; use log::{trace, warn}; + use parking_lot::Mutex; use std::collections::VecDeque; use std::io; - use std::sync::Arc; use std::sync::mpsc; - use super::Writer; - use crate::testutil; + use std::sync::Arc; #[derive(Clone)] struct MockDir(Arc>>); enum MockDirAction { - Create(CompositeId, Box Result + Send>), + Create( + CompositeId, + Box Result + Send>, + ), Sync(Box Result<(), nix::Error> + Send>), - Unlink(CompositeId, Box Result<(), nix::Error> + Send>), + Unlink( + CompositeId, + Box Result<(), nix::Error> + Send>, + ), } impl MockDir { - fn new() -> Self { MockDir(Arc::new(Mutex::new(VecDeque::new()))) } - fn expect(&self, action: MockDirAction) { self.0.lock().push_back(action); } - fn ensure_done(&self) { assert_eq!(self.0.lock().len(), 0); } + fn new() -> Self { + MockDir(Arc::new(Mutex::new(VecDeque::new()))) + } + fn expect(&self, action: MockDirAction) { + self.0.lock().push_back(action); + } + fn ensure_done(&self) { + assert_eq!(self.0.lock().len(), 0); + } } impl super::DirWriter for MockDir { type File = MockFile; fn create_file(&self, id: CompositeId) -> Result { - match self.0.lock().pop_front().expect("got create_file with no expectation") { + match self + .0 + .lock() + .pop_front() + .expect("got create_file with no expectation") + { MockDirAction::Create(expected_id, ref f) => { assert_eq!(id, expected_id); f(id) - }, + } _ => panic!("got create_file({}), expected something else", id), } } fn sync(&self) -> Result<(), nix::Error> { - match self.0.lock().pop_front().expect("got sync with no expectation") { + match self + .0 + .lock() + .pop_front() + .expect("got sync with no expectation") + { MockDirAction::Sync(f) => f(), _ => panic!("got sync, expected something else"), } } fn unlink_file(&self, id: CompositeId) -> Result<(), nix::Error> { - match self.0.lock().pop_front().expect("got unlink_file with no expectation") { + match self + .0 + .lock() + .pop_front() + .expect("got unlink_file with no expectation") + { MockDirAction::Unlink(expected_id, f) => { assert_eq!(id, expected_id); f(id) - }, + } _ => panic!("got unlink({}), expected something else", id), } } @@ -867,20 +1003,36 @@ mod tests { } impl MockFile { - fn new() -> Self { MockFile(Arc::new(Mutex::new(VecDeque::new()))) } - fn expect(&self, action: MockFileAction) { self.0.lock().push_back(action); } - fn ensure_done(&self) { assert_eq!(self.0.lock().len(), 0); } + fn new() -> Self { + MockFile(Arc::new(Mutex::new(VecDeque::new()))) + } + fn expect(&self, action: MockFileAction) { + self.0.lock().push_back(action); + } + fn ensure_done(&self) { + assert_eq!(self.0.lock().len(), 0); + } } impl super::FileWriter for MockFile { fn sync_all(&self) -> Result<(), io::Error> { - match self.0.lock().pop_front().expect("got sync_all with no expectation") { + match self + .0 + .lock() + .pop_front() + .expect("got sync_all with no expectation") + { MockFileAction::SyncAll(f) => f(), _ => panic!("got sync_all, expected something else"), } } fn write(&mut self, buf: &[u8]) -> Result { - match self.0.lock().pop_front().expect("got write with no expectation") { + match self + .0 + .lock() + .pop_front() + .expect("got write with no expectation") + { MockFileAction::Write(f) => f(buf), _ => panic!("got write({:?}), expected something else", buf), } @@ -900,7 +1052,13 @@ mod tests { fn new_harness(flush_if_sec: i64) -> Harness { let clocks = SimulatedClocks::new(::time::Timespec::new(0, 0)); let tdb = testutil::TestDb::new_with_flush_if_sec(clocks, flush_if_sec); - let dir_id = *tdb.db.lock().sample_file_dirs_by_id().keys().next().unwrap(); + let dir_id = *tdb + .db + .lock() + .sample_file_dirs_by_id() + .keys() + .next() + .unwrap(); // This starts a real fs-backed syncer. Get rid of it. tdb.db.lock().clear_on_flush(); @@ -910,7 +1068,13 @@ mod tests { // Start a mocker syncer. let dir = MockDir::new(); let syncer = super::Syncer { - dir_id: *tdb.db.lock().sample_file_dirs_by_id().keys().next().unwrap(), + dir_id: *tdb + .db + .lock() + .sample_file_dirs_by_id() + .keys() + .next() + .unwrap(), dir: dir.clone(), db: tdb.db.clone(), planned_flushes: std::collections::BinaryHeap::new(), @@ -918,8 +1082,10 @@ mod tests { let (syncer_snd, syncer_rcv) = mpsc::channel(); tdb.db.lock().on_flush(Box::new({ let snd = syncer_snd.clone(); - move || if let Err(e) = snd.send(super::SyncerCommand::DatabaseFlushed) { - warn!("Unable to notify syncer for dir {} of flush: {}", dir_id, e); + move || { + if let Err(e) = snd.send(super::SyncerCommand::DatabaseFlushed) { + warn!("Unable to notify syncer for dir {} of flush: {}", dir_id, e); + } } })); Harness { @@ -933,35 +1099,57 @@ mod tests { } } - fn eio() -> io::Error { io::Error::new(io::ErrorKind::Other, "got EIO") } - fn nix_eio() -> nix::Error { nix::Error::Sys(nix::errno::Errno::EIO) } + fn eio() -> io::Error { + io::Error::new(io::ErrorKind::Other, "got EIO") + } + fn nix_eio() -> nix::Error { + nix::Error::Sys(nix::errno::Errno::EIO) + } /// Tests the database flushing while a syncer is still processing a previous flush event. #[test] fn double_flush() { testutil::init(); let mut h = new_harness(0); - h.db.lock().update_retention(&[db::RetentionChange { - stream_id: testutil::TEST_STREAM_ID, - new_record: true, - new_limit: 0, - }]).unwrap(); + h.db.lock() + .update_retention(&[db::RetentionChange { + stream_id: testutil::TEST_STREAM_ID, + new_record: true, + new_limit: 0, + }]) + .unwrap(); // Setup: add a 3-byte recording. - let video_sample_entry_id = h.db.lock().insert_video_sample_entry(VideoSampleEntryToInsert { - width: 1920, - height: 1080, - pasp_h_spacing: 1, - pasp_v_spacing: 1, - data: [0u8; 100].to_vec(), - rfc6381_codec: "avc1.000000".to_owned(), - }).unwrap(); - let mut w = Writer::new(&h.dir, &h.db, &h.channel, testutil::TEST_STREAM_ID, - video_sample_entry_id); + let video_sample_entry_id = + h.db.lock() + .insert_video_sample_entry(VideoSampleEntryToInsert { + width: 1920, + height: 1080, + pasp_h_spacing: 1, + pasp_v_spacing: 1, + data: [0u8; 100].to_vec(), + rfc6381_codec: "avc1.000000".to_owned(), + }) + .unwrap(); + let mut w = Writer::new( + &h.dir, + &h.db, + &h.channel, + testutil::TEST_STREAM_ID, + video_sample_entry_id, + ); let f = MockFile::new(); - h.dir.expect(MockDirAction::Create(CompositeId::new(1, 0), - Box::new({ let f = f.clone(); move |_id| Ok(f.clone()) }))); - f.expect(MockFileAction::Write(Box::new(|buf| { assert_eq!(buf, b"123"); Ok(3) }))); + h.dir.expect(MockDirAction::Create( + CompositeId::new(1, 0), + Box::new({ + let f = f.clone(); + move |_id| Ok(f.clone()) + }), + )); + f.expect(MockFileAction::Write(Box::new(|buf| { + assert_eq!(buf, b"123"); + Ok(3) + }))); f.expect(MockFileAction::SyncAll(Box::new(|| Ok(())))); w.write(b"123", recording::Time(2), 0, true).unwrap(); h.dir.expect(MockDirAction::Sync(Box::new(|| Ok(())))); @@ -976,24 +1164,35 @@ mod tests { // Then a 1-byte recording. let f = MockFile::new(); - h.dir.expect(MockDirAction::Create(CompositeId::new(1, 1), - Box::new({ let f = f.clone(); move |_id| Ok(f.clone()) }))); - f.expect(MockFileAction::Write(Box::new(|buf| { assert_eq!(buf, b"4"); Ok(1) }))); + h.dir.expect(MockDirAction::Create( + CompositeId::new(1, 1), + Box::new({ + let f = f.clone(); + move |_id| Ok(f.clone()) + }), + )); + f.expect(MockFileAction::Write(Box::new(|buf| { + assert_eq!(buf, b"4"); + Ok(1) + }))); f.expect(MockFileAction::SyncAll(Box::new(|| Ok(())))); w.write(b"4", recording::Time(3), 1, true).unwrap(); h.dir.expect(MockDirAction::Sync(Box::new(|| Ok(())))); - h.dir.expect(MockDirAction::Unlink(CompositeId::new(1, 0), Box::new({ - let db = h.db.clone(); - move |_| { - // The drop(w) below should cause the old recording to be deleted (moved to - // garbage). When the database is flushed, the syncer forces garbage collection - // including this unlink. + h.dir.expect(MockDirAction::Unlink( + CompositeId::new(1, 0), + Box::new({ + let db = h.db.clone(); + move |_| { + // The drop(w) below should cause the old recording to be deleted (moved to + // garbage). When the database is flushed, the syncer forces garbage collection + // including this unlink. - // Do another database flush here, as if from another syncer. - db.lock().flush("another syncer running").unwrap(); - Ok(()) - } - }))); + // Do another database flush here, as if from another syncer. + db.lock().flush("another syncer running").unwrap(); + Ok(()) + } + }), + )); h.dir.expect(MockDirAction::Sync(Box::new(|| Ok(())))); drop(w); @@ -1028,8 +1227,10 @@ mod tests { // The syncer should shut down cleanly. drop(h.channel); h.db.lock().clear_on_flush(); - assert_eq!(h.syncer_rcv.try_recv().err(), - Some(std::sync::mpsc::TryRecvError::Disconnected)); + assert_eq!( + h.syncer_rcv.try_recv().err(), + Some(std::sync::mpsc::TryRecvError::Disconnected) + ); assert!(h.syncer.planned_flushes.is_empty()); } @@ -1037,20 +1238,36 @@ mod tests { fn write_path_retries() { testutil::init(); let mut h = new_harness(0); - let video_sample_entry_id = h.db.lock().insert_video_sample_entry(VideoSampleEntryToInsert { - width: 1920, - height: 1080, - pasp_h_spacing: 1, - pasp_v_spacing: 1, - data: [0u8; 100].to_vec(), - rfc6381_codec: "avc1.000000".to_owned(), - }).unwrap(); - let mut w = Writer::new(&h.dir, &h.db, &h.channel, testutil::TEST_STREAM_ID, - video_sample_entry_id); - h.dir.expect(MockDirAction::Create(CompositeId::new(1, 0), Box::new(|_id| Err(nix_eio())))); + let video_sample_entry_id = + h.db.lock() + .insert_video_sample_entry(VideoSampleEntryToInsert { + width: 1920, + height: 1080, + pasp_h_spacing: 1, + pasp_v_spacing: 1, + data: [0u8; 100].to_vec(), + rfc6381_codec: "avc1.000000".to_owned(), + }) + .unwrap(); + let mut w = Writer::new( + &h.dir, + &h.db, + &h.channel, + testutil::TEST_STREAM_ID, + video_sample_entry_id, + ); + h.dir.expect(MockDirAction::Create( + CompositeId::new(1, 0), + Box::new(|_id| Err(nix_eio())), + )); let f = MockFile::new(); - h.dir.expect(MockDirAction::Create(CompositeId::new(1, 0), - Box::new({ let f = f.clone(); move |_id| Ok(f.clone()) }))); + h.dir.expect(MockDirAction::Create( + CompositeId::new(1, 0), + Box::new({ + let f = f.clone(); + move |_id| Ok(f.clone()) + }), + )); f.expect(MockFileAction::Write(Box::new(|buf| { assert_eq!(buf, b"1234"); Err(eio()) @@ -1070,7 +1287,8 @@ mod tests { f.expect(MockFileAction::SyncAll(Box::new(|| Err(eio())))); f.expect(MockFileAction::SyncAll(Box::new(|| Ok(())))); w.write(b"1234", recording::Time(1), 0, true).unwrap(); - h.dir.expect(MockDirAction::Sync(Box::new(|| Err(nix_eio())))); + h.dir + .expect(MockDirAction::Sync(Box::new(|| Err(nix_eio())))); h.dir.expect(MockDirAction::Sync(Box::new(|| Ok(())))); drop(w); assert!(h.syncer.iter(&h.syncer_rcv)); // AsyncSave @@ -1091,8 +1309,10 @@ mod tests { // The syncer should shut down cleanly. drop(h.channel); h.db.lock().clear_on_flush(); - assert_eq!(h.syncer_rcv.try_recv().err(), - Some(std::sync::mpsc::TryRecvError::Disconnected)); + assert_eq!( + h.syncer_rcv.try_recv().err(), + Some(std::sync::mpsc::TryRecvError::Disconnected) + ); assert!(h.syncer.planned_flushes.is_empty()); } @@ -1100,27 +1320,45 @@ mod tests { fn gc_path_retries() { testutil::init(); let mut h = new_harness(0); - h.db.lock().update_retention(&[db::RetentionChange { - stream_id: testutil::TEST_STREAM_ID, - new_record: true, - new_limit: 0, - }]).unwrap(); + h.db.lock() + .update_retention(&[db::RetentionChange { + stream_id: testutil::TEST_STREAM_ID, + new_record: true, + new_limit: 0, + }]) + .unwrap(); // Setup: add a 3-byte recording. - let video_sample_entry_id = h.db.lock().insert_video_sample_entry(VideoSampleEntryToInsert { - width: 1920, - height: 1080, - pasp_h_spacing: 1, - pasp_v_spacing: 1, - data: [0u8; 100].to_vec(), - rfc6381_codec: "avc1.000000".to_owned(), - }).unwrap(); - let mut w = Writer::new(&h.dir, &h.db, &h.channel, testutil::TEST_STREAM_ID, - video_sample_entry_id); + let video_sample_entry_id = + h.db.lock() + .insert_video_sample_entry(VideoSampleEntryToInsert { + width: 1920, + height: 1080, + pasp_h_spacing: 1, + pasp_v_spacing: 1, + data: [0u8; 100].to_vec(), + rfc6381_codec: "avc1.000000".to_owned(), + }) + .unwrap(); + let mut w = Writer::new( + &h.dir, + &h.db, + &h.channel, + testutil::TEST_STREAM_ID, + video_sample_entry_id, + ); let f = MockFile::new(); - h.dir.expect(MockDirAction::Create(CompositeId::new(1, 0), - Box::new({ let f = f.clone(); move |_id| Ok(f.clone()) }))); - f.expect(MockFileAction::Write(Box::new(|buf| { assert_eq!(buf, b"123"); Ok(3) }))); + h.dir.expect(MockDirAction::Create( + CompositeId::new(1, 0), + Box::new({ + let f = f.clone(); + move |_id| Ok(f.clone()) + }), + )); + f.expect(MockFileAction::Write(Box::new(|buf| { + assert_eq!(buf, b"123"); + Ok(3) + }))); f.expect(MockFileAction::SyncAll(Box::new(|| Ok(())))); w.write(b"123", recording::Time(2), 0, true).unwrap(); h.dir.expect(MockDirAction::Sync(Box::new(|| Ok(())))); @@ -1136,31 +1374,46 @@ mod tests { // Then a 1-byte recording. let f = MockFile::new(); - h.dir.expect(MockDirAction::Create(CompositeId::new(1, 1), - Box::new({ let f = f.clone(); move |_id| Ok(f.clone()) }))); - f.expect(MockFileAction::Write(Box::new(|buf| { assert_eq!(buf, b"4"); Ok(1) }))); + h.dir.expect(MockDirAction::Create( + CompositeId::new(1, 1), + Box::new({ + let f = f.clone(); + move |_id| Ok(f.clone()) + }), + )); + f.expect(MockFileAction::Write(Box::new(|buf| { + assert_eq!(buf, b"4"); + Ok(1) + }))); f.expect(MockFileAction::SyncAll(Box::new(|| Ok(())))); w.write(b"4", recording::Time(3), 1, true).unwrap(); h.dir.expect(MockDirAction::Sync(Box::new(|| Ok(())))); - h.dir.expect(MockDirAction::Unlink(CompositeId::new(1, 0), Box::new({ - let db = h.db.clone(); - move |_| { - // The drop(w) below should cause the old recording to be deleted (moved to - // garbage). When the database is flushed, the syncer forces garbage collection - // including this unlink. + h.dir.expect(MockDirAction::Unlink( + CompositeId::new(1, 0), + Box::new({ + let db = h.db.clone(); + move |_| { + // The drop(w) below should cause the old recording to be deleted (moved to + // garbage). When the database is flushed, the syncer forces garbage collection + // including this unlink. - // This should have already applied the changes to sample file bytes, even - // though the garbage has yet to be collected. - let l = db.lock(); - let s = l.streams_by_id().get(&testutil::TEST_STREAM_ID).unwrap(); - assert_eq!(s.bytes_to_delete, 0); - assert_eq!(s.bytes_to_add, 0); - assert_eq!(s.sample_file_bytes, 1); - Err(nix_eio()) // force a retry. - } - }))); - h.dir.expect(MockDirAction::Unlink(CompositeId::new(1, 0), Box::new(|_| Ok(())))); - h.dir.expect(MockDirAction::Sync(Box::new(|| Err(nix_eio())))); + // This should have already applied the changes to sample file bytes, even + // though the garbage has yet to be collected. + let l = db.lock(); + let s = l.streams_by_id().get(&testutil::TEST_STREAM_ID).unwrap(); + assert_eq!(s.bytes_to_delete, 0); + assert_eq!(s.bytes_to_add, 0); + assert_eq!(s.sample_file_bytes, 1); + Err(nix_eio()) // force a retry. + } + }), + )); + h.dir.expect(MockDirAction::Unlink( + CompositeId::new(1, 0), + Box::new(|_| Ok(())), + )); + h.dir + .expect(MockDirAction::Sync(Box::new(|| Err(nix_eio())))); h.dir.expect(MockDirAction::Sync(Box::new(|| Ok(())))); drop(w); @@ -1190,36 +1443,60 @@ mod tests { // The syncer should shut down cleanly. drop(h.channel); h.db.lock().clear_on_flush(); - assert_eq!(h.syncer_rcv.try_recv().err(), - Some(std::sync::mpsc::TryRecvError::Disconnected)); + assert_eq!( + h.syncer_rcv.try_recv().err(), + Some(std::sync::mpsc::TryRecvError::Disconnected) + ); assert!(h.syncer.planned_flushes.is_empty()); } #[test] fn planned_flush() { testutil::init(); - let mut h = new_harness(60); // flush_if_sec=60 + let mut h = new_harness(60); // flush_if_sec=60 // There's a database constraint forbidding a recording starting at t=0, so advance. h.db.clocks().sleep(time::Duration::seconds(1)); // Setup: add a 3-byte recording. - let video_sample_entry_id = h.db.lock().insert_video_sample_entry(VideoSampleEntryToInsert { - width: 1920, - height: 1080, - pasp_h_spacing: 1, - pasp_v_spacing: 1, - data: [0u8; 100].to_vec(), - rfc6381_codec: "avc1.000000".to_owned(), - }).unwrap(); - let mut w = Writer::new(&h.dir, &h.db, &h.channel, testutil::TEST_STREAM_ID, - video_sample_entry_id); + let video_sample_entry_id = + h.db.lock() + .insert_video_sample_entry(VideoSampleEntryToInsert { + width: 1920, + height: 1080, + pasp_h_spacing: 1, + pasp_v_spacing: 1, + data: [0u8; 100].to_vec(), + rfc6381_codec: "avc1.000000".to_owned(), + }) + .unwrap(); + let mut w = Writer::new( + &h.dir, + &h.db, + &h.channel, + testutil::TEST_STREAM_ID, + video_sample_entry_id, + ); let f1 = MockFile::new(); - h.dir.expect(MockDirAction::Create(CompositeId::new(1, 0), - Box::new({ let f = f1.clone(); move |_id| Ok(f.clone()) }))); - f1.expect(MockFileAction::Write(Box::new(|buf| { assert_eq!(buf, b"123"); Ok(3) }))); + h.dir.expect(MockDirAction::Create( + CompositeId::new(1, 0), + Box::new({ + let f = f1.clone(); + move |_id| Ok(f.clone()) + }), + )); + f1.expect(MockFileAction::Write(Box::new(|buf| { + assert_eq!(buf, b"123"); + Ok(3) + }))); f1.expect(MockFileAction::SyncAll(Box::new(|| Ok(())))); - w.write(b"123", recording::Time(recording::TIME_UNITS_PER_SEC), 0, true).unwrap(); + w.write( + b"123", + recording::Time(recording::TIME_UNITS_PER_SEC), + 0, + true, + ) + .unwrap(); h.dir.expect(MockDirAction::Sync(Box::new(|| Ok(())))); drop(w); @@ -1233,14 +1510,33 @@ mod tests { h.db.clocks().sleep(time::Duration::seconds(30)); // Then, a 1-byte recording. - let mut w = Writer::new(&h.dir, &h.db, &h.channel, testutil::TEST_STREAM_ID, - video_sample_entry_id); + let mut w = Writer::new( + &h.dir, + &h.db, + &h.channel, + testutil::TEST_STREAM_ID, + video_sample_entry_id, + ); let f2 = MockFile::new(); - h.dir.expect(MockDirAction::Create(CompositeId::new(1, 1), - Box::new({ let f = f2.clone(); move |_id| Ok(f.clone()) }))); - f2.expect(MockFileAction::Write(Box::new(|buf| { assert_eq!(buf, b"4"); Ok(1) }))); + h.dir.expect(MockDirAction::Create( + CompositeId::new(1, 1), + Box::new({ + let f = f2.clone(); + move |_id| Ok(f.clone()) + }), + )); + f2.expect(MockFileAction::Write(Box::new(|buf| { + assert_eq!(buf, b"4"); + Ok(1) + }))); f2.expect(MockFileAction::SyncAll(Box::new(|| Ok(())))); - w.write(b"4", recording::Time(31*recording::TIME_UNITS_PER_SEC), 1, true).unwrap(); + w.write( + b"4", + recording::Time(31 * recording::TIME_UNITS_PER_SEC), + 1, + true, + ) + .unwrap(); h.dir.expect(MockDirAction::Sync(Box::new(|| Ok(())))); drop(w); @@ -1268,8 +1564,10 @@ mod tests { // The syncer should shut down cleanly. drop(h.channel); h.db.lock().clear_on_flush(); - assert_eq!(h.syncer_rcv.try_recv().err(), - Some(std::sync::mpsc::TryRecvError::Disconnected)); + assert_eq!( + h.syncer_rcv.try_recv().err(), + Some(std::sync::mpsc::TryRecvError::Disconnected) + ); assert!(h.syncer.planned_flushes.is_empty()); } } diff --git a/server/src/analytics.rs b/server/src/analytics.rs index 4580b1c..82889f3 100644 --- a/server/src/analytics.rs +++ b/server/src/analytics.rs @@ -50,7 +50,7 @@ //! interval to cut down on expense. use cstr::cstr; -use failure::{Error, format_err}; +use failure::{format_err, Error}; use ffmpeg; use log::info; use std::sync::Arc; @@ -163,17 +163,28 @@ impl ObjectDetector { let model = moonfire_tflite::Model::from_static(MODEL) .map_err(|()| format_err!("TensorFlow Lite model initialization failed"))?; let devices = moonfire_tflite::edgetpu::Devices::list(); - let device = devices.first().ok_or_else(|| format_err!("No Edge TPU device available"))?; - info!("Using device {:?}/{:?} for object detection", device.type_(), device.path()); + let device = devices + .first() + .ok_or_else(|| format_err!("No Edge TPU device available"))?; + info!( + "Using device {:?}/{:?} for object detection", + device.type_(), + device.path() + ); let mut builder = moonfire_tflite::Interpreter::builder(); - builder.add_owned_delegate(device.create_delegate() - .map_err(|()| format_err!("Unable to create delegate for {:?}/{:?}", - device.type_(), device.path()))?); - let interpreter = builder.build(&model) + builder.add_owned_delegate(device.create_delegate().map_err(|()| { + format_err!( + "Unable to create delegate for {:?}/{:?}", + device.type_(), + device.path() + ) + })?); + let interpreter = builder + .build(&model) .map_err(|()| format_err!("TensorFlow Lite initialization failed"))?; Ok(Arc::new(Self { interpreter: parking_lot::Mutex::new(interpreter), - width: 300, // TODO + width: 300, // TODO height: 300, })) } @@ -194,17 +205,19 @@ fn copy(from: &ffmpeg::avutil::VideoFrame, to: &mut moonfire_tflite::Tensor) { let mut from_i = 0; let mut to_i = 0; for _y in 0..h { - to[to_i..to_i+3*w].copy_from_slice(&from.data[from_i..from_i+3*w]); + to[to_i..to_i + 3 * w].copy_from_slice(&from.data[from_i..from_i + 3 * w]); from_i += from.linesize; - to_i += 3*w; + to_i += 3 * w; } } const SCORE_THRESHOLD: f32 = 0.5; impl ObjectDetectorStream { - pub fn new(par: ffmpeg::avcodec::InputCodecParameters<'_>, - detector: &ObjectDetector) -> Result { + pub fn new( + par: ffmpeg::avcodec::InputCodecParameters<'_>, + detector: &ObjectDetector, + ) -> Result { let mut dopt = ffmpeg::avutil::Dictionary::new(); dopt.set(cstr!("refcounted_frames"), cstr!("0"))?; let decoder = par.new_decoder(&mut dopt)?; @@ -223,15 +236,20 @@ impl ObjectDetectorStream { }) } - pub fn process_frame(&mut self, pkt: &ffmpeg::avcodec::Packet<'_>, - detector: &ObjectDetector) -> Result<(), Error> { + pub fn process_frame( + &mut self, + pkt: &ffmpeg::avcodec::Packet<'_>, + detector: &ObjectDetector, + ) -> Result<(), Error> { if !self.decoder.decode_video(pkt, &mut self.frame)? { return Ok(()); } self.scaler.scale(&self.frame, &mut self.scaled); let mut interpreter = detector.interpreter.lock(); copy(&self.scaled, &mut interpreter.inputs()[0]); - interpreter.invoke().map_err(|()| format_err!("TFLite interpreter invocation failed"))?; + interpreter + .invoke() + .map_err(|()| format_err!("TFLite interpreter invocation failed"))?; let outputs = interpreter.outputs(); let classes = outputs[1].f32s(); let scores = outputs[2].f32s(); diff --git a/server/src/body.rs b/server/src/body.rs index b064cea..361899e 100644 --- a/server/src/body.rs +++ b/server/src/body.rs @@ -31,7 +31,7 @@ //! Tools for implementing a `http_serve::Entity` body composed from many "slices". use base::Error; -use futures::{Stream, stream}; +use futures::{stream, Stream}; use reffers::ARefss; use std::error::Error as StdError; use std::pin::Pin; @@ -47,28 +47,42 @@ pub fn wrap_error(e: Error) -> BoxedError { } impl From> for Chunk { - fn from(r: ARefss<'static, [u8]>) -> Self { Chunk(r) } + fn from(r: ARefss<'static, [u8]>) -> Self { + Chunk(r) + } } impl From<&'static [u8]> for Chunk { - fn from(r: &'static [u8]) -> Self { Chunk(ARefss::new(r)) } + fn from(r: &'static [u8]) -> Self { + Chunk(ARefss::new(r)) + } } impl From<&'static str> for Chunk { - fn from(r: &'static str) -> Self { Chunk(ARefss::new(r.as_bytes())) } + fn from(r: &'static str) -> Self { + Chunk(ARefss::new(r.as_bytes())) + } } impl From for Chunk { - fn from(r: String) -> Self { Chunk(ARefss::new(r.into_bytes()).map(|v| &v[..])) } + fn from(r: String) -> Self { + Chunk(ARefss::new(r.into_bytes()).map(|v| &v[..])) + } } impl From> for Chunk { - fn from(r: Vec) -> Self { Chunk(ARefss::new(r).map(|v| &v[..])) } + fn from(r: Vec) -> Self { + Chunk(ARefss::new(r).map(|v| &v[..])) + } } impl hyper::body::Buf for Chunk { - fn remaining(&self) -> usize { self.0.len() } - fn chunk(&self) -> &[u8] { &*self.0 } + fn remaining(&self) -> usize { + self.0.len() + } + fn chunk(&self) -> &[u8] { + &*self.0 + } fn advance(&mut self, cnt: usize) { self.0 = ::std::mem::replace(&mut self.0, ARefss::new(&[][..])).map(|b| &b[cnt..]); } @@ -83,32 +97,46 @@ impl hyper::body::HttpBody for Body { type Data = Chunk; type Error = BoxedError; - fn poll_data(self: Pin<&mut Self>, cx: &mut std::task::Context) - -> std::task::Poll>> { - // This is safe because the pin is not structural. - // https://doc.rust-lang.org/std/pin/#pinning-is-not-structural-for-field - // (The field _holds_ a pin, but isn't itself pinned.) - unsafe { self.get_unchecked_mut() }.0.get_mut().as_mut().poll_next(cx) + fn poll_data( + self: Pin<&mut Self>, + cx: &mut std::task::Context, + ) -> std::task::Poll>> { + // This is safe because the pin is not structural. + // https://doc.rust-lang.org/std/pin/#pinning-is-not-structural-for-field + // (The field _holds_ a pin, but isn't itself pinned.) + unsafe { self.get_unchecked_mut() } + .0 + .get_mut() + .as_mut() + .poll_next(cx) } - fn poll_trailers(self: Pin<&mut Self>, _cx: &mut std::task::Context) - -> std::task::Poll, Self::Error>> { - std::task::Poll::Ready(Ok(None)) + fn poll_trailers( + self: Pin<&mut Self>, + _cx: &mut std::task::Context, + ) -> std::task::Poll, Self::Error>> { + std::task::Poll::Ready(Ok(None)) } } impl From for Body { - fn from(b: BodyStream) -> Self { Body(SyncWrapper::new(Pin::from(b))) } + fn from(b: BodyStream) -> Self { + Body(SyncWrapper::new(Pin::from(b))) + } } impl> From for Body { fn from(c: C) -> Self { - Body(SyncWrapper::new(Box::pin(stream::once(futures::future::ok(c.into()))))) + Body(SyncWrapper::new(Box::pin(stream::once( + futures::future::ok(c.into()), + )))) } } impl From for Body { fn from(e: Error) -> Self { - Body(SyncWrapper::new(Box::pin(stream::once(futures::future::err(wrap_error(e)))))) + Body(SyncWrapper::new(Box::pin(stream::once( + futures::future::err(wrap_error(e)), + )))) } } diff --git a/server/src/cmds/check.rs b/server/src/cmds/check.rs index 7d20bfc..bdb838e 100644 --- a/server/src/cmds/check.rs +++ b/server/src/cmds/check.rs @@ -38,8 +38,12 @@ use structopt::StructOpt; #[derive(StructOpt)] pub struct Args { /// Directory holding the SQLite3 index database. - #[structopt(long, default_value = "/var/lib/moonfire-nvr/db", value_name="path", - parse(from_os_str))] + #[structopt( + long, + default_value = "/var/lib/moonfire-nvr/db", + value_name = "path", + parse(from_os_str) + )] db_dir: PathBuf, /// Compare sample file lengths on disk to the database. @@ -70,10 +74,13 @@ pub struct Args { pub fn run(args: &Args) -> Result { let (_db_dir, mut conn) = super::open_conn(&args.db_dir, super::OpenMode::ReadWrite)?; - check::run(&mut conn, &check::Options { - compare_lens: args.compare_lens, - trash_orphan_sample_files: args.trash_orphan_sample_files, - delete_orphan_rows: args.delete_orphan_rows, - trash_corrupt_rows: args.trash_corrupt_rows, - }) + check::run( + &mut conn, + &check::Options { + compare_lens: args.compare_lens, + trash_orphan_sample_files: args.trash_orphan_sample_files, + delete_orphan_rows: args.delete_orphan_rows, + trash_corrupt_rows: args.trash_corrupt_rows, + }, + ) } diff --git a/server/src/cmds/config/cameras.rs b/server/src/cmds/config/cameras.rs index 5f7f645..cf6fa56 100644 --- a/server/src/cmds/config/cameras.rs +++ b/server/src/cmds/config/cameras.rs @@ -28,11 +28,11 @@ // You should have received a copy of the GNU General Public License // along with this program. If not, see . -use base::strutil::{decode_size, encode_size}; use crate::stream::{self, Opener, Stream}; -use cursive::Cursive; -use cursive::traits::{Boxable, Identifiable, Finder}; +use base::strutil::{decode_size, encode_size}; +use cursive::traits::{Boxable, Finder, Identifiable}; use cursive::views; +use cursive::Cursive; use db::writer; use failure::Error; use std::collections::BTreeMap; @@ -44,11 +44,35 @@ use url::Url; fn get_change(siv: &mut Cursive) -> db::CameraChange { // Note: these find_name calls are separate statements, which seems to be important: // https://github.com/gyscos/Cursive/issues/144 - let sn = siv.find_name::("short_name").unwrap().get_content().as_str().into(); - let d = siv.find_name::("description").unwrap().get_content().into(); - let h = siv.find_name::("onvif_host").unwrap().get_content().as_str().into(); - let u = siv.find_name::("username").unwrap().get_content().as_str().into(); - let p = siv.find_name::("password").unwrap().get_content().as_str().into(); + let sn = siv + .find_name::("short_name") + .unwrap() + .get_content() + .as_str() + .into(); + let d = siv + .find_name::("description") + .unwrap() + .get_content() + .into(); + let h = siv + .find_name::("onvif_host") + .unwrap() + .get_content() + .as_str() + .into(); + let u = siv + .find_name::("username") + .unwrap() + .get_content() + .as_str() + .into(); + let p = siv + .find_name::("password") + .unwrap() + .get_content() + .as_str() + .into(); let mut c = db::CameraChange { short_name: sn, description: d, @@ -58,16 +82,28 @@ fn get_change(siv: &mut Cursive) -> db::CameraChange { streams: Default::default(), }; for &t in &db::ALL_STREAM_TYPES { - let u = siv.find_name::(&format!("{}_rtsp_url", t.as_str())) - .unwrap().get_content().as_str().into(); - let r = siv.find_name::(&format!("{}_record", t.as_str())) - .unwrap().is_checked(); - let f = i64::from_str(siv.find_name::( - &format!("{}_flush_if_sec", t.as_str())).unwrap().get_content().as_str()) - .unwrap_or(0); - let d = *siv.find_name::>>( - &format!("{}_sample_file_dir", t.as_str())) - .unwrap().selection().unwrap(); + let u = siv + .find_name::(&format!("{}_rtsp_url", t.as_str())) + .unwrap() + .get_content() + .as_str() + .into(); + let r = siv + .find_name::(&format!("{}_record", t.as_str())) + .unwrap() + .is_checked(); + let f = i64::from_str( + siv.find_name::(&format!("{}_flush_if_sec", t.as_str())) + .unwrap() + .get_content() + .as_str(), + ) + .unwrap_or(0); + let d = *siv + .find_name::>>(&format!("{}_sample_file_dir", t.as_str())) + .unwrap() + .selection() + .unwrap(); c.streams[t.index()] = db::StreamChange { rtsp_url: u, sample_file_dir_id: d, @@ -90,11 +126,13 @@ fn press_edit(siv: &mut Cursive, db: &Arc, id: Option) { } }; if let Err(e) = result { - siv.add_layer(views::Dialog::text(format!("Unable to add camera: {}", e)) - .title("Error") - .dismiss_button("Abort")); + siv.add_layer( + views::Dialog::text(format!("Unable to add camera: {}", e)) + .title("Error") + .dismiss_button("Abort"), + ); } else { - siv.pop_layer(); // get rid of the add/edit camera dialog. + siv.pop_layer(); // get rid of the add/edit camera dialog. // Recreate the "Edit cameras" dialog from scratch; it's easier than adding the new entry. siv.pop_layer(); @@ -105,10 +143,13 @@ fn press_edit(siv: &mut Cursive, db: &Arc, id: Option) { fn press_test_inner(url: &Url) -> Result { let stream = stream::FFMPEG.open(stream::Source::Rtsp { url: url.as_str(), - redacted_url: url.as_str(), // don't need redaction in config UI. + redacted_url: url.as_str(), // don't need redaction in config UI. })?; let extra_data = stream.get_extra_data()?; - Ok(format!("{}x{} video stream", extra_data.entry.width, extra_data.entry.height)) + Ok(format!( + "{}x{} video stream", + extra_data.entry.width, extra_data.entry.height + )) } fn press_test(siv: &mut Cursive, t: db::StreamType) { @@ -116,22 +157,28 @@ fn press_test(siv: &mut Cursive, t: db::StreamType) { let mut url = match Url::parse(&c.streams[t.index()].rtsp_url) { Ok(u) => u, Err(e) => { - siv.add_layer(views::Dialog::text( - format!("Unparseable URL: {}", e)) + siv.add_layer( + views::Dialog::text(format!("Unparseable URL: {}", e)) .title("Stream test failed") - .dismiss_button("Back")); + .dismiss_button("Back"), + ); return; - }, + } }; if !c.username.is_empty() { let _ = url.set_username(&c.username); let _ = url.set_password(Some(&c.password)); } - siv.add_layer(views::Dialog::text(format!("Testing {} stream at {}. This may take a while \ - on timeout or if you have a long key frame interval", - t.as_str(), &url)) - .title("Testing")); + siv.add_layer( + views::Dialog::text(format!( + "Testing {} stream at {}. This may take a while \ + on timeout or if you have a long key frame interval", + t.as_str(), + &url + )) + .title("Testing"), + ); // Let siv have this thread for its event loop; do the work in a background thread. // siv.cb_sink doesn't actually wake up the event loop. Tell siv to poll, as a workaround. @@ -147,51 +194,75 @@ fn press_test(siv: &mut Cursive, t: db::StreamType) { Err(ref e) => { siv.add_layer( views::Dialog::text(format!("{} stream at {}:\n\n{}", t.as_str(), &url, e)) - .title("Stream test failed") - .dismiss_button("Back")); + .title("Stream test failed") + .dismiss_button("Back"), + ); return; - }, + } Ok(ref d) => d, }; - siv.add_layer(views::Dialog::text( - format!("{} stream at {}:\n\n{}", t.as_str(), &url, description)) - .title("Stream test succeeded") - .dismiss_button("Back")); - })).unwrap(); + siv.add_layer( + views::Dialog::text(format!( + "{} stream at {}:\n\n{}", + t.as_str(), + &url, + description + )) + .title("Stream test succeeded") + .dismiss_button("Back"), + ); + })) + .unwrap(); }); } fn press_delete(siv: &mut Cursive, db: &Arc, id: i32, name: String, to_delete: i64) { let dialog = if to_delete > 0 { - let prompt = format!("Camera {} has recorded video. Please confirm the amount \ - of data to delete by typing it back:\n\n{}", name, - encode_size(to_delete)); + let prompt = format!( + "Camera {} has recorded video. Please confirm the amount \ + of data to delete by typing it back:\n\n{}", + name, + encode_size(to_delete) + ); views::Dialog::around( views::LinearLayout::vertical() - .child(views::TextView::new(prompt)) - .child(views::DummyView) - .child(views::EditView::new().on_submit({ - let db = db.clone(); - move |siv, _| confirm_deletion(siv, &db, id, to_delete) - }).with_name("confirm"))) + .child(views::TextView::new(prompt)) + .child(views::DummyView) + .child( + views::EditView::new() + .on_submit({ + let db = db.clone(); + move |siv, _| confirm_deletion(siv, &db, id, to_delete) + }) + .with_name("confirm"), + ), + ) .button("Delete", { let db = db.clone(); move |siv| confirm_deletion(siv, &db, id, to_delete) }) } else { - views::Dialog::text(format!("Delete camera {}? This camera has no recorded video.", name)) + views::Dialog::text(format!( + "Delete camera {}? This camera has no recorded video.", + name + )) .button("Delete", { let db = db.clone(); move |s| actually_delete(s, &db, id) }) - }.title("Delete camera").dismiss_button("Cancel"); + } + .title("Delete camera") + .dismiss_button("Cancel"); siv.add_layer(dialog); } fn confirm_deletion(siv: &mut Cursive, db: &Arc, id: i32, to_delete: i64) { - let typed = siv.find_name::("confirm").unwrap().get_content(); + let typed = siv + .find_name::("confirm") + .unwrap() + .get_content(); if decode_size(typed.as_str()).ok() == Some(to_delete) { - siv.pop_layer(); // deletion confirmation dialog + siv.pop_layer(); // deletion confirmation dialog let mut zero_limits = BTreeMap::new(); { @@ -202,7 +273,9 @@ fn confirm_deletion(siv: &mut Cursive, db: &Arc, id: i32, to_delet Some(d) => d, None => continue, }; - let l = zero_limits.entry(dir_id).or_insert_with(|| Vec::with_capacity(2)); + let l = zero_limits + .entry(dir_id) + .or_insert_with(|| Vec::with_capacity(2)); l.push(writer::NewLimit { stream_id, limit: 0, @@ -211,21 +284,27 @@ fn confirm_deletion(siv: &mut Cursive, db: &Arc, id: i32, to_delet } } if let Err(e) = lower_retention(db, zero_limits) { - siv.add_layer(views::Dialog::text(format!("Unable to delete recordings: {}", e)) - .title("Error") - .dismiss_button("Abort")); + siv.add_layer( + views::Dialog::text(format!("Unable to delete recordings: {}", e)) + .title("Error") + .dismiss_button("Abort"), + ); return; } actually_delete(siv, db, id); } else { - siv.add_layer(views::Dialog::text("Please confirm amount.") - .title("Try again") - .dismiss_button("Back")); + siv.add_layer( + views::Dialog::text("Please confirm amount.") + .title("Try again") + .dismiss_button("Back"), + ); } } -fn lower_retention(db: &Arc, zero_limits: BTreeMap>) - -> Result<(), Error> { +fn lower_retention( + db: &Arc, + zero_limits: BTreeMap>, +) -> Result<(), Error> { let dirs_to_open: Vec<_> = zero_limits.keys().map(|id| *id).collect(); db.lock().open_sample_file_dirs(&dirs_to_open[..])?; for (&dir_id, l) in &zero_limits { @@ -235,15 +314,17 @@ fn lower_retention(db: &Arc, zero_limits: BTreeMap, id: i32) { - siv.pop_layer(); // get rid of the add/edit camera dialog. + siv.pop_layer(); // get rid of the add/edit camera dialog. let result = { let mut l = db.lock(); l.delete_camera(id) }; if let Err(e) = result { - siv.add_layer(views::Dialog::text(format!("Unable to delete camera: {}", e)) - .title("Error") - .dismiss_button("Abort")); + siv.add_layer( + views::Dialog::text(format!("Unable to delete camera: {}", e)) + .title("Error") + .dismiss_button("Abort"), + ); } else { // Recreate the "Edit cameras" dialog from scratch; it's easier than adding the new entry. siv.pop_layer(); @@ -255,10 +336,13 @@ fn actually_delete(siv: &mut Cursive, db: &Arc, id: i32) { /// (The former if `item` is None; the latter otherwise.) fn edit_camera_dialog(db: &Arc, siv: &mut Cursive, item: &Option) { let camera_list = views::ListView::new() - .child("id", views::TextView::new(match *item { - None => "".to_string(), - Some(id) => id.to_string(), - })) + .child( + "id", + views::TextView::new(match *item { + None => "".to_string(), + Some(id) => id.to_string(), + }), + ) .child("uuid", views::TextView::new("").with_name("uuid")) .child("short name", views::EditView::new().with_name("short_name")) .child("onvif_host", views::EditView::new().with_name("onvif_host")) @@ -268,32 +352,54 @@ fn edit_camera_dialog(db: &Arc, siv: &mut Cursive, item: &Option = ::std::iter::once(("".to_owned(), None)) - .chain(db.lock() - .sample_file_dirs_by_id() - .iter() - .map(|(&id, d)| (d.path.as_str().to_owned(), Some(id)))) - .collect(); + .chain( + db.lock() + .sample_file_dirs_by_id() + .iter() + .map(|(&id, d)| (d.path.as_str().to_owned(), Some(id))), + ) + .collect(); for &type_ in &db::ALL_STREAM_TYPES { let list = views::ListView::new() - .child("rtsp url", views::LinearLayout::horizontal() - .child(views::EditView::new() - .with_name(format!("{}_rtsp_url", type_.as_str())) - .full_width()) - .child(views::DummyView) - .child(views::Button::new("Test", move |siv| press_test(siv, type_)))) - .child("sample file dir", - views::SelectView::>::new() - .with_all(dirs.iter().map(|d| d.clone())) - .popup() - .with_name(format!("{}_sample_file_dir", type_.as_str()))) - .child("record", views::Checkbox::new().with_name(format!("{}_record", type_.as_str()))) - .child("flush_if_sec", views::EditView::new() - .with_name(format!("{}_flush_if_sec", type_.as_str()))) - .child("usage/capacity", - views::TextView::new("").with_name(format!("{}_usage_cap", type_.as_str()))) + .child( + "rtsp url", + views::LinearLayout::horizontal() + .child( + views::EditView::new() + .with_name(format!("{}_rtsp_url", type_.as_str())) + .full_width(), + ) + .child(views::DummyView) + .child(views::Button::new("Test", move |siv| { + press_test(siv, type_) + })), + ) + .child( + "sample file dir", + views::SelectView::>::new() + .with_all(dirs.iter().map(|d| d.clone())) + .popup() + .with_name(format!("{}_sample_file_dir", type_.as_str())), + ) + .child( + "record", + views::Checkbox::new().with_name(format!("{}_record", type_.as_str())), + ) + .child( + "flush_if_sec", + views::EditView::new().with_name(format!("{}_flush_if_sec", type_.as_str())), + ) + .child( + "usage/capacity", + views::TextView::new("").with_name(format!("{}_usage_cap", type_.as_str())), + ) .min_height(5); layout.add_child(views::DummyView); layout.add_child(views::TextView::new(format!("{} stream", type_.as_str()))); @@ -304,8 +410,11 @@ fn edit_camera_dialog(db: &Arc, siv: &mut Cursive, item: &Option, siv: &mut Cursive, item: &Option>| v.set_selection(selected_dir)); + |v: &mut views::SelectView>| v.set_selection(selected_dir), + ); } let name = camera.short_name.clone(); - for &(view_id, content) in &[("short_name", &*camera.short_name), - ("onvif_host", &*camera.onvif_host), - ("username", &*camera.username), - ("password", &*camera.password)] { - dialog.call_on_name(view_id, |v: &mut views::EditView| v.set_content(content.to_string())) - .expect("missing EditView"); + for &(view_id, content) in &[ + ("short_name", &*camera.short_name), + ("onvif_host", &*camera.onvif_host), + ("username", &*camera.username), + ("password", &*camera.password), + ] { + dialog + .call_on_name(view_id, |v: &mut views::EditView| { + v.set_content(content.to_string()) + }) + .expect("missing EditView"); } - dialog.call_on_name("description", - |v: &mut views::TextArea| v.set_content(camera.description.to_string())) - .expect("missing TextArea"); - dialog.title("Edit camera") - .button("Edit", { - let db = db.clone(); - move |s| press_edit(s, &db, Some(camera_id)) - }) - .button("Delete", { - let db = db.clone(); - move |s| press_delete(s, &db, camera_id, name.clone(), bytes) - }) + dialog + .call_on_name("description", |v: &mut views::TextArea| { + v.set_content(camera.description.to_string()) + }) + .expect("missing TextArea"); + dialog + .title("Edit camera") + .button("Edit", { + let db = db.clone(); + move |s| press_edit(s, &db, Some(camera_id)) + }) + .button("Delete", { + let db = db.clone(); + move |s| press_delete(s, &db, camera_id, name.clone(), bytes) + }) } else { for t in &db::ALL_STREAM_TYPES { - dialog.call_on_name(&format!("{}_usage_cap", t.as_str()), - |v: &mut views::TextView| v.set_content("")); + dialog.call_on_name( + &format!("{}_usage_cap", t.as_str()), + |v: &mut views::TextView| v.set_content(""), + ); } - dialog.title("Add camera") - .button("Add", { - let db = db.clone(); - move |s| press_edit(s, &db, None) - }) + dialog.title("Add camera").button("Add", { + let db = db.clone(); + move |s| press_edit(s, &db, None) + }) }; siv.add_layer(dialog.dismiss_button("Cancel")); } pub fn top_dialog(db: &Arc, siv: &mut Cursive) { - siv.add_layer(views::Dialog::around( - views::SelectView::new() - .on_submit({ - let db = db.clone(); - move |siv, item| edit_camera_dialog(&db, siv, item) - }) - .item("".to_string(), None) - .with_all(db.lock() + siv.add_layer( + views::Dialog::around( + views::SelectView::new() + .on_submit({ + let db = db.clone(); + move |siv, item| edit_camera_dialog(&db, siv, item) + }) + .item("".to_string(), None) + .with_all( + db.lock() .cameras_by_id() .iter() - .map(|(&id, camera)| (format!("{}: {}", id, camera.short_name), Some(id)))) - .full_width()) + .map(|(&id, camera)| (format!("{}: {}", id, camera.short_name), Some(id))), + ) + .full_width(), + ) .dismiss_button("Done") - .title("Edit cameras")); + .title("Edit cameras"), + ); } diff --git a/server/src/cmds/config/dirs.rs b/server/src/cmds/config/dirs.rs index 37d2324..288f30a 100644 --- a/server/src/cmds/config/dirs.rs +++ b/server/src/cmds/config/dirs.rs @@ -29,9 +29,9 @@ // along with this program. If not, see . use base::strutil::{decode_size, encode_size}; -use cursive::Cursive; use cursive::traits::{Boxable, Identifiable}; use cursive::views; +use cursive::Cursive; use db::writer; use failure::Error; use log::{debug, trace}; @@ -44,7 +44,7 @@ struct Stream { label: String, used: i64, record: bool, - retain: Option, // None if unparseable + retain: Option, // None if unparseable } struct Model { @@ -72,9 +72,11 @@ fn update_limits_inner(model: &Model) -> Result<(), Error> { fn update_limits(model: &Model, siv: &mut Cursive) { if let Err(e) = update_limits_inner(model) { - siv.add_layer(views::Dialog::text(format!("Unable to update limits: {}", e)) - .dismiss_button("Back") - .title("Error")); + siv.add_layer( + views::Dialog::text(format!("Unable to update limits: {}", e)) + .dismiss_button("Back") + .title("Error"), + ); } } @@ -111,8 +113,8 @@ fn edit_limit(model: &RefCell, siv: &mut Cursive, id: i32, content: &str) if (model.errors == 0) != (old_errors == 0) { trace!("toggling change state: errors={}", model.errors); siv.find_name::("change") - .unwrap() - .set_enabled(model.errors == 0); + .unwrap() + .set_enabled(model.errors == 0); } } @@ -124,35 +126,48 @@ fn edit_record(model: &RefCell, id: i32, record: bool) { } fn confirm_deletion(model: &RefCell, siv: &mut Cursive, to_delete: i64) { - let typed = siv.find_name::("confirm") - .unwrap() - .get_content(); - debug!("confirm, typed: {} vs expected: {}", typed.as_str(), to_delete); + let typed = siv + .find_name::("confirm") + .unwrap() + .get_content(); + debug!( + "confirm, typed: {} vs expected: {}", + typed.as_str(), + to_delete + ); if decode_size(typed.as_str()).ok() == Some(to_delete) { actually_delete(model, siv); } else { - siv.add_layer(views::Dialog::text("Please confirm amount.") - .title("Try again") - .dismiss_button("Back")); + siv.add_layer( + views::Dialog::text("Please confirm amount.") + .title("Try again") + .dismiss_button("Back"), + ); } } fn actually_delete(model: &RefCell, siv: &mut Cursive) { let model = &*model.borrow(); - let new_limits: Vec<_> = - model.streams.iter() - .map(|(&id, s)| writer::NewLimit {stream_id: id, limit: s.retain.unwrap()}) - .collect(); - siv.pop_layer(); // deletion confirmation - siv.pop_layer(); // retention dialog + let new_limits: Vec<_> = model + .streams + .iter() + .map(|(&id, s)| writer::NewLimit { + stream_id: id, + limit: s.retain.unwrap(), + }) + .collect(); + siv.pop_layer(); // deletion confirmation + siv.pop_layer(); // retention dialog { let mut l = model.db.lock(); - l.open_sample_file_dirs(&[model.dir_id]).unwrap(); // TODO: don't unwrap. + l.open_sample_file_dirs(&[model.dir_id]).unwrap(); // TODO: don't unwrap. } if let Err(e) = writer::lower_retention(model.db.clone(), model.dir_id, &new_limits[..]) { - siv.add_layer(views::Dialog::text(format!("Unable to delete excess video: {}", e)) - .title("Error") - .dismiss_button("Abort")); + siv.add_layer( + views::Dialog::text(format!("Unable to delete excess video: {}", e)) + .title("Error") + .dismiss_button("Abort"), + ); } else { update_limits(model, siv); } @@ -162,26 +177,38 @@ fn press_change(model: &Rc>, siv: &mut Cursive) { if model.borrow().errors > 0 { return; } - let to_delete = model.borrow().streams.values().map( - |s| ::std::cmp::max(s.used - s.retain.unwrap(), 0)).sum(); + let to_delete = model + .borrow() + .streams + .values() + .map(|s| ::std::cmp::max(s.used - s.retain.unwrap(), 0)) + .sum(); debug!("change press, to_delete={}", to_delete); if to_delete > 0 { - let prompt = format!("Some streams' usage exceeds new limit. Please confirm the amount \ - of data to delete by typing it back:\n\n{}", encode_size(to_delete)); + let prompt = format!( + "Some streams' usage exceeds new limit. Please confirm the amount \ + of data to delete by typing it back:\n\n{}", + encode_size(to_delete) + ); let dialog = views::Dialog::around( - views::LinearLayout::vertical() + views::LinearLayout::vertical() .child(views::TextView::new(prompt)) .child(views::DummyView) - .child(views::EditView::new().on_submit({ - let model = model.clone(); - move |siv, _| confirm_deletion(&model, siv, to_delete) - }).with_name("confirm"))) - .button("Confirm", { - let model = model.clone(); - move |siv| confirm_deletion(&model, siv, to_delete) - }) - .dismiss_button("Cancel") - .title("Confirm deletion"); + .child( + views::EditView::new() + .on_submit({ + let model = model.clone(); + move |siv, _| confirm_deletion(&model, siv, to_delete) + }) + .with_name("confirm"), + ), + ) + .button("Confirm", { + let model = model.clone(); + move |siv| confirm_deletion(&model, siv, to_delete) + }) + .dismiss_button("Cancel") + .title("Confirm deletion"); siv.add_layer(dialog); } else { siv.pop_layer(); @@ -190,23 +217,28 @@ fn press_change(model: &Rc>, siv: &mut Cursive) { } pub fn top_dialog(db: &Arc, siv: &mut Cursive) { - siv.add_layer(views::Dialog::around( - views::SelectView::new() - .on_submit({ - let db = db.clone(); - move |siv, item| match *item { - Some(d) => edit_dir_dialog(&db, siv, d), - None => add_dir_dialog(&db, siv), - } - }) - .item("".to_string(), None) - .with_all(db.lock() + siv.add_layer( + views::Dialog::around( + views::SelectView::new() + .on_submit({ + let db = db.clone(); + move |siv, item| match *item { + Some(d) => edit_dir_dialog(&db, siv, d), + None => add_dir_dialog(&db, siv), + } + }) + .item("".to_string(), None) + .with_all( + db.lock() .sample_file_dirs_by_id() .iter() - .map(|(&id, d)| (d.path.to_string(), Some(id)))) - .full_width()) + .map(|(&id, d)| (d.path.to_string(), Some(id))), + ) + .full_width(), + ) .dismiss_button("Done") - .title("Edit sample file directories")); + .title("Edit sample file directories"), + ); } fn add_dir_dialog(db: &Arc, siv: &mut Cursive) { @@ -214,29 +246,40 @@ fn add_dir_dialog(db: &Arc, siv: &mut Cursive) { views::Dialog::around( views::LinearLayout::vertical() .child(views::TextView::new("path")) - .child(views::EditView::new() - .on_submit({ - let db = db.clone(); - move |siv, path| add_dir(&db, siv, path) - }) - .with_name("path") - .fixed_width(60))) - .button("Add", { - let db = db.clone(); - move |siv| { - let path = siv.find_name::("path").unwrap().get_content(); - add_dir(&db, siv, &path) - } - }) - .button("Cancel", |siv| { siv.pop_layer(); }) - .title("Add sample file directory")); + .child( + views::EditView::new() + .on_submit({ + let db = db.clone(); + move |siv, path| add_dir(&db, siv, path) + }) + .with_name("path") + .fixed_width(60), + ), + ) + .button("Add", { + let db = db.clone(); + move |siv| { + let path = siv + .find_name::("path") + .unwrap() + .get_content(); + add_dir(&db, siv, &path) + } + }) + .button("Cancel", |siv| { + siv.pop_layer(); + }) + .title("Add sample file directory"), + ); } fn add_dir(db: &Arc, siv: &mut Cursive, path: &str) { if let Err(e) = db.lock().add_sample_file_dir(path.to_owned()) { - siv.add_layer(views::Dialog::text(format!("Unable to add path {}: {}", path, e)) - .dismiss_button("Back") - .title("Error")); + siv.add_layer( + views::Dialog::text(format!("Unable to add path {}: {}", path, e)) + .dismiss_button("Back") + .title("Error"), + ); return; } siv.pop_layer(); @@ -248,23 +291,25 @@ fn add_dir(db: &Arc, siv: &mut Cursive, path: &str) { fn delete_dir_dialog(db: &Arc, siv: &mut Cursive, dir_id: i32) { siv.add_layer( - views::Dialog::around( - views::TextView::new("Empty (no associated streams).")) + views::Dialog::around(views::TextView::new("Empty (no associated streams).")) .button("Delete", { let db = db.clone(); - move |siv| { - delete_dir(&db, siv, dir_id) - } + move |siv| delete_dir(&db, siv, dir_id) }) - .button("Cancel", |siv| { siv.pop_layer(); }) - .title("Delete sample file directory")); + .button("Cancel", |siv| { + siv.pop_layer(); + }) + .title("Delete sample file directory"), + ); } fn delete_dir(db: &Arc, siv: &mut Cursive, dir_id: i32) { if let Err(e) = db.lock().delete_sample_file_dir(dir_id) { - siv.add_layer(views::Dialog::text(format!("Unable to delete dir id {}: {}", dir_id, e)) - .dismiss_button("Back") - .title("Error")); + siv.add_layer( + views::Dialog::text(format!("Unable to delete dir id {}: {}", dir_id, e)) + .dismiss_button("Back") + .title("Error"), + ); return; } siv.pop_layer(); @@ -284,23 +329,29 @@ fn edit_dir_dialog(db: &Arc, siv: &mut Cursive, dir_id: i32) { { let mut l = db.lock(); for (&id, s) in l.streams_by_id() { - let c = l.cameras_by_id().get(&s.camera_id).expect("stream without camera"); + let c = l + .cameras_by_id() + .get(&s.camera_id) + .expect("stream without camera"); if s.sample_file_dir_id != Some(dir_id) { continue; } - streams.insert(id, Stream { - label: format!("{}: {}: {}", id, c.short_name, s.type_.as_str()), - used: s.fs_bytes, - record: s.record, - retain: Some(s.retain_bytes), - }); + streams.insert( + id, + Stream { + label: format!("{}: {}: {}", id, c.short_name, s.type_.as_str()), + used: s.fs_bytes, + record: s.record, + retain: Some(s.retain_bytes), + }, + ); total_used += s.fs_bytes; total_retain += s.retain_bytes; } if streams.is_empty() { return delete_dir_dialog(db, siv, dir_id); } - l.open_sample_file_dirs(&[dir_id]).unwrap(); // TODO: don't unwrap. + l.open_sample_file_dirs(&[dir_id]).unwrap(); // TODO: don't unwrap. let dir = l.sample_file_dirs_by_id().get(&dir_id).unwrap(); let stat = dir.get().unwrap().statfs().unwrap(); fs_capacity = stat.block_size() as i64 * stat.blocks_available() as i64 + total_used; @@ -326,7 +377,8 @@ fn edit_dir_dialog(db: &Arc, siv: &mut Cursive, dir_id: i32) { views::LinearLayout::horizontal() .child(views::TextView::new("record").fixed_width(RECORD_WIDTH)) .child(views::TextView::new("usage").fixed_width(BYTES_WIDTH)) - .child(views::TextView::new("limit").fixed_width(BYTES_WIDTH))); + .child(views::TextView::new("limit").fixed_width(BYTES_WIDTH)), + ); for (&id, stream) in &model.borrow().streams { let mut record_cb = views::Checkbox::new(); record_cb.set_checked(stream.record); @@ -339,50 +391,67 @@ fn edit_dir_dialog(db: &Arc, siv: &mut Cursive, dir_id: i32) { views::LinearLayout::horizontal() .child(record_cb.fixed_width(RECORD_WIDTH)) .child(views::TextView::new(encode_size(stream.used)).fixed_width(BYTES_WIDTH)) - .child(views::EditView::new() - .content(encode_size(stream.retain.unwrap())) - .on_edit({ - let model = model.clone(); - move |siv, content, _pos| edit_limit(&model, siv, id, content) - }) - .on_submit({ - let model = model.clone(); - move |siv, _| press_change(&model, siv) - }) - .fixed_width(20)) - .child(views::TextView::new("").with_name(format!("{}_ok", id)).fixed_width(1))); + .child( + views::EditView::new() + .content(encode_size(stream.retain.unwrap())) + .on_edit({ + let model = model.clone(); + move |siv, content, _pos| edit_limit(&model, siv, id, content) + }) + .on_submit({ + let model = model.clone(); + move |siv, _| press_change(&model, siv) + }) + .fixed_width(20), + ) + .child( + views::TextView::new("") + .with_name(format!("{}_ok", id)) + .fixed_width(1), + ), + ); } let over = model.borrow().total_retain > model.borrow().fs_capacity; list.add_child( "total", views::LinearLayout::horizontal() - .child(views::DummyView{}.fixed_width(RECORD_WIDTH)) - .child(views::TextView::new(encode_size(model.borrow().total_used)) - .fixed_width(BYTES_WIDTH)) - .child(views::TextView::new(encode_size(model.borrow().total_retain)) - .with_name("total_retain").fixed_width(BYTES_WIDTH)) - .child(views::TextView::new(if over { "*" } else { " " }).with_name("total_ok"))); + .child(views::DummyView {}.fixed_width(RECORD_WIDTH)) + .child( + views::TextView::new(encode_size(model.borrow().total_used)) + .fixed_width(BYTES_WIDTH), + ) + .child( + views::TextView::new(encode_size(model.borrow().total_retain)) + .with_name("total_retain") + .fixed_width(BYTES_WIDTH), + ) + .child(views::TextView::new(if over { "*" } else { " " }).with_name("total_ok")), + ); list.add_child( "filesystem", views::LinearLayout::horizontal() - .child(views::DummyView{}.fixed_width(3)) - .child(views::DummyView{}.fixed_width(20)) - .child(views::TextView::new(encode_size(model.borrow().fs_capacity)).fixed_width(25))); + .child(views::DummyView {}.fixed_width(3)) + .child(views::DummyView {}.fixed_width(20)) + .child(views::TextView::new(encode_size(model.borrow().fs_capacity)).fixed_width(25)), + ); let mut change_button = views::Button::new("Change", { let model = model.clone(); move |siv| press_change(&model, siv) }); change_button.set_enabled(!over); - let mut buttons = views::LinearLayout::horizontal() - .child(views::DummyView.full_width()); + let mut buttons = views::LinearLayout::horizontal().child(views::DummyView.full_width()); buttons.add_child(change_button.with_name("change")); buttons.add_child(views::DummyView); - buttons.add_child(views::Button::new("Cancel", |siv| { siv.pop_layer(); })); + buttons.add_child(views::Button::new("Cancel", |siv| { + siv.pop_layer(); + })); siv.add_layer( views::Dialog::around( views::LinearLayout::vertical() .child(list) .child(views::DummyView) - .child(buttons)) - .title(format!("Edit retention for {}", path))); + .child(buttons), + ) + .title(format!("Edit retention for {}", path)), + ); } diff --git a/server/src/cmds/config/mod.rs b/server/src/cmds/config/mod.rs index 05aef88..688dba7 100644 --- a/server/src/cmds/config/mod.rs +++ b/server/src/cmds/config/mod.rs @@ -34,8 +34,8 @@ //! configuration will likely be almost entirely done through a web-based UI. use base::clock; -use cursive::Cursive; use cursive::views; +use cursive::Cursive; use db; use failure::Error; use std::path::PathBuf; @@ -49,8 +49,12 @@ mod users; #[derive(StructOpt)] pub struct Args { /// Directory holding the SQLite3 index database. - #[structopt(long, default_value = "/var/lib/moonfire-nvr/db", value_name="path", - parse(from_os_str))] + #[structopt( + long, + default_value = "/var/lib/moonfire-nvr/db", + value_name = "path", + parse(from_os_str) + )] db_dir: PathBuf, } @@ -62,18 +66,20 @@ pub fn run(args: &Args) -> Result { let mut siv = cursive::default(); //siv.add_global_callback('q', |s| s.quit()); - siv.add_layer(views::Dialog::around( - views::SelectView::, &mut Cursive)>::new() - .on_submit({ - let db = db.clone(); - move |siv, item| item(&db, siv) - }) - .item("Cameras and streams".to_string(), cameras::top_dialog) - .item("Directories and retention".to_string(), dirs::top_dialog) - .item("Users".to_string(), users::top_dialog) - ) + siv.add_layer( + views::Dialog::around( + views::SelectView::, &mut Cursive)>::new() + .on_submit({ + let db = db.clone(); + move |siv, item| item(&db, siv) + }) + .item("Cameras and streams".to_string(), cameras::top_dialog) + .item("Directories and retention".to_string(), dirs::top_dialog) + .item("Users".to_string(), users::top_dialog), + ) .button("Quit", |siv| siv.quit()) - .title("Main menu")); + .title("Main menu"), + ); siv.run(); diff --git a/server/src/cmds/config/users.rs b/server/src/cmds/config/users.rs index 472346b..0b7b5b3 100644 --- a/server/src/cmds/config/users.rs +++ b/server/src/cmds/config/users.rs @@ -28,33 +28,51 @@ // You should have received a copy of the GNU General Public License // along with this program. If not, see . -use cursive::Cursive; use cursive::traits::{Boxable, Identifiable}; use cursive::views; +use cursive::Cursive; use log::info; use std::sync::Arc; /// Builds a `UserChange` from an active `edit_user_dialog`. -fn get_change(siv: &mut Cursive, db: &db::LockedDatabase, id: Option, - pw: PasswordChange) -> db::UserChange { +fn get_change( + siv: &mut Cursive, + db: &db::LockedDatabase, + id: Option, + pw: PasswordChange, +) -> db::UserChange { let mut change = match id { Some(id) => db.users_by_id().get(&id).unwrap().change(), None => db::UserChange::add_user(String::new()), }; change.username.clear(); - change.username += siv.find_name::("username").unwrap().get_content().as_str(); + change.username += siv + .find_name::("username") + .unwrap() + .get_content() + .as_str(); match pw { - PasswordChange::Leave => {}, + PasswordChange::Leave => {} PasswordChange::Set => { - let pwd = siv.find_name::("new_pw").unwrap().get_content(); + let pwd = siv + .find_name::("new_pw") + .unwrap() + .get_content(); change.set_password(pwd.as_str().into()); - }, + } PasswordChange::Clear => change.clear_password(), }; for (id, ref mut b) in &mut [ ("perm_view_video", &mut change.permissions.view_video), - ("perm_read_camera_configs", &mut change.permissions.read_camera_configs), - ("perm_update_signals", &mut change.permissions.update_signals)] { + ( + "perm_read_camera_configs", + &mut change.permissions.read_camera_configs, + ), + ( + "perm_update_signals", + &mut change.permissions.update_signals, + ), + ] { **b = siv.find_name::(id).unwrap().is_checked(); info!("{}: {}", id, **b); } @@ -68,11 +86,13 @@ fn press_edit(siv: &mut Cursive, db: &Arc, id: Option, pw: Pa l.apply_user_change(c).map(|_| ()) }; if let Err(e) = result { - siv.add_layer(views::Dialog::text(format!("Unable to apply change: {}", e)) - .title("Error") - .dismiss_button("Abort")); + siv.add_layer( + views::Dialog::text(format!("Unable to apply change: {}", e)) + .title("Error") + .dismiss_button("Abort"), + ); } else { - siv.pop_layer(); // get rid of the add/edit user dialog. + siv.pop_layer(); // get rid of the add/edit user dialog. // Recreate the "Edit users" dialog from scratch; it's easier than adding the new entry. siv.pop_layer(); @@ -81,24 +101,29 @@ fn press_edit(siv: &mut Cursive, db: &Arc, id: Option, pw: Pa } fn press_delete(siv: &mut Cursive, db: &Arc, id: i32, name: String) { - siv.add_layer(views::Dialog::text(format!("Delete user {}?", name)) - .button("Delete", { - let db = db.clone(); - move |s| actually_delete(s, &db, id) - }) - .title("Delete user").dismiss_button("Cancel")); + siv.add_layer( + views::Dialog::text(format!("Delete user {}?", name)) + .button("Delete", { + let db = db.clone(); + move |s| actually_delete(s, &db, id) + }) + .title("Delete user") + .dismiss_button("Cancel"), + ); } fn actually_delete(siv: &mut Cursive, db: &Arc, id: i32) { - siv.pop_layer(); // get rid of the add/edit user dialog. + siv.pop_layer(); // get rid of the add/edit user dialog. let result = { let mut l = db.lock(); l.delete_user(id) }; if let Err(e) = result { - siv.add_layer(views::Dialog::text(format!("Unable to delete user: {}", e)) - .title("Error") - .dismiss_button("Abort")); + siv.add_layer( + views::Dialog::text(format!("Unable to delete user: {}", e)) + .title("Error") + .dismiss_button("Abort"), + ); } else { // Recreate the "Edit users" dialog from scratch; it's easier than adding the new entry. siv.pop_layer(); @@ -114,7 +139,9 @@ enum PasswordChange { } fn select_set(siv: &mut Cursive) { - siv.find_name::>("pw_set").unwrap().select(); + siv.find_name::>("pw_set") + .unwrap() + .select(); } /// Adds or updates a user. @@ -128,13 +155,18 @@ fn edit_user_dialog(db: &Arc, siv: &mut Cursive, item: Option username = u.map(|u| u.username.clone()).unwrap_or(String::new()); id_str = item.map(|id| id.to_string()).unwrap_or("".to_string()); has_password = u.map(|u| u.has_password()).unwrap_or(false); - permissions = u.map(|u| u.permissions.clone()).unwrap_or(db::Permissions::default()); + permissions = u + .map(|u| u.permissions.clone()) + .unwrap_or(db::Permissions::default()); } let top_list = views::ListView::new() .child("id", views::TextView::new(id_str)) - .child("username", views::EditView::new() - .content(username.clone()) - .with_name("username")); + .child( + "username", + views::EditView::new() + .content(username.clone()) + .with_name("username"), + ); let mut layout = views::LinearLayout::vertical() .child(top_list) .child(views::DummyView) @@ -143,32 +175,48 @@ fn edit_user_dialog(db: &Arc, siv: &mut Cursive, item: Option if has_password { layout.add_child(pw_group.button(PasswordChange::Leave, "Leave set")); layout.add_child(pw_group.button(PasswordChange::Clear, "Clear")); - layout.add_child(views::LinearLayout::horizontal() - .child(pw_group.button(PasswordChange::Set, "Set to:") - .with_name("pw_set")) - .child(views::DummyView) - .child(views::EditView::new() - .on_edit(|siv, _, _| select_set(siv)) - .with_name("new_pw") - .full_width())); + layout.add_child( + views::LinearLayout::horizontal() + .child( + pw_group + .button(PasswordChange::Set, "Set to:") + .with_name("pw_set"), + ) + .child(views::DummyView) + .child( + views::EditView::new() + .on_edit(|siv, _, _| select_set(siv)) + .with_name("new_pw") + .full_width(), + ), + ); } else { layout.add_child(pw_group.button(PasswordChange::Leave, "Leave unset")); - layout.add_child(views::LinearLayout::horizontal() - .child(pw_group.button(PasswordChange::Set, "Reset to:") - .with_name("pw_set")) - .child(views::DummyView) - .child(views::EditView::new() - .on_edit(|siv, _, _| select_set(siv)) - .with_name("new_pw") - .full_width())); + layout.add_child( + views::LinearLayout::horizontal() + .child( + pw_group + .button(PasswordChange::Set, "Reset to:") + .with_name("pw_set"), + ) + .child(views::DummyView) + .child( + views::EditView::new() + .on_edit(|siv, _, _| select_set(siv)) + .with_name("new_pw") + .full_width(), + ), + ); } layout.add_child(views::DummyView); layout.add_child(views::TextView::new("permissions")); let mut perms = views::ListView::new(); - for (name, b) in &[("view_video", permissions.view_video), - ("read_camera_configs", permissions.read_camera_configs), - ("update_signals", permissions.update_signals)] { + for (name, b) in &[ + ("view_video", permissions.view_video), + ("read_camera_configs", permissions.read_camera_configs), + ("update_signals", permissions.update_signals), + ] { let mut checkbox = views::Checkbox::new(); checkbox.set_checked(*b); perms.add_child(name, checkbox.with_name(format!("perm_{}", name))); @@ -177,38 +225,43 @@ fn edit_user_dialog(db: &Arc, siv: &mut Cursive, item: Option let dialog = views::Dialog::around(layout); let dialog = if let Some(id) = item { - dialog.title("Edit user") - .button("Edit", { - let db = db.clone(); - move |s| press_edit(s, &db, item, *pw_group.selection()) - }) - .button("Delete", { - let db = db.clone(); - move |s| press_delete(s, &db, id, username.clone()) - }) + dialog + .title("Edit user") + .button("Edit", { + let db = db.clone(); + move |s| press_edit(s, &db, item, *pw_group.selection()) + }) + .button("Delete", { + let db = db.clone(); + move |s| press_delete(s, &db, id, username.clone()) + }) } else { - dialog.title("Add user") - .button("Add", { - let db = db.clone(); - move |s| press_edit(s, &db, item, *pw_group.selection()) - }) + dialog.title("Add user").button("Add", { + let db = db.clone(); + move |s| press_edit(s, &db, item, *pw_group.selection()) + }) }; siv.add_layer(dialog.dismiss_button("Cancel")); } pub fn top_dialog(db: &Arc, siv: &mut Cursive) { - siv.add_layer(views::Dialog::around( - views::SelectView::new() - .on_submit({ - let db = db.clone(); - move |siv, &item| edit_user_dialog(&db, siv, item) - }) - .item("".to_string(), None) - .with_all(db.lock() + siv.add_layer( + views::Dialog::around( + views::SelectView::new() + .on_submit({ + let db = db.clone(); + move |siv, &item| edit_user_dialog(&db, siv, item) + }) + .item("".to_string(), None) + .with_all( + db.lock() .users_by_id() .iter() - .map(|(&id, user)| (format!("{}: {}", id, user.username), Some(id)))) - .full_width()) + .map(|(&id, user)| (format!("{}: {}", id, user.username), Some(id))), + ) + .full_width(), + ) .dismiss_button("Done") - .title("Edit users")); + .title("Edit users"), + ); } diff --git a/server/src/cmds/init.rs b/server/src/cmds/init.rs index c69194c..a425df0 100644 --- a/server/src/cmds/init.rs +++ b/server/src/cmds/init.rs @@ -30,14 +30,18 @@ use failure::Error; use log::info; -use structopt::StructOpt; use std::path::PathBuf; +use structopt::StructOpt; #[derive(StructOpt)] pub struct Args { /// Directory holding the SQLite3 index database. - #[structopt(long, default_value = "/var/lib/moonfire-nvr/db", value_name="path", - parse(from_os_str))] + #[structopt( + long, + default_value = "/var/lib/moonfire-nvr/db", + value_name = "path", + parse(from_os_str) + )] db_dir: PathBuf, } @@ -55,12 +59,14 @@ pub fn run(args: &Args) -> Result { // page size (so reading large recording_playback rows doesn't require as many seeks). Changing // the page size requires doing a vacuum in non-WAL mode. This will be cheap on an empty // database. https://www.sqlite.org/pragma.html#pragma_page_size - conn.execute_batch(r#" + conn.execute_batch( + r#" pragma journal_mode = delete; pragma page_size = 16384; vacuum; pragma journal_mode = wal; - "#)?; + "#, + )?; db::init(&mut conn)?; info!("Database initialized."); Ok(0) diff --git a/server/src/cmds/login.rs b/server/src/cmds/login.rs index e2fdc16..5e4f0bd 100644 --- a/server/src/cmds/login.rs +++ b/server/src/cmds/login.rs @@ -32,17 +32,21 @@ use base::clock::{self, Clocks}; use db::auth::SessionFlag; -use failure::{Error, format_err}; -use std::os::unix::fs::OpenOptionsExt as _; +use failure::{format_err, Error}; use std::io::Write as _; +use std::os::unix::fs::OpenOptionsExt as _; use std::path::PathBuf; use structopt::StructOpt; #[derive(Debug, Default, StructOpt)] pub struct Args { /// Directory holding the SQLite3 index database. - #[structopt(long, default_value = "/var/lib/moonfire-nvr/db", value_name="path", - parse(from_os_str))] + #[structopt( + long, + default_value = "/var/lib/moonfire-nvr/db", + value_name = "path", + parse(from_os_str) + )] db_dir: PathBuf, /// Create a session with the given permissions. @@ -59,12 +63,16 @@ pub struct Args { /// Write the cookie to a new curl-compatible cookie-jar file. /// /// ---domain must be specified. This file can be used later with curl's --cookie flag. - #[structopt(long, requires("domain"), value_name="path")] + #[structopt(long, requires("domain"), value_name = "path")] curl_cookie_jar: Option, /// Set the given db::auth::SessionFlags. - #[structopt(long, default_value="http-only,secure,same-site,same-site-strict", - value_name="flags", use_delimiter=true)] + #[structopt( + long, + default_value = "http-only,secure,same-site,same-site-strict", + value_name = "flags", + use_delimiter = true + )] session_flags: Vec, /// Create the session for this username. @@ -76,7 +84,8 @@ pub fn run(args: &Args) -> Result { let (_db_dir, conn) = super::open_conn(&args.db_dir, super::OpenMode::ReadWrite)?; let db = std::sync::Arc::new(db::Database::new(clocks.clone(), conn, true).unwrap()); let mut l = db.lock(); - let u = l.get_user(&args.username) + let u = l + .get_user(&args.username) .ok_or_else(|| format_err!("no such user {:?}", &args.username))?; let permissions = args.permissions.as_ref().unwrap_or(&u.permissions).clone(); let creation = db::auth::Request { @@ -90,27 +99,36 @@ pub fn run(args: &Args) -> Result { } let uid = u.id; drop(u); - let (sid, _) = l.make_session(creation, uid, - args.domain.as_ref().map(|d| d.as_bytes().to_owned()), - flags, permissions)?; + let (sid, _) = l.make_session( + creation, + uid, + args.domain.as_ref().map(|d| d.as_bytes().to_owned()), + flags, + permissions, + )?; let mut encoded = [0u8; 64]; base64::encode_config_slice(&sid, base64::STANDARD_NO_PAD, &mut encoded); let encoded = std::str::from_utf8(&encoded[..]).expect("base64 is valid UTF-8"); if let Some(ref p) = args.curl_cookie_jar { - let d = args.domain.as_ref() - .ok_or_else(|| format_err!("--cookiejar requires --domain"))?; + let d = args + .domain + .as_ref() + .ok_or_else(|| format_err!("--cookiejar requires --domain"))?; let mut f = std::fs::OpenOptions::new() .write(true) .create_new(true) .mode(0o600) .open(p) .map_err(|e| format_err!("Unable to open {}: {}", p.display(), e))?; - write!(&mut f, - "# Netscape HTTP Cookie File\n\ - # https://curl.haxx.se/docs/http-cookies.html\n\ - # This file was generated by moonfire-nvr login! Edit at your own risk.\n\n\ - {}\n", curl_cookie(encoded, flags, d))?; + write!( + &mut f, + "# Netscape HTTP Cookie File\n\ + # https://curl.haxx.se/docs/http-cookies.html\n\ + # This file was generated by moonfire-nvr login! Edit at your own risk.\n\n\ + {}\n", + curl_cookie(encoded, flags, d) + )?; f.sync_all()?; println!("Wrote cookie to {}", p.display()); } else { @@ -120,15 +138,25 @@ pub fn run(args: &Args) -> Result { } fn curl_cookie(cookie: &str, flags: i32, domain: &str) -> String { - format!("{httponly}{domain}\t{tailmatch}\t{path}\t{secure}\t{expires}\t{name}\t{value}", - httponly=if (flags & SessionFlag::HttpOnly as i32) != 0 { "#HttpOnly_" } else { "" }, - domain=domain, - tailmatch="FALSE", - path="/", - secure=if (flags & SessionFlag::Secure as i32) != 0 { "TRUE" } else { "FALSE" }, - expires="9223372036854775807", // 64-bit CURL_OFF_T_MAX, never expires - name="s", - value=cookie) + format!( + "{httponly}{domain}\t{tailmatch}\t{path}\t{secure}\t{expires}\t{name}\t{value}", + httponly = if (flags & SessionFlag::HttpOnly as i32) != 0 { + "#HttpOnly_" + } else { + "" + }, + domain = domain, + tailmatch = "FALSE", + path = "/", + secure = if (flags & SessionFlag::Secure as i32) != 0 { + "TRUE" + } else { + "FALSE" + }, + expires = "9223372036854775807", // 64-bit CURL_OFF_T_MAX, never expires + name = "s", + value = cookie + ) } #[cfg(test)] @@ -137,9 +165,14 @@ mod tests { #[test] fn test_curl_cookie() { - assert_eq!(curl_cookie("o3mx3OntO7GzwwsD54OuyQ4IuipYrwPR2aiULPHSudAa+xIhwWjb+w1TnGRh8Z5Q", - SessionFlag::HttpOnly as i32, "localhost"), - "#HttpOnly_localhost\tFALSE\t/\tFALSE\t9223372036854775807\ts\t\ - o3mx3OntO7GzwwsD54OuyQ4IuipYrwPR2aiULPHSudAa+xIhwWjb+w1TnGRh8Z5Q"); + assert_eq!( + curl_cookie( + "o3mx3OntO7GzwwsD54OuyQ4IuipYrwPR2aiULPHSudAa+xIhwWjb+w1TnGRh8Z5Q", + SessionFlag::HttpOnly as i32, + "localhost" + ), + "#HttpOnly_localhost\tFALSE\t/\tFALSE\t9223372036854775807\ts\t\ + o3mx3OntO7GzwwsD54OuyQ4IuipYrwPR2aiULPHSudAa+xIhwWjb+w1TnGRh8Z5Q" + ); } } diff --git a/server/src/cmds/mod.rs b/server/src/cmds/mod.rs index af64962..0b6b5cc 100644 --- a/server/src/cmds/mod.rs +++ b/server/src/cmds/mod.rs @@ -37,8 +37,8 @@ use std::path::Path; pub mod check; pub mod config; -pub mod login; pub mod init; +pub mod login; pub mod run; pub mod sql; pub mod ts; @@ -48,23 +48,35 @@ pub mod upgrade; enum OpenMode { ReadOnly, ReadWrite, - Create + Create, } /// Locks the directory without opening the database. /// The returned `dir::Fd` holds the lock and should be kept open as long as the `Connection` is. fn open_dir(db_dir: &Path, mode: OpenMode) -> Result { - let dir = dir::Fd::open(db_dir, mode == OpenMode::Create) - .map_err(|e| e.context(if e == nix::Error::Sys(nix::errno::Errno::ENOENT) { - format!("db dir {} not found; try running moonfire-nvr init", - db_dir.display()) - } else { - format!("unable to open db dir {}", db_dir.display()) - }))?; + let dir = dir::Fd::open(db_dir, mode == OpenMode::Create).map_err(|e| { + e.context(if e == nix::Error::Sys(nix::errno::Errno::ENOENT) { + format!( + "db dir {} not found; try running moonfire-nvr init", + db_dir.display() + ) + } else { + format!("unable to open db dir {}", db_dir.display()) + }) + })?; let ro = mode == OpenMode::ReadOnly; - dir.lock(if ro { FlockArg::LockSharedNonblock } else { FlockArg::LockExclusiveNonblock }) - .map_err(|e| e.context(format!("unable to get {} lock on db dir {} ", - if ro { "shared" } else { "exclusive" }, db_dir.display())))?; + dir.lock(if ro { + FlockArg::LockSharedNonblock + } else { + FlockArg::LockExclusiveNonblock + }) + .map_err(|e| { + e.context(format!( + "unable to get {} lock on db dir {} ", + if ro { "shared" } else { "exclusive" }, + db_dir.display() + )) + })?; Ok(dir) } @@ -73,8 +85,12 @@ fn open_dir(db_dir: &Path, mode: OpenMode) -> Result { fn open_conn(db_dir: &Path, mode: OpenMode) -> Result<(dir::Fd, rusqlite::Connection), Error> { let dir = open_dir(db_dir, mode)?; let db_path = db_dir.join("db"); - info!("Opening {} in {:?} mode with SQLite version {}", - db_path.display(), mode, rusqlite::version()); + info!( + "Opening {} in {:?} mode with SQLite version {}", + db_path.display(), + mode, + rusqlite::version() + ); let conn = rusqlite::Connection::open_with_flags( db_path, match mode { @@ -86,6 +102,7 @@ fn open_conn(db_dir: &Path, mode: OpenMode) -> Result<(dir::Fd, rusqlite::Connec } | // rusqlite::Connection is not Sync, so there's no reason to tell SQLite3 to use the // serialized threading mode. - rusqlite::OpenFlags::SQLITE_OPEN_NO_MUTEX)?; + rusqlite::OpenFlags::SQLITE_OPEN_NO_MUTEX, + )?; Ok((dir, conn)) } diff --git a/server/src/cmds/run.rs b/server/src/cmds/run.rs index c7f706b..cce8671 100644 --- a/server/src/cmds/run.rs +++ b/server/src/cmds/run.rs @@ -28,34 +28,42 @@ // You should have received a copy of the GNU General Public License // along with this program. If not, see . -use base::clock; use crate::stream; use crate::streamer; use crate::web; +use base::clock; use db::{dir, writer}; -use failure::{Error, bail}; +use failure::{bail, Error}; use fnv::FnvHashMap; use futures::future::FutureExt; use hyper::service::{make_service_fn, service_fn}; use log::{info, warn}; use std::path::PathBuf; -use std::sync::Arc; use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::Arc; use std::thread; use structopt::StructOpt; use tokio; -use tokio::signal::unix::{SignalKind, signal}; +use tokio::signal::unix::{signal, SignalKind}; #[derive(StructOpt)] pub struct Args { /// Directory holding the SQLite3 index database. - #[structopt(long, default_value = "/var/lib/moonfire-nvr/db", value_name="path", - parse(from_os_str))] + #[structopt( + long, + default_value = "/var/lib/moonfire-nvr/db", + value_name = "path", + parse(from_os_str) + )] db_dir: PathBuf, /// Directory holding user interface files (.html, .js, etc). - #[structopt(long, default_value = "/usr/local/lib/moonfire-nvr/ui", value_name="path", - parse(from_os_str))] + #[structopt( + long, + default_value = "/usr/local/lib/moonfire-nvr/ui", + value_name = "path", + parse(from_os_str) + )] ui_dir: std::path::PathBuf, /// Bind address for unencrypted HTTP server. @@ -98,7 +106,7 @@ const LOCALTIME_PATH: &'static str = "/etc/localtime"; const TIMEZONE_PATH: &'static str = "/etc/timezone"; const ZONEINFO_PATHS: [&'static str; 2] = [ "/usr/share/zoneinfo/", // Linux, macOS < High Sierra - "/var/db/timezone/zoneinfo/" // macOS High Sierra + "/var/db/timezone/zoneinfo/", // macOS High Sierra ]; fn trim_zoneinfo(p: &str) -> &str { @@ -145,25 +153,32 @@ fn resolve_zone() -> Result { }; let p = trim_zoneinfo(localtime_dest); if p.starts_with('/') { - bail!("Unable to resolve {} symlink destination {} to a timezone.", - LOCALTIME_PATH, &localtime_dest); + bail!( + "Unable to resolve {} symlink destination {} to a timezone.", + LOCALTIME_PATH, + &localtime_dest + ); } return Ok(p.to_owned()); - }, + } Err(e) => { use ::std::io::ErrorKind; if e.kind() != ErrorKind::NotFound && e.kind() != ErrorKind::InvalidInput { bail!("Unable to read {} symlink: {}", LOCALTIME_PATH, e); } - }, + } }; // If `TIMEZONE_PATH` is a file, use its contents as the zone name. match ::std::fs::read_to_string(TIMEZONE_PATH) { Ok(z) => return Ok(z), Err(e) => { - bail!("Unable to resolve timezone from TZ env, {}, or {}. Last error: {}", - LOCALTIME_PATH, TIMEZONE_PATH, e); + bail!( + "Unable to resolve timezone from TZ env, {}, or {}. Last error: {}", + LOCALTIME_PATH, + TIMEZONE_PATH, + e + ); } } } @@ -179,7 +194,12 @@ pub async fn run(args: &Args) -> Result { let clocks = clock::RealClocks {}; let (_db_dir, conn) = super::open_conn( &args.db_dir, - if args.read_only { super::OpenMode::ReadOnly } else { super::OpenMode::ReadWrite })?; + if args.read_only { + super::OpenMode::ReadOnly + } else { + super::OpenMode::ReadWrite + }, + )?; let db = Arc::new(db::Database::new(clocks.clone(), conn, !args.read_only).unwrap()); info!("Database is loaded."); @@ -190,8 +210,11 @@ pub async fn run(args: &Args) -> Result { { let mut l = db.lock(); - let dirs_to_open: Vec<_> = - l.streams_by_id().values().filter_map(|s| s.sample_file_dir_id).collect(); + let dirs_to_open: Vec<_> = l + .streams_by_id() + .values() + .filter_map(|s| s.sample_file_dir_id) + .collect(); l.open_sample_file_dirs(&dirs_to_open)?; } info!("Directories are opened."); @@ -212,7 +235,9 @@ pub async fn run(args: &Args) -> Result { let syncers = if !args.read_only { let l = db.lock(); let mut dirs = FnvHashMap::with_capacity_and_hasher( - l.sample_file_dirs_by_id().len(), Default::default()); + l.sample_file_dirs_by_id().len(), + Default::default(), + ); let streams = l.streams_by_id().len(); let env = streamer::Environment { db: &db, @@ -236,11 +261,7 @@ pub async fn run(args: &Args) -> Result { let mut syncers = FnvHashMap::with_capacity_and_hasher(dirs.len(), Default::default()); for (id, dir) in dirs.drain() { let (channel, join) = writer::start_syncer(db.clone(), id)?; - syncers.insert(id, Syncer { - dir, - channel, - join, - }); + syncers.insert(id, Syncer { dir, channel, join }); } // Then start up streams. @@ -253,10 +274,14 @@ pub async fn run(args: &Args) -> Result { let sample_file_dir_id = match stream.sample_file_dir_id { Some(s) => s, None => { - warn!("Can't record stream {} ({}/{}) because it has no sample file dir", - id, camera.short_name, stream.type_.as_str()); + warn!( + "Can't record stream {} ({}/{}) because it has no sample file dir", + id, + camera.short_name, + stream.type_.as_str() + ); continue; - }, + } }; let rotate_offset_sec = streamer::ROTATE_INTERVAL_SEC * i as i64 / streams as i64; let syncer = syncers.get(&sample_file_dir_id).unwrap(); @@ -264,20 +289,33 @@ pub async fn run(args: &Args) -> Result { db::StreamType::SUB => object_detector.as_ref().map(|a| Arc::clone(a)), _ => None, }; - let mut streamer = streamer::Streamer::new(&env, syncer.dir.clone(), - syncer.channel.clone(), *id, camera, stream, - rotate_offset_sec, - streamer::ROTATE_INTERVAL_SEC, - object_detector)?; + let mut streamer = streamer::Streamer::new( + &env, + syncer.dir.clone(), + syncer.channel.clone(), + *id, + camera, + stream, + rotate_offset_sec, + streamer::ROTATE_INTERVAL_SEC, + object_detector, + )?; info!("Starting streamer for {}", streamer.short_name()); let name = format!("s-{}", streamer.short_name()); - streamers.push(thread::Builder::new().name(name).spawn(move|| { - streamer.run(); - }).expect("can't create thread")); + streamers.push( + thread::Builder::new() + .name(name) + .spawn(move || { + streamer.run(); + }) + .expect("can't create thread"), + ); } drop(l); Some(syncers) - } else { None }; + } else { + None + }; // Start the web interface. let make_svc = make_service_fn(move |_conn| { @@ -286,13 +324,13 @@ pub async fn run(args: &Args) -> Result { move |req| Arc::clone(&svc).serve(req) })) }); - let server = ::hyper::Server::bind(&args.http_addr).tcp_nodelay(true).serve(make_svc); + let server = ::hyper::Server::bind(&args.http_addr) + .tcp_nodelay(true) + .serve(make_svc); let mut int = signal(SignalKind::interrupt())?; let mut term = signal(SignalKind::terminate())?; - let shutdown = futures::future::select( - Box::pin(int.recv()), - Box::pin(term.recv())); + let shutdown = futures::future::select(Box::pin(int.recv()), Box::pin(term.recv())); let (shutdown_tx, shutdown_rx) = futures::channel::oneshot::channel(); let server = server.with_graceful_shutdown(shutdown_rx.map(|_| ())); diff --git a/server/src/cmds/sql.rs b/server/src/cmds/sql.rs index 7b25aa4..8277d23 100644 --- a/server/src/cmds/sql.rs +++ b/server/src/cmds/sql.rs @@ -30,19 +30,23 @@ //! Subcommand to run a SQLite shell. +use super::OpenMode; use failure::Error; use std::ffi::OsString; use std::os::unix::process::CommandExt; use std::path::PathBuf; use std::process::Command; -use super::OpenMode; use structopt::StructOpt; #[derive(StructOpt)] pub struct Args { /// Directory holding the SQLite3 index database. - #[structopt(long, default_value = "/var/lib/moonfire-nvr/db", value_name="path", - parse(from_os_str))] + #[structopt( + long, + default_value = "/var/lib/moonfire-nvr/db", + value_name = "path", + parse(from_os_str) + )] db_dir: PathBuf, /// Opens the database in read-only mode and locks it only for shared access. @@ -60,7 +64,11 @@ pub struct Args { } pub fn run(args: &Args) -> Result { - let mode = if args.read_only { OpenMode::ReadOnly } else { OpenMode::ReadWrite }; + let mode = if args.read_only { + OpenMode::ReadOnly + } else { + OpenMode::ReadWrite + }; let _db_dir = super::open_dir(&args.db_dir, mode)?; let mut db = OsString::new(); db.push("file:"); @@ -69,5 +77,9 @@ pub fn run(args: &Args) -> Result { if args.read_only { db.push("?mode=ro"); } - Err(Command::new("sqlite3").arg(&db).args(&args.arg).exec().into()) + Err(Command::new("sqlite3") + .arg(&db) + .args(&args.arg) + .exec() + .into()) } diff --git a/server/src/cmds/upgrade/mod.rs b/server/src/cmds/upgrade/mod.rs index 2e819c1..8f2c807 100644 --- a/server/src/cmds/upgrade/mod.rs +++ b/server/src/cmds/upgrade/mod.rs @@ -31,28 +31,35 @@ /// Upgrades the database schema. /// /// See `guide/schema.md` for more information. - use failure::Error; use structopt::StructOpt; #[derive(StructOpt)] pub struct Args { - #[structopt(long, - help = "Directory holding the SQLite3 index database.", - default_value = "/var/lib/moonfire-nvr/db", - parse(from_os_str))] + #[structopt( + long, + help = "Directory holding the SQLite3 index database.", + default_value = "/var/lib/moonfire-nvr/db", + parse(from_os_str) + )] db_dir: std::path::PathBuf, - #[structopt(help = "When upgrading from schema version 1 to 2, the sample file directory.", - long, parse(from_os_str))] + #[structopt( + help = "When upgrading from schema version 1 to 2, the sample file directory.", + long, + parse(from_os_str) + )] sample_file_dir: Option, - #[structopt(help = "Resets the SQLite journal_mode to the specified mode prior to the \ - upgrade. The default, delete, is recommended. off is very dangerous \ - but may be desirable in some circumstances. See guide/schema.md for \ - more information. The journal mode will be reset to wal after the \ - upgrade.", - long, default_value = "delete")] + #[structopt( + help = "Resets the SQLite journal_mode to the specified mode prior to \ + the upgrade. The default, delete, is recommended. off is very \ + dangerous but may be desirable in some circumstances. See \ + guide/schema.md for more information. The journal mode will be \ + reset to wal after the upgrade.", + long, + default_value = "delete" + )] preset_journal: String, #[structopt(help = "Skips the normal post-upgrade vacuum operation.", long)] @@ -62,10 +69,16 @@ pub struct Args { pub fn run(args: &Args) -> Result { let (_db_dir, mut conn) = super::open_conn(&args.db_dir, super::OpenMode::ReadWrite)?; - db::upgrade::run(&db::upgrade::Args { - sample_file_dir: args.sample_file_dir.as_ref().map(std::path::PathBuf::as_path), - preset_journal: &args.preset_journal, - no_vacuum: args.no_vacuum, - }, &mut conn)?; + db::upgrade::run( + &db::upgrade::Args { + sample_file_dir: args + .sample_file_dir + .as_ref() + .map(std::path::PathBuf::as_path), + preset_journal: &args.preset_journal, + no_vacuum: args.no_vacuum, + }, + &mut conn, + )?; Ok(0) } diff --git a/server/src/h264.rs b/server/src/h264.rs index 7e1ce52..73170a1 100644 --- a/server/src/h264.rs +++ b/server/src/h264.rs @@ -41,7 +41,7 @@ //! would be more trouble than it's worth. use byteorder::{BigEndian, ByteOrder, WriteBytesExt}; -use failure::{Error, bail, format_err}; +use failure::{bail, format_err, Error}; use std::convert::TryFrom; // See ISO/IEC 14496-10 table 7-1 - NAL unit type codes, syntax element categories, and NAL unit @@ -49,13 +49,13 @@ use std::convert::TryFrom; const NAL_UNIT_SEQ_PARAMETER_SET: u8 = 7; const NAL_UNIT_PIC_PARAMETER_SET: u8 = 8; -const NAL_UNIT_TYPE_MASK: u8 = 0x1F; // bottom 5 bits of first byte of unit. +const NAL_UNIT_TYPE_MASK: u8 = 0x1F; // bottom 5 bits of first byte of unit. // For certain common sub stream anamorphic resolutions, add a pixel aspect ratio box. const PIXEL_ASPECT_RATIOS: [((u16, u16), (u16, u16)); 4] = [ - ((320, 240), ( 4, 3)), + ((320, 240), (4, 3)), ((352, 240), (40, 33)), - ((640, 480), ( 4, 3)), + ((640, 480), (4, 3)), ((704, 480), (40, 33)), ]; @@ -90,18 +90,22 @@ fn default_pixel_aspect_ratio(width: u16, height: u16) -> (u16, u16) { /// TODO: detect invalid byte streams. For example, several 0x00s not followed by a 0x01, a stream /// stream not starting with 0x00 0x00 0x00 0x01, or an empty NAL unit. fn decode_h264_annex_b<'a, F>(mut data: &'a [u8], mut f: F) -> Result<(), Error> -where F: FnMut(&'a [u8]) -> Result<(), Error> { +where + F: FnMut(&'a [u8]) -> Result<(), Error>, +{ let start_code = &b"\x00\x00\x01"[..]; use nom::FindSubstring; 'outer: while let Some(pos) = data.find_substring(start_code) { let mut unit = &data[0..pos]; - data = &data[pos + start_code.len() ..]; + data = &data[pos + start_code.len()..]; // Have zero or more bytes that end in a start code. Strip out any trailing 0x00s and // process the unit if there's anything left. loop { match unit.last() { None => continue 'outer, - Some(b) if *b == 0 => { unit = &unit[..unit.len()-1]; }, + Some(b) if *b == 0 => { + unit = &unit[..unit.len() - 1]; + } Some(_) => break, } } @@ -139,8 +143,8 @@ fn parse_annex_b_extra_data(data: &[u8]) -> Result<(&[u8], &[u8]), Error> { /// . fn decode(encoded: &[u8]) -> Vec { struct NalRead(Vec); - use h264_reader::Context; use h264_reader::nal::NalHandler; + use h264_reader::Context; impl NalHandler for NalRead { type Ctx = (); fn start(&mut self, _ctx: &mut Context, _header: h264_reader::nal::NalHeader) {} @@ -177,8 +181,7 @@ impl ExtraData { let ctx; let sps_owner; let sps; // reference to either within ctx or to sps_owner. - if extradata.starts_with(b"\x00\x00\x00\x01") || - extradata.starts_with(b"\x00\x00\x01") { + if extradata.starts_with(b"\x00\x00\x00\x01") || extradata.starts_with(b"\x00\x00\x01") { // ffmpeg supplied "extradata" in Annex B format. let (s, p) = parse_annex_b_extra_data(extradata)?; let rbsp = decode(&s[1..]); @@ -196,9 +199,11 @@ impl ExtraData { if avcc.num_of_sequence_parameter_sets() != 1 { bail!("Multiple SPSs!"); } - ctx = avcc.create_context(()) + ctx = avcc + .create_context(()) .map_err(|e| format_err!("Can't load SPS+PPS: {:?}", e))?; - sps = ctx.sps_by_id(h264_reader::nal::pps::ParamSetId::from_u32(0).unwrap()) + sps = ctx + .sps_by_id(h264_reader::nal::pps::ParamSetId::from_u32(0).unwrap()) .ok_or_else(|| format_err!("No SPS 0"))?; }; @@ -212,23 +217,23 @@ impl ExtraData { sample_entry.extend_from_slice(b"\x00\x00\x00\x00avc1\x00\x00\x00\x00\x00\x00\x00\x01"); // VisualSampleEntry, ISO/IEC 14496-12 section 12.1.3. - sample_entry.extend_from_slice(&[0; 16]); // pre-defined + reserved + sample_entry.extend_from_slice(&[0; 16]); // pre-defined + reserved sample_entry.write_u16::(width)?; sample_entry.write_u16::(height)?; sample_entry.extend_from_slice(&[ - 0x00, 0x48, 0x00, 0x00, // horizresolution - 0x00, 0x48, 0x00, 0x00, // vertresolution - 0x00, 0x00, 0x00, 0x00, // reserved - 0x00, 0x01, // frame count - 0x00, 0x00, 0x00, 0x00, // compressorname - 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, - 0x00, 0x18, 0xff, 0xff, // depth + pre_defined + 0x00, 0x48, 0x00, 0x00, // horizresolution + 0x00, 0x48, 0x00, 0x00, // vertresolution + 0x00, 0x00, 0x00, 0x00, // reserved + 0x00, 0x01, // frame count + 0x00, 0x00, 0x00, 0x00, // compressorname + 0x00, 0x00, 0x00, 0x00, // + 0x00, 0x00, 0x00, 0x00, // + 0x00, 0x00, 0x00, 0x00, // + 0x00, 0x00, 0x00, 0x00, // + 0x00, 0x00, 0x00, 0x00, // + 0x00, 0x00, 0x00, 0x00, // + 0x00, 0x00, 0x00, 0x00, // + 0x00, 0x18, 0xff, 0xff, // depth + pre_defined ]); // AVCSampleEntry, ISO/IEC 14496-15 section 5.3.4.1. @@ -245,10 +250,10 @@ impl ExtraData { // "emulation_prevention_three_byte" in ISO/IEC 14496-10 section 7.4. // It looks like 00 is not a valid value of profile_idc, so this distinction // shouldn't be relevant here. And ffmpeg seems to ignore it. - sample_entry.push(1); // configurationVersion - sample_entry.push(sps[1]); // profile_idc . AVCProfileIndication - sample_entry.push(sps[2]); // ...misc bits... . profile_compatibility - sample_entry.push(sps[3]); // level_idc . AVCLevelIndication + sample_entry.push(1); // configurationVersion + sample_entry.push(sps[1]); // profile_idc . AVCProfileIndication + sample_entry.push(sps[2]); // ...misc bits... . profile_compatibility + sample_entry.push(sps[3]); // level_idc . AVCLevelIndication // Hardcode lengthSizeMinusOne to 3, matching TransformSampleData's 4-byte // lengths. @@ -260,41 +265,48 @@ impl ExtraData { sample_entry.push(0xe1); sample_entry.write_u16::(u16::try_from(sps.len())?)?; sample_entry.extend_from_slice(sps); - sample_entry.push(1); // # of PPSs. + sample_entry.push(1); // # of PPSs. sample_entry.write_u16::(u16::try_from(pps.len())?)?; sample_entry.extend_from_slice(pps); - } else { sample_entry.extend_from_slice(extradata); }; // Fix up avc1 and avcC box lengths. let cur_pos = sample_entry.len(); - BigEndian::write_u32(&mut sample_entry[avcc_len_pos .. avcc_len_pos + 4], - u32::try_from(cur_pos - avcc_len_pos)?); + BigEndian::write_u32( + &mut sample_entry[avcc_len_pos..avcc_len_pos + 4], + u32::try_from(cur_pos - avcc_len_pos)?, + ); // PixelAspectRatioBox, ISO/IEC 14496-12 section 12.1.4.2. // Write a PixelAspectRatioBox if necessary, as the sub streams can be be anamorphic. - let pasp = sps.vui_parameters.as_ref() - .and_then(|v| v.aspect_ratio_info.as_ref()) - .and_then(|a| a.clone().get()) - .unwrap_or_else(|| default_pixel_aspect_ratio(width, height)); + let pasp = sps + .vui_parameters + .as_ref() + .and_then(|v| v.aspect_ratio_info.as_ref()) + .and_then(|a| a.clone().get()) + .unwrap_or_else(|| default_pixel_aspect_ratio(width, height)); if pasp != (1, 1) { - sample_entry.extend_from_slice(b"\x00\x00\x00\x10pasp"); // length + box name + sample_entry.extend_from_slice(b"\x00\x00\x00\x10pasp"); // length + box name sample_entry.write_u32::(pasp.0.into())?; sample_entry.write_u32::(pasp.1.into())?; } let cur_pos = sample_entry.len(); - BigEndian::write_u32(&mut sample_entry[avc1_len_pos .. avc1_len_pos + 4], - u32::try_from(cur_pos - avc1_len_pos)?); + BigEndian::write_u32( + &mut sample_entry[avc1_len_pos..avc1_len_pos + 4], + u32::try_from(cur_pos - avc1_len_pos)?, + ); let profile_idc = sample_entry[103]; let constraint_flags = sample_entry[104]; let level_idc = sample_entry[105]; - let rfc6381_codec = - format!("avc1.{:02x}{:02x}{:02x}", profile_idc, constraint_flags, level_idc); + let rfc6381_codec = format!( + "avc1.{:02x}{:02x}{:02x}", + profile_idc, constraint_flags, level_idc + ); Ok(ExtraData { entry: db::VideoSampleEntryToInsert { data: sample_entry, @@ -321,7 +333,7 @@ pub fn transform_sample_data(annexb_sample: &[u8], avc_sample: &mut Vec) -> avc_sample.reserve(annexb_sample.len() + 4); decode_h264_annex_b(annexb_sample, |unit| { // 4-byte length; this must match ParseExtraData's lengthSizeMinusOne == 3. - avc_sample.write_u32::(unit.len() as u32)?; // length + avc_sample.write_u32::(unit.len() as u32)?; // length avc_sample.extend_from_slice(unit); Ok(()) })?; @@ -332,6 +344,7 @@ pub fn transform_sample_data(annexb_sample: &[u8], avc_sample: &mut Vec) -> mod tests { use db::testutil; + #[rustfmt::skip] const ANNEX_B_TEST_INPUT: [u8; 35] = [ 0x00, 0x00, 0x00, 0x01, 0x67, 0x4d, 0x00, 0x1f, 0x9a, 0x66, 0x02, 0x80, 0x2d, 0xff, 0x35, 0x01, @@ -340,6 +353,7 @@ mod tests { 0xee, 0x3c, 0x80, ]; + #[rustfmt::skip] const AVC_DECODER_CONFIG_TEST_INPUT: [u8; 38] = [ 0x01, 0x4d, 0x00, 0x1f, 0xff, 0xe1, 0x00, 0x17, 0x67, 0x4d, 0x00, 0x1f, 0x9a, 0x66, 0x02, 0x80, @@ -348,6 +362,7 @@ mod tests { 0x00, 0x04, 0x68, 0xee, 0x3c, 0x80, ]; + #[rustfmt::skip] const TEST_OUTPUT: [u8; 132] = [ 0x00, 0x00, 0x00, 0x84, 0x61, 0x76, 0x63, 0x31, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, @@ -376,8 +391,9 @@ mod tests { super::decode_h264_annex_b(data, |p| { pieces.push(p); Ok(()) - }).unwrap(); - assert_eq!(&pieces, &[&data[4 .. 27], &data[31 ..]]); + }) + .unwrap(); + assert_eq!(&pieces, &[&data[4..27], &data[31..]]); } #[test] @@ -404,6 +420,7 @@ mod tests { #[test] fn test_transform_sample_data() { testutil::init(); + #[rustfmt::skip] const INPUT: [u8; 64] = [ 0x00, 0x00, 0x00, 0x01, 0x67, 0x4d, 0x00, 0x1f, 0x9a, 0x66, 0x02, 0x80, 0x2d, 0xff, 0x35, 0x01, @@ -420,6 +437,7 @@ mod tests { 0xff, 0x8c, 0xd6, 0x35, // (truncated) ]; + #[rustfmt::skip] const EXPECTED_OUTPUT: [u8; 64] = [ 0x00, 0x00, 0x00, 0x17, 0x67, 0x4d, 0x00, 0x1f, 0x9a, 0x66, 0x02, 0x80, 0x2d, 0xff, 0x35, 0x01, diff --git a/server/src/json.rs b/server/src/json.rs index 21563e4..7a876eb 100644 --- a/server/src/json.rs +++ b/server/src/json.rs @@ -29,15 +29,15 @@ // along with this program. If not, see . use db::auth::SessionHash; -use failure::{Error, format_err}; -use serde::{Deserialize, Serialize}; +use failure::{format_err, Error}; use serde::ser::{Error as _, SerializeMap, SerializeSeq, Serializer}; +use serde::{Deserialize, Serialize}; use std::collections::BTreeMap; use std::ops::Not; use uuid::Uuid; #[derive(Serialize)] -#[serde(rename_all="camelCase")] +#[serde(rename_all = "camelCase")] pub struct TopLevel<'a> { pub time_zone_name: &'a str, @@ -57,7 +57,7 @@ pub struct TopLevel<'a> { } #[derive(Debug, Serialize)] -#[serde(rename_all="camelCase")] +#[serde(rename_all = "camelCase")] pub struct Session { pub username: String, @@ -67,7 +67,9 @@ pub struct Session { impl Session { fn serialize_csrf(csrf: &SessionHash, serializer: S) -> Result - where S: Serializer { + where + S: Serializer, + { let mut tmp = [0u8; 32]; csrf.encode_base64(&mut tmp); serializer.serialize_str(::std::str::from_utf8(&tmp[..]).expect("base64 is UTF-8")) @@ -77,7 +79,7 @@ impl Session { /// JSON serialization wrapper for a single camera when processing `/api/` and /// `/api/cameras//`. See `design/api.md` for details. #[derive(Debug, Serialize)] -#[serde(rename_all="camelCase")] +#[serde(rename_all = "camelCase")] pub struct Camera<'a> { pub uuid: Uuid, pub short_name: &'a str, @@ -91,7 +93,7 @@ pub struct Camera<'a> { } #[derive(Debug, Serialize)] -#[serde(rename_all="camelCase")] +#[serde(rename_all = "camelCase")] pub struct CameraConfig<'a> { pub onvif_host: &'a str, pub username: &'a str, @@ -99,7 +101,7 @@ pub struct CameraConfig<'a> { } #[derive(Debug, Serialize)] -#[serde(rename_all="camelCase")] +#[serde(rename_all = "camelCase")] pub struct Stream<'a> { pub retain_bytes: i64, pub min_start_time_90k: Option, @@ -117,13 +119,13 @@ pub struct Stream<'a> { } #[derive(Debug, Serialize)] -#[serde(rename_all="camelCase")] +#[serde(rename_all = "camelCase")] pub struct StreamConfig<'a> { pub rtsp_url: &'a str, } #[derive(Serialize)] -#[serde(rename_all="camelCase")] +#[serde(rename_all = "camelCase")] pub struct Signal<'a> { pub id: u32, #[serde(serialize_with = "Signal::serialize_cameras")] @@ -134,27 +136,27 @@ pub struct Signal<'a> { } #[derive(Deserialize)] -#[serde(rename_all="camelCase")] +#[serde(rename_all = "camelCase")] pub enum PostSignalsEndBase { Epoch, Now, } #[derive(Deserialize)] -#[serde(rename_all="camelCase")] +#[serde(rename_all = "camelCase")] pub struct LoginRequest<'a> { pub username: &'a str, pub password: String, } #[derive(Deserialize)] -#[serde(rename_all="camelCase")] +#[serde(rename_all = "camelCase")] pub struct LogoutRequest<'a> { pub csrf: &'a str, } #[derive(Deserialize)] -#[serde(rename_all="camelCase")] +#[serde(rename_all = "camelCase")] pub struct PostSignalsRequest { pub signal_ids: Vec, pub states: Vec, @@ -164,13 +166,13 @@ pub struct PostSignalsRequest { } #[derive(Serialize)] -#[serde(rename_all="camelCase")] +#[serde(rename_all = "camelCase")] pub struct PostSignalsResponse { pub time_90k: i64, } #[derive(Default, Serialize)] -#[serde(rename_all="camelCase")] +#[serde(rename_all = "camelCase")] pub struct Signals { pub times_90k: Vec, pub signal_ids: Vec, @@ -178,7 +180,7 @@ pub struct Signals { } #[derive(Debug, Serialize)] -#[serde(rename_all="camelCase")] +#[serde(rename_all = "camelCase")] pub struct SignalType<'a> { pub uuid: Uuid, @@ -187,7 +189,7 @@ pub struct SignalType<'a> { } #[derive(Debug, Serialize)] -#[serde(rename_all="camelCase")] +#[serde(rename_all = "camelCase")] pub struct SignalTypeState<'a> { value: u16, name: &'a str, @@ -198,8 +200,12 @@ pub struct SignalTypeState<'a> { } impl<'a> Camera<'a> { - pub fn wrap(c: &'a db::Camera, db: &'a db::LockedDatabase, include_days: bool, - include_config: bool) -> Result { + pub fn wrap( + c: &'a db::Camera, + db: &'a db::LockedDatabase, + include_days: bool, + include_config: bool, + ) -> Result { Ok(Camera { uuid: c.uuid, short_name: &c.short_name, @@ -220,11 +226,17 @@ impl<'a> Camera<'a> { } fn serialize_streams(streams: &[Option; 2], serializer: S) -> Result - where S: Serializer { + where + S: Serializer, + { let mut map = serializer.serialize_map(Some(streams.len()))?; for (i, s) in streams.iter().enumerate() { if let &Some(ref s) = s { - map.serialize_key(db::StreamType::from_index(i).expect("invalid stream type index").as_str())?; + map.serialize_key( + db::StreamType::from_index(i) + .expect("invalid stream type index") + .as_str(), + )?; map.serialize_value(s)?; } } @@ -233,13 +245,20 @@ impl<'a> Camera<'a> { } impl<'a> Stream<'a> { - fn wrap(db: &'a db::LockedDatabase, id: Option, include_days: bool, include_config: bool) - -> Result, Error> { + fn wrap( + db: &'a db::LockedDatabase, + id: Option, + include_days: bool, + include_config: bool, + ) -> Result, Error> { let id = match id { Some(id) => id, None => return Ok(None), }; - let s = db.streams_by_id().get(&id).ok_or_else(|| format_err!("missing stream {}", id))?; + let s = db + .streams_by_id() + .get(&id) + .ok_or_else(|| format_err!("missing stream {}", id))?; Ok(Some(Stream { retain_bytes: s.retain_bytes, min_start_time_90k: s.range.as_ref().map(|r| r.start.0), @@ -257,9 +276,13 @@ impl<'a> Stream<'a> { })) } - fn serialize_days(days: &Option>, - serializer: S) -> Result - where S: Serializer { + fn serialize_days( + days: &Option>, + serializer: S, + ) -> Result + where + S: Serializer, + { let days = match days.as_ref() { Some(d) => d, None => return serializer.serialize_none(), @@ -268,7 +291,7 @@ impl<'a> Stream<'a> { for (k, v) in days { map.serialize_key(k.as_ref())?; let bounds = k.bounds(); - map.serialize_value(&StreamDayValue{ + map.serialize_value(&StreamDayValue { start_time_90k: bounds.start.0, end_time_90k: bounds.end.0, total_duration_90k: v.duration.0, @@ -289,16 +312,19 @@ impl<'a> Signal<'a> { } } - fn serialize_cameras(cameras: &(&db::Signal, &db::LockedDatabase), - serializer: S) -> Result - where S: Serializer { + fn serialize_cameras( + cameras: &(&db::Signal, &db::LockedDatabase), + serializer: S, + ) -> Result + where + S: Serializer, + { let (s, db) = cameras; let mut map = serializer.serialize_map(Some(s.cameras.len()))?; for sc in &s.cameras { - let c = db.cameras_by_id() - .get(&sc.camera_id) - .ok_or_else(|| S::Error::custom(format!("signal has missing camera id {}", - sc.camera_id)))?; + let c = db.cameras_by_id().get(&sc.camera_id).ok_or_else(|| { + S::Error::custom(format!("signal has missing camera id {}", sc.camera_id)) + })?; map.serialize_key(&c.uuid)?; map.serialize_value(match sc.type_ { db::signal::SignalCameraType::Direct => "direct", @@ -317,9 +343,10 @@ impl<'a> SignalType<'a> { } } - fn serialize_states(type_: &db::signal::Type, - serializer: S) -> Result - where S: Serializer { + fn serialize_states(type_: &db::signal::Type, serializer: S) -> Result + where + S: Serializer, + { let mut seq = serializer.serialize_seq(Some(type_.states.len()))?; for s in &type_.states { seq.serialize_element(&SignalTypeState::wrap(s))?; @@ -340,7 +367,7 @@ impl<'a> SignalTypeState<'a> { } #[derive(Debug, Serialize)] -#[serde(rename_all="camelCase")] +#[serde(rename_all = "camelCase")] struct StreamDayValue { pub start_time_90k: i64, pub end_time_90k: i64, @@ -350,24 +377,33 @@ struct StreamDayValue { impl<'a> TopLevel<'a> { /// Serializes cameras as a list (rather than a map), optionally including the `days` and /// `cameras` fields. - fn serialize_cameras(cameras: &(&db::LockedDatabase, bool, bool), - serializer: S) -> Result - where S: Serializer { + fn serialize_cameras( + cameras: &(&db::LockedDatabase, bool, bool), + serializer: S, + ) -> Result + where + S: Serializer, + { let (db, include_days, include_config) = *cameras; let cs = db.cameras_by_id(); let mut seq = serializer.serialize_seq(Some(cs.len()))?; for (_, c) in cs { seq.serialize_element( &Camera::wrap(c, db, include_days, include_config) - .map_err(|e| S::Error::custom(e))?)?; + .map_err(|e| S::Error::custom(e))?, + )?; } seq.end() } /// Serializes signals as a list (rather than a map), optionally including the `days` field. - fn serialize_signals(signals: &(&db::LockedDatabase, bool), - serializer: S) -> Result - where S: Serializer { + fn serialize_signals( + signals: &(&db::LockedDatabase, bool), + serializer: S, + ) -> Result + where + S: Serializer, + { let (db, include_days) = *signals; let ss = db.signals_by_id(); let mut seq = serializer.serialize_seq(Some(ss.len()))?; @@ -378,9 +414,10 @@ impl<'a> TopLevel<'a> { } /// Serializes signals as a list (rather than a map), optionally including the `days` field. - fn serialize_signal_types(db: &db::LockedDatabase, - serializer: S) -> Result - where S: Serializer { + fn serialize_signal_types(db: &db::LockedDatabase, serializer: S) -> Result + where + S: Serializer, + { let ss = db.signal_types_by_uuid(); let mut seq = serializer.serialize_seq(Some(ss.len()))?; for (u, t) in ss { @@ -391,7 +428,7 @@ impl<'a> TopLevel<'a> { } #[derive(Serialize)] -#[serde(rename_all="camelCase")] +#[serde(rename_all = "camelCase")] pub struct ListRecordings<'a> { pub recordings: Vec, @@ -403,22 +440,27 @@ pub struct ListRecordings<'a> { } impl<'a> ListRecordings<'a> { - fn serialize_video_sample_entries(video_sample_entries: &(&db::LockedDatabase, Vec), - serializer: S) -> Result - where S: Serializer { + fn serialize_video_sample_entries( + video_sample_entries: &(&db::LockedDatabase, Vec), + serializer: S, + ) -> Result + where + S: Serializer, + { let (db, ref v) = *video_sample_entries; let mut map = serializer.serialize_map(Some(v.len()))?; for id in v { map.serialize_entry( id, - &VideoSampleEntry::from(&db.video_sample_entries_by_id().get(id).unwrap()))?; + &VideoSampleEntry::from(&db.video_sample_entries_by_id().get(id).unwrap()), + )?; } map.end() } } #[derive(Debug, Serialize)] -#[serde(rename_all="camelCase")] +#[serde(rename_all = "camelCase")] pub struct Recording { pub start_time_90k: i64, pub end_time_90k: i64, @@ -439,7 +481,7 @@ pub struct Recording { } #[derive(Debug, Serialize)] -#[serde(rename_all="camelCase")] +#[serde(rename_all = "camelCase")] pub struct VideoSampleEntry { pub width: u16, pub height: u16, diff --git a/server/src/main.rs b/server/src/main.rs index b3c06c3..49564f7 100644 --- a/server/src/main.rs +++ b/server/src/main.rs @@ -28,7 +28,7 @@ // You should have received a copy of the GNU General Public License // along with this program. If not, see . -#![cfg_attr(all(feature="nightly", test), feature(test))] +#![cfg_attr(all(feature = "nightly", test), feature(test))] use log::{debug, error}; use std::str::FromStr; @@ -40,7 +40,7 @@ mod analytics; /// Stub implementation of analytics module when not compiled with TensorFlow Lite. #[cfg(not(feature = "analytics"))] mod analytics { - use failure::{Error, bail}; + use failure::{bail, Error}; pub struct ObjectDetector; @@ -53,13 +53,18 @@ mod analytics { pub struct ObjectDetectorStream; impl ObjectDetectorStream { - pub fn new(_par: ffmpeg::avcodec::InputCodecParameters<'_>, - _detector: &ObjectDetector) -> Result { + pub fn new( + _par: ffmpeg::avcodec::InputCodecParameters<'_>, + _detector: &ObjectDetector, + ) -> Result { unimplemented!(); } - pub fn process_frame(&mut self, _pkt: &ffmpeg::avcodec::Packet<'_>, - _detector: &ObjectDetector) -> Result<(), Error> { + pub fn process_frame( + &mut self, + _pkt: &ffmpeg::avcodec::Packet<'_>, + _detector: &ObjectDetector, + ) -> Result<(), Error> { unimplemented!(); } } @@ -76,7 +81,10 @@ mod streamer; mod web; #[derive(StructOpt)] -#[structopt(name="moonfire-nvr", about="security camera network video recorder")] +#[structopt( + name = "moonfire-nvr", + about = "security camera network video recorder" +)] enum Args { /// Checks database integrity (like fsck). Check(cmds::check::Args), @@ -128,10 +136,12 @@ impl Args { fn main() { let args = Args::from_args(); let mut h = mylog::Builder::new() - .set_format(::std::env::var("MOONFIRE_FORMAT") - .map_err(|_| ()) - .and_then(|s| mylog::Format::from_str(&s)) - .unwrap_or(mylog::Format::Google)) + .set_format( + ::std::env::var("MOONFIRE_FORMAT") + .map_err(|_| ()) + .and_then(|s| mylog::Format::from_str(&s)) + .unwrap_or(mylog::Format::Google), + ) .set_spec(&::std::env::var("MOONFIRE_LOG").unwrap_or("info".to_owned())) .build(); h.clone().install().unwrap(); @@ -144,10 +154,10 @@ fn main() { Err(e) => { error!("Exiting due to error: {}", base::prettify_failure(&e)); ::std::process::exit(1); - }, + } Ok(rv) => { debug!("Exiting with status {}", rv); std::process::exit(rv) - }, + } } } diff --git a/server/src/mp4.rs b/server/src/mp4.rs index e6224a4..74b9728 100644 --- a/server/src/mp4.rs +++ b/server/src/mp4.rs @@ -77,14 +77,15 @@ //! * mdat (media data container) //! ``` -use base::{Error, ErrorKind, ResultExt, bail_t, format_err_t}; -use bytes::BytesMut; +use crate::body::{wrap_error, BoxedError, Chunk}; +use crate::slices::{self, Slices}; +use base::{bail_t, format_err_t, Error, ErrorKind, ResultExt}; use byteorder::{BigEndian, ByteOrder, WriteBytesExt}; -use crate::body::{Chunk, BoxedError, wrap_error}; +use bytes::BytesMut; use db::dir; -use db::recording::{self, TIME_UNITS_PER_SEC, rescale}; -use futures::Stream; +use db::recording::{self, rescale, TIME_UNITS_PER_SEC}; use futures::stream; +use futures::Stream; use http; use http::header::HeaderValue; use http_serve; @@ -93,15 +94,14 @@ use log::{debug, error, trace, warn}; use memmap; use parking_lot::Once; use reffers::ARefss; -use crate::slices::{self, Slices}; use smallvec::SmallVec; use std::cell::UnsafeCell; -use std::convert::TryFrom; use std::cmp; +use std::convert::TryFrom; use std::fmt; use std::io; -use std::ops::Range; use std::mem; +use std::ops::Range; use std::sync::Arc; use std::time::SystemTime; @@ -112,184 +112,177 @@ const FORMAT_VERSION: [u8; 1] = [0x07]; /// An `ftyp` (ISO/IEC 14496-12 section 4.3 `FileType`) box. const NORMAL_FTYP_BOX: &'static [u8] = &[ - 0x00, 0x00, 0x00, 0x20, // length = 32, sizeof(NORMAL_FTYP_BOX) - b'f', b't', b'y', b'p', // type - b'i', b's', b'o', b'm', // major_brand - 0x00, 0x00, 0x02, 0x00, // minor_version - b'i', b's', b'o', b'm', // compatible_brands[0] - b'i', b's', b'o', b'2', // compatible_brands[1] - b'a', b'v', b'c', b'1', // compatible_brands[2] - b'm', b'p', b'4', b'1', // compatible_brands[3] + 0x00, 0x00, 0x00, 0x20, // length = 32, sizeof(NORMAL_FTYP_BOX) + b'f', b't', b'y', b'p', // type + b'i', b's', b'o', b'm', // major_brand + 0x00, 0x00, 0x02, 0x00, // minor_version + b'i', b's', b'o', b'm', // compatible_brands[0] + b'i', b's', b'o', b'2', // compatible_brands[1] + b'a', b'v', b'c', b'1', // compatible_brands[2] + b'm', b'p', b'4', b'1', // compatible_brands[3] ]; /// An `ftyp` (ISO/IEC 14496-12 section 4.3 `FileType`) box for an initialization segment. /// More restrictive brands because of the default-base-is-moof flag. const INIT_SEGMENT_FTYP_BOX: &'static [u8] = &[ - 0x00, 0x00, 0x00, 0x10, // length = 16, sizeof(INIT_SEGMENT_FTYP_BOX) - b'f', b't', b'y', b'p', // type - b'i', b's', b'o', b'5', // major_brand - 0x00, 0x00, 0x02, 0x00, // minor_version + 0x00, 0x00, 0x00, 0x10, // length = 16, sizeof(INIT_SEGMENT_FTYP_BOX) + b'f', b't', b'y', b'p', // type + b'i', b's', b'o', b'5', // major_brand + 0x00, 0x00, 0x02, 0x00, // minor_version ]; /// An `hdlr` (ISO/IEC 14496-12 section 8.4.3 `HandlerBox`) box suitable for video. const VIDEO_HDLR_BOX: &'static [u8] = &[ - 0x00, 0x00, 0x00, 0x21, // length == sizeof(kHdlrBox) - b'h', b'd', b'l', b'r', // type == hdlr, ISO/IEC 14496-12 section 8.4.3. - 0x00, 0x00, 0x00, 0x00, // version + flags - 0x00, 0x00, 0x00, 0x00, // pre_defined - b'v', b'i', b'd', b'e', // handler = vide - 0x00, 0x00, 0x00, 0x00, // reserved[0] - 0x00, 0x00, 0x00, 0x00, // reserved[1] - 0x00, 0x00, 0x00, 0x00, // reserved[2] - 0x00, // name, zero-terminated (empty) + 0x00, 0x00, 0x00, 0x21, // length == sizeof(kHdlrBox) + b'h', b'd', b'l', b'r', // type == hdlr, ISO/IEC 14496-12 section 8.4.3. + 0x00, 0x00, 0x00, 0x00, // version + flags + 0x00, 0x00, 0x00, 0x00, // pre_defined + b'v', b'i', b'd', b'e', // handler = vide + 0x00, 0x00, 0x00, 0x00, // reserved[0] + 0x00, 0x00, 0x00, 0x00, // reserved[1] + 0x00, 0x00, 0x00, 0x00, // reserved[2] + 0x00, // name, zero-terminated (empty) ]; /// An `hdlr` (ISO/IEC 14496-12 section 8.4.3 `HandlerBox`) box suitable for subtitles. const SUBTITLE_HDLR_BOX: &'static [u8] = &[ - 0x00, 0x00, 0x00, 0x21, // length == sizeof(kHdlrBox) - b'h', b'd', b'l', b'r', // type == hdlr, ISO/IEC 14496-12 section 8.4.3. - 0x00, 0x00, 0x00, 0x00, // version + flags - 0x00, 0x00, 0x00, 0x00, // pre_defined - b's', b'b', b't', b'l', // handler = sbtl - 0x00, 0x00, 0x00, 0x00, // reserved[0] - 0x00, 0x00, 0x00, 0x00, // reserved[1] - 0x00, 0x00, 0x00, 0x00, // reserved[2] - 0x00, // name, zero-terminated (empty) + 0x00, 0x00, 0x00, 0x21, // length == sizeof(kHdlrBox) + b'h', b'd', b'l', b'r', // type == hdlr, ISO/IEC 14496-12 section 8.4.3. + 0x00, 0x00, 0x00, 0x00, // version + flags + 0x00, 0x00, 0x00, 0x00, // pre_defined + b's', b'b', b't', b'l', // handler = sbtl + 0x00, 0x00, 0x00, 0x00, // reserved[0] + 0x00, 0x00, 0x00, 0x00, // reserved[1] + 0x00, 0x00, 0x00, 0x00, // reserved[2] + 0x00, // name, zero-terminated (empty) ]; /// Part of an `mvhd` (`MovieHeaderBox` version 0, ISO/IEC 14496-12 section 8.2.2), used from /// `append_mvhd`. const MVHD_JUNK: &'static [u8] = &[ - 0x00, 0x01, 0x00, 0x00, // rate - 0x01, 0x00, // volume - 0x00, 0x00, // reserved - 0x00, 0x00, 0x00, 0x00, // reserved - 0x00, 0x00, 0x00, 0x00, // reserved - 0x00, 0x01, 0x00, 0x00, // matrix[0] - 0x00, 0x00, 0x00, 0x00, // matrix[1] - 0x00, 0x00, 0x00, 0x00, // matrix[2] - 0x00, 0x00, 0x00, 0x00, // matrix[3] - 0x00, 0x01, 0x00, 0x00, // matrix[4] - 0x00, 0x00, 0x00, 0x00, // matrix[5] - 0x00, 0x00, 0x00, 0x00, // matrix[6] - 0x00, 0x00, 0x00, 0x00, // matrix[7] - 0x40, 0x00, 0x00, 0x00, // matrix[8] - 0x00, 0x00, 0x00, 0x00, // pre_defined[0] - 0x00, 0x00, 0x00, 0x00, // pre_defined[1] - 0x00, 0x00, 0x00, 0x00, // pre_defined[2] - 0x00, 0x00, 0x00, 0x00, // pre_defined[3] - 0x00, 0x00, 0x00, 0x00, // pre_defined[4] - 0x00, 0x00, 0x00, 0x00, // pre_defined[5] + 0x00, 0x01, 0x00, 0x00, // rate + 0x01, 0x00, // volume + 0x00, 0x00, // reserved + 0x00, 0x00, 0x00, 0x00, // reserved + 0x00, 0x00, 0x00, 0x00, // reserved + 0x00, 0x01, 0x00, 0x00, // matrix[0] + 0x00, 0x00, 0x00, 0x00, // matrix[1] + 0x00, 0x00, 0x00, 0x00, // matrix[2] + 0x00, 0x00, 0x00, 0x00, // matrix[3] + 0x00, 0x01, 0x00, 0x00, // matrix[4] + 0x00, 0x00, 0x00, 0x00, // matrix[5] + 0x00, 0x00, 0x00, 0x00, // matrix[6] + 0x00, 0x00, 0x00, 0x00, // matrix[7] + 0x40, 0x00, 0x00, 0x00, // matrix[8] + 0x00, 0x00, 0x00, 0x00, // pre_defined[0] + 0x00, 0x00, 0x00, 0x00, // pre_defined[1] + 0x00, 0x00, 0x00, 0x00, // pre_defined[2] + 0x00, 0x00, 0x00, 0x00, // pre_defined[3] + 0x00, 0x00, 0x00, 0x00, // pre_defined[4] + 0x00, 0x00, 0x00, 0x00, // pre_defined[5] ]; /// Part of a `tkhd` (`TrackHeaderBox` version 0, ISO/IEC 14496-12 section 8.3.2), used from /// `append_video_tkhd` and `append_subtitle_tkhd`. const TKHD_JUNK: &'static [u8] = &[ - 0x00, 0x00, 0x00, 0x00, // reserved - 0x00, 0x00, 0x00, 0x00, // reserved - 0x00, 0x00, 0x00, 0x00, // layer + alternate_group - 0x00, 0x00, 0x00, 0x00, // volume + reserved - 0x00, 0x01, 0x00, 0x00, // matrix[0] - 0x00, 0x00, 0x00, 0x00, // matrix[1] - 0x00, 0x00, 0x00, 0x00, // matrix[2] - 0x00, 0x00, 0x00, 0x00, // matrix[3] - 0x00, 0x01, 0x00, 0x00, // matrix[4] - 0x00, 0x00, 0x00, 0x00, // matrix[5] - 0x00, 0x00, 0x00, 0x00, // matrix[6] - 0x00, 0x00, 0x00, 0x00, // matrix[7] - 0x40, 0x00, 0x00, 0x00, // matrix[8] + 0x00, 0x00, 0x00, 0x00, // reserved + 0x00, 0x00, 0x00, 0x00, // reserved + 0x00, 0x00, 0x00, 0x00, // layer + alternate_group + 0x00, 0x00, 0x00, 0x00, // volume + reserved + 0x00, 0x01, 0x00, 0x00, // matrix[0] + 0x00, 0x00, 0x00, 0x00, // matrix[1] + 0x00, 0x00, 0x00, 0x00, // matrix[2] + 0x00, 0x00, 0x00, 0x00, // matrix[3] + 0x00, 0x01, 0x00, 0x00, // matrix[4] + 0x00, 0x00, 0x00, 0x00, // matrix[5] + 0x00, 0x00, 0x00, 0x00, // matrix[6] + 0x00, 0x00, 0x00, 0x00, // matrix[7] + 0x40, 0x00, 0x00, 0x00, // matrix[8] ]; /// Part of a `minf` (`MediaInformationBox`, ISO/IEC 14496-12 section 8.4.4), used from /// `append_video_minf`. const VIDEO_MINF_JUNK: &'static [u8] = &[ - b'm', b'i', b'n', b'f', // type = minf, ISO/IEC 14496-12 section 8.4.4. + b'm', b'i', b'n', b'f', // type = minf, ISO/IEC 14496-12 section 8.4.4. // A vmhd box; the "graphicsmode" and "opcolor" values don't have any // meaningful use. - 0x00, 0x00, 0x00, 0x14, // length == sizeof(kVmhdBox) - b'v', b'm', b'h', b'd', // type = vmhd, ISO/IEC 14496-12 section 12.1.2. - 0x00, 0x00, 0x00, 0x01, // version + flags(1) - 0x00, 0x00, 0x00, 0x00, // graphicsmode (copy), opcolor[0] - 0x00, 0x00, 0x00, 0x00, // opcolor[1], opcolor[2] - + 0x00, 0x00, 0x00, 0x14, // length == sizeof(kVmhdBox) + b'v', b'm', b'h', b'd', // type = vmhd, ISO/IEC 14496-12 section 12.1.2. + 0x00, 0x00, 0x00, 0x01, // version + flags(1) + 0x00, 0x00, 0x00, 0x00, // graphicsmode (copy), opcolor[0] + 0x00, 0x00, 0x00, 0x00, // opcolor[1], opcolor[2] // A dinf box suitable for a "self-contained" .mp4 file (no URL/URN // references to external data). - 0x00, 0x00, 0x00, 0x24, // length == sizeof(kDinfBox) - b'd', b'i', b'n', b'f', // type = dinf, ISO/IEC 14496-12 section 8.7.1. - 0x00, 0x00, 0x00, 0x1c, // length - b'd', b'r', b'e', b'f', // type = dref, ISO/IEC 14496-12 section 8.7.2. - 0x00, 0x00, 0x00, 0x00, // version and flags - 0x00, 0x00, 0x00, 0x01, // entry_count - 0x00, 0x00, 0x00, 0x0c, // length - b'u', b'r', b'l', b' ', // type = url, ISO/IEC 14496-12 section 8.7.2. - 0x00, 0x00, 0x00, 0x01, // version=0, flags=self-contained + 0x00, 0x00, 0x00, 0x24, // length == sizeof(kDinfBox) + b'd', b'i', b'n', b'f', // type = dinf, ISO/IEC 14496-12 section 8.7.1. + 0x00, 0x00, 0x00, 0x1c, // length + b'd', b'r', b'e', b'f', // type = dref, ISO/IEC 14496-12 section 8.7.2. + 0x00, 0x00, 0x00, 0x00, // version and flags + 0x00, 0x00, 0x00, 0x01, // entry_count + 0x00, 0x00, 0x00, 0x0c, // length + b'u', b'r', b'l', b' ', // type = url, ISO/IEC 14496-12 section 8.7.2. + 0x00, 0x00, 0x00, 0x01, // version=0, flags=self-contained ]; /// Part of a `minf` (`MediaInformationBox`, ISO/IEC 14496-12 section 8.4.4), used from /// `append_subtitle_minf`. const SUBTITLE_MINF_JUNK: &'static [u8] = &[ - b'm', b'i', b'n', b'f', // type = minf, ISO/IEC 14496-12 section 8.4.4. + b'm', b'i', b'n', b'f', // type = minf, ISO/IEC 14496-12 section 8.4.4. // A nmhd box. - 0x00, 0x00, 0x00, 0x0c, // length == sizeof(kNmhdBox) - b'n', b'm', b'h', b'd', // type = nmhd, ISO/IEC 14496-12 section 12.1.2. - 0x00, 0x00, 0x00, 0x01, // version + flags(1) - + 0x00, 0x00, 0x00, 0x0c, // length == sizeof(kNmhdBox) + b'n', b'm', b'h', b'd', // type = nmhd, ISO/IEC 14496-12 section 12.1.2. + 0x00, 0x00, 0x00, 0x01, // version + flags(1) // A dinf box suitable for a "self-contained" .mp4 file (no URL/URN // references to external data). - 0x00, 0x00, 0x00, 0x24, // length == sizeof(kDinfBox) - b'd', b'i', b'n', b'f', // type = dinf, ISO/IEC 14496-12 section 8.7.1. - 0x00, 0x00, 0x00, 0x1c, // length - b'd', b'r', b'e', b'f', // type = dref, ISO/IEC 14496-12 section 8.7.2. - 0x00, 0x00, 0x00, 0x00, // version and flags - 0x00, 0x00, 0x00, 0x01, // entry_count - 0x00, 0x00, 0x00, 0x0c, // length - b'u', b'r', b'l', b' ', // type = url, ISO/IEC 14496-12 section 8.7.2. - 0x00, 0x00, 0x00, 0x01, // version=0, flags=self-contained + 0x00, 0x00, 0x00, 0x24, // length == sizeof(kDinfBox) + b'd', b'i', b'n', b'f', // type = dinf, ISO/IEC 14496-12 section 8.7.1. + 0x00, 0x00, 0x00, 0x1c, // length + b'd', b'r', b'e', b'f', // type = dref, ISO/IEC 14496-12 section 8.7.2. + 0x00, 0x00, 0x00, 0x00, // version and flags + 0x00, 0x00, 0x00, 0x01, // entry_count + 0x00, 0x00, 0x00, 0x0c, // length + b'u', b'r', b'l', b' ', // type = url, ISO/IEC 14496-12 section 8.7.2. + 0x00, 0x00, 0x00, 0x01, // version=0, flags=self-contained ]; /// Part of a `stbl` (`SampleTableBox`, ISO/IEC 14496 section 8.5.1) used from /// `append_subtitle_stbl`. +#[rustfmt::skip] const SUBTITLE_STBL_JUNK: &'static [u8] = &[ - b's', b't', b'b', b'l', // type = stbl, ISO/IEC 14496-12 section 8.5.1. - + b's', b't', b'b', b'l', // type = stbl, ISO/IEC 14496-12 section 8.5.1. // A stsd box. - 0x00, 0x00, 0x00, 0x54, // length - b's', b't', b's', b'd', // type == stsd, ISO/IEC 14496-12 section 8.5.2. - 0x00, 0x00, 0x00, 0x00, // version + flags - 0x00, 0x00, 0x00, 0x01, // entry_count == 1 - + 0x00, 0x00, 0x00, 0x54, // length + b's', b't', b's', b'd', // type == stsd, ISO/IEC 14496-12 section 8.5.2. + 0x00, 0x00, 0x00, 0x00, // version + flags + 0x00, 0x00, 0x00, 0x01, // entry_count == 1 // SampleEntry, ISO/IEC 14496-12 section 8.5.2.2. - 0x00, 0x00, 0x00, 0x44, // length - b't', b'x', b'3', b'g', // type == tx3g, 3GPP TS 26.245 section 5.16. - 0x00, 0x00, 0x00, 0x00, // reserved - 0x00, 0x00, 0x00, 0x01, // reserved, data_reference_index == 1 - + 0x00, 0x00, 0x00, 0x44, // length + b't', b'x', b'3', b'g', // type == tx3g, 3GPP TS 26.245 section 5.16. + 0x00, 0x00, 0x00, 0x00, // reserved + 0x00, 0x00, 0x00, 0x01, // reserved, data_reference_index == 1 // TextSampleEntry - 0x00, 0x00, 0x00, 0x00, // displayFlags == none - 0x00, // horizontal-justification == left - 0x00, // vertical-justification == top - 0x00, 0x00, 0x00, 0x00, // background-color-rgba == transparent - + 0x00, 0x00, 0x00, 0x00, // displayFlags == none + 0x00, // horizontal-justification == left + 0x00, // vertical-justification == top + 0x00, 0x00, 0x00, 0x00, // background-color-rgba == transparent // TextSampleEntry.BoxRecord - 0x00, 0x00, // top - 0x00, 0x00, // left - 0x00, 0x00, // bottom - 0x00, 0x00, // right - + 0x00, 0x00, // top + 0x00, 0x00, // left + 0x00, 0x00, // bottom + 0x00, 0x00, // right // TextSampleEntry.StyleRecord - 0x00, 0x00, // startChar - 0x00, 0x00, // endChar - 0x00, 0x01, // font-ID - 0x00, // face-style-flags - 0x12, // font-size == 18 px - 0xff, 0xff, 0xff, 0xff, // text-color-rgba == opaque white - + 0x00, 0x00, // startChar + 0x00, 0x00, // endChar + 0x00, 0x01, // font-ID + 0x00, // face-style-flags + 0x12, // font-size == 18 px + 0xff, 0xff, 0xff, 0xff, // text-color-rgba == opaque white // TextSampleEntry.FontTableBox - 0x00, 0x00, 0x00, 0x16, // length - b'f', b't', b'a', b'b', // type == ftab, section 5.16 - 0x00, 0x01, // entry-count == 1 - 0x00, 0x01, // font-ID == 1 - 0x09, // font-name-length == 9 + 0x00, 0x00, 0x00, 0x16, // length + b'f', b't', b'a', b'b', // type == ftab, section 5.16 + 0x00, 0x01, // entry-count == 1 + 0x00, 0x01, // font-ID == 1 + 0x09, // font-name-length == 9 b'M', b'o', b'n', b'o', b's', b'p', b'a', b'c', b'e', ]; @@ -329,7 +322,7 @@ enum StaticBytestring { const SUBTITLE_TEMPLATE: &'static str = "%Y-%m-%d %H:%M:%S %z"; /// The length of the output of `SUBTITLE_TEMPLATE`. -const SUBTITLE_LENGTH: usize = 25; // "2015-07-02 17:10:00 -0700".len(); +const SUBTITLE_LENGTH: usize = 25; // "2015-07-02 17:10:00 -0700".len(); /// The lengths of the indexes associated with a `Segment`; for use within `Segment` only. struct SegmentLengths { @@ -372,22 +365,33 @@ struct Segment { impl fmt::Debug for Segment { fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result { fmt.debug_struct("mp4::Segment") - .field("s", &self.s) - .field("recording_start", &self.recording_start) - .field("recording_wall_duration_90k", &self.recording_wall_duration_90k) - .field("recording_media_duration_90k", &self.recording_media_duration_90k) - .field("rel_media_range_90k", &self.rel_media_range_90k) - .field("first_frame_num", &self.first_frame_num) - .field("num_subtitle_samples", &self.num_subtitle_samples) - .finish() + .field("s", &self.s) + .field("recording_start", &self.recording_start) + .field( + "recording_wall_duration_90k", + &self.recording_wall_duration_90k, + ) + .field( + "recording_media_duration_90k", + &self.recording_media_duration_90k, + ) + .field("rel_media_range_90k", &self.rel_media_range_90k) + .field("first_frame_num", &self.first_frame_num) + .field("num_subtitle_samples", &self.num_subtitle_samples) + .finish() } } unsafe impl Sync for Segment {} impl Segment { - fn new(db: &db::LockedDatabase, row: &db::ListRecordingsRow, rel_media_range_90k: Range, - first_frame_num: u32, start_at_key: bool) -> Result { + fn new( + db: &db::LockedDatabase, + row: &db::ListRecordingsRow, + rel_media_range_90k: Range, + first_frame_num: u32, + start_at_key: bool, + ) -> Result { Ok(Segment { s: recording::Segment::new(db, row, rel_media_range_90k.clone(), start_at_key) .err_kind(ErrorKind::Unknown)?, @@ -403,20 +407,33 @@ impl Segment { } fn wall(&self, rel_media_90k: i32) -> i32 { - rescale(rel_media_90k, self.recording_media_duration_90k, self.recording_wall_duration_90k) + rescale( + rel_media_90k, + self.recording_media_duration_90k, + self.recording_wall_duration_90k, + ) } fn media(&self, rel_wall_90k: i32) -> i32 { - rescale(rel_wall_90k, self.recording_wall_duration_90k, self.recording_media_duration_90k) + rescale( + rel_wall_90k, + self.recording_wall_duration_90k, + self.recording_media_duration_90k, + ) } fn get_index<'a, F>(&'a self, db: &db::Database, f: F) -> Result<&'a [u8], Error> - where F: FnOnce(&[u8], SegmentLengths) -> &[u8] { + where + F: FnOnce(&[u8], SegmentLengths) -> &[u8], + { self.index_once.call_once(|| { let index = unsafe { &mut *self.index.get() }; - *index = db.lock() + *index = db + .lock() .with_recording_playback(self.s.id, &mut |playback| self.build_index(playback)) - .map_err(|e| { error!("Unable to build index for segment: {:?}", e); }); + .map_err(|e| { + error!("Unable to build index for segment: {:?}", e); + }); }); let index: &'a _ = unsafe { &*self.index.get() }; match *index { @@ -433,9 +450,15 @@ impl Segment { } } - fn stts(buf: &[u8], lens: SegmentLengths) -> &[u8] { &buf[.. lens.stts] } - fn stsz(buf: &[u8], lens: SegmentLengths) -> &[u8] { &buf[lens.stts .. lens.stts + lens.stsz] } - fn stss(buf: &[u8], lens: SegmentLengths) -> &[u8] { &buf[lens.stts + lens.stsz ..] } + fn stts(buf: &[u8], lens: SegmentLengths) -> &[u8] { + &buf[..lens.stts] + } + fn stsz(buf: &[u8], lens: SegmentLengths) -> &[u8] { + &buf[lens.stts..lens.stts + lens.stsz] + } + fn stss(buf: &[u8], lens: SegmentLengths) -> &[u8] { + &buf[lens.stts + lens.stsz..] + } fn build_index(&self, playback: &db::RecordingPlayback) -> Result, failure::Error> { let s = &self.s; @@ -455,12 +478,17 @@ impl Segment { let mut last_start_and_dur = None; s.foreach(playback, |it| { last_start_and_dur = Some((it.start_90k, it.duration_90k)); - BigEndian::write_u32(&mut stts[8*frame .. 8*frame+4], 1); - BigEndian::write_u32(&mut stts[8*frame+4 .. 8*frame+8], it.duration_90k as u32); - BigEndian::write_u32(&mut stsz[4*frame .. 4*frame+4], it.bytes as u32); + BigEndian::write_u32(&mut stts[8 * frame..8 * frame + 4], 1); + BigEndian::write_u32( + &mut stts[8 * frame + 4..8 * frame + 8], + it.duration_90k as u32, + ); + BigEndian::write_u32(&mut stsz[4 * frame..4 * frame + 4], it.bytes as u32); if it.is_key() { - BigEndian::write_u32(&mut stss[4*key_frame .. 4*key_frame+4], - self.first_frame_num + (frame as u32)); + BigEndian::write_u32( + &mut stss[4 * key_frame..4 * key_frame + 4], + self.first_frame_num + (frame as u32), + ); key_frame += 1; } frame += 1; @@ -472,7 +500,7 @@ impl Segment { // iteration. if let Some((last_start, dur)) = last_start_and_dur { let min = cmp::min(self.rel_media_range_90k.end - last_start, dur); - BigEndian::write_u32(&mut stts[8*frame-4 ..], u32::try_from(min).unwrap()); + BigEndian::write_u32(&mut stts[8 * frame - 4..], u32::try_from(min).unwrap()); } } @@ -480,14 +508,22 @@ impl Segment { } fn truns_len(&self) -> usize { - self.s.key_frames as usize * (mem::size_of::() * 6) + - self.s.frames as usize * (mem::size_of::() * 2) + - if self.s.starts_with_nonkey() { mem::size_of::() * 5 } else { 0 } + self.s.key_frames as usize * (mem::size_of::() * 6) + + self.s.frames as usize * (mem::size_of::() * 2) + + if self.s.starts_with_nonkey() { + mem::size_of::() * 5 + } else { + 0 + } } // TrackRunBox / trun (8.8.8). - fn truns(&self, playback: &db::RecordingPlayback, initial_pos: u64, len: usize) - -> Result, failure::Error> { + fn truns( + &self, + playback: &db::RecordingPlayback, + initial_pos: u64, + len: usize, + ) -> Result, failure::Error> { let mut v = Vec::with_capacity(len); struct RunInfo { @@ -499,87 +535,113 @@ impl Segment { } let mut run_info: Option = None; let mut data_pos = initial_pos; - self.s.foreach(playback, |it| { - let is_key = it.is_key(); - if is_key { - if let Some(r) = run_info.take() { - // Finish a non-terminal run. - let p = v.len(); - BigEndian::write_u32(&mut v[r.box_len_pos .. r.box_len_pos + 4], - (p - r.box_len_pos) as u32); - BigEndian::write_u32(&mut v[r.sample_count_pos .. r.sample_count_pos + 4], - r.count); + self.s + .foreach(playback, |it| { + let is_key = it.is_key(); + if is_key { + if let Some(r) = run_info.take() { + // Finish a non-terminal run. + let p = v.len(); + BigEndian::write_u32( + &mut v[r.box_len_pos..r.box_len_pos + 4], + (p - r.box_len_pos) as u32, + ); + BigEndian::write_u32( + &mut v[r.sample_count_pos..r.sample_count_pos + 4], + r.count, + ); + } } - } - let mut r = match run_info.take() { - None => { - let box_len_pos = v.len(); - v.extend_from_slice(&[ - 0x00, 0x00, 0x00, 0x00, // placeholder for size - b't', b'r', b'u', b'n', - - // version 0, tr_flags: - // 0x000001 data-offset-present - // 0x000004 first-sample-flags-present - // 0x000100 sample-duration-present - // 0x000200 sample-size-present - 0x00, 0x00, 0x03, 0x01 | if is_key { 0x04 } else { 0 }, + let mut r = match run_info.take() { + None => { + let box_len_pos = v.len(); + v.extend_from_slice(&[ + 0x00, + 0x00, + 0x00, + 0x00, // placeholder for size + b't', + b'r', + b'u', + b'n', + // version 0, tr_flags: + // 0x000001 data-offset-present + // 0x000004 first-sample-flags-present + // 0x000100 sample-duration-present + // 0x000200 sample-size-present + 0x00, + 0x00, + 0x03, + 0x01 | if is_key { 0x04 } else { 0 }, ]); - let sample_count_pos = v.len(); - v.write_u32::(0)?; // placeholder for sample count - v.write_u32::(data_pos as u32)?; + let sample_count_pos = v.len(); + v.write_u32::(0)?; // placeholder for sample count + v.write_u32::(data_pos as u32)?; - if is_key { - // first_sample_flags. See trex (8.8.3.1). - v.write_u32::( - // As defined by the Independent and Disposable Samples Box - // (sdp, 8.6.4). - (2 << 26) | // is_leading: this sample is not a leading sample + if is_key { + // first_sample_flags. See trex (8.8.3.1). + v.write_u32::( + // As defined by the Independent and Disposable Samples Box + // (sdp, 8.6.4). + (2 << 26) | // is_leading: this sample is not a leading sample (2 << 24) | // sample_depends_on: this sample does not depend on others (1 << 22) | // sample_is_depend_on: others may depend on this one (2 << 20) | // sample_has_redundancy: no redundant coding // As defined by the sample padding bits (padb, 8.7.6). (0 << 17) | // no padding (0 << 16) | // sample_is_non_sync_sample=0 - 0)?; // TODO: sample_degradation_priority + 0, + )?; // TODO: sample_degradation_priority + } + RunInfo { + box_len_pos, + sample_count_pos, + count: 0, + last_start: 0, + last_dur: 0, + } } - RunInfo { - box_len_pos, - sample_count_pos, - count: 0, - last_start: 0, - last_dur: 0, - } - }, - Some(r) => r, - }; - r.count += 1; - r.last_start = it.start_90k; - r.last_dur = it.duration_90k; - v.write_u32::(it.duration_90k as u32)?; - v.write_u32::(it.bytes as u32)?; - data_pos += it.bytes as u64; - run_info = Some(r); - Ok(()) - }).err_kind(ErrorKind::Internal)?; + Some(r) => r, + }; + r.count += 1; + r.last_start = it.start_90k; + r.last_dur = it.duration_90k; + v.write_u32::(it.duration_90k as u32)?; + v.write_u32::(it.bytes as u32)?; + data_pos += it.bytes as u64; + run_info = Some(r); + Ok(()) + }) + .err_kind(ErrorKind::Internal)?; if let Some(r) = run_info.take() { // Finish the run as in the non-terminal case above. let p = v.len(); - BigEndian::write_u32(&mut v[r.box_len_pos .. r.box_len_pos + 4], - (p - r.box_len_pos) as u32); - BigEndian::write_u32(&mut v[r.sample_count_pos .. r.sample_count_pos + 4], r.count); + BigEndian::write_u32( + &mut v[r.box_len_pos..r.box_len_pos + 4], + (p - r.box_len_pos) as u32, + ); + BigEndian::write_u32(&mut v[r.sample_count_pos..r.sample_count_pos + 4], r.count); // One more thing to do in the terminal case: fix up the final frame's duration. // Doing this after the fact is more efficient than having a condition on every // iteration. BigEndian::write_u32( - &mut v[p-8 .. p-4], - u32::try_from(cmp::min(self.rel_media_range_90k.end - r.last_start, r.last_dur)) - .unwrap()); - + &mut v[p - 8..p - 4], + u32::try_from(cmp::min( + self.rel_media_range_90k.end - r.last_start, + r.last_dur, + )) + .unwrap(), + ); } if len != v.len() { - bail_t!(Internal, "truns on {:?} expected len {} got len {}", self, len, v.len()); + bail_t!( + Internal, + "truns on {:?} expected len {} got len {}", + self, + len, + v.len() + ); } Ok(v) } @@ -631,24 +693,30 @@ struct Slice(u64); #[derive(Copy, Clone, Debug)] #[repr(u8)] enum SliceType { - Static = 0, // param is index into STATIC_BYTESTRINGS - Buf = 1, // param is index into m.buf - VideoSampleEntry = 2, // param is index into m.video_sample_entries - Stts = 3, // param is index into m.segments - Stsz = 4, // param is index into m.segments - Stss = 5, // param is index into m.segments - Co64 = 6, // param is unused - VideoSampleData = 7, // param is index into m.segments - SubtitleSampleData = 8, // param is index into m.segments - Truns = 9, // param is index into m.segments + Static = 0, // param is index into STATIC_BYTESTRINGS + Buf = 1, // param is index into m.buf + VideoSampleEntry = 2, // param is index into m.video_sample_entries + Stts = 3, // param is index into m.segments + Stsz = 4, // param is index into m.segments + Stss = 5, // param is index into m.segments + Co64 = 6, // param is unused + VideoSampleData = 7, // param is index into m.segments + SubtitleSampleData = 8, // param is index into m.segments + Truns = 9, // param is index into m.segments - // There must be no value > 15, as this is packed into 4 bits in Slice. + // There must be no value > 15, as this is packed into 4 bits in Slice. } impl Slice { fn new(end: u64, t: SliceType, p: usize) -> Result { - if end >= (1<<40) || p >= (1<<20) { - bail_t!(InvalidArgument, "end={} p={} too large for {:?} Slice", end, p, t); + if end >= (1 << 40) || p >= (1 << 20) { + bail_t!( + InvalidArgument, + "end={} p={} too large for {:?} Slice", + end, + p, + t + ); } Ok(Slice(end | ((t as u64) << 40) | ((p as u64) << 44))) @@ -659,46 +727,56 @@ impl Slice { // in Slice::new. unsafe { ::std::mem::transmute(((self.0 >> 40) & 0xF) as u8) } } - fn p(&self) -> usize { (self.0 >> 44) as usize } + fn p(&self) -> usize { + (self.0 >> 44) as usize + } fn wrap_index(&self, mp4: &File, r: Range, len: u64, f: &F) -> Result - where F: Fn(&[u8], SegmentLengths) -> &[u8] { + where + F: Fn(&[u8], SegmentLengths) -> &[u8], + { let mp4 = ARefss::new(mp4.0.clone()); - let r = r.start as usize .. r.end as usize; + let r = r.start as usize..r.end as usize; let p = self.p(); - Ok(mp4.try_map(|mp4| { - let i = mp4.segments[p].get_index(&mp4.db, f)?; - if u64::try_from(i.len()).unwrap() != len { - bail_t!(Internal, "expected len {} got {}", len, i.len()); - } - Ok::<_, Error>(&i[r]) - })?.into()) + Ok(mp4 + .try_map(|mp4| { + let i = mp4.segments[p].get_index(&mp4.db, f)?; + if u64::try_from(i.len()).unwrap() != len { + bail_t!(Internal, "expected len {} got {}", len, i.len()); + } + Ok::<_, Error>(&i[r]) + })? + .into()) } fn wrap_truns(&self, mp4: &File, r: Range, len: usize) -> Result { let s = &mp4.0.segments[self.p()]; let mut pos = mp4.0.initial_sample_byte_pos; - for ps in &mp4.0.segments[0 .. self.p()] { + for ps in &mp4.0.segments[0..self.p()] { let r = ps.s.sample_file_range(); pos += r.end - r.start; } - let truns = - mp4.0.db.lock() - .with_recording_playback(s.s.id, &mut |playback| s.truns(playback, pos, len)) - .err_kind(ErrorKind::Unknown)?; + let truns = mp4 + .0 + .db + .lock() + .with_recording_playback(s.s.id, &mut |playback| s.truns(playback, pos, len)) + .err_kind(ErrorKind::Unknown)?; let truns = ARefss::new(truns); - Ok(truns.map(|t| &t[r.start as usize .. r.end as usize]).into()) + Ok(truns.map(|t| &t[r.start as usize..r.end as usize]).into()) } fn wrap_video_sample_entry(&self, f: &File, r: Range, len: u64) -> Result { let mp4 = ARefss::new(f.0.clone()); - Ok(mp4.try_map(|mp4| { - let data = &mp4.video_sample_entries[self.p()].data; - if u64::try_from(data.len()).unwrap() != len { - bail_t!(Internal, "expected len {} got len {}", len, data.len()); - } - Ok::<_, Error>(&data[r.start as usize .. r.end as usize]) - })?.into()) + Ok(mp4 + .try_map(|mp4| { + let data = &mp4.video_sample_entries[self.p()].data; + if u64::try_from(data.len()).unwrap() != len { + bail_t!(Internal, "expected len {} got len {}", len, data.len()); + } + Ok::<_, Error>(&data[r.start as usize..r.end as usize]) + })? + .into()) } } @@ -706,25 +784,39 @@ impl slices::Slice for Slice { type Ctx = File; type Chunk = Chunk; - fn end(&self) -> u64 { return self.0 & 0xFF_FF_FF_FF_FF } - fn get_range(&self, f: &File, range: Range, len: u64) - -> Box> + Send + Sync> { + fn end(&self) -> u64 { + return self.0 & 0xFF_FF_FF_FF_FF; + } + fn get_range( + &self, + f: &File, + range: Range, + len: u64, + ) -> Box> + Send + Sync> { trace!("getting mp4 slice {:?}'s range {:?} / {}", self, range, len); let p = self.p(); let res = match self.t() { SliceType::Static => { let s = STATIC_BYTESTRINGS[p]; if u64::try_from(s.len()).unwrap() != len { - Err(format_err_t!(Internal, "expected len {} got len {}", len, s.len())) + Err(format_err_t!( + Internal, + "expected len {} got len {}", + len, + s.len() + )) } else { - let part = &s[range.start as usize .. range.end as usize]; + let part = &s[range.start as usize..range.end as usize]; Ok(part.into()) } - }, + } SliceType::Buf => { let r = ARefss::new(f.0.clone()); - Ok(r.map(|f| &f.buf[p+range.start as usize .. p+range.end as usize]).into()) - }, + Ok( + r.map(|f| &f.buf[p + range.start as usize..p + range.end as usize]) + .into(), + ) + } SliceType::VideoSampleEntry => self.wrap_video_sample_entry(f, range.clone(), len), SliceType::Stts => self.wrap_index(f, range.clone(), len, &Segment::stts), SliceType::Stsz => self.wrap_index(f, range.clone(), len, &Segment::stsz), @@ -734,20 +826,25 @@ impl slices::Slice for Slice { SliceType::SubtitleSampleData => f.0.get_subtitle_sample_data(p, range.clone(), len), SliceType::Truns => self.wrap_truns(f, range.clone(), len as usize), }; - Box::new(stream::once(futures::future::ready(res - .map_err(|e| wrap_error(e)) - .and_then(move |c| { + Box::new(stream::once(futures::future::ready( + res.map_err(|e| wrap_error(e)).and_then(move |c| { if c.remaining() != (range.end - range.start) as usize { return Err(wrap_error(format_err_t!( Internal, "Error producing {:?}: range {:?} produced incorrect len {}.", - self, range, c.remaining()))); + self, + range, + c.remaining() + ))); } Ok(c) - })))) + }), + ))) } - fn get_slices(ctx: &File) -> &Slices { &ctx.0.slices } + fn get_slices(ctx: &File) -> &Slices { + &ctx.0.slices + } } impl fmt::Debug for Slice { @@ -759,23 +856,29 @@ impl fmt::Debug for Slice { /// Converts from seconds since Unix epoch (1970-01-01 00:00:00 UTC) to seconds since /// ISO-14496 epoch (1904-01-01 00:00:00 UTC). -fn to_iso14496_timestamp(unix_secs: i64) -> u32 { unix_secs as u32 + 24107 * 86400 } +fn to_iso14496_timestamp(unix_secs: i64) -> u32 { + unix_secs as u32 + 24107 * 86400 +} /// Writes a box length for everything appended in the supplied scope. /// Used only within FileBuilder::build (and methods it calls internally). macro_rules! write_length { ($_self:ident, $b:block) => {{ let len_pos = $_self.body.buf.len(); - let len_start = $_self.body.slices.len() + $_self.body.buf.len() as u64 - - $_self.body.unflushed_buf_pos as u64; - $_self.body.append_u32(0); // placeholder. - { $b; } - let len_end = $_self.body.slices.len() + $_self.body.buf.len() as u64 - - $_self.body.unflushed_buf_pos as u64; - BigEndian::write_u32(&mut $_self.body.buf[len_pos .. len_pos + 4], - (len_end - len_start) as u32); + let len_start = $_self.body.slices.len() + $_self.body.buf.len() as u64 + - $_self.body.unflushed_buf_pos as u64; + $_self.body.append_u32(0); // placeholder. + { + $b; + } + let len_end = $_self.body.slices.len() + $_self.body.buf.len() as u64 + - $_self.body.unflushed_buf_pos as u64; + BigEndian::write_u32( + &mut $_self.body.buf[len_pos..len_pos + 4], + (len_end - len_start) as u32, + ); Ok::<_, Error>(()) - }} + }}; } #[derive(Copy, Clone, Debug, PartialEq, Eq)] @@ -794,7 +897,7 @@ impl FileBuilder { media_duration_90k: 0, num_subtitle_samples: 0, subtitle_co64_pos: None, - body: BodyState{ + body: BodyState { slices: Slices::new(), buf: Vec::new(), unflushed_buf_pos: 0, @@ -813,7 +916,10 @@ impl FileBuilder { // There's no support today for timestamp truns or for timestamps without edit lists. // The latter would invalidate the code's assumption that desired timespan == actual // timespan in the timestamp track. - bail_t!(InvalidArgument, "timestamp subtitles aren't supported on media segments"); + bail_t!( + InvalidArgument, + "timestamp subtitles aren't supported on media segments" + ); } self.include_timestamp_subtitle_track = b; Ok(()) @@ -831,42 +937,67 @@ impl FileBuilder { /// Appends a segment for (a subset of) the given recording. /// `rel_media_range_90k` is the media time range within the recording. /// Eg `0 .. row.media_duration_90k` means the full recording. - pub fn append(&mut self, db: &db::LockedDatabase, row: db::ListRecordingsRow, - rel_media_range_90k: Range, start_at_key: bool) -> Result<(), Error> { + pub fn append( + &mut self, + db: &db::LockedDatabase, + row: db::ListRecordingsRow, + rel_media_range_90k: Range, + start_at_key: bool, + ) -> Result<(), Error> { if let Some(prev) = self.segments.last() { if prev.s.have_trailing_zero() { - bail_t!(InvalidArgument, - "unable to append recording {} after recording {} with trailing zero", - row.id, prev.s.id); + bail_t!( + InvalidArgument, + "unable to append recording {} after recording {} with trailing zero", + row.id, + prev.s.id + ); } } else { // Include the current run in this count here, as we're not propagating the // run_offset_id further. - self.prev_media_duration_and_cur_runs = row.prev_media_duration_and_runs + self.prev_media_duration_and_cur_runs = row + .prev_media_duration_and_runs .map(|(d, r)| (d, r + if row.open_id == 0 { 1 } else { 0 })); } - let s = Segment::new(db, &row, rel_media_range_90k, self.next_frame_num, start_at_key)?; + let s = Segment::new( + db, + &row, + rel_media_range_90k, + self.next_frame_num, + start_at_key, + )?; self.next_frame_num += s.s.frames as u32; self.segments.push(s); - if !self.video_sample_entries.iter().any(|e| e.id == row.video_sample_entry_id) { - let vse = db.video_sample_entries_by_id().get(&row.video_sample_entry_id).unwrap(); + if !self + .video_sample_entries + .iter() + .any(|e| e.id == row.video_sample_entry_id) + { + let vse = db + .video_sample_entries_by_id() + .get(&row.video_sample_entry_id) + .unwrap(); self.video_sample_entries.push(vse.clone()); } Ok(()) } pub fn set_filename(&mut self, filename: &str) -> Result<(), Error> { - self.content_disposition = - Some(HeaderValue::try_from(format!("attachment; filename=\"{}\"", filename)) - .err_kind(ErrorKind::InvalidArgument)?); + self.content_disposition = Some( + HeaderValue::try_from(format!("attachment; filename=\"{}\"", filename)) + .err_kind(ErrorKind::InvalidArgument)?, + ); Ok(()) } /// Builds the `File`, consuming the builder. - pub fn build(mut self, db: Arc, - dirs_by_stream_id: Arc<::fnv::FnvHashMap>>) - -> Result { + pub fn build( + mut self, + db: Arc, + dirs_by_stream_id: Arc<::fnv::FnvHashMap>>, + ) -> Result { let mut max_end = None; let mut etag = blake3::Hasher::new(); etag.update(&FORMAT_VERSION[..]); @@ -878,9 +1009,13 @@ impl FileBuilder { etag.update(cd.as_bytes()); } match self.type_ { - Type::Normal => {}, - Type::InitSegment => { etag.update(b":init:"); }, - Type::MediaSegment => { etag.update(b":media:"); }, + Type::Normal => {} + Type::InitSegment => { + etag.update(b":init:"); + } + Type::MediaSegment => { + etag.update(b":media:"); + } }; for s in &mut self.segments { let md = &s.rel_media_range_90k; @@ -892,9 +1027,8 @@ impl FileBuilder { _ => md.start, }; self.media_duration_90k += u64::try_from(md.end - start).unwrap(); - let wall = - s.recording_start + recording::Duration(i64::from(s.wall(md.start))) .. - s.recording_start + recording::Duration(i64::from(s.wall(md.end))); + let wall = s.recording_start + recording::Duration(i64::from(s.wall(md.start))) + ..s.recording_start + recording::Duration(i64::from(s.wall(md.end))); max_end = match max_end { None => Some(wall.end), Some(v) => Some(cmp::max(v, wall.end)), @@ -912,11 +1046,21 @@ impl FileBuilder { // Update the etag to reflect this segment. let mut data = [0_u8; 28]; let mut cursor = io::Cursor::new(&mut data[..]); - cursor.write_i64::(s.s.id.0).err_kind(ErrorKind::Internal)?; - cursor.write_i64::(s.recording_start.0).err_kind(ErrorKind::Internal)?; - cursor.write_u32::(s.s.open_id).err_kind(ErrorKind::Internal)?; - cursor.write_i32::(md.start).err_kind(ErrorKind::Internal)?; - cursor.write_i32::(md.end).err_kind(ErrorKind::Internal)?; + cursor + .write_i64::(s.s.id.0) + .err_kind(ErrorKind::Internal)?; + cursor + .write_i64::(s.recording_start.0) + .err_kind(ErrorKind::Internal)?; + cursor + .write_u32::(s.s.open_id) + .err_kind(ErrorKind::Internal)?; + cursor + .write_i32::(md.start) + .err_kind(ErrorKind::Internal)?; + cursor + .write_i32::(md.end) + .err_kind(ErrorKind::Internal)?; etag.update(cursor.into_inner()); } let max_end = match max_end { @@ -939,42 +1083,59 @@ impl FileBuilder { // If the segment is > 4 GiB, the 32-bit trun data offsets are untrustworthy. // We'd need multiple moof+mdat sequences to support large media segments properly. if self.body.slices.len() > u32::max_value() as u64 { - bail_t!(InvalidArgument, - "media segment has length {}, greater than allowed 4 GiB", - self.body.slices.len()); + bail_t!( + InvalidArgument, + "media segment has length {}, greater than allowed 4 GiB", + self.body.slices.len() + ); } p - }, + } Type::InitSegment => { - self.body.append_static(StaticBytestring::InitSegmentFtypBox)?; + self.body + .append_static(StaticBytestring::InitSegmentFtypBox)?; self.append_moov(creation_ts)?; self.body.flush_buf()?; 0 - }, + } Type::Normal => { self.body.append_static(StaticBytestring::NormalFtypBox)?; self.append_moov(creation_ts)?; self.append_mdat()? - }, + } }; if est_slices < self.body.slices.num() { - warn!("Estimated {} slices; actually were {} slices", est_slices, - self.body.slices.num()); + warn!( + "Estimated {} slices; actually were {} slices", + est_slices, + self.body.slices.num() + ); } else { - debug!("Estimated {} slices; actually were {} slices", est_slices, - self.body.slices.num()); + debug!( + "Estimated {} slices; actually were {} slices", + est_slices, + self.body.slices.num() + ); } if EST_BUF_LEN < self.body.buf.len() { - warn!("Estimated {} buf bytes; actually were {}", EST_BUF_LEN, self.body.buf.len()); + warn!( + "Estimated {} buf bytes; actually were {}", + EST_BUF_LEN, + self.body.buf.len() + ); } else { - debug!("Estimated {} buf bytes; actually were {}", EST_BUF_LEN, self.body.buf.len()); + debug!( + "Estimated {} buf bytes; actually were {}", + EST_BUF_LEN, + self.body.buf.len() + ); } debug!("segments: {:#?}", self.segments); debug!("slices: {:?}", self.body.slices); - let last_modified = ::std::time::UNIX_EPOCH + - ::std::time::Duration::from_secs(max_end as u64); + let last_modified = + ::std::time::UNIX_EPOCH + ::std::time::Duration::from_secs(max_end as u64); let etag = etag.finalize(); Ok(File(Arc::new(FileInner { db, @@ -986,7 +1147,7 @@ impl FileBuilder { initial_sample_byte_pos, last_modified, etag: HeaderValue::try_from(format!("\"{}\"", etag.to_hex().as_str())) - .expect("hex string should be valid UTF-8"), + .expect("hex string should be valid UTF-8"), content_disposition: self.content_disposition, prev_media_duration_and_cur_runs: self.prev_media_duration_and_cur_runs, type_: self.type_, @@ -997,27 +1158,34 @@ impl FileBuilder { // Write the mdat header. Use the large format to support files over 2^32-1 bytes long. // Write zeroes for the length as a placeholder; fill it in after it's known. // It'd be nice to use the until-EOF form, but QuickTime Player doesn't support it. - self.body.buf.extend_from_slice(b"\x00\x00\x00\x01mdat\x00\x00\x00\x00\x00\x00\x00\x00"); + self.body + .buf + .extend_from_slice(b"\x00\x00\x00\x01mdat\x00\x00\x00\x00\x00\x00\x00\x00"); let mdat_len_pos = self.body.buf.len() - 8; self.body.flush_buf()?; let initial_sample_byte_pos = self.body.slices.len(); for (i, s) in self.segments.iter().enumerate() { let r = s.s.sample_file_range(); - self.body.append_slice(r.end - r.start, SliceType::VideoSampleData, i)?; + self.body + .append_slice(r.end - r.start, SliceType::VideoSampleData, i)?; } if let Some(p) = self.subtitle_co64_pos { - BigEndian::write_u64(&mut self.body.buf[p .. p + 8], self.body.slices.len()); + BigEndian::write_u64(&mut self.body.buf[p..p + 8], self.body.slices.len()); for (i, s) in self.segments.iter().enumerate() { self.body.append_slice( - s.num_subtitle_samples as u64 * - (mem::size_of::() + SUBTITLE_LENGTH) as u64, - SliceType::SubtitleSampleData, i)?; + s.num_subtitle_samples as u64 + * (mem::size_of::() + SUBTITLE_LENGTH) as u64, + SliceType::SubtitleSampleData, + i, + )?; } } // Fill in the length left as a placeholder above. Note the 16 here is the length // of the mdat header. - BigEndian::write_u64(&mut self.body.buf[mdat_len_pos .. mdat_len_pos + 8], - 16 + self.body.slices.len() - initial_sample_byte_pos); + BigEndian::write_u64( + &mut self.body.buf[mdat_len_pos..mdat_len_pos + 8], + 16 + self.body.slices.len() - initial_sample_byte_pos, + ); Ok(initial_sample_byte_pos) } @@ -1045,20 +1213,20 @@ impl FileBuilder { // track. write_length!(self, { self.body.buf.extend_from_slice(&[ - b't', b'r', b'e', b'x', - 0x00, 0x00, 0x00, 0x00, // version + flags - 0x00, 0x00, 0x00, 0x01, // track_id - 0x00, 0x00, 0x00, 0x01, // default_sample_description_index - 0x00, 0x00, 0x00, 0x00, // default_sample_duration - 0x00, 0x00, 0x00, 0x00, // default_sample_size - 0x09, 0x21, 0x00, 0x00, // default_sample_flags (non sync): - // is_leading: not a leading sample - // sample_depends_on: does depend on others - // sample_is_depend_on: unknown - // sample_has_redundancy: no - // no padding - // sample_is_non_sync_sample: 1 - // sample_degradation_priority: 0 + b't', b'r', b'e', b'x', 0x00, 0x00, 0x00, 0x00, // version + flags + 0x00, 0x00, 0x00, 0x01, // track_id + 0x00, 0x00, 0x00, 0x01, // default_sample_description_index + 0x00, 0x00, 0x00, 0x00, // default_sample_duration + 0x00, 0x00, 0x00, 0x00, // default_sample_size + 0x09, 0x21, 0x00, + 0x00, // default_sample_flags (non sync): + // is_leading: not a leading sample + // sample_depends_on: does depend on others + // sample_is_depend_on: unknown + // sample_has_redundancy: no + // no padding + // sample_is_non_sync_sample: 1 + // sample_degradation_priority: 0 ]); })?; }) @@ -1072,7 +1240,7 @@ impl FileBuilder { // MovieFragmentHeaderBox (ISO/IEC 14496-12 section 8.8.5). write_length!(self, { self.body.buf.extend_from_slice(b"mfhd\x00\x00\x00\x00"); - self.body.append_u32(1); // sequence_number + self.body.append_u32(1); // sequence_number })?; // TrackFragmentBox (ISO/IEC 14496-12 section 8.8.6). @@ -1082,9 +1250,9 @@ impl FileBuilder { // TrackFragmentHeaderBox, tfhd (ISO/IEC 14496-12 section 8.8.7). write_length!(self, { self.body.buf.extend_from_slice(&[ - b't', b'f', b'h', b'd', - 0x00, 0x02, 0x00, 0x00, // version + flags (default-base-is-moof) - 0x00, 0x00, 0x00, 0x01, // track_id = 1 + b't', b'f', b'h', b'd', 0x00, 0x02, 0x00, + 0x00, // version + flags (default-base-is-moof) + 0x00, 0x00, 0x00, 0x01, // track_id = 1 ]); })?; self.append_truns()?; @@ -1092,9 +1260,8 @@ impl FileBuilder { // `TrackFragmentBaseMediaDecodeTimeBox` (ISO/IEC 14496-12 section 8.8.12). write_length!(self, { self.body.buf.extend_from_slice(&[ - b't', b'f', b'd', b't', - 0x00, 0x00, 0x00, 0x00, // version + flags - 0x00, 0x00, 0x00, 0x00, // TODO: baseMediaDecodeTime + b't', b'f', b'd', b't', 0x00, 0x00, 0x00, 0x00, // version + flags + 0x00, 0x00, 0x00, 0x00, // TODO: baseMediaDecodeTime ]); })?; })?; @@ -1104,7 +1271,8 @@ impl FileBuilder { fn append_truns(&mut self) -> Result<(), Error> { self.body.flush_buf()?; for (i, s) in self.segments.iter().enumerate() { - self.body.append_slice(s.truns_len() as u64, SliceType::Truns, i)?; + self.body + .append_slice(s.truns_len() as u64, SliceType::Truns, i)?; } Ok(()) } @@ -1119,7 +1287,11 @@ impl FileBuilder { let d = self.media_duration_90k; self.body.append_u64(d); self.body.append_static(StaticBytestring::MvhdJunk)?; - let next_track_id = if self.include_timestamp_subtitle_track { 3 } else { 2 }; + let next_track_id = if self.include_timestamp_subtitle_track { + 3 + } else { + 2 + }; self.body.append_u32(next_track_id); }) } @@ -1150,17 +1322,19 @@ impl FileBuilder { self.body.buf.extend_from_slice(b"tkhd\x00\x00\x00\x07"); self.body.append_u32(creation_ts); self.body.append_u32(creation_ts); - self.body.append_u32(1); // track_id - self.body.append_u32(0); // reserved + self.body.append_u32(1); // track_id + self.body.append_u32(0); // reserved self.body.append_u32(self.media_duration_90k as u32); self.body.append_static(StaticBytestring::TkhdJunk)?; - let (width, height) = self.video_sample_entries.iter().fold(None, |m, e| { - match m { + let (width, height) = self + .video_sample_entries + .iter() + .fold(None, |m, e| match m { None => Some((e.width, e.height)), Some((w, h)) => Some((cmp::max(w, e.width), cmp::max(h, e.height))), - } - }).ok_or_else(|| format_err_t!(InvalidArgument, "no video_sample_entries"))?; + }) + .ok_or_else(|| format_err_t!(InvalidArgument, "no video_sample_entries"))?; self.body.append_u32((width as u32) << 16); self.body.append_u32((height as u32) << 16); }) @@ -1173,12 +1347,12 @@ impl FileBuilder { self.body.buf.extend_from_slice(b"tkhd\x01\x00\x00\x07"); self.body.append_u64(creation_ts as u64); self.body.append_u64(creation_ts as u64); - self.body.append_u32(2); // track_id - self.body.append_u32(0); // reserved + self.body.append_u32(2); // track_id + self.body.append_u32(0); // reserved self.body.append_u64(self.media_duration_90k); self.body.append_static(StaticBytestring::TkhdJunk)?; - self.body.append_u32(0); // width, unused. - self.body.append_u32(0); // height, unused. + self.body.append_u32(0); // width, unused. + self.body.append_u32(0); // height, unused. }) } @@ -1219,7 +1393,7 @@ impl FileBuilder { } if flushed.is_empty() && unflushed.media_time == 0 { - return Ok(()); // use implicit one-to-one mapping. + return Ok(()); // use implicit one-to-one mapping. } flushed.push(unflushed); @@ -1271,7 +1445,7 @@ impl FileBuilder { self.body.append_u64(creation_ts as u64); self.body.append_u32(TIME_UNITS_PER_SEC as u32); self.body.append_u64(self.media_duration_90k); - self.body.append_u32(0x55c40000); // language=und + pre_defined + self.body.append_u32(0x55c40000); // language=und + pre_defined }) } @@ -1286,7 +1460,8 @@ impl FileBuilder { /// Appends a `MediaInformationBox` (ISO/IEC 14496-12 section 8.4.4) suitable for subtitles. fn append_subtitle_minf(&mut self) -> Result<(), Error> { write_length!(self, { - self.body.append_static(StaticBytestring::SubtitleMinfJunk)?; + self.body + .append_static(StaticBytestring::SubtitleMinfJunk)?; self.append_subtitle_stbl()?; }) } @@ -1307,7 +1482,8 @@ impl FileBuilder { /// Appends a `SampleTableBox` (ISO/IEC 14496-12 section 8.5.1) suitable for subtitles. fn append_subtitle_stbl(&mut self) -> Result<(), Error> { write_length!(self, { - self.body.append_static(StaticBytestring::SubtitleStblJunk)?; + self.body + .append_static(StaticBytestring::SubtitleStblJunk)?; self.append_subtitle_stts()?; self.append_subtitle_stsc()?; self.append_subtitle_stsz()?; @@ -1323,7 +1499,8 @@ impl FileBuilder { self.body.append_u32(n_entries); self.body.flush_buf()?; for (i, e) in self.video_sample_entries.iter().enumerate() { - self.body.append_slice(e.data.len() as u64, SliceType::VideoSampleEntry, i)?; + self.body + .append_slice(e.data.len() as u64, SliceType::VideoSampleEntry, i)?; } }) } @@ -1342,7 +1519,9 @@ impl FileBuilder { for (i, s) in self.segments.iter().enumerate() { self.body.append_slice( 2 * (mem::size_of::() as u64) * (s.s.frames as u64), - SliceType::Stts, i)?; + SliceType::Stts, + i, + )?; } } }) @@ -1354,7 +1533,7 @@ impl FileBuilder { self.body.buf.extend_from_slice(b"stts\x00\x00\x00\x00"); let entry_count_pos = self.body.buf.len(); - self.body.append_u32(0); // placeholder for entry_count + self.body.append_u32(0); // placeholder for entry_count let mut entry_count = 0; for s in &self.segments { @@ -1363,19 +1542,19 @@ impl FileBuilder { let mr = &s.rel_media_range_90k; let start = s.recording_start + recording::Duration(i64::from(s.wall(mr.start))); let end = s.recording_start + recording::Duration(i64::from(s.wall(mr.end))); - let start_next_sec = recording::Time( - start.0 + TIME_UNITS_PER_SEC - (start.0 % TIME_UNITS_PER_SEC)); + let start_next_sec = + recording::Time(start.0 + TIME_UNITS_PER_SEC - (start.0 % TIME_UNITS_PER_SEC)); if end <= start_next_sec { // Segment doesn't last past the next second. Just write one entry. entry_count += 1; self.body.append_u32(1); - self.body.append_u32(u32::try_from(mr.end - mr.start).unwrap()); + self.body + .append_u32(u32::try_from(mr.end - mr.start).unwrap()); } else { // The first subtitle lasts until the next second. // media_off is relative to the start of the desired range. - let mut media_off = - s.media(i32::try_from((start_next_sec - start).0).unwrap()); + let mut media_off = s.media(i32::try_from((start_next_sec - start).0).unwrap()); entry_count += 1; self.body.append_u32(1); self.body.append_u32(u32::try_from(media_off).unwrap()); @@ -1388,23 +1567,26 @@ impl FileBuilder { // second but it's not worth bloating the moov over 3 ms. let end_prev_sec = recording::Time(end.0 - (end.0 % TIME_UNITS_PER_SEC)); if start_next_sec < end_prev_sec { - let onesec_media_dur = - s.media(i32::try_from(TIME_UNITS_PER_SEC).unwrap()); + let onesec_media_dur = s.media(i32::try_from(TIME_UNITS_PER_SEC).unwrap()); let interior = (end_prev_sec - start_next_sec).0 / TIME_UNITS_PER_SEC; entry_count += 1; - self.body.append_u32(interior as u32); // count - self.body.append_u32(u32::try_from(onesec_media_dur).unwrap()); + self.body.append_u32(interior as u32); // count + self.body + .append_u32(u32::try_from(onesec_media_dur).unwrap()); media_off += onesec_media_dur * i32::try_from(interior).unwrap(); } // Then there's a final subtitle for the remaining fraction of a second. entry_count += 1; self.body.append_u32(1); - self.body.append_u32(u32::try_from(mr.end - mr.start - media_off).unwrap()); + self.body + .append_u32(u32::try_from(mr.end - mr.start - media_off).unwrap()); } } - BigEndian::write_u32(&mut self.body.buf[entry_count_pos .. entry_count_pos + 4], - entry_count); + BigEndian::write_u32( + &mut self.body.buf[entry_count_pos..entry_count_pos + 4], + entry_count, + ); }) } @@ -1418,8 +1600,11 @@ impl FileBuilder { self.body.append_u32(s.s.frames as u32); // Write sample_description_index. - let i = self.video_sample_entries.iter().position( - |e| e.id == s.s.video_sample_entry_id()).unwrap(); + let i = self + .video_sample_entries + .iter() + .position(|e| e.id == s.s.video_sample_entry_id()) + .unwrap(); self.body.append_u32((i + 1) as u32); } }) @@ -1428,8 +1613,9 @@ impl FileBuilder { /// Appends a `SampleToChunkBox` (ISO/IEC 14496-12 section 8.7.4) suitable for subtitles. fn append_subtitle_stsc(&mut self) -> Result<(), Error> { write_length!(self, { - self.body.buf.extend_from_slice( - b"stsc\x00\x00\x00\x00\x00\x00\x00\x01\x00\x00\x00\x01"); + self.body + .buf + .extend_from_slice(b"stsc\x00\x00\x00\x00\x00\x00\x00\x01\x00\x00\x00\x01"); self.body.append_u32(self.num_subtitle_samples as u32); self.body.append_u32(1); }) @@ -1438,7 +1624,9 @@ impl FileBuilder { /// Appends a `SampleSizeBox` (ISO/IEC 14496-12 section 8.7.3) suitable for video. fn append_video_stsz(&mut self) -> Result<(), Error> { write_length!(self, { - self.body.buf.extend_from_slice(b"stsz\x00\x00\x00\x00\x00\x00\x00\x00"); + self.body + .buf + .extend_from_slice(b"stsz\x00\x00\x00\x00\x00\x00\x00\x00"); let mut entry_count = 0; for s in &self.segments { entry_count += s.s.frames as u32; @@ -1448,7 +1636,10 @@ impl FileBuilder { self.body.flush_buf()?; for (i, s) in self.segments.iter().enumerate() { self.body.append_slice( - (mem::size_of::()) as u64 * (s.s.frames as u64), SliceType::Stsz, i)?; + (mem::size_of::()) as u64 * (s.s.frames as u64), + SliceType::Stsz, + i, + )?; } } }) @@ -1458,7 +1649,8 @@ impl FileBuilder { fn append_subtitle_stsz(&mut self) -> Result<(), Error> { write_length!(self, { self.body.buf.extend_from_slice(b"stsz\x00\x00\x00\x00"); - self.body.append_u32((mem::size_of::() + SUBTITLE_LENGTH) as u32); + self.body + .append_u32((mem::size_of::() + SUBTITLE_LENGTH) as u32); self.body.append_u32(self.num_subtitle_samples as u32); }) } @@ -1472,7 +1664,9 @@ impl FileBuilder { self.body.flush_buf()?; self.body.append_slice( (mem::size_of::()) as u64 * (self.segments.len() as u64), - SliceType::Co64, 0)?; + SliceType::Co64, + 0, + )?; } }) } @@ -1482,7 +1676,8 @@ impl FileBuilder { write_length!(self, { // Write a placeholder; the actual value will be filled in later. self.body.buf.extend_from_slice( - b"co64\x00\x00\x00\x00\x00\x00\x00\x01\x00\x00\x00\x00\x00\x00\x00\x00"); + b"co64\x00\x00\x00\x00\x00\x00\x00\x01\x00\x00\x00\x00\x00\x00\x00\x00", + ); self.subtitle_co64_pos = Some(self.body.buf.len() - 8); }) } @@ -1501,7 +1696,9 @@ impl FileBuilder { for (i, s) in self.segments.iter().enumerate() { self.body.append_slice( (mem::size_of::() as u64) * (s.s.key_frames as u64), - SliceType::Stss, i)?; + SliceType::Stss, + i, + )?; } } }) @@ -1510,11 +1707,15 @@ impl FileBuilder { impl BodyState { fn append_u32(&mut self, v: u32) { - self.buf.write_u32::(v).expect("Vec write shouldn't fail"); + self.buf + .write_u32::(v) + .expect("Vec write shouldn't fail"); } fn append_u64(&mut self, v: u64) { - self.buf.write_u64::(v).expect("Vec write shouldn't fail"); + self.buf + .write_u64::(v) + .expect("Vec write shouldn't fail"); } /// Flushes the buffer: appends a slice for everything written into the buffer so far, @@ -1532,7 +1733,9 @@ impl BodyState { fn append_slice(&mut self, len: u64, t: SliceType, p: usize) -> Result<(), Error> { let l = self.slices.len(); - self.slices.append(Slice::new(l + len, t, p)?).err_kind(ErrorKind::Internal) + self.slices + .append(Slice::new(l + len, t, p)?) + .err_kind(ErrorKind::Internal) } /// Appends a static bytestring, flushing the buffer if necessary. @@ -1563,11 +1766,14 @@ impl FileInner { let mut v = Vec::with_capacity(l as usize); let mut pos = self.initial_sample_byte_pos; for s in &self.segments { - v.write_u64::(pos).err_kind(ErrorKind::Internal)?; + v.write_u64::(pos) + .err_kind(ErrorKind::Internal)?; let r = s.s.sample_file_range(); pos += r.end - r.start; } - Ok(ARefss::new(v).map(|v| &v[r.start as usize .. r.end as usize]).into()) + Ok(ARefss::new(v) + .map(|v| &v[r.start as usize..r.end as usize]) + .into()) } /// Gets a `Chunk` of video sample data from disk. @@ -1581,17 +1787,20 @@ impl FileInner { /// happen because nothing should be touching Moonfire NVR's files but itself. fn get_video_sample_data(&self, i: usize, r: Range) -> Result { let s = &self.segments[i]; - let f = self.dirs_by_stream_id - .get(&s.s.id.stream()) - .ok_or_else(|| format_err_t!(NotFound, "{}: stream not found", s.s.id))? - .open_file(s.s.id).err_kind(ErrorKind::Unknown)?; + let f = self + .dirs_by_stream_id + .get(&s.s.id.stream()) + .ok_or_else(|| format_err_t!(NotFound, "{}: stream not found", s.s.id))? + .open_file(s.s.id) + .err_kind(ErrorKind::Unknown)?; let start = s.s.sample_file_range().start + r.start; let mmap = Box::new(unsafe { memmap::MmapOptions::new() .offset(start) .len((r.end - r.start) as usize) - .map(&f).err_kind(ErrorKind::Internal)? - }); + .map(&f) + .err_kind(ErrorKind::Internal)? + }); use core::ops::Deref; Ok(ARefss::new(mmap).map(|m| m.deref()).into()) } @@ -1599,24 +1808,32 @@ impl FileInner { fn get_subtitle_sample_data(&self, i: usize, r: Range, l: u64) -> Result { let s = &self.segments[i]; let md = &s.rel_media_range_90k; - let wd = s.wall(md.start) .. s.wall(md.end); + let wd = s.wall(md.start)..s.wall(md.end); let start_sec = (s.recording_start + recording::Duration(i64::from(wd.start))).unix_seconds(); - let end_sec = - (s.recording_start + recording::Duration(i64::from(wd.end) + TIME_UNITS_PER_SEC - 1)) - .unix_seconds(); + let end_sec = (s.recording_start + + recording::Duration(i64::from(wd.end) + TIME_UNITS_PER_SEC - 1)) + .unix_seconds(); let l = usize::try_from(l).unwrap(); let mut v = Vec::with_capacity(l); // TODO(slamb): is this right?!? might have an off-by-one here. - for ts in start_sec .. end_sec { - v.write_u16::(SUBTITLE_LENGTH as u16).expect("Vec write shouldn't fail"); - let tm = time::at(time::Timespec{sec: ts, nsec: 0}); - use std::io::Write; - write!(v, "{}", tm.strftime(SUBTITLE_TEMPLATE).err_kind(ErrorKind::Internal)?) + for ts in start_sec..end_sec { + v.write_u16::(SUBTITLE_LENGTH as u16) .expect("Vec write shouldn't fail"); + let tm = time::at(time::Timespec { sec: ts, nsec: 0 }); + use std::io::Write; + write!( + v, + "{}", + tm.strftime(SUBTITLE_TEMPLATE) + .err_kind(ErrorKind::Internal)? + ) + .expect("Vec write shouldn't fail"); } assert_eq!(l, v.len()); - Ok(ARefss::new(v).map(|v| &v[r.start as usize .. r.end as usize]).into()) + Ok(ARefss::new(v) + .map(|v| &v[r.start as usize..r.end as usize]) + .into()) } } @@ -1626,10 +1843,14 @@ pub struct File(Arc); impl File { pub async fn append_into_vec(self, v: &mut Vec) -> Result<(), Error> { use http_serve::Entity; - v.reserve(usize::try_from(self.len()) - .map_err(|_| format_err_t!(InvalidArgument, "{}-byte mp4 is too big to send over WebSockets!", - self.len()))?); - let mut b = std::pin::Pin::from(self.get_range(0 .. self.len())); + v.reserve(usize::try_from(self.len()).map_err(|_| { + format_err_t!( + InvalidArgument, + "{}-byte mp4 is too big to send over WebSockets!", + self.len() + ) + })?); + let mut b = std::pin::Pin::from(self.get_range(0..self.len())); loop { use futures::stream::StreamExt; match b.next().await { @@ -1643,7 +1864,7 @@ impl File { let len = c.len(); chunk.advance(len); } - }, + } None => return Ok(()), } } @@ -1667,8 +1888,10 @@ impl http_serve::Entity for File { mime.extend_from_slice(e.rfc6381_codec.as_bytes()); } mime.extend_from_slice(b"\""); - hdrs.insert(http::header::CONTENT_TYPE, - http::header::HeaderValue::from_maybe_shared(mime.freeze()).unwrap()); + hdrs.insert( + http::header::CONTENT_TYPE, + http::header::HeaderValue::from_maybe_shared(mime.freeze()).unwrap(), + ); if let Some(cd) = self.0.content_disposition.as_ref() { hdrs.insert(http::header::CONTENT_DISPOSITION, cd.clone()); @@ -1677,26 +1900,37 @@ impl http_serve::Entity for File { if let Some((d, r)) = self.0.prev_media_duration_and_cur_runs { hdrs.insert( "X-Prev-Media-Duration", - HeaderValue::try_from(d.0.to_string()).expect("ints are valid headers")); + HeaderValue::try_from(d.0.to_string()).expect("ints are valid headers"), + ); hdrs.insert( "X-Runs", - HeaderValue::try_from(r.to_string()).expect("ints are valid headers")); + HeaderValue::try_from(r.to_string()).expect("ints are valid headers"), + ); } if let Some(s) = self.0.segments.first() { let skip = s.rel_media_range_90k.start - s.s.actual_start_90k(); if skip > 0 { hdrs.insert( "X-Leading-Media-Duration", - HeaderValue::try_from(skip.to_string()).expect("ints are valid headers")); + HeaderValue::try_from(skip.to_string()).expect("ints are valid headers"), + ); } } } } - fn last_modified(&self) -> Option { Some(self.0.last_modified) } - fn etag(&self) -> Option { Some(self.0.etag.clone()) } - fn len(&self) -> u64 { self.0.slices.len() } - fn get_range(&self, range: Range) - -> Box> + Send + Sync> { + fn last_modified(&self) -> Option { + Some(self.0.last_modified) + } + fn etag(&self) -> Option { + Some(self.0.etag.clone()) + } + fn len(&self) -> u64 { + self.0.slices.len() + } + fn get_range( + &self, + range: Range, + ) -> Box> + Send + Sync> { self.0.slices.get_range(self, range) } } @@ -1724,53 +1958,57 @@ impl fmt::Debug for File { /// to verify the output is byte-for-byte as expected. #[cfg(test)] mod tests { + use super::*; + use crate::stream::{self, Opener, Stream}; use base::clock::RealClocks; use byteorder::{BigEndian, ByteOrder}; - use hyper::body::Buf; - use crate::stream::{self, Opener, Stream}; use db::recording::{self, TIME_UNITS_PER_SEC}; use db::testutil::{self, TestDb, TEST_STREAM_ID}; use db::writer; use futures::stream::TryStreamExt; - use log::info; use http_serve::{self, Entity}; + use hyper::body::Buf; + use log::info; use std::fs; use std::ops::Range; use std::path::Path; use std::pin::Pin; use std::str; - use super::*; async fn fill_slice(slice: &mut [u8], e: &E, start: u64) - where E::Error : ::std::fmt::Debug { + where + E::Error: ::std::fmt::Debug, + { let mut p = 0; - Pin::from(e.get_range(start .. start + slice.len() as u64)) - .try_for_each(|mut chunk| { - let len = chunk.remaining(); - chunk.copy_to_slice(&mut slice[p .. p + len]); - p += len; - futures::future::ok::<_, E::Error>(()) - }) - .await - .unwrap(); + Pin::from(e.get_range(start..start + slice.len() as u64)) + .try_for_each(|mut chunk| { + let len = chunk.remaining(); + chunk.copy_to_slice(&mut slice[p..p + len]); + p += len; + futures::future::ok::<_, E::Error>(()) + }) + .await + .unwrap(); } /// Returns the Blake3 digest of the given `Entity`. async fn digest(e: &E) -> blake3::Hash - where E::Error : ::std::fmt::Debug { - Pin::from(e.get_range(0 .. e.len())) - .try_fold(blake3::Hasher::new(), |mut hasher, mut chunk| { - while chunk.has_remaining() { - let c = chunk.chunk(); - hasher.update(c); - let len = c.len(); - chunk.advance(len); - } - futures::future::ok::<_, E::Error>(hasher) - }) - .await - .unwrap() - .finalize() + where + E::Error: ::std::fmt::Debug, + { + Pin::from(e.get_range(0..e.len())) + .try_fold(blake3::Hasher::new(), |mut hasher, mut chunk| { + while chunk.has_remaining() { + let c = chunk.chunk(); + hasher.update(c); + let len = c.len(); + chunk.advance(len); + } + futures::future::ok::<_, E::Error>(hasher) + }) + .await + .unwrap() + .finalize() } /// Information used within `BoxCursor` to describe a box on the stack. @@ -1790,7 +2028,7 @@ mod tests { impl BoxCursor { pub fn new(mp4: File) -> BoxCursor { - BoxCursor{ + BoxCursor { mp4: mp4, stack: Vec::new(), } @@ -1799,7 +2037,9 @@ mod tests { /// Pushes the box at the given position onto the stack (returning true), or returns /// false if pos == max. async fn internal_push(&mut self, pos: u64, max: u64) -> bool { - if pos == max { return false; } + if pos == max { + return false; + } let mut hdr = [0u8; 16]; fill_slice(&mut hdr[..8], &self.mp4, pos).await; let (len, hdr_len, boxtype_slice) = match BigEndian::read_u32(&hdr[..4]) { @@ -1807,15 +2047,22 @@ mod tests { 1 => { fill_slice(&mut hdr[8..], &self.mp4, pos + 8).await; (BigEndian::read_u64(&hdr[8..16]), 16, &hdr[4..8]) - }, + } l => (l as u64, 8, &hdr[4..8]), }; let mut boxtype = [0u8; 4]; assert!(pos + (hdr_len as u64) <= max); - assert!(pos + len <= max, "path={} pos={} len={} max={}", self.path(), pos, len, max); + assert!( + pos + len <= max, + "path={} pos={} len={} max={}", + self.path(), + pos, + len, + max + ); boxtype[..].copy_from_slice(boxtype_slice); - self.stack.push(Mp4Box{ - interior: pos + hdr_len as u64 .. pos + len, + self.stack.push(Mp4Box { + interior: pos + hdr_len as u64..pos + len, boxtype: boxtype, }); trace!("positioned at {}", self.path()); @@ -1839,16 +2086,23 @@ mod tests { str::from_utf8(&self.stack.last().expect("at root").boxtype[..]).unwrap() } - pub fn depth(&self) -> usize { self.stack.len() } + pub fn depth(&self) -> usize { + self.stack.len() + } /// Gets the specified byte range within the current box (excluding length and type). /// Must not be at EOF. pub async fn get(&self, start: u64, buf: &mut [u8]) { let interior = &self.stack.last().expect("at root").interior; - assert!(start + (buf.len() as u64) <= interior.end - interior.start, - "path={} start={} buf.len={} interior={:?}", - self.path(), start, buf.len(), interior); - fill_slice(buf, &self.mp4, start+interior.start).await; + assert!( + start + (buf.len() as u64) <= interior.end - interior.start, + "path={} start={} buf.len={} interior={:?}", + self.path(), + start, + buf.len(), + interior + ); + fill_slice(buf, &self.mp4, start + interior.start).await; } pub async fn get_all(&self) -> Vec { @@ -1877,8 +2131,15 @@ mod tests { /// Navigates to the next box after the current one, or up if the current one is last. pub async fn next(&mut self) -> bool { - let old = self.stack.pop().expect("positioned at root; there is no next"); - let max = self.stack.last().map(|b| b.interior.end).unwrap_or_else(|| self.mp4.len()); + let old = self + .stack + .pop() + .expect("positioned at root; there is no next"); + let max = self + .stack + .last() + .map(|b| b.interior.end) + .unwrap_or_else(|| self.mp4.len()); self.internal_push(old.interior.end, max).await } @@ -1896,14 +2157,22 @@ mod tests { } /// Moves up the stack. Must not be at root. - pub fn up(&mut self) { self.stack.pop(); } + pub fn up(&mut self) { + self.stack.pop(); + } /// Moves down the stack. Must be positioned on a box with children. pub async fn down(&mut self) { - let range = self.stack.last().map(|b| b.interior.clone()) - .unwrap_or_else(|| 0 .. self.mp4.len()); - assert!(self.internal_push(range.start, range.end).await, - "no children in {}", self.path()); + let range = self + .stack + .last() + .map(|b| b.interior.clone()) + .unwrap_or_else(|| 0..self.mp4.len()); + assert!( + self.internal_push(range.start, range.end).await, + "no children in {}", + self.path() + ); } } @@ -1929,12 +2198,16 @@ mod tests { // Let id_pos be the offset after the FullBox section of the track_id. let id_pos = match version[0] { - 0 => 8, // track_id follows 32-bit creation_time and modification_time - 1 => 16, // ...64-bit times... + 0 => 8, // track_id follows 32-bit creation_time and modification_time + 1 => 16, // ...64-bit times... v => panic!("unexpected tkhd version {}", v), }; let cur_track_id = cursor.get_u32(4 + id_pos).await; - trace!("found moov/trak/tkhd with id {}; want {}", cur_track_id, track_id); + trace!( + "found moov/trak/tkhd with id {}; want {}", + cur_track_id, + track_id + ); if cur_track_id == track_id { break; } @@ -1954,7 +2227,7 @@ mod tests { assert!(cursor.find(b"minf").await); cursor.down().await; assert!(cursor.find(b"stbl").await); - Track{ + Track { edts_cursor: edts_cursor, stbl_cursor: cursor, } @@ -1970,17 +2243,26 @@ mod tests { } fn copy_mp4_to_db(db: &TestDb) { - let mut input = - stream::FFMPEG.open(stream::Source::File("src/testdata/clip.mp4")).unwrap(); + let mut input = stream::FFMPEG + .open(stream::Source::File("src/testdata/clip.mp4")) + .unwrap(); // 2015-04-26 00:00:00 UTC. const START_TIME: recording::Time = recording::Time(1430006400i64 * TIME_UNITS_PER_SEC); let extra_data = input.get_extra_data().unwrap(); - let video_sample_entry_id = - db.db.lock().insert_video_sample_entry(extra_data.entry).unwrap(); + let video_sample_entry_id = db + .db + .lock() + .insert_video_sample_entry(extra_data.entry) + .unwrap(); let dir = db.dirs_by_stream_id.get(&TEST_STREAM_ID).unwrap(); - let mut output = writer::Writer::new(dir, &db.db, &db.syncer_channel, TEST_STREAM_ID, - video_sample_entry_id); + let mut output = writer::Writer::new( + dir, + &db.db, + &db.syncer_channel, + TEST_STREAM_ID, + video_sample_entry_id, + ); // end_pts is the pts of the end of the most recent frame (start + duration). // It's needed because dir::Writer calculates a packet's duration from its pts and the @@ -1994,75 +2276,118 @@ mod tests { loop { let pkt = match input.get_next() { Ok(p) => p, - Err(e) if e.is_eof() => { break; }, - Err(e) => { panic!("unexpected input error: {}", e); }, + Err(e) if e.is_eof() => { + break; + } + Err(e) => { + panic!("unexpected input error: {}", e); + } }; let pts = pkt.pts().unwrap(); frame_time += recording::Duration(pkt.duration() as i64); - output.write(pkt.data().expect("packet without data"), frame_time, pts, - pkt.is_key()).unwrap(); + output + .write( + pkt.data().expect("packet without data"), + frame_time, + pts, + pkt.is_key(), + ) + .unwrap(); end_pts = Some(pts + pkt.duration() as i64); } output.close(end_pts).unwrap(); db.syncer_channel.flush(); } - pub fn create_mp4_from_db(tdb: &TestDb, skip_90k: i32, shorten_90k: i32, - include_subtitles: bool) -> File { + pub fn create_mp4_from_db( + tdb: &TestDb, + skip_90k: i32, + shorten_90k: i32, + include_subtitles: bool, + ) -> File { let mut builder = FileBuilder::new(Type::Normal); - builder.include_timestamp_subtitle_track(include_subtitles).unwrap(); - let all_time = recording::Time(i64::min_value()) .. recording::Time(i64::max_value()); + builder + .include_timestamp_subtitle_track(include_subtitles) + .unwrap(); + let all_time = recording::Time(i64::min_value())..recording::Time(i64::max_value()); { let db = tdb.db.lock(); db.list_recordings_by_time(TEST_STREAM_ID, all_time, &mut |r| { let d = r.media_duration_90k; - assert!(skip_90k + shorten_90k < d, "skip_90k={} shorten_90k={} r={:?}", - skip_90k, shorten_90k, r); - builder.append(&*db, r, skip_90k .. d - shorten_90k, true).unwrap(); + assert!( + skip_90k + shorten_90k < d, + "skip_90k={} shorten_90k={} r={:?}", + skip_90k, + shorten_90k, + r + ); + builder + .append(&*db, r, skip_90k..d - shorten_90k, true) + .unwrap(); Ok(()) - }).unwrap(); + }) + .unwrap(); } - builder.build(tdb.db.clone(), tdb.dirs_by_stream_id.clone()).unwrap() + builder + .build(tdb.db.clone(), tdb.dirs_by_stream_id.clone()) + .unwrap() } async fn write_mp4(mp4: &File, dir: &Path) -> String { let mut filename = dir.to_path_buf(); filename.push("clip.new.mp4"); - let mut out = fs::OpenOptions::new().write(true).create_new(true).open(&filename).unwrap(); + let mut out = fs::OpenOptions::new() + .write(true) + .create_new(true) + .open(&filename) + .unwrap(); use ::std::io::Write; - Pin::from(mp4.get_range(0 .. mp4.len())) - .try_for_each(|mut chunk| { - while chunk.has_remaining() { + Pin::from(mp4.get_range(0..mp4.len())) + .try_for_each(|mut chunk| { + while chunk.has_remaining() { let c = chunk.chunk(); let len = match out.write(c) { Err(e) => return futures::future::err(BoxedError::from(e)), Ok(l) => l, }; chunk.advance(len); - } - futures::future::ok(()) - }) - .await - .unwrap(); + } + futures::future::ok(()) + }) + .await + .unwrap(); info!("wrote {:?}", filename); filename.to_str().unwrap().to_string() } fn compare_mp4s(new_filename: &str, pts_offset: i64, shorten: i64) { - let mut orig = stream::FFMPEG.open(stream::Source::File("src/testdata/clip.mp4")).unwrap(); - let mut new = stream::FFMPEG.open(stream::Source::File(new_filename)).unwrap(); - assert_eq!(orig.get_extra_data().unwrap(), new.get_extra_data().unwrap()); + let mut orig = stream::FFMPEG + .open(stream::Source::File("src/testdata/clip.mp4")) + .unwrap(); + let mut new = stream::FFMPEG + .open(stream::Source::File(new_filename)) + .unwrap(); + assert_eq!( + orig.get_extra_data().unwrap(), + new.get_extra_data().unwrap() + ); let mut final_durations = None; loop { let orig_pkt = match orig.get_next() { Ok(p) => Some(p), Err(e) if e.is_eof() => None, - Err(e) => { panic!("unexpected input error: {}", e); }, + Err(e) => { + panic!("unexpected input error: {}", e); + } }; let new_pkt = match new.get_next() { Ok(p) => Some(p), - Err(e) if e.is_eof() => { break; }, - Err(e) => { panic!("unexpected input error: {}", e); }, + Err(e) if e.is_eof() => { + break; + } + Err(e) => { + panic!("unexpected input error: {}", e); + } }; let (orig_pkt, new_pkt) = match (orig_pkt, new_pkt) { (Some(o), Some(n)) => (o, n), @@ -2081,32 +2406,44 @@ mod tests { // edit list, ffmpeg 3.x appears to extend the last packet's duration by the amount // skipped at the beginning. ffmpeg 4.x behaves properly. Allow either behavior. // See . - assert!(orig_dur - shorten + pts_offset == new_dur || - orig_dur - shorten == new_dur, - "orig_dur={} new_dur={} shorten={} pts_offset={}", - orig_dur, new_dur, shorten, pts_offset); + assert!( + orig_dur - shorten + pts_offset == new_dur || orig_dur - shorten == new_dur, + "orig_dur={} new_dur={} shorten={} pts_offset={}", + orig_dur, + new_dur, + shorten, + pts_offset + ); } } /// Makes a `.mp4` file which is only good for exercising the `Slice` logic for producing /// sample tables that match the supplied encoder. - fn make_mp4_from_encoders(type_: Type, db: &TestDb, - mut recordings: Vec, - desired_range_90k: Range, - start_at_key: bool) -> Result { + fn make_mp4_from_encoders( + type_: Type, + db: &TestDb, + mut recordings: Vec, + desired_range_90k: Range, + start_at_key: bool, + ) -> Result { let mut builder = FileBuilder::new(type_); let mut duration_so_far = 0; for r in recordings.drain(..) { let row = db.insert_recording_from_encoder(r); - let d_start = if desired_range_90k.start < duration_so_far { 0 } - else { desired_range_90k.start - duration_so_far }; + let d_start = if desired_range_90k.start < duration_so_far { + 0 + } else { + desired_range_90k.start - duration_so_far + }; let d_end = if desired_range_90k.end > duration_so_far + row.media_duration_90k { row.media_duration_90k } else { desired_range_90k.end - duration_so_far }; duration_so_far += row.media_duration_90k; - builder.append(&db.db.lock(), row, d_start .. d_end, start_at_key).unwrap(); + builder + .append(&db.db.lock(), row, d_start..d_end, start_at_key) + .unwrap(); } builder.build(db.db.clone(), db.dirs_by_stream_id.clone()) } @@ -2125,45 +2462,50 @@ mod tests { } // Time range [2, 2+4+6+8) means the 2nd, 3rd, and 4th samples should be included. - let mp4 = make_mp4_from_encoders(Type::Normal, &db, vec![r], 2 .. 2+4+6+8, true).unwrap(); + let mp4 = + make_mp4_from_encoders(Type::Normal, &db, vec![r], 2..2 + 4 + 6 + 8, true).unwrap(); traverse(mp4.clone()).await; let track = find_track(mp4, 1).await; assert!(track.edts_cursor.is_none()); let mut cursor = track.stbl_cursor; cursor.down().await; cursor.find(b"stts").await; - assert_eq!(cursor.get_all().await, &[ - 0x00, 0x00, 0x00, 0x00, // version + flags - 0x00, 0x00, 0x00, 0x03, // entry_count - - // entries - 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x04, // run length / timestamps. - 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x06, - 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x08, - ]); + assert_eq!( + cursor.get_all().await, + &[ + 0x00, 0x00, 0x00, 0x00, // version + flags + 0x00, 0x00, 0x00, 0x03, // entry_count + // entries + 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x04, // run length / timestamps. + 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x06, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, + 0x00, 0x08, + ] + ); cursor.find(b"stsz").await; - assert_eq!(cursor.get_all().await, &[ - 0x00, 0x00, 0x00, 0x00, // version + flags - 0x00, 0x00, 0x00, 0x00, // sample_size - 0x00, 0x00, 0x00, 0x03, // sample_count - - // entries - 0x00, 0x00, 0x00, 0x06, // size - 0x00, 0x00, 0x00, 0x09, - 0x00, 0x00, 0x00, 0x0c, - ]); + assert_eq!( + cursor.get_all().await, + &[ + 0x00, 0x00, 0x00, 0x00, // version + flags + 0x00, 0x00, 0x00, 0x00, // sample_size + 0x00, 0x00, 0x00, 0x03, // sample_count + // entries + 0x00, 0x00, 0x00, 0x06, // size + 0x00, 0x00, 0x00, 0x09, 0x00, 0x00, 0x00, 0x0c, + ] + ); cursor.find(b"stss").await; - assert_eq!(cursor.get_all().await, &[ - 0x00, 0x00, 0x00, 0x00, // version + flags - 0x00, 0x00, 0x00, 0x03, // entry_count - - // entries - 0x00, 0x00, 0x00, 0x01, // sample_number - 0x00, 0x00, 0x00, 0x02, - 0x00, 0x00, 0x00, 0x03, - ]); + assert_eq!( + cursor.get_all().await, + &[ + 0x00, 0x00, 0x00, 0x00, // version + flags + 0x00, 0x00, 0x00, 0x03, // entry_count + // entries + 0x00, 0x00, 0x00, 0x01, // sample_number + 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x03, + ] + ); } /// Tests sample table and edit list for a video index with half sync frames. @@ -2181,8 +2523,9 @@ mod tests { // Time range [2+4+6, 2+4+6+8) means the 4th sample should be included. // The 3rd gets pulled in also because it's a sync frame and the 4th isn't. - let mp4 = make_mp4_from_encoders(Type::Normal, &db, vec![r], 2+4+6 .. 2+4+6+8, true) - .unwrap(); + let mp4 = + make_mp4_from_encoders(Type::Normal, &db, vec![r], 2 + 4 + 6..2 + 4 + 6 + 8, true) + .unwrap(); traverse(mp4.clone()).await; let track = find_track(mp4, 1).await; @@ -2190,53 +2533,64 @@ mod tests { let mut cursor = track.edts_cursor.unwrap(); cursor.down().await; cursor.find(b"elst").await; - assert_eq!(cursor.get_all().await, &[ - 0x01, 0x00, 0x00, 0x00, // version + flags - 0x00, 0x00, 0x00, 0x01, // length - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x08, // segment_duration - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x06, // media_time - 0x00, 0x01, 0x00, 0x00, // media_rate_{integer,fraction} - ]); + assert_eq!( + cursor.get_all().await, + &[ + 0x01, 0x00, 0x00, 0x00, // version + flags + 0x00, 0x00, 0x00, 0x01, // length + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x08, // segment_duration + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x06, // media_time + 0x00, 0x01, 0x00, 0x00, // media_rate_{integer,fraction} + ] + ); // Examine stbl. let mut cursor = track.stbl_cursor; cursor.down().await; cursor.find(b"stts").await; - assert_eq!(cursor.get_all().await, &[ - 0x00, 0x00, 0x00, 0x00, // version + flags - 0x00, 0x00, 0x00, 0x02, // entry_count - - // entries - 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x06, // run length / timestamps. - 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x08, - ]); + assert_eq!( + cursor.get_all().await, + &[ + 0x00, 0x00, 0x00, 0x00, // version + flags + 0x00, 0x00, 0x00, 0x02, // entry_count + // entries + 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x06, // run length / timestamps. + 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x08, + ] + ); cursor.find(b"stsz").await; - assert_eq!(cursor.get_all().await, &[ - 0x00, 0x00, 0x00, 0x00, // version + flags - 0x00, 0x00, 0x00, 0x00, // sample_size - 0x00, 0x00, 0x00, 0x02, // sample_count - - // entries - 0x00, 0x00, 0x00, 0x09, // size - 0x00, 0x00, 0x00, 0x0c, - ]); + assert_eq!( + cursor.get_all().await, + &[ + 0x00, 0x00, 0x00, 0x00, // version + flags + 0x00, 0x00, 0x00, 0x00, // sample_size + 0x00, 0x00, 0x00, 0x02, // sample_count + // entries + 0x00, 0x00, 0x00, 0x09, // size + 0x00, 0x00, 0x00, 0x0c, + ] + ); cursor.find(b"stss").await; - assert_eq!(cursor.get_all().await, &[ - 0x00, 0x00, 0x00, 0x00, // version + flags - 0x00, 0x00, 0x00, 0x01, // entry_count - - // entries - 0x00, 0x00, 0x00, 0x01, // sample_number - ]); + assert_eq!( + cursor.get_all().await, + &[ + 0x00, 0x00, 0x00, 0x00, // version + flags + 0x00, 0x00, 0x00, 0x01, // entry_count + // entries + 0x00, 0x00, 0x00, 0x01, // sample_number + ] + ); } #[tokio::test] async fn test_no_segments() { testutil::init(); let db = TestDb::new(RealClocks {}); - let e = make_mp4_from_encoders(Type::Normal, &db, vec![], 0 .. 0, true).err().unwrap(); + let e = make_mp4_from_encoders(Type::Normal, &db, vec![], 0..0, true) + .err() + .unwrap(); assert_eq!(e.to_string(), "Invalid argument: no video_sample_entries"); } @@ -2258,7 +2612,7 @@ mod tests { encoders.push(r); // This should include samples 3 and 4 only, both sync frames. - let mp4 = make_mp4_from_encoders(Type::Normal, &db, encoders, 1+2 .. 1+2+3+4, true) + let mp4 = make_mp4_from_encoders(Type::Normal, &db, encoders, 1 + 2..1 + 2 + 3 + 4, true) .unwrap(); traverse(mp4.clone()).await; let mut cursor = BoxCursor::new(mp4); @@ -2274,7 +2628,7 @@ mod tests { assert!(cursor.find(b"stbl").await); cursor.down().await; assert!(cursor.find(b"stss").await); - assert_eq!(cursor.get_u32(4).await, 2); // entry_count + assert_eq!(cursor.get_u32(4).await, 2); // entry_count assert_eq!(cursor.get_u32(8).await, 1); assert_eq!(cursor.get_u32(12).await, 2); } @@ -2295,15 +2649,15 @@ mod tests { encoders.push(r); // Multi-segment recording with an edit list, encoding with a zero-duration recording. - let mp4 = make_mp4_from_encoders(Type::Normal, &db, encoders, 1 .. 2+3, true).unwrap(); + let mp4 = make_mp4_from_encoders(Type::Normal, &db, encoders, 1..2 + 3, true).unwrap(); traverse(mp4.clone()).await; let track = find_track(mp4, 1).await; let mut cursor = track.edts_cursor.unwrap(); cursor.down().await; cursor.find(b"elst").await; - assert_eq!(cursor.get_u32(4).await, 1); // entry_count - assert_eq!(cursor.get_u64(8).await, 4); // segment_duration - assert_eq!(cursor.get_u64(16).await, 1); // media_time + assert_eq!(cursor.get_u32(4).await, 1); // entry_count + assert_eq!(cursor.get_u64(8).await, 4); // segment_duration + assert_eq!(cursor.get_u64(16).await, 1); // media_time } #[tokio::test] @@ -2320,8 +2674,14 @@ mod tests { // Time range [2+4+6, 2+4+6+8+1) means the 4th sample and part of the 5th are included. // The 3rd gets pulled in also because it's a sync frame and the 4th isn't. - let mp4 = make_mp4_from_encoders(Type::MediaSegment, &db, vec![r], - 2+4+6 .. 2+4+6+8+1, true).unwrap(); + let mp4 = make_mp4_from_encoders( + Type::MediaSegment, + &db, + vec![r], + 2 + 4 + 6..2 + 4 + 6 + 8 + 1, + true, + ) + .unwrap(); traverse(mp4.clone()).await; let mut cursor = BoxCursor::new(mp4); cursor.down().await; @@ -2336,18 +2696,21 @@ mod tests { assert!(cursor.find(b"trun").await); assert_eq!(cursor.get_u32(4).await, 2); assert_eq!(cursor.get_u32(8).await as u64, mdat.interior().start); - assert_eq!(cursor.get_u32(12).await, 174063616); // first_sample_flags - assert_eq!(cursor.get_u32(16).await, 6); // sample duration - assert_eq!(cursor.get_u32(20).await, 9); // sample size - assert_eq!(cursor.get_u32(24).await, 8); // sample duration - assert_eq!(cursor.get_u32(28).await, 12); // sample size + assert_eq!(cursor.get_u32(12).await, 174063616); // first_sample_flags + assert_eq!(cursor.get_u32(16).await, 6); // sample duration + assert_eq!(cursor.get_u32(20).await, 9); // sample size + assert_eq!(cursor.get_u32(24).await, 8); // sample duration + assert_eq!(cursor.get_u32(28).await, 12); // sample size assert!(cursor.next().await); assert_eq!(cursor.name(), "trun"); assert_eq!(cursor.get_u32(4).await, 1); - assert_eq!(cursor.get_u32(8).await as u64, mdat.interior().start + 9 + 12); - assert_eq!(cursor.get_u32(12).await, 174063616); // first_sample_flags - assert_eq!(cursor.get_u32(16).await, 1); // sample duration - assert_eq!(cursor.get_u32(20).await, 15); // sample size + assert_eq!( + cursor.get_u32(8).await as u64, + mdat.interior().start + 9 + 12 + ); + assert_eq!(cursor.get_u32(12).await, 174063616); // first_sample_flags + assert_eq!(cursor.get_u32(16).await, 1); // sample duration + assert_eq!(cursor.get_u32(20).await, 15); // sample size } /// Tests `.mp4` files which represent a single frame, as in the live view WebSocket stream. @@ -2366,8 +2729,14 @@ mod tests { let mut pos = 0; for i in 1..6 { let duration_90k = 2 * i; - let mp4 = make_mp4_from_encoders(Type::MediaSegment, &db, vec![r.clone()], - pos .. pos+duration_90k, false).unwrap(); + let mp4 = make_mp4_from_encoders( + Type::MediaSegment, + &db, + vec![r.clone()], + pos..pos + duration_90k, + false, + ) + .unwrap(); traverse(mp4.clone()).await; let mut cursor = BoxCursor::new(mp4); cursor.down().await; @@ -2394,11 +2763,16 @@ mod tests { // here fails, it can be updated, but the etag must change as well! Otherwise clients may // combine ranges from the new format with ranges from the old format. let hash = digest(&mp4).await; - assert_eq!("e95f2d261cdebac5b9983abeea59e8eb053dc4efac866722544c665d9de7c49d", - hash.to_hex().as_str()); + assert_eq!( + "e95f2d261cdebac5b9983abeea59e8eb053dc4efac866722544c665d9de7c49d", + hash.to_hex().as_str() + ); const EXPECTED_ETAG: &'static str = "\"61031ab36449b4d1186e9513b5e40df84e78bfb2807c0035b360437bb905cdd5\""; - assert_eq!(Some(HeaderValue::from_str(EXPECTED_ETAG).unwrap()), mp4.etag()); + assert_eq!( + Some(HeaderValue::from_str(EXPECTED_ETAG).unwrap()), + mp4.etag() + ); drop(db.syncer_channel); db.db.lock().clear_on_flush(); db.syncer_join.join().unwrap(); @@ -2418,11 +2792,16 @@ mod tests { // here fails, it can be updated, but the etag must change as well! Otherwise clients may // combine ranges from the new format with ranges from the old format. let hash = digest(&mp4).await; - assert_eq!("77e09be8ee5ca353ca56f9a80bb7420680713c80a0831d236fac45a96aa3b3d4", - hash.to_hex().as_str()); + assert_eq!( + "77e09be8ee5ca353ca56f9a80bb7420680713c80a0831d236fac45a96aa3b3d4", + hash.to_hex().as_str() + ); const EXPECTED_ETAG: &'static str = "\"8e048b22b21c9b93d889e8dfbeeb56fa1b17dc0956190f5c3acc84f6f674089f\""; - assert_eq!(Some(HeaderValue::from_str(EXPECTED_ETAG).unwrap()), mp4.etag()); + assert_eq!( + Some(HeaderValue::from_str(EXPECTED_ETAG).unwrap()), + mp4.etag() + ); drop(db.syncer_channel); db.db.lock().clear_on_flush(); db.syncer_join.join().unwrap(); @@ -2442,11 +2821,16 @@ mod tests { // here fails, it can be updated, but the etag must change as well! Otherwise clients may // combine ranges from the new format with ranges from the old format. let hash = digest(&mp4).await; - assert_eq!("f9807cfc6b96a399f3a5ad62d090f55a18543a9eeb1f48d59f86564ffd9b1e84", - hash.to_hex().as_str()); + assert_eq!( + "f9807cfc6b96a399f3a5ad62d090f55a18543a9eeb1f48d59f86564ffd9b1e84", + hash.to_hex().as_str() + ); const EXPECTED_ETAG: &'static str = "\"196192eccd8be2c840dfc4073355efe5c917999641e3d0a2b87e0d2eab40267f\""; - assert_eq!(Some(HeaderValue::from_str(EXPECTED_ETAG).unwrap()), mp4.etag()); + assert_eq!( + Some(HeaderValue::from_str(EXPECTED_ETAG).unwrap()), + mp4.etag() + ); drop(db.syncer_channel); db.db.lock().clear_on_flush(); db.syncer_join.join().unwrap(); @@ -2457,8 +2841,8 @@ mod tests { testutil::init(); let db = TestDb::new(RealClocks {}); copy_mp4_to_db(&db); - let off = 2*TIME_UNITS_PER_SEC; - let mp4 = create_mp4_from_db(&db, i32::try_from(off).unwrap(), 0, true); + let off = 2 * TIME_UNITS_PER_SEC; + let mp4 = create_mp4_from_db(&db, i32::try_from(off).unwrap(), 0, true); traverse(mp4.clone()).await; let new_filename = write_mp4(&mp4, db.tmpdir.path()).await; compare_mp4s(&new_filename, off, 0); @@ -2467,11 +2851,16 @@ mod tests { // here fails, it can be updated, but the etag must change as well! Otherwise clients may // combine ranges from the new format with ranges from the old format. let hash = digest(&mp4).await; - assert_eq!("7aef371e9ab5e871893fd9b1963cb71c1c9b093b5d4ff36cb1340b65155a8aa2", - hash.to_hex().as_str()); + assert_eq!( + "7aef371e9ab5e871893fd9b1963cb71c1c9b093b5d4ff36cb1340b65155a8aa2", + hash.to_hex().as_str() + ); const EXPECTED_ETAG: &'static str = "\"84cafd9db7a5c0c32961d9848582d2dca436a58d25cbedfb02d7450bc6ce1229\""; - assert_eq!(Some(HeaderValue::from_str(EXPECTED_ETAG).unwrap()), mp4.etag()); + assert_eq!( + Some(HeaderValue::from_str(EXPECTED_ETAG).unwrap()), + mp4.etag() + ); drop(db.syncer_channel); db.db.lock().clear_on_flush(); db.syncer_join.join().unwrap(); @@ -2491,29 +2880,34 @@ mod tests { // here fails, it can be updated, but the etag must change as well! Otherwise clients may // combine ranges from the new format with ranges from the old format. let hash = digest(&mp4).await; - assert_eq!("5211104e1fdfe3bbc0d7d7d479933940305ff7f23201e97308db23a022ee6339", - hash.to_hex().as_str()); + assert_eq!( + "5211104e1fdfe3bbc0d7d7d479933940305ff7f23201e97308db23a022ee6339", + hash.to_hex().as_str() + ); const EXPECTED_ETAG: &'static str = "\"9e50099d86ae1c742e65f7a4151c4427f42051a87158405a35b4e5550fd05c30\""; - assert_eq!(Some(HeaderValue::from_str(EXPECTED_ETAG).unwrap()), mp4.etag()); + assert_eq!( + Some(HeaderValue::from_str(EXPECTED_ETAG).unwrap()), + mp4.etag() + ); drop(db.syncer_channel); db.db.lock().clear_on_flush(); db.syncer_join.join().unwrap(); } } -#[cfg(all(test, feature="nightly"))] +#[cfg(all(test, feature = "nightly"))] mod bench { extern crate test; + use super::tests::create_mp4_from_db; use base::clock::RealClocks; use db::recording; use db::testutil::{self, TestDb}; use futures::future; - use hyper; use http_serve; + use hyper; use lazy_static::lazy_static; - use super::tests::create_mp4_from_db; use url::Url; /// An HTTP server for benchmarking. @@ -2537,8 +2931,11 @@ mod bench { let make_svc = hyper::service::make_service_fn(move |_conn| { future::ok::<_, std::convert::Infallible>(hyper::service::service_fn({ let mp4 = mp4.clone(); - move |req| future::ok::, hyper::Error>( - http_serve::serve(mp4.clone(), &req)) + move |req| { + future::ok::, hyper::Error>( + http_serve::serve(mp4.clone(), &req), + ) + } })) }); let rt = tokio::runtime::Runtime::new().unwrap(); @@ -2549,7 +2946,7 @@ mod bench { .tcp_nodelay(true) .serve(make_svc) }; - let addr = srv.local_addr(); // resolve port 0 to a real ephemeral port number. + let addr = srv.local_addr(); // resolve port 0 to a real ephemeral port number. ::std::thread::spawn(move || { rt.block_on(srv).unwrap(); }); @@ -2572,22 +2969,24 @@ mod bench { let db = db.db.lock(); let segment = { - let all_time = recording::Time(i64::min_value()) .. recording::Time(i64::max_value()); + let all_time = recording::Time(i64::min_value())..recording::Time(i64::max_value()); let mut row = None; db.list_recordings_by_time(testutil::TEST_STREAM_ID, all_time, &mut |r| { row = Some(r); Ok(()) - }).unwrap(); + }) + .unwrap(); let row = row.unwrap(); - let rel_range_90k = 0 .. row.media_duration_90k; + let rel_range_90k = 0..row.media_duration_90k; super::Segment::new(&db, &row, rel_range_90k, 1, true).unwrap() }; db.with_recording_playback(segment.s.id, &mut |playback| { - let v = segment.build_index(playback).unwrap(); // warm. - b.bytes = v.len() as u64; // define the benchmark performance in terms of output bytes. + let v = segment.build_index(playback).unwrap(); // warm. + b.bytes = v.len() as u64; // define the benchmark performance in terms of output bytes. b.iter(|| segment.build_index(playback).unwrap()); Ok(()) - }).unwrap(); + }) + .unwrap(); } /// Benchmarks serving the generated part of a `.mp4` file (up to the first byte from disk). @@ -2601,17 +3000,17 @@ mod bench { let rt = tokio::runtime::Runtime::new().unwrap(); let run = || { rt.block_on(async { - let resp = - client.get(server.url.clone()) - .header(reqwest::header::RANGE, format!("bytes=0-{}", p - 1)) - .send() - .await - .unwrap(); + let resp = client + .get(server.url.clone()) + .header(reqwest::header::RANGE, format!("bytes=0-{}", p - 1)) + .send() + .await + .unwrap(); let b = resp.bytes().await.unwrap(); assert_eq!(p, b.len() as u64); }); }; - run(); // warm. + run(); // warm. b.iter(run); } diff --git a/server/src/slices.rs b/server/src/slices.rs index 8004d3e..b0b3b2f 100644 --- a/server/src/slices.rs +++ b/server/src/slices.rs @@ -30,17 +30,17 @@ //! Tools for implementing a `http_serve::Entity` body composed from many "slices". +use crate::body::{wrap_error, BoxedError}; use base::format_err_t; -use crate::body::{BoxedError, wrap_error}; -use failure::{Error, bail}; -use futures::{Stream, stream, stream::StreamExt}; +use failure::{bail, Error}; +use futures::{stream, stream::StreamExt, Stream}; use std::fmt; use std::ops::Range; use std::pin::Pin; /// Gets a byte range given a context argument. /// Each `Slice` instance belongs to a single `Slices`. -pub trait Slice : fmt::Debug + Sized + Sync + 'static { +pub trait Slice: fmt::Debug + Sized + Sync + 'static { type Ctx: Send + Sync + Clone; type Chunk: Send + Sync; @@ -52,15 +52,22 @@ pub trait Slice : fmt::Debug + Sized + Sync + 'static { /// Gets the body bytes indicated by `r`, which is relative to this slice's start. /// The additional argument `ctx` is as supplied to the `Slices`. /// The additional argument `l` is the length of this slice, as determined by the `Slices`. - fn get_range(&self, ctx: &Self::Ctx, r: Range, len: u64) - -> Box> + Sync + Send>; + fn get_range( + &self, + ctx: &Self::Ctx, + r: Range, + len: u64, + ) -> Box> + Sync + Send>; fn get_slices(ctx: &Self::Ctx) -> &Slices; } /// Helper to serve byte ranges from a body which is broken down into many "slices". /// This is used to implement `.mp4` serving in `mp4::File` from `mp4::Slice` enums. -pub struct Slices where S: Slice { +pub struct Slices +where + S: Slice, +{ /// The total byte length of the `Slices`. /// Equivalent to `self.slices.back().map(|s| s.end()).unwrap_or(0)`; kept for convenience and /// to avoid a branch. @@ -70,22 +77,45 @@ pub struct Slices where S: Slice { slices: Vec, } -impl fmt::Debug for Slices where S: Slice { +impl fmt::Debug for Slices +where + S: Slice, +{ fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - write!(f, "{} slices with overall length {}:", self.slices.len(), self.len)?; + write!( + f, + "{} slices with overall length {}:", + self.slices.len(), + self.len + )?; let mut start = 0; for (i, s) in self.slices.iter().enumerate() { let end = s.end(); - write!(f, "\ni {:7}: range [{:12}, {:12}) len {:12}: {:?}", - i, start, end, end - start, s)?; + write!( + f, + "\ni {:7}: range [{:12}, {:12}) len {:12}: {:?}", + i, + start, + end, + end - start, + s + )?; start = end; } Ok(()) } } -impl Slices where S: Slice { - pub fn new() -> Self { Slices{len: 0, slices: Vec::new()} } +impl Slices +where + S: Slice, +{ + pub fn new() -> Self { + Slices { + len: 0, + slices: Vec::new(), + } + } /// Reserves space for at least `additional` more slices to be appended. pub fn reserve(&mut self, additional: usize) { @@ -95,8 +125,13 @@ impl Slices where S: Slice { /// Appends the given slice, which must have end > the Slices's current len. pub fn append(&mut self, slice: S) -> Result<(), Error> { if slice.end() <= self.len { - bail!("end {} <= len {} while adding slice {:?} to slices:\n{:?}", - slice.end(), self.len, slice, self); + bail!( + "end {} <= len {} while adding slice {:?} to slices:\n{:?}", + slice.end(), + self.len, + slice, + self + ); } self.len = slice.end(); self.slices.push(slice); @@ -104,59 +139,78 @@ impl Slices where S: Slice { } /// Returns the total byte length of all slices. - pub fn len(&self) -> u64 { self.len } + pub fn len(&self) -> u64 { + self.len + } /// Returns the number of slices. - pub fn num(&self) -> usize { self.slices.len() } + pub fn num(&self) -> usize { + self.slices.len() + } /// Writes `range` to `out`. /// This interface mirrors `http_serve::Entity::write_to`, with the additional `ctx` argument. - pub fn get_range(&self, ctx: &S::Ctx, range: Range) - -> Box> + Sync + Send> { + pub fn get_range( + &self, + ctx: &S::Ctx, + range: Range, + ) -> Box> + Sync + Send> { if range.start > range.end || range.end > self.len { - return Box::new(stream::once(futures::future::err(wrap_error(format_err_t!( - Internal, "Bad range {:?} for slice of length {}", range, self.len))))); + return Box::new(stream::once(futures::future::err(wrap_error( + format_err_t!( + Internal, + "Bad range {:?} for slice of length {}", + range, + self.len + ), + )))); } // Binary search for the first slice of the range to write, determining its index and // (from the preceding slice) the start of its range. let (i, slice_start) = match self.slices.binary_search_by_key(&range.start, |s| s.end()) { - Ok(i) => (i+1, self.slices[i].end()), // desired start == slice i's end; first is i+1! + Ok(i) => (i + 1, self.slices[i].end()), // desired start == slice i's end; first is i+1! Err(i) if i == 0 => (0, 0), // desired start < slice 0's end; first is 0. - Err(i) => (i, self.slices[i-1].end()), // desired start < slice i's end; first is i. + Err(i) => (i, self.slices[i - 1].end()), // desired start < slice i's end; first is i. }; // Iterate through and write each slice until the end. let start_pos = range.start - slice_start; let bodies = stream::unfold( - (ctx.clone(), i, start_pos, slice_start), move |(c, i, start_pos, slice_start)| { - let (body, min_end); - { - let self_ = S::get_slices(&c); - if i == self_.slices.len() { return futures::future::ready(None) } - let s = &self_.slices[i]; - if range.end == slice_start + start_pos { return futures::future::ready(None) } - let s_end = s.end(); - min_end = ::std::cmp::min(range.end, s_end); - let l = s_end - slice_start; - body = s.get_range(&c, start_pos .. min_end - slice_start, l); - }; - futures::future::ready(Some((Pin::from(body), (c, i+1, 0, min_end)))) - }); + (ctx.clone(), i, start_pos, slice_start), + move |(c, i, start_pos, slice_start)| { + let (body, min_end); + { + let self_ = S::get_slices(&c); + if i == self_.slices.len() { + return futures::future::ready(None); + } + let s = &self_.slices[i]; + if range.end == slice_start + start_pos { + return futures::future::ready(None); + } + let s_end = s.end(); + min_end = ::std::cmp::min(range.end, s_end); + let l = s_end - slice_start; + body = s.get_range(&c, start_pos..min_end - slice_start, l); + }; + futures::future::ready(Some((Pin::from(body), (c, i + 1, 0, min_end)))) + }, + ); Box::new(bodies.flatten()) } } #[cfg(test)] mod tests { + use super::{Slice, Slices}; use crate::body::BoxedError; use db::testutil; use futures::stream::{self, Stream, TryStreamExt}; use lazy_static::lazy_static; use std::ops::Range; use std::pin::Pin; - use super::{Slice, Slices}; #[derive(Debug, Eq, PartialEq)] pub struct FakeChunk { @@ -174,30 +228,45 @@ mod tests { type Ctx = &'static Slices; type Chunk = FakeChunk; - fn end(&self) -> u64 { self.end } - - fn get_range(&self, _ctx: &&'static Slices, r: Range, _l: u64) - -> Box> + Send + Sync> { - Box::new(stream::once(futures::future::ok(FakeChunk{slice: self.name, range: r}))) + fn end(&self) -> u64 { + self.end } - fn get_slices(ctx: &&'static Slices) -> &'static Slices { *ctx } + fn get_range( + &self, + _ctx: &&'static Slices, + r: Range, + _l: u64, + ) -> Box> + Send + Sync> { + Box::new(stream::once(futures::future::ok(FakeChunk { + slice: self.name, + range: r, + }))) + } + + fn get_slices(ctx: &&'static Slices) -> &'static Slices { + *ctx + } } lazy_static! { + #[rustfmt::skip] static ref SLICES: Slices = { let mut s = Slices::new(); - s.append(FakeSlice{end: 5, name: "a"}).unwrap(); - s.append(FakeSlice{end: 5+13, name: "b"}).unwrap(); - s.append(FakeSlice{end: 5+13+7, name: "c"}).unwrap(); - s.append(FakeSlice{end: 5+13+7+17, name: "d"}).unwrap(); - s.append(FakeSlice{end: 5+13+7+17+19, name: "e"}).unwrap(); + s.append(FakeSlice { end: 5, name: "a" }).unwrap(); + s.append(FakeSlice { end: 5 + 13, name: "b" }).unwrap(); + s.append(FakeSlice { end: 5 + 13 + 7, name: "c" }).unwrap(); + s.append(FakeSlice { end: 5 + 13 + 7 + 17, name: "d" }).unwrap(); + s.append(FakeSlice { end: 5 + 13 + 7 + 17 + 19, name: "e" }).unwrap(); s }; } async fn get_range(r: Range) -> Vec { - Pin::from(SLICES.get_range(&&*SLICES, r)).try_collect().await.unwrap() + Pin::from(SLICES.get_range(&&*SLICES, r)) + .try_collect() + .await + .unwrap() } #[test] @@ -210,48 +279,68 @@ mod tests { pub async fn exact_slice() { // Test writing exactly slice b. testutil::init(); - let out = get_range(5 .. 18).await; - assert_eq!(&[FakeChunk{slice: "b", range: 0 .. 13}], &out[..]); + let out = get_range(5..18).await; + assert_eq!( + &[FakeChunk { + slice: "b", + range: 0..13 + }], + &out[..] + ); } #[tokio::test] pub async fn offset_first() { // Test writing part of slice a. testutil::init(); - let out = get_range(1 .. 3).await; - assert_eq!(&[FakeChunk{slice: "a", range: 1 .. 3}], &out[..]); + let out = get_range(1..3).await; + assert_eq!( + &[FakeChunk { + slice: "a", + range: 1..3 + }], + &out[..] + ); } #[tokio::test] pub async fn offset_mid() { // Test writing part of slice b, all of slice c, and part of slice d. testutil::init(); - let out = get_range(17 .. 26).await; - assert_eq!(&[ - FakeChunk{slice: "b", range: 12 .. 13}, - FakeChunk{slice: "c", range: 0 .. 7}, - FakeChunk{slice: "d", range: 0 .. 1}, - ], &out[..]); + let out = get_range(17..26).await; + #[rustfmt::skip] + assert_eq!( + &[ + FakeChunk { slice: "b", range: 12..13 }, + FakeChunk { slice: "c", range: 0..7 }, + FakeChunk { slice: "d", range: 0..1 }, + ], + &out[..] + ); } #[tokio::test] pub async fn everything() { // Test writing the whole Slices. testutil::init(); - let out = get_range(0 .. 61).await; - assert_eq!(&[ - FakeChunk{slice: "a", range: 0 .. 5}, - FakeChunk{slice: "b", range: 0 .. 13}, - FakeChunk{slice: "c", range: 0 .. 7}, - FakeChunk{slice: "d", range: 0 .. 17}, - FakeChunk{slice: "e", range: 0 .. 19}, - ], &out[..]); + let out = get_range(0..61).await; + #[rustfmt::skip] + assert_eq!( + &[ + FakeChunk { slice: "a", range: 0..5 }, + FakeChunk { slice: "b", range: 0..13 }, + FakeChunk { slice: "c", range: 0..7 }, + FakeChunk { slice: "d", range: 0..17 }, + FakeChunk { slice: "e", range: 0..19 }, + ], + &out[..] + ); } #[tokio::test] pub async fn at_end() { testutil::init(); - let out = get_range(61 .. 61).await; + let out = get_range(61..61).await; let empty: &[FakeChunk] = &[]; assert_eq!(empty, &out[..]); } diff --git a/server/src/stream.rs b/server/src/stream.rs index a1f5ca2..df3e964 100644 --- a/server/src/stream.rs +++ b/server/src/stream.rs @@ -30,7 +30,7 @@ use crate::h264; use cstr::cstr; -use failure::{Error, bail}; +use failure::{bail, Error}; use ffmpeg; use lazy_static::lazy_static; use log::{debug, warn}; @@ -50,13 +50,10 @@ pub enum Source<'a> { File(&'a str), /// An RTSP stream, for production use. - Rtsp { - url: &'a str, - redacted_url: &'a str - }, + Rtsp { url: &'a str, redacted_url: &'a str }, } -pub trait Opener : Sync { +pub trait Opener: Sync { fn open(&self, src: Source) -> Result; } @@ -70,8 +67,10 @@ pub struct Ffmpeg {} impl Ffmpeg { fn new() -> Ffmpeg { - START.call_once(|| { ffmpeg::Ffmpeg::new(); }); - Ffmpeg{} + START.call_once(|| { + ffmpeg::Ffmpeg::new(); + }); + Ffmpeg {} } } @@ -84,39 +83,57 @@ impl Opener for Ffmpeg { let mut open_options = ffmpeg::avutil::Dictionary::new(); // Work around https://github.com/scottlamb/moonfire-nvr/issues/10 - open_options.set(cstr!("advanced_editlist"), cstr!("false")).unwrap(); + open_options + .set(cstr!("advanced_editlist"), cstr!("false")) + .unwrap(); let url = format!("file:{}", filename); - let i = InputFormatContext::open(&CString::new(url.clone()).unwrap(), - &mut open_options)?; + let i = InputFormatContext::open( + &CString::new(url.clone()).unwrap(), + &mut open_options, + )?; if !open_options.empty() { - warn!("While opening URL {}, some options were not understood: {}", - url, open_options); + warn!( + "While opening URL {}, some options were not understood: {}", + url, open_options + ); } i } - Source::Rtsp{url, redacted_url} => { + Source::Rtsp { url, redacted_url } => { let mut open_options = ffmpeg::avutil::Dictionary::new(); - open_options.set(cstr!("rtsp_transport"), cstr!("tcp")).unwrap(); - open_options.set(cstr!("user-agent"), cstr!("moonfire-nvr")).unwrap(); + open_options + .set(cstr!("rtsp_transport"), cstr!("tcp")) + .unwrap(); + open_options + .set(cstr!("user-agent"), cstr!("moonfire-nvr")) + .unwrap(); // 10-second socket timeout, in microseconds. - open_options.set(cstr!("stimeout"), cstr!("10000000")).unwrap(); + open_options + .set(cstr!("stimeout"), cstr!("10000000")) + .unwrap(); // Without this option, the first packet has an incorrect pts. // https://trac.ffmpeg.org/ticket/5018 - open_options.set(cstr!("fflags"), cstr!("nobuffer")).unwrap(); + open_options + .set(cstr!("fflags"), cstr!("nobuffer")) + .unwrap(); // Moonfire NVR currently only supports video, so receiving audio is wasteful. // It also triggers . - open_options.set(cstr!("allowed_media_types"), cstr!("video")).unwrap(); + open_options + .set(cstr!("allowed_media_types"), cstr!("video")) + .unwrap(); let i = InputFormatContext::open(&CString::new(url).unwrap(), &mut open_options)?; if !open_options.empty() { - warn!("While opening URL {}, some options were not understood: {}", - redacted_url, open_options); + warn!( + "While opening URL {}, some options were not understood: {}", + redacted_url, open_options + ); } i - }, + } }; input.find_stream_info()?; @@ -125,7 +142,7 @@ impl Opener for Ffmpeg { let mut video_i = None; { let s = input.streams(); - for i in 0 .. s.len() { + for i in 0..s.len() { if s.get(i).codecpar().codec_type().is_video() { debug!("Video stream index is {}", i); video_i = Some(i); @@ -138,10 +155,7 @@ impl Opener for Ffmpeg { None => bail!("no video stream"), }; - Ok(FfmpegStream { - input, - video_i, - }) + Ok(FfmpegStream { input, video_i }) } } @@ -159,7 +173,11 @@ impl Stream for FfmpegStream { let video = self.input.streams().get(self.video_i); let tb = video.time_base(); if tb.num != 1 || tb.den != 90000 { - bail!("video stream has timebase {}/{}; expected 1/90000", tb.num, tb.den); + bail!( + "video stream has timebase {}/{}; expected 1/90000", + tb.num, + tb.den + ); } let codec = video.codecpar(); let codec_id = codec.codec_id(); @@ -167,8 +185,11 @@ impl Stream for FfmpegStream { bail!("stream's video codec {:?} is not h264", codec_id); } let dims = codec.dims(); - h264::ExtraData::parse(codec.extradata(), u16::try_from(dims.width)?, - u16::try_from(dims.height)?) + h264::ExtraData::parse( + codec.extradata(), + u16::try_from(dims.width)?, + u16::try_from(dims.height)?, + ) } fn get_next<'i>(&'i mut self) -> Result, ffmpeg::Error> { diff --git a/server/src/streamer.rs b/server/src/streamer.rs index aaa65f5..0a4d639 100644 --- a/server/src/streamer.rs +++ b/server/src/streamer.rs @@ -28,11 +28,11 @@ // You should have received a copy of the GNU General Public License // along with this program. If not, see . -use base::clock::{Clocks, TimerGuard}; use crate::h264; use crate::stream; -use db::{Camera, Database, Stream, dir, recording, writer}; -use failure::{Error, bail, format_err}; +use base::clock::{Clocks, TimerGuard}; +use db::{dir, recording, writer, Camera, Database, Stream}; +use failure::{bail, format_err, Error}; use log::{debug, info, trace, warn}; use std::result::Result; use std::sync::atomic::{AtomicBool, Ordering}; @@ -43,13 +43,21 @@ use url::Url; pub static ROTATE_INTERVAL_SEC: i64 = 60; /// Common state that can be used by multiple `Streamer` instances. -pub struct Environment<'a, 'b, C, S> where C: Clocks + Clone, S: 'a + stream::Stream { +pub struct Environment<'a, 'b, C, S> +where + C: Clocks + Clone, + S: 'a + stream::Stream, +{ pub opener: &'a dyn stream::Opener, pub db: &'b Arc>, pub shutdown: &'b Arc, } -pub struct Streamer<'a, C, S> where C: Clocks + Clone, S: 'a + stream::Stream { +pub struct Streamer<'a, C, S> +where + C: Clocks + Clone, + S: 'a + stream::Stream, +{ shutdown: Arc, // State below is only used by the thread in Run. @@ -66,17 +74,27 @@ pub struct Streamer<'a, C, S> where C: Clocks + Clone, S: 'a + stream::Stream { detector: Option>, } -impl<'a, C, S> Streamer<'a, C, S> where C: 'a + Clocks + Clone, S: 'a + stream::Stream { - pub fn new<'b>(env: &Environment<'a, 'b, C, S>, dir: Arc, - syncer_channel: writer::SyncerChannel<::std::fs::File>, - stream_id: i32, c: &Camera, s: &Stream, rotate_offset_sec: i64, - rotate_interval_sec: i64, - detector: Option>) - -> Result { +impl<'a, C, S> Streamer<'a, C, S> +where + C: 'a + Clocks + Clone, + S: 'a + stream::Stream, +{ + pub fn new<'b>( + env: &Environment<'a, 'b, C, S>, + dir: Arc, + syncer_channel: writer::SyncerChannel<::std::fs::File>, + stream_id: i32, + c: &Camera, + s: &Stream, + rotate_offset_sec: i64, + rotate_interval_sec: i64, + detector: Option>, + ) -> Result { let mut url = Url::parse(&s.rtsp_url)?; let mut redacted_url = url.clone(); if !c.username.is_empty() { - url.set_username(&c.username).map_err(|_| format_err!("can't set username"))?; + url.set_username(&c.username) + .map_err(|_| format_err!("can't set username"))?; redacted_url.set_username(&c.username).unwrap(); url.set_password(Some(&c.password)).unwrap(); redacted_url.set_password(Some("redacted")).unwrap(); @@ -97,14 +115,20 @@ impl<'a, C, S> Streamer<'a, C, S> where C: 'a + Clocks + Clone, S: 'a + stream:: }) } - pub fn short_name(&self) -> &str { &self.short_name } + pub fn short_name(&self) -> &str { + &self.short_name + } pub fn run(&mut self) { while !self.shutdown.load(Ordering::SeqCst) { if let Err(e) = self.run_once() { let sleep_time = time::Duration::seconds(1); - warn!("{}: sleeping for {:?} after error: {}", - self.short_name, sleep_time, base::prettify_failure(&e)); + warn!( + "{}: sleeping for {:?} after error: {}", + self.short_name, + sleep_time, + base::prettify_failure(&e) + ); self.db.clocks().sleep(sleep_time); } } @@ -127,21 +151,31 @@ impl<'a, C, S> Streamer<'a, C, S> where C: 'a + Clocks + Clone, S: 'a + stream:: let mut detector_stream = match self.detector.as_ref() { None => None, Some(od) => Some(crate::analytics::ObjectDetectorStream::new( - stream.get_video_codecpar(), &od)?), + stream.get_video_codecpar(), + &od, + )?), }; let extra_data = stream.get_extra_data()?; let video_sample_entry_id = { let _t = TimerGuard::new(&clocks, || "inserting video sample entry"); self.db.lock().insert_video_sample_entry(extra_data.entry)? }; - debug!("{}: video_sample_entry_id={}", self.short_name, video_sample_entry_id); + debug!( + "{}: video_sample_entry_id={}", + self.short_name, video_sample_entry_id + ); let mut seen_key_frame = false; // Seconds since epoch at which to next rotate. let mut rotate: Option = None; let mut transformed = Vec::new(); - let mut w = writer::Writer::new(&self.dir, &self.db, &self.syncer_channel, self.stream_id, - video_sample_entry_id); + let mut w = writer::Writer::new( + &self.dir, + &self.db, + &self.syncer_channel, + self.stream_id, + video_sample_entry_id, + ); while !self.shutdown.load(Ordering::SeqCst) { let pkt = { let _t = TimerGuard::new(&clocks, || "getting next packet"); @@ -168,22 +202,32 @@ impl<'a, C, S> Streamer<'a, C, S> where C: 'a + Clocks + Clone, S: 'a + stream:: } else { Some(r) } - } else { None }; + } else { + None + }; let r = match rotate { Some(r) => r, None => { let sec = frame_realtime.sec; let r = sec - (sec % self.rotate_interval_sec) + self.rotate_offset_sec; - let r = r + if r <= sec { self.rotate_interval_sec } else { 0 }; + let r = r + if r <= sec { + self.rotate_interval_sec + } else { + 0 + }; // On the first recording, set rotate time to not the next rotate offset, but // the one after, so that it's longer than usual rather than shorter than // usual. This ensures there's plenty of frame times to use when calculating // the start time. - let r = r + if w.previously_opened()? { 0 } else { self.rotate_interval_sec }; + let r = r + if w.previously_opened()? { + 0 + } else { + self.rotate_interval_sec + }; let _t = TimerGuard::new(&clocks, || "creating writer"); r - }, + } }; let orig_data = match pkt.data() { Some(d) => d, @@ -195,8 +239,9 @@ impl<'a, C, S> Streamer<'a, C, S> where C: 'a + Clocks + Clone, S: 'a + stream:: } else { orig_data }; - let _t = TimerGuard::new(&clocks, - || format!("writing {} bytes", transformed_data.len())); + let _t = TimerGuard::new(&clocks, || { + format!("writing {} bytes", transformed_data.len()) + }); w.write(transformed_data, local_time, pts, pkt.is_key())?; rotate = Some(r); } @@ -210,17 +255,17 @@ impl<'a, C, S> Streamer<'a, C, S> where C: 'a + Clocks + Clone, S: 'a + stream:: #[cfg(test)] mod tests { - use base::clock::{self, Clocks}; use crate::h264; use crate::stream::{self, Opener, Stream}; - use db::{CompositeId, recording, testutil}; - use failure::{Error, bail}; + use base::clock::{self, Clocks}; + use db::{recording, testutil, CompositeId}; + use failure::{bail, Error}; use log::trace; use parking_lot::Mutex; use std::cmp; use std::convert::TryFrom; - use std::sync::Arc; use std::sync::atomic::{AtomicBool, Ordering}; + use std::sync::Arc; use time; struct ProxyingStream<'a> { @@ -234,8 +279,11 @@ mod tests { } impl<'a> ProxyingStream<'a> { - fn new(clocks: &'a clock::SimulatedClocks, buffered: time::Duration, - inner: stream::FfmpegStream) -> ProxyingStream { + fn new( + clocks: &'a clock::SimulatedClocks, + buffered: time::Duration, + inner: stream::FfmpegStream, + ) -> ProxyingStream { clocks.sleep(buffered); ProxyingStream { clocks: clocks, @@ -265,7 +313,8 @@ mod tests { { let goal = pkt.pts().unwrap() + pkt.duration() as i64; let goal = time::Duration::nanoseconds( - goal * 1_000_000_000 / recording::TIME_UNITS_PER_SEC); + goal * 1_000_000_000 / recording::TIME_UNITS_PER_SEC, + ); let duration = goal - self.slept; let buf_part = cmp::min(self.buffered, duration); self.buffered = self.buffered - buf_part; @@ -293,7 +342,9 @@ mod tests { self.inner.get_video_codecpar() } - fn get_extra_data(&self) -> Result { self.inner.get_extra_data() } + fn get_extra_data(&self) -> Result { + self.inner.get_extra_data() + } } struct MockOpener<'a> { @@ -305,7 +356,7 @@ mod tests { impl<'a> stream::Opener> for MockOpener<'a> { fn open(&self, src: stream::Source) -> Result, Error> { match src { - stream::Source::Rtsp{url, ..} => assert_eq!(url, &self.expected_url), + stream::Source::Rtsp { url, .. } => assert_eq!(url, &self.expected_url), stream::Source::File(_) => panic!("expected rtsp url"), }; let mut l = self.streams.lock(); @@ -313,12 +364,12 @@ mod tests { Some(stream) => { trace!("MockOpener returning next stream"); Ok(stream) - }, + } None => { trace!("MockOpener shutting down"); self.shutdown.store(true, Ordering::SeqCst); bail!("done") - }, + } } } } @@ -335,14 +386,15 @@ mod tests { let mut it = recording::SampleIndexIterator::new(); let mut frames = Vec::new(); while it.next(&rec.video_index).unwrap() { - frames.push(Frame{ + frames.push(Frame { start_90k: it.start_90k, duration_90k: it.duration_90k, is_key: it.is_key(), }); } Ok(frames) - }).unwrap() + }) + .unwrap() } #[test] @@ -350,14 +402,16 @@ mod tests { testutil::init(); // 2015-04-25 00:00:00 UTC let clocks = clock::SimulatedClocks::new(time::Timespec::new(1429920000, 0)); - clocks.sleep(time::Duration::seconds(86400)); // to 2015-04-26 00:00:00 UTC + clocks.sleep(time::Duration::seconds(86400)); // to 2015-04-26 00:00:00 UTC - let stream = stream::FFMPEG.open(stream::Source::File("src/testdata/clip.mp4")).unwrap(); + let stream = stream::FFMPEG + .open(stream::Source::File("src/testdata/clip.mp4")) + .unwrap(); let mut stream = ProxyingStream::new(&clocks, time::Duration::seconds(2), stream); - stream.ts_offset = 123456; // starting pts of the input should be irrelevant + stream.ts_offset = 123456; // starting pts of the input should be irrelevant stream.ts_offset_pkts_left = u32::max_value(); stream.pkts_left = u32::max_value(); - let opener = MockOpener{ + let opener = MockOpener { expected_url: "rtsp://foo:bar@test-camera/main".to_owned(), streams: Mutex::new(vec![stream]), shutdown: Arc::new(AtomicBool::new(false)), @@ -373,9 +427,23 @@ mod tests { let l = db.db.lock(); let camera = l.cameras_by_id().get(&testutil::TEST_CAMERA_ID).unwrap(); let s = l.streams_by_id().get(&testutil::TEST_STREAM_ID).unwrap(); - let dir = db.dirs_by_stream_id.get(&testutil::TEST_STREAM_ID).unwrap().clone(); - stream = super::Streamer::new(&env, dir, db.syncer_channel.clone(), - testutil::TEST_STREAM_ID, camera, s, 0, 3, None).unwrap(); + let dir = db + .dirs_by_stream_id + .get(&testutil::TEST_STREAM_ID) + .unwrap() + .clone(); + stream = super::Streamer::new( + &env, + dir, + db.syncer_channel.clone(), + testutil::TEST_STREAM_ID, + camera, + s, + 0, + 3, + None, + ) + .unwrap(); } stream.run(); assert!(opener.streams.lock().is_empty()); @@ -386,25 +454,28 @@ mod tests { // 3-second boundaries (such as 2016-04-26 00:00:03), rotation happens somewhat later: // * the first rotation is always skipped // * the second rotation is deferred until a key frame. + #[rustfmt::skip] assert_eq!(get_frames(&db, CompositeId::new(testutil::TEST_STREAM_ID, 0)), &[ - Frame{start_90k: 0, duration_90k: 90379, is_key: true}, - Frame{start_90k: 90379, duration_90k: 89884, is_key: false}, - Frame{start_90k: 180263, duration_90k: 89749, is_key: false}, - Frame{start_90k: 270012, duration_90k: 89981, is_key: false}, // pts_time 3.0001... - Frame{start_90k: 359993, duration_90k: 90055, is_key: true}, - Frame{start_90k: 450048, duration_90k: 89967, is_key: false}, - Frame{start_90k: 540015, duration_90k: 90021, is_key: false}, // pts_time 6.0001... - Frame{start_90k: 630036, duration_90k: 89958, is_key: false}, + Frame { start_90k: 0, duration_90k: 90379, is_key: true }, + Frame { start_90k: 90379, duration_90k: 89884, is_key: false }, + Frame { start_90k: 180263, duration_90k: 89749, is_key: false }, + Frame { start_90k: 270012, duration_90k: 89981, is_key: false }, // pts_time 3.0001... + Frame { start_90k: 359993, duration_90k: 90055, is_key: true }, + Frame { start_90k: 450048, duration_90k: 89967, is_key: false }, + Frame { start_90k: 540015, duration_90k: 90021, is_key: false }, // pts_time 6.0001... + Frame { start_90k: 630036, duration_90k: 89958, is_key: false }, ]); + #[rustfmt::skip] assert_eq!(get_frames(&db, CompositeId::new(testutil::TEST_STREAM_ID, 1)), &[ - Frame{start_90k: 0, duration_90k: 90011, is_key: true}, - Frame{start_90k: 90011, duration_90k: 0, is_key: false}, + Frame { start_90k: 0, duration_90k: 90011, is_key: true }, + Frame { start_90k: 90011, duration_90k: 0, is_key: false }, ]); let mut recordings = Vec::new(); db.list_recordings_by_id(testutil::TEST_STREAM_ID, 0..2, &mut |r| { recordings.push(r); Ok(()) - }).unwrap(); + }) + .unwrap(); assert_eq!(2, recordings.len()); assert_eq!(0, recordings[0].id.recording()); assert_eq!(recording::Time(128700575999999), recordings[0].start); diff --git a/server/src/web.rs b/server/src/web.rs index 4af41c9..cdc71e6 100644 --- a/server/src/web.rs +++ b/server/src/web.rs @@ -28,32 +28,32 @@ // You should have received a copy of the GNU General Public License // along with this program. If not, see . -use base::clock::Clocks; -use base::{ErrorKind, bail_t}; -use hyper::body::Bytes; use crate::body::Body; use crate::json; use crate::mp4; +use base::clock::Clocks; +use base::{bail_t, ErrorKind}; use bytes::{BufMut, BytesMut}; use core::borrow::Borrow; use core::str::FromStr; -use db::{auth, recording}; use db::dir::SampleFileDir; -use failure::{Error, bail, format_err}; +use db::{auth, recording}; +use failure::{bail, format_err, Error}; use fnv::FnvHashMap; use futures::sink::SinkExt; use futures::stream::StreamExt; -use http::{Request, Response, status::StatusCode}; use http::header::{self, HeaderValue}; +use http::{status::StatusCode, Request, Response}; use http_serve::dir::FsDir; +use hyper::body::Bytes; use log::{debug, info, warn}; use memchr::memchr; -use nom::IResult; -use nom::bytes::complete::{take_while1, tag}; +use nom::bytes::complete::{tag, take_while1}; use nom::combinator::{all_consuming, map, map_res, opt}; use nom::sequence::{preceded, tuple}; -use std::convert::TryFrom; +use nom::IResult; use std::cmp; +use std::convert::TryFrom; use std::net::IpAddr; use std::ops::Range; use std::sync::Arc; @@ -92,11 +92,11 @@ impl Path { "/logout" => return Path::Logout, "/request" => return Path::Request, "/signals" => return Path::Signals, - _ => {}, + _ => {} }; if path.starts_with("/init/") { let (debug, path) = if path.ends_with(".txt") { - (true, &path[0 .. path.len() - 4]) + (true, &path[0..path.len() - 4]) } else { (false, path) }; @@ -105,7 +105,7 @@ impl Path { } let id_start = "/init/".len(); let id_end = path.len() - ".mp4".len(); - if let Ok(id) = i32::from_str(&path[id_start .. id_end]) { + if let Ok(id) = i32::from_str(&path[id_start..id_end]) { return Path::InitSegment(id, debug); } return Path::NotFound; @@ -115,16 +115,18 @@ impl Path { } let path = &path["/cameras/".len()..]; let slash = match path.find('/') { - None => { return Path::NotFound; }, + None => { + return Path::NotFound; + } Some(s) => s, }; - let uuid = &path[0 .. slash]; - let path = &path[slash+1 .. ]; + let uuid = &path[0..slash]; + let path = &path[slash + 1..]; // TODO(slamb): require uuid to be in canonical format. let uuid = match Uuid::parse_str(uuid) { Ok(u) => u, - Err(_) => { return Path::NotFound }, + Err(_) => return Path::NotFound, }; if path.is_empty() { @@ -132,13 +134,17 @@ impl Path { } let slash = match path.find('/') { - None => { return Path::NotFound; }, + None => { + return Path::NotFound; + } Some(s) => s, }; let (type_, path) = path.split_at(slash); let type_ = match db::StreamType::parse(type_) { - None => { return Path::NotFound; }, + None => { + return Path::NotFound; + } Some(t) => t, }; match path { @@ -157,7 +163,8 @@ fn plain_response>(status: http::StatusCode, body: B) -> Response< Response::builder() .status(status) .header(header::CONTENT_TYPE, HeaderValue::from_static("text/plain")) - .body(body.into()).expect("hardcoded head should be valid") + .body(body.into()) + .expect("hardcoded head should be valid") } fn not_found>(body: B) -> Response { @@ -200,21 +207,32 @@ impl Segments { fn parse(i: &str) -> IResult<&str, Segments> { // Parse START_ID[-END_ID] into Range. // Note that END_ID is inclusive, but Ranges are half-open. - let (i, ids) = map(tuple((num::(), opt(preceded(tag("-"), num::())))), - |(start, end)| start .. end.unwrap_or(start) + 1)(i)?; + let (i, ids) = map( + tuple((num::(), opt(preceded(tag("-"), num::())))), + |(start, end)| start..end.unwrap_or(start) + 1, + )(i)?; // Parse [@OPEN_ID] into Option. let (i, open_id) = opt(preceded(tag("@"), num::()))(i)?; // Parse [.[REL_START_TIME]-[REL_END_TIME]] into (i64, Option). let (i, (start_time, end_time)) = map( - opt(preceded(tag("."), tuple((opt(num::()), tag("-"), opt(num::()))))), - |t| { - t.map(|(s, _, e)| (s.unwrap_or(0), e)) - .unwrap_or((0, None)) - })(i)?; + opt(preceded( + tag("."), + tuple((opt(num::()), tag("-"), opt(num::()))), + )), + |t| t.map(|(s, _, e)| (s.unwrap_or(0), e)).unwrap_or((0, None)), + )(i)?; - Ok((i, Segments { ids, open_id, start_time, end_time, })) + Ok(( + i, + Segments { + ids, + open_id, + start_time, + end_time, + }, + )) } } @@ -244,8 +262,10 @@ type ResponseResult = Result, Response>; fn serve_json(req: &Request, out: &T) -> ResponseResult { let (mut resp, writer) = http_serve::streaming_body(&req).build(); - resp.headers_mut().insert(header::CONTENT_TYPE, - HeaderValue::from_static("application/json")); + resp.headers_mut().insert( + header::CONTENT_TYPE, + HeaderValue::from_static("application/json"), + ); if let Some(mut w) = writer { serde_json::to_writer(&mut w, out).map_err(internal_server_err)?; } @@ -285,18 +305,22 @@ fn extract_sid(req: &Request) -> Option { /// that borrows from the bytes. async fn extract_json_body(req: &mut Request) -> Result> { if *req.method() != http::method::Method::POST { - return Err(plain_response(StatusCode::METHOD_NOT_ALLOWED, "POST expected")); + return Err(plain_response( + StatusCode::METHOD_NOT_ALLOWED, + "POST expected", + )); } let correct_mime_type = match req.headers().get(header::CONTENT_TYPE) { Some(t) if t == "application/json" => true, Some(t) if t == "application/json; charset=UTF-8" => true, - _ => false, + _ => false, }; if !correct_mime_type { return Err(bad_req("expected application/json request body")); } let b = ::std::mem::replace(req.body_mut(), hyper::Body::empty()); - hyper::body::to_bytes(b).await + hyper::body::to_bytes(b) + .await .map_err(|e| internal_server_err(format_err!("unable to read request body: {}", e))) } @@ -337,9 +361,12 @@ impl Service { if let Some(d) = config.ui_dir { match FsDir::builder().for_path(&d) { Err(e) => { - warn!("Unable to load --ui-dir={}; will serve no static files: {}", - d.display(), e); - }, + warn!( + "Unable to load --ui-dir={}; will serve no static files: {}", + d.display(), + e + ); + } Ok(d) => ui_dir = Some(d), }; } @@ -352,10 +379,7 @@ impl Service { Some(d) => d, None => continue, }; - d.insert(id, l.sample_file_dirs_by_id() - .get(&dir_id) - .unwrap() - .get()?); + d.insert(id, l.sample_file_dirs_by_id().get(&dir_id).unwrap().get()?); } Arc::new(d) }; @@ -370,10 +394,18 @@ impl Service { }) } - fn stream_live_m4s(self: Arc, req: Request<::hyper::Body>, caller: Caller, uuid: Uuid, - stream_type: db::StreamType) -> ResponseResult { + fn stream_live_m4s( + self: Arc, + req: Request<::hyper::Body>, + caller: Caller, + uuid: Uuid, + stream_type: db::StreamType, + ) -> ResponseResult { if !caller.permissions.view_video { - return Err(plain_response(StatusCode::UNAUTHORIZED, "view_video required")); + return Err(plain_response( + StatusCode::UNAUTHORIZED, + "view_video required", + )); } let stream_id; @@ -382,26 +414,33 @@ impl Service { { let mut db = self.db.lock(); open_id = match db.open { - None => return Err(plain_response( + None => { + return Err(plain_response( StatusCode::PRECONDITION_FAILED, - "database is read-only; there are no live streams")), + "database is read-only; there are no live streams", + )) + } Some(o) => o.id, }; - let camera = db.get_camera(uuid) - .ok_or_else(|| plain_response(StatusCode::NOT_FOUND, - format!("no such camera {}", uuid)))?; - stream_id = camera.streams[stream_type.index()] - .ok_or_else(|| plain_response(StatusCode::NOT_FOUND, - format!("no such stream {}/{}", uuid, - stream_type)))?; - db.watch_live(stream_id, Box::new(move |l| sub_tx.unbounded_send(l).is_ok())) - .expect("stream_id refed by camera"); + let camera = db.get_camera(uuid).ok_or_else(|| { + plain_response(StatusCode::NOT_FOUND, format!("no such camera {}", uuid)) + })?; + stream_id = camera.streams[stream_type.index()].ok_or_else(|| { + plain_response( + StatusCode::NOT_FOUND, + format!("no such stream {}/{}", uuid, stream_type), + ) + })?; + db.watch_live( + stream_id, + Box::new(move |l| sub_tx.unbounded_send(l).is_ok()), + ) + .expect("stream_id refed by camera"); } let response = - tungstenite::handshake::server::create_response_with_body( - &req, hyper::Body::empty) - .map_err(|e| bad_req(e.to_string()))?; + tungstenite::handshake::server::create_response_with_body(&req, hyper::Body::empty) + .map_err(|e| bad_req(e.to_string()))?; let (parts, _) = response.into_parts(); tokio::spawn(self.stream_live_m4s_ws(stream_id, open_id, req, sub_rx)); @@ -410,20 +449,25 @@ impl Service { } async fn stream_live_m4s_ws( - self: Arc, stream_id: i32, open_id: u32, req: hyper::Request, - mut sub_rx: futures::channel::mpsc::UnboundedReceiver) { + self: Arc, + stream_id: i32, + open_id: u32, + req: hyper::Request, + mut sub_rx: futures::channel::mpsc::UnboundedReceiver, + ) { let upgraded = match hyper::upgrade::on(req).await { Ok(u) => u, Err(e) => { warn!("Unable to upgrade stream to websocket: {}", e); return; - }, + } }; let mut ws = tokio_tungstenite::WebSocketStream::from_raw_socket( upgraded, tungstenite::protocol::Role::Server, None, - ).await; + ) + .await; // Start the first segment at a key frame to reduce startup latency. let mut start_at_key = true; @@ -434,8 +478,10 @@ impl Service { }; info!("chunk: is_key={:?}", live.is_key); - if let Err(e) = self.stream_live_m4s_chunk(open_id, stream_id, &mut ws, live, - start_at_key).await { + if let Err(e) = self + .stream_live_m4s_chunk(open_id, stream_id, &mut ws, live, start_at_key) + .await + { info!("Dropping WebSocket after error: {}", e); return; } @@ -444,15 +490,19 @@ impl Service { } async fn stream_live_m4s_chunk( - &self, open_id: u32, stream_id: i32, + &self, + open_id: u32, + stream_id: i32, ws: &mut tokio_tungstenite::WebSocketStream, - live: db::LiveSegment, start_at_key: bool) -> Result<(), Error> { + live: db::LiveSegment, + start_at_key: bool, + ) -> Result<(), Error> { let mut builder = mp4::FileBuilder::new(mp4::Type::MediaSegment); let mut row = None; { let db = self.db.lock(); let mut rows = 0; - db.list_recordings_by_id(stream_id, live.recording .. live.recording+1, &mut |r| { + db.list_recordings_by_id(stream_id, live.recording..live.recording + 1, &mut |r| { rows += 1; row = Some(r); builder.append(&db, r, live.media_off_90k.clone(), start_at_key)?; @@ -485,7 +535,8 @@ impl Service { live.media_off_90k.end, prev_media_duration.0, prev_runs + if row.run_offset == 0 { 1 } else { 0 }, - &row.video_sample_entry_id); + &row.video_sample_entry_id + ); let mut v = hdr.into_bytes(); mp4.append_into_vec(&mut v).await?; ws.send(tungstenite::Message::Binary(v)).await?; @@ -497,56 +548,74 @@ impl Service { match *req.method() { Method::POST => self.post_signals(req, caller).await, Method::GET | Method::HEAD => self.get_signals(&req), - _ => Err(plain_response(StatusCode::METHOD_NOT_ALLOWED, - "POST, GET, or HEAD expected")), + _ => Err(plain_response( + StatusCode::METHOD_NOT_ALLOWED, + "POST, GET, or HEAD expected", + )), } } - async fn serve_inner(self: Arc, req: Request<::hyper::Body>, p: Path, caller: Caller) - -> ResponseResult { + async fn serve_inner( + self: Arc, + req: Request<::hyper::Body>, + p: Path, + caller: Caller, + ) -> ResponseResult { let (cache, mut response) = match p { - Path::InitSegment(sha1, debug) => { - (CacheControl::PrivateStatic, self.init_segment(sha1, debug, &req)?) - }, + Path::InitSegment(sha1, debug) => ( + CacheControl::PrivateStatic, + self.init_segment(sha1, debug, &req)?, + ), Path::TopLevel => (CacheControl::PrivateDynamic, self.top_level(&req, caller)?), Path::Request => (CacheControl::PrivateDynamic, self.request(&req)?), Path::Camera(uuid) => (CacheControl::PrivateDynamic, self.camera(&req, uuid)?), - Path::StreamRecordings(uuid, type_) => { - (CacheControl::PrivateDynamic, self.stream_recordings(&req, uuid, type_)?) - }, - Path::StreamViewMp4(uuid, type_, debug) => { - (CacheControl::PrivateStatic, - self.stream_view_mp4(&req, caller, uuid, type_, mp4::Type::Normal, debug)?) - }, - Path::StreamViewMp4Segment(uuid, type_, debug) => { - (CacheControl::PrivateStatic, - self.stream_view_mp4(&req, caller, uuid, type_, mp4::Type::MediaSegment, debug)?) - }, - Path::StreamLiveMp4Segments(uuid, type_) => { - (CacheControl::PrivateDynamic, self.stream_live_m4s(req, caller, uuid, type_)?) - }, + Path::StreamRecordings(uuid, type_) => ( + CacheControl::PrivateDynamic, + self.stream_recordings(&req, uuid, type_)?, + ), + Path::StreamViewMp4(uuid, type_, debug) => ( + CacheControl::PrivateStatic, + self.stream_view_mp4(&req, caller, uuid, type_, mp4::Type::Normal, debug)?, + ), + Path::StreamViewMp4Segment(uuid, type_, debug) => ( + CacheControl::PrivateStatic, + self.stream_view_mp4(&req, caller, uuid, type_, mp4::Type::MediaSegment, debug)?, + ), + Path::StreamLiveMp4Segments(uuid, type_) => ( + CacheControl::PrivateDynamic, + self.stream_live_m4s(req, caller, uuid, type_)?, + ), Path::NotFound => return Err(not_found("path not understood")), Path::Login => (CacheControl::PrivateDynamic, self.login(req).await?), Path::Logout => (CacheControl::PrivateDynamic, self.logout(req).await?), - Path::Signals => (CacheControl::PrivateDynamic, self.signals(req, caller).await?), - Path::Static => (CacheControl::None, self.static_file(req).await?) + Path::Signals => ( + CacheControl::PrivateDynamic, + self.signals(req, caller).await?, + ), + Path::Static => (CacheControl::None, self.static_file(req).await?), }; match cache { CacheControl::PrivateStatic => { - response.headers_mut().insert(header::CACHE_CONTROL, - HeaderValue::from_static("private, max-age=3600")); - }, + response.headers_mut().insert( + header::CACHE_CONTROL, + HeaderValue::from_static("private, max-age=3600"), + ); + } CacheControl::PrivateDynamic => { - response.headers_mut().insert(header::CACHE_CONTROL, - HeaderValue::from_static("private, no-cache")); - }, - CacheControl::None => {}, + response.headers_mut().insert( + header::CACHE_CONTROL, + HeaderValue::from_static("private, no-cache"), + ); + } + CacheControl::None => {} } Ok(response) } - pub async fn serve(self: Arc, req: Request<::hyper::Body>) - -> Result, std::convert::Infallible> { + pub async fn serve( + self: Arc, + req: Request<::hyper::Body>, + ) -> Result, std::convert::Infallible> { let p = Path::decode(req.uri().path()); let always_allow_unauthenticated = match p { Path::NotFound | Path::Request | Path::Login | Path::Logout | Path::Static => true, @@ -569,39 +638,52 @@ impl Service { match key { "days" => days = value == "true", "cameraConfigs" => camera_configs = value == "true", - _ => {}, + _ => {} }; } } if camera_configs { if !caller.permissions.read_camera_configs { - return Err(plain_response(StatusCode::UNAUTHORIZED, - "read_camera_configs required")); + return Err(plain_response( + StatusCode::UNAUTHORIZED, + "read_camera_configs required", + )); } } let db = self.db.lock(); - serve_json(req, &json::TopLevel { - time_zone_name: &self.time_zone_name, - cameras: (&db, days, camera_configs), - session: caller.session, - signals: (&db, days), - signal_types: &db, - }) + serve_json( + req, + &json::TopLevel { + time_zone_name: &self.time_zone_name, + cameras: (&db, days, camera_configs), + session: caller.session, + signals: (&db, days), + signal_types: &db, + }, + ) } fn camera(&self, req: &Request<::hyper::Body>, uuid: Uuid) -> ResponseResult { let db = self.db.lock(); - let camera = db.get_camera(uuid) - .ok_or_else(|| not_found(format!("no such camera {}", uuid)))?; - serve_json(req, &json::Camera::wrap(camera, &db, true, false).map_err(internal_server_err)?) + let camera = db + .get_camera(uuid) + .ok_or_else(|| not_found(format!("no such camera {}", uuid)))?; + serve_json( + req, + &json::Camera::wrap(camera, &db, true, false).map_err(internal_server_err)?, + ) } - fn stream_recordings(&self, req: &Request<::hyper::Body>, uuid: Uuid, type_: db::StreamType) - -> ResponseResult { + fn stream_recordings( + &self, + req: &Request<::hyper::Body>, + uuid: Uuid, + type_: db::StreamType, + ) -> ResponseResult { let (r, split) = { - let mut time = recording::Time::min_value() .. recording::Time::max_value(); + let mut time = recording::Time::min_value()..recording::Time::max_value(); let mut split = recording::Duration(i64::max_value()); if let Some(q) = req.uri().query() { for (key, value) in form_urlencoded::parse(q.as_bytes()) { @@ -610,18 +692,20 @@ impl Service { "startTime90k" => { time.start = recording::Time::parse(value) .map_err(|_| bad_req("unparseable startTime90k"))? - }, + } "endTime90k" => { time.end = recording::Time::parse(value) .map_err(|_| bad_req("unparseable endTime90k"))? - }, + } "split90k" => { - split = recording::Duration(i64::from_str(value) - .map_err(|_| bad_req("unparseable split90k"))?) - }, - _ => {}, + split = recording::Duration( + i64::from_str(value) + .map_err(|_| bad_req("unparseable split90k"))?, + ) + } + _ => {} } - }; + } } (time, split) }; @@ -630,17 +714,24 @@ impl Service { recordings: Vec::new(), video_sample_entries: (&db, Vec::new()), }; - let camera = db.get_camera(uuid) - .ok_or_else(|| plain_response(StatusCode::NOT_FOUND, - format!("no such camera {}", uuid)))?; - let stream_id = camera.streams[type_.index()] - .ok_or_else(|| plain_response(StatusCode::NOT_FOUND, - format!("no such stream {}/{}", uuid, type_)))?; + let camera = db.get_camera(uuid).ok_or_else(|| { + plain_response(StatusCode::NOT_FOUND, format!("no such camera {}", uuid)) + })?; + let stream_id = camera.streams[type_.index()].ok_or_else(|| { + plain_response( + StatusCode::NOT_FOUND, + format!("no such stream {}/{}", uuid, type_), + ) + })?; db.list_aggregated_recordings(stream_id, r, split, &mut |row| { - let end = row.ids.end - 1; // in api, ids are inclusive. + let end = row.ids.end - 1; // in api, ids are inclusive. out.recordings.push(json::Recording { start_id: row.ids.start, - end_id: if end == row.ids.start { None } else { Some(end) }, + end_id: if end == row.ids.start { + None + } else { + Some(end) + }, start_time_90k: row.time.start.0, end_time_90k: row.time.end.0, sample_file_bytes: row.sample_file_bytes, @@ -650,22 +741,29 @@ impl Service { video_sample_entry_id: row.video_sample_entry_id.to_string(), growing: row.growing, }); - if !out.video_sample_entries.1.contains(&row.video_sample_entry_id) { + if !out + .video_sample_entries + .1 + .contains(&row.video_sample_entry_id) + { out.video_sample_entries.1.push(row.video_sample_entry_id); } Ok(()) - }).map_err(internal_server_err)?; + }) + .map_err(internal_server_err)?; serve_json(req, &out) } - fn init_segment(&self, id: i32, debug: bool, req: &Request<::hyper::Body>) - -> ResponseResult { + fn init_segment(&self, id: i32, debug: bool, req: &Request<::hyper::Body>) -> ResponseResult { let mut builder = mp4::FileBuilder::new(mp4::Type::InitSegment); let db = self.db.lock(); - let ent = db.video_sample_entries_by_id().get(&id) + let ent = db + .video_sample_entries_by_id() + .get(&id) .ok_or_else(|| not_found("not such init segment"))?; builder.append_video_sample_entry(ent.clone()); - let mp4 = builder.build(self.db.clone(), self.dirs_by_stream_id.clone()) + let mp4 = builder + .build(self.db.clone(), self.dirs_by_stream_id.clone()) .map_err(from_base_error)?; if debug { Ok(plain_response(StatusCode::OK, format!("{:#?}", mp4))) @@ -674,24 +772,34 @@ impl Service { } } - fn stream_view_mp4(&self, req: &Request<::hyper::Body>, caller: Caller, uuid: Uuid, - stream_type: db::StreamType, mp4_type: mp4::Type, debug: bool) - -> ResponseResult { + fn stream_view_mp4( + &self, + req: &Request<::hyper::Body>, + caller: Caller, + uuid: Uuid, + stream_type: db::StreamType, + mp4_type: mp4::Type, + debug: bool, + ) -> ResponseResult { if !caller.permissions.view_video { - return Err(plain_response(StatusCode::UNAUTHORIZED, "view_video required")); + return Err(plain_response( + StatusCode::UNAUTHORIZED, + "view_video required", + )); } let (stream_id, camera_name); { let db = self.db.lock(); - let camera = db.get_camera(uuid) - .ok_or_else(|| plain_response(StatusCode::NOT_FOUND, - format!("no such camera {}", uuid)))?; + let camera = db.get_camera(uuid).ok_or_else(|| { + plain_response(StatusCode::NOT_FOUND, format!("no such camera {}", uuid)) + })?; camera_name = camera.short_name.clone(); - stream_id = camera.streams[stream_type.index()] - .ok_or_else(|| plain_response(StatusCode::NOT_FOUND, - format!("no such stream {}/{}", uuid, - stream_type)))?; - + stream_id = camera.streams[stream_type.index()].ok_or_else(|| { + plain_response( + StatusCode::NOT_FOUND, + format!("no such stream {}/{}", uuid, stream_type), + ) + })?; }; let mut start_time_for_filename = None; let mut builder = mp4::FileBuilder::new(mp4_type); @@ -700,9 +808,12 @@ impl Service { let (key, value) = (key.borrow(), value.borrow()); match key { "s" => { - let s = Segments::from_str(value).map_err( - |()| plain_response(StatusCode::BAD_REQUEST, - format!("invalid s parameter: {}", value)))?; + let s = Segments::from_str(value).map_err(|()| { + plain_response( + StatusCode::BAD_REQUEST, + format!("invalid s parameter: {}", value), + ) + })?; debug!("stream_view_mp4: appending s={:?}", s); let mut est_segments = usize::try_from(s.ids.end - s.ids.start).unwrap(); if let Some(end) = s.end_time { @@ -711,9 +822,10 @@ impl Service { // there are no gaps or overlap, possibly another for misalignment of // the requested timespan with the rotate offset and another because // rotation only happens at key frames. - let ceil_durations = (end - s.start_time + - recording::DESIRED_RECORDING_WALL_DURATION - 1) / - recording::DESIRED_RECORDING_WALL_DURATION; + let ceil_durations = (end - s.start_time + + recording::DESIRED_RECORDING_WALL_DURATION + - 1) + / recording::DESIRED_RECORDING_WALL_DURATION; est_segments = cmp::min(est_segments, (ceil_durations + 2) as usize); } builder.reserve(est_segments); @@ -725,19 +837,23 @@ impl Service { if let Some(o) = s.open_id { if r.open_id != o { - bail!("recording {} has open id {}, requested {}", - r.id, r.open_id, o); + bail!( + "recording {} has open id {}, requested {}", + r.id, + r.open_id, + o + ); } } // Check for missing recordings. match prev { - None if recording_id == s.ids.start => {}, + None if recording_id == s.ids.start => {} None => bail!("no such recording {}/{}", stream_id, s.ids.start), Some(id) if r.id.recording() != id + 1 => { bail!("no such recording {}/{}", stream_id, id + 1); - }, - _ => {}, + } + _ => {} }; prev = Some(recording_id); @@ -748,63 +864,94 @@ impl Service { if s.start_time <= cur_off + wd && cur_off < end_time { let start = cmp::max(0, s.start_time - cur_off); let end = cmp::min(wd, end_time - cur_off); - let wr = i32::try_from(start).unwrap() .. - i32::try_from(end).unwrap(); - debug!("...appending recording {} with wall duration {:?} \ - (out of total {})", r.id, wr, wd); + let wr = i32::try_from(start).unwrap()..i32::try_from(end).unwrap(); + debug!( + "...appending recording {} with wall duration {:?} \ + (out of total {})", + r.id, wr, wd + ); if start_time_for_filename.is_none() { start_time_for_filename = Some(r.start + recording::Duration(start)); } use recording::rescale; let mr = - rescale(wr.start, r.wall_duration_90k, r.media_duration_90k) .. - rescale(wr.end, r.wall_duration_90k, r.media_duration_90k); + rescale(wr.start, r.wall_duration_90k, r.media_duration_90k) + ..rescale( + wr.end, + r.wall_duration_90k, + r.media_duration_90k, + ); builder.append(&db, r, mr, true)?; } else { debug!("...skipping recording {} wall dur {}", r.id, wd); } cur_off += wd; Ok(()) - }).map_err(internal_server_err)?; + }) + .map_err(internal_server_err)?; // Check for missing recordings. match prev { Some(id) if s.ids.end != id + 1 => { - return Err(not_found(format!("no such recording {}/{}", - stream_id, s.ids.end - 1))); - }, + return Err(not_found(format!( + "no such recording {}/{}", + stream_id, + s.ids.end - 1 + ))); + } None => { - return Err(not_found(format!("no such recording {}/{}", - stream_id, s.ids.start))); - }, - _ => {}, + return Err(not_found(format!( + "no such recording {}/{}", + stream_id, s.ids.start + ))); + } + _ => {} }; if let Some(end) = s.end_time { if end > cur_off { return Err(plain_response( - StatusCode::BAD_REQUEST, - format!("end time {} is beyond specified recordings", - end))); + StatusCode::BAD_REQUEST, + format!("end time {} is beyond specified recordings", end), + )); } } - }, - "ts" => builder.include_timestamp_subtitle_track(value == "true") - .map_err(from_base_error)?, + } + "ts" => builder + .include_timestamp_subtitle_track(value == "true") + .map_err(from_base_error)?, _ => return Err(bad_req(format!("parameter {} not understood", key))), } - }; + } } if let Some(start) = start_time_for_filename { - let tm = time::at(time::Timespec{sec: start.unix_seconds(), nsec: 0}); - let stream_abbrev = if stream_type == db::StreamType::MAIN { "main" } else { "sub" }; - let suffix = if mp4_type == mp4::Type::Normal { "mp4" } else { "m4s" }; - builder.set_filename(&format!("{}-{}-{}.{}", tm.strftime("%Y%m%d%H%M%S").unwrap(), - camera_name, stream_abbrev, suffix)) + let tm = time::at(time::Timespec { + sec: start.unix_seconds(), + nsec: 0, + }); + let stream_abbrev = if stream_type == db::StreamType::MAIN { + "main" + } else { + "sub" + }; + let suffix = if mp4_type == mp4::Type::Normal { + "mp4" + } else { + "m4s" + }; + builder + .set_filename(&format!( + "{}-{}-{}.{}", + tm.strftime("%Y%m%d%H%M%S").unwrap(), + camera_name, + stream_abbrev, + suffix + )) .map_err(from_base_error)?; } - let mp4 = builder.build(self.db.clone(), self.dirs_by_stream_id.clone()) - .map_err(from_base_error)?; + let mp4 = builder + .build(self.db.clone(), self.dirs_by_stream_id.clone()) + .map_err(from_base_error)?; if debug { return Ok(plain_response(StatusCode::OK, format!("{:#?}", mp4))); } @@ -812,28 +959,37 @@ impl Service { } async fn static_file(&self, req: Request) -> ResponseResult { - let dir = self.ui_dir.clone() + let dir = self + .ui_dir + .clone() .ok_or_else(|| not_found("--ui-dir not configured; no static files available."))?; let static_req = match StaticFileRequest::parse(req.uri().path()) { None => return Err(not_found("static file not found")), Some(r) => r, }; let f = dir.get(static_req.path, req.headers()); - let node = f.await - .map_err(|e| if e.kind() == std::io::ErrorKind::NotFound { + let node = f.await.map_err(|e| { + if e.kind() == std::io::ErrorKind::NotFound { not_found("no such static file") } else { internal_server_err(e) - })?; + } + })?; let mut hdrs = http::HeaderMap::new(); node.add_encoding_headers(&mut hdrs); - hdrs.insert(header::CACHE_CONTROL, HeaderValue::from_static(if static_req.immutable { - // https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Cache-Control#Caching_static_assets - "public, max-age=604800, immutable" - } else { - "public" - })); - hdrs.insert(header::CONTENT_TYPE, HeaderValue::from_static(static_req.mime)); + hdrs.insert( + header::CACHE_CONTROL, + HeaderValue::from_static(if static_req.immutable { + // https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Cache-Control#Caching_static_assets + "public, max-age=604800, immutable" + } else { + "public" + }), + ); + hdrs.insert( + header::CONTENT_TYPE, + HeaderValue::from_static(static_req.mime), + ); let e = node.into_file_entity(hdrs).map_err(internal_server_err)?; Ok(http_serve::serve(e, &req)) } @@ -842,60 +998,89 @@ impl Service { auth::Request { when_sec: Some(self.db.clocks().realtime().sec), addr: if self.trust_forward_hdrs { - req.headers().get("X-Real-IP") - .and_then(|v| v.to_str().ok()) - .and_then(|v| IpAddr::from_str(v).ok()) - } else { None }, - user_agent: req.headers().get(header::USER_AGENT).map(|ua| ua.as_bytes().to_vec()), + req.headers() + .get("X-Real-IP") + .and_then(|v| v.to_str().ok()) + .and_then(|v| IpAddr::from_str(v).ok()) + } else { + None + }, + user_agent: req + .headers() + .get(header::USER_AGENT) + .map(|ua| ua.as_bytes().to_vec()), } } fn request(&self, req: &Request<::hyper::Body>) -> ResponseResult { let authreq = self.authreq(req); - let host = req.headers().get(header::HOST).map(|h| String::from_utf8_lossy(h.as_bytes())); - let agent = authreq.user_agent.as_ref().map(|u| String::from_utf8_lossy(&u[..])); - Ok(plain_response(StatusCode::OK, format!( - "when: {}\n\ + let host = req + .headers() + .get(header::HOST) + .map(|h| String::from_utf8_lossy(h.as_bytes())); + let agent = authreq + .user_agent + .as_ref() + .map(|u| String::from_utf8_lossy(&u[..])); + Ok(plain_response( + StatusCode::OK, + format!( + "when: {}\n\ host: {:?}\n\ addr: {:?}\n\ user_agent: {:?}\n\ secure: {:?}", - time::at(time::Timespec{sec: authreq.when_sec.unwrap(), nsec: 0}) - .strftime("%FT%T") - .map(|f| f.to_string()) - .unwrap_or_else(|e| e.to_string()), - host.as_ref().map(|h| &*h), - &authreq.addr, - agent.as_ref().map(|a| &*a), - self.is_secure(req)))) + time::at(time::Timespec { + sec: authreq.when_sec.unwrap(), + nsec: 0 + }) + .strftime("%FT%T") + .map(|f| f.to_string()) + .unwrap_or_else(|e| e.to_string()), + host.as_ref().map(|h| &*h), + &authreq.addr, + agent.as_ref().map(|a| &*a), + self.is_secure(req) + ), + )) } fn is_secure(&self, req: &Request<::hyper::Body>) -> bool { - self.trust_forward_hdrs && - req.headers().get("X-Forwarded-Proto") - .map(|v| v.as_bytes() == b"https") - .unwrap_or(false) + self.trust_forward_hdrs + && req + .headers() + .get("X-Forwarded-Proto") + .map(|v| v.as_bytes() == b"https") + .unwrap_or(false) } async fn login(&self, mut req: Request<::hyper::Body>) -> ResponseResult { let r = extract_json_body(&mut req).await?; - let r: json::LoginRequest = serde_json::from_slice(&r) - .map_err(|e| bad_req(e.to_string()))?; + let r: json::LoginRequest = + serde_json::from_slice(&r).map_err(|e| bad_req(e.to_string()))?; let authreq = self.authreq(&req); - let host = req.headers().get(header::HOST).ok_or_else(|| bad_req("missing Host header!"))?; + let host = req + .headers() + .get(header::HOST) + .ok_or_else(|| bad_req("missing Host header!"))?; let host = host.as_bytes(); let domain = match memchr(b':', host) { Some(colon) => &host[0..colon], None => host, - }.to_owned(); + } + .to_owned(); let mut l = self.db.lock(); let is_secure = self.is_secure(&req); - let flags = (auth::SessionFlag::HttpOnly as i32) | - (auth::SessionFlag::SameSite as i32) | - (auth::SessionFlag::SameSiteStrict as i32) | - if is_secure { auth::SessionFlag::Secure as i32 } else { 0 }; - let (sid, _) = l.login_by_password(authreq, &r.username, r.password, Some(domain), - flags) + let flags = (auth::SessionFlag::HttpOnly as i32) + | (auth::SessionFlag::SameSite as i32) + | (auth::SessionFlag::SameSiteStrict as i32) + | if is_secure { + auth::SessionFlag::Secure as i32 + } else { + 0 + }; + let (sid, _) = l + .login_by_password(authreq, &r.username, r.password, Some(domain), flags) .map_err(|e| plain_response(StatusCode::UNAUTHORIZED, e.to_string()))?; let s_suffix = if is_secure { &b"; HttpOnly; Secure; SameSite=Strict; Max-Age=2147483648; Path=/"[..] @@ -909,16 +1094,20 @@ impl Service { cookie.put(&encoded[..]); cookie.put(s_suffix); Ok(Response::builder() - .header(header::SET_COOKIE, HeaderValue::from_maybe_shared(cookie.freeze()) - .expect("cookie can't have invalid bytes")) + .header( + header::SET_COOKIE, + HeaderValue::from_maybe_shared(cookie.freeze()) + .expect("cookie can't have invalid bytes"), + ) .status(StatusCode::NO_CONTENT) - .body(b""[..].into()).unwrap()) + .body(b""[..].into()) + .unwrap()) } async fn logout(&self, mut req: Request) -> ResponseResult { let r = extract_json_body(&mut req).await?; - let r: json::LogoutRequest = serde_json::from_slice(&r) - .map_err(|e| bad_req(e.to_string()))?; + let r: json::LogoutRequest = + serde_json::from_slice(&r).map_err(|e| bad_req(e.to_string()))?; let mut res = Response::new(b""[..].into()); if let Some(sid) = extract_sid(&req) { @@ -933,56 +1122,62 @@ impl Service { } info!("revoking session"); true - }, + } Err(e) => { // TODO: distinguish "no such session", "session is no longer valid", and // "user ... is disabled" (which are all client error / bad state) from database // errors. warn!("logout failed: {}", e); false - }, + } }; if need_revoke { // TODO: inline this above with non-lexical lifetimes. l.revoke_session(auth::RevocationReason::LoggedOut, None, authreq, &hash) - .map_err(internal_server_err)?; + .map_err(internal_server_err)?; } // By now the session is invalid (whether it was valid to start with or not). // Clear useless cookie. - res.headers_mut().append(header::SET_COOKIE, - HeaderValue::from_str("s=; Max-Age=0; Path=/").unwrap()); + res.headers_mut().append( + header::SET_COOKIE, + HeaderValue::from_str("s=; Max-Age=0; Path=/").unwrap(), + ); } *res.status_mut() = StatusCode::NO_CONTENT; Ok(res) } - async fn post_signals(&self, mut req: Request, caller: Caller) - -> ResponseResult { + async fn post_signals(&self, mut req: Request, caller: Caller) -> ResponseResult { if !caller.permissions.update_signals { - return Err(plain_response(StatusCode::UNAUTHORIZED, "update_signals required")); + return Err(plain_response( + StatusCode::UNAUTHORIZED, + "update_signals required", + )); } let r = extract_json_body(&mut req).await?; - let r: json::PostSignalsRequest = serde_json::from_slice(&r) - .map_err(|e| bad_req(e.to_string()))?; + let r: json::PostSignalsRequest = + serde_json::from_slice(&r).map_err(|e| bad_req(e.to_string()))?; let mut l = self.db.lock(); let now = recording::Time::new(self.db.clocks().realtime()); let start = r.start_time_90k.map(recording::Time).unwrap_or(now); let end = match r.end_base { - json::PostSignalsEndBase::Epoch => recording::Time(r.rel_end_time_90k.ok_or_else( - || bad_req("must specify rel_end_time_90k when end_base is epoch"))?), + json::PostSignalsEndBase::Epoch => { + recording::Time(r.rel_end_time_90k.ok_or_else(|| { + bad_req("must specify rel_end_time_90k when end_base is epoch") + })?) + } json::PostSignalsEndBase::Now => { now + recording::Duration(r.rel_end_time_90k.unwrap_or(0)) - }, + } }; - l.update_signals(start .. end, &r.signal_ids, &r.states).map_err(from_base_error)?; - serve_json(&req, &json::PostSignalsResponse { - time_90k: now.0, - }) + l.update_signals(start..end, &r.signal_ids, &r.states) + .map_err(from_base_error)?; + serve_json(&req, &json::PostSignalsResponse { time_90k: now.0 }) } fn get_signals(&self, req: &Request) -> ResponseResult { - let mut time = recording::Time::min_value() .. recording::Time::max_value(); + let mut time = recording::Time::min_value()..recording::Time::max_value(); if let Some(q) = req.uri().query() { for (key, value) in form_urlencoded::parse(q.as_bytes()) { let (key, value) = (key.borrow(), value.borrow()); @@ -990,33 +1185,42 @@ impl Service { "startTime90k" => { time.start = recording::Time::parse(value) .map_err(|_| bad_req("unparseable startTime90k"))? - }, + } "endTime90k" => { time.end = recording::Time::parse(value) .map_err(|_| bad_req("unparseable endTime90k"))? - }, - _ => {}, + } + _ => {} } } } let mut signals = json::Signals::default(); - self.db.lock().list_changes_by_time(time, &mut |c: &db::signal::ListStateChangesRow| { - signals.times_90k.push(c.when.0); - signals.signal_ids.push(c.signal); - signals.states.push(c.state); - }); + self.db + .lock() + .list_changes_by_time(time, &mut |c: &db::signal::ListStateChangesRow| { + signals.times_90k.push(c.when.0); + signals.signal_ids.push(c.signal); + signals.states.push(c.state); + }); serve_json(req, &signals) } - fn authenticate(&self, req: &Request, unauth_path: bool) - -> Result { + fn authenticate( + &self, + req: &Request, + unauth_path: bool, + ) -> Result { if let Some(sid) = extract_sid(req) { let authreq = self.authreq(req); // TODO: real error handling! this assumes all errors are due to lack of // authentication, when they could be logic errors in SQL or such. - if let Ok((s, u)) = self.db.lock().authenticate_session(authreq.clone(), &sid.hash()) { + if let Ok((s, u)) = self + .db + .lock() + .authenticate_session(authreq.clone(), &sid.hash()) + { return Ok(Caller { permissions: s.permissions.clone(), session: Some(json::Session { @@ -1039,7 +1243,7 @@ impl Service { return Ok(Caller { permissions: db::Permissions::default(), session: None, - }) + }); } bail_t!(Unauthenticated, "unauthenticated"); @@ -1068,7 +1272,7 @@ impl<'a> StaticFileRequest<'a> { None => return None, Some(d) => d, }; - let ext = &path[last_dot+1..]; + let ext = &path[last_dot + 1..]; let mime = match ext { "html" => "text/html", "ico" => "image/x-icon", @@ -1076,7 +1280,7 @@ impl<'a> StaticFileRequest<'a> { "json" => "application/json", "png" => "image/png", "webapp" => "application/x-web-app-manifest+json", - _ => return None + _ => return None, }; Some(StaticFileRequest { @@ -1089,13 +1293,13 @@ impl<'a> StaticFileRequest<'a> { #[cfg(test)] mod tests { + use super::{Segments, StaticFileRequest}; use db::testutil::{self, TestDb}; use futures::future::FutureExt; use log::info; use std::collections::HashMap; use std::str::FromStr; use std::sync::Arc; - use super::{Segments, StaticFileRequest}; struct Server { db: TestDb, @@ -1109,13 +1313,16 @@ mod tests { fn new(allow_unauthenticated_permissions: Option) -> Server { let db = TestDb::new(base::clock::RealClocks {}); let (shutdown_tx, shutdown_rx) = futures::channel::oneshot::channel::<()>(); - let service = Arc::new(super::Service::new(super::Config { - db: db.db.clone(), - ui_dir: None, - allow_unauthenticated_permissions, - trust_forward_hdrs: true, - time_zone_name: "".to_owned(), - }).unwrap()); + let service = Arc::new( + super::Service::new(super::Config { + db: db.db.clone(), + ui_dir: None, + allow_unauthenticated_permissions, + trust_forward_hdrs: true, + time_zone_name: "".to_owned(), + }) + .unwrap(), + ); let make_svc = hyper::service::make_service_fn(move |_conn| { futures::future::ok::<_, std::convert::Infallible>(hyper::service::service_fn({ let s = Arc::clone(&service); @@ -1128,11 +1335,14 @@ mod tests { let rt = tokio::runtime::Runtime::new().unwrap(); let srv = { let _guard = rt.enter(); - hyper::server::Server::bind(&addr).tcp_nodelay(true).serve(make_svc) + hyper::server::Server::bind(&addr) + .tcp_nodelay(true) + .serve(make_svc) }; - let addr = srv.local_addr(); // resolve port 0 to a real ephemeral port number. + let addr = srv.local_addr(); // resolve port 0 to a real ephemeral port number. tx.send(addr).unwrap(); - rt.block_on(srv.with_graceful_shutdown(shutdown_rx.map(|_| ()))).unwrap(); + rt.block_on(srv.with_graceful_shutdown(shutdown_rx.map(|_| ()))) + .unwrap(); }); let addr = rx.recv().unwrap(); @@ -1197,39 +1407,56 @@ mod tests { let cam_uuid = Uuid::parse_str("35144640-ff1e-4619-b0d5-4c74c185741c").unwrap(); assert_eq!(Path::decode("/foo"), Path::Static); assert_eq!(Path::decode("/api/"), Path::TopLevel); - assert_eq!(Path::decode("/api/init/42.mp4"), Path::InitSegment(42, false)); - assert_eq!(Path::decode("/api/init/42.mp4.txt"), Path::InitSegment(42, true)); - assert_eq!(Path::decode("/api/init/x.mp4"), Path::NotFound); // non-digit - assert_eq!(Path::decode("/api/cameras/35144640-ff1e-4619-b0d5-4c74c185741c/"), - Path::Camera(cam_uuid)); + assert_eq!( + Path::decode("/api/init/42.mp4"), + Path::InitSegment(42, false) + ); + assert_eq!( + Path::decode("/api/init/42.mp4.txt"), + Path::InitSegment(42, true) + ); + assert_eq!(Path::decode("/api/init/x.mp4"), Path::NotFound); // non-digit + assert_eq!( + Path::decode("/api/cameras/35144640-ff1e-4619-b0d5-4c74c185741c/"), + Path::Camera(cam_uuid) + ); assert_eq!(Path::decode("/api/cameras/asdf/"), Path::NotFound); assert_eq!( Path::decode("/api/cameras/35144640-ff1e-4619-b0d5-4c74c185741c/main/recordings"), - Path::StreamRecordings(cam_uuid, db::StreamType::MAIN)); + Path::StreamRecordings(cam_uuid, db::StreamType::MAIN) + ); assert_eq!( Path::decode("/api/cameras/35144640-ff1e-4619-b0d5-4c74c185741c/sub/recordings"), - Path::StreamRecordings(cam_uuid, db::StreamType::SUB)); + Path::StreamRecordings(cam_uuid, db::StreamType::SUB) + ); assert_eq!( Path::decode("/api/cameras/35144640-ff1e-4619-b0d5-4c74c185741c/junk/recordings"), - Path::NotFound); + Path::NotFound + ); assert_eq!( Path::decode("/api/cameras/35144640-ff1e-4619-b0d5-4c74c185741c/main/view.mp4"), - Path::StreamViewMp4(cam_uuid, db::StreamType::MAIN, false)); + Path::StreamViewMp4(cam_uuid, db::StreamType::MAIN, false) + ); assert_eq!( Path::decode("/api/cameras/35144640-ff1e-4619-b0d5-4c74c185741c/main/view.mp4.txt"), - Path::StreamViewMp4(cam_uuid, db::StreamType::MAIN, true)); + Path::StreamViewMp4(cam_uuid, db::StreamType::MAIN, true) + ); assert_eq!( Path::decode("/api/cameras/35144640-ff1e-4619-b0d5-4c74c185741c/main/view.m4s"), - Path::StreamViewMp4Segment(cam_uuid, db::StreamType::MAIN, false)); + Path::StreamViewMp4Segment(cam_uuid, db::StreamType::MAIN, false) + ); assert_eq!( Path::decode("/api/cameras/35144640-ff1e-4619-b0d5-4c74c185741c/main/view.m4s.txt"), - Path::StreamViewMp4Segment(cam_uuid, db::StreamType::MAIN, true)); + Path::StreamViewMp4Segment(cam_uuid, db::StreamType::MAIN, true) + ); assert_eq!( Path::decode("/api/cameras/35144640-ff1e-4619-b0d5-4c74c185741c/main/live.m4s"), - Path::StreamLiveMp4Segments(cam_uuid, db::StreamType::MAIN)); + Path::StreamLiveMp4Segments(cam_uuid, db::StreamType::MAIN) + ); assert_eq!( Path::decode("/api/cameras/35144640-ff1e-4619-b0d5-4c74c185741c/main/junk"), - Path::NotFound); + Path::NotFound + ); assert_eq!(Path::decode("/api/login"), Path::Login); assert_eq!(Path::decode("/api/logout"), Path::Logout); assert_eq!(Path::decode("/api/signals"), Path::Signals); @@ -1240,43 +1467,70 @@ mod tests { fn static_file() { testutil::init(); let r = StaticFileRequest::parse("/jquery-ui.b6d3d46c828800e78499.js").unwrap(); - assert_eq!(r, StaticFileRequest { - path: "jquery-ui.b6d3d46c828800e78499.js", - mime: "text/javascript", - immutable: true, - }); + assert_eq!( + r, + StaticFileRequest { + path: "jquery-ui.b6d3d46c828800e78499.js", + mime: "text/javascript", + immutable: true, + } + ); let r = StaticFileRequest::parse("/").unwrap(); - assert_eq!(r, StaticFileRequest { - path: "index.html", - mime: "text/html", - immutable: false, - }); + assert_eq!( + r, + StaticFileRequest { + path: "index.html", + mime: "text/html", + immutable: false, + } + ); } #[test] + #[rustfmt::skip] fn test_segments() { testutil::init(); - assert_eq!(Segments{ids: 1..2, open_id: None, start_time: 0, end_time: None}, - Segments::from_str("1").unwrap()); - assert_eq!(Segments{ids: 1..2, open_id: Some(42), start_time: 0, end_time: None}, - Segments::from_str("1@42").unwrap()); - assert_eq!(Segments{ids: 1..2, open_id: None, start_time: 26, end_time: None}, - Segments::from_str("1.26-").unwrap()); - assert_eq!(Segments{ids: 1..2, open_id: Some(42), start_time: 26, end_time: None}, - Segments::from_str("1@42.26-").unwrap()); - assert_eq!(Segments{ids: 1..2, open_id: None, start_time: 0, end_time: Some(42)}, - Segments::from_str("1.-42").unwrap()); - assert_eq!(Segments{ids: 1..2, open_id: None, start_time: 26, end_time: Some(42)}, - Segments::from_str("1.26-42").unwrap()); - assert_eq!(Segments{ids: 1..6, open_id: None, start_time: 0, end_time: None}, - Segments::from_str("1-5").unwrap()); - assert_eq!(Segments{ids: 1..6, open_id: None, start_time: 26, end_time: None}, - Segments::from_str("1-5.26-").unwrap()); - assert_eq!(Segments{ids: 1..6, open_id: None, start_time: 0, end_time: Some(42)}, - Segments::from_str("1-5.-42").unwrap()); - assert_eq!(Segments{ids: 1..6, open_id: None, start_time: 26, end_time: Some(42)}, - Segments::from_str("1-5.26-42").unwrap()); + assert_eq!( + Segments { ids: 1..2, open_id: None, start_time: 0, end_time: None }, + Segments::from_str("1").unwrap() + ); + assert_eq!( + Segments { ids: 1..2, open_id: Some(42), start_time: 0, end_time: None }, + Segments::from_str("1@42").unwrap() + ); + assert_eq!( + Segments { ids: 1..2, open_id: None, start_time: 26, end_time: None }, + Segments::from_str("1.26-").unwrap() + ); + assert_eq!( + Segments { ids: 1..2, open_id: Some(42), start_time: 26, end_time: None }, + Segments::from_str("1@42.26-").unwrap() + ); + assert_eq!( + Segments { ids: 1..2, open_id: None, start_time: 0, end_time: Some(42) }, + Segments::from_str("1.-42").unwrap() + ); + assert_eq!( + Segments { ids: 1..2, open_id: None, start_time: 26, end_time: Some(42) }, + Segments::from_str("1.26-42").unwrap() + ); + assert_eq!( + Segments { ids: 1..6, open_id: None, start_time: 0, end_time: None }, + Segments::from_str("1-5").unwrap() + ); + assert_eq!( + Segments { ids: 1..6, open_id: None, start_time: 26, end_time: None }, + Segments::from_str("1-5.26-").unwrap() + ); + assert_eq!( + Segments { ids: 1..6, open_id: None, start_time: 0, end_time: Some(42) }, + Segments::from_str("1-5.-42").unwrap() + ); + assert_eq!( + Segments { ids: 1..6, open_id: None, start_time: 26, end_time: Some(42) }, + Segments::from_str("1-5.26-42").unwrap() + ); } #[tokio::test] @@ -1284,7 +1538,11 @@ mod tests { testutil::init(); let s = Server::new(None); let cli = reqwest::Client::new(); - let resp = cli.get(&format!("{}/api/", &s.base_url)).send().await.unwrap(); + let resp = cli + .get(&format!("{}/api/", &s.base_url)) + .send() + .await + .unwrap(); assert_eq!(resp.status(), reqwest::StatusCode::UNAUTHORIZED); } @@ -1314,11 +1572,12 @@ mod tests { info!("cookie: {:?}", cookie); info!("header: {}", cookie.header()); - let resp = cli.get(&format!("{}/api/", &s.base_url)) - .header(reqwest::header::COOKIE, cookie.header()) - .send() - .await - .unwrap(); + let resp = cli + .get(&format!("{}/api/", &s.base_url)) + .header(reqwest::header::COOKIE, cookie.header()) + .send() + .await + .unwrap(); assert_eq!(resp.status(), reqwest::StatusCode::OK); } @@ -1330,41 +1589,59 @@ mod tests { let mut p = HashMap::new(); p.insert("username", "slamb"); p.insert("password", "hunter2"); - let resp = cli.post(&format!("{}/api/login", &s.base_url)).json(&p).send().await.unwrap(); + let resp = cli + .post(&format!("{}/api/login", &s.base_url)) + .json(&p) + .send() + .await + .unwrap(); assert_eq!(resp.status(), reqwest::StatusCode::NO_CONTENT); let cookie = SessionCookie::new(resp.headers()); // A GET shouldn't work. - let resp = cli.get(&format!("{}/api/logout", &s.base_url)) - .header(reqwest::header::COOKIE, cookie.header()) - .send() - .await - .unwrap(); + let resp = cli + .get(&format!("{}/api/logout", &s.base_url)) + .header(reqwest::header::COOKIE, cookie.header()) + .send() + .await + .unwrap(); assert_eq!(resp.status(), reqwest::StatusCode::METHOD_NOT_ALLOWED); // Neither should a POST without a csrf token. - let resp = cli.post(&format!("{}/api/logout", &s.base_url)) - .header(reqwest::header::COOKIE, cookie.header()) - .send() - .await - .unwrap(); + let resp = cli + .post(&format!("{}/api/logout", &s.base_url)) + .header(reqwest::header::COOKIE, cookie.header()) + .send() + .await + .unwrap(); assert_eq!(resp.status(), reqwest::StatusCode::BAD_REQUEST); // But it should work with the csrf token. // Retrieve that from the toplevel API request. - let toplevel: serde_json::Value = cli.post(&format!("{}/api/", &s.base_url)) - .header(reqwest::header::COOKIE, cookie.header()) - .send().await.unwrap() - .json().await.unwrap(); - let csrf = toplevel.get("session").unwrap().get("csrf").unwrap().as_str(); + let toplevel: serde_json::Value = cli + .post(&format!("{}/api/", &s.base_url)) + .header(reqwest::header::COOKIE, cookie.header()) + .send() + .await + .unwrap() + .json() + .await + .unwrap(); + let csrf = toplevel + .get("session") + .unwrap() + .get("csrf") + .unwrap() + .as_str(); let mut p = HashMap::new(); p.insert("csrf", csrf); - let resp = cli.post(&format!("{}/api/logout", &s.base_url)) - .header(reqwest::header::COOKIE, cookie.header()) - .json(&p) - .send() - .await - .unwrap(); + let resp = cli + .post(&format!("{}/api/logout", &s.base_url)) + .header(reqwest::header::COOKIE, cookie.header()) + .json(&p) + .send() + .await + .unwrap(); assert_eq!(resp.status(), reqwest::StatusCode::NO_CONTENT); let mut updated_cookie = cookie.clone(); updated_cookie.update(resp.headers()); @@ -1373,11 +1650,12 @@ mod tests { assert!(updated_cookie.0.is_none()); // It should also be invalidated server-side. - let resp = cli.get(&format!("{}/api/", &s.base_url)) - .header(reqwest::header::COOKIE, cookie.header()) - .send() - .await - .unwrap(); + let resp = cli + .get(&format!("{}/api/", &s.base_url)) + .header(reqwest::header::COOKIE, cookie.header()) + .send() + .await + .unwrap(); assert_eq!(resp.status(), reqwest::StatusCode::UNAUTHORIZED); } @@ -1388,14 +1666,19 @@ mod tests { permissions.view_video = true; let s = Server::new(Some(permissions)); let cli = reqwest::Client::new(); - let resp = cli.get( - &format!("{}/api/cameras/{}/main/view.mp4", &s.base_url, s.db.test_camera_uuid)) - .send().await.unwrap(); + let resp = cli + .get(&format!( + "{}/api/cameras/{}/main/view.mp4", + &s.base_url, s.db.test_camera_uuid + )) + .send() + .await + .unwrap(); assert_eq!(resp.status(), reqwest::StatusCode::BAD_REQUEST); } } -#[cfg(all(test, feature="nightly"))] +#[cfg(all(test, feature = "nightly"))] mod bench { extern crate test; @@ -1415,13 +1698,16 @@ mod bench { let db = TestDb::new(::base::clock::RealClocks {}); let test_camera_uuid = db.test_camera_uuid; testutil::add_dummy_recordings_to_db(&db.db, 1440); - let service = Arc::new(super::Service::new(super::Config { - db: db.db.clone(), - ui_dir: None, - allow_unauthenticated_permissions: Some(db::Permissions::default()), - trust_forward_hdrs: false, - time_zone_name: "".to_owned(), - }).unwrap()); + let service = Arc::new( + super::Service::new(super::Config { + db: db.db.clone(), + ui_dir: None, + allow_unauthenticated_permissions: Some(db::Permissions::default()), + trust_forward_hdrs: false, + time_zone_name: "".to_owned(), + }) + .unwrap(), + ); let make_svc = hyper::service::make_service_fn(move |_conn| { futures::future::ok::<_, std::convert::Infallible>(hyper::service::service_fn({ let s = Arc::clone(&service); @@ -1436,7 +1722,7 @@ mod bench { .tcp_nodelay(true) .serve(make_svc) }; - let addr = srv.local_addr(); // resolve port 0 to a real ephemeral port number. + let addr = srv.local_addr(); // resolve port 0 to a real ephemeral port number. ::std::thread::spawn(move || { rt.block_on(srv).unwrap(); }); @@ -1455,8 +1741,11 @@ mod bench { fn serve_stream_recordings(b: &mut test::Bencher) { testutil::init(); let server = &*SERVER; - let url = reqwest::Url::parse(&format!("{}/api/cameras/{}/main/recordings", server.base_url, - server.test_camera_uuid)).unwrap(); + let url = reqwest::Url::parse(&format!( + "{}/api/cameras/{}/main/recordings", + server.base_url, server.test_camera_uuid + )) + .unwrap(); let client = reqwest::Client::new(); let rt = tokio::runtime::Runtime::new().unwrap(); let f = || { @@ -1466,7 +1755,7 @@ mod bench { let _b = resp.bytes().await.unwrap(); }); }; - f(); // warm. + f(); // warm. b.iter(f); } }