MyFavMirrors
/
moonfire-nvr
mirror of https://github.com/scottlamb/moonfire-nvr.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

WIP: support H.265 (#33) and MJPEG too 

This isn't ready to merge because it's depending on an unreleased
version of `h264-reader` and an unreleased (and WIP) version of retina.
But it appears to work in a quick test.

There's no transcoding, so if the browser/player doesn't support these
formats, they don't play. But it will record and allow downloads, and
it seems to be working with Chrome on macOS at least.
This commit is contained in:
Scott Lamb 2024-08-14 20:43:33 -07:00
parent 0422593ec6
commit 6481ab86cb
11 changed files with 169 additions and 428 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.github/workflows/ci.yml vendored
View File

@ -15,7 +15,7 @@ jobs:
name: Rust ${{ matrix.rust }} name: Rust ${{ matrix.rust }}
strategy: strategy:
matrix: matrix:
rust: [ "stable", "1.70", "nightly" ] rust: [ "stable", "1.79", "nightly" ]
include: include:
- rust: nightly - rust: nightly
extra_args: "--features nightly --benches" extra_args: "--features nightly --benches"

2
CHANGELOG.md
View File

@ -10,6 +10,8 @@ even on minor releases, e.g. `v0.7.5` -> `v0.7.6`.
## unreleased ## unreleased
* support recording H.265 (#3) and MJPEG video. Browser support may vary.
* bump minimum Rust version to 1.79.
* in UI's list view, add a tooltip on the end time which shows why the * in UI's list view, add a tooltip on the end time which shows why the
recording ended. recording ended.

2
guide/build.md
View File

@ -68,7 +68,7 @@ following command:
$ brew install node $ brew install node
``` ```
Next, you need Rust 1.65+ and Cargo. The easiest way to install them is by Next, you need Rust 1.79+ and Cargo. The easiest way to install them is by
following the instructions at [rustup.rs](https://www.rustup.rs/). Avoid following the instructions at [rustup.rs](https://www.rustup.rs/). Avoid
your Linux distribution's Rust packages, which tend to be too old. your Linux distribution's Rust packages, which tend to be too old.
(At least on Debian-based systems; Arch and Gentoo might be okay.) (At least on Debian-based systems; Arch and Gentoo might be okay.)

49
server/Cargo.lock generated
View File

@ -129,6 +129,12 @@ version = "1.10.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e445576659fd04a57b44cbd00aa37aaa815ebefa0aa3cb677a6b5e63d883074f" checksum = "e445576659fd04a57b44cbd00aa37aaa815ebefa0aa3cb677a6b5e63d883074f"
[[package]]
name = "bitstream-io"
version = "2.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3dcde5f311c85b8ca30c2e4198d4326bc342c76541590106f5fa4a50946ea499"
[[package]] [[package]]
name = "blake3" name = "blake3"
version = "1.5.0" version = "1.5.0"
@ -365,7 +371,7 @@ dependencies = [
"log", "log",
"num", "num",
"owning_ref", "owning_ref",
"time 0.3.31", "time 0.3.36",
"unicode-segmentation", "unicode-segmentation",
"unicode-width", "unicode-width",
"xi-unicode", "xi-unicode",
@ -554,9 +560,9 @@ dependencies = [
[[package]] [[package]]
name = "four-cc" name = "four-cc"
version = "0.1.0" version = "0.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3958af68a31b1d1384d3f39b6aa33eb14b6009065b5ca305ddd9712a4237124f" checksum = "795cbfc56d419a7ce47ccbb7504dd9a5b7c484c083c356e797de08bd988d9629"
[[package]] [[package]]
name = "futures" name = "futures"
@ -715,11 +721,10 @@ dependencies = [
[[package]] [[package]]
name = "h264-reader" name = "h264-reader"
version = "0.7.0" version = "0.8.0-dev"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "git+https://github.com/scottlamb/h264-reader?rev=35968d5ca67f317fd35c1e344adb8ae14ee2efd6#35968d5ca67f317fd35c1e344adb8ae14ee2efd6"
checksum = "bd118dcc322cc71cfc33254a19ebece92cfaaf6d4b4793fec3f7f44fbc4150df"
dependencies = [ dependencies = [
"bitstream-io", "bitstream-io 2.5.0",
"hex-slice", "hex-slice",
"log", "log",
"memchr", "memchr",
@ -1234,9 +1239,9 @@ dependencies = [
[[package]] [[package]]
name = "mp4ra-rust" name = "mp4ra-rust"
version = "0.1.0" version = "0.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "be9daf03b43bf3842962947c62ba40f411e46a58774c60838038f04a67d17626" checksum = "fdbc3d3867085d66ac6270482e66f3dd2c5a18451a3dc9ad7269e94844a536b7"
dependencies = [ dependencies = [
"four-cc", "four-cc",
] ]
@ -1312,6 +1317,12 @@ dependencies = [
"num-traits", "num-traits",
] ]
[[package]]
name = "num-conv"
version = "0.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "51d515d32fb182ee37cda2ccdcb92950d6a3c2893aa280e540671c2cd0f3b1d9"
[[package]] [[package]]
name = "num-integer" name = "num-integer"
version = "0.1.45" version = "0.1.45"
@ -1743,11 +1754,10 @@ dependencies = [
[[package]] [[package]]
name = "retina" name = "retina"
version = "0.4.8" version = "0.4.8"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "git+https://github.com/scottlamb/retina?rev=27a4ebf286945ecf3f1863a24f0cc568cd50bdb8#27a4ebf286945ecf3f1863a24f0cc568cd50bdb8"
checksum = "fdd73fbdea4177bdc50179d23a85d1db7c329bfbe06e064947a6b92d87332d81"
dependencies = [ dependencies = [
"base64", "base64",
"bitstream-io", "bitstream-io 1.10.0",
"bytes", "bytes",
"futures", "futures",
"h264-reader", "h264-reader",
@ -1770,11 +1780,10 @@ dependencies = [
[[package]] [[package]]
name = "rfc6381-codec" name = "rfc6381-codec"
version = "0.1.0" version = "0.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4395f46a67f0d57c57f6a5361f3a9a0c0183a19cab3998892ecdc003de6d8037" checksum = "ed54c20f5c3ec82eab6d998b313dc75ec5d5650d4f57675e61d72489040297fd"
dependencies = [ dependencies = [
"four-cc",
"mp4ra-rust", "mp4ra-rust",
"mpeg4-audio-const", "mpeg4-audio-const",
] ]
@ -2155,13 +2164,14 @@ dependencies = [
[[package]] [[package]]
name = "time" name = "time"
version = "0.3.31" version = "0.3.36"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f657ba42c3f86e7680e53c8cd3af8abbe56b5491790b46e22e19c0d57463583e" checksum = "5dfd88e563464686c916c7e46e623e520ddc6d79fa6641390f2e3fa86e83e885"
dependencies = [ dependencies = [
"deranged", "deranged",
"itoa", "itoa",
"libc", "libc",
"num-conv",
"num_threads", "num_threads",
"powerfmt", "powerfmt",
"serde", "serde",
@ -2177,10 +2187,11 @@ checksum = "ef927ca75afb808a4d64dd374f00a2adf8d0fcff8e7b184af886c3c87ec4a3f3"
[[package]] [[package]]
name = "time-macros" name = "time-macros"
version = "0.2.16" version = "0.2.18"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "26197e33420244aeb70c3e8c78376ca46571bc4e701e4791c2cd9f57dcb3a43f" checksum = "3f252a68540fde3a3877aeea552b832b40ab9a69e318efd078774a01ddee1ccf"
dependencies = [ dependencies = [
"num-conv",
"time-core", "time-core",
] ]

8
server/Cargo.toml
View File

@ -5,7 +5,7 @@ authors = ["Scott Lamb <slamb@slamb.org>"]
edition = "2021" edition = "2021"
resolver = "2" resolver = "2"
license-file = "../LICENSE.txt" license-file = "../LICENSE.txt"
rust-version = "1.70" rust-version = "1.79"
publish = false publish = false
[features] [features]
@ -24,7 +24,8 @@ members = ["base", "db"]
[workspace.dependencies] [workspace.dependencies]
base64 = "0.21.0" base64 = "0.21.0"
h264-reader = "0.7.0" #h264-reader = "0.7.0"
h264-reader = { git = "https://github.com/scottlamb/h264-reader", rev = "35968d5ca67f317fd35c1e344adb8ae14ee2efd6" }
itertools = "0.12.0" itertools = "0.12.0"
nix = "0.27.0" nix = "0.27.0"
pretty-hex = "0.4.0" pretty-hex = "0.4.0"
@ -58,7 +59,8 @@ password-hash = "0.5.0"
pretty-hex = { workspace = true } pretty-hex = { workspace = true }
protobuf = "3.0" protobuf = "3.0"
reffers = "0.7.0" reffers = "0.7.0"
retina = "0.4.0" #retina = "0.4.0"
retina = { git = "https://github.com/scottlamb/retina", rev = "27a4ebf286945ecf3f1863a24f0cc568cd50bdb8", features = ["unstable-sample-entry", "unstable-h265"] }
ring = { workspace = true } ring = { workspace = true }
rusqlite = { workspace = true } rusqlite = { workspace = true }
serde = { version = "1.0", features = ["derive"] } serde = { version = "1.0", features = ["derive"] }

2
server/db/Cargo.toml
View File

@ -5,7 +5,7 @@ authors = ["Scott Lamb <slamb@slamb.org>"]
readme = "../README.md" readme = "../README.md"
edition = "2021" edition = "2021"
license-file = "../../LICENSE.txt" license-file = "../../LICENSE.txt"
rust-version = "1.70" rust-version = "1.79"
publish = false publish = false
[features] [features]

2
server/db/upgrade/v5_to_v6.rs
View File

@ -133,7 +133,7 @@ pub fn run(_args: &super::Args, tx: &rusqlite::Transaction) -> Result<(), Error>
) )
})?; })?;
let sps = ctx let sps = ctx
.sps_by_id(h264_reader::nal::pps::ParamSetId::from_u32(0).unwrap()) .sps_by_id(h264_reader::nal::sps::SeqParamSetId::from_u32(0).unwrap())
.ok_or_else(|| { .ok_or_else(|| {
err!( err!(
Unimplemented, Unimplemented,

383
server/src/h264.rs
View File

@ -1,383 +0,0 @@
// This file is part of Moonfire NVR, a security camera network video recorder.
// Copyright (C) 2021 The Moonfire NVR Authors; see AUTHORS and LICENSE.txt.
// SPDX-License-Identifier: GPL-v3.0-or-later WITH GPL-3.0-linking-exception.
//! H.264 decoding
//!
//! For the most part, Moonfire NVR does not try to understand the video codec. However, H.264 has
//! two byte stream encodings: ISO/IEC 14496-10 Annex B, and ISO/IEC 14496-15 AVC access units.
//! When streaming from RTSP, ffmpeg supplies the former. We need the latter to stick into `.mp4`
//! files. This file manages the conversion, both for the ffmpeg "extra data" (which should become
//! the ISO/IEC 14496-15 section 5.2.4.1 `AVCDecoderConfigurationRecord`) and the actual samples.
//!
//! See the [wiki page on standards and
//! specifications](https://github.com/scottlamb/moonfire-nvr/wiki/Standards-and-specifications)
//! for help finding a copy of the relevant standards. This code won't make much sense without them!
//!
//! ffmpeg of course has logic to do the same thing, but unfortunately it is not exposed except
//! through ffmpeg's own generated `.mp4` file. Extracting just this part of their `.mp4` files
//! would be more trouble than it's worth.
use base::{bail, err, Error};
use byteorder::{BigEndian, ByteOrder, WriteBytesExt};
use db::VideoSampleEntryToInsert;
use h264_reader::nal::Nal;
use pretty_hex::PrettyHex as _;
use std::convert::TryFrom;
// For certain common sub stream anamorphic resolutions, add a pixel aspect ratio box.
// Assume the camera is 16x9. These are just the standard wide mode; default_pixel_aspect_ratio
// tries the transpose also.
const PIXEL_ASPECT_RATIOS: [((u16, u16), (u16, u16)); 6] = [
((320, 240), (4, 3)),
((352, 240), (40, 33)),
((640, 352), (44, 45)),
((640, 480), (4, 3)),
((704, 480), (40, 33)),
((720, 480), (32, 27)),
];
/// Get the pixel aspect ratio to use if none is specified.
///
/// The Dahua IPC-HDW5231R-Z sets the aspect ratio in the H.264 SPS (correctly) for both square and
/// non-square pixels. The Hikvision DS-2CD2032-I doesn't set it, even though the sub stream's
/// pixels aren't square. So define a default based on the pixel dimensions to use if the camera
/// doesn't tell us what to do.
///
/// Note that at least in the case of .mp4 muxing, we don't need to fix up the underlying SPS.
/// PixelAspectRatioBox's definition says that it overrides the H.264-level declaration.
fn default_pixel_aspect_ratio(width: u16, height: u16) -> (u16, u16) {
if width >= height {
PIXEL_ASPECT_RATIOS
.iter()
.find(|r| r.0 == (width, height))
.map(|r| r.1)
.unwrap_or((1, 1))
} else {
PIXEL_ASPECT_RATIOS
.iter()
.find(|r| r.0 == (height, width))
.map(|r| (r.1 .1, r.1 .0))
.unwrap_or((1, 1))
}
}
/// `h264_reader::rbsp::BitRead` impl that does not care about extra trailing data.
///
/// Some (Reolink) cameras appear to have a stray extra byte at the end. Follow the lead of most
/// other RTSP implementations in tolerating this.
#[derive(Debug)]
struct TolerantBitReader<R> {
inner: R,
}
impl<R: h264_reader::rbsp::BitRead> h264_reader::rbsp::BitRead for TolerantBitReader<R> {
fn read_ue(&mut self, name: &'static str) -> Result<u32, h264_reader::rbsp::BitReaderError> {
self.inner.read_ue(name)
}
fn read_se(&mut self, name: &'static str) -> Result<i32, h264_reader::rbsp::BitReaderError> {
self.inner.read_se(name)
}
fn read_bool(&mut self, name: &'static str) -> Result<bool, h264_reader::rbsp::BitReaderError> {
self.inner.read_bool(name)
}
fn read_u8(
&mut self,
bit_count: u32,
name: &'static str,
) -> Result<u8, h264_reader::rbsp::BitReaderError> {
self.inner.read_u8(bit_count, name)
}
fn read_u16(
&mut self,
bit_count: u32,
name: &'static str,
) -> Result<u16, h264_reader::rbsp::BitReaderError> {
self.inner.read_u16(bit_count, name)
}
fn read_u32(
&mut self,
bit_count: u32,
name: &'static str,
) -> Result<u32, h264_reader::rbsp::BitReaderError> {
self.inner.read_u32(bit_count, name)
}
fn read_i32(
&mut self,
bit_count: u32,
name: &'static str,
) -> Result<i32, h264_reader::rbsp::BitReaderError> {
self.inner.read_i32(bit_count, name)
}
fn has_more_rbsp_data(
&mut self,
name: &'static str,
) -> Result<bool, h264_reader::rbsp::BitReaderError> {
self.inner.has_more_rbsp_data(name)
}
fn finish_rbsp(self) -> Result<(), h264_reader::rbsp::BitReaderError> {
match self.inner.finish_rbsp() {
Ok(()) => Ok(()),
Err(h264_reader::rbsp::BitReaderError::RemainingData) => {
tracing::debug!("extra data at end of NAL unit");
Ok(())
}
Err(e) => Err(e),
}
}
fn finish_sei_payload(self) -> Result<(), h264_reader::rbsp::BitReaderError> {
self.inner.finish_sei_payload()
}
}
fn parse_extra_data_inner(extradata: &[u8]) -> Result<VideoSampleEntryToInsert, Error> {
let avcc =
h264_reader::avcc::AvcDecoderConfigurationRecord::try_from(extradata).map_err(|e| {
err!(
InvalidArgument,
msg("bad AvcDecoderConfigurationRecord: {:?}", e)
)
})?;
if avcc.num_of_sequence_parameter_sets() != 1 {
bail!(Unimplemented, msg("multiple SPSs!"));
}
// This logic is essentially copied from
// `h264_reader::avcc::AvcDecoderConfigurationRecord::create_context` but
// using our `TolerantBitReader` wrapper.
let mut ctx = h264_reader::Context::new();
for sps in avcc.sequence_parameter_sets() {
let sps = h264_reader::nal::RefNal::new(
sps.map_err(|e| err!(InvalidArgument, msg("bad SPS: {e:?}")))?,
&[],
true,
);
let sps = h264_reader::nal::sps::SeqParameterSet::from_bits(TolerantBitReader {
inner: sps.rbsp_bits(),
})
.map_err(|e| err!(InvalidArgument, msg("bad SPS: {e:?}")))?;
ctx.put_seq_param_set(sps);
}
for pps in avcc.picture_parameter_sets() {
let pps = h264_reader::nal::RefNal::new(
pps.map_err(|e| err!(InvalidArgument, msg("bad PPS: {e:?}")))?,
&[],
true,
);
let pps = h264_reader::nal::pps::PicParameterSet::from_bits(
&ctx,
TolerantBitReader {
inner: pps.rbsp_bits(),
},
)
.map_err(|e| err!(InvalidArgument, msg("bad PPS: {e:?}")))?;
ctx.put_pic_param_set(pps);
}
let sps = ctx
.sps_by_id(h264_reader::nal::pps::ParamSetId::from_u32(0).unwrap())
.ok_or_else(|| err!(Unimplemented, msg("no SPS 0")))?;
let pixel_dimensions = sps.pixel_dimensions().map_err(|e| {
err!(
InvalidArgument,
msg("SPS has invalid pixel dimensions: {:?}", e)
)
})?;
let (Ok(width), Ok(height)) = (
u16::try_from(pixel_dimensions.0),
u16::try_from(pixel_dimensions.1),
) else {
bail!(
InvalidArgument,
msg(
"bad dimensions {}x{}",
pixel_dimensions.0,
pixel_dimensions.1
)
);
};
let mut sample_entry = Vec::with_capacity(256);
// This is a concatenation of the following boxes/classes.
// SampleEntry, ISO/IEC 14496-12 section 8.5.2.
let avc1_len_pos = sample_entry.len();
// length placeholder + type + reserved + data_reference_index = 1
sample_entry.extend_from_slice(b"\x00\x00\x00\x00avc1\x00\x00\x00\x00\x00\x00\x00\x01");
// VisualSampleEntry, ISO/IEC 14496-12 section 12.1.3.
sample_entry.extend_from_slice(&[0; 16]); // pre-defined + reserved
sample_entry.write_u16::<BigEndian>(width)?;
sample_entry.write_u16::<BigEndian>(height)?;
sample_entry.extend_from_slice(&[
0x00, 0x48, 0x00, 0x00, // horizresolution
0x00, 0x48, 0x00, 0x00, // vertresolution
0x00, 0x00, 0x00, 0x00, // reserved
0x00, 0x01, // frame count
0x00, 0x00, 0x00, 0x00, // compressorname
0x00, 0x00, 0x00, 0x00, //
0x00, 0x00, 0x00, 0x00, //
0x00, 0x00, 0x00, 0x00, //
0x00, 0x00, 0x00, 0x00, //
0x00, 0x00, 0x00, 0x00, //
0x00, 0x00, 0x00, 0x00, //
0x00, 0x00, 0x00, 0x00, //
0x00, 0x18, 0xff, 0xff, // depth + pre_defined
]);
// AVCSampleEntry, ISO/IEC 14496-15 section 5.3.4.1.
// AVCConfigurationBox, ISO/IEC 14496-15 section 5.3.4.1.
let avcc_len_pos = sample_entry.len();
sample_entry.extend_from_slice(b"\x00\x00\x00\x00avcC");
sample_entry.extend_from_slice(extradata);
// Fix up avc1 and avcC box lengths.
let cur_pos = sample_entry.len();
BigEndian::write_u32(
&mut sample_entry[avcc_len_pos..avcc_len_pos + 4],
u32::try_from(cur_pos - avcc_len_pos).map_err(|_| err!(OutOfRange))?,
);
// PixelAspectRatioBox, ISO/IEC 14496-12 section 12.1.4.2.
// Write a PixelAspectRatioBox if necessary, as the sub streams can be be anamorphic.
let pasp = sps
.vui_parameters
.as_ref()
.and_then(|v| v.aspect_ratio_info.as_ref())
.and_then(|a| a.clone().get())
.unwrap_or_else(|| default_pixel_aspect_ratio(width, height));
if pasp != (1, 1) {
sample_entry.extend_from_slice(b"\x00\x00\x00\x10pasp"); // length + box name
sample_entry.write_u32::<BigEndian>(pasp.0.into())?;
sample_entry.write_u32::<BigEndian>(pasp.1.into())?;
}
let cur_pos = sample_entry.len();
BigEndian::write_u32(
&mut sample_entry[avc1_len_pos..avc1_len_pos + 4],
u32::try_from(cur_pos - avc1_len_pos).map_err(|_| err!(OutOfRange))?,
);
let profile_idc = sample_entry[103];
let constraint_flags = sample_entry[104];
let level_idc = sample_entry[105];
let rfc6381_codec = format!("avc1.{profile_idc:02x}{constraint_flags:02x}{level_idc:02x}");
Ok(VideoSampleEntryToInsert {
data: sample_entry,
rfc6381_codec,
width,
height,
pasp_h_spacing: pasp.0,
pasp_v_spacing: pasp.1,
})
}
/// Parses the `AvcDecoderConfigurationRecord` in the "extra data".
pub fn parse_extra_data(extradata: &[u8]) -> Result<VideoSampleEntryToInsert, Error> {
parse_extra_data_inner(extradata).map_err(|e| {
err!(
e,
msg(
"can't parse AvcDecoderRecord {}",
extradata.hex_conf(pretty_hex::HexConfig {
width: 0,
group: 0,
chunk: 0,
..Default::default()
})
)
)
})
}
#[cfg(test)]
mod tests {
use db::testutil;
#[rustfmt::skip]
const AVC_DECODER_CONFIG_TEST_INPUT: [u8; 38] = [
0x01, 0x4d, 0x00, 0x1f, 0xff,
0xe1, 0x00, 0x17, // 1 SPS, length 0x17
0x67, 0x4d, 0x00, 0x1f, 0x9a, 0x66, 0x02, 0x80,
0x2d, 0xff, 0x35, 0x01, 0x01, 0x01, 0x40, 0x00,
0x00, 0xfa, 0x00, 0x00, 0x1d, 0x4c, 0x01,
0x01, 0x00, 0x04, // 1 PPS, length 0x04
0x68, 0xee, 0x3c, 0x80,
];
#[rustfmt::skip]
const AVC_DECODER_CONFIG_TEST_INPUT_WITH_TRAILING_GARBAGE: [u8; 40] = [
0x01, 0x4d, 0x00, 0x1f, 0xff,
0xe1, 0x00, 0x18, // 1 SPS, length 0x18
0x67, 0x4d, 0x00, 0x1f, 0x9a, 0x66, 0x02, 0x80,
0x2d, 0xff, 0x35, 0x01, 0x01, 0x01, 0x40, 0x00,
0x00, 0xfa, 0x00, 0x00, 0x1d, 0x4c, 0x01, 0x01,
0x01, 0x00, 0x04, // 1 PPS, length 0x05
0x68, 0xee, 0x3c, 0x80, 0x80,
];
#[rustfmt::skip]
const TEST_OUTPUT: [u8; 132] = [
0x00, 0x00, 0x00, 0x84, 0x61, 0x76, 0x63, 0x31,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x05, 0x00, 0x02, 0xd0, 0x00, 0x48, 0x00, 0x00,
0x00, 0x48, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x18, 0xff, 0xff, 0x00, 0x00,
0x00, 0x2e, 0x61, 0x76, 0x63, 0x43, 0x01, 0x4d,
0x00, 0x1f, 0xff, 0xe1, 0x00, 0x17, 0x67, 0x4d,
0x00, 0x1f, 0x9a, 0x66, 0x02, 0x80, 0x2d, 0xff,
0x35, 0x01, 0x01, 0x01, 0x40, 0x00, 0x00, 0xfa,
0x00, 0x00, 0x1d, 0x4c, 0x01, 0x01, 0x00, 0x04,
0x68, 0xee, 0x3c, 0x80,
];
#[test]
fn test_sample_entry_from_avc_decoder_config() {
testutil::init();
let e = super::parse_extra_data(&AVC_DECODER_CONFIG_TEST_INPUT).unwrap();
assert_eq!(&e.data[..], &TEST_OUTPUT[..]);
assert_eq!(e.width, 1280);
assert_eq!(e.height, 720);
assert_eq!(e.rfc6381_codec, "avc1.4d001f");
}
#[test]
fn pixel_aspect_ratios() {
use super::default_pixel_aspect_ratio;
use num_rational::Ratio;
for &((w, h), _) in &super::PIXEL_ASPECT_RATIOS {
let (h_spacing, v_spacing) = default_pixel_aspect_ratio(w, h);
assert_eq!(Ratio::new(w * h_spacing, h * v_spacing), Ratio::new(16, 9));
// 90 or 270 degree rotation.
let (h_spacing, v_spacing) = default_pixel_aspect_ratio(h, w);
assert_eq!(Ratio::new(h * h_spacing, w * v_spacing), Ratio::new(9, 16));
}
}
#[test]
fn extra_sps_data() {
super::parse_extra_data(&AVC_DECODER_CONFIG_TEST_INPUT_WITH_TRAILING_GARBAGE).unwrap();
}
}

1
server/src/main.rs
View File

@ -12,7 +12,6 @@ use tracing::{debug, error};
mod body; mod body;
mod cmds; mod cmds;
mod h264;
mod json; mod json;
mod mp4; mod mp4;
mod slices; mod slices;

10
server/src/mp4.rs
View File

@ -2844,7 +2844,7 @@ mod tests {
// combine ranges from the new format with ranges from the old format. // combine ranges from the new format with ranges from the old format.
let hash = digest(&mp4).await; let hash = digest(&mp4).await;
assert_eq!( assert_eq!(
"64f23b856692702b13d1811cd02dc83395b3d501dead7fd16f175eb26b4d8eee", "123e2cf075125c81e80820bffa412d38729aff05c252c7ea2ab3384905903bb7",
hash.to_hex().as_str() hash.to_hex().as_str()
); );
const EXPECTED_ETAG: &str = const EXPECTED_ETAG: &str =
@ -2873,7 +2873,7 @@ mod tests {
// combine ranges from the new format with ranges from the old format. // combine ranges from the new format with ranges from the old format.
let hash = digest(&mp4).await; let hash = digest(&mp4).await;
assert_eq!( assert_eq!(
"f9e4ed946187b2dd22ef049c4c1869d0f6c4f377ef08f8f53570850b61a06701", "1f85ec7ea7f061b7d8f696c337a3258abc2bf830e81ac23c1342131669d7bb14",
hash.to_hex().as_str() hash.to_hex().as_str()
); );
const EXPECTED_ETAG: &str = const EXPECTED_ETAG: &str =
@ -2902,7 +2902,7 @@ mod tests {
// combine ranges from the new format with ranges from the old format. // combine ranges from the new format with ranges from the old format.
let hash = digest(&mp4).await; let hash = digest(&mp4).await;
assert_eq!( assert_eq!(
"f913d46d0119a03291e85459455b9a75a84cc9a1a5e3b88ca7e93eb718d73190", "1debe76fc6277546209454919550ff4c3a379560f481fa0ce78378cbf3c646f8",
hash.to_hex().as_str() hash.to_hex().as_str()
); );
const EXPECTED_ETAG: &str = const EXPECTED_ETAG: &str =
@ -2932,7 +2932,7 @@ mod tests {
// combine ranges from the new format with ranges from the old format. // combine ranges from the new format with ranges from the old format.
let hash = digest(&mp4).await; let hash = digest(&mp4).await;
assert_eq!( assert_eq!(
"64cc763fa2533118bc6bf0b01249f02524ae87e0c97815079447b235722c1e2d", "9c0302294f8f34d14fc8069fea1a65c1593a4c01134c07ab994b7398004f2b63",
hash.to_hex().as_str() hash.to_hex().as_str()
); );
const EXPECTED_ETAG: &str = const EXPECTED_ETAG: &str =
@ -2961,7 +2961,7 @@ mod tests {
// combine ranges from the new format with ranges from the old format. // combine ranges from the new format with ranges from the old format.
let hash = digest(&mp4).await; let hash = digest(&mp4).await;
assert_eq!( assert_eq!(
"6886b36ae6df9ce538f6db7ebd6159e68c2936b9d43307f7728fe75e0b62cad2", "e06b5627788828b73b98726dfb6466d32305df64af0acbe6164fc8ab296de473",
hash.to_hex().as_str() hash.to_hex().as_str()
); );
const EXPECTED_ETAG: &str = const EXPECTED_ETAG: &str =

134
server/src/stream.rs
View File

@ -2,7 +2,6 @@
// Copyright (C) 2016 The Moonfire NVR Authors; see AUTHORS and LICENSE.txt. // Copyright (C) 2016 The Moonfire NVR Authors; see AUTHORS and LICENSE.txt.
// SPDX-License-Identifier: GPL-v3.0-or-later WITH GPL-3.0-linking-exception. // SPDX-License-Identifier: GPL-v3.0-or-later WITH GPL-3.0-linking-exception.
use crate::h264;
use base::{bail, err, Error}; use base::{bail, err, Error};
use bytes::Bytes; use bytes::Bytes;
use futures::StreamExt; use futures::StreamExt;
@ -15,6 +14,43 @@ use url::Url;
static RETINA_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(30); static RETINA_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(30);
// For certain common sub stream anamorphic resolutions, add a pixel aspect ratio box.
// Assume the camera is 16x9. These are just the standard wide mode; default_pixel_aspect_ratio
// tries the transpose also.
const PIXEL_ASPECT_RATIOS: [((u16, u16), (u16, u16)); 6] = [
((320, 240), (4, 3)),
((352, 240), (40, 33)),
((640, 352), (44, 45)),
((640, 480), (4, 3)),
((704, 480), (40, 33)),
((720, 480), (32, 27)),
];
/// Gets the pixel aspect ratio to use if none is specified.
///
/// The Dahua IPC-HDW5231R-Z sets the aspect ratio in the H.264 SPS (correctly) for both square and
/// non-square pixels. The Hikvision DS-2CD2032-I doesn't set it, even though the sub stream's
/// pixels aren't square. So define a default based on the pixel dimensions to use if the camera
/// doesn't tell us what to do.
///
/// Note that at least in the case of .mp4 muxing, we don't need to fix up the underlying SPS.
/// PixelAspectRatioBox's definition says that it overrides the H.264-level declaration.
fn default_pixel_aspect_ratio(width: u16, height: u16) -> (u16, u16) {
if width >= height {
PIXEL_ASPECT_RATIOS
.iter()
.find(|r| r.0 == (width, height))
.map(|r| r.1)
.unwrap_or((1, 1))
} else {
PIXEL_ASPECT_RATIOS
.iter()
.find(|r| r.0 == (height, width))
.map(|r| (r.1 .1, r.1 .0))
.unwrap_or((1, 1))
}
}
pub struct Options { pub struct Options {
pub session: retina::client::SessionOptions, pub session: retina::client::SessionOptions,
pub setup: retina::client::SetupOptions, pub setup: retina::client::SetupOptions,
@ -34,6 +70,7 @@ pub struct VideoFrame {
/// An estimate of the duration of the frame, or zero. /// An estimate of the duration of the frame, or zero.
/// This can be deceptive and is only used by some testing code. /// This can be deceptive and is only used by some testing code.
#[cfg(test)]
pub duration: i32, pub duration: i32,
pub is_key: bool, pub is_key: bool,
@ -114,6 +151,27 @@ struct RetinaStreamInner {
video_sample_entry: db::VideoSampleEntryToInsert, video_sample_entry: db::VideoSampleEntryToInsert,
} }
fn params_to_sample_entry(
params: &retina::codec::VideoParameters,
) -> Result<db::VideoSampleEntryToInsert, Error> {
let (width, height) = params.pixel_dimensions();
let width = u16::try_from(width).map_err(|e| err!(Unknown, source(e)))?;
let height = u16::try_from(height).map_err(|e| err!(Unknown, source(e)))?;
let aspect = default_pixel_aspect_ratio(width, height);
Ok(db::VideoSampleEntryToInsert {
data: params
.sample_entry()
.with_aspect_ratio(aspect)
.build()
.map_err(|e| err!(Unknown, source(e)))?,
rfc6381_codec: "avc1.4d401e".to_string(),
width,
height,
pasp_h_spacing: aspect.0,
pasp_v_spacing: aspect.1,
})
}
impl RetinaStreamInner { impl RetinaStreamInner {
/// Plays to first frame. No timeout; that's the caller's responsibility. /// Plays to first frame. No timeout; that's the caller's responsibility.
async fn play( async fn play(
@ -128,8 +186,15 @@ impl RetinaStreamInner {
let video_i = session let video_i = session
.streams() .streams()
.iter() .iter()
.position(|s| s.media() == "video" && s.encoding_name() == "h264") .position(|s| {
.ok_or_else(|| err!(FailedPrecondition, msg("couldn't find H.264 video stream")))?; s.media() == "video" && matches!(s.encoding_name(), "h264" | "h265" | "jpeg")
})
.ok_or_else(|| {
err!(
FailedPrecondition,
msg("couldn't find supported video stream")
)
})?;
session session
.setup(video_i, options.setup) .setup(video_i, options.setup)
.await .await
@ -156,9 +221,9 @@ impl RetinaStreamInner {
let video_params = match session.streams()[video_i].parameters() { let video_params = match session.streams()[video_i].parameters() {
Some(retina::codec::ParametersRef::Video(v)) => v.clone(), Some(retina::codec::ParametersRef::Video(v)) => v.clone(),
Some(_) => unreachable!(), Some(_) => unreachable!(),
None => bail!(Unknown, msg("couldn't find H.264 parameters")), None => bail!(Unknown, msg("couldn't find video parameters")),
}; };
let video_sample_entry = h264::parse_extra_data(video_params.extra_data())?; let video_sample_entry = params_to_sample_entry(&video_params)?;
let self_ = Box::new(Self { let self_ = Box::new(Self {
label, label,
session, session,
@ -245,7 +310,7 @@ impl Stream for RetinaStream {
})??; })??;
let mut new_video_sample_entry = false; let mut new_video_sample_entry = false;
if let Some(p) = new_parameters { if let Some(p) = new_parameters {
let video_sample_entry = h264::parse_extra_data(p.extra_data())?; let video_sample_entry = params_to_sample_entry(&p)?;
if video_sample_entry != inner.video_sample_entry { if video_sample_entry != inner.video_sample_entry {
tracing::debug!( tracing::debug!(
"{}: parameter change:\nold: {:?}\nnew: {:?}", "{}: parameter change:\nold: {:?}\nnew: {:?}",
@ -262,6 +327,7 @@ impl Stream for RetinaStream {
})?; })?;
Ok(VideoFrame { Ok(VideoFrame {
pts: frame.timestamp().elapsed(), pts: frame.timestamp().elapsed(),
#[cfg(test)]
duration: 0, duration: 0,
is_key: frame.is_random_access_point(), is_key: frame.is_random_access_point(),
data: frame.into_data().into(), data: frame.into_data().into(),
@ -272,6 +338,8 @@ impl Stream for RetinaStream {
#[cfg(test)] #[cfg(test)]
pub mod testutil { pub mod testutil {
use mp4::mp4box::WriteBox as _;
use super::*; use super::*;
use std::convert::TryFrom; use std::convert::TryFrom;
use std::io::Cursor; use std::io::Cursor;
@ -298,14 +366,35 @@ pub mod testutil {
.values() .values()
.find(|t| matches!(t.media_type(), Ok(mp4::MediaType::H264))) .find(|t| matches!(t.media_type(), Ok(mp4::MediaType::H264)))
{ {
None => bail!(InvalidArgument, msg("expected a H.264 track")), None => bail!(
InvalidArgument,
msg(
"expected a H.264 track, tracks were: {:#?}",
reader.tracks()
)
),
Some(t) => t, Some(t) => t,
}; };
let video_sample_entry = h264::parse_extra_data( let mut data = Vec::new();
&h264_track h264_track
.extra_data() .trak
.map_err(|e| err!(Unknown, source(e)))?[..], .mdia
)?; .minf
.stbl
.stsd
.avc1
.as_ref()
.unwrap()
.write_box(&mut data)
.unwrap();
let video_sample_entry = db::VideoSampleEntryToInsert {
data,
rfc6381_codec: "avc1.4d401e".to_string(),
width: h264_track.width(),
height: h264_track.height(),
pasp_h_spacing: 1,
pasp_v_spacing: 1,
};
let h264_track_id = h264_track.track_id(); let h264_track_id = h264_track.track_id();
let stream = Mp4Stream { let stream = Mp4Stream {
reader, reader,
@ -345,6 +434,7 @@ pub mod testutil {
self.next_sample_id += 1; self.next_sample_id += 1;
Ok(VideoFrame { Ok(VideoFrame {
pts: sample.start_time as i64, pts: sample.start_time as i64,
#[cfg(test)]
duration: sample.duration as i32, duration: sample.duration as i32,
is_key: sample.is_sync, is_key: sample.is_sync,
data: sample.bytes, data: sample.bytes,
@ -357,3 +447,23 @@ pub mod testutil {
} }
} }
} }
#[cfg(test)]
mod tests {
use db::testutil;
#[test]
fn pixel_aspect_ratios() {
testutil::init();
use super::default_pixel_aspect_ratio;
use num_rational::Ratio;
for &((w, h), _) in &super::PIXEL_ASPECT_RATIOS {
let (h_spacing, v_spacing) = default_pixel_aspect_ratio(w, h);
assert_eq!(Ratio::new(w * h_spacing, h * v_spacing), Ratio::new(16, 9));
// 90 or 270 degree rotation.
let (h_spacing, v_spacing) = default_pixel_aspect_ratio(h, w);
assert_eq!(Ratio::new(h * h_spacing, w * v_spacing), Ratio::new(9, 16));
}
}
}