MyFavMirrors
/
moonfire-nvr
mirror of https://github.com/scottlamb/moonfire-nvr.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

shrink mp4::Segment 128 -> 112 bytes (on 64-bit) 

* don't store sizes of mp4-format sample indexes; recalculate them.
   * keep SampleIndexIterator position as a u32 rather than a usize.

This is 960 bytes for a 60-minute mp4; another small cache usage improvement.
This commit is contained in:
Scott Lamb 2017-02-26 00:02:49 -08:00
parent 21212be18a
commit f24daba299
5 changed files with 65 additions and 77 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
Cargo.lock generated
View File

@ -13,6 +13,7 @@ dependencies = [
"http-entity 0.0.1 (git+https://github.com/scottlamb/http-entity)",
"hyper 0.10.4 (registry+https://github.com/rust-lang/crates.io-index)",
"lazy_static 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)",
"lazycell 0.5.0 (registry+https://github.com/rust-lang/crates.io-index)",
"libc 0.2.20 (registry+https://github.com/rust-lang/crates.io-index)",
"log 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)",
"lru-cache 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
@ -300,6 +301,11 @@ name = "lazy_static"
version = "0.2.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
[[package]]
name = "lazycell"
version = "0.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
[[package]]
name = "libc"
version = "0.2.20"
@ -903,6 +909,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
"checksum language-tags 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "a91d884b6667cd606bb5a69aa0c99ba811a115fc68915e7056ec08a46e93199a"
"checksum lazy_static 0.1.16 (registry+https://github.com/rust-lang/crates.io-index)" = "cf186d1a8aa5f5bee5fd662bc9c1b949e0259e1bcc379d1f006847b0080c7417"
"checksum lazy_static 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "6abe0ee2e758cd6bc8a2cd56726359007748fbf4128da998b65d0b70f881e19b"
"checksum lazycell 0.5.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ec38a5c22f1ef3e30d2642aa875620d60edeef36cef43c4739d86215ce816331"
"checksum libc 0.2.20 (registry+https://github.com/rust-lang/crates.io-index)" = "684f330624d8c3784fb9558ca46c4ce488073a8d22450415c5eb4f4cfb0d11b5"
"checksum libsqlite3-sys 0.6.2 (registry+https://github.com/rust-lang/crates.io-index)" = "b6de3eea39ba6ed0cddf04e1c7a78486e3f750441e0a0b15b6ea39d0dd8e1b8c"
"checksum linked-hash-map 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "bda158e0dabeb97ee8a401f4d17e479d6b891a14de0bba79d5cc2d4d325b5e48"

1
Cargo.toml
View File

@ -14,6 +14,7 @@ docopt = "0.7"
fnv = "1.0"
http-entity = { git = "https://github.com/scottlamb/http-entity" }
hyper = "0.10"
lazycell = "0.5"
lazy_static = "0.2"
libc = "0.2"
log = { version = "0.3", features = ["release_max_level_info"] }

1
src/main.rs
View File

@ -41,6 +41,7 @@ extern crate fnv;
extern crate http_entity;
extern crate hyper;
#[macro_use] extern crate lazy_static;
extern crate lazycell;
extern crate libc;
#[macro_use] extern crate log;
extern crate lru_cache;

121
src/mp4.rs
View File

@ -90,7 +90,7 @@ use openssl::hash;
use recording::{self, TIME_UNITS_PER_SEC};
use slices::{self, Slices};
use smallvec::SmallVec;
use std::cell::RefCell;
use lazycell::LazyCell;
use std::cmp;
use std::io;
use std::ops::Range;
@ -314,30 +314,23 @@ const SUBTITLE_TEMPLATE: &'static str = "%Y-%m-%d %H:%M:%S %z";
/// The length of the output of `SUBTITLE_TEMPLATE`.
const SUBTITLE_LENGTH: usize = 25; // "2015-07-02 17:10:00 -0700".len();
/// Holds the sample indexes for a given video segment: `stts`, `stsz`, and `stss`.
struct SegmentIndex {
/// Holds all three sample indexes:
/// &buf[.. stsz_start] is stts.
/// &buf[stsz_start .. stss_start] is stsz.
/// &buf[stss_start ..] is stss.
buf: Box<[u8]>,
stsz_start: u32,
stss_start: u32,
}
impl SegmentIndex {
fn stts(&self) -> &[u8] { &self.buf[.. self.stsz_start as usize] }
fn stsz(&self) -> &[u8] { &self.buf[self.stsz_start as usize .. self.stss_start as usize] }
fn stss(&self) -> &[u8] { &self.buf[self.stss_start as usize ..] }
/// The lengths of the indexes associated with a `Segment`; for use within `Segment` only.
struct SegmentLengths {
stts: usize,
stsz: usize,
stss: usize,
}
/// A wrapper around `recording::Segment` that keeps some additional `.mp4`-specific state.
struct Segment {
s: recording::Segment,
/// Holds the `stts`, `stsz`, and `stss` if they've been generated.
/// Access only through `with_index`.
index: RefCell<Option<SegmentIndex>>,
/// If generated, the `.mp4`-format sample indexes.
/// 1. stts: `slice[.. stsz_start]`
/// 2. stsz: `slice[stsz_start .. stss_start]`
/// 3. stss: `slice[stss_start ..]`
/// Access only through `write_index`.
index: LazyCell<Result<Box<[u8]>, ()>>,
/// The 1-indexed frame number in the `File` of the first frame in this segment.
first_frame_num: u32,
@ -345,32 +338,46 @@ struct Segment {
}
impl Segment {
fn with_index<F, R>(&self, db: &db::Database, f: F) -> Result<R, Error>
where F: FnOnce(&SegmentIndex) -> Result<R, Error> {
let mut i = self.index.borrow_mut();
if let Some(ref i) = *i {
return f(i);
}
let index = self.build_index(db)?;
let r = f(&index);
*i = Some(index);
r
fn write_index<F>(&self, r: Range<u64>, db: &db::Database, out: &mut io::Write, f: F)
-> Result<(), Error>
where F: FnOnce(&[u8], SegmentLengths) -> &[u8] {
let lens = SegmentLengths {
stts: mem::size_of::<u32>() * 2 * (self.s.frames as usize),
stsz: mem::size_of::<u32>() * self.s.frames as usize,
stss: mem::size_of::<u32>() * self.s.key_frames as usize,
};
let index = self.index.borrow_with(|| {
self.build_index(&lens, db)
.map_err(|e| {
error!("Unable to build index for segment: {:?}", e);
()
})
});
let index = match *index {
Ok(ref b) => &b[..],
Err(()) => {
return Err(Error::new("Unable to build index; see previous error.".to_owned()))
},
};
out.write_all(&f(&index, lens)[r.start as usize .. r.end as usize])?;
Ok(())
}
fn build_index(&self, db: &db::Database) -> Result<SegmentIndex, Error> {
fn stts(buf: &[u8], lens: SegmentLengths) -> &[u8] { &buf[.. lens.stts] }
fn stsz(buf: &[u8], lens: SegmentLengths) -> &[u8] { &buf[lens.stts .. lens.stts + lens.stsz] }
fn stss(buf: &[u8], lens: SegmentLengths) -> &[u8] { &buf[lens.stts + lens.stsz ..] }
fn build_index(&self, lens: &SegmentLengths, db: &db::Database) -> Result<Box<[u8]>, Error> {
let s = &self.s;
let stts_len = mem::size_of::<u32>() * 2 * (s.frames as usize);
let stsz_len = mem::size_of::<u32>() * s.frames as usize;
let stss_len = mem::size_of::<u32>() * s.key_frames as usize;
let len = stts_len + stsz_len + stss_len;
let mut buf = unsafe {
let len = lens.stts + lens.stsz + lens.stss;
let mut buf = {
let mut v = Vec::with_capacity(len);
v.set_len(len);
unsafe { v.set_len(len) };
v.into_boxed_slice()
};
{
let (stts, mut rest) = buf.split_at_mut(stts_len);
let (stsz, stss) = rest.split_at_mut(stsz_len);
let (stts, mut rest) = buf.split_at_mut(lens.stts);
let (stsz, stss) = rest.split_at_mut(lens.stsz);
let mut frame = 0;
let mut key_frame = 0;
let mut last_start_and_dur = None;
@ -396,11 +403,7 @@ impl Segment {
cmp::min(s.desired_range_90k.end - last_start, dur) as u32);
}
}
Ok(SegmentIndex{
buf: buf,
stsz_start: stts_len as u32,
stss_start: (stts_len + stsz_len) as u32,
})
Ok(buf)
}
}
@ -501,10 +504,10 @@ impl slices::Slice for Slice {
out.write_all(&f.video_sample_entries[p].data[r.start as usize .. r.end as usize])?;
Ok(())
},
SliceType::Stts => f.write_stts(p, r, l, out),
SliceType::Stsz => f.write_stsz(p, r, l, out),
SliceType::Stts => f.segments[p].write_index(r, &f.db, out, Segment::stts),
SliceType::Stsz => f.segments[p].write_index(r, &f.db, out, Segment::stsz),
SliceType::Stss => f.segments[p].write_index(r, &f.db, out, Segment::stss),
SliceType::Co64 => f.write_co64(r, l, out),
SliceType::Stss => f.write_stss(p, r, l, out),
SliceType::VideoSampleData => f.write_video_sample_data(p, r, out),
SliceType::SubtitleSampleData => f.write_subtitle_sample_data(p, r, l, out),
}
@ -580,7 +583,7 @@ impl FileBuilder {
}
self.segments.push(Segment{
s: recording::Segment::new(db, &row, rel_range_90k)?,
index: RefCell::new(None),
index: LazyCell::new(),
first_frame_num: self.next_frame_num,
num_subtitle_samples: 0,
});
@ -1125,22 +1128,6 @@ pub struct File {
}
impl File {
fn write_stts(&self, i: usize, r: Range<u64>, _l: u64, out: &mut io::Write)
-> Result<(), Error> {
self.segments[i].with_index(&self.db, |i| {
out.write_all(&i.stts()[r.start as usize .. r.end as usize])?;
Ok(())
})
}
fn write_stsz(&self, i: usize, r: Range<u64>, _l: u64, out: &mut io::Write)
-> Result<(), Error> {
self.segments[i].with_index(&self.db, |i| {
out.write_all(&i.stsz()[r.start as usize .. r.end as usize])?;
Ok(())
})
}
fn write_co64(&self, r: Range<u64>, l: u64, out: &mut io::Write) -> Result<(), Error> {
slices::clip_to_range(r, l, out, |w| {
let mut pos = self.initial_sample_byte_pos;
@ -1153,14 +1140,6 @@ impl File {
})
}
fn write_stss(&self, i: usize, r: Range<u64>, _l: u64, out: &mut io::Write)
-> Result<(), Error> {
self.segments[i].with_index(&self.db, |i| {
out.write_all(&i.stss()[r.start as usize .. r.end as usize])?;
Ok(())
})
}
fn write_video_sample_data(&self, i: usize, r: Range<u64>, out: &mut io::Write)
-> Result<(), Error> {
let s = &self.segments[i];

12
src/recording.rs
View File

@ -210,7 +210,7 @@ impl ops::SubAssign for Duration {
#[derive(Clone, Copy, Debug)]
pub struct SampleIndexIterator {
i: usize,
i: u32,
pub pos: i32,
pub start_90k: i32,
pub duration_90k: i32,
@ -251,10 +251,10 @@ impl SampleIndexIterator {
pub fn next(&mut self, data: &[u8]) -> Result<bool, Error> {
self.pos += self.bytes;
self.start_90k += self.duration_90k;
if self.i == data.len() {
if self.i as usize == data.len() {
return Ok(false)
}
let (raw1, i1) = match decode_varint32(data, self.i) {
let (raw1, i1) = match decode_varint32(data, self.i as usize) {
Ok(tuple) => tuple,
Err(()) => return Err(Error::new(format!("bad varint 1 at offset {}", self.i))),
};
@ -262,7 +262,7 @@ impl SampleIndexIterator {
Ok(tuple) => tuple,
Err(()) => return Err(Error::new(format!("bad varint 2 at offset {}", i1))),
};
self.i = i2;
self.i = i2 as u32;
let duration_90k_delta = unzigzag32(raw1 >> 1);
self.duration_90k += duration_90k_delta;
if self.duration_90k < 0 {
@ -271,10 +271,10 @@ impl SampleIndexIterator {
self.duration_90k, duration_90k_delta),
cause: None});
}
if self.duration_90k == 0 && data.len() > self.i {
if self.duration_90k == 0 && data.len() > self.i as usize {
return Err(Error{
description: format!("zero duration only allowed at end; have {} bytes left",
data.len() - self.i),
data.len() - self.i as usize),
cause: None});
}
self.is_key = (raw1 & 1) == 1;