shrink mp4::Segment 128 -> 112 bytes (on 64-bit)

* don't store sizes of mp4-format sample indexes; recalculate them.
   * keep SampleIndexIterator position as a u32 rather than a usize.

This is 960 bytes for a 60-minute mp4; another small cache usage improvement.

Cargo.lock

@@ -13,6 +13,7 @@ dependencies = [
  "http-entity 0.0.1 (git+https://github.com/scottlamb/http-entity)",
  "hyper 0.10.4 (registry+https://github.com/rust-lang/crates.io-index)",
  "lazy_static 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)",
+ "lazycell 0.5.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "libc 0.2.20 (registry+https://github.com/rust-lang/crates.io-index)",
  "log 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)",
  "lru-cache 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
@@ -300,6 +301,11 @@ name = "lazy_static"
 version = "0.2.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 
+[[package]]
+name = "lazycell"
+version = "0.5.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+
 [[package]]
 name = "libc"
 version = "0.2.20"
@@ -903,6 +909,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 "checksum language-tags 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "a91d884b6667cd606bb5a69aa0c99ba811a115fc68915e7056ec08a46e93199a"
 "checksum lazy_static 0.1.16 (registry+https://github.com/rust-lang/crates.io-index)" = "cf186d1a8aa5f5bee5fd662bc9c1b949e0259e1bcc379d1f006847b0080c7417"
 "checksum lazy_static 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "6abe0ee2e758cd6bc8a2cd56726359007748fbf4128da998b65d0b70f881e19b"
+"checksum lazycell 0.5.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ec38a5c22f1ef3e30d2642aa875620d60edeef36cef43c4739d86215ce816331"
 "checksum libc 0.2.20 (registry+https://github.com/rust-lang/crates.io-index)" = "684f330624d8c3784fb9558ca46c4ce488073a8d22450415c5eb4f4cfb0d11b5"
 "checksum libsqlite3-sys 0.6.2 (registry+https://github.com/rust-lang/crates.io-index)" = "b6de3eea39ba6ed0cddf04e1c7a78486e3f750441e0a0b15b6ea39d0dd8e1b8c"
 "checksum linked-hash-map 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "bda158e0dabeb97ee8a401f4d17e479d6b891a14de0bba79d5cc2d4d325b5e48"

Cargo.toml

@@ -14,6 +14,7 @@ docopt = "0.7"
 fnv = "1.0"
 http-entity = { git = "https://github.com/scottlamb/http-entity" }
 hyper = "0.10"
+lazycell = "0.5"
 lazy_static = "0.2"
 libc = "0.2"
 log = { version = "0.3", features = ["release_max_level_info"] }

src/main.rs

@@ -41,6 +41,7 @@ extern crate fnv;
 extern crate http_entity;
 extern crate hyper;
 #[macro_use] extern crate lazy_static;
+extern crate lazycell;
 extern crate libc;
 #[macro_use] extern crate log;
 extern crate lru_cache;

src/mp4.rs

@@ -90,7 +90,7 @@ use openssl::hash;
 use recording::{self, TIME_UNITS_PER_SEC};
 use slices::{self, Slices};
 use smallvec::SmallVec;
-use std::cell::RefCell;
+use lazycell::LazyCell;
 use std::cmp;
 use std::io;
 use std::ops::Range;
@@ -314,30 +314,23 @@ const SUBTITLE_TEMPLATE: &'static str = "%Y-%m-%d %H:%M:%S %z";
 /// The length of the output of `SUBTITLE_TEMPLATE`.
 const SUBTITLE_LENGTH: usize = 25;  // "2015-07-02 17:10:00 -0700".len();
 
-/// Holds the sample indexes for a given video segment: `stts`, `stsz`, and `stss`.
+/// The lengths of the indexes associated with a `Segment`; for use within `Segment` only.
-struct SegmentIndex {
+struct SegmentLengths {
-    /// Holds all three sample indexes:
+    stts: usize,
-    /// &buf[.. stsz_start] is stts.
+    stsz: usize,
-    /// &buf[stsz_start .. stss_start] is stsz.
+    stss: usize,
-    /// &buf[stss_start ..] is stss.
-    buf: Box<[u8]>,
-    stsz_start: u32,
-    stss_start: u32,
-}
-
-impl SegmentIndex {
-    fn stts(&self) -> &[u8] { &self.buf[.. self.stsz_start as usize] }
-    fn stsz(&self) -> &[u8] { &self.buf[self.stsz_start as usize .. self.stss_start as usize] }
-    fn stss(&self) -> &[u8] { &self.buf[self.stss_start as usize ..] }
 }
 
 /// A wrapper around `recording::Segment` that keeps some additional `.mp4`-specific state.
 struct Segment {
     s: recording::Segment,
 
-    /// Holds the `stts`, `stsz`, and `stss` if they've been generated.
+    /// If generated, the `.mp4`-format sample indexes.
-    /// Access only through `with_index`.
+    ///    1. stts: `slice[.. stsz_start]`
-    index: RefCell<Option<SegmentIndex>>,
+    ///    2. stsz: `slice[stsz_start .. stss_start]`
+    ///    3. stss: `slice[stss_start ..]`
+    /// Access only through `write_index`.
+    index: LazyCell<Result<Box<[u8]>, ()>>,
 
     /// The 1-indexed frame number in the `File` of the first frame in this segment.
     first_frame_num: u32,
@@ -345,32 +338,46 @@ struct Segment {
 }
 
 impl Segment {
-    fn with_index<F, R>(&self, db: &db::Database, f: F) -> Result<R, Error>
+    fn write_index<F>(&self, r: Range<u64>, db: &db::Database, out: &mut io::Write, f: F)
-    where F: FnOnce(&SegmentIndex) -> Result<R, Error> {
+                      -> Result<(), Error>
-        let mut i = self.index.borrow_mut();
+    where F: FnOnce(&[u8], SegmentLengths) -> &[u8] {
-        if let Some(ref i) = *i {
+        let lens = SegmentLengths {
-            return f(i);
+            stts: mem::size_of::<u32>() * 2 * (self.s.frames as usize),
-        }
+            stsz: mem::size_of::<u32>() * self.s.frames as usize,
-        let index = self.build_index(db)?;
+            stss: mem::size_of::<u32>() * self.s.key_frames as usize,
-        let r = f(&index);
+        };
-        *i = Some(index);
+        let index = self.index.borrow_with(|| {
-        r
+            self.build_index(&lens, db)
+                .map_err(|e| {
+                    error!("Unable to build index for segment: {:?}", e);
+                    ()
+                })
+        });
+        let index = match *index {
+            Ok(ref b) => &b[..],
+            Err(()) => {
+                return Err(Error::new("Unable to build index; see previous error.".to_owned()))
+            },
+        };
+        out.write_all(&f(&index, lens)[r.start as usize .. r.end as usize])?;
+        Ok(())
     }
 
-    fn build_index(&self, db: &db::Database) -> Result<SegmentIndex, Error> {
+    fn stts(buf: &[u8], lens: SegmentLengths) -> &[u8] { &buf[.. lens.stts] }
+    fn stsz(buf: &[u8], lens: SegmentLengths) -> &[u8] { &buf[lens.stts .. lens.stts + lens.stsz] }
+    fn stss(buf: &[u8], lens: SegmentLengths) -> &[u8] { &buf[lens.stts + lens.stsz ..] }
+
+    fn build_index(&self, lens: &SegmentLengths, db: &db::Database) -> Result<Box<[u8]>, Error> {
         let s = &self.s;
-        let stts_len = mem::size_of::<u32>() * 2 * (s.frames as usize);
+        let len = lens.stts + lens.stsz + lens.stss;
-        let stsz_len = mem::size_of::<u32>() * s.frames as usize;
+        let mut buf = {
-        let stss_len = mem::size_of::<u32>() * s.key_frames as usize;
-        let len = stts_len + stsz_len + stss_len;
-        let mut buf = unsafe {
             let mut v = Vec::with_capacity(len);
-            v.set_len(len);
+            unsafe { v.set_len(len) };
             v.into_boxed_slice()
         };
         {
-            let (stts, mut rest) = buf.split_at_mut(stts_len);
+            let (stts, mut rest) = buf.split_at_mut(lens.stts);
-            let (stsz, stss) = rest.split_at_mut(stsz_len);
+            let (stsz, stss) = rest.split_at_mut(lens.stsz);
             let mut frame = 0;
             let mut key_frame = 0;
             let mut last_start_and_dur = None;
@@ -396,11 +403,7 @@ impl Segment {
                                      cmp::min(s.desired_range_90k.end - last_start, dur) as u32);
             }
         }
-        Ok(SegmentIndex{
+        Ok(buf)
-            buf: buf,
-            stsz_start: stts_len as u32,
-            stss_start: (stts_len + stsz_len) as u32,
-        })
     }
 }
 
@@ -501,10 +504,10 @@ impl slices::Slice for Slice {
                 out.write_all(&f.video_sample_entries[p].data[r.start as usize .. r.end as usize])?;
                 Ok(())
             },
-            SliceType::Stts => f.write_stts(p, r, l, out),
+            SliceType::Stts => f.segments[p].write_index(r, &f.db, out, Segment::stts),
-            SliceType::Stsz => f.write_stsz(p, r, l, out),
+            SliceType::Stsz => f.segments[p].write_index(r, &f.db, out, Segment::stsz),
+            SliceType::Stss => f.segments[p].write_index(r, &f.db, out, Segment::stss),
             SliceType::Co64 => f.write_co64(r, l, out),
-            SliceType::Stss => f.write_stss(p, r, l, out),
             SliceType::VideoSampleData => f.write_video_sample_data(p, r, out),
             SliceType::SubtitleSampleData => f.write_subtitle_sample_data(p, r, l, out),
         }
@@ -580,7 +583,7 @@ impl FileBuilder {
         }
         self.segments.push(Segment{
             s: recording::Segment::new(db, &row, rel_range_90k)?,
-            index: RefCell::new(None),
+            index: LazyCell::new(),
             first_frame_num: self.next_frame_num,
             num_subtitle_samples: 0,
         });
@@ -1125,22 +1128,6 @@ pub struct File {
 }
 
 impl File {
-    fn write_stts(&self, i: usize, r: Range<u64>, _l: u64, out: &mut io::Write)
-                  -> Result<(), Error> {
-        self.segments[i].with_index(&self.db, |i| {
-            out.write_all(&i.stts()[r.start as usize .. r.end as usize])?;
-            Ok(())
-        })
-    }
-
-    fn write_stsz(&self, i: usize, r: Range<u64>, _l: u64, out: &mut io::Write)
-                  -> Result<(), Error> {
-        self.segments[i].with_index(&self.db, |i| {
-            out.write_all(&i.stsz()[r.start as usize .. r.end as usize])?;
-            Ok(())
-        })
-    }
-
     fn write_co64(&self, r: Range<u64>, l: u64, out: &mut io::Write) -> Result<(), Error> {
         slices::clip_to_range(r, l, out, |w| {
             let mut pos = self.initial_sample_byte_pos;
@@ -1153,14 +1140,6 @@ impl File {
         })
     }
 
-    fn write_stss(&self, i: usize, r: Range<u64>, _l: u64, out: &mut io::Write)
-                  -> Result<(), Error> {
-        self.segments[i].with_index(&self.db, |i| {
-            out.write_all(&i.stss()[r.start as usize .. r.end as usize])?;
-            Ok(())
-        })
-    }
-
     fn write_video_sample_data(&self, i: usize, r: Range<u64>, out: &mut io::Write)
                                -> Result<(), Error> {
         let s = &self.segments[i];

src/recording.rs

@@ -210,7 +210,7 @@ impl ops::SubAssign for Duration {
 
 #[derive(Clone, Copy, Debug)]
 pub struct SampleIndexIterator {
-    i: usize,
+    i: u32,
     pub pos: i32,
     pub start_90k: i32,
     pub duration_90k: i32,
@@ -251,10 +251,10 @@ impl SampleIndexIterator {
     pub fn next(&mut self, data: &[u8]) -> Result<bool, Error> {
         self.pos += self.bytes;
         self.start_90k += self.duration_90k;
-        if self.i == data.len() {
+        if self.i as usize == data.len() {
             return Ok(false)
         }
-        let (raw1, i1) = match decode_varint32(data, self.i) {
+        let (raw1, i1) = match decode_varint32(data, self.i as usize) {
             Ok(tuple) => tuple,
             Err(()) => return Err(Error::new(format!("bad varint 1 at offset {}", self.i))),
         };
@@ -262,7 +262,7 @@ impl SampleIndexIterator {
             Ok(tuple) => tuple,
             Err(()) => return Err(Error::new(format!("bad varint 2 at offset {}", i1))),
         };
-        self.i = i2;
+        self.i = i2 as u32;
         let duration_90k_delta = unzigzag32(raw1 >> 1);
         self.duration_90k += duration_90k_delta;
         if self.duration_90k < 0 {
@@ -271,10 +271,10 @@ impl SampleIndexIterator {
                                      self.duration_90k, duration_90k_delta),
                 cause: None});
         }
-        if self.duration_90k == 0 && data.len() > self.i {
+        if self.duration_90k == 0 && data.len() > self.i as usize {
             return Err(Error{
                 description: format!("zero duration only allowed at end; have {} bytes left",
-                                     data.len() - self.i),
+                                     data.len() - self.i as usize),
                 cause: None});
         }
         self.is_key = (raw1 & 1) == 1;