live stream frame-by-frame rather than GOP-by-GOP (#59)

This should reduce live stream latency by two seconds when my cameras
are at their default setting (I frame interval = 2 * frame rate)!

I was under the impression that every HTML5 Media Source Extensions
media segment had to start with a Random Access Point. This used to
be true, but apparently changed quite a while ago:
https://bugs.chromium.org/p/chromium/issues/detail?id=229412

Support generating segments that don't start with a key frame, and
plumb this through the mp4 media segment generation logic. Add some
extra error checking in mp4 slice handling, as my first attempts had a
mismatch between expected and actual lengths that silently returned
corrupted .m4s files.

Also pull everything from the most recent key frame on along with the
first live segment to reduce startup latency. Live view is quite a bit
more pleasant now.

db/db.rs

@@ -512,13 +512,16 @@ pub struct Stream {
     on_live_segment: Vec<Box<dyn FnMut(LiveSegment) -> bool + Send>>,
 }
 
-/// Bounds of a single keyframe and the frames dependent on it.
+/// Bounds of a live view segment. Currently this is a single frame of video.
 /// This is used for live stream recordings. The stream id should already be known to the
 /// subscriber.
 #[derive(Clone, Debug)]
 pub struct LiveSegment {
     pub recording: i32,
 
+    /// If the segment's one frame is a key frame.
+    pub is_key: bool,
+
     /// The pts, relative to the start of the recording, of the start and end of this live segment,
     /// in 90kHz units.
     pub media_off_90k: Range<i32>,

db/recording.rs

@@ -143,7 +143,7 @@ impl SampleIndexIterator {
         Ok(true)
     }
 
-    pub fn uninitialized(&self) -> bool { self.i_and_is_key == 0 }
+    #[inline]
     pub fn is_key(&self) -> bool { (self.i_and_is_key & 0x8000_0000) != 0 }
 }
 
@@ -208,14 +208,20 @@ impl Segment {
     /// Creates a segment.
     ///
     /// `desired_media_range_90k` represents the desired range of the segment relative to the start
-    /// of the recording, in media time units. The actual range will start at the first key frame
-    /// at or before the desired start time. (The caller is responsible for creating an edit list
-    /// to skip the undesired portion.) It will end at the first frame after the desired range
-    /// (unless the desired range extends beyond the recording). (Likewise, the caller is
-    /// responsible for trimming the final frame's duration if desired.)
+    /// of the recording, in media time units.
+    ///
+    /// The actual range will start at the most recent acceptable frame's start at or before the
+    /// desired start time. If `start_at_key` is true, only key frames are acceptable; otherwise
+    /// any frame is. The caller is responsible for skipping over the undesired prefix, perhaps
+    /// with an edit list in the case of a `.mp4`.
+    ///
+    /// The actual range will end at the first frame after the desired range (unless the desired
+    /// range extends beyond the recording). Likewise, the caller is responsible for trimming the
+    /// final frame's duration if desired.
     pub fn new(db: &db::LockedDatabase,
                recording: &db::ListRecordingsRow,
-               desired_media_range_90k: Range<i32>) -> Result<Segment, Error> {
+               desired_media_range_90k: Range<i32>,
+               start_at_key: bool) -> Result<Segment, Error> {
         let mut self_ = Segment {
             id: recording.id,
             open_id: recording.open_id,
@@ -268,7 +274,8 @@ impl Segment {
             };
 
             loop {
-                if it.start_90k <= desired_media_range_90k.start && it.is_key() {
+                if it.start_90k <= desired_media_range_90k.start &&
+                    (!start_at_key || it.is_key()) {
                     // new start candidate.
                     *begin = it;
                     self_.frames = 0;
@@ -317,16 +324,17 @@ impl Segment {
         let data = &(&playback).video_index;
         let mut it = match self.begin {
             Some(ref b) => **b,
-            None => SampleIndexIterator::new(),
+            None => {
+                let mut it = SampleIndexIterator::new();
+                if !it.next(data)? {
+                    bail!("recording {} has no frames", self.id);
+                }
+                if !it.is_key() {
+                    bail!("recording {} doesn't start with key frame", self.id);
+                }
+                it
+            }
         };
-        if it.uninitialized() {
-            if !it.next(data)? {
-                bail!("recording {}: no frames", self.id);
-            }
-            if !it.is_key() {
-                bail!("recording {}: doesn't start with key frame", self.id);
-            }
-        }
         let mut have_frame = true;
         let mut key_frame = 0;
 
@@ -359,6 +367,17 @@ impl Segment {
         }
         Ok(())
     }
+
+    /// Returns true if this starts with a non-key frame.
+    pub fn starts_with_nonkey(&self) -> bool {
+        match self.begin {
+            Some(ref b) => !b.is_key(),
+
+            // Fast-path case, in which this holds an entire recording. They always start with a
+            // key frame.
+            None => false,
+        }
+    }
 }
 
 #[cfg(test)]
@@ -467,7 +486,7 @@ mod tests {
         let row = db.insert_recording_from_encoder(r);
         // Time range [2, 2 + 4 + 6 + 8) means the 2nd, 3rd, 4th samples should be
         // included.
-        let segment = Segment::new(&db.db.lock(), &row, 2 .. 2+4+6+8).unwrap();
+        let segment = Segment::new(&db.db.lock(), &row, 2 .. 2+4+6+8, true).unwrap();
         assert_eq!(&get_frames(&db.db, &segment, |it| it.duration_90k), &[4, 6, 8]);
     }
 
@@ -486,7 +505,7 @@ mod tests {
         let row = db.insert_recording_from_encoder(r);
         // Time range [2 + 4 + 6, 2 + 4 + 6 + 8) means the 4th sample should be included.
         // The 3rd also gets pulled in because it is a sync frame and the 4th is not.
-        let segment = Segment::new(&db.db.lock(), &row, 2+4+6 .. 2+4+6+8).unwrap();
+        let segment = Segment::new(&db.db.lock(), &row, 2+4+6 .. 2+4+6+8, true).unwrap();
         assert_eq!(&get_frames(&db.db, &segment, |it| it.duration_90k), &[6, 8]);
     }
 
@@ -500,7 +519,7 @@ mod tests {
         encoder.add_sample(0, 3, true, &mut r);
         let db = TestDb::new(RealClocks {});
         let row = db.insert_recording_from_encoder(r);
-        let segment = Segment::new(&db.db.lock(), &row, 1 .. 2).unwrap();
+        let segment = Segment::new(&db.db.lock(), &row, 1 .. 2, true).unwrap();
         assert_eq!(&get_frames(&db.db, &segment, |it| it.bytes), &[2, 3]);
     }
 
@@ -513,7 +532,7 @@ mod tests {
         encoder.add_sample(1, 1, true, &mut r);
         let db = TestDb::new(RealClocks {});
         let row = db.insert_recording_from_encoder(r);
-        let segment = Segment::new(&db.db.lock(), &row, 0 .. 0).unwrap();
+        let segment = Segment::new(&db.db.lock(), &row, 0 .. 0, true).unwrap();
         assert_eq!(&get_frames(&db.db, &segment, |it| it.bytes), &[1]);
     }
 
@@ -531,7 +550,7 @@ mod tests {
         }
         let db = TestDb::new(RealClocks {});
         let row = db.insert_recording_from_encoder(r);
-        let segment = Segment::new(&db.db.lock(), &row, 0 .. 2+4+6+8+10).unwrap();
+        let segment = Segment::new(&db.db.lock(), &row, 0 .. 2+4+6+8+10, true).unwrap();
         assert_eq!(&get_frames(&db.db, &segment, |it| it.duration_90k), &[2, 4, 6, 8, 10]);
     }
 
@@ -545,7 +564,7 @@ mod tests {
         encoder.add_sample(0, 3, true, &mut r);
         let db = TestDb::new(RealClocks {});
         let row = db.insert_recording_from_encoder(r);
-        let segment = Segment::new(&db.db.lock(), &row, 0 .. 2).unwrap();
+        let segment = Segment::new(&db.db.lock(), &row, 0 .. 2, true).unwrap();
         assert_eq!(&get_frames(&db.db, &segment, |it| it.bytes), &[1, 2, 3]);
     }
 

db/testutil.rs

@@ -187,7 +187,8 @@ pub fn add_dummy_recordings_to_db(db: &db::Database, num: usize) {
     let mut recording = db::RecordingToInsert {
         sample_file_bytes: 30104460,
         start: recording::Time(1430006400i64 * TIME_UNITS_PER_SEC),
-        duration_90k: 5399985,
+        media_duration_90k: 5399985,
+        wall_duration_90k: 5399985,
         video_samples: 1800,
         video_sync_samples: 60,
         video_sample_entry_id: video_sample_entry_id,
@@ -197,7 +198,7 @@ pub fn add_dummy_recordings_to_db(db: &db::Database, num: usize) {
     };
     for _ in 0..num {
         let (id, _) = db.add_recording(TEST_STREAM_ID, recording.clone()).unwrap();
-        recording.start += recording::Duration(recording.duration_90k as i64);
+        recording.start += recording::Duration(recording.wall_duration_90k as i64);
         recording.run_offset += 1;
         db.mark_synced(id).unwrap();
     }

db/writer.rs

@@ -558,10 +558,6 @@ struct InnerWriter<F: FileWriter> {
     e: recording::SampleIndexEncoder,
     id: CompositeId,
 
-    /// The pts, relative to the start of this segment and in 90kHz units, up until which live
-    /// segments have been sent out. Initially 0.
-    completed_live_segment_media_off_90k: i32,
-
     hasher: blake3::Hasher,
 
     /// The start time of this segment, based solely on examining the local clock after frames in
@@ -634,7 +630,6 @@ impl<'a, C: Clocks + Clone, D: DirWriter> Writer<'a, C, D> {
             r,
             e: recording::SampleIndexEncoder::new(),
             id,
-            completed_live_segment_media_off_90k: 0,
             hasher: blake3::Hasher::new(),
             local_start: recording::Time(i64::max_value()),
             unindexed_sample: None,
@@ -666,29 +661,14 @@ impl<'a, C: Clocks + Clone, D: DirWriter> Writer<'a, C, D> {
         if let Some(unindexed) = w.unindexed_sample.take() {
             let duration = i32::try_from(pts_90k - i64::from(unindexed.pts_90k))?;
             if duration <= 0 {
-                // Restore invariant.
-                w.unindexed_sample = Some(unindexed);
+                w.unindexed_sample = Some(unindexed); // restore invariant.
                 bail!("pts not monotonically increasing; got {} then {}",
                       unindexed.pts_90k, pts_90k);
             }
-            let d = match w.add_sample(duration, unindexed.len, unindexed.is_key,
-                                       unindexed.local_time) {
-                Ok(d) => d,
-                Err(e) => {
-                    // Restore invariant.
-                    w.unindexed_sample = Some(unindexed);
-                    return Err(e);
-                },
-            };
-
-            // If the sample `write` was called on is a key frame, then the prior frames (including
-            // the one we just flushed) represent a live segment. Send it out.
-            if is_key {
-                self.db.lock().send_live_segment(self.stream_id, db::LiveSegment {
-                    recording: w.id.recording(),
-                    media_off_90k: w.completed_live_segment_media_off_90k .. d,
-                }).unwrap();
-                w.completed_live_segment_media_off_90k = d;
+            if let Err(e) = w.add_sample(duration, unindexed.len, unindexed.is_key,
+                                         unindexed.local_time, self.db, self.stream_id) {
+                w.unindexed_sample = Some(unindexed); // restore invariant.
+                return Err(e);
             }
         }
         let mut remaining = pkt;
@@ -726,12 +706,14 @@ fn clamp(v: i64, min: i64, max: i64) -> i64 {
 }
 
 impl<F: FileWriter> InnerWriter<F> {
-    /// Returns the total media duration of the `RecordingToInsert` (needed for live view path).
-    fn add_sample(&mut self, duration_90k: i32, bytes: i32, is_key: bool,
-                  pkt_local_time: recording::Time) -> Result<i32, Error> {
+    fn add_sample<C: Clocks + Clone>(&mut self, duration_90k: i32, bytes: i32, is_key: bool,
+                                     pkt_local_time: recording::Time, db: &db::Database<C>,
+                                     stream_id: i32)
+                                     -> Result<(), Error> {
         let mut l = self.r.lock();
 
         // design/time.md explains these time manipulations in detail.
+        let prev_media_duration_90k = l.media_duration_90k;
         let media_duration_90k = l.media_duration_90k + duration_90k;
         let local_start =
             cmp::min(self.local_start,
@@ -754,7 +736,13 @@ impl<F: FileWriter> InnerWriter<F> {
         l.start = start;
         self.local_start = local_start;
         self.e.add_sample(duration_90k, bytes, is_key, &mut l);
-        Ok(media_duration_90k)
+        drop(l);
+        db.lock().send_live_segment(stream_id, db::LiveSegment {
+            recording: self.id.recording(),
+            is_key,
+            media_off_90k: prev_media_duration_90k .. media_duration_90k,
+        }).unwrap();
+        Ok(())
     }
 
     fn close<C: Clocks + Clone>(mut self, channel: &SyncerChannel<F>, next_pts: Option<i64>,
@@ -766,14 +754,10 @@ impl<F: FileWriter> InnerWriter<F> {
         };
         let blake3 = self.hasher.finalize();
         let (run_offset, end);
-        let d = self.add_sample(last_sample_duration, unindexed.len, unindexed.is_key,
-                                unindexed.local_time)?;
+        self.add_sample(last_sample_duration, unindexed.len, unindexed.is_key,
+                        unindexed.local_time, db, stream_id)?;
 
         // This always ends a live segment.
-        db.lock().send_live_segment(stream_id, db::LiveSegment {
-            recording: self.id.recording(),
-            media_off_90k: self.completed_live_segment_media_off_90k .. d,
-        }).unwrap();
         let wall_duration;
         {
             let mut l = self.r.lock();