make Writer enforce maximum recording duration

My installation recently somehow ended up with a recording with a
duration of 503793844 90,000ths of a second, way over the maximum of 5
minutes. (Looks like the machine was pretty unresponsive at the time
and/or having network problems.)

When this happens, the system really spirals. Every flush afterward (12
per minute with my installation) fails with a CHECK constraint failure
on the recording table. It never gives up on that recording. /var/log
fills pretty quickly as this failure is extremely verbose (a stack
trace, and a line for each byte of video_index). Eventually the sample
file dirs fill up too as it continues writing video samples while GC is
stuck. The video samples are useless anyway; given that they're not
referenced in the database, they'll be deleted on next startup.

This ensures the offending recording is never added to the database, so
we don't get the same persistent problem. Instead, writing to the
recording will fail. The stream will drop and be retried. If the
underlying condition that caused a too-long recording (many
non-key-frames, or the camera returning a crazy duration, or the
monotonic clock jumping forward extremely, or something) has gone away,
the system should recover.
This commit is contained in:
Scott Lamb
2019-01-29 08:26:36 -08:00
parent 3ba3bf2b18
commit c271cfa2b5
4 changed files with 56 additions and 42 deletions

View File

@@ -1916,7 +1916,7 @@ mod tests {
for i in 1..6 {
let duration_90k = 2 * i;
let bytes = 3 * i;
encoder.add_sample(duration_90k, bytes, true, &mut r);
encoder.add_sample(duration_90k, bytes, true, &mut r).unwrap();
}
// Time range [2, 2+4+6+8) means the 2nd, 3rd, and 4th samples should be included.
@@ -1970,7 +1970,7 @@ mod tests {
for i in 1..6 {
let duration_90k = 2 * i;
let bytes = 3 * i;
encoder.add_sample(duration_90k, bytes, (i % 2) == 1, &mut r);
encoder.add_sample(duration_90k, bytes, (i % 2) == 1, &mut r).unwrap();
}
// Time range [2+4+6, 2+4+6+8) means the 4th sample should be included.
@@ -2039,14 +2039,14 @@ mod tests {
let mut encoders = Vec::new();
let mut r = db::RecordingToInsert::default();
let mut encoder = recording::SampleIndexEncoder::new();
encoder.add_sample(1, 1, true, &mut r);
encoder.add_sample(2, 2, false, &mut r);
encoder.add_sample(3, 3, true, &mut r);
encoder.add_sample(1, 1, true, &mut r).unwrap();
encoder.add_sample(2, 2, false, &mut r).unwrap();
encoder.add_sample(3, 3, true, &mut r).unwrap();
encoders.push(r);
let mut r = db::RecordingToInsert::default();
let mut encoder = recording::SampleIndexEncoder::new();
encoder.add_sample(4, 4, true, &mut r);
encoder.add_sample(5, 5, false, &mut r);
encoder.add_sample(4, 4, true, &mut r).unwrap();
encoder.add_sample(5, 5, false, &mut r).unwrap();
encoders.push(r);
// This should include samples 3 and 4 only, both sync frames.
@@ -2076,12 +2076,12 @@ mod tests {
let mut encoders = Vec::new();
let mut r = db::RecordingToInsert::default();
let mut encoder = recording::SampleIndexEncoder::new();
encoder.add_sample(2, 1, true, &mut r);
encoder.add_sample(3, 2, false, &mut r);
encoder.add_sample(2, 1, true, &mut r).unwrap();
encoder.add_sample(3, 2, false, &mut r).unwrap();
encoders.push(r);
let mut r = db::RecordingToInsert::default();
let mut encoder = recording::SampleIndexEncoder::new();
encoder.add_sample(0, 3, true, &mut r);
encoder.add_sample(0, 3, true, &mut r).unwrap();
encoders.push(r);
// Multi-segment recording with an edit list, encoding with a zero-duration recording.
@@ -2104,7 +2104,7 @@ mod tests {
for i in 1..6 {
let duration_90k = 2 * i;
let bytes = 3 * i;
encoder.add_sample(duration_90k, bytes, (i % 2) == 1, &mut r);
encoder.add_sample(duration_90k, bytes, (i % 2) == 1, &mut r).unwrap();
}
// Time range [2+4+6, 2+4+6+8+1) means the 4th sample and part of the 5th are included.

View File

@@ -140,7 +140,7 @@ impl<'a, C, S> Streamer<'a, C, S> where C: 'a + Clocks + Clone, S: 'a + stream::
if frame_realtime.sec > r && pkt.is_key() {
trace!("{}: write on normal rotation", self.short_name);
let _t = TimerGuard::new(&clocks, || "closing writer");
w.close(Some(pts));
w.close(Some(pts))?;
None
} else {
Some(r)
@@ -179,7 +179,7 @@ impl<'a, C, S> Streamer<'a, C, S> where C: 'a + Clocks + Clone, S: 'a + stream::
}
if rotate.is_some() {
let _t = TimerGuard::new(&clocks, || "closing writer");
w.close(None);
w.close(None)?;
}
Ok(())
}