allow listing and viewing uncommitted recordings

There may be considerable lag between being fully written and being committed
when using the flush_if_sec feature. Additionally, this is a step toward
listing and viewing recordings before they're fully written. That's a
considerable delay: 60 to 120 seconds for the first recording of a run,
0 to 60 seconds for subsequent recordings.

These recordings aren't yet included in the information returned by
/api/?days=true. They probably should be, but small steps.
This commit is contained in:
Scott Lamb
2018-03-02 11:38:11 -08:00
parent b17761e871
commit 45f7b30619
8 changed files with 171 additions and 34 deletions

View File

@@ -173,6 +173,10 @@ pub struct Recording {
pub video_samples: i64,
pub video_sample_entry_sha1: String,
pub start_id: i32,
pub open_id: u32,
#[serde(skip_serializing_if = "Option::is_none")]
pub first_uncommitted: Option<i32>,
#[serde(skip_serializing_if = "Option::is_none")]
pub end_id: Option<i32>,

View File

@@ -804,10 +804,11 @@ impl FileBuilder {
}
// Update the etag to reflect this segment.
let mut data = [0_u8; 24];
let mut data = [0_u8; 28];
let mut cursor = io::Cursor::new(&mut data[..]);
cursor.write_i64::<BigEndian>(s.s.id.0)?;
cursor.write_i64::<BigEndian>(s.s.start.0)?;
cursor.write_u32::<BigEndian>(s.s.open_id)?;
cursor.write_i32::<BigEndian>(d.start)?;
cursor.write_i32::<BigEndian>(d.end)?;
etag.update(cursor.into_inner())?;
@@ -2103,7 +2104,7 @@ mod tests {
// combine ranges from the new format with ranges from the old format.
let sha1 = digest(&mp4);
assert_eq!("1e5331e8371bd97ac3158b3a86494abc87cdc70e", strutil::hex(&sha1[..]));
const EXPECTED_ETAG: &'static str = "c56ef7eb3b4a713ceafebc3dc7958bd9e62a2fae";
const EXPECTED_ETAG: &'static str = "04298efb2df0cc45a6cea65dfdf2e817a3b42ca8";
assert_eq!(Some(header::EntityTag::strong(EXPECTED_ETAG.to_owned())), mp4.etag());
drop(db.syncer_channel);
db.db.lock().clear_on_flush();
@@ -2124,7 +2125,7 @@ mod tests {
// combine ranges from the new format with ranges from the old format.
let sha1 = digest(&mp4);
assert_eq!("de382684a471f178e4e3a163762711b0653bfd83", strutil::hex(&sha1[..]));
const EXPECTED_ETAG: &'static str = "3bdc2c8ce521df50155d0ca4d7497ada448fa7c3";
const EXPECTED_ETAG: &'static str = "16a4f6348560c3de0d149675dccba21ef7906be3";
assert_eq!(Some(header::EntityTag::strong(EXPECTED_ETAG.to_owned())), mp4.etag());
drop(db.syncer_channel);
db.db.lock().clear_on_flush();
@@ -2145,7 +2146,7 @@ mod tests {
// combine ranges from the new format with ranges from the old format.
let sha1 = digest(&mp4);
assert_eq!("d655945f94e18e6ed88a2322d27522aff6f76403", strutil::hex(&sha1[..]));
const EXPECTED_ETAG: &'static str = "3986d3bd9b866c3455fb7359fb134aa2d9107af7";
const EXPECTED_ETAG: &'static str = "80e418b029e81aa195f90aa6b806015a5030e5be";
assert_eq!(Some(header::EntityTag::strong(EXPECTED_ETAG.to_owned())), mp4.etag());
drop(db.syncer_channel);
db.db.lock().clear_on_flush();
@@ -2166,7 +2167,7 @@ mod tests {
// combine ranges from the new format with ranges from the old format.
let sha1 = digest(&mp4);
assert_eq!("e0d28ddf08e24575a82657b1ce0b2da73f32fd88", strutil::hex(&sha1[..]));
const EXPECTED_ETAG: &'static str = "9e789398c9a71ca834fec8fbc55b389f99d12dda";
const EXPECTED_ETAG: &'static str = "5bfea0f20108a7c5b77ef1e21d82ef2abc29540f";
assert_eq!(Some(header::EntityTag::strong(EXPECTED_ETAG.to_owned())), mp4.etag());
drop(db.syncer_channel);
db.db.lock().clear_on_flush();

View File

@@ -61,8 +61,9 @@ use uuid::Uuid;
lazy_static! {
/// Regex used to parse the `s` query parameter to `view.mp4`.
/// As described in `design/api.md`, this is of the form
/// `START_ID[-END_ID][.[REL_START_TIME]-[REL_END_TIME]]`.
static ref SEGMENTS_RE: Regex = Regex::new(r"^(\d+)(-\d+)?(?:\.(\d+)?-(\d+)?)?$").unwrap();
/// `START_ID[-END_ID][@OPEN_ID][.[REL_START_TIME]-[REL_END_TIME]]`.
static ref SEGMENTS_RE: Regex =
Regex::new(r"^(\d+)(-\d+)?(@\d+)?(?:\.(\d+)?-(\d+)?)?$").unwrap();
}
enum Path {
@@ -135,6 +136,7 @@ fn decode_path(path: &str) -> Path {
#[derive(Debug, Eq, PartialEq)]
struct Segments {
ids: Range<i32>,
open_id: Option<u32>,
start_time: i64,
end_time: Option<i64>,
}
@@ -144,17 +146,21 @@ impl Segments {
let caps = SEGMENTS_RE.captures(input).ok_or(())?;
let ids_start = i32::from_str(caps.get(1).unwrap().as_str()).map_err(|_| ())?;
let ids_end = match caps.get(2) {
Some(e) => i32::from_str(&e.as_str()[1..]).map_err(|_| ())?,
Some(m) => i32::from_str(&m.as_str()[1..]).map_err(|_| ())?,
None => ids_start,
} + 1;
let open_id = match caps.get(3) {
Some(m) => Some(u32::from_str(&m.as_str()[1..]).map_err(|_| ())?),
None => None,
};
if ids_start < 0 || ids_end <= ids_start {
return Err(());
}
let start_time = caps.get(3).map_or(Ok(0), |m| i64::from_str(m.as_str())).map_err(|_| ())?;
let start_time = caps.get(4).map_or(Ok(0), |m| i64::from_str(m.as_str())).map_err(|_| ())?;
if start_time < 0 {
return Err(());
}
let end_time = match caps.get(4) {
let end_time = match caps.get(5) {
Some(v) => {
let e = i64::from_str(v.as_str()).map_err(|_| ())?;
if e <= start_time {
@@ -164,10 +170,11 @@ impl Segments {
},
None => None
};
Ok(Segments{
Ok(Segments {
ids: ids_start .. ids_end,
start_time: start_time,
end_time: end_time,
open_id,
start_time,
end_time,
})
}
}
@@ -262,10 +269,12 @@ impl ServiceInner {
let vse = db.video_sample_entries_by_id().get(&row.video_sample_entry_id).unwrap();
out.recordings.push(json::Recording {
start_id: row.ids.start,
end_id: if end == row.ids.start + 1 { None } else { Some(end) },
end_id: if end == row.ids.start { None } else { Some(end) },
start_time_90k: row.time.start.0,
end_time_90k: row.time.end.0,
sample_file_bytes: row.sample_file_bytes,
open_id: row.open_id,
first_uncommitted: row.first_uncommitted,
video_samples: row.video_samples,
video_sample_entry_width: vse.width,
video_sample_entry_height: vse.height,
@@ -331,6 +340,13 @@ impl ServiceInner {
db.list_recordings_by_id(stream_id, s.ids.clone(), &mut |r| {
let recording_id = r.id.recording();
if let Some(o) = s.open_id {
if r.open_id != o {
bail!("recording {} has open id {}, requested {}",
r.id, r.open_id, o);
}
}
// Check for missing recordings.
match prev {
None if recording_id == s.ids.start => {},
@@ -507,21 +523,25 @@ mod tests {
#[test]
fn test_segments() {
testutil::init();
assert_eq!(Segments{ids: 1..2, start_time: 0, end_time: None},
assert_eq!(Segments{ids: 1..2, open_id: None, start_time: 0, end_time: None},
Segments::parse("1").unwrap());
assert_eq!(Segments{ids: 1..2, start_time: 26, end_time: None},
assert_eq!(Segments{ids: 1..2, open_id: Some(42), start_time: 0, end_time: None},
Segments::parse("1@42").unwrap());
assert_eq!(Segments{ids: 1..2, open_id: None, start_time: 26, end_time: None},
Segments::parse("1.26-").unwrap());
assert_eq!(Segments{ids: 1..2, start_time: 0, end_time: Some(42)},
assert_eq!(Segments{ids: 1..2, open_id: Some(42), start_time: 26, end_time: None},
Segments::parse("1@42.26-").unwrap());
assert_eq!(Segments{ids: 1..2, open_id: None, start_time: 0, end_time: Some(42)},
Segments::parse("1.-42").unwrap());
assert_eq!(Segments{ids: 1..2, start_time: 26, end_time: Some(42)},
assert_eq!(Segments{ids: 1..2, open_id: None, start_time: 26, end_time: Some(42)},
Segments::parse("1.26-42").unwrap());
assert_eq!(Segments{ids: 1..6, start_time: 0, end_time: None},
assert_eq!(Segments{ids: 1..6, open_id: None, start_time: 0, end_time: None},
Segments::parse("1-5").unwrap());
assert_eq!(Segments{ids: 1..6, start_time: 26, end_time: None},
assert_eq!(Segments{ids: 1..6, open_id: None, start_time: 26, end_time: None},
Segments::parse("1-5.26-").unwrap());
assert_eq!(Segments{ids: 1..6, start_time: 0, end_time: Some(42)},
assert_eq!(Segments{ids: 1..6, open_id: None, start_time: 0, end_time: Some(42)},
Segments::parse("1-5.-42").unwrap());
assert_eq!(Segments{ids: 1..6, start_time: 26, end_time: Some(42)},
assert_eq!(Segments{ids: 1..6, open_id: None, start_time: 26, end_time: Some(42)},
Segments::parse("1-5.26-42").unwrap());
}
}