fully implement json handling as in spec

This is a significant milestone; now the Rust branch matches the C++ branch's
features.

In the process, I switched from using serde_derive (which requires nightly
Rust) to serde_codegen (which does not). It was easier than I thought it'd
be. I'm getting close to no longer requiring nightly Rust.

src/db.rs

@@ -61,7 +61,6 @@ use lru_cache::LruCache;
 use openssl::crypto::hash;
 use recording::{self, TIME_UNITS_PER_SEC};
 use rusqlite;
-use serde::ser::{Serialize, Serializer};
 use std::collections::BTreeMap;
 use std::cell::RefCell;
 use std::cmp;
@@ -231,18 +230,26 @@ impl CameraDayKey {
         write!(&mut s.0[..], "{}", tm.strftime("%Y-%m-%d")?)?;
         Ok(s)
     }
-}
 
-impl Serialize for CameraDayKey {
-    /// Serializes as a string, not as the default bytes.
-    /// serde_json will only allow string keys for objects.
-    fn serialize<S>(&self, serializer: &mut S) -> Result<(), S::Error> where S: Serializer {
-        serializer.serialize_str(str::from_utf8(&self.0[..]).expect("days are always UTF-8"))
+    pub fn bounds(&self) -> Range<recording::Time> {
+        let mut my_tm = time::strptime(self.as_ref(), "%Y-%m-%d").expect("days must be parseable");
+        let start = recording::Time(my_tm.to_timespec().sec * recording::TIME_UNITS_PER_SEC);
+        my_tm.tm_isdst = -1;
+        my_tm.tm_hour = 0;
+        my_tm.tm_min = 0;
+        my_tm.tm_sec = 0;
+        my_tm.tm_mday += 1;
+        let end = recording::Time(my_tm.to_timespec().sec * recording::TIME_UNITS_PER_SEC);
+        start .. end
     }
 }
 
+impl AsRef<str> for CameraDayKey {
+    fn as_ref(&self) -> &str { str::from_utf8(&self.0[..]).expect("days are always UTF-8") }
+}
+
 /// In-memory state about a particular camera on a particular day.
-#[derive(Copy, Clone, Debug, PartialEq, Eq, Serialize)]
+#[derive(Copy, Clone, Debug, PartialEq, Eq)]
 pub struct CameraDayValue {
     /// The number of recordings that overlap with this day. Note that `adjust_day` automatically
     /// prunes days with 0 recordings.
@@ -255,7 +262,7 @@ pub struct CameraDayValue {
 }
 
 /// In-memory state about a camera.
-#[derive(Debug, Serialize)]
+#[derive(Debug)]
 pub struct Camera {
     pub id: i32,
     pub uuid: Uuid,
@@ -270,7 +277,6 @@ pub struct Camera {
 
     /// The time range of recorded data associated with this camera (minimum start time and maximum
     /// end time). `None` iff there are no recordings for this camera.
-    #[serde(skip_serializing)]
     pub range: Option<Range<recording::Time>>,
     pub sample_file_bytes: i64,
 

src/main.rs

@@ -52,7 +52,6 @@ extern crate openssl;
 extern crate regex;
 extern crate rustc_serialize;
 extern crate serde;
-#[macro_use] extern crate serde_derive;
 extern crate serde_json;
 extern crate slog;
 extern crate slog_envlogger;
@@ -82,6 +81,7 @@ mod recording;
 mod resource;
 mod stream;
 mod streamer;
+mod strutil;
 #[cfg(test)] mod testutil;
 mod web;
 

src/mp4.rs

@@ -99,6 +99,7 @@ use std::io;
 use std::ops::Range;
 use std::mem;
 use std::sync::{Arc, MutexGuard};
+use strutil;
 use time::Timespec;
 
 /// This value should be incremented any time a change is made to this file that causes different
@@ -513,18 +514,6 @@ macro_rules! write_length {
     }}
 }
 
-/// Returns a hex-encoded version of the input.
-fn hex(raw: &[u8]) -> String {
-    const HEX_CHARS: [u8; 16] = [b'0', b'1', b'2', b'3', b'4', b'5', b'6', b'7',
-                                 b'8', b'9', b'a', b'b', b'c', b'd', b'e', b'f'];
-    let mut hex = Vec::with_capacity(2 * raw.len());
-    for b in raw {
-        hex.push(HEX_CHARS[((b & 0xf0) >> 4) as usize]);
-        hex.push(HEX_CHARS[( b & 0x0f      ) as usize]);
-    }
-    unsafe { String::from_utf8_unchecked(hex) }
-}
-
 impl Mp4FileBuilder {
     pub fn new() -> Self {
         Mp4FileBuilder{
@@ -669,7 +658,7 @@ impl Mp4FileBuilder {
             video_sample_entries: self.video_sample_entries,
             initial_sample_byte_pos: initial_sample_byte_pos,
             last_modified: header::HttpDate(time::at(Timespec::new(max_end.unix_seconds(), 0))),
-            etag: header::EntityTag::strong(hex(&etag.finish()?)),
+            etag: header::EntityTag::strong(strutil::hex(&etag.finish()?)),
         })
     }
 
@@ -1202,6 +1191,7 @@ mod tests {
     use std::path::Path;
     use std::sync::Arc;
     use std::str;
+    use strutil;
     use super::*;
     use stream::{self, Opener, Stream};
     use testutil::{self, TestDb};
@@ -1663,7 +1653,7 @@ mod tests {
         // here fails, it can be updated, but the etag must change as well! Otherwise clients may
         // combine ranges from the new format with ranges from the old format.
         let sha1 = digest(&mp4);
-        assert_eq!("1e5331e8371bd97ac3158b3a86494abc87cdc70e", super::hex(&sha1[..]));
+        assert_eq!("1e5331e8371bd97ac3158b3a86494abc87cdc70e", strutil::hex(&sha1[..]));
         const EXPECTED_ETAG: &'static str = "3c48af4dbce2024db07f27a00789b6af774a8c89";
         assert_eq!(Some(&header::EntityTag::strong(EXPECTED_ETAG.to_owned())), mp4.etag());
         drop(db.syncer_channel);
@@ -1683,7 +1673,7 @@ mod tests {
         // here fails, it can be updated, but the etag must change as well! Otherwise clients may
         // combine ranges from the new format with ranges from the old format.
         let sha1 = digest(&mp4);
-        assert_eq!("de382684a471f178e4e3a163762711b0653bfd83", super::hex(&sha1[..]));
+        assert_eq!("de382684a471f178e4e3a163762711b0653bfd83", strutil::hex(&sha1[..]));
         const EXPECTED_ETAG: &'static str = "c24d7af372e5d8f66f4feb6e3a5cd43828392371";
         assert_eq!(Some(&header::EntityTag::strong(EXPECTED_ETAG.to_owned())), mp4.etag());
         drop(db.syncer_channel);
@@ -1703,7 +1693,7 @@ mod tests {
         // here fails, it can be updated, but the etag must change as well! Otherwise clients may
         // combine ranges from the new format with ranges from the old format.
         let sha1 = digest(&mp4);
-        assert_eq!("685e026af44204bc9cc52115c5e17058e9fb7c70", super::hex(&sha1[..]));
+        assert_eq!("685e026af44204bc9cc52115c5e17058e9fb7c70", strutil::hex(&sha1[..]));
         const EXPECTED_ETAG: &'static str = "870e2b3cfef4a988951344b32e53af0d4496894d";
         assert_eq!(Some(&header::EntityTag::strong(EXPECTED_ETAG.to_owned())), mp4.etag());
         drop(db.syncer_channel);
@@ -1723,7 +1713,7 @@ mod tests {
         // here fails, it can be updated, but the etag must change as well! Otherwise clients may
         // combine ranges from the new format with ranges from the old format.
         let sha1 = digest(&mp4);
-        assert_eq!("e0d28ddf08e24575a82657b1ce0b2da73f32fd88", super::hex(&sha1[..]));
+        assert_eq!("e0d28ddf08e24575a82657b1ce0b2da73f32fd88", strutil::hex(&sha1[..]));
         const EXPECTED_ETAG: &'static str = "71c329188a2cd175c8d61492a9789e242af06c05";
         assert_eq!(Some(&header::EntityTag::strong(EXPECTED_ETAG.to_owned())), mp4.etag());
         drop(db.syncer_channel);

src/recording.rs

@@ -46,7 +46,7 @@ pub const DESIRED_RECORDING_DURATION: i64 = 60 * TIME_UNITS_PER_SEC;
 pub const MAX_RECORDING_DURATION: i64 = 5 * 60 * TIME_UNITS_PER_SEC;
 
 /// A time specified as 90,000ths of a second since 1970-01-01 00:00:00 UTC.
-#[derive(Clone, Copy, Debug, Eq, Ord, PartialEq, PartialOrd, Serialize)]
+#[derive(Clone, Copy, Debug, Eq, Ord, PartialEq, PartialOrd)]
 pub struct Time(pub i64);
 
 impl Time {
@@ -86,7 +86,7 @@ impl fmt::Display for Time {
 
 /// A duration specified in 1/90,000ths of a second.
 /// Durations are typically non-negative, but a `db::CameraDayValue::duration` may be negative.
-#[derive(Clone, Copy, Debug, Eq, Ord, PartialEq, PartialOrd, Serialize)]
+#[derive(Clone, Copy, Debug, Eq, Ord, PartialEq, PartialOrd)]
 pub struct Duration(pub i64);
 
 impl fmt::Display for Duration {

src/serde_types.in.rs

@@ -0,0 +1,131 @@
+// This file is part of Moonfire NVR, a security camera digital video recorder.
+// Copyright (C) 2016 Scott Lamb <slamb@slamb.org>
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// In addition, as a special exception, the copyright holders give
+// permission to link the code of portions of this program with the
+// OpenSSL library under certain conditions as described in each
+// individual source file, and distribute linked combinations including
+// the two.
+//
+// You must obey the GNU General Public License in all respects for all
+// of the code used other than OpenSSL. If you modify file(s) with this
+// exception, you may extend this exception to your version of the
+// file(s), but you are not obligated to do so. If you do not wish to do
+// so, delete this exception statement from your version. If you delete
+// this exception statement from all source files in the program, then
+// also delete it here.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+use db;
+use serde::ser::Serializer;
+use std::collections::BTreeMap;
+use uuid::Uuid;
+
+#[derive(Debug, Serialize)]
+pub struct ListCameras<'a> {
+    // Use a custom serializer which presents the map's values as a sequence.
+    #[serde(serialize_with = "ListCameras::serialize_cameras")]
+    pub cameras: &'a BTreeMap<i32, db::Camera>,
+}
+
+/// JSON serialization wrapper for a single camera when processing `/cameras/` and
+/// `/cameras/<uuid>/`. See `design/api.md` for details.
+#[derive(Debug, Serialize)]
+pub struct Camera<'a> {
+    pub uuid: Uuid,
+    pub short_name: &'a str,
+    pub description: &'a str,
+    pub retain_bytes: i64,
+    pub min_start_time_90k: Option<i64>,
+    pub max_end_time_90k: Option<i64>,
+    pub total_duration_90k: i64,
+    pub total_sample_file_bytes: i64,
+
+    #[serde(serialize_with = "Camera::serialize_days")]
+    pub days: Option<&'a BTreeMap<db::CameraDayKey, db::CameraDayValue>>,
+}
+
+impl<'a> Camera<'a> {
+    pub fn new(c: &'a db::Camera, include_days: bool) -> Self {
+        Camera{
+            uuid: c.uuid,
+            short_name: &c.short_name,
+            description: &c.description,
+            retain_bytes: c.retain_bytes,
+            min_start_time_90k: c.range.as_ref().map(|r| r.start.0),
+            max_end_time_90k: c.range.as_ref().map(|r| r.end.0),
+            total_duration_90k: c.duration.0,
+            total_sample_file_bytes: c.sample_file_bytes,
+            days: if include_days { Some(&c.days) } else { None },
+        }
+    }
+
+    fn serialize_days<S>(days: &Option<&BTreeMap<db::CameraDayKey, db::CameraDayValue>>,
+                         serializer: &mut S) -> Result<(), S::Error>
+    where S: Serializer {
+        let days = match *days {
+            Some(d) => d,
+            None => return Ok(()),
+        };
+        let mut state = serializer.serialize_map(Some(days.len()))?;
+        for (k, v) in days {
+            serializer.serialize_map_key(&mut state, k.as_ref())?;
+            let bounds = k.bounds();
+            serializer.serialize_map_value(&mut state, &CameraDayValue{
+                start_time_90k: bounds.start.0,
+                end_time_90k: bounds.end.0,
+                total_duration_90k: v.duration.0,
+            })?;
+        }
+        serializer.serialize_map_end(state)
+    }
+}
+
+#[derive(Debug, Serialize)]
+struct CameraDayValue {
+    pub start_time_90k: i64,
+    pub end_time_90k: i64,
+    pub total_duration_90k: i64,
+}
+
+impl<'a> ListCameras<'a> {
+    /// Serializes cameras as a list (rather than a map), wrapping each camera in the
+    /// `ListCamerasCamera` type to tweak the data returned.
+    fn serialize_cameras<S>(cameras: &BTreeMap<i32, db::Camera>,
+                            serializer: &mut S) -> Result<(), S::Error>
+    where S: Serializer {
+        let mut state = serializer.serialize_seq(Some(cameras.len()))?;
+        for c in cameras.values() {
+            serializer.serialize_seq_elt(&mut state, &Camera::new(c, false))?;
+        }
+        serializer.serialize_seq_end(state)
+    }
+}
+
+#[derive(Debug, Serialize)]
+pub struct ListRecordings {
+    pub recordings: Vec<Recording>,
+}
+
+#[derive(Debug, Serialize)]
+pub struct Recording {
+    pub start_time_90k: i64,
+    pub end_time_90k: i64,
+    pub sample_file_bytes: i64,
+    pub video_samples: i64,
+    pub video_sample_entry_width: u16,
+    pub video_sample_entry_height: u16,
+    pub video_sample_entry_sha1: String,
+}

src/strutil.rs

@@ -0,0 +1,41 @@
+// This file is part of Moonfire NVR, a security camera digital video recorder.
+// Copyright (C) 2016 Scott Lamb <slamb@slamb.org>
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// In addition, as a special exception, the copyright holders give
+// permission to link the code of portions of this program with the
+// OpenSSL library under certain conditions as described in each
+// individual source file, and distribute linked combinations including
+// the two.
+//
+// You must obey the GNU General Public License in all respects for all
+// of the code used other than OpenSSL. If you modify file(s) with this
+// exception, you may extend this exception to your version of the
+// file(s), but you are not obligated to do so. If you do not wish to do
+// so, delete this exception statement from your version. If you delete
+// this exception statement from all source files in the program, then
+// also delete it here.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+/// Returns a hex-encoded version of the input.
+pub fn hex(raw: &[u8]) -> String {
+    const HEX_CHARS: [u8; 16] = [b'0', b'1', b'2', b'3', b'4', b'5', b'6', b'7',
+                                 b'8', b'9', b'a', b'b', b'c', b'd', b'e', b'f'];
+    let mut hex = Vec::with_capacity(2 * raw.len());
+    for b in raw {
+        hex.push(HEX_CHARS[((b & 0xf0) >> 4) as usize]);
+        hex.push(HEX_CHARS[( b & 0x0f      ) as usize]);
+    }
+    unsafe { String::from_utf8_unchecked(hex) }
+}

src/web.rs

@@ -42,12 +42,11 @@ use mp4;
 use recording;
 use resource;
 use serde_json;
-use serde::ser::Serializer;
-use std::collections::BTreeMap;
 use std::fmt;
 use std::io::Write;
-use std::result;
+use std::ops::Range;
 use std::sync::{Arc,MutexGuard};
+use strutil;
 use time;
 use url::form_urlencoded;
 use uuid::Uuid;
@@ -60,6 +59,8 @@ lazy_static! {
     static ref HTML: mime::Mime = mime!(Text/Html);
 }
 
+mod json { include!(concat!(env!("OUT_DIR"), "/serde_types.rs")); }
+
 enum Path {
     CamerasList,              // "/" or "/cameras/"
     Camera(Uuid),             // "/cameras/<uuid>/"
@@ -180,25 +181,6 @@ pub struct Handler {
     dir: Arc<SampleFileDir>,
 }
 
-#[derive(Serialize)]
-struct ListCameras<'a> {
-    // Use a custom serializer which presents the map's values as a sequence.
-    #[serde(serialize_with = "ListCameras::serialize_cameras")]
-    cameras: &'a BTreeMap<i32, db::Camera>,
-}
-
-impl<'a> ListCameras<'a> {
-    fn serialize_cameras<S>(cameras: &BTreeMap<i32, db::Camera>,
-                            serializer: &mut S) -> result::Result<(), S::Error>
-    where S: Serializer {
-        let mut state = serializer.serialize_seq(Some(cameras.len()))?;
-        for c in cameras.values() {
-            serializer.serialize_seq_elt(&mut state, c)?;
-        }
-        serializer.serialize_seq_end(state)
-    }
-}
-
 impl Handler {
     pub fn new(db: Arc<db::Database>, dir: Arc<SampleFileDir>) -> Self {
         Handler{db: db, dir: dir}
@@ -215,7 +197,7 @@ impl Handler {
         let buf = {
             let db = self.db.lock();
             if json {
-                serde_json::to_vec(&ListCameras{cameras: db.cameras_by_id()})?
+                serde_json::to_vec(&json::ListCameras{cameras: db.cameras_by_id()})?
             } else {
                 self.list_cameras_html(db)?
             }
@@ -259,16 +241,17 @@ impl Handler {
         Ok(buf)
     }
 
-    fn camera(&self, uuid: Uuid, req: &server::Request, mut res: server::Response) -> Result<()> {
+    fn camera(&self, uuid: Uuid, query: &str, req: &server::Request, mut res: server::Response)
+              -> Result<()> {
         let json = is_json(req);
         let buf = {
             let db = self.db.lock();
             if json {
                 let camera = db.get_camera(uuid)
                                .ok_or_else(|| Error::new("no such camera".to_owned()))?;
-                serde_json::to_vec(&camera)?
+                serde_json::to_vec(&json::Camera::new(camera, true))?
             } else {
-                self.camera_html(db, uuid)?
+                self.camera_html(db, query, uuid)?
             }
         };
         res.headers_mut().set(header::ContentType(if json { JSON.clone() } else { HTML.clone() }));
@@ -276,7 +259,9 @@ impl Handler {
         Ok(())
     }
 
-    fn camera_html(&self, db: MutexGuard<db::LockedDatabase>, uuid: Uuid) -> Result<Vec<u8>> {
+    fn camera_html(&self, db: MutexGuard<db::LockedDatabase>, query: &str,
+                   uuid: Uuid) -> Result<Vec<u8>> {
+        let r = Handler::get_optional_range(query)?;
         let camera = db.get_camera(uuid)
                        .ok_or_else(|| Error::new("no such camera".to_owned()))?;
         let mut buf = Vec::new();
@@ -299,7 +284,6 @@ impl Handler {
             <th>fps</th><th>size</th><th>bitrate</th>\
             </tr>\n",
             HtmlEscaped(&camera.short_name), HtmlEscaped(&camera.description))?;
-        let r = recording::Time(i64::min_value()) .. recording::Time(i64::max_value());
 
         // Rather than listing each 60-second recording, generate a HTML row for aggregated .mp4
         // files of up to FORCE_SPLIT_DURATION each, provided there is no gap or change in video
@@ -324,10 +308,36 @@ impl Handler {
         Ok(buf)
     }
 
-    fn camera_recordings(&self, _uuid: Uuid, _req: &server::Request,
+    fn camera_recordings(&self, uuid: Uuid, query: &str, req: &server::Request,
                          mut res: server::Response) -> Result<()> {
-        *res.status_mut() = status::StatusCode::NotImplemented;
-        res.send(b"not implemented")?;
+        let r = Handler::get_optional_range(query)?;
+        if !is_json(req) {
+            *res.status_mut() = status::StatusCode::NotAcceptable;
+            res.send(b"only available for JSON requests")?;
+            return Ok(());
+        }
+        let mut out = json::ListRecordings{recordings: Vec::new()};
+        {
+            let db = self.db.lock();
+            let camera = db.get_camera(uuid)
+                           .ok_or_else(|| Error::new("no such camera".to_owned()))?;
+            db.list_aggregated_recordings(camera.id, &r, recording::Duration(i64::max_value()),
+                                          |row| {
+                out.recordings.push(json::Recording{
+                    start_time_90k: row.range.start.0,
+                    end_time_90k: row.range.end.0,
+                    sample_file_bytes: row.sample_file_bytes,
+                    video_samples: row.video_samples,
+                    video_sample_entry_width: row.video_sample_entry.width,
+                    video_sample_entry_height: row.video_sample_entry.height,
+                    video_sample_entry_sha1: strutil::hex(&row.video_sample_entry.sha1),
+                });
+                Ok(())
+            })?;
+        }
+        let buf = serde_json::to_vec(&out)?;
+        res.headers_mut().set(header::ContentType(JSON.clone()));
+        res.send(&buf)?;
         Ok(())
     }
 
@@ -409,6 +419,22 @@ impl Handler {
         resource::serve(&mp4, req, res)?;
         Ok(())
     }
+
+    /// Parses optional `start_time_90k` and `end_time_90k` query parameters, defaulting to the
+    /// full range of possible values.
+    fn get_optional_range(query: &str) -> Result<Range<recording::Time>> {
+        let mut start = i64::min_value();
+        let mut end = i64::max_value();
+        for (key, value) in form_urlencoded::parse(query.as_bytes()) {
+            let (key, value) = (key.borrow(), value.borrow());
+            match key {
+                "start_time_90k" => start = i64::from_str(value)?,
+                "end_time_90k" => end = i64::from_str(value)?,
+                _ => {},
+            }
+        };
+        Ok(recording::Time(start) .. recording::Time(end))
+    }
 }
 
 impl server::Handler for Handler {
@@ -416,8 +442,8 @@ impl server::Handler for Handler {
         let (path, query) = get_path_and_query(&req.uri);
         let res = match decode_path(path) {
             Path::CamerasList => self.list_cameras(&req, res),
-            Path::Camera(uuid) => self.camera(uuid, &req, res),
-            Path::CameraRecordings(uuid) => self.camera_recordings(uuid, &req, res),
+            Path::Camera(uuid) => self.camera(uuid, query, &req, res),
+            Path::CameraRecordings(uuid) => self.camera_recordings(uuid, query, &req, res),
             Path::CameraViewMp4(uuid) => self.camera_view_mp4(uuid, query, &req, res),
             Path::NotFound => self.not_found(res),
         };