version the sqlite3 database schema

See guide/schema.md for instructions on upgrading past this commit.

README.md

@@ -148,15 +148,18 @@ Once prerequisites are installed, Moonfire NVR can be built as follows:
 Moonfire NVR should be run under a dedicated user. It keeps two kinds of
 state:
 
-* a SQLite database, typically <1 GiB. It should be stored on flash if
-  available.
-* the "sample file directory", which holds the actual samples/frames of H.264
-  video. This should be quite large and typically is stored on a hard drive.
+   * a SQLite database, typically <1 GiB. It should be stored on flash if
+     available.
+   * the "sample file directory", which holds the actual samples/frames of
+     H.264 video. This should be quite large and typically is stored on a hard
+     drive.
 
-Both are intended to be accessed only by Moonfire NVR itself. However, the
-interface for adding new cameras is not yet written, so you will have to
-manually create the database and insert cameras with the `sqlite3` command line
-tool prior to starting Moonfire NVR.
+(See [guide/schema.md](guide/schema.md) for more information.)
+
+Both kinds of state are intended to be accessed only by Moonfire NVR itself.
+However, the interface for adding new cameras is not yet written, so you will
+have to manually create the database and insert cameras with the `sqlite3`
+command line tool prior to starting Moonfire NVR.
 
 Manual commands would look something like this:
 

design/schema.md

@@ -3,6 +3,10 @@
 Status: **current**. This is largely implemented; there is optimization and
 testing work left to do.
 
+This is the initial design for the most fundamental parts of the Moonfire NVR
+storage schema. See also [guide/schema.md](../guide/schema.md) for more
+administrator-focused documentation.
+
 ## Objective
 
 Goals:
@@ -40,8 +44,6 @@ Possible future goals:
 * record audio and/or other types of timestamped samples (such as
   [Xandem][xandem] tomography data).
 
-## Background
-
 ### Cameras
 
 Inexpensive modern ONVIF/PSIA IP security cameras, such as the $100

guide/schema.md

@@ -0,0 +1,79 @@
+# Moonfire NVR Schema Guide
+
+This document has notes about the Moonfire NVR storage schema. As described in
+[README.md](../README.md), this consists of two kinds of state:
+
+   * a SQLite database, typically <1 GiB. It should be stored on flash if
+     available.
+   * the "sample file directory", which holds the actual samples/frames of
+     H.264 video. This should be quite large and typically is stored on a hard
+     drive.
+
+## Upgrading
+
+The database schema includes a version number to quickly identify if a
+the database is compatible with a particular version of the software. Some
+software upgrades will require you to upgrade the database.
+
+### Unversioned to version 0
+
+Early versions of Moonfire NVR did not include the version information in the
+schema. You can manually add this information to your schema using the
+`sqlite3` commandline. This process is backward compatible, meaning that
+software versions that accept an unversioned database will also accept a
+version 0 database.
+
+Version 0 makes two changes:
+
+    * schema versioning, as described above.
+    * adding a column (`video_sync_samples`) to a database index to speed up
+      certain operations.
+
+First ensure Moonfire NVR is not running; if you are using systemd with the
+service name `moonfire-nvr`, you can do this as follows:
+
+    $ sudo systemctl stop moonfire-nvr
+
+The service takes a moment to shut down; wait until the following command
+reports that it is not running:
+
+    $ sudo systemctl status moonfire-nvr
+
+Then use `sqlite3` to manually edit the database. The default path is
+`/var/lib/moonfire-nvr/db/db`; if you've specified a different `--db_dir`,
+use that directory with a suffix of `/db`.
+
+   $ sudo -u moonfire-nvr sqlite3 /var/lib/moonfire-nvr/db/db
+   sqlite3>
+
+At the prompt, run the following commands:
+
+```sql
+begin transaction;
+
+create table version (
+  id integer primary key,
+  unix_time integer not null,
+  notes text
+);
+
+insert into version values (0, cast(strftime('%s', 'now') as int),
+                            'manual upgrade to version 0');
+
+drop index recording_cover;
+
+create index recording_cover on recording (
+  camera_id,
+  start_time_90k,
+  duration_90k,
+  video_samples,
+  video_sample_entry_id,
+  sample_file_bytes
+);
+
+commit transaction;
+```
+
+When you are done, you can restart the service:
+
+    $ sudo systemctl start moonfire-nvr

src/db.rs

@@ -69,6 +69,9 @@ use std::vec::Vec;
 use time;
 use uuid::Uuid;
 
+/// Expected schema version. See `guide/schema.md` for more information.
+pub const EXPECTED_VERSION: i32 = 0;
+
 const GET_RECORDING_SQL: &'static str =
     "select sample_file_uuid, video_index from recording where id = :id";
 
@@ -414,6 +417,7 @@ fn init_recordings(conn: &mut rusqlite::Connection, camera_id: i32, camera: &mut
     Ok(())
 }
 
+#[derive(Debug)]
 pub struct LockedDatabase {
     conn: rusqlite::Connection,
     state: State,
@@ -422,6 +426,7 @@ pub struct LockedDatabase {
 /// In-memory state from the database.
 /// This is separated out of `LockedDatabase` so that `Transaction` can mutably borrow `state`
 /// while its underlying `rusqlite::Transaction` is borrowing `conn`.
+#[derive(Debug)]
 struct State {
     cameras_by_id: BTreeMap<i32, Camera>,
     cameras_by_uuid: BTreeMap<Uuid, i32>,
@@ -959,6 +964,7 @@ impl LockedDatabase {
 /// The recording database. Abstracts away SQLite queries. Also maintains in-memory state
 /// (loaded on startup, and updated on successful commit) to avoid expensive scans over the
 /// recording table on common queries.
+#[derive(Debug)]
 pub struct Database(Mutex<LockedDatabase>);
 
 impl Database {
@@ -983,6 +989,36 @@ impl Database {
             order by
                 recording.start_time_90k
         "#, recording::MAX_RECORDING_DURATION);
+        {
+            use std::error::Error as E;
+            let ver: i32 = match conn.query_row("select max(id) from version", &[],
+                                                |row| row.get_checked::<_, i32>(0)) {
+                Ok(r) => r?,
+                Err(ref e) if e.description().starts_with("no such table: version") => {
+                    return Err(Error::new("no such table: version. \
+                                          \
+                                          If you are starting from an \
+                                          empty database, see README.md to complete the \
+                                          installation. If you are starting from 
+                                          complete the schema. If you are starting from a database \
+                                          that predates schema versioning, see guide/schema.md."
+                                          .to_owned()));
+                },
+                Err(e) => return Err(e.into()),
+            };
+            if ver < EXPECTED_VERSION {
+                return Err(Error::new(format!(
+                            "Database schema version {} is too old (expected {}); \
+                            see upgrade instructions in guide/upgrade.md.",
+                            ver, EXPECTED_VERSION)));
+            } else if ver > EXPECTED_VERSION {
+                return Err(Error::new(format!(
+                            "Database schema version {} is too new (expected {}); \
+                            must use a newer binary to match.", ver,
+                            EXPECTED_VERSION)));
+
+            }
+        }
         let db = Database(Mutex::new(LockedDatabase{
             conn: conn,
             state: State{
@@ -1024,6 +1060,7 @@ mod tests {
     use recording::{self, TIME_UNITS_PER_SEC};
     use rusqlite::Connection;
     use std::collections::BTreeMap;
+    use std::error::Error as E;
     use std::fmt::Debug;
     use testutil;
     use super::*;
@@ -1203,9 +1240,37 @@ mod tests {
         assert_eq!(0, m.len());
     }
 
-    /// Basic test of running some queries on an empty database.
     #[test]
-    fn test_empty_db() {
+    fn test_no_version() {
+        testutil::init();
+        let e = Database::new(Connection::open_in_memory().unwrap()).unwrap_err();
+        assert!(e.description().starts_with("no such table: version"));
+    }
+
+    #[test]
+    fn test_version_too_old() {
+        testutil::init();
+        let c = setup_conn();
+        c.execute_batch("delete from version; insert into version values (-1, 0, '');").unwrap();
+        let e = Database::new(c).unwrap_err();
+        assert!(e.description().starts_with(
+                "Database schema version -1 is too old (expected 0)"), "got: {:?}",
+                e.description());
+    }
+
+    #[test]
+    fn test_version_too_new() {
+        testutil::init();
+        let c = setup_conn();
+        c.execute_batch("delete from version; insert into version values (1, 0, '');").unwrap();
+        let e = Database::new(c).unwrap_err();
+        assert!(e.description().starts_with(
+                "Database schema version 1 is too new (expected 0)"), "got: {:?}", e.description());
+    }
+
+    /// Basic test of running some queries on a fresh database.
+    #[test]
+    fn test_fresh_db() {
         testutil::init();
         let conn = setup_conn();
         let db = Database::new(conn).unwrap();

src/schema.sql

@@ -33,6 +33,20 @@
 
 --pragma journal_mode = wal;
 
+-- This table tracks the schema version.
+-- There is one row for the initial database creation (inserted below, after the
+-- create statements) and one for each upgrade procedure (if any).
+create table version (
+  id integer primary key,
+
+  -- The unix time as of the creation/upgrade, as determined by
+  -- cast(strftime('%s', 'now') as int).
+  unix_time integer not null,
+
+  -- Optional notes on the creation/upgrade; could include the binary version.
+  notes text
+);
+
 create table camera (
   id integer primary key,
   uuid blob unique,-- not null check (length(uuid) = 16),
@@ -140,3 +154,6 @@ create table video_sample_entry (
   -- the case of H.264).
   data blob not null check (length(data) > 86)
 );
+
+insert into version (id, unix_time,                           notes)
+             values (0,  cast(strftime('%s', 'now') as int), 'db creation');