move db upgrade logic into db crate

This allows shrinking db's API surface.

db/dir.rs

@@ -226,8 +226,7 @@ impl SampleFileDir {
         Ok(meta)
     }
 
-    // TODO: this should be exposed only to the db layer.
-    pub fn write_meta(&self, meta: &schema::DirMeta) -> Result<(), Error> {
+    pub(crate) fn write_meta(&self, meta: &schema::DirMeta) -> Result<(), Error> {
         let (tmp_path, final_path) = unsafe {
             (ffi::CStr::from_ptr("meta.tmp\0".as_ptr() as *const c_char),
              ffi::CStr::from_ptr("meta\0".as_ptr() as *const c_char))

db/lib.rs

@@ -50,7 +50,8 @@ pub mod db;
 pub mod dir;
 mod raw;
 pub mod recording;
-pub mod schema;
+mod schema;
+pub mod upgrade;
 
 // This is only for #[cfg(test)], but it's also used by the dependent crate, and it appears that
 // #[cfg(test)] is not passed on to dependencies.

db/upgrade/mod.rs

@@ -0,0 +1,111 @@
+// This file is part of Moonfire NVR, a security camera digital video recorder.
+// Copyright (C) 2016 Scott Lamb <slamb@slamb.org>
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// In addition, as a special exception, the copyright holders give
+// permission to link the code of portions of this program with the
+// OpenSSL library under certain conditions as described in each
+// individual source file, and distribute linked combinations including
+// the two.
+//
+// You must obey the GNU General Public License in all respects for all
+// of the code used other than OpenSSL. If you modify file(s) with this
+// exception, you may extend this exception to your version of the
+// file(s), but you are not obligated to do so. If you do not wish to do
+// so, delete this exception statement from your version. If you delete
+// this exception statement from all source files in the program, then
+// also delete it here.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+/// Upgrades the database schema.
+///
+/// See `guide/schema.md` for more information.
+
+use db;
+use failure::Error;
+use rusqlite;
+
+mod v0_to_v1;
+mod v1_to_v2;
+mod v2_to_v3;
+
+const UPGRADE_NOTES: &'static str =
+    concat!("upgraded using moonfire-db ", env!("CARGO_PKG_VERSION"));
+
+pub trait Upgrader {
+    fn in_tx(&mut self, &rusqlite::Transaction) -> Result<(), Error> { Ok(()) }
+    fn post_tx(&mut self) -> Result<(), Error> { Ok(()) }
+}
+
+#[derive(Debug)]
+pub struct Args<'a> {
+    pub flag_sample_file_dir: Option<&'a str>,
+    pub flag_preset_journal: &'a str,
+    pub flag_no_vacuum: bool,
+}
+
+fn set_journal_mode(conn: &rusqlite::Connection, requested: &str) -> Result<(), Error> {
+    assert!(!requested.contains(';'));  // quick check for accidental sql injection.
+    let actual = conn.query_row(&format!("pragma journal_mode = {}", requested), &[],
+                                |row| row.get_checked::<_, String>(0))??;
+    info!("...database now in journal_mode {} (requested {}).", actual, requested);
+    Ok(())
+}
+
+pub fn run(args: &Args, conn: &mut rusqlite::Connection) -> Result<(), Error> {
+    let upgraders = [
+        v0_to_v1::new,
+        v1_to_v2::new,
+        v2_to_v3::new,
+    ];
+
+    {
+        assert_eq!(upgraders.len(), db::EXPECTED_VERSION as usize);
+        let old_ver =
+            conn.query_row("select max(id) from version", &[], |row| row.get_checked(0))??;
+        if old_ver > db::EXPECTED_VERSION {
+            bail!("Database is at version {}, later than expected {}",
+                  old_ver, db::EXPECTED_VERSION);
+        } else if old_ver < 0 {
+            bail!("Database is at negative version {}!", old_ver);
+        }
+        info!("Upgrading database from version {} to version {}...", old_ver, db::EXPECTED_VERSION);
+        set_journal_mode(&conn, args.flag_preset_journal).unwrap();
+        for ver in old_ver .. db::EXPECTED_VERSION {
+            info!("...from version {} to version {}", ver, ver + 1);
+            let mut u = upgraders[ver as usize](&args)?;
+            let tx = conn.transaction()?;
+            u.in_tx(&tx)?;
+            tx.execute(r#"
+                insert into version (id, unix_time, notes)
+                             values (?, cast(strftime('%s', 'now') as int32), ?)
+            "#, &[&(ver + 1), &UPGRADE_NOTES])?;
+            tx.commit()?;
+            u.post_tx()?;
+        }
+    }
+
+    // WAL is the preferred journal mode for normal operation; it reduces the number of syncs
+    // without compromising safety.
+    set_journal_mode(&conn, "wal").unwrap();
+    if !args.flag_no_vacuum {
+        info!("...vacuuming database after upgrade.");
+        conn.execute_batch(r#"
+            pragma page_size = 16384;
+            vacuum;
+        "#).unwrap();
+    }
+    info!("...done.");
+    Ok(())
+}

db/upgrade/v0_to_v1.rs

@@ -0,0 +1,256 @@
+// This file is part of Moonfire NVR, a security camera digital video recorder.
+// Copyright (C) 2016 Scott Lamb <slamb@slamb.org>
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// In addition, as a special exception, the copyright holders give
+// permission to link the code of portions of this program with the
+// OpenSSL library under certain conditions as described in each
+// individual source file, and distribute linked combinations including
+// the two.
+//
+// You must obey the GNU General Public License in all respects for all
+// of the code used other than OpenSSL. If you modify file(s) with this
+// exception, you may extend this exception to your version of the
+// file(s), but you are not obligated to do so. If you do not wish to do
+// so, delete this exception statement from your version. If you delete
+// this exception statement from all source files in the program, then
+// also delete it here.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+/// Upgrades a version 0 schema to a version 1 schema.
+
+use db;
+use failure::Error;
+use recording;
+use rusqlite;
+use std::collections::HashMap;
+
+pub struct U;
+
+pub fn new<'a>(_args: &'a super::Args) -> Result<Box<super::Upgrader + 'a>, Error> {
+    Ok(Box::new(U))
+}
+
+impl super::Upgrader for U {
+    fn in_tx(&mut self, tx: &rusqlite::Transaction) -> Result<(), Error> {
+        // These create statements match the schema.sql when version 1 was the latest.
+        tx.execute_batch(r#"
+            alter table camera rename to old_camera;
+            create table camera (
+              id integer primary key,
+              uuid blob unique,
+              short_name text not null,
+              description text,
+              host text,
+              username text,
+              password text,
+              main_rtsp_path text,
+              sub_rtsp_path text,
+              retain_bytes integer not null check (retain_bytes >= 0),
+              next_recording_id integer not null check (next_recording_id >= 0)
+            );
+            alter table recording rename to old_recording;
+            drop index recording_cover;
+            create table recording (
+              composite_id integer primary key,
+              camera_id integer not null references camera (id),
+              run_offset integer not null,
+              flags integer not null,
+              sample_file_bytes integer not null check (sample_file_bytes > 0),
+              start_time_90k integer not null check (start_time_90k > 0),
+              duration_90k integer not null
+                  check (duration_90k >= 0 and duration_90k < 5*60*90000),
+              local_time_delta_90k integer not null,
+              video_samples integer not null check (video_samples > 0),
+              video_sync_samples integer not null check (video_samples > 0),
+              video_sample_entry_id integer references video_sample_entry (id),
+              check (composite_id >> 32 = camera_id)
+            );
+            create index recording_cover on recording (
+              camera_id,
+              start_time_90k,
+              duration_90k,
+              video_samples,
+              video_sync_samples,
+              video_sample_entry_id,
+              sample_file_bytes,
+              run_offset,
+              flags
+            );
+            create table recording_playback (
+              composite_id integer primary key references recording (composite_id),
+              sample_file_uuid blob not null check (length(sample_file_uuid) = 16),
+              sample_file_sha1 blob not null check (length(sample_file_sha1) = 20),
+              video_index blob not null check (length(video_index) > 0)
+            );
+        "#)?;
+        let camera_state = fill_recording(tx).unwrap();
+        fill_camera(tx, camera_state).unwrap();
+        tx.execute_batch(r#"
+          drop table old_camera;
+          drop table old_recording;
+        "#)?;
+        Ok(())
+    }
+}
+
+struct CameraState {
+    /// tuple of (run_start_id, next_start_90k).
+    current_run: Option<(i64, i64)>,
+
+    /// As in the `next_recording_id` field of the `camera` table.
+    next_recording_id: i32,
+}
+
+fn has_trailing_zero(video_index: &[u8]) -> Result<bool, Error> {
+    let mut it = recording::SampleIndexIterator::new();
+    while it.next(video_index)? {}
+    Ok(it.duration_90k == 0)
+}
+
+/// Fills the `recording` and `recording_playback` tables from `old_recording`, returning
+/// the `camera_state` map for use by a following call to `fill_cameras`.
+fn fill_recording(tx: &rusqlite::Transaction) -> Result<HashMap<i32, CameraState>, Error> {
+    let mut select = tx.prepare(r#"
+      select
+        camera_id,
+        sample_file_bytes,
+        start_time_90k,
+        duration_90k,
+        local_time_delta_90k,
+        video_samples,
+        video_sync_samples,
+        video_sample_entry_id,
+        sample_file_uuid,
+        sample_file_sha1,
+        video_index,
+        id
+      from
+        old_recording
+    "#)?;
+    let mut insert1 = tx.prepare(r#"
+      insert into recording values (:composite_id, :camera_id, :run_offset, :flags,
+                                    :sample_file_bytes, :start_time_90k, :duration_90k,
+                                    :local_time_delta_90k, :video_samples, :video_sync_samples,
+                                    :video_sample_entry_id)
+    "#)?;
+    let mut insert2 = tx.prepare(r#"
+      insert into recording_playback values (:composite_id, :sample_file_uuid, :sample_file_sha1,
+                                             :video_index)
+    "#)?;
+    let mut rows = select.query(&[])?;
+    let mut camera_state: HashMap<i32, CameraState> = HashMap::new();
+    while let Some(row) = rows.next() {
+        let row = row?;
+        let camera_id: i32 = row.get_checked(0)?;
+        let camera_state = camera_state.entry(camera_id).or_insert_with(|| {
+            CameraState{
+                current_run: None,
+                next_recording_id: 1,
+            }
+        });
+        let composite_id = ((camera_id as i64) << 32) | (camera_state.next_recording_id as i64);
+        camera_state.next_recording_id += 1;
+        let sample_file_bytes: i32 = row.get_checked(1)?;
+        let start_time_90k: i64 = row.get_checked(2)?;
+        let duration_90k: i32 = row.get_checked(3)?;
+        let local_time_delta_90k: i64 = row.get_checked(4)?;
+        let video_samples: i32 = row.get_checked(5)?;
+        let video_sync_samples: i32 = row.get_checked(6)?;
+        let video_sample_entry_id: i32 = row.get_checked(7)?;
+        let sample_file_uuid: db::FromSqlUuid = row.get_checked(8)?;
+        let sample_file_sha1: Vec<u8> = row.get_checked(9)?;
+        let video_index: Vec<u8> = row.get_checked(10)?;
+        let old_id: i32 = row.get_checked(11)?;
+        let trailing_zero = has_trailing_zero(&video_index).unwrap_or_else(|e| {
+            warn!("recording {}/{} (sample file {}, formerly recording {}) has corrupt \
+                  video_index: {}",
+                  camera_id, composite_id & 0xFFFF, sample_file_uuid.0, old_id, e);
+            false
+        });
+        let run_id = match camera_state.current_run {
+            Some((run_id, expected_start)) if expected_start == start_time_90k => run_id,
+            _ => composite_id,
+        };
+        insert1.execute_named(&[
+            (":composite_id", &composite_id),
+            (":camera_id", &camera_id),
+            (":run_offset", &(composite_id - run_id)),
+            (":flags", &(if trailing_zero { db::RecordingFlags::TrailingZero as i32 } else { 0 })),
+            (":sample_file_bytes", &sample_file_bytes),
+            (":start_time_90k", &start_time_90k),
+            (":duration_90k", &duration_90k),
+            (":local_time_delta_90k", &local_time_delta_90k),
+            (":video_samples", &video_samples),
+            (":video_sync_samples", &video_sync_samples),
+            (":video_sample_entry_id", &video_sample_entry_id),
+        ])?;
+        insert2.execute_named(&[
+            (":composite_id", &composite_id),
+            (":sample_file_uuid", &&sample_file_uuid.0.as_bytes()[..]),
+            (":sample_file_sha1", &sample_file_sha1),
+            (":video_index", &video_index),
+        ])?;
+        camera_state.current_run = if trailing_zero {
+            None
+        } else {
+            Some((run_id, start_time_90k + duration_90k as i64))
+        };
+    }
+    Ok(camera_state)
+}
+
+fn fill_camera(tx: &rusqlite::Transaction, camera_state: HashMap<i32, CameraState>)
+                -> Result<(), Error> {
+    let mut select = tx.prepare(r#"
+      select
+        id, uuid, short_name, description, host, username, password, main_rtsp_path,
+        sub_rtsp_path, retain_bytes
+      from
+        old_camera
+    "#)?;
+    let mut insert = tx.prepare(r#"
+      insert into camera values (:id, :uuid, :short_name, :description, :host, :username, :password,
+      :main_rtsp_path, :sub_rtsp_path, :retain_bytes, :next_recording_id)
+    "#)?;
+    let mut rows = select.query(&[])?;
+    while let Some(row) = rows.next() {
+        let row = row?;
+        let id: i32 = row.get_checked(0)?;
+        let uuid: Vec<u8> = row.get_checked(1)?;
+        let short_name: String = row.get_checked(2)?;
+        let description: String = row.get_checked(3)?;
+        let host: String = row.get_checked(4)?;
+        let username: String = row.get_checked(5)?;
+        let password: String = row.get_checked(6)?;
+        let main_rtsp_path: String = row.get_checked(7)?;
+        let sub_rtsp_path: String = row.get_checked(8)?;
+        let retain_bytes: i64 = row.get_checked(9)?;
+        insert.execute_named(&[
+            (":id", &id),
+            (":uuid", &uuid),
+            (":short_name", &short_name),
+            (":description", &description),
+            (":host", &host),
+            (":username", &username),
+            (":password", &password),
+            (":main_rtsp_path", &main_rtsp_path),
+            (":sub_rtsp_path", &sub_rtsp_path),
+            (":retain_bytes", &retain_bytes),
+            (":next_recording_id",
+             &camera_state.get(&id).map(|s| s.next_recording_id).unwrap_or(1)),
+        ])?;
+    }
+    Ok(())
+}

db/upgrade/v1_to_v2.rs

@@ -0,0 +1,332 @@
+// This file is part of Moonfire NVR, a security camera digital video recorder.
+// Copyright (C) 2018 Scott Lamb <slamb@slamb.org>
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// In addition, as a special exception, the copyright holders give
+// permission to link the code of portions of this program with the
+// OpenSSL library under certain conditions as described in each
+// individual source file, and distribute linked combinations including
+// the two.
+//
+// You must obey the GNU General Public License in all respects for all
+// of the code used other than OpenSSL. If you modify file(s) with this
+// exception, you may extend this exception to your version of the
+// file(s), but you are not obligated to do so. If you do not wish to do
+// so, delete this exception statement from your version. If you delete
+// this exception statement from all source files in the program, then
+// also delete it here.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+/// Upgrades a version 1 schema to a version 2 schema.
+
+use dir;
+use failure::Error;
+use rusqlite;
+use schema::DirMeta;
+use std::fs;
+use uuid::Uuid;
+
+pub struct U<'a> {
+    sample_file_path: &'a str,
+    dir_meta: Option<DirMeta>,
+}
+
+pub fn new<'a>(args: &'a super::Args) -> Result<Box<super::Upgrader + 'a>, Error> {
+    let sample_file_path =
+        args.flag_sample_file_dir
+            .ok_or_else(|| format_err!("--sample-file-dir required when upgrading from \
+                                        schema version 1 to 2."))?;
+    Ok(Box::new(U { sample_file_path, dir_meta: None }))
+}
+
+impl<'a> U<'a> {
+    /// Ensures there are sample files in the directory for all listed recordings.
+    /// Among other problems, this catches a fat-fingered `--sample-file-dir`.
+    fn verify_sample_files(&self, tx: &rusqlite::Transaction) -> Result<(), Error> {
+        // Build a hash of the uuids found in sample_file_path. Ignore other files.
+        let n: i64 = tx.query_row("select count(*) from recording", &[], |r| r.get_checked(0))??;
+        let mut files = ::fnv::FnvHashSet::with_capacity_and_hasher(n as usize, Default::default());
+        for e in fs::read_dir(self.sample_file_path)? {
+            let e = e?;
+            let f = e.file_name();
+            let s = match f.to_str() {
+                Some(s) => s,
+                None => continue,
+            };
+            let uuid = match Uuid::parse_str(s) {
+                Ok(u) => u,
+                Err(_) => continue,
+            };
+            if s != uuid.hyphenated().to_string() {  // non-canonical form.
+                continue;
+            }
+            files.insert(uuid);
+        }
+
+        // Iterate through the database and check that everything has a matching file.
+        let mut stmt = tx.prepare(r"select sample_file_uuid from recording_playback")?;
+        let mut rows = stmt.query(&[])?;
+        while let Some(row) = rows.next() {
+            let row = row?;
+            let uuid: ::db::FromSqlUuid = row.get_checked(0)?;
+            if !files.contains(&uuid.0) {
+                bail!("{} is missing from dir {}!", uuid.0, self.sample_file_path);
+            }
+        }
+        Ok(())
+    }
+}
+
+impl<'a> super::Upgrader for U<'a> {
+    fn in_tx(&mut self, tx: &rusqlite::Transaction) -> Result<(), Error> {
+        self.verify_sample_files(tx)?;
+
+        // These create statements match the schema.sql when version 2 was the latest.
+        tx.execute_batch(r#"
+            create table meta (
+              uuid blob not null check (length(uuid) = 16)
+            );
+            create table open (
+              id integer primary key,
+              uuid blob unique not null check (length(uuid) = 16)
+            );
+            create table sample_file_dir (
+              id integer primary key,
+              path text unique not null,
+              uuid blob unique not null check (length(uuid) = 16),
+              last_complete_open_id integer references open (id)
+            );
+        "#)?;
+        let db_uuid = ::uuid::Uuid::new_v4();
+        let db_uuid_bytes = &db_uuid.as_bytes()[..];
+        tx.execute("insert into meta (uuid) values (?)", &[&db_uuid_bytes])?;
+        let open_uuid = ::uuid::Uuid::new_v4();
+        let open_uuid_bytes = &open_uuid.as_bytes()[..];
+        tx.execute("insert into open (uuid) values (?)", &[&open_uuid_bytes])?;
+        let open_id = tx.last_insert_rowid() as u32;
+        let dir_uuid = ::uuid::Uuid::new_v4();
+        let dir_uuid_bytes = &dir_uuid.as_bytes()[..];
+
+        let mut meta = DirMeta::default();
+        {
+            meta.db_uuid.extend_from_slice(db_uuid_bytes);
+            meta.dir_uuid.extend_from_slice(dir_uuid_bytes);
+            let open = meta.mut_in_progress_open();
+            open.id = open_id;
+            open.uuid.extend_from_slice(&open_uuid_bytes);
+        }
+
+        tx.execute(r#"
+            insert into sample_file_dir (path,  uuid, last_complete_open_id)
+                                 values (?,     ?,    ?)
+        "#, &[&self.sample_file_path, &dir_uuid_bytes, &open_id])?;
+        self.dir_meta = Some(meta);
+
+        tx.execute_batch(r#"
+            drop table reserved_sample_files;
+            alter table camera rename to old_camera;
+            alter table recording rename to old_recording;
+            alter table video_sample_entry rename to old_video_sample_entry;
+            drop index recording_cover;
+
+            create table camera (
+              id integer primary key,
+              uuid blob unique not null check (length(uuid) = 16),
+              short_name text not null,
+              description text,
+              host text,
+              username text,
+              password text
+            );
+
+            create table stream (
+              id integer primary key,
+              camera_id integer not null references camera (id),
+              sample_file_dir_id integer references sample_file_dir (id),
+              type text not null check (type in ('main', 'sub')),
+              record integer not null check (record in (1, 0)),
+              rtsp_path text not null,
+              retain_bytes integer not null check (retain_bytes >= 0),
+              flush_if_sec integer not null,
+              next_recording_id integer not null check (next_recording_id >= 0),
+              unique (camera_id, type)
+            );
+
+            create table recording (
+              composite_id integer primary key,
+              stream_id integer not null references stream (id),
+              open_id integer not null,
+              run_offset integer not null,
+              flags integer not null,
+              sample_file_bytes integer not null check (sample_file_bytes > 0),
+              start_time_90k integer not null check (start_time_90k > 0),
+              duration_90k integer not null
+                  check (duration_90k >= 0 and duration_90k < 5*60*90000),
+              local_time_delta_90k integer not null,
+              video_samples integer not null check (video_samples > 0),
+              video_sync_samples integer not null check (video_sync_samples > 0),
+              video_sample_entry_id integer references video_sample_entry (id),
+              check (composite_id >> 32 = stream_id)
+            );
+
+            create index recording_cover on recording (
+              stream_id,
+              start_time_90k,
+              open_id,
+              duration_90k,
+              video_samples,
+              video_sync_samples,
+              video_sample_entry_id,
+              sample_file_bytes,
+              run_offset,
+              flags
+            );
+
+            create table video_sample_entry (
+              id integer primary key,
+              sha1 blob unique not null check (length(sha1) = 20),
+              width integer not null check (width > 0),
+              height integer not null check (height > 0),
+              rfc6381_codec text not null,
+              data blob not null check (length(data) > 86)
+            );
+
+            create table garbage (
+              sample_file_dir_id integer references sample_file_dir (id),
+              composite_id integer,
+              primary key (sample_file_dir_id, composite_id)
+            ) without rowid;
+
+            insert into camera
+            select
+              id,
+              uuid,
+              short_name,
+              description,
+              host,
+              username,
+              password
+            from old_camera;
+
+            -- Insert main streams using the same id as the camera, to ease changing recordings.
+            insert into stream
+            select
+              old_camera.id,
+              old_camera.id,
+              sample_file_dir.id,
+              'main',
+              1,
+              old_camera.main_rtsp_path,
+              old_camera.retain_bytes,
+              0,
+              old_camera.next_recording_id
+            from
+              old_camera cross join sample_file_dir;
+
+            -- Insert sub stream (if path is non-empty) using any id.
+            insert into stream (camera_id, sample_file_dir_id, type, record, rtsp_path,
+                                retain_bytes, flush_if_sec, next_recording_id)
+            select
+              old_camera.id,
+              sample_file_dir.id,
+              'sub',
+              0,
+              old_camera.sub_rtsp_path,
+              0,
+              90,
+              1
+            from
+              old_camera cross join sample_file_dir
+            where
+              old_camera.sub_rtsp_path != '';
+
+            insert into recording
+            select
+              r.composite_id,
+              r.camera_id,
+              o.open_id,
+              r.run_offset,
+              r.flags,
+              r.sample_file_bytes,
+              r.start_time_90k,
+              r.duration_90k,
+              r.local_time_delta_90k,
+              r.video_samples,
+              r.video_sync_samples,
+              r.video_sample_entry_id
+            from
+              old_recording r cross join open o;
+        "#)?;
+
+        fix_video_sample_entry(tx)?;
+
+        tx.execute_batch(r#"
+            drop table old_camera;
+            drop table old_recording;
+            drop table old_video_sample_entry;
+        "#)?;
+
+        Ok(())
+    }
+
+    fn post_tx(&mut self) -> Result<(), Error> {
+        let mut meta = self.dir_meta.take().unwrap();
+        let d = dir::SampleFileDir::create(self.sample_file_path, &meta)?;
+        ::std::mem::swap(&mut meta.last_complete_open, &mut meta.in_progress_open);
+        d.write_meta(&meta)?;
+        Ok(())
+    }
+}
+
+fn fix_video_sample_entry(tx: &rusqlite::Transaction) -> Result<(), Error> {
+    let mut select = tx.prepare(r#"
+        select
+          id,
+          sha1,
+          width,
+          height,
+          data
+        from
+          old_video_sample_entry
+    "#)?;
+    let mut insert = tx.prepare(r#"
+        insert into video_sample_entry values (:id, :sha1, :width, :height, :rfc6381_codec, :data)
+    "#)?;
+    let mut rows = select.query(&[])?;
+    while let Some(row) = rows.next() {
+        let row = row?;
+        let data: Vec<u8> = row.get_checked(4)?;
+        insert.execute_named(&[
+            (":id", &row.get_checked::<_, i32>(0)?),
+            (":sha1", &row.get_checked::<_, Vec<u8>>(1)?),
+            (":width", &row.get_checked::<_, i32>(2)?),
+            (":height", &row.get_checked::<_, i32>(3)?),
+            (":rfc6381_codec", &rfc6381_codec_from_sample_entry(&data)?),
+            (":data", &data),
+        ])?;
+    }
+    Ok(())
+}
+
+// This previously lived in h264.rs. As of version 1, H.264 is the only supported codec.
+fn rfc6381_codec_from_sample_entry(sample_entry: &[u8]) -> Result<String, Error> {
+    if sample_entry.len() < 99 || &sample_entry[4..8] != b"avc1" ||
+       &sample_entry[90..94] != b"avcC" {
+        bail!("not a valid AVCSampleEntry");
+    }
+    let profile_idc = sample_entry[103];
+    let constraint_flags_byte = sample_entry[104];
+    let level_idc = sample_entry[105];
+    Ok(format!("avc1.{:02x}{:02x}{:02x}", profile_idc, constraint_flags_byte, level_idc))
+}

db/upgrade/v2_to_v3.rs

@@ -0,0 +1,116 @@
+// This file is part of Moonfire NVR, a security camera digital video recorder.
+// Copyright (C) 2018 Scott Lamb <slamb@slamb.org>
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// In addition, as a special exception, the copyright holders give
+// permission to link the code of portions of this program with the
+// OpenSSL library under certain conditions as described in each
+// individual source file, and distribute linked combinations including
+// the two.
+//
+// You must obey the GNU General Public License in all respects for all
+// of the code used other than OpenSSL. If you modify file(s) with this
+// exception, you may extend this exception to your version of the
+// file(s), but you are not obligated to do so. If you do not wish to do
+// so, delete this exception statement from your version. If you delete
+// this exception statement from all source files in the program, then
+// also delete it here.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+/// Upgrades a version 2 schema to a version 3 schema.
+
+use db::{self, FromSqlUuid};
+use dir;
+use failure::Error;
+use libc;
+use std::io::{self, Write};
+use std::mem;
+use rusqlite;
+use uuid::Uuid;
+
+pub struct U;
+
+pub fn new<'a>(_args: &'a super::Args) -> Result<Box<super::Upgrader + 'a>, Error> {
+    Ok(Box::new(U))
+}
+
+/// Gets a pathname for a sample file suitable for passing to open or unlink.
+fn get_uuid_pathname(uuid: Uuid) -> [libc::c_char; 37] {
+    let mut buf = [0u8; 37];
+    write!(&mut buf[..36], "{}", uuid.hyphenated()).expect("can't format uuid to pathname buf");
+
+    // libc::c_char seems to be i8 on some platforms (Linux/arm) and u8 on others (Linux/amd64).
+    unsafe { mem::transmute::<[u8; 37], [libc::c_char; 37]>(buf) }
+}
+
+fn get_id_pathname(id: db::CompositeId) -> [libc::c_char; 17] {
+    let mut buf = [0u8; 17];
+    write!(&mut buf[..16], "{:016x}", id.0).expect("can't format id to pathname buf");
+    unsafe { mem::transmute::<[u8; 17], [libc::c_char; 17]>(buf) }
+}
+
+impl super::Upgrader for U {
+    fn in_tx(&mut self, tx: &rusqlite::Transaction) -> Result<(), Error> {
+        let path: String = tx.query_row(r#"
+            select path from sample_file_dir
+        "#, &[], |row| { row.get_checked(0) })??;
+
+        // Build map of stream -> dirname.
+        let d = dir::Fd::open(None, &path, false)?;
+        //let stream_to_dir = build_stream_to_dir(&d, tx)?;
+
+        let mut stmt = tx.prepare(r#"
+            select
+              composite_id,
+              sample_file_uuid
+            from
+              recording_playback
+        "#)?;
+        let mut rows = stmt.query(&[])?;
+        while let Some(row) = rows.next() {
+            let row = row?;
+            let id = db::CompositeId(row.get_checked(0)?);
+            let sample_file_uuid: FromSqlUuid = row.get_checked(1)?;
+            let from_path = get_uuid_pathname(sample_file_uuid.0);
+            let to_path = get_id_pathname(id);
+            //let to_dir: &dir::Fd = stream_to_dir[stream_id as usize].as_ref().unwrap();
+            let r = unsafe { dir::renameat(&d, from_path.as_ptr(), &d, to_path.as_ptr()) };
+            if let Err(e) = r {
+                if e.kind() == io::ErrorKind::NotFound {
+                    continue;  // assume it was already moved.
+                }
+                Err(e)?;
+            }
+        }
+
+        // These create statements match the schema.sql when version 3 was the latest.
+        tx.execute_batch(r#"
+            alter table recording_playback rename to old_recording_playback;
+            create table recording_playback (
+              composite_id integer primary key references recording (composite_id),
+              sample_file_sha1 blob not null check (length(sample_file_sha1) = 20),
+              video_index blob not null check (length(video_index) > 0)
+            );
+            insert into recording_playback
+            select
+              composite_id,
+              sample_file_sha1,
+              video_index
+            from
+              old_recording_playback;
+            drop table old_recording_playback;
+        "#)?;
+        Ok(())
+    }
+}