initial split of database to a separate crate

It should reduce compile time / memory usage to put quite a bit of the code
into a separate crate. I also intend to limit visibility of some things to
only within the db crate, but that's for a future change. This is the smallest
move that will compile.

Cargo.lock

@@ -489,6 +489,27 @@ dependencies = [
  "ws2_32-sys 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
+[[package]]
+name = "moonfire-db"
+version = "0.0.1"
+dependencies = [
+ "failure 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)",
+ "fnv 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)",
+ "lazy_static 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)",
+ "libc 0.2.36 (registry+https://github.com/rust-lang/crates.io-index)",
+ "log 0.4.1 (registry+https://github.com/rust-lang/crates.io-index)",
+ "lru-cache 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)",
+ "mylog 0.1.0 (git+https://github.com/scottlamb/mylog)",
+ "openssl 0.10.2 (registry+https://github.com/rust-lang/crates.io-index)",
+ "parking_lot 0.5.3 (registry+https://github.com/rust-lang/crates.io-index)",
+ "protobuf 1.4.3 (registry+https://github.com/rust-lang/crates.io-index)",
+ "regex 0.2.5 (registry+https://github.com/rust-lang/crates.io-index)",
+ "rusqlite 0.13.0 (registry+https://github.com/rust-lang/crates.io-index)",
+ "tempdir 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)",
+ "time 0.1.39 (registry+https://github.com/rust-lang/crates.io-index)",
+ "uuid 0.6.0 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
 [[package]]
 name = "moonfire-ffmpeg"
 version = "0.0.1"
@@ -515,14 +536,13 @@ dependencies = [
  "lazy_static 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "libc 0.2.36 (registry+https://github.com/rust-lang/crates.io-index)",
  "log 0.4.1 (registry+https://github.com/rust-lang/crates.io-index)",
- "lru-cache 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)",
  "memmap 0.6.2 (registry+https://github.com/rust-lang/crates.io-index)",
  "mime 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)",
+ "moonfire-db 0.0.1",
  "moonfire-ffmpeg 0.0.1",
  "mylog 0.1.0 (git+https://github.com/scottlamb/mylog)",
  "openssl 0.10.2 (registry+https://github.com/rust-lang/crates.io-index)",
  "parking_lot 0.5.3 (registry+https://github.com/rust-lang/crates.io-index)",
- "protobuf 1.4.3 (registry+https://github.com/rust-lang/crates.io-index)",
  "reffers 0.4.2 (git+https://github.com/diwic/reffers-rs)",
  "regex 0.2.5 (registry+https://github.com/rust-lang/crates.io-index)",
  "reqwest 0.8.4 (registry+https://github.com/rust-lang/crates.io-index)",
@@ -536,7 +556,7 @@ dependencies = [
  "tokio-core 0.1.12 (registry+https://github.com/rust-lang/crates.io-index)",
  "tokio-signal 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)",
  "url 1.6.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "uuid 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)",
+ "uuid 0.6.0 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
 [[package]]
@@ -1221,6 +1241,14 @@ version = "0.5.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 dependencies = [
  "rand 0.3.20 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
+[[package]]
+name = "uuid"
+version = "0.6.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "rand 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)",
  "serde 1.0.27 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
@@ -1418,6 +1446,7 @@ dependencies = [
 "checksum url 1.6.0 (registry+https://github.com/rust-lang/crates.io-index)" = "fa35e768d4daf1d85733418a49fb42e10d7f633e394fccab4ab7aba897053fe2"
 "checksum utf8-ranges 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)" = "662fab6525a98beff2921d7f61a39e7d59e0b425ebc7d0d9e66d316e55124122"
 "checksum uuid 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)" = "bcc7e3b898aa6f6c08e5295b6c89258d1331e9ac578cc992fb818759951bdc22"
+"checksum uuid 0.6.0 (registry+https://github.com/rust-lang/crates.io-index)" = "2bcacdce7c75ad110c0dba131d6628c64398593e3895025d043ee2fcf97c4b6e"
 "checksum vcpkg 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "9e0a7d8bed3178a8fb112199d466eeca9ed09a14ba8ad67718179b4fd5487d0b"
 "checksum version_check 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)" = "6b772017e347561807c1aa192438c5fd74242a670a6cffacc40f2defd1dc069d"
 "checksum void 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)" = "6a02e4885ed3bc0f2de90ea6dd45ebcbb66dacffe03547fadbb0eeae2770887d"

Cargo.toml

@@ -7,12 +7,15 @@ authors = ["Scott Lamb <slamb@slamb.org>"]
 
 # The nightly feature is used within moonfire-nvr itself to gate the
 # benchmarks. Also pass it along to crates that can benefit from it.
-nightly = ["parking_lot/nightly"]
+nightly = ["moonfire-db/nightly", "parking_lot/nightly"]
 
 # The bundled feature includes bundled (aka statically linked) versions of
 # native libraries where possible.
 bundled = ["rusqlite/bundled"]
 
+[workspace]
+members = ["db", "ffmpeg"]
+
 [dependencies]
 byteorder = "1.0"
 docopt = "0.8"
@@ -25,14 +28,13 @@ hyper = "0.11.16"
 lazy_static = "1.0"
 libc = "0.2"
 log = { version = "0.4", features = ["release_max_level_info"] }
-lru-cache = "0.1"
 memmap = "0.6"
 mime = "0.3"
+moonfire-db = { path = "db" }
 moonfire-ffmpeg = { path = "ffmpeg" }
 mylog = { git = "https://github.com/scottlamb/mylog" }
 openssl = "0.10"
 parking_lot = { version = "0.5", features = [] }
-protobuf = "1.4"
 reffers = { git = "https://github.com/diwic/reffers-rs" }
 regex = "0.2"
 rusqlite = "0.13"
@@ -44,7 +46,7 @@ time = "0.1"
 tokio-core = "0.1"
 tokio-signal = "0.1"
 url = "1.4"
-uuid = { version = "0.5", features = ["serde", "v4"] }
+uuid = { version = "0.6", features = ["serde", "std", "v4"] }
 
 [dev-dependencies]
 reqwest = "0.8"

db/Cargo.toml

@@ -0,0 +1,28 @@
+[package]
+name = "moonfire-db"
+version = "0.0.1"
+authors = ["Scott Lamb <slamb@slamb.org>"]
+readme = "../README.md"
+
+[features]
+nightly = []
+
+[lib]
+path = "lib.rs"
+
+[dependencies]
+failure = "0.1.1"
+fnv = "1.0"
+lazy_static = "1.0"
+libc = "0.2"
+log = { version = "0.4", features = ["release_max_level_info"] }
+lru-cache = "0.1"
+mylog = { git = "https://github.com/scottlamb/mylog" }
+openssl = "0.10"
+parking_lot = { version = "0.5", features = [] }
+protobuf = "1.4"
+regex = "0.2"
+rusqlite = "0.13"
+tempdir = "0.3"
+time = "0.1"
+uuid = { version = "0.6", features = ["std", "v4"] }

db/dir.rs

@@ -335,8 +335,6 @@ struct SharedMutableState {
 enum SyncerCommand {
     AsyncSaveRecording(db::RecordingToInsert, fs::File),
     AsyncAbandonRecording(CompositeId),
-
-    #[cfg(test)]
     Flush(mpsc::SyncSender<()>),
 }
 
@@ -451,7 +449,6 @@ impl SyncerChannel {
     }
 
     /// For testing: flushes the syncer, waiting for all currently-queued commands to complete.
-    #[cfg(test)]
     pub fn flush(&self) {
         let (snd, rcv) = mpsc::sync_channel(0);
         self.0.send(SyncerCommand::Flush(snd)).unwrap();
@@ -520,7 +517,6 @@ impl Syncer {
                 Err(_) => return,  // all senders have closed the channel; shutdown
                 Ok(SyncerCommand::AsyncSaveRecording(recording, f)) => self.save(recording, f),
                 Ok(SyncerCommand::AsyncAbandonRecording(uuid)) => self.abandon(uuid),
-                #[cfg(test)]
                 Ok(SyncerCommand::Flush(_)) => {},  // just drop the supplied sender, closing it.
             };
         }

db/lib.rs

@@ -0,0 +1,58 @@
+// This file is part of Moonfire NVR, a security camera digital video recorder.
+// Copyright (C) 2018 Scott Lamb <slamb@slamb.org>
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// In addition, as a special exception, the copyright holders give
+// permission to link the code of portions of this program with the
+// OpenSSL library under certain conditions as described in each
+// individual source file, and distribute linked combinations including
+// the two.
+//
+// You must obey the GNU General Public License in all respects for all
+// of the code used other than OpenSSL. If you modify file(s) with this
+// exception, you may extend this exception to your version of the
+// file(s), but you are not obligated to do so. If you do not wish to do
+// so, delete this exception statement from your version. If you delete
+// this exception statement from all source files in the program, then
+// also delete it here.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+#![cfg_attr(all(feature="nightly", test), feature(test))]
+
+#[macro_use] extern crate failure;
+extern crate fnv;
+#[macro_use] extern crate lazy_static;
+extern crate libc;
+#[macro_use] extern crate log;
+extern crate lru_cache;
+extern crate mylog;
+extern crate openssl;
+extern crate parking_lot;
+extern crate protobuf;
+extern crate regex;
+extern crate rusqlite;
+extern crate time;
+extern crate uuid;
+
+mod coding;
+pub mod db;
+pub mod dir;
+pub mod recording;
+pub mod schema;
+
+// This is only for #[cfg(test)], but it's also used by the dependent crate, and it appears that
+// #[cfg(test)] is not passed on to dependencies.
+pub mod testutil;
+
+pub use db::*;

db/recording.rs

@@ -29,13 +29,13 @@
 // along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 use coding::{append_varint32, decode_varint32, unzigzag32, zigzag32};
-use core::str::FromStr;
 use db;
 use failure::Error;
 use regex::Regex;
 use std::ops;
 use std::fmt;
 use std::ops::Range;
+use std::str::FromStr;
 use time;
 
 pub const TIME_UNITS_PER_SEC: i64 = 90000;

ffmpeg/Cargo.lock

@@ -1,35 +0,0 @@
-[root]
-name = "moonfire-ffmpeg"
-version = "0.0.1"
-dependencies = [
- "cc 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)",
- "libc 0.2.24 (registry+https://github.com/rust-lang/crates.io-index)",
- "log 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)",
- "pkg-config 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)",
-]
-
-[[package]]
-name = "cc"
-version = "1.0.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-
-[[package]]
-name = "libc"
-version = "0.2.24"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-
-[[package]]
-name = "log"
-version = "0.3.8"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-
-[[package]]
-name = "pkg-config"
-version = "0.3.9"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-
-[metadata]
-"checksum cc 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)" = "ef4019bdb99c0c1ddd56c12c2f507c174d729c6915eca6bd9d27c42f3d93b0f4"
-"checksum libc 0.2.24 (registry+https://github.com/rust-lang/crates.io-index)" = "38f5c2b18a287cf78b4097db62e20f43cace381dc76ae5c0a3073067f78b7ddc"
-"checksum log 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)" = "880f77541efa6e5cc74e76910c9884d9859683118839d6a1dc3b11e63512565b"
-"checksum pkg-config 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)" = "3a8b4c6b8165cd1a1cd4b9b120978131389f64bdaf456435caa41e630edba903"

src/cmds/check.rs

@@ -30,9 +30,8 @@
 
 //! Subcommand to check the database and sample file dir for errors.
 
-use db;
+use db::{self, recording};
 use failure::Error;
-use recording;
 use std::fs;
 use uuid::Uuid;
 

src/cmds/config/cameras.rs

@@ -33,8 +33,7 @@ extern crate cursive;
 use self::cursive::Cursive;
 use self::cursive::traits::{Boxable, Identifiable, Finder};
 use self::cursive::views;
-use db;
+use db::{self, dir};
-use dir;
 use failure::Error;
 use std::collections::BTreeMap;
 use std::sync::Arc;

src/cmds/config/dirs.rs

@@ -33,8 +33,7 @@ extern crate cursive;
 use self::cursive::Cursive;
 use self::cursive::traits::{Boxable, Identifiable};
 use self::cursive::views;
-use db;
+use db::{self, dir};
-use dir;
 use failure::Error;
 use std::cell::RefCell;
 use std::collections::BTreeMap;

src/cmds/mod.rs

@@ -28,7 +28,7 @@
 // You should have received a copy of the GNU General Public License
 // along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-use dir;
+use db::dir;
 use docopt;
 use failure::{Error, Fail};
 use libc;

src/cmds/run.rs

@@ -29,8 +29,7 @@
 // along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 use clock;
-use db;
+use db::{self, dir};
-use dir;
 use failure::Error;
 use fnv::FnvHashMap;
 use futures::{Future, Stream};

src/cmds/ts.rs

@@ -28,8 +28,8 @@
 // You should have received a copy of the GNU General Public License
 // along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
+use db::recording;
 use failure::Error;
-use recording;
 
 const USAGE: &'static str = r#"
 Usage: moonfire-nvr ts <ts>...

src/cmds/upgrade/v0_to_v1.rs

@@ -30,9 +30,8 @@
 
 /// Upgrades a version 0 schema to a version 1 schema.
 
-use db;
+use db::{self, recording};
 use failure::Error;
-use recording;
 use rusqlite;
 use std::collections::HashMap;
 use strutil;

src/cmds/upgrade/v1_to_v2.rs

@@ -30,10 +30,11 @@
 
 /// Upgrades a version 1 schema to a version 2 schema.
 
+use db::schema::DirMeta;
+use db::dir;
 use failure::Error;
 use std::fs;
 use rusqlite;
-use schema::DirMeta;
 use uuid::Uuid;
 
 pub struct U<'a> {
@@ -118,7 +119,7 @@ impl<'a> super::Upgrader for U<'a> {
         let dir_uuid = ::uuid::Uuid::new_v4();
         let dir_uuid_bytes = &dir_uuid.as_bytes()[..];
 
-        let mut meta = ::schema::DirMeta::default();
+        let mut meta = DirMeta::default();
         {
             meta.db_uuid.extend_from_slice(db_uuid_bytes);
             meta.dir_uuid.extend_from_slice(dir_uuid_bytes);
@@ -276,7 +277,7 @@ impl<'a> super::Upgrader for U<'a> {
 
     fn post_tx(&mut self) -> Result<(), Error> {
         let mut meta = self.dir_meta.take().unwrap();
-        let d = ::dir::SampleFileDir::create(self.sample_file_path, &meta)?;
+        let d = dir::SampleFileDir::create(self.sample_file_path, &meta)?;
         ::std::mem::swap(&mut meta.last_complete_open, &mut meta.in_progress_open);
         d.write_meta(&meta)?;
         Ok(())

src/cmds/upgrade/v2_to_v3.rs

@@ -30,8 +30,7 @@
 
 /// Upgrades a version 2 schema to a version 3 schema.
 
-use db::{self, FromSqlUuid};
+use db::{self, dir, FromSqlUuid};
-use dir;
 use failure::Error;
 use libc;
 use std::io::{self, Write};

src/h264.rs

@@ -248,7 +248,7 @@ pub fn transform_sample_data(annexb_sample: &[u8], avc_sample: &mut Vec<u8>) ->
 
 #[cfg(test)]
 mod tests {
-    use testutil;
+    use db::testutil;
 
     const ANNEX_B_TEST_INPUT: [u8; 35] = [
         0x00, 0x00, 0x00, 0x01, 0x67, 0x4d, 0x00, 0x1f,

src/main.rs

@@ -42,16 +42,15 @@ extern crate hyper;
 #[macro_use] extern crate lazy_static;
 extern crate libc;
 #[macro_use] extern crate log;
-extern crate lru_cache;
 extern crate reffers;
 extern crate rusqlite;
 extern crate memmap;
 extern crate mime;
+extern crate moonfire_db as db;
 extern crate moonfire_ffmpeg;
 extern crate mylog;
 extern crate openssl;
 extern crate parking_lot;
-extern crate protobuf;
 extern crate regex;
 extern crate serde;
 #[macro_use] extern crate serde_derive;
@@ -64,20 +63,14 @@ extern crate url;
 extern crate uuid;
 
 mod clock;
-mod coding;
 mod cmds;
-mod db;
-mod dir;
 mod h264;
 mod json;
 mod mp4;
-mod recording;
-mod schema;
 mod slices;
 mod stream;
 mod streamer;
 mod strutil;
-#[cfg(test)] mod testutil;
 mod web;
 
 /// Commandline usage string. This is in the particular format expected by the `docopt` crate.

src/mp4.rs

@@ -79,8 +79,8 @@
 extern crate time;
 
 use byteorder::{BigEndian, ByteOrder, WriteBytesExt};
-use db;
+use db::recording::{self, TIME_UNITS_PER_SEC};
-use dir;
+use db::{self, dir};
 use failure::Error;
 use futures::stream;
 use http_serve;
@@ -88,7 +88,6 @@ use hyper::header;
 use memmap;
 use openssl::hash;
 use parking_lot::{Once, ONCE_INIT};
-use recording::{self, TIME_UNITS_PER_SEC};
 use reffers::ARefs;
 use slices::{self, Body, Chunk, Slices};
 use smallvec::SmallVec;
@@ -1517,11 +1516,12 @@ impl http_serve::Entity for File {
 #[cfg(test)]
 mod tests {
     use byteorder::{BigEndian, ByteOrder};
+    use db::recording::{self, TIME_UNITS_PER_SEC};
+    use db::testutil::{self, TestDb, TEST_STREAM_ID};
     use futures::Future;
     use futures::Stream as FuturesStream;
     use hyper::header;
     use openssl::hash;
-    use recording::{self, TIME_UNITS_PER_SEC};
     use http_serve::{self, Entity};
     use std::fs;
     use std::ops::Range;
@@ -1530,7 +1530,6 @@ mod tests {
     use strutil;
     use super::*;
     use stream::{self, Opener, Stream};
-    use testutil::{self, TestDb, TEST_STREAM_ID};
 
     fn fill_slice<E: http_serve::Entity>(slice: &mut [u8], e: &E, start: u64) {
         let mut p = 0;
@@ -2175,15 +2174,15 @@ mod bench {
     extern crate reqwest;
     extern crate test;
 
+    use db::recording;
+    use db::testutil::{self, TestDb};
     use futures::Stream;
     use futures::future;
     use hyper;
     use http_serve;
-    use recording;
     use reffers::ARefs;
     use self::test::Bencher;
     use super::tests::create_mp4_from_db;
-    use testutil::{self, TestDb};
     use url::Url;
 
     /// An HTTP server for benchmarking.

src/slices.rs

@@ -151,11 +151,11 @@ impl<S> Slices<S> where S: Slice {
 
 #[cfg(test)]
 mod tests {
+    use db::testutil;
     use futures::{Future, Stream};
     use futures::stream;
     use std::ops::Range;
     use super::{Slice, Slices};
-    use testutil;
 
     #[derive(Debug, Eq, PartialEq)]
     pub struct FakeChunk {

src/streamer.rs

@@ -29,11 +29,9 @@
 // along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 use clock::{Clocks, TimerGuard};
-use db::{Camera, Database, Stream};
+use db::{Camera, Database, Stream, dir, recording};
-use dir;
 use failure::Error;
 use h264;
-use recording;
 use std::result::Result;
 use std::sync::atomic::{AtomicBool, Ordering};
 use std::sync::Arc;
@@ -202,15 +200,15 @@ impl<'a, C, S> Streamer<'a, C, S> where C: 'a + Clocks, S: 'a + stream::Stream {
 mod tests {
     use clock::{self, Clocks};
     use db::{self, CompositeId};
+    use db::recording;
+    use db::testutil;
     use failure::Error;
     use h264;
     use moonfire_ffmpeg;
-    use recording;
     use std::cmp;
     use std::sync::{Arc, Mutex};
     use std::sync::atomic::{AtomicBool, Ordering};
     use stream::{self, Opener, Stream};
-    use testutil;
     use time;
 
     struct ProxyingStream<'a> {

src/web.rs

@@ -32,8 +32,8 @@ extern crate hyper;
 
 use core::borrow::Borrow;
 use core::str::FromStr;
-use db;
+use db::{self, recording};
-use dir::SampleFileDir;
+use db::dir::SampleFileDir;
 use failure::Error;
 use fnv::FnvHashMap;
 use futures::{future, stream};
@@ -44,7 +44,6 @@ use hyper::header;
 use hyper::server::{self, Request, Response};
 use mime;
 use mp4;
-use recording;
 use reffers::ARefs;
 use regex::Regex;
 use serde_json;
@@ -501,8 +500,8 @@ impl server::Service for Service {
 
 #[cfg(test)]
 mod tests {
+    use db::testutil;
     use super::Segments;
-    use testutil;
 
     #[test]
     fn test_segments() {
@@ -531,9 +530,9 @@ mod bench {
     extern crate reqwest;
     extern crate test;
 
+    use db::testutil::{self, TestDb};
     use hyper;
     use self::test::Bencher;
-    use testutil::{self, TestDb};
     use uuid::Uuid;
 
     struct Server {