From c5a4af15ff4bef37bf1f939a85e208e241b7f59f Mon Sep 17 00:00:00 2001 From: Scott Lamb Date: Fri, 17 Jul 2020 20:08:33 -0700 Subject: [PATCH 1/7] fix --ui-dir parsing Thanks Jack Challen for reporting the breakage: https://groups.google.com/g/moonfire-nvr-users/c/WB-TIW3bBZI/m/Gqh-L6I9BgAJ --- src/cmds/run.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/cmds/run.rs b/src/cmds/run.rs index 01e6498..86a0d27 100644 --- a/src/cmds/run.rs +++ b/src/cmds/run.rs @@ -54,7 +54,7 @@ pub struct Args { db_dir: PathBuf, /// Directory holding user interface files (.html, .js, etc). - #[structopt(default_value = "/usr/local/lib/moonfire-nvr/ui", value_name="path", + #[structopt(long, default_value = "/usr/local/lib/moonfire-nvr/ui", value_name="path", parse(from_os_str))] ui_dir: std::path::PathBuf, From 9370732ed9258b6da4289e901dcbb35abf33b8a5 Mon Sep 17 00:00:00 2001 From: Jack Challen Date: Sat, 18 Jul 2020 09:26:58 +0100 Subject: [PATCH 2/7] Add note about initializing empty DB The first time moonfire's run it needs an (empty) db. The docs appear to miss that step. Made surrounding documentation slightly more explicit. --- guide/install.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/guide/install.md b/guide/install.md index e9719d4..c1a324a 100644 --- a/guide/install.md +++ b/guide/install.md @@ -85,13 +85,13 @@ RequiresMountsFor=/media/nvr ## Completing configuration through the UI -Once setup is complete, it is time to add sample file directory and camera -configurations to the database. - -You can configure the system's database through a text-based user interface: +Once your system is set up, it's time to initialize an empty database, +and add the cameras and sample directories to moonfire. You can do this +by using the `moonfire-nvr` binary's text-based configuration tool. ``` -$ sudo -u moonfire-nvr moonfire-nvr config 2>debug-log +$ sudo -u moonfire-nvr moonfire-nvr init # Initialize empty db +$ sudo -u moonfire-nvr moonfire-nvr config 2>debug-log # Configure cameras and storage ``` In the user interface, From 459615a616cdf1783f5eec636aa74860b77cc245 Mon Sep 17 00:00:00 2001 From: Scott Lamb Date: Sat, 18 Jul 2020 11:57:17 -0700 Subject: [PATCH 3/7] include all recordings in days map (fixes #57) This is a quick fix to a problem that gives a confusing/poor initial experience, as in this thread: https://groups.google.com/g/moonfire-nvr-users/c/WB-TIW3bBZI/m/Gqh-L6I9BgAJ I don't think it's a permanent solution. In particular, when we implement an event stream (#40), I don't want to have a separate event for every frame, so having the days map change that often won't work. The client side will likely manipulate the days map then to include a special entry for a growing recording, representing "from this time to now". --- db/db.rs | 24 ++++++++++++++++++------ design/api.md | 7 +++++-- src/json.rs | 19 ++++++++++--------- 3 files changed, 33 insertions(+), 17 deletions(-) diff --git a/db/db.rs b/db/db.rs index c95313a..a6e2e31 100644 --- a/db/db.rs +++ b/db/db.rs @@ -447,8 +447,9 @@ pub struct Stream { /// gaps and overlap. pub duration: recording::Duration, - /// Mapping of calendar day (in the server's time zone) to a summary of recordings on that day. - pub days: BTreeMap, + /// Mapping of calendar day (in the server's time zone) to a summary of committed recordings on + /// that day. + pub committed_days: BTreeMap, pub record: bool, /// The `next_recording_id` currently committed to the database. @@ -588,7 +589,18 @@ impl Stream { self.duration += r.end - r.start; self.sample_file_bytes += sample_file_bytes as i64; self.fs_bytes += round_up(i64::from(sample_file_bytes)); - adjust_days(r, 1, &mut self.days); + adjust_days(r, 1, &mut self.committed_days); + } + + /// Returns a days map including unflushed recordings. + pub fn days(&self) -> BTreeMap { + let mut days = self.committed_days.clone(); + for u in &self.uncommitted { + let l = u.lock(); + adjust_days(l.start .. l.start + recording::Duration(i64::from(l.duration_90k)), + 1, &mut days); + } + days } } @@ -789,7 +801,7 @@ impl StreamStateChanger { bytes_to_add: 0, fs_bytes_to_add: 0, duration: recording::Duration(0), - days: BTreeMap::new(), + committed_days: BTreeMap::new(), record: sc.record, next_recording_id: 1, uncommitted: VecDeque::new(), @@ -1031,7 +1043,7 @@ impl LockedDatabase { dir.garbage_needs_unlink.insert(row.id); let d = recording::Duration(row.duration as i64); s.duration -= d; - adjust_days(row.start .. row.start + d, -1, &mut s.days); + adjust_days(row.start .. row.start + d, -1, &mut s.committed_days); } // Process add_recordings. @@ -1530,7 +1542,7 @@ impl LockedDatabase { bytes_to_add: 0, fs_bytes_to_add: 0, duration: recording::Duration(0), - days: BTreeMap::new(), + committed_days: BTreeMap::new(), next_recording_id: row.get(7)?, record: row.get(8)?, uncommitted: VecDeque::new(), diff --git a/design/api.md b/design/api.md index 92216d5..0274e7e 100644 --- a/design/api.md +++ b/design/api.md @@ -96,8 +96,11 @@ The `application/json` response will have a dict as follows: filesystem block allocated to each file. * `days`: (only included if request pararameter `days` is true) dictionary representing calendar days (in the server's time zone) - with non-zero total duration of recordings for that day. The keys - are of the form `YYYY-mm-dd`; the values are objects with the + with non-zero total duration of recordings for that day. Currently + this includes uncommitted and growing recordings. This is likely + to change in a future release for + [#40](https://github.com/scottlamb/moonfire-nvr/issues/40). The + keys are of the form `YYYY-mm-dd`; the values are objects with the following attributes: * `totalDuration90k` is the total duration recorded during that day. If a recording spans a day boundary, some portion of it diff --git a/src/json.rs b/src/json.rs index 47838c4..83b68e1 100644 --- a/src/json.rs +++ b/src/json.rs @@ -87,7 +87,7 @@ pub struct Camera<'a> { pub config: Option>, #[serde(serialize_with = "Camera::serialize_streams")] - pub streams: [Option>; 2], + pub streams: [Option; 2], } #[derive(Debug, Serialize)] @@ -100,7 +100,7 @@ pub struct CameraConfig<'a> { #[derive(Debug, Serialize)] #[serde(rename_all="camelCase")] -pub struct Stream<'a> { +pub struct Stream { pub retain_bytes: i64, pub min_start_time_90k: Option, pub max_end_time_90k: Option, @@ -110,7 +110,7 @@ pub struct Stream<'a> { #[serde(skip_serializing_if = "Option::is_none")] #[serde(serialize_with = "Stream::serialize_days")] - pub days: Option<&'a BTreeMap>, + pub days: Option>, } #[derive(Serialize)] @@ -210,7 +210,7 @@ impl<'a> Camera<'a> { }) } - fn serialize_streams(streams: &[Option>; 2], serializer: S) -> Result + fn serialize_streams(streams: &[Option; 2], serializer: S) -> Result where S: Serializer { let mut map = serializer.serialize_map(Some(streams.len()))?; for (i, s) in streams.iter().enumerate() { @@ -223,8 +223,9 @@ impl<'a> Camera<'a> { } } -impl<'a> Stream<'a> { - fn wrap(db: &'a db::LockedDatabase, id: Option, include_days: bool) -> Result, Error> { +impl Stream { + fn wrap(db: &db::LockedDatabase, id: Option, include_days: bool) + -> Result, Error> { let id = match id { Some(id) => id, None => return Ok(None), @@ -237,14 +238,14 @@ impl<'a> Stream<'a> { total_duration_90k: s.duration.0, total_sample_file_bytes: s.sample_file_bytes, fs_bytes: s.fs_bytes, - days: if include_days { Some(&s.days) } else { None }, + days: if include_days { Some(s.days()) } else { None }, })) } - fn serialize_days(days: &Option<&BTreeMap>, + fn serialize_days(days: &Option>, serializer: S) -> Result where S: Serializer { - let days = match *days { + let days = match days.as_ref() { Some(d) => d, None => return serializer.serialize_none(), }; From 2d5c2a4de855363c6d3495419898d71ab3d96bba Mon Sep 17 00:00:00 2001 From: main Date: Mon, 7 Sep 2020 16:56:20 -0400 Subject: [PATCH 4/7] Added pwa meta tags --- ui-src/index.html | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ui-src/index.html b/ui-src/index.html index c63ebef..657f02c 100644 --- a/ui-src/index.html +++ b/ui-src/index.html @@ -1,6 +1,8 @@ Moonfire NVR + + From d83bb1bf4dc11a7566c83670145b91a66f70a3c2 Mon Sep 17 00:00:00 2001 From: Scott Lamb Date: Sat, 17 Oct 2020 16:53:57 -0700 Subject: [PATCH 5/7] better error msg when db dir open fails The previous poor error message was reported by Marlon Hendred: https://groups.google.com/g/moonfire-nvr-users/c/X9k2cOCJUDQ/m/1y1ikrWoAAAJ --- src/cmds/mod.rs | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/src/cmds/mod.rs b/src/cmds/mod.rs index e635bc5..7025602 100644 --- a/src/cmds/mod.rs +++ b/src/cmds/mod.rs @@ -53,11 +53,17 @@ enum OpenMode { /// Locks the directory without opening the database. /// The returned `dir::Fd` holds the lock and should be kept open as long as the `Connection` is. fn open_dir(db_dir: &Path, mode: OpenMode) -> Result { - let dir = dir::Fd::open(db_dir, mode == OpenMode::Create)?; + let dir = dir::Fd::open(db_dir, mode == OpenMode::Create) + .map_err(|e| e.context(if e == nix::Error::Sys(nix::errno::Errno::ENOENT) { + format!("db dir {} not found; try running moonfire-nvr init", + db_dir.display()) + } else { + format!("unable to open db dir {}: {}", db_dir.display(), &e) + }))?; let ro = mode == OpenMode::ReadOnly; dir.lock(if ro { FlockArg::LockSharedNonblock } else { FlockArg::LockExclusiveNonblock }) - .map_err(|e| e.context(format!("db dir {:?} already in use; can't get {} lock", - db_dir, if ro { "shared" } else { "exclusive" })))?; + .map_err(|e| e.context(format!("db dir {} already in use; can't get {} lock", + db_dir.display(), if ro { "shared" } else { "exclusive" })))?; Ok(dir) } From 16825537b91581383667cdb711f0616ccaa35f93 Mon Sep 17 00:00:00 2001 From: Scott Lamb Date: Sun, 22 Nov 2020 17:37:55 -0800 Subject: [PATCH 6/7] upgrade rusqlite, linked-hash-map/hashlink With Rust 1.48.0, I was getting panics like "attempted to leave type `linked_hash_map::Node` uninitialized, which is invalid". Looks like this is due to a bug in linked-hash-map 0.5.2, fixed with 0.5.3. Along the way, I see that rusqlite no longer uses this crate; it switched to hashlink. Upgrade rusqlite and match that change for my own use (video index LRU cache). I'm still pulling in linked-hash-map via serde_yaml. I didn't even realize I was using serde_yaml, but libpasta wants it. Seems a little silly to me but that's something I might explore another day. --- Cargo.lock | 53 +++++++++++++++++++++++++++++++++------------------ Cargo.toml | 2 +- db/Cargo.toml | 4 ++-- db/db.rs | 48 ++++++++++++++++++++++++++++++---------------- 4 files changed, 69 insertions(+), 38 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 2a6cb15..c16f077 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -15,6 +15,12 @@ dependencies = [ "const-random", ] +[[package]] +name = "ahash" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6789e291be47ace86a60303502173d84af8327e3627ecf334356ee0f87a164c" + [[package]] name = "ansi_term" version = "0.9.0" @@ -390,7 +396,7 @@ version = "0.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "341b03eec276c30c6cdc640d8bd8c08eac9605064c3f9c4838f958dac06973bb" dependencies = [ - "ahash", + "ahash 0.2.18", "cfg-if", "chrono", "crossbeam-channel", @@ -791,6 +797,24 @@ dependencies = [ "tokio-util", ] +[[package]] +name = "hashbrown" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7afe4a420e3fe79967a00898cc1f4db7c8a49a9333a29f8a4bd76a253d5cd04" +dependencies = [ + "ahash 0.4.6", +] + +[[package]] +name = "hashlink" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d99cf782f0dc4372d26846bec3de7804ceb5df083c2d4462c0b8d2330e894fa8" +dependencies = [ + "hashbrown", +] + [[package]] name = "heck" version = "0.3.1" @@ -1035,9 +1059,9 @@ dependencies = [ [[package]] name = "libsqlite3-sys" -version = "0.17.3" +version = "0.20.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "56d90181c2904c287e5390186be820e5ef311a3c62edebb7d6ca3d6a48ce041d" +checksum = "64d31059f22935e6c31830db5249ba2b7ecd54fd73a9909286f0a67aa55c2fbd" dependencies = [ "cc", "pkg-config", @@ -1046,9 +1070,9 @@ dependencies = [ [[package]] name = "linked-hash-map" -version = "0.5.2" +version = "0.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ae91b68aebc4ddb91978b11a1b02ddd8602a05ec19002801c5666000e05e0f83" +checksum = "8dd5a6d5999d9907cda8ed67bbd137d3af8085216c2ac62de5be860bd41f304a" [[package]] name = "lock_api" @@ -1068,15 +1092,6 @@ dependencies = [ "cfg-if", ] -[[package]] -name = "lru-cache" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "31e24f1ad8321ca0e8a1e0ac13f23cb668e6f5466c2c57319f6a5cf1cc8e3b1c" -dependencies = [ - "linked-hash-map", -] - [[package]] name = "maplit" version = "1.0.2" @@ -1200,12 +1215,12 @@ dependencies = [ "cstr", "failure", "fnv", + "hashlink", "itertools", "lazy_static", "libc", "libpasta", "log", - "lru-cache", "moonfire-base", "mylog", "nix", @@ -1925,17 +1940,17 @@ dependencies = [ [[package]] name = "rusqlite" -version = "0.22.0" +version = "0.24.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "57edf4c4cea4d7e0fab069acb5da9e8e8e5403c78abc81b1f37d83af02148ea5" +checksum = "7e3d4791ab5517217f51216a84a688b53c1ebf7988736469c538d02f46ddba68" dependencies = [ "bitflags", "fallible-iterator", "fallible-streaming-iterator", + "hashlink", "libsqlite3-sys", - "lru-cache", "memchr", - "time 0.1.43", + "smallvec", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 4c2bab4..d98f26e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -47,7 +47,7 @@ parking_lot = { version = "0.10", features = [] } protobuf = { git = "https://github.com/stepancheg/rust-protobuf" } reffers = "0.6.0" ring = "0.14.6" -rusqlite = "0.22.0" +rusqlite = "0.24.1" serde = { version = "1.0", features = ["derive"] } serde_json = "1.0" smallvec = "1.0" diff --git a/db/Cargo.toml b/db/Cargo.toml index 00a24c5..daa3c53 100644 --- a/db/Cargo.toml +++ b/db/Cargo.toml @@ -18,11 +18,11 @@ blake2-rfc = "0.2.18" cstr = "0.1.7" failure = "0.1.1" fnv = "1.0" +hashlink = "0.6.0" lazy_static = "1.0" libc = "0.2" libpasta = "0.1.0-rc2" log = "0.4" -lru-cache = "0.1" mylog = { git = "https://github.com/scottlamb/mylog" } nix = "0.17.0" odds = { version = "0.4.0", features = ["std-vec"] } @@ -30,7 +30,7 @@ openssl = "0.10" parking_lot = { version = "0.10", features = [] } prettydiff = "0.3.1" protobuf = { git = "https://github.com/stepancheg/rust-protobuf" } -rusqlite = "0.22.0" +rusqlite = "0.24.1" smallvec = "1.0" tempdir = "0.3" time = "0.1" diff --git a/db/db.rs b/db/db.rs index a6e2e31..a4ebe7c 100644 --- a/db/db.rs +++ b/db/db.rs @@ -62,9 +62,9 @@ use crate::schema; use crate::signal; use failure::{Error, bail, format_err}; use fnv::{FnvHashMap, FnvHashSet}; +use hashlink::LinkedHashMap; use itertools::Itertools; use log::{error, info, trace}; -use lru_cache::LruCache; use openssl::hash; use parking_lot::{Mutex,MutexGuard}; use protobuf::prelude::MessageField; @@ -87,6 +87,11 @@ use uuid::Uuid; /// Expected schema version. See `guide/schema.md` for more information. pub const EXPECTED_VERSION: i32 = 5; +/// Length of the video index cache. +/// The actual data structure is one bigger than this because we insert before we remove. +/// Make it one less than a power of two so that the data structure's size is efficient. +const VIDEO_INDEX_CACHE_LEN: usize = 1023; + const GET_RECORDING_PLAYBACK_SQL: &'static str = r#" select video_index @@ -652,7 +657,7 @@ pub struct LockedDatabase { streams_by_id: BTreeMap, cameras_by_uuid: BTreeMap, // values are ids. video_sample_entries_by_id: BTreeMap>, - video_index_cache: RefCell, fnv::FnvBuildHasher>>, + video_index_cache: RefCell, fnv::FnvBuildHasher>>, on_flush: Vec>, } @@ -1339,20 +1344,30 @@ impl LockedDatabase { // Committed path. let mut cache = self.video_index_cache.borrow_mut(); - if let Some(video_index) = cache.get_mut(&id.0) { - trace!("cache hit for recording {}", id); - return f(&RecordingPlayback { video_index }); + use hashlink::linked_hash_map::RawEntryMut; + match cache.raw_entry_mut().from_key(&id.0) { + RawEntryMut::Occupied(mut occupied) => { + trace!("cache hit for recording {}", id); + occupied.to_back(); + let video_index = occupied.get(); + return f(&RecordingPlayback { video_index }); + }, + RawEntryMut::Vacant(vacant) => { + trace!("cache miss for recording {}", id); + let mut stmt = self.conn.prepare_cached(GET_RECORDING_PLAYBACK_SQL)?; + let mut rows = stmt.query_named(named_params!{":composite_id": id.0})?; + if let Some(row) = rows.next()? { + let video_index: VideoIndex = row.get(0)?; + let result = f(&RecordingPlayback { video_index: &video_index.0[..] }); + vacant.insert(id.0, video_index.0); + if cache.len() > VIDEO_INDEX_CACHE_LEN { + cache.pop_front(); + } + return result; + } + Err(format_err!("no such recording {}", id)) + }, } - trace!("cache miss for recording {}", id); - let mut stmt = self.conn.prepare_cached(GET_RECORDING_PLAYBACK_SQL)?; - let mut rows = stmt.query_named(named_params!{":composite_id": id.0})?; - if let Some(row) = rows.next()? { - let video_index: VideoIndex = row.get(0)?; - let result = f(&RecordingPlayback { video_index: &video_index.0[..] }); - cache.insert(id.0, video_index.0); - return result; - } - Err(format_err!("no such recording {}", id)) } /// Queues for deletion the oldest recordings that aren't already queued. @@ -2020,7 +2035,8 @@ impl Database { cameras_by_uuid: BTreeMap::new(), streams_by_id: BTreeMap::new(), video_sample_entries_by_id: BTreeMap::new(), - video_index_cache: RefCell::new(LruCache::with_hasher(1024, Default::default())), + video_index_cache: RefCell::new(LinkedHashMap::with_capacity_and_hasher( + VIDEO_INDEX_CACHE_LEN + 1, Default::default())), on_flush: Vec::new(), })), clocks, From 642eb4f60b155f5556a590039284a611d7fb74d6 Mon Sep 17 00:00:00 2001 From: Scott Lamb Date: Sun, 22 Nov 2020 18:45:54 -0800 Subject: [PATCH 7/7] update min Rust version to 1.42.0 rusqlite 0.24.1 uses matches!, which was introduced in this version. So 1682553 does't work on prior versions. https://travis-ci.org/github/scottlamb/moonfire-nvr/jobs/745309747 --- .travis.yml | 2 +- guide/install-manual.md | 2 +- scripts/script-functions.sh | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index f0705de..287862d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -23,7 +23,7 @@ matrix: script: - ci/script-rust.sh - language: rust - rust: 1.40.0 + rust: 1.42.0 script: - ci/script-rust.sh - language: node_js diff --git a/guide/install-manual.md b/guide/install-manual.md index 2fd7f09..9df65ba 100644 --- a/guide/install-manual.md +++ b/guide/install-manual.md @@ -48,7 +48,7 @@ $ sudo apt-get install \ tzdata ``` -Next, you need Rust 1.40+ and Cargo. The easiest way to install them is by +Next, you need Rust 1.42+ and Cargo. The easiest way to install them is by following the instructions at [rustup.rs](https://www.rustup.rs/). Finally, building the UI requires [yarn](https://yarnpkg.com/en/). diff --git a/scripts/script-functions.sh b/scripts/script-functions.sh index d2b4a7b..0faec02 100755 --- a/scripts/script-functions.sh +++ b/scripts/script-functions.sh @@ -40,7 +40,7 @@ fi NODE_MIN_VERSION="10" YARN_MIN_VERSION="1.0" CARGO_MIN_VERSION="0.2" -RUSTC_MIN_VERSION="1.40" +RUSTC_MIN_VERSION="1.42" normalizeDirPath() {