mirror of
https://github.com/scottlamb/moonfire-nvr.git
synced 2025-11-09 13:39:46 -05:00
restructure into "server" and "ui" subdirs
Besides being more clear about what belongs to which, this helps with docker caching. The server and ui parts are only rebuilt when their respective subdirectories change. Extend this a bit further by making the webpack build not depend on the target architecture. And adding cache dirs so parts of the server and ui build process can be reused when layer-wide caching fails.
This commit is contained in:
Scott Lamb
308
server/db/upgrade/mod.rs
Normal file
308
server/db/upgrade/mod.rs
Normal file
@@ -0,0 +1,308 @@
|
||||
// This file is part of Moonfire NVR, a security camera network video recorder.
|
||||
// Copyright (C) 2016-2020 The Moonfire NVR Authors
|
||||
//
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// In addition, as a special exception, the copyright holders give
|
||||
// permission to link the code of portions of this program with the
|
||||
// OpenSSL library under certain conditions as described in each
|
||||
// individual source file, and distribute linked combinations including
|
||||
// the two.
|
||||
//
|
||||
// You must obey the GNU General Public License in all respects for all
|
||||
// of the code used other than OpenSSL. If you modify file(s) with this
|
||||
// exception, you may extend this exception to your version of the
|
||||
// file(s), but you are not obligated to do so. If you do not wish to do
|
||||
// so, delete this exception statement from your version. If you delete
|
||||
// this exception statement from all source files in the program, then
|
||||
// also delete it here.
|
||||
//
|
||||
// This program is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
/// Upgrades the database schema.
|
||||
///
|
||||
/// See `guide/schema.md` for more information.
|
||||
|
||||
use crate::db;
|
||||
use failure::{Error, bail};
|
||||
use log::info;
|
||||
use std::ffi::CStr;
|
||||
use std::io::Write;
|
||||
use nix::NixPath;
|
||||
use rusqlite::params;
|
||||
use uuid::Uuid;
|
||||
|
||||
mod v0_to_v1;
|
||||
mod v1_to_v2;
|
||||
mod v2_to_v3;
|
||||
mod v3_to_v4;
|
||||
mod v4_to_v5;
|
||||
mod v5_to_v6;
|
||||
|
||||
const UPGRADE_NOTES: &'static str =
|
||||
concat!("upgraded using moonfire-db ", env!("CARGO_PKG_VERSION"));
|
||||
|
||||
#[derive(Debug)]
|
||||
pub struct Args<'a> {
|
||||
pub sample_file_dir: Option<&'a std::path::Path>,
|
||||
pub preset_journal: &'a str,
|
||||
pub no_vacuum: bool,
|
||||
}
|
||||
|
||||
fn set_journal_mode(conn: &rusqlite::Connection, requested: &str) -> Result<(), Error> {
|
||||
assert!(!requested.contains(';')); // quick check for accidental sql injection.
|
||||
let actual = conn.query_row(&format!("pragma journal_mode = {}", requested), params![],
|
||||
|row| row.get::<_, String>(0))?;
|
||||
info!("...database now in journal_mode {} (requested {}).", actual, requested);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn upgrade(args: &Args, target_ver: i32, conn: &mut rusqlite::Connection) -> Result<(), Error> {
|
||||
let upgraders = [
|
||||
v0_to_v1::run,
|
||||
v1_to_v2::run,
|
||||
v2_to_v3::run,
|
||||
v3_to_v4::run,
|
||||
v4_to_v5::run,
|
||||
v5_to_v6::run,
|
||||
];
|
||||
|
||||
{
|
||||
assert_eq!(upgraders.len(), db::EXPECTED_VERSION as usize);
|
||||
let old_ver =
|
||||
conn.query_row("select max(id) from version", params![],
|
||||
|row| row.get(0))?;
|
||||
if old_ver > db::EXPECTED_VERSION {
|
||||
bail!("Database is at version {}, later than expected {}",
|
||||
old_ver, db::EXPECTED_VERSION);
|
||||
} else if old_ver < 0 {
|
||||
bail!("Database is at negative version {}!", old_ver);
|
||||
}
|
||||
info!("Upgrading database from version {} to version {}...", old_ver, target_ver);
|
||||
set_journal_mode(&conn, args.preset_journal)?;
|
||||
for ver in old_ver .. target_ver {
|
||||
info!("...from version {} to version {}", ver, ver + 1);
|
||||
let tx = conn.transaction()?;
|
||||
upgraders[ver as usize](&args, &tx)?;
|
||||
tx.execute(r#"
|
||||
insert into version (id, unix_time, notes)
|
||||
values (?, cast(strftime('%s', 'now') as int32), ?)
|
||||
"#, params![ver + 1, UPGRADE_NOTES])?;
|
||||
tx.commit()?;
|
||||
}
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn run(args: &Args, conn: &mut rusqlite::Connection) -> Result<(), Error> {
|
||||
db::set_integrity_pragmas(conn)?;
|
||||
upgrade(args, db::EXPECTED_VERSION, conn)?;
|
||||
|
||||
// WAL is the preferred journal mode for normal operation; it reduces the number of syncs
|
||||
// without compromising safety.
|
||||
set_journal_mode(&conn, "wal")?;
|
||||
if !args.no_vacuum {
|
||||
info!("...vacuuming database after upgrade.");
|
||||
conn.execute_batch(r#"
|
||||
pragma page_size = 16384;
|
||||
vacuum;
|
||||
"#)?;
|
||||
}
|
||||
info!("...done.");
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// A uuid-based path, as used in version 0 and version 1 schemas.
|
||||
struct UuidPath([u8; 37]);
|
||||
|
||||
impl UuidPath {
|
||||
pub(crate) fn from(uuid: Uuid) -> Self {
|
||||
let mut buf = [0u8; 37];
|
||||
write!(&mut buf[..36], "{}", uuid.to_hyphenated_ref())
|
||||
.expect("can't format uuid to pathname buf");
|
||||
UuidPath(buf)
|
||||
}
|
||||
}
|
||||
|
||||
impl NixPath for UuidPath {
|
||||
fn is_empty(&self) -> bool { false }
|
||||
fn len(&self) -> usize { 36 }
|
||||
|
||||
fn with_nix_path<T, F>(&self, f: F) -> Result<T, nix::Error>
|
||||
where F: FnOnce(&CStr) -> T {
|
||||
let p = CStr::from_bytes_with_nul(&self.0[..]).expect("no interior nuls");
|
||||
Ok(f(p))
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use crate::compare;
|
||||
use crate::testutil;
|
||||
use failure::ResultExt;
|
||||
use fnv::FnvHashMap;
|
||||
use super::*;
|
||||
|
||||
const BAD_ANAMORPHIC_VIDEO_SAMPLE_ENTRY: &[u8] =
|
||||
b"\x00\x00\x00\x84\x61\x76\x63\x31\x00\x00\
|
||||
\x00\x00\x00\x00\x00\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\
|
||||
\x00\x00\x00\x00\x00\x00\x01\x40\x00\xf0\x00\x48\x00\x00\x00\x48\
|
||||
\x00\x00\x00\x00\x00\x00\x00\x01\x00\x00\x00\x00\x00\x00\x00\x00\
|
||||
\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\
|
||||
\x00\x00\x00\x00\x00\x00\x00\x00\x00\x18\xff\xff\x00\x00\x00\x2e\
|
||||
\x61\x76\x63\x43\x01\x4d\x40\x1e\xff\xe1\x00\x17\x67\x4d\x40\x1e\
|
||||
\x9a\x66\x0a\x0f\xff\x35\x01\x01\x01\x40\x00\x00\xfa\x00\x00\x03\
|
||||
\x01\xf4\x01\x01\x00\x04\x68\xee\x3c\x80";
|
||||
|
||||
const GOOD_ANAMORPHIC_VIDEO_SAMPLE_ENTRY: &[u8] =
|
||||
b"\x00\x00\x00\x9f\x61\x76\x63\x31\x00\x00\x00\x00\x00\x00\x00\x01\
|
||||
\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\
|
||||
\x02\xc0\x01\xe0\x00\x48\x00\x00\x00\x48\x00\x00\x00\x00\x00\x00\
|
||||
\x00\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\
|
||||
\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\
|
||||
\x00\x00\x00\x18\xff\xff\x00\x00\x00\x49\x61\x76\x63\x43\x01\x64\
|
||||
\x00\x16\xff\xe1\x00\x31\x67\x64\x00\x16\xac\x1b\x1a\x80\xb0\x3d\
|
||||
\xff\xff\x00\x28\x00\x21\x6e\x0c\x0c\x0c\x80\x00\x01\xf4\x00\x00\
|
||||
\x27\x10\x74\x30\x07\xd0\x00\x07\xa1\x25\xde\x5c\x68\x60\x0f\xa0\
|
||||
\x00\x0f\x42\x4b\xbc\xb8\x50\x01\x00\x05\x68\xee\x38\x30\x00";
|
||||
|
||||
fn new_conn() -> Result<rusqlite::Connection, Error> {
|
||||
let conn = rusqlite::Connection::open_in_memory()?;
|
||||
conn.execute("pragma foreign_keys = on", params![])?;
|
||||
conn.execute("pragma fullfsync = on", params![])?;
|
||||
conn.execute("pragma synchronous = 2", params![])?;
|
||||
Ok(conn)
|
||||
}
|
||||
|
||||
fn compare(c: &rusqlite::Connection, ver: i32, fresh_sql: &str) -> Result<(), Error> {
|
||||
let fresh = new_conn()?;
|
||||
fresh.execute_batch(fresh_sql)?;
|
||||
if let Some(diffs) = compare::get_diffs("upgraded", &c, "fresh", &fresh)? {
|
||||
panic!("Version {}: differences found:\n{}", ver, diffs);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Upgrades and compares schemas.
|
||||
/// Doesn't (yet) compare any actual data.
|
||||
#[test]
|
||||
fn upgrade_and_compare() -> Result<(), Error> {
|
||||
testutil::init();
|
||||
let tmpdir = tempdir::TempDir::new("moonfire-nvr-test")?;
|
||||
//let path = tmpdir.path().to_str().ok_or_else(|| format_err!("invalid UTF-8"))?.to_owned();
|
||||
let mut upgraded = new_conn()?;
|
||||
upgraded.execute_batch(include_str!("v0.sql"))?;
|
||||
upgraded.execute_batch(r#"
|
||||
insert into camera (id, uuid, short_name, description, host, username, password,
|
||||
main_rtsp_path, sub_rtsp_path, retain_bytes)
|
||||
values (1, zeroblob(16), 'test camera', 'desc', 'host', 'user', 'pass',
|
||||
'main', 'sub', 42);
|
||||
"#)?;
|
||||
upgraded.execute(r#"
|
||||
insert into video_sample_entry (id, sha1, width, height, data)
|
||||
values (1, X'0000000000000000000000000000000000000000', 1920, 1080, ?);
|
||||
"#, params![testutil::TEST_VIDEO_SAMPLE_ENTRY_DATA])?;
|
||||
upgraded.execute(r#"
|
||||
insert into video_sample_entry (id, sha1, width, height, data)
|
||||
values (2, X'0000000000000000000000000000000000000001', 320, 240, ?);
|
||||
"#, params![BAD_ANAMORPHIC_VIDEO_SAMPLE_ENTRY])?;
|
||||
upgraded.execute(r#"
|
||||
insert into video_sample_entry (id, sha1, width, height, data)
|
||||
values (3, X'0000000000000000000000000000000000000002', 704, 480, ?);
|
||||
"#, params![GOOD_ANAMORPHIC_VIDEO_SAMPLE_ENTRY])?;
|
||||
upgraded.execute(r#"
|
||||
insert into video_sample_entry (id, sha1, width, height, data)
|
||||
values (4, X'0000000000000000000000000000000000000003', 704, 480, ?);
|
||||
"#, params![GOOD_ANAMORPHIC_VIDEO_SAMPLE_ENTRY])?;
|
||||
upgraded.execute_batch(r#"
|
||||
insert into recording (id, camera_id, sample_file_bytes, start_time_90k, duration_90k,
|
||||
local_time_delta_90k, video_samples, video_sync_samples,
|
||||
video_sample_entry_id, sample_file_uuid, sample_file_sha1,
|
||||
video_index)
|
||||
values (1, 1, 42, 140063580000000, 90000, 0, 1, 1, 1,
|
||||
X'E69D45E8CBA64DC1BA2ECB1585983A10', zeroblob(20), X'00'),
|
||||
(2, 1, 42, 140063580090000, 90000, 0, 1, 1, 2,
|
||||
X'94DE8484FF874A5295D488C8038A0312', zeroblob(20), X'00'),
|
||||
(3, 1, 42, 140063580180000, 90000, 0, 1, 1, 3,
|
||||
X'C94D4D0B533746059CD40B29039E641E', zeroblob(20), X'00');
|
||||
insert into reserved_sample_files values (X'51EF700C933E4197AAE4EE8161E94221', 0),
|
||||
(X'E69D45E8CBA64DC1BA2ECB1585983A10', 1);
|
||||
"#)?;
|
||||
let rec1 = tmpdir.path().join("e69d45e8-cba6-4dc1-ba2e-cb1585983a10");
|
||||
let rec2 = tmpdir.path().join("94de8484-ff87-4a52-95d4-88c8038a0312");
|
||||
let rec3 = tmpdir.path().join("c94d4d0b-5337-4605-9cd4-0b29039e641e");
|
||||
let garbage = tmpdir.path().join("51ef700c-933e-4197-aae4-ee8161e94221");
|
||||
std::fs::File::create(&rec1)?;
|
||||
std::fs::File::create(&rec2)?;
|
||||
std::fs::File::create(&rec3)?;
|
||||
std::fs::File::create(&garbage)?;
|
||||
|
||||
for (ver, fresh_sql) in &[(1, Some(include_str!("v1.sql"))),
|
||||
(2, None), // transitional; don't compare schemas.
|
||||
(3, Some(include_str!("v3.sql"))),
|
||||
(4, None), // transitional; don't compare schemas.
|
||||
(5, Some(include_str!("v5.sql"))),
|
||||
(6, Some(include_str!("../schema.sql")))] {
|
||||
upgrade(&Args {
|
||||
sample_file_dir: Some(&tmpdir.path()),
|
||||
preset_journal: "delete",
|
||||
no_vacuum: false,
|
||||
}, *ver, &mut upgraded).context(format!("upgrading to version {}", ver))?;
|
||||
if let Some(f) = fresh_sql {
|
||||
compare(&upgraded, *ver, f)?;
|
||||
}
|
||||
if *ver == 3 {
|
||||
// Check that the garbage files is cleaned up properly, but also add it back
|
||||
// to simulate a bug prior to 433be217. The v5 upgrade should take care of
|
||||
// anything left over.
|
||||
assert!(!garbage.exists());
|
||||
std::fs::File::create(&garbage)?;
|
||||
}
|
||||
if *ver == 6 {
|
||||
// Check that the pasp was set properly.
|
||||
let mut stmt = upgraded.prepare(r#"
|
||||
select
|
||||
id,
|
||||
pasp_h_spacing,
|
||||
pasp_v_spacing
|
||||
from
|
||||
video_sample_entry
|
||||
"#)?;
|
||||
let mut rows = stmt.query(params![])?;
|
||||
let mut pasp_by_id = FnvHashMap::default();
|
||||
while let Some(row) = rows.next()? {
|
||||
let id: i32 = row.get(0)?;
|
||||
let pasp_h_spacing: i32 = row.get(1)?;
|
||||
let pasp_v_spacing: i32 = row.get(2)?;
|
||||
pasp_by_id.insert(id, (pasp_h_spacing, pasp_v_spacing));
|
||||
}
|
||||
assert_eq!(pasp_by_id.get(&1), Some(&(1, 1)));
|
||||
assert_eq!(pasp_by_id.get(&2), Some(&(4, 3)));
|
||||
assert_eq!(pasp_by_id.get(&3), Some(&(40, 33)));
|
||||
|
||||
// No recording references this video_sample_entry, so it gets dropped on upgrade.
|
||||
assert_eq!(pasp_by_id.get(&4), None);
|
||||
}
|
||||
}
|
||||
|
||||
// Check that recording files get renamed.
|
||||
assert!(!rec1.exists());
|
||||
assert!(tmpdir.path().join("0000000100000001").exists());
|
||||
|
||||
// Check that garbage files get cleaned up.
|
||||
assert!(!garbage.exists());
|
||||
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
159
server/db/upgrade/v0.sql
Normal file
159
server/db/upgrade/v0.sql
Normal file
@@ -0,0 +1,159 @@
|
||||
-- This file is part of Moonfire NVR, a security camera network video recorder.
|
||||
-- Copyright (C) 2016 The Moonfire NVR Authors
|
||||
--
|
||||
-- This program is free software: you can redistribute it and/or modify
|
||||
-- it under the terms of the GNU General Public License as published by
|
||||
-- the Free Software Foundation, either version 3 of the License, or
|
||||
-- (at your option) any later version.
|
||||
--
|
||||
-- In addition, as a special exception, the copyright holders give
|
||||
-- permission to link the code of portions of this program with the
|
||||
-- OpenSSL library under certain conditions as described in each
|
||||
-- individual source file, and distribute linked combinations including
|
||||
-- the two.
|
||||
--
|
||||
-- You must obey the GNU General Public License in all respects for all
|
||||
-- of the code used other than OpenSSL. If you modify file(s) with this
|
||||
-- exception, you may extend this exception to your version of the
|
||||
-- file(s), but you are not obligated to do so. If you do not wish to do
|
||||
-- so, delete this exception statement from your version. If you delete
|
||||
-- this exception statement from all source files in the program, then
|
||||
-- also delete it here.
|
||||
--
|
||||
-- This program is distributed in the hope that it will be useful,
|
||||
-- but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
-- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
-- GNU General Public License for more details.
|
||||
--
|
||||
-- You should have received a copy of the GNU General Public License
|
||||
-- along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
--
|
||||
-- schema.sql: SQLite3 database schema for Moonfire NVR.
|
||||
-- See also design/schema.md.
|
||||
|
||||
--pragma journal_mode = wal;
|
||||
|
||||
-- This table tracks the schema version.
|
||||
-- There is one row for the initial database creation (inserted below, after the
|
||||
-- create statements) and one for each upgrade procedure (if any).
|
||||
create table version (
|
||||
id integer primary key,
|
||||
|
||||
-- The unix time as of the creation/upgrade, as determined by
|
||||
-- cast(strftime('%s', 'now') as int).
|
||||
unix_time integer not null,
|
||||
|
||||
-- Optional notes on the creation/upgrade; could include the binary version.
|
||||
notes text
|
||||
);
|
||||
|
||||
create table camera (
|
||||
id integer primary key,
|
||||
uuid blob unique,-- not null check (length(uuid) = 16),
|
||||
|
||||
-- A short name of the camera, used in log messages.
|
||||
short_name text,-- not null,
|
||||
|
||||
-- A short description of the camera.
|
||||
description text,
|
||||
|
||||
-- The host (or IP address) to use in rtsp:// URLs when accessing the camera.
|
||||
host text,
|
||||
|
||||
-- The username to use when accessing the camera.
|
||||
-- If empty, no username or password will be supplied.
|
||||
username text,
|
||||
|
||||
-- The password to use when accessing the camera.
|
||||
password text,
|
||||
|
||||
-- The path (starting with "/") to use in rtsp:// URLs to reference this
|
||||
-- camera's "main" (full-quality) video stream.
|
||||
main_rtsp_path text,
|
||||
|
||||
-- The path (starting with "/") to use in rtsp:// URLs to reference this
|
||||
-- camera's "sub" (low-bandwidth) video stream.
|
||||
sub_rtsp_path text,
|
||||
|
||||
-- The number of bytes of video to retain, excluding the currently-recording
|
||||
-- file. Older files will be deleted as necessary to stay within this limit.
|
||||
retain_bytes integer not null check (retain_bytes >= 0)
|
||||
);
|
||||
|
||||
-- Each row represents a single completed recorded segment of video.
|
||||
-- Recordings are typically ~60 seconds; never more than 5 minutes.
|
||||
create table recording (
|
||||
id integer primary key,
|
||||
camera_id integer references camera (id) not null,
|
||||
|
||||
sample_file_bytes integer not null check (sample_file_bytes > 0),
|
||||
|
||||
-- The starting time of the recording, in 90 kHz units since
|
||||
-- 1970-01-01 00:00:00 UTC. Currently on initial connection, this is taken
|
||||
-- from the local system time; on subsequent recordings, it exactly
|
||||
-- matches the previous recording's end time.
|
||||
start_time_90k integer not null check (start_time_90k > 0),
|
||||
|
||||
-- The duration of the recording, in 90 kHz units.
|
||||
duration_90k integer not null
|
||||
check (duration_90k >= 0 and duration_90k < 5*60*90000),
|
||||
|
||||
-- The number of 90 kHz units the local system time is ahead of the
|
||||
-- recording; negative numbers indicate the local system time is behind
|
||||
-- the recording. Large values would indicate that the local time has jumped
|
||||
-- during recording or that the local time and camera time frequencies do
|
||||
-- not match.
|
||||
local_time_delta_90k integer not null,
|
||||
|
||||
video_samples integer not null check (video_samples > 0),
|
||||
video_sync_samples integer not null check (video_samples > 0),
|
||||
video_sample_entry_id integer references video_sample_entry (id),
|
||||
|
||||
sample_file_uuid blob not null check (length(sample_file_uuid) = 16),
|
||||
sample_file_sha1 blob not null check (length(sample_file_sha1) = 20),
|
||||
video_index blob not null check (length(video_index) > 0)
|
||||
);
|
||||
|
||||
create index recording_cover on recording (
|
||||
-- Typical queries use "where camera_id = ? order by start_time_90k (desc)?".
|
||||
camera_id,
|
||||
start_time_90k,
|
||||
|
||||
-- These fields are not used for ordering; they cover most queries so
|
||||
-- that only database verification and actual viewing of recordings need
|
||||
-- to consult the underlying row.
|
||||
duration_90k,
|
||||
video_samples,
|
||||
video_sync_samples,
|
||||
video_sample_entry_id,
|
||||
sample_file_bytes
|
||||
);
|
||||
|
||||
-- Files in the sample file directory which may be present but should simply be
|
||||
-- discarded on startup. (Recordings which were never completed or have been
|
||||
-- marked for completion.)
|
||||
create table reserved_sample_files (
|
||||
uuid blob primary key check (length(uuid) = 16),
|
||||
state integer not null -- 0 (writing) or 1 (deleted)
|
||||
) without rowid;
|
||||
|
||||
-- A concrete box derived from a ISO/IEC 14496-12 section 8.5.2
|
||||
-- VisualSampleEntry box. Describes the codec, width, height, etc.
|
||||
create table video_sample_entry (
|
||||
id integer primary key,
|
||||
|
||||
-- A SHA-1 hash of |bytes|.
|
||||
sha1 blob unique not null check (length(sha1) = 20),
|
||||
|
||||
-- The width and height in pixels; must match values within
|
||||
-- |sample_entry_bytes|.
|
||||
width integer not null check (width > 0),
|
||||
height integer not null check (height > 0),
|
||||
|
||||
-- The serialized box, including the leading length and box type (avcC in
|
||||
-- the case of H.264).
|
||||
data blob not null check (length(data) > 86)
|
||||
);
|
||||
|
||||
insert into version (id, unix_time, notes)
|
||||
values (0, cast(strftime('%s', 'now') as int), 'db creation');
|
||||
233
server/db/upgrade/v0_to_v1.rs
Normal file
233
server/db/upgrade/v0_to_v1.rs
Normal file
@@ -0,0 +1,233 @@
|
||||
// This file is part of Moonfire NVR, a security camera network video recorder.
|
||||
// Copyright (C) 2016 The Moonfire NVR Authors
|
||||
//
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// In addition, as a special exception, the copyright holders give
|
||||
// permission to link the code of portions of this program with the
|
||||
// OpenSSL library under certain conditions as described in each
|
||||
// individual source file, and distribute linked combinations including
|
||||
// the two.
|
||||
//
|
||||
// You must obey the GNU General Public License in all respects for all
|
||||
// of the code used other than OpenSSL. If you modify file(s) with this
|
||||
// exception, you may extend this exception to your version of the
|
||||
// file(s), but you are not obligated to do so. If you do not wish to do
|
||||
// so, delete this exception statement from your version. If you delete
|
||||
// this exception statement from all source files in the program, then
|
||||
// also delete it here.
|
||||
//
|
||||
// This program is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
/// Upgrades a version 0 schema to a version 1 schema.
|
||||
|
||||
use crate::db;
|
||||
use crate::recording;
|
||||
use failure::Error;
|
||||
use log::warn;
|
||||
use rusqlite::params;
|
||||
use std::collections::HashMap;
|
||||
|
||||
pub fn run(_args: &super::Args, tx: &rusqlite::Transaction) -> Result<(), Error> {
|
||||
// These create statements match the schema.sql when version 1 was the latest.
|
||||
tx.execute_batch(r#"
|
||||
alter table camera rename to old_camera;
|
||||
create table camera (
|
||||
id integer primary key,
|
||||
uuid blob unique not null check (length(uuid) = 16),
|
||||
short_name text not null,
|
||||
description text,
|
||||
host text,
|
||||
username text,
|
||||
password text,
|
||||
main_rtsp_path text,
|
||||
sub_rtsp_path text,
|
||||
retain_bytes integer not null check (retain_bytes >= 0),
|
||||
next_recording_id integer not null check (next_recording_id >= 0)
|
||||
);
|
||||
alter table recording rename to old_recording;
|
||||
drop index recording_cover;
|
||||
create table recording (
|
||||
composite_id integer primary key,
|
||||
camera_id integer not null references camera (id),
|
||||
run_offset integer not null,
|
||||
flags integer not null,
|
||||
sample_file_bytes integer not null check (sample_file_bytes > 0),
|
||||
start_time_90k integer not null check (start_time_90k > 0),
|
||||
duration_90k integer not null
|
||||
check (duration_90k >= 0 and duration_90k < 5*60*90000),
|
||||
local_time_delta_90k integer not null,
|
||||
video_samples integer not null check (video_samples > 0),
|
||||
video_sync_samples integer not null check (video_samples > 0),
|
||||
video_sample_entry_id integer references video_sample_entry (id),
|
||||
check (composite_id >> 32 = camera_id)
|
||||
);
|
||||
create index recording_cover on recording (
|
||||
camera_id,
|
||||
start_time_90k,
|
||||
duration_90k,
|
||||
video_samples,
|
||||
video_sync_samples,
|
||||
video_sample_entry_id,
|
||||
sample_file_bytes,
|
||||
run_offset,
|
||||
flags
|
||||
);
|
||||
create table recording_playback (
|
||||
composite_id integer primary key references recording (composite_id),
|
||||
sample_file_uuid blob not null check (length(sample_file_uuid) = 16),
|
||||
sample_file_sha1 blob not null check (length(sample_file_sha1) = 20),
|
||||
video_index blob not null check (length(video_index) > 0)
|
||||
);
|
||||
insert into camera
|
||||
select
|
||||
id,
|
||||
uuid,
|
||||
short_name,
|
||||
description,
|
||||
host,
|
||||
username,
|
||||
password,
|
||||
main_rtsp_path,
|
||||
sub_rtsp_path,
|
||||
retain_bytes,
|
||||
1 as next_recording_id
|
||||
from
|
||||
old_camera;
|
||||
"#)?;
|
||||
let camera_state = fill_recording(tx)?;
|
||||
update_camera(tx, camera_state)?;
|
||||
tx.execute_batch(r#"
|
||||
drop table old_recording;
|
||||
drop table old_camera;
|
||||
"#)?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
struct CameraState {
|
||||
/// tuple of (run_start_id, next_start_90k).
|
||||
current_run: Option<(i64, i64)>,
|
||||
|
||||
/// As in the `next_recording_id` field of the `camera` table.
|
||||
next_recording_id: i32,
|
||||
}
|
||||
|
||||
fn has_trailing_zero(video_index: &[u8]) -> Result<bool, Error> {
|
||||
let mut it = recording::SampleIndexIterator::new();
|
||||
while it.next(video_index)? {}
|
||||
Ok(it.duration_90k == 0)
|
||||
}
|
||||
|
||||
/// Fills the `recording` and `recording_playback` tables from `old_recording`, returning
|
||||
/// the `camera_state` map for use by a following call to `fill_cameras`.
|
||||
fn fill_recording(tx: &rusqlite::Transaction) -> Result<HashMap<i32, CameraState>, Error> {
|
||||
let mut select = tx.prepare(r#"
|
||||
select
|
||||
camera_id,
|
||||
sample_file_bytes,
|
||||
start_time_90k,
|
||||
duration_90k,
|
||||
local_time_delta_90k,
|
||||
video_samples,
|
||||
video_sync_samples,
|
||||
video_sample_entry_id,
|
||||
sample_file_uuid,
|
||||
sample_file_sha1,
|
||||
video_index,
|
||||
id
|
||||
from
|
||||
old_recording
|
||||
"#)?;
|
||||
let mut insert1 = tx.prepare(r#"
|
||||
insert into recording values (:composite_id, :camera_id, :run_offset, :flags,
|
||||
:sample_file_bytes, :start_time_90k, :duration_90k,
|
||||
:local_time_delta_90k, :video_samples, :video_sync_samples,
|
||||
:video_sample_entry_id)
|
||||
"#)?;
|
||||
let mut insert2 = tx.prepare(r#"
|
||||
insert into recording_playback values (:composite_id, :sample_file_uuid, :sample_file_sha1,
|
||||
:video_index)
|
||||
"#)?;
|
||||
let mut rows = select.query(params![])?;
|
||||
let mut camera_state: HashMap<i32, CameraState> = HashMap::new();
|
||||
while let Some(row) = rows.next()? {
|
||||
let camera_id: i32 = row.get(0)?;
|
||||
let camera_state = camera_state.entry(camera_id).or_insert_with(|| {
|
||||
CameraState{
|
||||
current_run: None,
|
||||
next_recording_id: 1,
|
||||
}
|
||||
});
|
||||
let composite_id = ((camera_id as i64) << 32) | (camera_state.next_recording_id as i64);
|
||||
camera_state.next_recording_id += 1;
|
||||
let sample_file_bytes: i32 = row.get(1)?;
|
||||
let start_time_90k: i64 = row.get(2)?;
|
||||
let duration_90k: i32 = row.get(3)?;
|
||||
let local_time_delta_90k: i64 = row.get(4)?;
|
||||
let video_samples: i32 = row.get(5)?;
|
||||
let video_sync_samples: i32 = row.get(6)?;
|
||||
let video_sample_entry_id: i32 = row.get(7)?;
|
||||
let sample_file_uuid: db::FromSqlUuid = row.get(8)?;
|
||||
let sample_file_sha1: Vec<u8> = row.get(9)?;
|
||||
let video_index: Vec<u8> = row.get(10)?;
|
||||
let old_id: i32 = row.get(11)?;
|
||||
let trailing_zero = has_trailing_zero(&video_index).unwrap_or_else(|e| {
|
||||
warn!("recording {}/{} (sample file {}, formerly recording {}) has corrupt \
|
||||
video_index: {}",
|
||||
camera_id, composite_id & 0xFFFF, sample_file_uuid.0, old_id, e);
|
||||
false
|
||||
});
|
||||
let run_id = match camera_state.current_run {
|
||||
Some((run_id, expected_start)) if expected_start == start_time_90k => run_id,
|
||||
_ => composite_id,
|
||||
};
|
||||
insert1.execute_named(&[
|
||||
(":composite_id", &composite_id),
|
||||
(":camera_id", &camera_id),
|
||||
(":run_offset", &(composite_id - run_id)),
|
||||
(":flags", &(if trailing_zero { db::RecordingFlags::TrailingZero as i32 } else { 0 })),
|
||||
(":sample_file_bytes", &sample_file_bytes),
|
||||
(":start_time_90k", &start_time_90k),
|
||||
(":duration_90k", &duration_90k),
|
||||
(":local_time_delta_90k", &local_time_delta_90k),
|
||||
(":video_samples", &video_samples),
|
||||
(":video_sync_samples", &video_sync_samples),
|
||||
(":video_sample_entry_id", &video_sample_entry_id),
|
||||
])?;
|
||||
insert2.execute_named(&[
|
||||
(":composite_id", &composite_id),
|
||||
(":sample_file_uuid", &&sample_file_uuid.0.as_bytes()[..]),
|
||||
(":sample_file_sha1", &sample_file_sha1),
|
||||
(":video_index", &video_index),
|
||||
])?;
|
||||
camera_state.current_run = if trailing_zero {
|
||||
None
|
||||
} else {
|
||||
Some((run_id, start_time_90k + duration_90k as i64))
|
||||
};
|
||||
}
|
||||
Ok(camera_state)
|
||||
}
|
||||
|
||||
fn update_camera(tx: &rusqlite::Transaction, camera_state: HashMap<i32, CameraState>)
|
||||
-> Result<(), Error> {
|
||||
let mut stmt = tx.prepare(r#"
|
||||
update camera set next_recording_id = :next_recording_id where id = :id
|
||||
"#)?;
|
||||
for (ref id, ref state) in &camera_state {
|
||||
stmt.execute_named(&[
|
||||
(":id", &id),
|
||||
(":next_recording_id", &state.next_recording_id),
|
||||
])?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
205
server/db/upgrade/v1.sql
Normal file
205
server/db/upgrade/v1.sql
Normal file
@@ -0,0 +1,205 @@
|
||||
-- This file is part of Moonfire NVR, a security camera network video recorder.
|
||||
-- Copyright (C) 2016 The Moonfire NVR Authors
|
||||
--
|
||||
-- This program is free software: you can redistribute it and/or modify
|
||||
-- it under the terms of the GNU General Public License as published by
|
||||
-- the Free Software Foundation, either version 3 of the License, or
|
||||
-- (at your option) any later version.
|
||||
--
|
||||
-- In addition, as a special exception, the copyright holders give
|
||||
-- permission to link the code of portions of this program with the
|
||||
-- OpenSSL library under certain conditions as described in each
|
||||
-- individual source file, and distribute linked combinations including
|
||||
-- the two.
|
||||
--
|
||||
-- You must obey the GNU General Public License in all respects for all
|
||||
-- of the code used other than OpenSSL. If you modify file(s) with this
|
||||
-- exception, you may extend this exception to your version of the
|
||||
-- file(s), but you are not obligated to do so. If you do not wish to do
|
||||
-- so, delete this exception statement from your version. If you delete
|
||||
-- this exception statement from all source files in the program, then
|
||||
-- also delete it here.
|
||||
--
|
||||
-- This program is distributed in the hope that it will be useful,
|
||||
-- but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
-- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
-- GNU General Public License for more details.
|
||||
--
|
||||
-- You should have received a copy of the GNU General Public License
|
||||
-- along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
--
|
||||
-- schema.sql: SQLite3 database schema for Moonfire NVR.
|
||||
-- See also design/schema.md.
|
||||
|
||||
-- This table tracks the schema version.
|
||||
-- There is one row for the initial database creation (inserted below, after the
|
||||
-- create statements) and one for each upgrade procedure (if any).
|
||||
create table version (
|
||||
id integer primary key,
|
||||
|
||||
-- The unix time as of the creation/upgrade, as determined by
|
||||
-- cast(strftime('%s', 'now') as int).
|
||||
unix_time integer not null,
|
||||
|
||||
-- Optional notes on the creation/upgrade; could include the binary version.
|
||||
notes text
|
||||
);
|
||||
|
||||
create table camera (
|
||||
id integer primary key,
|
||||
uuid blob unique not null check (length(uuid) = 16),
|
||||
|
||||
-- A short name of the camera, used in log messages.
|
||||
short_name text not null,
|
||||
|
||||
-- A short description of the camera.
|
||||
description text,
|
||||
|
||||
-- The host (or IP address) to use in rtsp:// URLs when accessing the camera.
|
||||
host text,
|
||||
|
||||
-- The username to use when accessing the camera.
|
||||
-- If empty, no username or password will be supplied.
|
||||
username text,
|
||||
|
||||
-- The password to use when accessing the camera.
|
||||
password text,
|
||||
|
||||
-- The path (starting with "/") to use in rtsp:// URLs to reference this
|
||||
-- camera's "main" (full-quality) video stream.
|
||||
main_rtsp_path text,
|
||||
|
||||
-- The path (starting with "/") to use in rtsp:// URLs to reference this
|
||||
-- camera's "sub" (low-bandwidth) video stream.
|
||||
sub_rtsp_path text,
|
||||
|
||||
-- The number of bytes of video to retain, excluding the currently-recording
|
||||
-- file. Older files will be deleted as necessary to stay within this limit.
|
||||
retain_bytes integer not null check (retain_bytes >= 0),
|
||||
|
||||
-- The low 32 bits of the next recording id to assign for this camera.
|
||||
-- Typically this is the maximum current recording + 1, but it does
|
||||
-- not decrease if that recording is deleted.
|
||||
next_recording_id integer not null check (next_recording_id >= 0)
|
||||
);
|
||||
|
||||
-- Each row represents a single completed recorded segment of video.
|
||||
-- Recordings are typically ~60 seconds; never more than 5 minutes.
|
||||
create table recording (
|
||||
-- The high 32 bits of composite_id are taken from the camera's id, which
|
||||
-- improves locality. The low 32 bits are taken from the camera's
|
||||
-- next_recording_id (which should be post-incremented in the same
|
||||
-- transaction). It'd be simpler to use a "without rowid" table and separate
|
||||
-- fields to make up the primary key, but
|
||||
-- <https://www.sqlite.org/withoutrowid.html> points out that "without rowid"
|
||||
-- is not appropriate when the average row size is in excess of 50 bytes.
|
||||
-- recording_cover rows (which match this id format) are typically 1--5 KiB.
|
||||
composite_id integer primary key,
|
||||
|
||||
-- This field is redundant with id above, but used to enforce the reference
|
||||
-- constraint and to structure the recording_start_time index.
|
||||
camera_id integer not null references camera (id),
|
||||
|
||||
-- The offset of this recording within a run. 0 means this was the first
|
||||
-- recording made from a RTSP session. The start of the run has id
|
||||
-- (id-run_offset).
|
||||
run_offset integer not null,
|
||||
|
||||
-- flags is a bitmask:
|
||||
--
|
||||
-- * 1, or "trailing zero", indicates that this recording is the last in a
|
||||
-- stream. As the duration of a sample is not known until the next sample
|
||||
-- is received, the final sample in this recording will have duration 0.
|
||||
flags integer not null,
|
||||
|
||||
sample_file_bytes integer not null check (sample_file_bytes > 0),
|
||||
|
||||
-- The starting time of the recording, in 90 kHz units since
|
||||
-- 1970-01-01 00:00:00 UTC. Currently on initial connection, this is taken
|
||||
-- from the local system time; on subsequent recordings, it exactly
|
||||
-- matches the previous recording's end time.
|
||||
start_time_90k integer not null check (start_time_90k > 0),
|
||||
|
||||
-- The duration of the recording, in 90 kHz units.
|
||||
duration_90k integer not null
|
||||
check (duration_90k >= 0 and duration_90k < 5*60*90000),
|
||||
|
||||
-- The number of 90 kHz units the local system time is ahead of the
|
||||
-- recording; negative numbers indicate the local system time is behind
|
||||
-- the recording. Large absolute values would indicate that the local time
|
||||
-- has jumped during recording or that the local time and camera time
|
||||
-- frequencies do not match.
|
||||
local_time_delta_90k integer not null,
|
||||
|
||||
video_samples integer not null check (video_samples > 0),
|
||||
video_sync_samples integer not null check (video_sync_samples > 0),
|
||||
video_sample_entry_id integer references video_sample_entry (id),
|
||||
|
||||
check (composite_id >> 32 = camera_id)
|
||||
);
|
||||
|
||||
create index recording_cover on recording (
|
||||
-- Typical queries use "where camera_id = ? order by start_time_90k".
|
||||
camera_id,
|
||||
start_time_90k,
|
||||
|
||||
-- These fields are not used for ordering; they cover most queries so
|
||||
-- that only database verification and actual viewing of recordings need
|
||||
-- to consult the underlying row.
|
||||
duration_90k,
|
||||
video_samples,
|
||||
video_sync_samples,
|
||||
video_sample_entry_id,
|
||||
sample_file_bytes,
|
||||
run_offset,
|
||||
flags
|
||||
);
|
||||
|
||||
-- Large fields for a recording which are not needed when simply listing all
|
||||
-- of the recordings in a given range. In particular, when serving a byte
|
||||
-- range within a .mp4 file, the recording_playback row is needed for the
|
||||
-- recording(s) corresponding to that particular byte range, needed, but the
|
||||
-- recording rows suffice for all other recordings in the .mp4.
|
||||
create table recording_playback (
|
||||
-- See description on recording table.
|
||||
composite_id integer primary key references recording (composite_id),
|
||||
|
||||
-- The binary representation of the sample file's uuid. The canonical text
|
||||
-- representation of this uuid is the filename within the sample file dir.
|
||||
sample_file_uuid blob not null check (length(sample_file_uuid) = 16),
|
||||
|
||||
-- The sha1 hash of the contents of the sample file.
|
||||
sample_file_sha1 blob not null check (length(sample_file_sha1) = 20),
|
||||
|
||||
-- See design/schema.md#video_index for a description of this field.
|
||||
video_index blob not null check (length(video_index) > 0)
|
||||
);
|
||||
|
||||
-- Files in the sample file directory which may be present but should simply be
|
||||
-- discarded on startup. (Recordings which were never completed or have been
|
||||
-- marked for completion.)
|
||||
create table reserved_sample_files (
|
||||
uuid blob primary key check (length(uuid) = 16),
|
||||
state integer not null -- 0 (writing) or 1 (deleted)
|
||||
) without rowid;
|
||||
|
||||
-- A concrete box derived from a ISO/IEC 14496-12 section 8.5.2
|
||||
-- VisualSampleEntry box. Describes the codec, width, height, etc.
|
||||
create table video_sample_entry (
|
||||
id integer primary key,
|
||||
|
||||
-- A SHA-1 hash of |bytes|.
|
||||
sha1 blob unique not null check (length(sha1) = 20),
|
||||
|
||||
-- The width and height in pixels; must match values within
|
||||
-- |sample_entry_bytes|.
|
||||
width integer not null check (width > 0),
|
||||
height integer not null check (height > 0),
|
||||
|
||||
-- The serialized box, including the leading length and box type (avcC in
|
||||
-- the case of H.264).
|
||||
data blob not null check (length(data) > 86)
|
||||
);
|
||||
|
||||
insert into version (id, unix_time, notes)
|
||||
values (1, cast(strftime('%s', 'now') as int), 'db creation');
|
||||
409
server/db/upgrade/v1_to_v2.rs
Normal file
409
server/db/upgrade/v1_to_v2.rs
Normal file
@@ -0,0 +1,409 @@
|
||||
// This file is part of Moonfire NVR, a security camera network video recorder.
|
||||
// Copyright (C) 2018 The Moonfire NVR Authors
|
||||
//
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// In addition, as a special exception, the copyright holders give
|
||||
// permission to link the code of portions of this program with the
|
||||
// OpenSSL library under certain conditions as described in each
|
||||
// individual source file, and distribute linked combinations including
|
||||
// the two.
|
||||
//
|
||||
// You must obey the GNU General Public License in all respects for all
|
||||
// of the code used other than OpenSSL. If you modify file(s) with this
|
||||
// exception, you may extend this exception to your version of the
|
||||
// file(s), but you are not obligated to do so. If you do not wish to do
|
||||
// so, delete this exception statement from your version. If you delete
|
||||
// this exception statement from all source files in the program, then
|
||||
// also delete it here.
|
||||
//
|
||||
// This program is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
/// Upgrades a version 1 schema to a version 2 schema.
|
||||
|
||||
use crate::dir;
|
||||
use failure::{Error, bail, format_err};
|
||||
use nix::fcntl::{FlockArg, OFlag};
|
||||
use nix::sys::stat::Mode;
|
||||
use rusqlite::params;
|
||||
use crate::schema::DirMeta;
|
||||
use std::os::unix::io::AsRawFd;
|
||||
use uuid::Uuid;
|
||||
|
||||
pub fn run(args: &super::Args, tx: &rusqlite::Transaction) -> Result<(), Error> {
|
||||
let sample_file_path =
|
||||
args.sample_file_dir
|
||||
.ok_or_else(|| format_err!("--sample-file-dir required when upgrading from \
|
||||
schema version 1 to 2."))?;
|
||||
|
||||
let mut d = nix::dir::Dir::open(sample_file_path, OFlag::O_DIRECTORY | OFlag::O_RDONLY,
|
||||
Mode::empty())?;
|
||||
nix::fcntl::flock(d.as_raw_fd(), FlockArg::LockExclusiveNonblock)?;
|
||||
verify_dir_contents(sample_file_path, &mut d, tx)?;
|
||||
|
||||
// These create statements match the schema.sql when version 2 was the latest.
|
||||
tx.execute_batch(r#"
|
||||
create table meta (
|
||||
uuid blob not null check (length(uuid) = 16)
|
||||
);
|
||||
create table open (
|
||||
id integer primary key,
|
||||
uuid blob unique not null check (length(uuid) = 16),
|
||||
start_time_90k integer,
|
||||
end_time_90k integer,
|
||||
duration_90k integer
|
||||
);
|
||||
create table sample_file_dir (
|
||||
id integer primary key,
|
||||
path text unique not null,
|
||||
uuid blob unique not null check (length(uuid) = 16),
|
||||
last_complete_open_id integer references open (id)
|
||||
);
|
||||
create table user (
|
||||
id integer primary key,
|
||||
username unique not null,
|
||||
flags integer not null,
|
||||
password_hash text,
|
||||
password_id integer not null default 0,
|
||||
password_failure_count integer not null default 0,
|
||||
unix_uid integer
|
||||
);
|
||||
create table user_session (
|
||||
session_id_hash blob primary key not null,
|
||||
user_id integer references user (id) not null,
|
||||
seed blob not null,
|
||||
flags integer not null,
|
||||
domain text,
|
||||
description text,
|
||||
creation_password_id integer,
|
||||
creation_time_sec integer not null,
|
||||
creation_user_agent text,
|
||||
creation_peer_addr blob,
|
||||
revocation_time_sec integer,
|
||||
revocation_user_agent text,
|
||||
revocation_peer_addr blob,
|
||||
revocation_reason integer,
|
||||
revocation_reason_detail text,
|
||||
last_use_time_sec integer,
|
||||
last_use_user_agent text,
|
||||
last_use_peer_addr blob,
|
||||
use_count not null default 0
|
||||
) without rowid;
|
||||
create index user_session_uid on user_session (user_id);
|
||||
"#)?;
|
||||
let db_uuid = ::uuid::Uuid::new_v4();
|
||||
let db_uuid_bytes = &db_uuid.as_bytes()[..];
|
||||
tx.execute("insert into meta (uuid) values (?)", params![db_uuid_bytes])?;
|
||||
let open_uuid = ::uuid::Uuid::new_v4();
|
||||
let open_uuid_bytes = &open_uuid.as_bytes()[..];
|
||||
tx.execute("insert into open (uuid) values (?)", params![open_uuid_bytes])?;
|
||||
let open_id = tx.last_insert_rowid() as u32;
|
||||
let dir_uuid = ::uuid::Uuid::new_v4();
|
||||
let dir_uuid_bytes = &dir_uuid.as_bytes()[..];
|
||||
|
||||
// Write matching metadata to the directory.
|
||||
let mut meta = DirMeta::default();
|
||||
{
|
||||
meta.db_uuid.extend_from_slice(db_uuid_bytes);
|
||||
meta.dir_uuid.extend_from_slice(dir_uuid_bytes);
|
||||
let open = meta.last_complete_open.set_default();
|
||||
open.id = open_id;
|
||||
open.uuid.extend_from_slice(&open_uuid_bytes);
|
||||
}
|
||||
dir::write_meta(d.as_raw_fd(), &meta)?;
|
||||
|
||||
let sample_file_path = sample_file_path.to_str()
|
||||
.ok_or_else(|| format_err!("sample file dir {} is not a valid string",
|
||||
sample_file_path.display()))?;
|
||||
tx.execute(r#"
|
||||
insert into sample_file_dir (path, uuid, last_complete_open_id)
|
||||
values (?, ?, ?)
|
||||
"#, params![sample_file_path, dir_uuid_bytes, open_id])?;
|
||||
|
||||
tx.execute_batch(r#"
|
||||
drop table reserved_sample_files;
|
||||
alter table camera rename to old_camera;
|
||||
alter table recording rename to old_recording;
|
||||
alter table video_sample_entry rename to old_video_sample_entry;
|
||||
drop index recording_cover;
|
||||
|
||||
create table camera (
|
||||
id integer primary key,
|
||||
uuid blob unique not null check (length(uuid) = 16),
|
||||
short_name text not null,
|
||||
description text,
|
||||
host text,
|
||||
username text,
|
||||
password text
|
||||
);
|
||||
|
||||
create table stream (
|
||||
id integer primary key,
|
||||
camera_id integer not null references camera (id),
|
||||
sample_file_dir_id integer references sample_file_dir (id),
|
||||
type text not null check (type in ('main', 'sub')),
|
||||
record integer not null check (record in (1, 0)),
|
||||
rtsp_path text not null,
|
||||
retain_bytes integer not null check (retain_bytes >= 0),
|
||||
flush_if_sec integer not null,
|
||||
next_recording_id integer not null check (next_recording_id >= 0),
|
||||
unique (camera_id, type)
|
||||
);
|
||||
|
||||
create table recording (
|
||||
composite_id integer primary key,
|
||||
open_id integer not null,
|
||||
stream_id integer not null references stream (id),
|
||||
run_offset integer not null,
|
||||
flags integer not null,
|
||||
sample_file_bytes integer not null check (sample_file_bytes > 0),
|
||||
start_time_90k integer not null check (start_time_90k > 0),
|
||||
duration_90k integer not null
|
||||
check (duration_90k >= 0 and duration_90k < 5*60*90000),
|
||||
video_samples integer not null check (video_samples > 0),
|
||||
video_sync_samples integer not null check (video_sync_samples > 0),
|
||||
video_sample_entry_id integer references video_sample_entry (id),
|
||||
check (composite_id >> 32 = stream_id)
|
||||
);
|
||||
|
||||
create index recording_cover on recording (
|
||||
stream_id,
|
||||
start_time_90k,
|
||||
open_id,
|
||||
duration_90k,
|
||||
video_samples,
|
||||
video_sync_samples,
|
||||
video_sample_entry_id,
|
||||
sample_file_bytes,
|
||||
run_offset,
|
||||
flags
|
||||
);
|
||||
|
||||
create table recording_integrity (
|
||||
composite_id integer primary key references recording (composite_id),
|
||||
local_time_delta_90k integer,
|
||||
local_time_since_open_90k integer,
|
||||
wall_time_delta_90k integer,
|
||||
sample_file_sha1 blob check (length(sample_file_sha1) <= 20)
|
||||
);
|
||||
|
||||
create table video_sample_entry (
|
||||
id integer primary key,
|
||||
sha1 blob unique not null check (length(sha1) = 20),
|
||||
width integer not null check (width > 0),
|
||||
height integer not null check (height > 0),
|
||||
rfc6381_codec text not null,
|
||||
data blob not null check (length(data) > 86)
|
||||
);
|
||||
|
||||
create table garbage (
|
||||
sample_file_dir_id integer references sample_file_dir (id),
|
||||
composite_id integer,
|
||||
primary key (sample_file_dir_id, composite_id)
|
||||
) without rowid;
|
||||
|
||||
insert into camera
|
||||
select
|
||||
id,
|
||||
uuid,
|
||||
short_name,
|
||||
description,
|
||||
host,
|
||||
username,
|
||||
password
|
||||
from old_camera;
|
||||
|
||||
-- Insert main streams using the same id as the camera, to ease changing recordings.
|
||||
insert into stream
|
||||
select
|
||||
old_camera.id,
|
||||
old_camera.id,
|
||||
sample_file_dir.id,
|
||||
'main',
|
||||
1,
|
||||
old_camera.main_rtsp_path,
|
||||
old_camera.retain_bytes,
|
||||
0,
|
||||
old_camera.next_recording_id
|
||||
from
|
||||
old_camera cross join sample_file_dir;
|
||||
|
||||
-- Insert sub stream (if path is non-empty) using any id.
|
||||
insert into stream (camera_id, sample_file_dir_id, type, record, rtsp_path,
|
||||
retain_bytes, flush_if_sec, next_recording_id)
|
||||
select
|
||||
old_camera.id,
|
||||
sample_file_dir.id,
|
||||
'sub',
|
||||
0,
|
||||
old_camera.sub_rtsp_path,
|
||||
0,
|
||||
90,
|
||||
1
|
||||
from
|
||||
old_camera cross join sample_file_dir
|
||||
where
|
||||
old_camera.sub_rtsp_path != '';
|
||||
"#)?;
|
||||
|
||||
// Add the new video_sample_entry rows, before inserting the recordings referencing them.
|
||||
fix_video_sample_entry(tx)?;
|
||||
|
||||
tx.execute_batch(r#"
|
||||
insert into recording
|
||||
select
|
||||
r.composite_id,
|
||||
r.camera_id,
|
||||
o.id,
|
||||
r.run_offset,
|
||||
r.flags,
|
||||
r.sample_file_bytes,
|
||||
r.start_time_90k,
|
||||
r.duration_90k,
|
||||
r.video_samples,
|
||||
r.video_sync_samples,
|
||||
r.video_sample_entry_id
|
||||
from
|
||||
old_recording r cross join open o;
|
||||
|
||||
insert into recording_integrity (composite_id, local_time_delta_90k, sample_file_sha1)
|
||||
select
|
||||
r.composite_id,
|
||||
case when r.run_offset > 0 then local_time_delta_90k else null end,
|
||||
p.sample_file_sha1
|
||||
from
|
||||
old_recording r join recording_playback p on (r.composite_id = p.composite_id);
|
||||
"#)?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Ensures the sample file directory has the expected contents.
|
||||
/// Among other problems, this catches a fat-fingered `--sample-file-dir`.
|
||||
/// The expected contents are:
|
||||
///
|
||||
/// * required: recording uuids.
|
||||
/// * optional: reserved sample file uuids.
|
||||
/// * optional: meta and meta-tmp from half-completed update attempts.
|
||||
/// * forbidden: anything else.
|
||||
fn verify_dir_contents(sample_file_path: &std::path::Path, dir: &mut nix::dir::Dir,
|
||||
tx: &rusqlite::Transaction) -> Result<(), Error> {
|
||||
// Build a hash of the uuids found in the directory.
|
||||
let n: i64 = tx.query_row(r#"
|
||||
select
|
||||
a.c + b.c
|
||||
from
|
||||
(select count(*) as c from recording) a,
|
||||
(select count(*) as c from reserved_sample_files) b;
|
||||
"#, params![], |r| r.get(0))?;
|
||||
let mut files = ::fnv::FnvHashSet::with_capacity_and_hasher(n as usize, Default::default());
|
||||
for e in dir.iter() {
|
||||
let e = e?;
|
||||
let f = e.file_name();
|
||||
match f.to_bytes() {
|
||||
b"." | b".." => continue,
|
||||
b"meta" | b"meta-tmp" => {
|
||||
// Ignore metadata files. These might from a half-finished update attempt.
|
||||
// If the directory is actually an in-use >v3 format, other contents won't match.
|
||||
continue;
|
||||
},
|
||||
_ => {},
|
||||
};
|
||||
let s = match f.to_str() {
|
||||
Ok(s) => s,
|
||||
Err(_) => bail!("unexpected file {:?} in {:?}", f, sample_file_path),
|
||||
};
|
||||
let uuid = match Uuid::parse_str(s) {
|
||||
Ok(u) => u,
|
||||
Err(_) => bail!("unexpected file {:?} in {:?}", f, sample_file_path),
|
||||
};
|
||||
if s != uuid.to_hyphenated_ref().to_string() { // non-canonical form.
|
||||
bail!("unexpected file {:?} in {:?}", f, sample_file_path);
|
||||
}
|
||||
files.insert(uuid);
|
||||
}
|
||||
|
||||
// Iterate through the database and check that everything has a matching file.
|
||||
{
|
||||
let mut stmt = tx.prepare(r"select sample_file_uuid from recording_playback")?;
|
||||
let mut rows = stmt.query(params![])?;
|
||||
while let Some(row) = rows.next()? {
|
||||
let uuid: crate::db::FromSqlUuid = row.get(0)?;
|
||||
if !files.remove(&uuid.0) {
|
||||
bail!("{} is missing from dir {}!", uuid.0, sample_file_path.display());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let mut stmt = tx.prepare(r"select uuid from reserved_sample_files")?;
|
||||
let mut rows = stmt.query(params![])?;
|
||||
while let Some(row) = rows.next()? {
|
||||
let uuid: crate::db::FromSqlUuid = row.get(0)?;
|
||||
if files.remove(&uuid.0) {
|
||||
// Also remove the garbage file. For historical reasons (version 2 was originally
|
||||
// defined as not having a garbage table so still is), do this here rather than with
|
||||
// the other path manipulations in v2_to_v3.rs. There's no harm anyway in deleting
|
||||
// a garbage file so if the upgrade transation fails this is still a valid and complete
|
||||
// version 1 database.
|
||||
let p = super::UuidPath::from(uuid.0);
|
||||
nix::unistd::unlinkat(Some(dir.as_raw_fd()), &p,
|
||||
nix::unistd::UnlinkatFlags::NoRemoveDir)?;
|
||||
}
|
||||
}
|
||||
|
||||
if !files.is_empty() {
|
||||
bail!("{} unexpected sample file uuids in dir {}: {:?}!",
|
||||
files.len(), sample_file_path.display(), files);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn fix_video_sample_entry(tx: &rusqlite::Transaction) -> Result<(), Error> {
|
||||
let mut select = tx.prepare(r#"
|
||||
select
|
||||
id,
|
||||
sha1,
|
||||
width,
|
||||
height,
|
||||
data
|
||||
from
|
||||
old_video_sample_entry
|
||||
"#)?;
|
||||
let mut insert = tx.prepare(r#"
|
||||
insert into video_sample_entry values (:id, :sha1, :width, :height, :rfc6381_codec, :data)
|
||||
"#)?;
|
||||
let mut rows = select.query(params![])?;
|
||||
while let Some(row) = rows.next()? {
|
||||
let data: Vec<u8> = row.get(4)?;
|
||||
insert.execute_named(&[
|
||||
(":id", &row.get::<_, i32>(0)?),
|
||||
(":sha1", &row.get::<_, Vec<u8>>(1)?),
|
||||
(":width", &row.get::<_, i32>(2)?),
|
||||
(":height", &row.get::<_, i32>(3)?),
|
||||
(":rfc6381_codec", &rfc6381_codec_from_sample_entry(&data)?),
|
||||
(":data", &data),
|
||||
])?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
// This previously lived in h264.rs. As of version 1, H.264 is the only supported codec.
|
||||
fn rfc6381_codec_from_sample_entry(sample_entry: &[u8]) -> Result<String, Error> {
|
||||
if sample_entry.len() < 99 || &sample_entry[4..8] != b"avc1" ||
|
||||
&sample_entry[90..94] != b"avcC" {
|
||||
bail!("not a valid AVCSampleEntry");
|
||||
}
|
||||
let profile_idc = sample_entry[103];
|
||||
let constraint_flags_byte = sample_entry[104];
|
||||
let level_idc = sample_entry[105];
|
||||
Ok(format!("avc1.{:02x}{:02x}{:02x}", profile_idc, constraint_flags_byte, level_idc))
|
||||
}
|
||||
118
server/db/upgrade/v2_to_v3.rs
Normal file
118
server/db/upgrade/v2_to_v3.rs
Normal file
@@ -0,0 +1,118 @@
|
||||
// This file is part of Moonfire NVR, a security camera network video recorder.
|
||||
// Copyright (C) 2018 The Moonfire NVR Authors
|
||||
//
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// In addition, as a special exception, the copyright holders give
|
||||
// permission to link the code of portions of this program with the
|
||||
// OpenSSL library under certain conditions as described in each
|
||||
// individual source file, and distribute linked combinations including
|
||||
// the two.
|
||||
//
|
||||
// You must obey the GNU General Public License in all respects for all
|
||||
// of the code used other than OpenSSL. If you modify file(s) with this
|
||||
// exception, you may extend this exception to your version of the
|
||||
// file(s), but you are not obligated to do so. If you do not wish to do
|
||||
// so, delete this exception statement from your version. If you delete
|
||||
// this exception statement from all source files in the program, then
|
||||
// also delete it here.
|
||||
//
|
||||
// This program is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
/// Upgrades a version 2 schema to a version 3 schema.
|
||||
/// Note that a version 2 schema is never actually used; so we know the upgrade from version 1 was
|
||||
/// completed, and possibly an upgrade from 2 to 3 is half-finished.
|
||||
|
||||
use crate::db::{self, FromSqlUuid};
|
||||
use crate::dir;
|
||||
use failure::Error;
|
||||
use crate::schema;
|
||||
use rusqlite::params;
|
||||
use std::os::unix::io::AsRawFd;
|
||||
use std::sync::Arc;
|
||||
|
||||
/// Opens the sample file dir.
|
||||
///
|
||||
/// Makes a couple simplifying assumptions valid for version 2:
|
||||
/// * there's only one dir.
|
||||
/// * it has a last completed open.
|
||||
fn open_sample_file_dir(tx: &rusqlite::Transaction) -> Result<Arc<dir::SampleFileDir>, Error> {
|
||||
let (p, s_uuid, o_id, o_uuid, db_uuid): (String, FromSqlUuid, i32, FromSqlUuid, FromSqlUuid) =
|
||||
tx.query_row(r#"
|
||||
select
|
||||
s.path, s.uuid, s.last_complete_open_id, o.uuid, m.uuid
|
||||
from
|
||||
sample_file_dir s
|
||||
join open o on (s.last_complete_open_id = o.id)
|
||||
cross join meta m
|
||||
"#, params![], |row| {
|
||||
Ok((row.get(0)?,
|
||||
row.get(1)?,
|
||||
row.get(2)?,
|
||||
row.get(3)?,
|
||||
row.get(4)?))
|
||||
})?;
|
||||
let mut meta = schema::DirMeta::default();
|
||||
meta.db_uuid.extend_from_slice(&db_uuid.0.as_bytes()[..]);
|
||||
meta.dir_uuid.extend_from_slice(&s_uuid.0.as_bytes()[..]);
|
||||
{
|
||||
let open = meta.last_complete_open.set_default();
|
||||
open.id = o_id as u32;
|
||||
open.uuid.extend_from_slice(&o_uuid.0.as_bytes()[..]);
|
||||
}
|
||||
dir::SampleFileDir::open(&p, &meta)
|
||||
}
|
||||
|
||||
pub fn run(_args: &super::Args, tx: &rusqlite::Transaction) -> Result<(), Error> {
|
||||
let d = open_sample_file_dir(&tx)?;
|
||||
let mut stmt = tx.prepare(r#"
|
||||
select
|
||||
composite_id,
|
||||
sample_file_uuid
|
||||
from
|
||||
recording_playback
|
||||
"#)?;
|
||||
let mut rows = stmt.query(params![])?;
|
||||
while let Some(row) = rows.next()? {
|
||||
let id = db::CompositeId(row.get(0)?);
|
||||
let sample_file_uuid: FromSqlUuid = row.get(1)?;
|
||||
let from_path = super::UuidPath::from(sample_file_uuid.0);
|
||||
let to_path = crate::dir::CompositeIdPath::from(id);
|
||||
if let Err(e) = nix::fcntl::renameat(Some(d.fd.as_raw_fd()), &from_path,
|
||||
Some(d.fd.as_raw_fd()), &to_path) {
|
||||
if e == nix::Error::Sys(nix::errno::Errno::ENOENT) {
|
||||
continue; // assume it was already moved.
|
||||
}
|
||||
Err(e)?;
|
||||
}
|
||||
}
|
||||
|
||||
// These create statements match the schema.sql when version 3 was the latest.
|
||||
tx.execute_batch(r#"
|
||||
alter table recording_playback rename to old_recording_playback;
|
||||
create table recording_playback (
|
||||
composite_id integer primary key references recording (composite_id),
|
||||
video_index blob not null check (length(video_index) > 0)
|
||||
);
|
||||
insert into recording_playback
|
||||
select
|
||||
composite_id,
|
||||
video_index
|
||||
from
|
||||
old_recording_playback;
|
||||
drop table old_recording_playback;
|
||||
drop table old_recording;
|
||||
drop table old_camera;
|
||||
drop table old_video_sample_entry;
|
||||
"#)?;
|
||||
Ok(())
|
||||
}
|
||||
400
server/db/upgrade/v3.sql
Normal file
400
server/db/upgrade/v3.sql
Normal file
@@ -0,0 +1,400 @@
|
||||
-- This file is part of Moonfire NVR, a security camera network video recorder.
|
||||
-- Copyright (C) 2016 The Moonfire NVR Authors
|
||||
--
|
||||
-- This program is free software: you can redistribute it and/or modify
|
||||
-- it under the terms of the GNU General Public License as published by
|
||||
-- the Free Software Foundation, either version 3 of the License, or
|
||||
-- (at your option) any later version.
|
||||
--
|
||||
-- In addition, as a special exception, the copyright holders give
|
||||
-- permission to link the code of portions of this program with the
|
||||
-- OpenSSL library under certain conditions as described in each
|
||||
-- individual source file, and distribute linked combinations including
|
||||
-- the two.
|
||||
--
|
||||
-- You must obey the GNU General Public License in all respects for all
|
||||
-- of the code used other than OpenSSL. If you modify file(s) with this
|
||||
-- exception, you may extend this exception to your version of the
|
||||
-- file(s), but you are not obligated to do so. If you do not wish to do
|
||||
-- so, delete this exception statement from your version. If you delete
|
||||
-- this exception statement from all source files in the program, then
|
||||
-- also delete it here.
|
||||
--
|
||||
-- This program is distributed in the hope that it will be useful,
|
||||
-- but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
-- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
-- GNU General Public License for more details.
|
||||
--
|
||||
-- You should have received a copy of the GNU General Public License
|
||||
-- along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
--
|
||||
-- schema.sql: SQLite3 database schema for Moonfire NVR.
|
||||
-- See also design/schema.md.
|
||||
|
||||
-- Database metadata. There should be exactly one row in this table.
|
||||
create table meta (
|
||||
uuid blob not null check (length(uuid) = 16)
|
||||
);
|
||||
|
||||
-- This table tracks the schema version.
|
||||
-- There is one row for the initial database creation (inserted below, after the
|
||||
-- create statements) and one for each upgrade procedure (if any).
|
||||
create table version (
|
||||
id integer primary key,
|
||||
|
||||
-- The unix time as of the creation/upgrade, as determined by
|
||||
-- cast(strftime('%s', 'now') as int).
|
||||
unix_time integer not null,
|
||||
|
||||
-- Optional notes on the creation/upgrade; could include the binary version.
|
||||
notes text
|
||||
);
|
||||
|
||||
-- Tracks every time the database has been opened in read/write mode.
|
||||
-- This is used to ensure directories are in sync with the database (see
|
||||
-- schema.proto:DirMeta), to disambiguate uncommitted recordings, and
|
||||
-- potentially to understand time problems.
|
||||
create table open (
|
||||
id integer primary key,
|
||||
uuid blob unique not null check (length(uuid) = 16),
|
||||
|
||||
-- Information about when / how long the database was open. These may be all
|
||||
-- null, for example in the open that represents all information written
|
||||
-- prior to database version 3.
|
||||
|
||||
-- System time when the database was opened, in 90 kHz units since
|
||||
-- 1970-01-01 00:00:00Z excluding leap seconds.
|
||||
start_time_90k integer,
|
||||
|
||||
-- System time when the database was closed or (on crash) last flushed.
|
||||
end_time_90k integer,
|
||||
|
||||
-- How long the database was open. This is end_time_90k - start_time_90k if
|
||||
-- there were no time steps or leap seconds during this time.
|
||||
duration_90k integer
|
||||
);
|
||||
|
||||
create table sample_file_dir (
|
||||
id integer primary key,
|
||||
path text unique not null,
|
||||
uuid blob unique not null check (length(uuid) = 16),
|
||||
|
||||
-- The last (read/write) open of this directory which fully completed.
|
||||
-- See schema.proto:DirMeta for a more complete description.
|
||||
last_complete_open_id integer references open (id)
|
||||
);
|
||||
|
||||
create table camera (
|
||||
id integer primary key,
|
||||
uuid blob unique not null check (length(uuid) = 16),
|
||||
|
||||
-- A short name of the camera, used in log messages.
|
||||
short_name text not null,
|
||||
|
||||
-- A short description of the camera.
|
||||
description text,
|
||||
|
||||
-- The host (or IP address) to use in rtsp:// URLs when accessing the camera.
|
||||
host text,
|
||||
|
||||
-- The username to use when accessing the camera.
|
||||
-- If empty, no username or password will be supplied.
|
||||
username text,
|
||||
|
||||
-- The password to use when accessing the camera.
|
||||
password text
|
||||
);
|
||||
|
||||
create table stream (
|
||||
id integer primary key,
|
||||
camera_id integer not null references camera (id),
|
||||
sample_file_dir_id integer references sample_file_dir (id),
|
||||
type text not null check (type in ('main', 'sub')),
|
||||
|
||||
-- If record is true, the stream should start recording when moonfire
|
||||
-- starts. If false, no new recordings will be made, but old recordings
|
||||
-- will not be deleted.
|
||||
record integer not null check (record in (1, 0)),
|
||||
|
||||
-- The path (starting with "/") to use in rtsp:// URLs to for this stream.
|
||||
rtsp_path text not null,
|
||||
|
||||
-- The number of bytes of video to retain, excluding the currently-recording
|
||||
-- file. Older files will be deleted as necessary to stay within this limit.
|
||||
retain_bytes integer not null check (retain_bytes >= 0),
|
||||
|
||||
-- Flush the database when the first instant of completed recording is this
|
||||
-- many seconds old. A value of 0 means that every completed recording will
|
||||
-- cause an immediate flush. Higher values may allow flushes to be combined,
|
||||
-- reducing SSD write cycles. For example, if all streams have a flush_if_sec
|
||||
-- >= x sec, there will be:
|
||||
--
|
||||
-- * at most one flush per x sec in total
|
||||
-- * at most x sec of completed but unflushed recordings per stream.
|
||||
-- * at most x completed but unflushed recordings per stream, in the worst
|
||||
-- case where a recording instantly fails, waits the 1-second retry delay,
|
||||
-- then fails again, forever.
|
||||
flush_if_sec integer not null,
|
||||
|
||||
-- The low 32 bits of the next recording id to assign for this stream.
|
||||
-- Typically this is the maximum current recording + 1, but it does
|
||||
-- not decrease if that recording is deleted.
|
||||
next_recording_id integer not null check (next_recording_id >= 0),
|
||||
|
||||
unique (camera_id, type)
|
||||
);
|
||||
|
||||
-- Each row represents a single completed recorded segment of video.
|
||||
-- Recordings are typically ~60 seconds; never more than 5 minutes.
|
||||
create table recording (
|
||||
-- The high 32 bits of composite_id are taken from the stream's id, which
|
||||
-- improves locality. The low 32 bits are taken from the stream's
|
||||
-- next_recording_id (which should be post-incremented in the same
|
||||
-- transaction). It'd be simpler to use a "without rowid" table and separate
|
||||
-- fields to make up the primary key, but
|
||||
-- <https://www.sqlite.org/withoutrowid.html> points out that "without rowid"
|
||||
-- is not appropriate when the average row size is in excess of 50 bytes.
|
||||
-- recording_cover rows (which match this id format) are typically 1--5 KiB.
|
||||
composite_id integer primary key,
|
||||
|
||||
-- The open in which this was committed to the database. For a given
|
||||
-- composite_id, only one recording will ever be committed to the database,
|
||||
-- but in-memory state may reflect a recording which never gets committed.
|
||||
-- This field allows disambiguation in etags and such.
|
||||
open_id integer not null references open (id),
|
||||
|
||||
-- This field is redundant with id above, but used to enforce the reference
|
||||
-- constraint and to structure the recording_start_time index.
|
||||
stream_id integer not null references stream (id),
|
||||
|
||||
-- The offset of this recording within a run. 0 means this was the first
|
||||
-- recording made from a RTSP session. The start of the run has id
|
||||
-- (id-run_offset).
|
||||
run_offset integer not null,
|
||||
|
||||
-- flags is a bitmask:
|
||||
--
|
||||
-- * 1, or "trailing zero", indicates that this recording is the last in a
|
||||
-- stream. As the duration of a sample is not known until the next sample
|
||||
-- is received, the final sample in this recording will have duration 0.
|
||||
flags integer not null,
|
||||
|
||||
sample_file_bytes integer not null check (sample_file_bytes > 0),
|
||||
|
||||
-- The starting time of the recording, in 90 kHz units since
|
||||
-- 1970-01-01 00:00:00 UTC excluding leap seconds. Currently on initial
|
||||
-- connection, this is taken from the local system time; on subsequent
|
||||
-- recordings, it exactly matches the previous recording's end time.
|
||||
start_time_90k integer not null check (start_time_90k > 0),
|
||||
|
||||
-- The duration of the recording, in 90 kHz units.
|
||||
duration_90k integer not null
|
||||
check (duration_90k >= 0 and duration_90k < 5*60*90000),
|
||||
|
||||
video_samples integer not null check (video_samples > 0),
|
||||
video_sync_samples integer not null check (video_sync_samples > 0),
|
||||
video_sample_entry_id integer references video_sample_entry (id),
|
||||
|
||||
check (composite_id >> 32 = stream_id)
|
||||
);
|
||||
|
||||
create index recording_cover on recording (
|
||||
-- Typical queries use "where stream_id = ? order by start_time_90k".
|
||||
stream_id,
|
||||
start_time_90k,
|
||||
|
||||
-- These fields are not used for ordering; they cover most queries so
|
||||
-- that only database verification and actual viewing of recordings need
|
||||
-- to consult the underlying row.
|
||||
open_id,
|
||||
duration_90k,
|
||||
video_samples,
|
||||
video_sync_samples,
|
||||
video_sample_entry_id,
|
||||
sample_file_bytes,
|
||||
run_offset,
|
||||
flags
|
||||
);
|
||||
|
||||
-- Fields which are only needed to check/correct database integrity problems
|
||||
-- (such as incorrect timestamps).
|
||||
create table recording_integrity (
|
||||
-- See description on recording table.
|
||||
composite_id integer primary key references recording (composite_id),
|
||||
|
||||
-- The number of 90 kHz units the local system's monotonic clock has
|
||||
-- advanced more than the stated duration of recordings in a run since the
|
||||
-- first recording ended. Negative numbers indicate the local system time is
|
||||
-- behind the recording.
|
||||
--
|
||||
-- The first recording of a run (that is, one with run_offset=0) has null
|
||||
-- local_time_delta_90k because errors are assumed to
|
||||
-- be the result of initial buffering rather than frequency mismatch.
|
||||
--
|
||||
-- This value should be near 0 even on long runs in which the camera's clock
|
||||
-- and local system's clock frequency differ because each recording's delta
|
||||
-- is used to correct the durations of the next (up to 500 ppm error).
|
||||
local_time_delta_90k integer,
|
||||
|
||||
-- The number of 90 kHz units the local system's monotonic clock had
|
||||
-- advanced since the database was opened, as of the start of recording.
|
||||
-- TODO: fill this in!
|
||||
local_time_since_open_90k integer,
|
||||
|
||||
-- The difference between start_time_90k+duration_90k and a wall clock
|
||||
-- timestamp captured at end of this recording. This is meaningful for all
|
||||
-- recordings in a run, even the initial one (run_offset=0), because
|
||||
-- start_time_90k is derived from the wall time as of when recording
|
||||
-- starts, not when it ends.
|
||||
-- TODO: fill this in!
|
||||
wall_time_delta_90k integer,
|
||||
|
||||
-- The sha1 hash of the contents of the sample file.
|
||||
sample_file_sha1 blob check (length(sample_file_sha1) <= 20)
|
||||
);
|
||||
|
||||
-- Large fields for a recording which are needed ony for playback.
|
||||
-- In particular, when serving a byte range within a .mp4 file, the
|
||||
-- recording_playback row is needed for the recording(s) corresponding to that
|
||||
-- particular byte range, needed, but the recording rows suffice for all other
|
||||
-- recordings in the .mp4.
|
||||
create table recording_playback (
|
||||
-- See description on recording table.
|
||||
composite_id integer primary key references recording (composite_id),
|
||||
|
||||
-- See design/schema.md#video_index for a description of this field.
|
||||
video_index blob not null check (length(video_index) > 0)
|
||||
|
||||
-- audio_index could be added here in the future.
|
||||
);
|
||||
|
||||
-- Files which are to be deleted (may or may not still exist).
|
||||
-- Note that besides these files, for each stream, any recordings >= its
|
||||
-- next_recording_id should be discarded on startup.
|
||||
create table garbage (
|
||||
-- This is _mostly_ redundant with composite_id, which contains the stream
|
||||
-- id and thus a linkage to the sample file directory. Listing it here
|
||||
-- explicitly means that streams can be deleted without losing the
|
||||
-- association of garbage to directory.
|
||||
sample_file_dir_id integer not null references sample_file_dir (id),
|
||||
|
||||
-- See description on recording table.
|
||||
composite_id integer not null,
|
||||
|
||||
-- Organize the table first by directory, as that's how it will be queried.
|
||||
primary key (sample_file_dir_id, composite_id)
|
||||
) without rowid;
|
||||
|
||||
-- A concrete box derived from a ISO/IEC 14496-12 section 8.5.2
|
||||
-- VisualSampleEntry box. Describes the codec, width, height, etc.
|
||||
create table video_sample_entry (
|
||||
id integer primary key,
|
||||
|
||||
-- A SHA-1 hash of |bytes|.
|
||||
sha1 blob unique not null check (length(sha1) = 20),
|
||||
|
||||
-- The width and height in pixels; must match values within
|
||||
-- |sample_entry_bytes|.
|
||||
width integer not null check (width > 0),
|
||||
height integer not null check (height > 0),
|
||||
|
||||
-- The codec in RFC-6381 format, such as "avc1.4d001f".
|
||||
rfc6381_codec text not null,
|
||||
|
||||
-- The serialized box, including the leading length and box type (avcC in
|
||||
-- the case of H.264).
|
||||
data blob not null check (length(data) > 86)
|
||||
);
|
||||
|
||||
create table user (
|
||||
id integer primary key,
|
||||
username unique not null,
|
||||
|
||||
-- Bitwise mask of flags:
|
||||
-- 1: disabled. If set, no method of authentication for this user will succeed.
|
||||
flags integer not null,
|
||||
|
||||
-- If set, a hash for password authentication, as generated by `libpasta::hash_password`.
|
||||
password_hash text,
|
||||
|
||||
-- A counter which increments with every password reset or clear.
|
||||
password_id integer not null default 0,
|
||||
|
||||
-- Updated lazily on database flush; reset when password_id is incremented.
|
||||
-- This could be used to automatically disable the password on hitting a threshold.
|
||||
password_failure_count integer not null default 0,
|
||||
|
||||
-- If set, a Unix UID that is accepted for authentication when using HTTP over
|
||||
-- a Unix domain socket. (Additionally, the UID running Moonfire NVR can authenticate
|
||||
-- as anyone; there's no point in trying to do otherwise.) This might be an easy
|
||||
-- bootstrap method once configuration happens through a web UI rather than text UI.
|
||||
unix_uid integer
|
||||
);
|
||||
|
||||
-- A single session, whether for browser or robot use.
|
||||
-- These map at the HTTP layer to an "s" cookie (exact format described
|
||||
-- elsewhere), which holds the session id and an encrypted sequence number for
|
||||
-- replay protection.
|
||||
create table user_session (
|
||||
-- The session id is a 48-byte blob. This is the unencoded, unsalted Blake2b-192
|
||||
-- (24 bytes) of the unencoded session id. Much like `password_hash`, a
|
||||
-- hash is used here so that a leaked database backup can't be trivially used
|
||||
-- to steal credentials.
|
||||
session_id_hash blob primary key not null,
|
||||
|
||||
user_id integer references user (id) not null,
|
||||
|
||||
-- A 32-byte random number. Used to derive keys for the replay protection
|
||||
-- and CSRF tokens.
|
||||
seed blob not null,
|
||||
|
||||
-- A bitwise mask of flags, currently all properties of the HTTP cookie
|
||||
-- used to hold the session:
|
||||
-- 1: HttpOnly
|
||||
-- 2: Secure
|
||||
-- 4: SameSite=Lax
|
||||
-- 8: SameSite=Strict - 4 must also be set.
|
||||
flags integer not null,
|
||||
|
||||
-- The domain of the HTTP cookie used to store this session. The outbound
|
||||
-- `Set-Cookie` header never specifies a scope, so this matches the `Host:` of
|
||||
-- the inbound HTTP request (minus the :port, if any was specified).
|
||||
domain text,
|
||||
|
||||
-- An editable description which might describe the device/program which uses
|
||||
-- this session, such as "Chromebook", "iPhone", or "motion detection worker".
|
||||
description text,
|
||||
|
||||
creation_password_id integer, -- the id it was created from, if created via password
|
||||
creation_time_sec integer not null, -- sec since epoch
|
||||
creation_user_agent text, -- User-Agent header from inbound HTTP request.
|
||||
creation_peer_addr blob, -- IPv4 or IPv6 address, or null for Unix socket.
|
||||
|
||||
revocation_time_sec integer, -- sec since epoch
|
||||
revocation_user_agent text, -- User-Agent header from inbound HTTP request.
|
||||
revocation_peer_addr blob, -- IPv4 or IPv6 address, or null for Unix socket/no peer.
|
||||
|
||||
-- A value indicating the reason for revocation, with optional additional
|
||||
-- text detail. Enumeration values:
|
||||
-- 0: logout link clicked (i.e. from within the session itself)
|
||||
--
|
||||
-- This might be extended for a variety of other reasons:
|
||||
-- x: user revoked (while authenticated in another way)
|
||||
-- x: password change invalidated all sessions created with that password
|
||||
-- x: expired (due to fixed total time or time inactive)
|
||||
-- x: evicted (due to too many sessions)
|
||||
-- x: suspicious activity
|
||||
revocation_reason integer,
|
||||
revocation_reason_detail text,
|
||||
|
||||
-- Information about requests which used this session, updated lazily on database flush.
|
||||
last_use_time_sec integer, -- sec since epoch
|
||||
last_use_user_agent text, -- User-Agent header from inbound HTTP request.
|
||||
last_use_peer_addr blob, -- IPv4 or IPv6 address, or null for Unix socket.
|
||||
use_count not null default 0
|
||||
) without rowid;
|
||||
|
||||
create index user_session_uid on user_session (user_id);
|
||||
|
||||
insert into version (id, unix_time, notes)
|
||||
values (3, cast(strftime('%s', 'now') as int), 'db creation');
|
||||
196
server/db/upgrade/v3_to_v4.rs
Normal file
196
server/db/upgrade/v3_to_v4.rs
Normal file
@@ -0,0 +1,196 @@
|
||||
// This file is part of Moonfire NVR, a security camera network video recorder.
|
||||
// Copyright (C) 2019 The Moonfire NVR Authors
|
||||
//
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// In addition, as a special exception, the copyright holders give
|
||||
// permission to link the code of portions of this program with the
|
||||
// OpenSSL library under certain conditions as described in each
|
||||
// individual source file, and distribute linked combinations including
|
||||
// the two.
|
||||
//
|
||||
// You must obey the GNU General Public License in all respects for all
|
||||
// of the code used other than OpenSSL. If you modify file(s) with this
|
||||
// exception, you may extend this exception to your version of the
|
||||
// file(s), but you are not obligated to do so. If you do not wish to do
|
||||
// so, delete this exception statement from your version. If you delete
|
||||
// this exception statement from all source files in the program, then
|
||||
// also delete it here.
|
||||
//
|
||||
// This program is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
/// Upgrades a version 3 schema to a version 4 schema.
|
||||
|
||||
use failure::Error;
|
||||
|
||||
pub fn run(_args: &super::Args, tx: &rusqlite::Transaction) -> Result<(), Error> {
|
||||
// These create statements match the schema.sql when version 4 was the latest.
|
||||
tx.execute_batch(r#"
|
||||
alter table meta add column max_signal_changes integer check (max_signal_changes >= 0);
|
||||
|
||||
create table signal (
|
||||
id integer primary key,
|
||||
source_uuid blob not null check (length(source_uuid) = 16),
|
||||
type_uuid blob not null check (length(type_uuid) = 16),
|
||||
short_name not null,
|
||||
unique (source_uuid, type_uuid)
|
||||
);
|
||||
|
||||
create table signal_type_enum (
|
||||
type_uuid blob not null check (length(type_uuid) = 16),
|
||||
value integer not null check (value > 0 and value < 16),
|
||||
name text not null,
|
||||
motion int not null check (motion in (0, 1)) default 0,
|
||||
color text
|
||||
);
|
||||
|
||||
create table signal_camera (
|
||||
signal_id integer references signal (id),
|
||||
camera_id integer references camera (id),
|
||||
type integer not null,
|
||||
primary key (signal_id, camera_id)
|
||||
) without rowid;
|
||||
|
||||
create table signal_change (
|
||||
time_90k integer primary key,
|
||||
changes blob not null
|
||||
);
|
||||
|
||||
alter table user add column permissions blob not null default X'';
|
||||
alter table user_session add column permissions blob not null default X'';
|
||||
|
||||
-- Set permissions to "view_video" on existing users and sessions to preserve their
|
||||
-- behavior. Newly created users won't have prepopulated permissions like this.
|
||||
update user set permissions = X'0801';
|
||||
update user_session set permissions = X'0801';
|
||||
|
||||
alter table camera rename to old_camera;
|
||||
create table camera (
|
||||
id integer primary key,
|
||||
uuid blob unique not null check (length(uuid) = 16),
|
||||
short_name text not null,
|
||||
description text,
|
||||
onvif_host text,
|
||||
username text,
|
||||
password text
|
||||
);
|
||||
insert into camera
|
||||
select
|
||||
id,
|
||||
uuid,
|
||||
short_name,
|
||||
description,
|
||||
host,
|
||||
username,
|
||||
password
|
||||
from
|
||||
old_camera;
|
||||
|
||||
alter table stream rename to old_stream;
|
||||
create table stream (
|
||||
id integer primary key,
|
||||
camera_id integer not null references camera (id),
|
||||
sample_file_dir_id integer references sample_file_dir (id),
|
||||
type text not null check (type in ('main', 'sub')),
|
||||
record integer not null check (record in (1, 0)),
|
||||
rtsp_url text not null,
|
||||
retain_bytes integer not null check (retain_bytes >= 0),
|
||||
flush_if_sec integer not null,
|
||||
next_recording_id integer not null check (next_recording_id >= 0),
|
||||
unique (camera_id, type)
|
||||
);
|
||||
insert into stream
|
||||
select
|
||||
s.id,
|
||||
s.camera_id,
|
||||
s.sample_file_dir_id,
|
||||
s.type,
|
||||
s.record,
|
||||
'rtsp://' || c.onvif_host || s.rtsp_path as rtsp_url,
|
||||
retain_bytes,
|
||||
flush_if_sec,
|
||||
next_recording_id
|
||||
from
|
||||
old_stream s join camera c on (s.camera_id = c.id);
|
||||
|
||||
alter table recording rename to old_recording;
|
||||
create table recording (
|
||||
composite_id integer primary key,
|
||||
open_id integer not null,
|
||||
stream_id integer not null references stream (id),
|
||||
run_offset integer not null,
|
||||
flags integer not null,
|
||||
sample_file_bytes integer not null check (sample_file_bytes > 0),
|
||||
start_time_90k integer not null check (start_time_90k > 0),
|
||||
duration_90k integer not null
|
||||
check (duration_90k >= 0 and duration_90k < 5*60*90000),
|
||||
video_samples integer not null check (video_samples > 0),
|
||||
video_sync_samples integer not null check (video_sync_samples > 0),
|
||||
video_sample_entry_id integer references video_sample_entry (id),
|
||||
check (composite_id >> 32 = stream_id)
|
||||
);
|
||||
insert into recording select
|
||||
composite_id,
|
||||
open_id,
|
||||
stream_id,
|
||||
run_offset,
|
||||
flags,
|
||||
sample_file_bytes,
|
||||
start_time_90k,
|
||||
duration_90k,
|
||||
video_samples,
|
||||
video_sync_samples,
|
||||
video_sample_entry_id
|
||||
from old_recording;
|
||||
drop index recording_cover;
|
||||
create index recording_cover on recording (
|
||||
stream_id,
|
||||
start_time_90k,
|
||||
open_id,
|
||||
duration_90k,
|
||||
video_samples,
|
||||
video_sync_samples,
|
||||
video_sample_entry_id,
|
||||
sample_file_bytes,
|
||||
run_offset,
|
||||
flags
|
||||
);
|
||||
|
||||
alter table recording_integrity rename to old_recording_integrity;
|
||||
create table recording_integrity (
|
||||
composite_id integer primary key references recording (composite_id),
|
||||
local_time_delta_90k integer,
|
||||
local_time_since_open_90k integer,
|
||||
wall_time_delta_90k integer,
|
||||
sample_file_sha1 blob check (length(sample_file_sha1) <= 20)
|
||||
);
|
||||
insert into recording_integrity select * from old_recording_integrity;
|
||||
|
||||
alter table recording_playback rename to old_recording_playback;
|
||||
create table recording_playback (
|
||||
composite_id integer primary key references recording (composite_id),
|
||||
video_index blob not null check (length(video_index) > 0)
|
||||
);
|
||||
insert into recording_playback select * from old_recording_playback;
|
||||
|
||||
drop table old_recording_playback;
|
||||
drop table old_recording_integrity;
|
||||
drop table old_recording;
|
||||
drop table old_stream;
|
||||
drop table old_camera;
|
||||
|
||||
-- This was supposed to be present in version 2, but the upgrade procedure used to miss it.
|
||||
-- Catch up so we know a version 4 database is right.
|
||||
create index if not exists user_session_uid on user_session (user_id);
|
||||
"#)?;
|
||||
Ok(())
|
||||
}
|
||||
162
server/db/upgrade/v4_to_v5.rs
Normal file
162
server/db/upgrade/v4_to_v5.rs
Normal file
@@ -0,0 +1,162 @@
|
||||
// This file is part of Moonfire NVR, a security camera network video recorder.
|
||||
// Copyright (C) 2019 The Moonfire NVR Authors
|
||||
//
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// In addition, as a special exception, the copyright holders give
|
||||
// permission to link the code of portions of this program with the
|
||||
// OpenSSL library under certain conditions as described in each
|
||||
// individual source file, and distribute linked combinations including
|
||||
// the two.
|
||||
//
|
||||
// You must obey the GNU General Public License in all respects for all
|
||||
// of the code used other than OpenSSL. If you modify file(s) with this
|
||||
// exception, you may extend this exception to your version of the
|
||||
// file(s), but you are not obligated to do so. If you do not wish to do
|
||||
// so, delete this exception statement from your version. If you delete
|
||||
// this exception statement from all source files in the program, then
|
||||
// also delete it here.
|
||||
//
|
||||
// This program is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
/// Upgrades a version 4 schema to a version 5 schema.
|
||||
///
|
||||
/// This just handles the directory meta files. If they're already in the new format, great.
|
||||
/// Otherwise, verify they are consistent with the database then upgrade them.
|
||||
|
||||
use crate::db::FromSqlUuid;
|
||||
use crate::{dir, schema};
|
||||
use cstr::cstr;
|
||||
use failure::{Error, Fail, bail};
|
||||
use log::info;
|
||||
use nix::fcntl::{FlockArg, OFlag};
|
||||
use nix::sys::stat::Mode;
|
||||
use protobuf::Message;
|
||||
use rusqlite::params;
|
||||
use std::io::{Read, Write};
|
||||
use std::os::unix::io::AsRawFd;
|
||||
use uuid::Uuid;
|
||||
|
||||
const FIXED_DIR_META_LEN: usize = 512;
|
||||
|
||||
/// Maybe upgrades the `meta` file, returning if an upgrade happened (and thus a sync is needed).
|
||||
fn maybe_upgrade_meta(dir: &dir::Fd, db_meta: &schema::DirMeta) -> Result<bool, Error> {
|
||||
let tmp_path = cstr!("meta.tmp");
|
||||
let meta_path = cstr!("meta");
|
||||
let mut f = crate::fs::openat(dir.as_raw_fd(), meta_path, OFlag::O_RDONLY, Mode::empty())?;
|
||||
let mut data = Vec::new();
|
||||
f.read_to_end(&mut data)?;
|
||||
if data.len() == FIXED_DIR_META_LEN {
|
||||
return Ok(false);
|
||||
}
|
||||
|
||||
let mut s = protobuf::CodedInputStream::from_bytes(&data);
|
||||
let mut dir_meta = schema::DirMeta::new();
|
||||
dir_meta.merge_from(&mut s)
|
||||
.map_err(|e| e.context("Unable to parse metadata proto: {}"))?;
|
||||
if !dir::SampleFileDir::consistent(&db_meta, &dir_meta) {
|
||||
bail!("Inconsistent db_meta={:?} dir_meta={:?}", &db_meta, &dir_meta);
|
||||
}
|
||||
let mut f = crate::fs::openat(dir.as_raw_fd(), tmp_path,
|
||||
OFlag::O_CREAT | OFlag::O_TRUNC | OFlag::O_WRONLY,
|
||||
Mode::S_IRUSR | Mode::S_IWUSR)?;
|
||||
let mut data =
|
||||
dir_meta.write_length_delimited_to_bytes().expect("proto3->vec is infallible");
|
||||
if data.len() > FIXED_DIR_META_LEN {
|
||||
bail!("Length-delimited DirMeta message requires {} bytes, over limit of {}",
|
||||
data.len(), FIXED_DIR_META_LEN);
|
||||
}
|
||||
data.resize(FIXED_DIR_META_LEN, 0); // pad to required length.
|
||||
f.write_all(&data)?;
|
||||
f.sync_all()?;
|
||||
|
||||
nix::fcntl::renameat(Some(dir.as_raw_fd()), tmp_path, Some(dir.as_raw_fd()), meta_path)?;
|
||||
Ok(true)
|
||||
}
|
||||
|
||||
/// Looks for uuid-based filenames and deletes them.
|
||||
///
|
||||
/// The v1->v3 migration failed to remove garbage files prior to 433be217. Let's have a clean slate
|
||||
/// at v5.
|
||||
///
|
||||
/// Returns true if something was done (and thus a sync is needed).
|
||||
fn maybe_cleanup_garbage_uuids(dir: &dir::Fd) -> Result<bool, Error> {
|
||||
let mut need_sync = false;
|
||||
let mut dir2 = nix::dir::Dir::openat(dir.as_raw_fd(), ".",
|
||||
OFlag::O_DIRECTORY | OFlag::O_RDONLY, Mode::empty())?;
|
||||
for e in dir2.iter() {
|
||||
let e = e?;
|
||||
let f = e.file_name();
|
||||
info!("file: {}", f.to_str().unwrap());
|
||||
let f_str = match f.to_str() {
|
||||
Ok(f) => f,
|
||||
Err(_) => continue,
|
||||
};
|
||||
if Uuid::parse_str(f_str).is_ok() {
|
||||
info!("removing leftover garbage file {}", f_str);
|
||||
nix::unistd::unlinkat(Some(dir.as_raw_fd()), f,
|
||||
nix::unistd::UnlinkatFlags::NoRemoveDir)?;
|
||||
need_sync = true;
|
||||
}
|
||||
}
|
||||
|
||||
Ok(need_sync)
|
||||
}
|
||||
|
||||
pub fn run(_args: &super::Args, tx: &rusqlite::Transaction) -> Result<(), Error> {
|
||||
let db_uuid: FromSqlUuid =
|
||||
tx.query_row_and_then(r"select uuid from meta", params![], |row| row.get(0))?;
|
||||
let mut stmt = tx.prepare(r#"
|
||||
select
|
||||
d.path,
|
||||
d.uuid,
|
||||
d.last_complete_open_id,
|
||||
o.uuid
|
||||
from
|
||||
sample_file_dir d
|
||||
left join open o on (d.last_complete_open_id = o.id);
|
||||
"#)?;
|
||||
let mut rows = stmt.query(params![])?;
|
||||
while let Some(row) = rows.next()? {
|
||||
let path = row.get_raw_checked(0)?.as_str()?;
|
||||
info!("path: {}", path);
|
||||
let dir_uuid: FromSqlUuid = row.get(1)?;
|
||||
let open_id: Option<u32> = row.get(2)?;
|
||||
let open_uuid: Option<FromSqlUuid> = row.get(3)?;
|
||||
let mut db_meta = schema::DirMeta::new();
|
||||
db_meta.db_uuid.extend_from_slice(&db_uuid.0.as_bytes()[..]);
|
||||
db_meta.dir_uuid.extend_from_slice(&dir_uuid.0.as_bytes()[..]);
|
||||
match (open_id, open_uuid) {
|
||||
(Some(id), Some(uuid)) => {
|
||||
let mut o = db_meta.last_complete_open.set_default();
|
||||
o.id = id;
|
||||
o.uuid.extend_from_slice(&uuid.0.as_bytes()[..]);
|
||||
},
|
||||
(None, None) => {},
|
||||
_ => bail!("open table missing id"),
|
||||
}
|
||||
|
||||
let dir = dir::Fd::open(path, false)?;
|
||||
dir.lock(FlockArg::LockExclusiveNonblock)?;
|
||||
|
||||
let mut need_sync = maybe_upgrade_meta(&dir, &db_meta)?;
|
||||
if maybe_cleanup_garbage_uuids(&dir)? {
|
||||
need_sync = true;
|
||||
}
|
||||
|
||||
if need_sync {
|
||||
dir.sync()?;
|
||||
}
|
||||
info!("done with path: {}", path);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
497
server/db/upgrade/v5.sql
Normal file
497
server/db/upgrade/v5.sql
Normal file
@@ -0,0 +1,497 @@
|
||||
-- This file is part of Moonfire NVR, a security camera network video recorder.
|
||||
-- Copyright (C) 2016 The Moonfire NVR Authors
|
||||
--
|
||||
-- This program is free software: you can redistribute it and/or modify
|
||||
-- it under the terms of the GNU General Public License as published by
|
||||
-- the Free Software Foundation, either version 3 of the License, or
|
||||
-- (at your option) any later version.
|
||||
--
|
||||
-- In addition, as a special exception, the copyright holders give
|
||||
-- permission to link the code of portions of this program with the
|
||||
-- OpenSSL library under certain conditions as described in each
|
||||
-- individual source file, and distribute linked combinations including
|
||||
-- the two.
|
||||
--
|
||||
-- You must obey the GNU General Public License in all respects for all
|
||||
-- of the code used other than OpenSSL. If you modify file(s) with this
|
||||
-- exception, you may extend this exception to your version of the
|
||||
-- file(s), but you are not obligated to do so. If you do not wish to do
|
||||
-- so, delete this exception statement from your version. If you delete
|
||||
-- this exception statement from all source files in the program, then
|
||||
-- also delete it here.
|
||||
--
|
||||
-- This program is distributed in the hope that it will be useful,
|
||||
-- but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
-- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
-- GNU General Public License for more details.
|
||||
--
|
||||
-- You should have received a copy of the GNU General Public License
|
||||
-- along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
--
|
||||
-- schema.sql: SQLite3 database schema for Moonfire NVR.
|
||||
-- See also design/schema.md.
|
||||
|
||||
-- Database metadata. There should be exactly one row in this table.
|
||||
create table meta (
|
||||
uuid blob not null check (length(uuid) = 16),
|
||||
|
||||
-- The maximum number of entries in the signal_state table. If an update
|
||||
-- causes this to be exceeded, older times will be garbage collected to stay
|
||||
-- within the limit.
|
||||
max_signal_changes integer check (max_signal_changes >= 0)
|
||||
);
|
||||
|
||||
-- This table tracks the schema version.
|
||||
-- There is one row for the initial database creation (inserted below, after the
|
||||
-- create statements) and one for each upgrade procedure (if any).
|
||||
create table version (
|
||||
id integer primary key,
|
||||
|
||||
-- The unix time as of the creation/upgrade, as determined by
|
||||
-- cast(strftime('%s', 'now') as int).
|
||||
unix_time integer not null,
|
||||
|
||||
-- Optional notes on the creation/upgrade; could include the binary version.
|
||||
notes text
|
||||
);
|
||||
|
||||
-- Tracks every time the database has been opened in read/write mode.
|
||||
-- This is used to ensure directories are in sync with the database (see
|
||||
-- schema.proto:DirMeta), to disambiguate uncommitted recordings, and
|
||||
-- potentially to understand time problems.
|
||||
create table open (
|
||||
id integer primary key,
|
||||
uuid blob unique not null check (length(uuid) = 16),
|
||||
|
||||
-- Information about when / how long the database was open. These may be all
|
||||
-- null, for example in the open that represents all information written
|
||||
-- prior to database version 3.
|
||||
|
||||
-- System time when the database was opened, in 90 kHz units since
|
||||
-- 1970-01-01 00:00:00Z excluding leap seconds.
|
||||
start_time_90k integer,
|
||||
|
||||
-- System time when the database was closed or (on crash) last flushed.
|
||||
end_time_90k integer,
|
||||
|
||||
-- How long the database was open. This is end_time_90k - start_time_90k if
|
||||
-- there were no time steps or leap seconds during this time.
|
||||
duration_90k integer
|
||||
);
|
||||
|
||||
create table sample_file_dir (
|
||||
id integer primary key,
|
||||
path text unique not null,
|
||||
uuid blob unique not null check (length(uuid) = 16),
|
||||
|
||||
-- The last (read/write) open of this directory which fully completed.
|
||||
-- See schema.proto:DirMeta for a more complete description.
|
||||
last_complete_open_id integer references open (id)
|
||||
);
|
||||
|
||||
create table camera (
|
||||
id integer primary key,
|
||||
uuid blob unique not null check (length(uuid) = 16),
|
||||
|
||||
-- A short name of the camera, used in log messages.
|
||||
short_name text not null,
|
||||
|
||||
-- A short description of the camera.
|
||||
description text,
|
||||
|
||||
-- The host part of the http:// URL when accessing ONVIF, optionally
|
||||
-- including ":<port>". Eg with ONVIF host "192.168.1.110:85", the full URL
|
||||
-- of the devie management service will be
|
||||
-- "http://192.168.1.110:85/device_service".
|
||||
onvif_host text,
|
||||
|
||||
-- The username to use when accessing the camera.
|
||||
-- If empty, no username or password will be supplied.
|
||||
username text,
|
||||
|
||||
-- The password to use when accessing the camera.
|
||||
password text
|
||||
);
|
||||
|
||||
create table stream (
|
||||
id integer primary key,
|
||||
camera_id integer not null references camera (id),
|
||||
sample_file_dir_id integer references sample_file_dir (id),
|
||||
type text not null check (type in ('main', 'sub')),
|
||||
|
||||
-- If record is true, the stream should start recording when moonfire
|
||||
-- starts. If false, no new recordings will be made, but old recordings
|
||||
-- will not be deleted.
|
||||
record integer not null check (record in (1, 0)),
|
||||
|
||||
-- The rtsp:// URL to use for this stream, excluding username and password.
|
||||
-- (Those are taken from the camera row's respective fields.)
|
||||
rtsp_url text not null,
|
||||
|
||||
-- The number of bytes of video to retain, excluding the currently-recording
|
||||
-- file. Older files will be deleted as necessary to stay within this limit.
|
||||
retain_bytes integer not null check (retain_bytes >= 0),
|
||||
|
||||
-- Flush the database when the first instant of completed recording is this
|
||||
-- many seconds old. A value of 0 means that every completed recording will
|
||||
-- cause an immediate flush. Higher values may allow flushes to be combined,
|
||||
-- reducing SSD write cycles. For example, if all streams have a flush_if_sec
|
||||
-- >= x sec, there will be:
|
||||
--
|
||||
-- * at most one flush per x sec in total
|
||||
-- * at most x sec of completed but unflushed recordings per stream.
|
||||
-- * at most x completed but unflushed recordings per stream, in the worst
|
||||
-- case where a recording instantly fails, waits the 1-second retry delay,
|
||||
-- then fails again, forever.
|
||||
flush_if_sec integer not null,
|
||||
|
||||
-- The low 32 bits of the next recording id to assign for this stream.
|
||||
-- Typically this is the maximum current recording + 1, but it does
|
||||
-- not decrease if that recording is deleted.
|
||||
next_recording_id integer not null check (next_recording_id >= 0),
|
||||
|
||||
unique (camera_id, type)
|
||||
);
|
||||
|
||||
-- Each row represents a single completed recorded segment of video.
|
||||
-- Recordings are typically ~60 seconds; never more than 5 minutes.
|
||||
create table recording (
|
||||
-- The high 32 bits of composite_id are taken from the stream's id, which
|
||||
-- improves locality. The low 32 bits are taken from the stream's
|
||||
-- next_recording_id (which should be post-incremented in the same
|
||||
-- transaction). It'd be simpler to use a "without rowid" table and separate
|
||||
-- fields to make up the primary key, but
|
||||
-- <https://www.sqlite.org/withoutrowid.html> points out that "without rowid"
|
||||
-- is not appropriate when the average row size is in excess of 50 bytes.
|
||||
-- recording_cover rows (which match this id format) are typically 1--5 KiB.
|
||||
composite_id integer primary key,
|
||||
|
||||
-- The open in which this was committed to the database. For a given
|
||||
-- composite_id, only one recording will ever be committed to the database,
|
||||
-- but in-memory state may reflect a recording which never gets committed.
|
||||
-- This field allows disambiguation in etags and such.
|
||||
open_id integer not null references open (id),
|
||||
|
||||
-- This field is redundant with id above, but used to enforce the reference
|
||||
-- constraint and to structure the recording_start_time index.
|
||||
stream_id integer not null references stream (id),
|
||||
|
||||
-- The offset of this recording within a run. 0 means this was the first
|
||||
-- recording made from a RTSP session. The start of the run has id
|
||||
-- (id-run_offset).
|
||||
run_offset integer not null,
|
||||
|
||||
-- flags is a bitmask:
|
||||
--
|
||||
-- * 1, or "trailing zero", indicates that this recording is the last in a
|
||||
-- stream. As the duration of a sample is not known until the next sample
|
||||
-- is received, the final sample in this recording will have duration 0.
|
||||
flags integer not null,
|
||||
|
||||
sample_file_bytes integer not null check (sample_file_bytes > 0),
|
||||
|
||||
-- The starting time of the recording, in 90 kHz units since
|
||||
-- 1970-01-01 00:00:00 UTC excluding leap seconds. Currently on initial
|
||||
-- connection, this is taken from the local system time; on subsequent
|
||||
-- recordings, it exactly matches the previous recording's end time.
|
||||
start_time_90k integer not null check (start_time_90k > 0),
|
||||
|
||||
-- The duration of the recording, in 90 kHz units.
|
||||
duration_90k integer not null
|
||||
check (duration_90k >= 0 and duration_90k < 5*60*90000),
|
||||
|
||||
video_samples integer not null check (video_samples > 0),
|
||||
video_sync_samples integer not null check (video_sync_samples > 0),
|
||||
video_sample_entry_id integer references video_sample_entry (id),
|
||||
|
||||
check (composite_id >> 32 = stream_id)
|
||||
);
|
||||
|
||||
create index recording_cover on recording (
|
||||
-- Typical queries use "where stream_id = ? order by start_time_90k".
|
||||
stream_id,
|
||||
start_time_90k,
|
||||
|
||||
-- These fields are not used for ordering; they cover most queries so
|
||||
-- that only database verification and actual viewing of recordings need
|
||||
-- to consult the underlying row.
|
||||
open_id,
|
||||
duration_90k,
|
||||
video_samples,
|
||||
video_sync_samples,
|
||||
video_sample_entry_id,
|
||||
sample_file_bytes,
|
||||
run_offset,
|
||||
flags
|
||||
);
|
||||
|
||||
-- Fields which are only needed to check/correct database integrity problems
|
||||
-- (such as incorrect timestamps).
|
||||
create table recording_integrity (
|
||||
-- See description on recording table.
|
||||
composite_id integer primary key references recording (composite_id),
|
||||
|
||||
-- The number of 90 kHz units the local system's monotonic clock has
|
||||
-- advanced more than the stated duration of recordings in a run since the
|
||||
-- first recording ended. Negative numbers indicate the local system time is
|
||||
-- behind the recording.
|
||||
--
|
||||
-- The first recording of a run (that is, one with run_offset=0) has null
|
||||
-- local_time_delta_90k because errors are assumed to
|
||||
-- be the result of initial buffering rather than frequency mismatch.
|
||||
--
|
||||
-- This value should be near 0 even on long runs in which the camera's clock
|
||||
-- and local system's clock frequency differ because each recording's delta
|
||||
-- is used to correct the durations of the next (up to 500 ppm error).
|
||||
local_time_delta_90k integer,
|
||||
|
||||
-- The number of 90 kHz units the local system's monotonic clock had
|
||||
-- advanced since the database was opened, as of the start of recording.
|
||||
-- TODO: fill this in!
|
||||
local_time_since_open_90k integer,
|
||||
|
||||
-- The difference between start_time_90k+duration_90k and a wall clock
|
||||
-- timestamp captured at end of this recording. This is meaningful for all
|
||||
-- recordings in a run, even the initial one (run_offset=0), because
|
||||
-- start_time_90k is derived from the wall time as of when recording
|
||||
-- starts, not when it ends.
|
||||
-- TODO: fill this in!
|
||||
wall_time_delta_90k integer,
|
||||
|
||||
-- The sha1 hash of the contents of the sample file.
|
||||
sample_file_sha1 blob check (length(sample_file_sha1) <= 20)
|
||||
);
|
||||
|
||||
-- Large fields for a recording which are needed ony for playback.
|
||||
-- In particular, when serving a byte range within a .mp4 file, the
|
||||
-- recording_playback row is needed for the recording(s) corresponding to that
|
||||
-- particular byte range, needed, but the recording rows suffice for all other
|
||||
-- recordings in the .mp4.
|
||||
create table recording_playback (
|
||||
-- See description on recording table.
|
||||
composite_id integer primary key references recording (composite_id),
|
||||
|
||||
-- See design/schema.md#video_index for a description of this field.
|
||||
video_index blob not null check (length(video_index) > 0)
|
||||
|
||||
-- audio_index could be added here in the future.
|
||||
);
|
||||
|
||||
-- Files which are to be deleted (may or may not still exist).
|
||||
-- Note that besides these files, for each stream, any recordings >= its
|
||||
-- next_recording_id should be discarded on startup.
|
||||
create table garbage (
|
||||
-- This is _mostly_ redundant with composite_id, which contains the stream
|
||||
-- id and thus a linkage to the sample file directory. Listing it here
|
||||
-- explicitly means that streams can be deleted without losing the
|
||||
-- association of garbage to directory.
|
||||
sample_file_dir_id integer not null references sample_file_dir (id),
|
||||
|
||||
-- See description on recording table.
|
||||
composite_id integer not null,
|
||||
|
||||
-- Organize the table first by directory, as that's how it will be queried.
|
||||
primary key (sample_file_dir_id, composite_id)
|
||||
) without rowid;
|
||||
|
||||
-- A concrete box derived from a ISO/IEC 14496-12 section 8.5.2
|
||||
-- VisualSampleEntry box. Describes the codec, width, height, etc.
|
||||
create table video_sample_entry (
|
||||
id integer primary key,
|
||||
|
||||
-- A SHA-1 hash of |bytes|.
|
||||
sha1 blob unique not null check (length(sha1) = 20),
|
||||
|
||||
-- The width and height in pixels; must match values within
|
||||
-- |sample_entry_bytes|.
|
||||
width integer not null check (width > 0),
|
||||
height integer not null check (height > 0),
|
||||
|
||||
-- The codec in RFC-6381 format, such as "avc1.4d001f".
|
||||
rfc6381_codec text not null,
|
||||
|
||||
-- The serialized box, including the leading length and box type (avcC in
|
||||
-- the case of H.264).
|
||||
data blob not null check (length(data) > 86)
|
||||
);
|
||||
|
||||
create table user (
|
||||
id integer primary key,
|
||||
username unique not null,
|
||||
|
||||
-- Bitwise mask of flags:
|
||||
-- 1: disabled. If set, no method of authentication for this user will succeed.
|
||||
flags integer not null,
|
||||
|
||||
-- If set, a hash for password authentication, as generated by `libpasta::hash_password`.
|
||||
password_hash text,
|
||||
|
||||
-- A counter which increments with every password reset or clear.
|
||||
password_id integer not null default 0,
|
||||
|
||||
-- Updated lazily on database flush; reset when password_id is incremented.
|
||||
-- This could be used to automatically disable the password on hitting a threshold.
|
||||
password_failure_count integer not null default 0,
|
||||
|
||||
-- If set, a Unix UID that is accepted for authentication when using HTTP over
|
||||
-- a Unix domain socket. (Additionally, the UID running Moonfire NVR can authenticate
|
||||
-- as anyone; there's no point in trying to do otherwise.) This might be an easy
|
||||
-- bootstrap method once configuration happens through a web UI rather than text UI.
|
||||
unix_uid integer,
|
||||
|
||||
-- Permissions available for newly created tokens or when authenticating via
|
||||
-- unix_uid above. A serialized "Permissions" protobuf.
|
||||
permissions blob not null default X''
|
||||
);
|
||||
|
||||
-- A single session, whether for browser or robot use.
|
||||
-- These map at the HTTP layer to an "s" cookie (exact format described
|
||||
-- elsewhere), which holds the session id and an encrypted sequence number for
|
||||
-- replay protection.
|
||||
create table user_session (
|
||||
-- The session id is a 48-byte blob. This is the unencoded, unsalted Blake2b-192
|
||||
-- (24 bytes) of the unencoded session id. Much like `password_hash`, a
|
||||
-- hash is used here so that a leaked database backup can't be trivially used
|
||||
-- to steal credentials.
|
||||
session_id_hash blob primary key not null,
|
||||
|
||||
user_id integer references user (id) not null,
|
||||
|
||||
-- A 32-byte random number. Used to derive keys for the replay protection
|
||||
-- and CSRF tokens.
|
||||
seed blob not null,
|
||||
|
||||
-- A bitwise mask of flags, currently all properties of the HTTP cookie
|
||||
-- used to hold the session:
|
||||
-- 1: HttpOnly
|
||||
-- 2: Secure
|
||||
-- 4: SameSite=Lax
|
||||
-- 8: SameSite=Strict - 4 must also be set.
|
||||
flags integer not null,
|
||||
|
||||
-- The domain of the HTTP cookie used to store this session. The outbound
|
||||
-- `Set-Cookie` header never specifies a scope, so this matches the `Host:` of
|
||||
-- the inbound HTTP request (minus the :port, if any was specified).
|
||||
domain text,
|
||||
|
||||
-- An editable description which might describe the device/program which uses
|
||||
-- this session, such as "Chromebook", "iPhone", or "motion detection worker".
|
||||
description text,
|
||||
|
||||
creation_password_id integer, -- the id it was created from, if created via password
|
||||
creation_time_sec integer not null, -- sec since epoch
|
||||
creation_user_agent text, -- User-Agent header from inbound HTTP request.
|
||||
creation_peer_addr blob, -- IPv4 or IPv6 address, or null for Unix socket.
|
||||
|
||||
revocation_time_sec integer, -- sec since epoch
|
||||
revocation_user_agent text, -- User-Agent header from inbound HTTP request.
|
||||
revocation_peer_addr blob, -- IPv4 or IPv6 address, or null for Unix socket/no peer.
|
||||
|
||||
-- A value indicating the reason for revocation, with optional additional
|
||||
-- text detail. Enumeration values:
|
||||
-- 0: logout link clicked (i.e. from within the session itself)
|
||||
--
|
||||
-- This might be extended for a variety of other reasons:
|
||||
-- x: user revoked (while authenticated in another way)
|
||||
-- x: password change invalidated all sessions created with that password
|
||||
-- x: expired (due to fixed total time or time inactive)
|
||||
-- x: evicted (due to too many sessions)
|
||||
-- x: suspicious activity
|
||||
revocation_reason integer,
|
||||
revocation_reason_detail text,
|
||||
|
||||
-- Information about requests which used this session, updated lazily on database flush.
|
||||
last_use_time_sec integer, -- sec since epoch
|
||||
last_use_user_agent text, -- User-Agent header from inbound HTTP request.
|
||||
last_use_peer_addr blob, -- IPv4 or IPv6 address, or null for Unix socket.
|
||||
use_count not null default 0,
|
||||
|
||||
-- Permissions associated with this token; a serialized "Permissions" protobuf.
|
||||
permissions blob not null default X''
|
||||
) without rowid;
|
||||
|
||||
create index user_session_uid on user_session (user_id);
|
||||
|
||||
create table signal (
|
||||
id integer primary key,
|
||||
|
||||
-- a uuid describing the originating object, such as the uuid of the camera
|
||||
-- for built-in motion detection. There will be a JSON interface for adding
|
||||
-- events; it will require this UUID to be supplied. An external uuid might
|
||||
-- indicate "my house security system's zone 23".
|
||||
source_uuid blob not null check (length(source_uuid) = 16),
|
||||
|
||||
-- a uuid describing the type of event. A registry (TBD) will list built-in
|
||||
-- supported types, such as "Hikvision on-camera motion detection", or
|
||||
-- "ONVIF on-camera motion detection". External programs can use their own
|
||||
-- uuids, such as "Elk security system watcher".
|
||||
type_uuid blob not null check (length(type_uuid) = 16),
|
||||
|
||||
-- a short human-readable description of the event to use in mouseovers or event
|
||||
-- lists, such as "driveway motion" or "front door open".
|
||||
short_name not null,
|
||||
|
||||
unique (source_uuid, type_uuid)
|
||||
);
|
||||
|
||||
-- e.g. "moving/still", "disarmed/away/stay", etc.
|
||||
-- TODO: just do a protobuf for each type? might be simpler, more flexible.
|
||||
create table signal_type_enum (
|
||||
type_uuid blob not null check (length(type_uuid) = 16),
|
||||
value integer not null check (value > 0 and value < 16),
|
||||
name text not null,
|
||||
|
||||
-- true/1 iff this signal value should be considered "motion" for directly associated cameras.
|
||||
motion int not null check (motion in (0, 1)) default 0,
|
||||
|
||||
color text
|
||||
);
|
||||
|
||||
-- Associations between event sources and cameras.
|
||||
-- For example, if two cameras have overlapping fields of view, they might be
|
||||
-- configured such that each camera is associated with both its own motion and
|
||||
-- the other camera's motion.
|
||||
create table signal_camera (
|
||||
signal_id integer references signal (id),
|
||||
camera_id integer references camera (id),
|
||||
|
||||
-- type:
|
||||
--
|
||||
-- 0 means direct association, as if the event source if the camera's own
|
||||
-- motion detection. Here are a couple ways this could be used:
|
||||
--
|
||||
-- * when viewing the camera, hotkeys to go to the start of the next or
|
||||
-- previous event should respect this event.
|
||||
-- * a list of events might include the recordings associated with the
|
||||
-- camera in the same timespan.
|
||||
--
|
||||
-- 1 means indirect association. A screen associated with the camera should
|
||||
-- given some indication of this event, but there should be no assumption
|
||||
-- that the camera will have a direct view of the event. For example, all
|
||||
-- cameras might be indirectly associated with a doorknob press. Cameras at
|
||||
-- the back of the house shouldn't be expected to have a direct view of this
|
||||
-- event, but motion events shortly afterward might warrant extra scrutiny.
|
||||
type integer not null,
|
||||
|
||||
primary key (signal_id, camera_id)
|
||||
) without rowid;
|
||||
|
||||
-- Changes to signals as of a given timestamp.
|
||||
create table signal_change (
|
||||
-- Event time, in 90 kHz units since 1970-01-01 00:00:00Z excluding leap seconds.
|
||||
time_90k integer primary key,
|
||||
|
||||
-- Changes at this timestamp.
|
||||
--
|
||||
-- A blob of varints representing a list of
|
||||
-- (signal number - next allowed, state) pairs, where signal number is
|
||||
-- non-decreasing. For example,
|
||||
-- input signals: 1 3 200 (must be sorted)
|
||||
-- delta: 1 1 196 (must be non-negative)
|
||||
-- states: 1 1 2
|
||||
-- varint: \x01 \x01 \x01 \x01 \xc4 \x01 \x02
|
||||
changes blob not null
|
||||
);
|
||||
|
||||
insert into version (id, unix_time, notes)
|
||||
values (5, cast(strftime('%s', 'now') as int), 'db creation');
|
||||
333
server/db/upgrade/v5_to_v6.rs
Normal file
333
server/db/upgrade/v5_to_v6.rs
Normal file
@@ -0,0 +1,333 @@
|
||||
// This file is part of Moonfire NVR, a security camera network video recorder.
|
||||
// Copyright (C) 2020 The Moonfire NVR Authors
|
||||
//
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// In addition, as a special exception, the copyright holders give
|
||||
// permission to link the code of portions of this program with the
|
||||
// OpenSSL library under certain conditions as described in each
|
||||
// individual source file, and distribute linked combinations including
|
||||
// the two.
|
||||
//
|
||||
// You must obey the GNU General Public License in all respects for all
|
||||
// of the code used other than OpenSSL. If you modify file(s) with this
|
||||
// exception, you may extend this exception to your version of the
|
||||
// file(s), but you are not obligated to do so. If you do not wish to do
|
||||
// so, delete this exception statement from your version. If you delete
|
||||
// this exception statement from all source files in the program, then
|
||||
// also delete it here.
|
||||
//
|
||||
// This program is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
/// Upgrades a version 4 schema to a version 5 schema.
|
||||
|
||||
use byteorder::{BigEndian, ByteOrder, WriteBytesExt};
|
||||
use failure::{Error, ResultExt, bail, format_err};
|
||||
use h264_reader::avcc::AvcDecoderConfigurationRecord;
|
||||
use rusqlite::{named_params, params};
|
||||
use std::convert::{TryFrom, TryInto};
|
||||
|
||||
// Copied from src/h264.rs. h264 stuff really doesn't belong in the db crate, but we do what we
|
||||
// must for schema upgrades.
|
||||
const PIXEL_ASPECT_RATIOS: [((u16, u16), (u16, u16)); 4] = [
|
||||
((320, 240), ( 4, 3)),
|
||||
((352, 240), (40, 33)),
|
||||
((640, 480), ( 4, 3)),
|
||||
((704, 480), (40, 33)),
|
||||
];
|
||||
fn default_pixel_aspect_ratio(width: u16, height: u16) -> (u16, u16) {
|
||||
let dims = (width, height);
|
||||
for r in &PIXEL_ASPECT_RATIOS {
|
||||
if r.0 == dims {
|
||||
return r.1;
|
||||
}
|
||||
}
|
||||
(1, 1)
|
||||
}
|
||||
|
||||
fn parse(data: &[u8]) -> Result<AvcDecoderConfigurationRecord, Error> {
|
||||
if data.len() < 94 || &data[4..8] != b"avc1" || &data[90..94] != b"avcC" {
|
||||
bail!("data of len {} doesn't have an avcC", data.len());
|
||||
}
|
||||
let avcc_len = BigEndian::read_u32(&data[86..90]);
|
||||
if avcc_len < 8 { // length and type.
|
||||
bail!("invalid avcc len {}", avcc_len);
|
||||
}
|
||||
let end_pos = 86 + usize::try_from(avcc_len)?;
|
||||
if end_pos != data.len() {
|
||||
bail!("expected avcC to be end of extradata; there are {} more bytes.",
|
||||
data.len() - end_pos);
|
||||
}
|
||||
AvcDecoderConfigurationRecord::try_from(&data[94..end_pos])
|
||||
.map_err(|e| format_err!("Bad AvcDecoderConfigurationRecord: {:?}", e))
|
||||
}
|
||||
|
||||
pub fn run(_args: &super::Args, tx: &rusqlite::Transaction) -> Result<(), Error> {
|
||||
// These create statements match the schema.sql when version 5 was the latest.
|
||||
tx.execute_batch(r#"
|
||||
alter table video_sample_entry rename to old_video_sample_entry;
|
||||
|
||||
create table video_sample_entry (
|
||||
id integer primary key,
|
||||
width integer not null check (width > 0),
|
||||
height integer not null check (height > 0),
|
||||
rfc6381_codec text not null,
|
||||
data blob not null check (length(data) > 86),
|
||||
pasp_h_spacing integer not null default 1 check (pasp_h_spacing > 0),
|
||||
pasp_v_spacing integer not null default 1 check (pasp_v_spacing > 0)
|
||||
);
|
||||
"#)?;
|
||||
|
||||
let mut insert = tx.prepare(r#"
|
||||
insert into video_sample_entry (id, width, height, rfc6381_codec, data,
|
||||
pasp_h_spacing, pasp_v_spacing)
|
||||
values (:id, :width, :height, :rfc6381_codec, :data,
|
||||
:pasp_h_spacing, :pasp_v_spacing)
|
||||
"#)?;
|
||||
|
||||
// Only insert still-referenced video sample entries. I've had problems with
|
||||
// no-longer-referenced ones (perhaps from some ancient, buggy version of Moonfire NVR) for
|
||||
// which avcc.create_context(()) fails.
|
||||
let mut stmt = tx.prepare(r#"
|
||||
select
|
||||
id,
|
||||
width,
|
||||
height,
|
||||
rfc6381_codec,
|
||||
data
|
||||
from
|
||||
old_video_sample_entry v
|
||||
where
|
||||
exists (
|
||||
select
|
||||
'x'
|
||||
from
|
||||
recording r
|
||||
where
|
||||
r.video_sample_entry_id = v.id)
|
||||
"#)?;
|
||||
let mut rows = stmt.query(params![])?;
|
||||
while let Some(row) = rows.next()? {
|
||||
let id: i32 = row.get(0)?;
|
||||
let width: u16 = row.get::<_, i32>(1)?.try_into()?;
|
||||
let height: u16 = row.get::<_, i32>(2)?.try_into()?;
|
||||
let rfc6381_codec: &str = row.get_raw_checked(3)?.as_str()?;
|
||||
let mut data: Vec<u8> = row.get(4)?;
|
||||
let avcc = parse(&data)?;
|
||||
if avcc.num_of_sequence_parameter_sets() != 1 {
|
||||
bail!("Multiple SPSs!");
|
||||
}
|
||||
let ctx = avcc.create_context(())
|
||||
.map_err(|e| format_err!("Can't load SPS+PPS for video_sample_entry_id {}: {:?}",
|
||||
id, e))?;
|
||||
let sps = ctx.sps_by_id(h264_reader::nal::pps::ParamSetId::from_u32(0).unwrap())
|
||||
.ok_or_else(|| format_err!("No SPS 0 for video_sample_entry_id {}", id))?;
|
||||
let pasp = sps.vui_parameters.as_ref()
|
||||
.and_then(|v| v.aspect_ratio_info.as_ref())
|
||||
.and_then(|a| a.clone().get())
|
||||
.unwrap_or_else(|| default_pixel_aspect_ratio(width, height));
|
||||
if pasp != (1, 1) {
|
||||
data.extend_from_slice(b"\x00\x00\x00\x10pasp"); // length + box name
|
||||
data.write_u32::<BigEndian>(pasp.0.into())?;
|
||||
data.write_u32::<BigEndian>(pasp.1.into())?;
|
||||
let len = data.len();
|
||||
BigEndian::write_u32(&mut data[0..4], u32::try_from(len)?);
|
||||
}
|
||||
|
||||
insert.execute_named(named_params!{
|
||||
":id": id,
|
||||
":width": width,
|
||||
":height": height,
|
||||
":rfc6381_codec": rfc6381_codec,
|
||||
":data": &data,
|
||||
":pasp_h_spacing": pasp.0,
|
||||
":pasp_v_spacing": pasp.1,
|
||||
})?;
|
||||
}
|
||||
tx.execute_batch(r#"
|
||||
alter table stream rename to old_stream;
|
||||
create table stream (
|
||||
id integer primary key,
|
||||
camera_id integer not null references camera (id),
|
||||
sample_file_dir_id integer references sample_file_dir (id),
|
||||
type text not null check (type in ('main', 'sub')),
|
||||
record integer not null check (record in (1, 0)),
|
||||
rtsp_url text not null,
|
||||
retain_bytes integer not null check (retain_bytes >= 0),
|
||||
flush_if_sec integer not null,
|
||||
cum_recordings integer not null check (cum_recordings >= 0),
|
||||
cum_media_duration_90k integer not null check (cum_media_duration_90k >= 0),
|
||||
cum_runs integer not null check (cum_runs >= 0),
|
||||
unique (camera_id, type)
|
||||
);
|
||||
insert into stream
|
||||
select
|
||||
s.id,
|
||||
s.camera_id,
|
||||
s.sample_file_dir_id,
|
||||
s.type,
|
||||
s.record,
|
||||
s.rtsp_url,
|
||||
s.retain_bytes,
|
||||
s.flush_if_sec,
|
||||
s.next_recording_id as cum_recordings,
|
||||
coalesce(sum(r.duration_90k), 0) as cum_duration_90k,
|
||||
coalesce(sum(case when r.run_offset = 0 then 1 else 0 end), 0) as cum_runs
|
||||
from
|
||||
old_stream s left join recording r on (s.id = r.stream_id)
|
||||
group by 1;
|
||||
|
||||
alter table recording rename to old_recording;
|
||||
create table recording (
|
||||
composite_id integer primary key,
|
||||
open_id integer not null,
|
||||
stream_id integer not null references stream (id),
|
||||
run_offset integer not null,
|
||||
flags integer not null,
|
||||
sample_file_bytes integer not null check (sample_file_bytes > 0),
|
||||
start_time_90k integer not null check (start_time_90k > 0),
|
||||
prev_media_duration_90k integer not null check (prev_media_duration_90k >= 0),
|
||||
prev_runs integer not null check (prev_runs >= 0),
|
||||
wall_duration_90k integer not null
|
||||
check (wall_duration_90k >= 0 and wall_duration_90k < 5*60*90000),
|
||||
media_duration_delta_90k integer not null,
|
||||
video_samples integer not null check (video_samples > 0),
|
||||
video_sync_samples integer not null check (video_sync_samples > 0),
|
||||
video_sample_entry_id integer references video_sample_entry (id),
|
||||
check (composite_id >> 32 = stream_id)
|
||||
);
|
||||
"#)?;
|
||||
|
||||
// SQLite added window functions in 3.25.0. macOS still ships SQLite 3.24.0 (no support).
|
||||
// Compute cumulative columns by hand.
|
||||
let mut cur_stream_id = None;
|
||||
let mut cum_duration_90k = 0;
|
||||
let mut cum_runs = 0;
|
||||
let mut stmt = tx.prepare(r#"
|
||||
select
|
||||
composite_id,
|
||||
open_id,
|
||||
stream_id,
|
||||
run_offset,
|
||||
flags,
|
||||
sample_file_bytes,
|
||||
start_time_90k,
|
||||
duration_90k,
|
||||
video_samples,
|
||||
video_sync_samples,
|
||||
video_sample_entry_id
|
||||
from
|
||||
old_recording
|
||||
order by composite_id
|
||||
"#)?;
|
||||
let mut insert = tx.prepare(r#"
|
||||
insert into recording (composite_id, open_id, stream_id, run_offset, flags,
|
||||
sample_file_bytes, start_time_90k, prev_media_duration_90k,
|
||||
prev_runs, wall_duration_90k, media_duration_delta_90k,
|
||||
video_samples, video_sync_samples, video_sample_entry_id)
|
||||
values (:composite_id, :open_id, :stream_id, :run_offset, :flags,
|
||||
:sample_file_bytes, :start_time_90k, :prev_media_duration_90k,
|
||||
:prev_runs, :wall_duration_90k, 0, :video_samples,
|
||||
:video_sync_samples, :video_sample_entry_id)
|
||||
"#)?;
|
||||
let mut rows = stmt.query(params![])?;
|
||||
while let Some(row) = rows.next()? {
|
||||
let composite_id: i64 = row.get(0)?;
|
||||
let open_id: i32 = row.get(1)?;
|
||||
let stream_id: i32 = row.get(2)?;
|
||||
let run_offset: i32 = row.get(3)?;
|
||||
let flags: i32 = row.get(4)?;
|
||||
let sample_file_bytes: i32 = row.get(5)?;
|
||||
let start_time_90k: i64 = row.get(6)?;
|
||||
let wall_duration_90k: i32 = row.get(7)?;
|
||||
let video_samples: i32 = row.get(8)?;
|
||||
let video_sync_samples: i32 = row.get(9)?;
|
||||
let video_sample_entry_id: i32 = row.get(10)?;
|
||||
if cur_stream_id != Some(stream_id) {
|
||||
cum_duration_90k = 0;
|
||||
cum_runs = 0;
|
||||
cur_stream_id = Some(stream_id);
|
||||
}
|
||||
insert.execute_named(named_params!{
|
||||
":composite_id": composite_id,
|
||||
":open_id": open_id,
|
||||
":stream_id": stream_id,
|
||||
":run_offset": run_offset,
|
||||
":flags": flags,
|
||||
":sample_file_bytes": sample_file_bytes,
|
||||
":start_time_90k": start_time_90k,
|
||||
":prev_media_duration_90k": cum_duration_90k,
|
||||
":prev_runs": cum_runs,
|
||||
":wall_duration_90k": wall_duration_90k,
|
||||
":video_samples": video_samples,
|
||||
":video_sync_samples": video_sync_samples,
|
||||
":video_sample_entry_id": video_sample_entry_id,
|
||||
}).with_context(|_| format!("Unable to insert composite_id {}", composite_id))?;
|
||||
cum_duration_90k += i64::from(wall_duration_90k);
|
||||
cum_runs += if run_offset == 0 { 1 } else { 0 };
|
||||
}
|
||||
tx.execute_batch(r#"
|
||||
drop index recording_cover;
|
||||
create index recording_cover on recording (
|
||||
stream_id,
|
||||
start_time_90k,
|
||||
open_id,
|
||||
wall_duration_90k,
|
||||
media_duration_delta_90k,
|
||||
video_samples,
|
||||
video_sync_samples,
|
||||
video_sample_entry_id,
|
||||
sample_file_bytes,
|
||||
run_offset,
|
||||
flags
|
||||
);
|
||||
|
||||
alter table recording_integrity rename to old_recording_integrity;
|
||||
create table recording_integrity (
|
||||
composite_id integer primary key references recording (composite_id),
|
||||
local_time_delta_90k integer,
|
||||
local_time_since_open_90k integer,
|
||||
wall_time_delta_90k integer,
|
||||
sample_file_blake3 blob check (length(sample_file_blake3) <= 32)
|
||||
);
|
||||
insert into recording_integrity
|
||||
select
|
||||
composite_id,
|
||||
local_time_delta_90k,
|
||||
local_time_since_open_90k,
|
||||
wall_time_delta_90k,
|
||||
null
|
||||
from
|
||||
old_recording_integrity;
|
||||
|
||||
alter table recording_playback rename to old_recording_playback;
|
||||
create table recording_playback (
|
||||
composite_id integer primary key references recording (composite_id),
|
||||
video_index blob not null check (length(video_index) > 0)
|
||||
);
|
||||
insert into recording_playback select * from old_recording_playback;
|
||||
|
||||
drop table old_recording_playback;
|
||||
drop table old_recording_integrity;
|
||||
drop table old_recording;
|
||||
drop table old_stream;
|
||||
drop table old_video_sample_entry;
|
||||
|
||||
update user_session
|
||||
set
|
||||
revocation_reason = 1,
|
||||
revocation_reason_detail = 'Blake2b->Blake3 upgrade'
|
||||
where
|
||||
revocation_reason is null;
|
||||
"#)?;
|
||||
Ok(())
|
||||
}
|
||||
Reference in New Issue
Block a user