From dad349840df89b83658c4c68e854bbd1dbd3eb0e Mon Sep 17 00:00:00 2001 From: Scott Lamb Date: Tue, 26 Oct 2021 10:12:19 -0700 Subject: [PATCH] more flexible signals Now there's room to add arbitrary configuration to signals and types. Several things are no longer fixed columns/tables but instead within the configuration types. --- design/api.md | 2 +- server/db/Cargo.toml | 2 +- server/db/check.rs | 8 +- server/db/db.rs | 23 ++-- server/db/json.rs | 146 ++++++++++++++++++++- server/db/raw.rs | 14 +- server/db/schema.sql | 70 ++-------- server/db/signal.rs | 238 ++++++++++++++-------------------- server/db/upgrade/v0_to_v1.rs | 2 +- server/db/upgrade/v1_to_v2.rs | 4 +- server/db/upgrade/v2_to_v3.rs | 8 +- server/db/upgrade/v4_to_v5.rs | 8 +- server/db/upgrade/v6_to_v7.rs | 160 ++++++++++++++++++++++- server/src/json.rs | 37 +++--- 14 files changed, 464 insertions(+), 258 deletions(-) diff --git a/design/api.md b/design/api.md index 397f21c..6f29511 100644 --- a/design/api.md +++ b/design/api.md @@ -142,7 +142,7 @@ The `application/json` response will have a JSON object as follows: * `signals`: a list of all *signals* known to the server. Each is a JSON object with the following properties: * `id`: an integer identifier. - * `source`: a UUID representing the signal source (could be a camera UUID) + * `uuid`: a UUID identifier. * `shortName`: a unique, human-readable description of the signal * `cameras`: a map of associated cameras' UUIDs to the type of association: `direct` or `indirect`. See `db/schema.sql` for more description. diff --git a/server/db/Cargo.toml b/server/db/Cargo.toml index e3b664d..b136bf1 100644 --- a/server/db/Cargo.toml +++ b/server/db/Cargo.toml @@ -44,7 +44,7 @@ tempfile = "3.2.0" time = "0.1" tokio = { version = "1.0", features = ["macros", "parking_lot", "rt-multi-thread", "sync"] } url = { version = "2.1.1", features = ["serde"] } -uuid = { version = "0.8", features = ["std", "v4"] } +uuid = { version = "0.8", features = ["serde", "std", "v4"] } itertools = "0.10.0" [build-dependencies] diff --git a/server/db/check.rs b/server/db/check.rs index 7aa7f20..a683373 100644 --- a/server/db/check.rs +++ b/server/db/check.rs @@ -5,7 +5,7 @@ //! Subcommand to check the database and sample file dir for errors. use crate::compare; -use crate::db::{self, CompositeId, FromSqlUuid}; +use crate::db::{self, CompositeId, SqlUuid}; use crate::dir; use crate::raw; use crate::recording; @@ -72,7 +72,7 @@ pub fn run(conn: &mut rusqlite::Connection, opts: &Options) -> Result = FnvHashMap::default(); @@ -90,9 +90,9 @@ pub fn run(conn: &mut rusqlite::Connection, opts: &Options) -> Result i64 { (bytes + blk - 1) / blk * blk } -pub struct FromSqlUuid(pub Uuid); +/// A wrapper around `Uuid` which implements `FromSql` and `ToSql`. +pub struct SqlUuid(pub Uuid); -impl rusqlite::types::FromSql for FromSqlUuid { +impl rusqlite::types::FromSql for SqlUuid { fn column_result(value: rusqlite::types::ValueRef) -> rusqlite::types::FromSqlResult { let uuid = Uuid::from_slice(value.as_blob()?) .map_err(|e| rusqlite::types::FromSqlError::Other(Box::new(e)))?; - Ok(FromSqlUuid(uuid)) + Ok(SqlUuid(uuid)) } } -impl rusqlite::types::ToSql for FromSqlUuid { +impl rusqlite::types::ToSql for SqlUuid { fn to_sql(&self) -> rusqlite::Result> { Ok(self.0.as_bytes()[..].into()) } @@ -1554,9 +1555,9 @@ impl LockedDatabase { let mut rows = stmt.query(params![])?; while let Some(row) = rows.next()? { let id = row.get(0)?; - let dir_uuid: FromSqlUuid = row.get(2)?; + let dir_uuid: SqlUuid = row.get(2)?; let open_id: Option = row.get(3)?; - let open_uuid: Option = row.get(4)?; + let open_uuid: Option = row.get(4)?; let last_complete_open = match (open_id, open_uuid) { (Some(id), Some(uuid)) => Some(Open { id, uuid: uuid.0 }), (None, None) => None, @@ -1600,7 +1601,7 @@ impl LockedDatabase { let mut rows = stmt.query(params![])?; while let Some(row) = rows.next()? { let id = row.get(0)?; - let uuid: FromSqlUuid = row.get(1)?; + let uuid: SqlUuid = row.get(1)?; self.cameras_by_id.insert( id, Camera { @@ -2247,19 +2248,19 @@ impl Database { // Note: the meta check comes after the version check to improve the error message when // trying to open a version 0 or version 1 database (which lacked the meta table). - let uuid = raw::get_db_uuid(&conn)?; + let (uuid, config) = raw::read_meta(&conn)?; let open_monotonic = recording::Time::new(clocks.monotonic()); let open = if read_write { let real = recording::Time::new(clocks.realtime()); let mut stmt = conn .prepare(" insert into open (uuid, start_time_90k, boot_uuid) values (?, ?, ?)")?; - let open_uuid = FromSqlUuid(Uuid::new_v4()); + let open_uuid = SqlUuid(Uuid::new_v4()); let boot_uuid = match get_boot_uuid() { Err(e) => { warn!("Unable to get boot uuid: {}", e); None } - Ok(id) => id.map(FromSqlUuid), + Ok(id) => id.map(SqlUuid), }; stmt.execute(params![open_uuid, real.0, boot_uuid])?; let id = conn.last_insert_rowid() as u32; @@ -2268,7 +2269,7 @@ impl Database { None }; let auth = auth::State::init(&conn)?; - let signal = signal::State::init(&conn)?; + let signal = signal::State::init(&conn, &config)?; let db = Database { db: Some(Mutex::new(LockedDatabase { conn, diff --git a/server/db/json.rs b/server/db/json.rs index 9a34f2f..041a1ff 100644 --- a/server/db/json.rs +++ b/server/db/json.rs @@ -3,12 +3,44 @@ // SPDX-License-Identifier: GPL-v3.0-or-later WITH GPL-3.0-linking-exception. //! JSON types for use in the database schema. See references from `schema.sql`. +//! +//! In general, every table in the database with reasonably low expected row +//! count should have a JSON config column. This allows the schema to be +//! modified without a major migration. +//! +//! JSON should be avoided for very high-row-count tables (eg `reocrding`) for +//! storage efficiency, in favor of separate columns or a binary type. +//! (Currently protobuf is used within `user_session`. A future schema version +//! might switch to a more JSON-like binary format to minimize impedance +//! mismatch.) +//! +//! JSON types should be designed for extensibility with forward and backward +//! compatibility: +//! +//! * Every struct has a flattened `unknown` so that if an unknown attribute is +//! written with a newer version of the binary, then the config is saved +//! (read and re-written) with an older version, the value will be +//! preserved. +//! * If a field is only for use by the UI and there's no need for the server +//! to constrain it, leave it in `unknown`. +//! * Fields shouldn't use closed enumerations or other restrictive types, +//! so that parsing the config with a non-understood value will not fail. If +//! the behavior of unknown values is not obvious, it should be clarified +//! via a comment. +//! * Fields should generally parse without values, via `#[serde(default)]`, +//! so that they can be removed in a future version if they no longer make +//! sense. It also makes sense to avoid serializing them when empty. + +use std::collections::BTreeMap; use rusqlite::types::{FromSqlError, ValueRef}; use serde::{Deserialize, Serialize}; use serde_json::{Map, Value}; use url::Url; +use uuid::Uuid; +/// Serializes and deserializes JSON as a SQLite3 `text` column, compatible with the +/// [JSON1 extension](https://www.sqlite.org/json1.html). macro_rules! sql { ($l:ident) => { impl rusqlite::types::FromSql for $l { @@ -18,6 +50,7 @@ macro_rules! sql { Ok(serde_json::from_slice(t) .map_err(|e| FromSqlError::Other(Box::new(e)))?) } + ValueRef::Null => Ok($l::default()), _ => Err(FromSqlError::InvalidType), } } @@ -33,24 +66,98 @@ macro_rules! sql { }; } +/// Global configuration, used in the `config` column of the `meta` table. +#[derive(Clone, Debug, Default, Deserialize, Eq, PartialEq, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct GlobalConfig { + /// The maximum number of entries in the `signal_state` table (or `None` for unlimited). + /// + /// If an update causes this to be exceeded, older times will be garbage + /// collected to stay within the limit. + pub max_signal_changes: Option, + + /// Information about signal types. + #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] + pub signal_types: BTreeMap, + + /// Information about signals. + #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] + pub signals: BTreeMap, + + #[serde(flatten)] + pub unknown: Map, +} +sql!(GlobalConfig); + +#[derive(Clone, Debug, Default, Deserialize, Eq, PartialEq, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct SignalTypeConfig { + /// Information about possible enumeration values of this signal type. + /// + /// 0 always means `unknown`. Other values may be specified here to set + /// their configuration. It's more efficient in terms of encoded length + /// and RAM at runtime for common values (eg, `still`, `normal`, or + /// `disarmed`) to be numerically lower than rarer values (eg `motion`, + /// `violated`, or `armed`) and for the value space to be dense (eg, to use + /// values 1, 2, 3 rather than 1, 10, 20). + /// + /// Currently values must be in the range `[0, 16)`. + /// + /// Nothing enforces that only values specified here may be set for a signal + /// of this type. + #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] + pub values: BTreeMap, + + #[serde(flatten)] + pub unknown: Map, +} +sql!(SignalTypeConfig); + +/// Information about a signal type value; used in `SignalTypeConfig::values`. +#[derive(Clone, Debug, Default, Deserialize, Eq, PartialEq, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct SignalTypeValueConfig { + pub name: String, + + #[serde(default, skip_serializing_if = "std::ops::Not::not")] + pub motion: bool, + + #[serde(default, skip_serializing_if = "String::is_empty")] + pub color: String, + + #[serde(flatten)] + pub unknown: Map, +} + +impl SignalTypeValueConfig { + pub fn is_empty(&self) -> bool { + self.unknown.is_empty() + } +} + +/// Camera configuration, used in the `config` column of the `camera` table. #[derive(Clone, Debug, Default, Deserialize, Eq, PartialEq, Serialize)] #[serde(rename_all = "camelCase")] pub struct CameraConfig { /// A short description of the camera. + #[serde(default, skip_serializing_if = "String::is_empty")] pub description: String, /// The base URL for accessing ONVIF; `device_service` will be joined on /// automatically to form the device management service URL. /// Eg with `onvif_base=http://192.168.1.110:85`, the full - /// URL of the devie management service will be + /// URL of the device management service will be /// `http://192.168.1.110:85/device_service`. + #[serde(default, skip_serializing_if = "Option::is_none")] pub onvif_base_url: Option, /// The username to use when accessing the camera. /// If empty, no username or password will be supplied. + #[serde(default, skip_serializing_if = "String::is_empty")] pub username: String, /// The password to use when accessing the camera. + #[serde(default, skip_serializing_if = "String::is_empty")] pub password: String, #[serde(flatten)] @@ -68,6 +175,7 @@ impl CameraConfig { } } +/// Stream configuration, used in the `config` column of the `stream` table. #[derive(Clone, Debug, Default, Deserialize, Eq, PartialEq, Serialize)] #[serde(rename_all = "camelCase")] pub struct StreamConfig { @@ -75,7 +183,7 @@ pub struct StreamConfig { /// /// Null means entirely disabled. At present, so does any value other than /// `record`. - #[serde(default)] + #[serde(default, skip_serializing_if = "String::is_empty")] pub mode: String, /// The `rtsp://` URL to use for this stream, excluding username and @@ -86,7 +194,7 @@ pub struct StreamConfig { /// protocol](https://github.com/thirtythreeforty/neolink). /// /// (Credentials are taken from [`CameraConfig`]'s respective fields.) - // TODO: should this really be Option? + #[serde(default, skip_serializing_if = "Option::is_none")] pub url: Option, /// The number of bytes of video to retain, excluding the @@ -126,3 +234,35 @@ impl StreamConfig { && self.unknown.is_empty() } } + +/// Signal configuration, used in the `config` column of the `signal` table. +#[derive(Clone, Debug, Default, Deserialize, Eq, PartialEq, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct SignalConfig { + #[serde(default, skip_serializing_if = "String::is_empty")] + pub short_name: String, + + /// Map of associated cameras to the type of association. + /// + /// `direct` is as if the event source is the camera's own motion detection. + /// Here are a couple ways this could be used: + /// + /// * when viewing the camera, hotkeys to go to the start of the next or + /// previous event should respect this event. + /// * a list of events might include the recordings associated with the + /// camera in the same timespan. + /// + /// `indirect` might mean a screen associated with the camera should given + /// some indication of this event, but there should be no assumption that + /// the camera will have a direct view of the event. For example, all + /// cameras might be indirectly associated with a doorknob press. Cameras + /// at the back of the house shouldn't be expected to have a direct view of + /// this event, but motion events shortly afterward might warrant extra + /// scrutiny. + #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] + pub camera_associations: BTreeMap, + + #[serde(flatten)] + pub unknown: Map, +} +sql!(SignalConfig); diff --git a/server/db/raw.rs b/server/db/raw.rs index 29e55c9..8f2e8c4 100644 --- a/server/db/raw.rs +++ b/server/db/raw.rs @@ -4,7 +4,8 @@ //! Raw database access: SQLite statements which do not touch any cached state. -use crate::db::{self, CompositeId, FromSqlUuid}; +use crate::db::{self, CompositeId, SqlUuid}; +use crate::json::GlobalConfig; use crate::recording; use base::{ErrorKind, ResultExt as _}; use failure::{bail, Error, ResultExt as _}; @@ -170,13 +171,14 @@ fn list_recordings_inner( Ok(()) } -pub(crate) fn get_db_uuid(conn: &rusqlite::Connection) -> Result { +pub(crate) fn read_meta(conn: &rusqlite::Connection) -> Result<(Uuid, GlobalConfig), Error> { Ok(conn.query_row( - "select uuid from meta", + "select uuid, config from meta", params![], - |row| -> rusqlite::Result { - let uuid: FromSqlUuid = row.get(0)?; - Ok(uuid.0) + |row| -> rusqlite::Result<(Uuid, GlobalConfig)> { + let uuid: SqlUuid = row.get(0)?; + let config: GlobalConfig = row.get(1)?; + Ok((uuid.0, config)) }, )?) } diff --git a/server/db/schema.sql b/server/db/schema.sql index 3734090..440ebbf 100644 --- a/server/db/schema.sql +++ b/server/db/schema.sql @@ -9,10 +9,8 @@ create table meta ( uuid blob not null check (length(uuid) = 16), - -- The maximum number of entries in the signal_state table. If an update - -- causes this to be exceeded, older times will be garbage collected to stay - -- within the limit. - max_signal_changes integer check (max_signal_changes >= 0) + -- Holds a json.GlobalConfig. + config text ); -- This table tracks the schema version. @@ -393,65 +391,19 @@ create index user_session_uid on user_session (user_id); -- * security system zone status (unknown, normal, violated, trouble) create table signal ( id integer primary key, + uuid blob unique not null check (length(uuid) = 16), + type_uuid blob not null references signal_type (uuid) + check (length(type_uuid) = 16), - -- a uuid describing the originating object, such as the uuid of the camera - -- for built-in motion detection. There will be a JSON interface for adding - -- events; it will require this UUID to be supplied. An external uuid might - -- indicate "my house security system's zone 23". - source_uuid blob not null check (length(source_uuid) = 16), - - -- a uuid describing the type of event. A registry (TBD) will list built-in - -- supported types, such as "Hikvision on-camera motion detection", or - -- "ONVIF on-camera motion detection". External programs can use their own - -- uuids, such as "Elk security system watcher". - type_uuid blob not null check (length(type_uuid) = 16), - - -- a short human-readable description to use in mouseovers or event lists, - -- such as "driveway motion" or "front door open". - short_name not null, - - unique (source_uuid, type_uuid) + -- Holds a json.SignalConfig + config text ); --- e.g. "still/moving", "disarmed/away/stay", etc. -create table signal_type_enum ( - type_uuid blob not null check (length(type_uuid) = 16), - value integer not null check (value > 0 and value < 16), - name text not null, +create table signal_type ( + uuid blob primary key check (length(uuid) = 16), - -- true/1 iff this signal value should be considered "motion" for directly associated cameras. - motion int not null check (motion in (0, 1)) default 0, - - color text -); - --- Associations between event sources and cameras. --- For example, if two cameras have overlapping fields of view, they might be --- configured such that each camera is associated with both its own motion and --- the other camera's motion. -create table signal_camera ( - signal_id integer references signal (id), - camera_id integer references camera (id), - - -- type: - -- - -- 0 means direct association, as if the event source if the camera's own - -- motion detection. Here are a couple ways this could be used: - -- - -- * when viewing the camera, hotkeys to go to the start of the next or - -- previous event should respect this event. - -- * a list of events might include the recordings associated with the - -- camera in the same timespan. - -- - -- 1 means indirect association. A screen associated with the camera should - -- given some indication of this event, but there should be no assumption - -- that the camera will have a direct view of the event. For example, all - -- cameras might be indirectly associated with a doorknob press. Cameras at - -- the back of the house shouldn't be expected to have a direct view of this - -- event, but motion events shortly afterward might warrant extra scrutiny. - type integer not null, - - primary key (signal_id, camera_id) + -- Holds a json.SignalTypeConfig + config text ) without rowid; -- Changes to signals as of a given timestamp. diff --git a/server/db/signal.rs b/server/db/signal.rs index 39fcce8..330b9b7 100644 --- a/server/db/signal.rs +++ b/server/db/signal.rs @@ -5,9 +5,9 @@ //! Schema for "signals": enum-valued timeserieses. //! See the `signal` table within `schema.sql` for more information. -use crate::db::FromSqlUuid; -use crate::recording; +use crate::json::{SignalConfig, SignalTypeConfig}; use crate::{coding, days}; +use crate::{recording, SqlUuid}; use base::bail_t; use failure::{bail, format_err, Error}; use fnv::FnvHashMap; @@ -15,6 +15,7 @@ use log::debug; use rusqlite::{params, Connection, Transaction}; use std::collections::btree_map::Entry; use std::collections::{BTreeMap, BTreeSet}; +use std::convert::TryFrom; use std::ops::Range; use uuid::Uuid; @@ -38,7 +39,7 @@ pub(crate) struct State { /// These either have a matching `points_by_time` entry or represent a removal. dirty_by_time: BTreeSet, - max_signal_changes: Option, + max_signal_changes: Option, } /// Representation of all signals at a point in time. @@ -186,21 +187,6 @@ impl<'a> PointDataIterator<'a> { } } -/// Representation of a `signal_camera` row. -/// `signal_id` is implied by the `Signal` which owns this struct. -#[derive(Debug)] -pub struct SignalCamera { - pub camera_id: i32, - pub type_: SignalCameraType, -} - -/// Representation of the `type` field in a `signal_camera` row. -#[derive(Debug)] -pub enum SignalCameraType { - Direct = 0, - Indirect = 1, -} - #[derive(Copy, Clone, Debug, PartialEq, Eq)] pub struct ListStateChangesRow { pub when: recording::Time, @@ -209,17 +195,12 @@ pub struct ListStateChangesRow { } impl State { - pub fn init(conn: &Connection) -> Result { - let max_signal_changes: Option = - conn.query_row("select max_signal_changes from meta", params![], |row| { - row.get(0) - })?; + pub fn init(conn: &Connection, config: &crate::json::GlobalConfig) -> Result { let mut signals_by_id = State::init_signals(conn)?; - State::fill_signal_cameras(conn, &mut signals_by_id)?; let mut points_by_time = BTreeMap::new(); State::fill_points(conn, &mut points_by_time, &mut signals_by_id)?; let s = State { - max_signal_changes, + max_signal_changes: config.max_signal_changes, signals_by_id, types_by_uuid: State::init_types(conn)?, points_by_time, @@ -291,8 +272,6 @@ impl State { fn gc(&mut self) { let max = match self.max_signal_changes { None => return, - Some(m) if m < 0 => 0_usize, - Some(m) if m > (isize::max_value() as i64) => return, Some(m) => m as usize, }; let to_remove = match self.points_by_time.len().checked_sub(max) { @@ -366,13 +345,12 @@ impl State { match self.signals_by_id.get(&signal) { None => bail_t!(InvalidArgument, "unknown signal {}", signal), Some(ref s) => { - let empty = Vec::new(); let states = self .types_by_uuid .get(&s.type_) - .map(|t| &t.states) - .unwrap_or(&empty); - if state != 0 && states.binary_search_by_key(&state, |s| s.value).is_err() { + .map(|t| t.valid_states) + .unwrap_or(0); + if state >= 16 || (states & (1 << state)) == 0 { bail_t!( FailedPrecondition, "signal {} specifies unknown state {}", @@ -671,33 +649,67 @@ impl State { r#" select id, - source_uuid, + uuid, type_uuid, - short_name + config from signal "#, )?; let mut rows = stmt.query(params![])?; while let Some(row) = rows.next()? { - let id = row.get(0)?; - let source: FromSqlUuid = row.get(1)?; - let type_: FromSqlUuid = row.get(2)?; + let id: i32 = row.get(0)?; + let id = u32::try_from(id)?; + let uuid: SqlUuid = row.get(1)?; + let type_: SqlUuid = row.get(2)?; + let config: SignalConfig = row.get(3)?; signals.insert( id, Signal { id, - source: source.0, - type_: type_.0, - short_name: row.get(3)?, - cameras: Vec::new(), + uuid: uuid.0, days: days::Map::default(), + type_: type_.0, + config, }, ); } Ok(signals) } + fn init_types(conn: &Connection) -> Result, Error> { + let mut types = FnvHashMap::default(); + let mut stmt = conn.prepare( + r#" + select + uuid, + config + from + signal_type + "#, + )?; + let mut rows = stmt.query(params![])?; + while let Some(row) = rows.next()? { + let uuid: SqlUuid = row.get(0)?; + let mut type_ = Type { + valid_states: 1, // bit 0 (unknown state) is always valid. + config: row.get(1)?, + }; + for &value in type_.config.values.keys() { + if value == 0 || value >= 16 { + bail!( + "signal type {} value {} out of accepted range [0, 16)", + uuid.0, + value + ); + } + type_.valid_states |= 1 << value; + } + types.insert(uuid.0, type_); + } + Ok(types) + } + /// Fills `points_by_time` from the database, also filling the `days` /// index of each signal. fn fill_points( @@ -755,73 +767,6 @@ impl State { Ok(()) } - /// Fills the `cameras` field of the `Signal` structs within the supplied `signals`. - fn fill_signal_cameras( - conn: &Connection, - signals: &mut BTreeMap, - ) -> Result<(), Error> { - let mut stmt = conn.prepare( - r#" - select - signal_id, - camera_id, - type - from - signal_camera - order by signal_id, camera_id - "#, - )?; - let mut rows = stmt.query(params![])?; - while let Some(row) = rows.next()? { - let signal_id = row.get(0)?; - let s = signals.get_mut(&signal_id).ok_or_else(|| { - format_err!("signal_camera row for unknown signal id {}", signal_id) - })?; - let type_ = row.get(2)?; - s.cameras.push(SignalCamera { - camera_id: row.get(1)?, - type_: match type_ { - 0 => SignalCameraType::Direct, - 1 => SignalCameraType::Indirect, - _ => bail!("unknown signal_camera type {}", type_), - }, - }); - } - Ok(()) - } - - fn init_types(conn: &Connection) -> Result, Error> { - let mut types = FnvHashMap::default(); - let mut stmt = conn.prepare( - r#" - select - type_uuid, - value, - name, - motion, - color - from - signal_type_enum - order by type_uuid, value - "#, - )?; - let mut rows = stmt.query(params![])?; - while let Some(row) = rows.next()? { - let type_: FromSqlUuid = row.get(0)?; - types - .entry(type_.0) - .or_insert_with(Type::default) - .states - .push(TypeState { - value: row.get(1)?, - name: row.get(2)?, - motion: row.get(3)?, - color: row.get(4)?, - }); - } - Ok(types) - } - pub fn signals_by_id(&self) -> &BTreeMap { &self.signals_by_id } @@ -853,37 +798,26 @@ impl State { #[derive(Debug)] pub struct Signal { pub id: u32, - pub source: Uuid, + pub uuid: Uuid, pub type_: Uuid, - pub short_name: String, - - /// The cameras this signal is associated with. Sorted by camera id, which is unique. - pub cameras: Vec, - pub days: days::Map, + pub config: SignalConfig, } -/// Representation of a `signal_type_enum` row. -/// `type_uuid` is implied by the `Type` which owns this struct. -#[derive(Debug)] -pub struct TypeState { - pub value: u16, - pub name: String, - pub motion: bool, - pub color: String, -} - -/// Representation of a signal type; currently this just gathers together the TypeStates. #[derive(Debug, Default)] pub struct Type { - /// The possible states associated with this type. They are sorted by value, which is unique. - pub states: Vec, + pub valid_states: u16, + pub config: SignalTypeConfig, } #[cfg(test)] mod tests { use super::*; - use crate::{db, testutil}; + use crate::{ + db, + json::{GlobalConfig, SignalTypeConfig, SignalTypeValueConfig}, + testutil, + }; use rusqlite::Connection; use smallvec::smallvec; @@ -903,7 +837,7 @@ mod tests { testutil::init(); let mut conn = Connection::open_in_memory().unwrap(); db::init(&mut conn).unwrap(); - let s = State::init(&conn).unwrap(); + let s = State::init(&conn, &GlobalConfig::default()).unwrap(); s.list_changes_by_time( recording::Time::min_value()..recording::Time::max_value(), &mut |_r| panic!("no changes expected"), @@ -915,23 +849,49 @@ mod tests { testutil::init(); let mut conn = Connection::open_in_memory().unwrap(); db::init(&mut conn).unwrap(); + let mut type_config = SignalTypeConfig::default(); + type_config.values.insert( + 1, + SignalTypeValueConfig { + name: "still".to_owned(), + motion: false, + color: "black".to_owned(), + ..Default::default() + }, + ); + type_config.values.insert( + 2, + SignalTypeValueConfig { + name: "moving".to_owned(), + motion: true, + color: "red".to_owned(), + ..Default::default() + }, + ); + conn.execute( + "insert into signal_type (uuid, config) values (?, ?)", + params![ + SqlUuid(Uuid::parse_str("ee66270f-d9c6-4819-8b33-9720d4cbca6b").unwrap()), + &type_config, + ], + ) + .unwrap(); conn.execute_batch( r#" - update meta set max_signal_changes = 2; - - insert into signal (id, source_uuid, type_uuid, short_name) + insert into signal (id, uuid, type_uuid, config) values (1, x'1B3889C0A59F400DA24C94EBEB19CC3A', - x'EE66270FD9C648198B339720D4CBCA6B', 'a'), + x'EE66270FD9C648198B339720D4CBCA6B', '{"name": "a"}'), (2, x'A4A73D9A53424EBCB9F6366F1E5617FA', - x'EE66270FD9C648198B339720D4CBCA6B', 'b'); + x'EE66270FD9C648198B339720D4CBCA6B', '{"name": "b"}'); - insert into signal_type_enum (type_uuid, value, name, motion, color) - values (x'EE66270FD9C648198B339720D4CBCA6B', 1, 'still', 0, 'black'), - (x'EE66270FD9C648198B339720D4CBCA6B', 2, 'moving', 1, 'red'); "#, ) .unwrap(); - let mut s = State::init(&conn).unwrap(); + let config = GlobalConfig { + max_signal_changes: Some(2), + ..Default::default() + }; + let mut s = State::init(&conn, &config).unwrap(); s.list_changes_by_time( recording::Time::min_value()..recording::Time::max_value(), &mut |_r| panic!("no changes expected"), @@ -997,7 +957,7 @@ mod tests { } drop(s); - let mut s = State::init(&conn).unwrap(); + let mut s = State::init(&conn, &config).unwrap(); rows.clear(); s.list_changes_by_time( recording::Time::min_value()..recording::Time::max_value(), @@ -1044,7 +1004,7 @@ mod tests { tx.commit().unwrap(); } drop(s); - let s = State::init(&conn).unwrap(); + let s = State::init(&conn, &config).unwrap(); rows.clear(); s.list_changes_by_time( recording::Time::min_value()..recording::Time::max_value(), diff --git a/server/db/upgrade/v0_to_v1.rs b/server/db/upgrade/v0_to_v1.rs index 052694b..5bca566 100644 --- a/server/db/upgrade/v0_to_v1.rs +++ b/server/db/upgrade/v0_to_v1.rs @@ -159,7 +159,7 @@ fn fill_recording(tx: &rusqlite::Transaction) -> Result = row.get(9)?; let video_index: Vec = row.get(10)?; let old_id: i32 = row.get(11)?; diff --git a/server/db/upgrade/v1_to_v2.rs b/server/db/upgrade/v1_to_v2.rs index 71c0c20..f718c68 100644 --- a/server/db/upgrade/v1_to_v2.rs +++ b/server/db/upgrade/v1_to_v2.rs @@ -335,7 +335,7 @@ fn verify_dir_contents( let mut stmt = tx.prepare(r"select sample_file_uuid from recording_playback")?; let mut rows = stmt.query(params![])?; while let Some(row) = rows.next()? { - let uuid: crate::db::FromSqlUuid = row.get(0)?; + let uuid: crate::db::SqlUuid = row.get(0)?; if !files.remove(&uuid.0) { bail!( "{} is missing from dir {}!", @@ -349,7 +349,7 @@ fn verify_dir_contents( let mut stmt = tx.prepare(r"select uuid from reserved_sample_files")?; let mut rows = stmt.query(params![])?; while let Some(row) = rows.next()? { - let uuid: crate::db::FromSqlUuid = row.get(0)?; + let uuid: crate::db::SqlUuid = row.get(0)?; if files.remove(&uuid.0) { // Also remove the garbage file. For historical reasons (version 2 was originally // defined as not having a garbage table so still is), do this here rather than with diff --git a/server/db/upgrade/v2_to_v3.rs b/server/db/upgrade/v2_to_v3.rs index 8e21f12..644fc39 100644 --- a/server/db/upgrade/v2_to_v3.rs +++ b/server/db/upgrade/v2_to_v3.rs @@ -5,7 +5,7 @@ /// Upgrades a version 2 schema to a version 3 schema. /// Note that a version 2 schema is never actually used; so we know the upgrade from version 1 was /// completed, and possibly an upgrade from 2 to 3 is half-finished. -use crate::db::{self, FromSqlUuid}; +use crate::db::{self, SqlUuid}; use crate::dir; use crate::schema; use failure::Error; @@ -19,8 +19,8 @@ use std::sync::Arc; /// * there's only one dir. /// * it has a last completed open. fn open_sample_file_dir(tx: &rusqlite::Transaction) -> Result, Error> { - let (p, s_uuid, o_id, o_uuid, db_uuid): (String, FromSqlUuid, i32, FromSqlUuid, FromSqlUuid) = - tx.query_row( + let (p, s_uuid, o_id, o_uuid, db_uuid): (String, SqlUuid, i32, SqlUuid, SqlUuid) = tx + .query_row( r#" select s.path, s.uuid, s.last_complete_open_id, o.uuid, m.uuid @@ -65,7 +65,7 @@ pub fn run(_args: &super::Args, tx: &rusqlite::Transaction) -> Result<(), Error> let mut rows = stmt.query(params![])?; while let Some(row) = rows.next()? { let id = db::CompositeId(row.get(0)?); - let sample_file_uuid: FromSqlUuid = row.get(1)?; + let sample_file_uuid: SqlUuid = row.get(1)?; let from_path = super::UuidPath::from(sample_file_uuid.0); let to_path = crate::dir::CompositeIdPath::from(id); if let Err(e) = nix::fcntl::renameat( diff --git a/server/db/upgrade/v4_to_v5.rs b/server/db/upgrade/v4_to_v5.rs index 3940114..0e7ce7f 100644 --- a/server/db/upgrade/v4_to_v5.rs +++ b/server/db/upgrade/v4_to_v5.rs @@ -6,7 +6,7 @@ /// /// This just handles the directory meta files. If they're already in the new format, great. /// Otherwise, verify they are consistent with the database then upgrade them. -use crate::db::FromSqlUuid; +use crate::db::SqlUuid; use crate::{dir, schema}; use cstr::cstr; use failure::{bail, Error, Fail}; @@ -111,7 +111,7 @@ fn maybe_cleanup_garbage_uuids(dir: &dir::Fd) -> Result { } pub fn run(_args: &super::Args, tx: &rusqlite::Transaction) -> Result<(), Error> { - let db_uuid: FromSqlUuid = + let db_uuid: SqlUuid = tx.query_row_and_then(r"select uuid from meta", params![], |row| row.get(0))?; let mut stmt = tx.prepare( r#" @@ -129,9 +129,9 @@ pub fn run(_args: &super::Args, tx: &rusqlite::Transaction) -> Result<(), Error> while let Some(row) = rows.next()? { let path = row.get_ref(0)?.as_str()?; info!("path: {}", path); - let dir_uuid: FromSqlUuid = row.get(1)?; + let dir_uuid: SqlUuid = row.get(1)?; let open_id: Option = row.get(2)?; - let open_uuid: Option = row.get(3)?; + let open_uuid: Option = row.get(3)?; let mut db_meta = schema::DirMeta::new(); db_meta.db_uuid.extend_from_slice(&db_uuid.0.as_bytes()[..]); db_meta diff --git a/server/db/upgrade/v6_to_v7.rs b/server/db/upgrade/v6_to_v7.rs index e28320f..4769305 100644 --- a/server/db/upgrade/v6_to_v7.rs +++ b/server/db/upgrade/v6_to_v7.rs @@ -3,11 +3,138 @@ // SPDX-License-Identifier: GPL-v3.0-or-later WITH GPL-3.0-linking-exception /// Upgrades a version 6 schema to a version 7 schema. -use failure::Error; +use failure::{format_err, Error}; +use fnv::FnvHashMap; use rusqlite::{named_params, params}; +use std::convert::TryFrom; use url::Url; +use uuid::Uuid; -use crate::{json::CameraConfig, FromSqlUuid}; +use crate::{ + json::{CameraConfig, GlobalConfig, SignalConfig, SignalTypeConfig}, + SqlUuid, +}; + +fn copy_meta(tx: &rusqlite::Transaction) -> Result<(), Error> { + let mut stmt = tx.prepare("select uuid, max_signal_changes from old_meta")?; + let mut insert = tx.prepare("insert into meta (uuid, config) values (:uuid, :config)")?; + let mut rows = stmt.query(params![])?; + while let Some(row) = rows.next()? { + let uuid: SqlUuid = row.get(0)?; + let max_signal_changes: Option = row.get(1)?; + let config = GlobalConfig { + max_signal_changes: max_signal_changes + .map(|s| { + u32::try_from(s).map_err(|_| format_err!("max_signal_changes out of range")) + }) + .transpose()?, + ..Default::default() + }; + insert.execute(named_params! { + ":uuid": uuid, + ":config": &config, + })?; + } + + Ok(()) +} + +fn copy_signal_types(tx: &rusqlite::Transaction) -> Result<(), Error> { + let mut types_ = FnvHashMap::default(); + let mut stmt = tx.prepare("select type_uuid, value, name from signal_type_enum")?; + let mut rows = stmt.query(params![])?; + while let Some(row) = rows.next()? { + let type_uuid: SqlUuid = row.get(0)?; + let value: i32 = row.get(1)?; + let name: Option = row.get(2)?; + let type_ = types_ + .entry(type_uuid.0) + .or_insert_with(SignalTypeConfig::default); + let value = u8::try_from(value).map_err(|_| format_err!("bad signal type value"))?; + let value_config = type_.values.entry(value).or_insert_with(Default::default); + if let Some(n) = name { + value_config.name = n; + } + } + let mut insert = tx.prepare("insert into signal_type (uuid, config) values (?, ?)")?; + for (&uuid, config) in &types_ { + insert.execute(params![SqlUuid(uuid), config])?; + } + Ok(()) +} + +struct Signal { + uuid: Uuid, + type_uuid: Uuid, + config: SignalConfig, +} + +fn copy_signals(tx: &rusqlite::Transaction) -> Result<(), Error> { + let mut signals = FnvHashMap::default(); + + // Read from signal table. + { + let mut stmt = + tx.prepare("select id, source_uuid, type_uuid, short_name from old_signal")?; + let mut rows = stmt.query(params![])?; + while let Some(row) = rows.next()? { + let id: i32 = row.get(0)?; + let id = u32::try_from(id)?; + let source_uuid: SqlUuid = row.get(1)?; + let type_uuid: SqlUuid = row.get(2)?; + let short_name: String = row.get(3)?; + signals.insert( + id, + Signal { + uuid: source_uuid.0, + type_uuid: type_uuid.0, + config: SignalConfig { + short_name, + ..Default::default() + }, + }, + ); + } + } + + // Read from the signal_camera table. + { + let mut stmt = tx.prepare("select signal_id, camera_id, type from signal_camera")?; + let mut rows = stmt.query(params![])?; + while let Some(row) = rows.next()? { + let signal_id: i32 = row.get(0)?; + let signal_id = u32::try_from(signal_id)?; + let camera_id: i32 = row.get(1)?; + let type_: i32 = row.get(2)?; + let signal = signals.get_mut(&signal_id).unwrap(); + signal.config.camera_associations.insert( + camera_id, + match type_ { + 0 => "direct", + _ => "indirect", + } + .to_owned(), + ); + } + } + + let mut insert = tx.prepare( + r#" + insert into signal (id, uuid, type_uuid, config) + values (:id, :uuid, :type_uuid, :config) + "#, + )?; + for (&id, signal) in &signals { + insert.execute(named_params! { + ":id": id, + ":uuid": SqlUuid(signal.uuid), + ":type_uuid": SqlUuid(signal.type_uuid), + ":config": &signal.config, + })?; + } + + Ok(()) +} fn copy_cameras(tx: &rusqlite::Transaction) -> Result<(), Error> { let mut insert = tx.prepare( @@ -34,7 +161,7 @@ fn copy_cameras(tx: &rusqlite::Transaction) -> Result<(), Error> { let mut rows = stmt.query(params![])?; while let Some(row) = rows.next()? { let id: i32 = row.get(0)?; - let uuid: FromSqlUuid = row.get(1)?; + let uuid: SqlUuid = row.get(1)?; let uuid_bytes = &uuid.0.as_bytes()[..]; let short_name: String = row.get(2)?; let mut description: Option = row.get(3)?; @@ -134,6 +261,13 @@ pub fn run(_args: &super::Args, tx: &rusqlite::Transaction) -> Result<(), Error> alter table user add preferences text; alter table camera rename to old_camera; alter table stream rename to old_stream; + alter table signal rename to old_signal; + alter table meta rename to old_meta; + + create table meta ( + uuid blob not null check (length(uuid) = 16), + config text + ); create table camera ( id integer primary key, @@ -153,9 +287,25 @@ pub fn run(_args: &super::Args, tx: &rusqlite::Transaction) -> Result<(), Error> cum_runs integer not null check (cum_runs >= 0), unique (camera_id, type) ); + + create table signal ( + id integer primary key, + uuid blob unique not null check (length(uuid) = 16), + type_uuid blob not null references signal_type (uuid) + check (length(type_uuid) = 16), + config text + ); + + create table signal_type ( + uuid blob primary key check (length(uuid) = 16), + config text + ) without rowid; "#, )?; + copy_meta(tx)?; copy_cameras(tx)?; + copy_signal_types(tx)?; + copy_signals(tx)?; copy_streams(tx)?; tx.execute_batch( r#" @@ -224,6 +374,10 @@ pub fn run(_args: &super::Args, tx: &rusqlite::Transaction) -> Result<(), Error> drop table old_recording; drop table old_stream; drop table old_camera; + drop table old_meta; + drop table old_signal; + drop table signal_type_enum; + drop table signal_camera; "#, )?; Ok(()) diff --git a/server/src/json.rs b/server/src/json.rs index 86abfde..0d08df7 100644 --- a/server/src/json.rs +++ b/server/src/json.rs @@ -98,7 +98,7 @@ pub struct Signal<'a> { pub id: u32, #[serde(serialize_with = "Signal::serialize_cameras")] pub cameras: (&'a db::Signal, &'a db::LockedDatabase), - pub source: Uuid, + pub uuid: Uuid, pub type_: Uuid, pub short_name: &'a str, @@ -162,7 +162,7 @@ pub struct SignalType<'a> { #[derive(Debug, Serialize)] #[serde(rename_all = "camelCase")] pub struct SignalTypeState<'a> { - value: u16, + value: u8, name: &'a str, #[serde(skip_serializing_if = "Not::not")] @@ -273,9 +273,9 @@ impl<'a> Signal<'a> { Signal { id: s.id, cameras: (s, db), - source: s.source, + uuid: s.uuid, type_: s.type_, - short_name: &s.short_name, + short_name: &s.config.short_name, days: if include_days { Some(&s.days) } else { None }, } } @@ -288,16 +288,13 @@ impl<'a> Signal<'a> { S: Serializer, { let (s, db) = cameras; - let mut map = serializer.serialize_map(Some(s.cameras.len()))?; - for sc in &s.cameras { - let c = db.cameras_by_id().get(&sc.camera_id).ok_or_else(|| { - S::Error::custom(format!("signal has missing camera id {}", sc.camera_id)) + let mut map = serializer.serialize_map(Some(s.config.camera_associations.len()))?; + for (camera_id, association) in &s.config.camera_associations { + let c = db.cameras_by_id().get(camera_id).ok_or_else(|| { + S::Error::custom(format!("signal has missing camera id {}", camera_id)) })?; map.serialize_key(&c.uuid)?; - map.serialize_value(match sc.type_ { - db::signal::SignalCameraType::Direct => "direct", - db::signal::SignalCameraType::Indirect => "indirect", - })?; + map.serialize_value(association.as_str())?; } map.end() } @@ -339,21 +336,21 @@ impl<'a> SignalType<'a> { where S: Serializer, { - let mut seq = serializer.serialize_seq(Some(type_.states.len()))?; - for s in &type_.states { - seq.serialize_element(&SignalTypeState::wrap(s))?; + let mut seq = serializer.serialize_seq(Some(type_.config.values.len()))?; + for (&value, config) in &type_.config.values { + seq.serialize_element(&SignalTypeState::wrap(value, config))?; } seq.end() } } impl<'a> SignalTypeState<'a> { - pub fn wrap(s: &'a db::signal::TypeState) -> Self { + pub fn wrap(value: u8, config: &'a db::json::SignalTypeValueConfig) -> Self { SignalTypeState { - value: s.value, - name: &s.name, - motion: s.motion, - color: &s.color, + value, + name: &config.name, + motion: config.motion, + color: &config.color, } } }