MyFavMirrors/moonfire-nvr
1
0
Fork 0
mirror of https://github.com/scottlamb/moonfire-nvr.git synced 2025-11-24 19:46:17 -05:00
Code Issues Packages Projects Releases Wiki Activity

use Blake3 instead of SHA-1 or Blake2b

Browse Source 
Benefits:

* Blake3 is faster. This is most noticeable for the hashing of the
  sample file data.
* we no longer need OpenSSL, which helps with shrinking the binary size
  (#70). sha1 basically forced OpenSSL usage; ring deliberately doesn't
  support this old algorithm, and the pure-Rust sha1 crate is painfully
  slow. OpenSSL might still be a better choice than ring/rustls for TLS
  but it's nice to have the option.

For the video sample entries, I decided we don't need to hash at all. I
think the id number is sufficiently stable, and it's okay---perhaps even
desirable---if an existing init segment changes for fixes like e5b83c2.
This commit is contained in:
Scott Lamb
2020-03-20 20:52:30 -07:00
parent e5b83c21e1
commit 00991733f2
18 changed files with 248 additions and 187 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
db/Cargo.toml
View File

@@ -14,7 +14,7 @@ path = "lib.rs"
[dependencies]
base = { package = "moonfire-base", path = "../base" }
base64 = "0.11.0"
blake2-rfc = "0.2.18"
blake3 = "0.2.2"
byteorder = "1.0"
cstr = "0.1.7"
failure = "0.1.1"
@@ -28,11 +28,11 @@ lru-cache = "0.1"
mylog = { git = "https://github.com/scottlamb/mylog" }
nix = "0.16.1"
odds = { version = "0.3.1", features = ["std-vec"] }
openssl = "0.10"
parking_lot = { version = "0.9", features = [] }
prettydiff = "0.3.1"
protobuf = { git = "https://github.com/stepancheg/rust-protobuf" }
regex = "1.0"
ring = "0.14.6"
rusqlite = "0.21.0"
smallvec = "1.0"
tempdir = "0.3"

33
db/auth.rs
View File

@@ -30,7 +30,6 @@
use log::info;
use base::strutil;
use blake2_rfc::blake2b::blake2b;
use crate::schema::Permissions;
use failure::{Error, bail, format_err};
use fnv::FnvHashMap;
@@ -38,6 +37,7 @@ use lazy_static::lazy_static;
use libpasta;
use parking_lot::Mutex;
use protobuf::Message;
use ring::rand::{SecureRandom, SystemRandom};
use rusqlite::{Connection, Transaction, params};
use std::collections::BTreeMap;
use std::fmt;
@@ -204,6 +204,7 @@ pub enum SessionFlags {
#[derive(Copy, Clone)]
pub enum RevocationReason {
LoggedOut = 1,
AlgorithmChange = 2,
}
#[derive(Debug, Default)]
@@ -230,9 +231,9 @@ pub struct Session {
impl Session {
pub fn csrf(&self) -> SessionHash {
let r = blake2b(24, b"csrf", &self.seed.0[..]);
let r = blake3::keyed_hash(&self.seed.0, b"csrf");
let mut h = SessionHash([0u8; 24]);
h.0.copy_from_slice(r.as_bytes());
h.0.copy_from_slice(&r.as_bytes()[0..24]);
h
}
}
@@ -253,9 +254,9 @@ impl RawSessionId {
}
pub fn hash(&self) -> SessionHash {
let r = blake2b(24, &[], &self.0[..]);
let r = blake3::hash(&self.0[..]);
let mut h = SessionHash([0u8; 24]);
h.0.copy_from_slice(r.as_bytes());
h.0.copy_from_slice(&r.as_bytes()[0..24]);
h
}
}
@@ -276,8 +277,8 @@ impl fmt::Debug for RawSessionId {
/// A Blake2b-256 (48 bytes) of data associated with the session.
/// This is currently used in two ways:
/// * the csrf token is a blake2b drived from the session's seed. This is put into the `sc`
/// cookie.
/// * the csrf token is a truncated blake3 derived from the session's seed. This is put into the
/// `sc` cookie.
/// * the 48-byte session id is hashed to be used as a database key.
#[derive(Copy, Clone, Default, PartialEq, Eq, Hash)]
pub struct SessionHash(pub [u8; 24]);
@@ -333,6 +334,8 @@ pub(crate) struct State {
/// evict the oldest when its size exceeds a threshold. Or just evict everything on every flush
/// (and accept more frequent database accesses).
sessions: FnvHashMap<SessionHash, Session>,
rand: SystemRandom,
}
impl State {
@@ -341,6 +344,7 @@ impl State {
users_by_id: BTreeMap::new(),
users_by_name: BTreeMap::new(),
sessions: FnvHashMap::default(),
rand: ring::rand::SystemRandom::new(),
};
let mut stmt = conn.prepare(r#"
select
@@ -525,7 +529,7 @@ impl State {
u.dirty = true;
}
let password_id = u.password_id;
State::make_session_int(conn, req, u, domain, Some(password_id), session_flags,
State::make_session_int(&self.rand, conn, req, u, domain, Some(password_id), session_flags,
&mut self.sessions, u.permissions.clone())
}
@@ -537,19 +541,20 @@ impl State {
if u.disabled() {
bail!("user is disabled");
}
State::make_session_int(conn, creation, u, domain, None, flags, &mut self.sessions,
permissions)
State::make_session_int(&self.rand, conn, creation, u, domain, None, flags,
&mut self.sessions, permissions)
}
fn make_session_int<'s>(conn: &Connection, creation: Request, user: &mut User,
domain: Option<Vec<u8>>, creation_password_id: Option<i32>, flags: i32,
fn make_session_int<'s>(rand: &SystemRandom, conn: &Connection, creation: Request,
user: &mut User, domain: Option<Vec<u8>>,
creation_password_id: Option<i32>, flags: i32,
sessions: &'s mut FnvHashMap<SessionHash, Session>,
permissions: Permissions)
-> Result<(RawSessionId, &'s Session), Error> {
let mut session_id = RawSessionId::new();
::openssl::rand::rand_bytes(&mut session_id.0).unwrap();
rand.fill(&mut session_id.0).unwrap();
let mut seed = [0u8; 32];
::openssl::rand::rand_bytes(&mut seed).unwrap();
rand.fill(&mut seed).unwrap();
let hash = session_id.hash();
let mut stmt = conn.prepare_cached(r#"
insert into user_session (session_id_hash, user_id, seed, flags, domain,

40
db/db.rs
View File

@@ -97,9 +97,9 @@ const GET_RECORDING_PLAYBACK_SQL: &'static str = r#"
"#;
const INSERT_VIDEO_SAMPLE_ENTRY_SQL: &'static str = r#"
insert into video_sample_entry (sha1, width, height, pasp_h_spacing, pasp_v_spacing,
insert into video_sample_entry (width, height, pasp_h_spacing, pasp_v_spacing,
rfc6381_codec, data)
values (:sha1, :width, :height, :pasp_h_spacing, :pasp_v_spacing,
values (:width, :height, :pasp_h_spacing, :pasp_v_spacing,
:rfc6381_codec, :data)
"#;
@@ -129,7 +129,6 @@ impl rusqlite::types::FromSql for VideoIndex {
#[derive(Debug)]
pub struct VideoSampleEntry {
pub id: i32,
pub sha1: [u8; 20],
// Fields matching VideoSampleEntryToInsert below.
@@ -234,7 +233,7 @@ pub struct RecordingToInsert {
pub video_sync_samples: i32,
pub video_sample_entry_id: i32,
pub video_index: Vec<u8>,
pub sample_file_sha1: [u8; 20],
pub sample_file_blake3: Option<[u8; 32]>,
}
impl RecordingToInsert {
@@ -1358,7 +1357,6 @@ impl LockedDatabase {
let mut stmt = self.conn.prepare(r#"
select
id,
sha1,
width,
height,
pasp_h_spacing,
@@ -1371,23 +1369,16 @@ impl LockedDatabase {
let mut rows = stmt.query(params![])?;
while let Some(row) = rows.next()? {
let id = row.get(0)?;
let mut sha1 = [0u8; 20];
let sha1_vec: Vec<u8> = row.get(1)?;
if sha1_vec.len() != 20 {
bail!("video sample entry id {} has sha1 {} of wrong length", id, sha1_vec.len());
}
sha1.copy_from_slice(&sha1_vec);
let data: Vec<u8> = row.get(7)?;
let data: Vec<u8> = row.get(6)?;
self.video_sample_entries_by_id.insert(id, Arc::new(VideoSampleEntry {
id,
width: row.get::<_, i32>(2)?.try_into()?,
height: row.get::<_, i32>(3)?.try_into()?,
pasp_h_spacing: row.get::<_, i32>(4)?.try_into()?,
pasp_v_spacing: row.get::<_, i32>(5)?.try_into()?,
sha1,
width: row.get::<_, i32>(1)?.try_into()?,
height: row.get::<_, i32>(2)?.try_into()?,
pasp_h_spacing: row.get::<_, i32>(3)?.try_into()?,
pasp_v_spacing: row.get::<_, i32>(4)?.try_into()?,
data,
rfc6381_codec: row.get(6)?,
rfc6381_codec: row.get(5)?,
}));
}
info!("Loaded {} video sample entries",
@@ -1532,19 +1523,16 @@ impl LockedDatabase {
/// On success, returns the id of a new or existing row.
pub fn insert_video_sample_entry(&mut self, entry: VideoSampleEntryToInsert)
-> Result<i32, Error> {
let sha1 = crate::sha1(&entry.data)?;
// Check if it already exists.
// There shouldn't be too many entries, so it's fine to enumerate everything.
for (&id, v) in &self.video_sample_entries_by_id {
if v.sha1 == sha1 {
// A hash collision (different data with the same hash) is unlikely.
// The other fields are derived from data, so differences there indicate a bug.
if v.data == entry.data {
// The other fields are derived from data, so differences indicate a bug.
if v.width != entry.width || v.height != entry.height ||
v.pasp_h_spacing != entry.pasp_h_spacing ||
v.pasp_v_spacing != entry.pasp_v_spacing {
bail!("video_sample_entry SHA-1 {} mismatch: existing entry {:?}, new {:?}",
base::strutil::hex(&sha1[..]), v, &entry);
bail!("video_sample_entry id {}: existing entry {:?}, new {:?}", id, v, &entry);
}
return Ok(id);
}
@@ -1552,7 +1540,6 @@ impl LockedDatabase {
let mut stmt = self.conn.prepare_cached(INSERT_VIDEO_SAMPLE_ENTRY_SQL)?;
stmt.execute_named(named_params!{
":sha1": &sha1[..],
":width": i32::from(entry.width),
":height": i32::from(entry.height),
":pasp_h_spacing": i32::from(entry.pasp_h_spacing),
@@ -1568,7 +1555,6 @@ impl LockedDatabase {
height: entry.height,
pasp_h_spacing: entry.pasp_h_spacing,
pasp_v_spacing: entry.pasp_v_spacing,
sha1,
data: entry.data,
rfc6381_codec: entry.rfc6381_codec,
}));
@@ -2356,7 +2342,7 @@ mod tests {
video_sync_samples: 1,
video_sample_entry_id: vse_id,
video_index: [0u8; 100].to_vec(),
sample_file_sha1: [0u8; 20],
sample_file_blake3: None,
};
let id = {
let mut db = db.lock();

9
db/lib.rs
View File

@@ -51,12 +51,3 @@ pub mod testutil;
pub use crate::db::*;
pub use crate::schema::Permissions;
pub use crate::signal::Signal;
use openssl::hash;
fn sha1(input: &[u8]) -> Result<[u8; 20], failure::Error> {
let sha1 = hash::hash(hash::MessageDigest::sha1(), &input)?;
let mut sha1_bytes = [0u8; 20];
sha1_bytes.copy_from_slice(&sha1);
Ok(sha1_bytes)
}

8
db/raw.rs
View File

@@ -200,10 +200,10 @@ pub(crate) fn insert_recording(tx: &rusqlite::Transaction, o: &db::Open, id: Com
id, r, e))?;
let mut stmt = tx.prepare_cached(r#"
insert into recording_integrity (composite_id, local_time_delta_90k, sample_file_sha1)
values (:composite_id, :local_time_delta_90k, :sample_file_sha1)
insert into recording_integrity (composite_id, local_time_delta_90k, sample_file_blake3)
values (:composite_id, :local_time_delta_90k, :sample_file_blake3)
"#).with_context(|e| format!("can't prepare recording_integrity insert: {}", e))?;
let sha1 = &r.sample_file_sha1[..];
let blake3 = r.sample_file_blake3.as_ref().map(|b| &b[..]);
let delta = match r.run_offset {
0 => None,
_ => Some(r.local_time_delta.0),
@@ -211,7 +211,7 @@ pub(crate) fn insert_recording(tx: &rusqlite::Transaction, o: &db::Open, id: Com
stmt.execute_named(named_params!{
":composite_id": id.0,
":local_time_delta_90k": delta,
":sample_file_sha1": sha1,
":sample_file_blake3": blake3,
}).with_context(|e| format!("unable to insert recording_integrity for {:#?}: {}", r, e))?;
let mut stmt = tx.prepare_cached(r#"

19
db/schema.sql
View File

@@ -258,8 +258,9 @@ create table recording_integrity (
-- TODO: fill this in!
wall_time_delta_90k integer,
-- The sha1 hash of the contents of the sample file.
sample_file_sha1 blob check (length(sample_file_sha1) <= 20)
-- The (possibly truncated) raw blake3 hash of the contents of the sample
-- file.
sample_file_blake3 blob check (length(sample_file_blake3) <= 32)
);
-- Large fields for a recording which are needed ony for playback.
@@ -299,11 +300,8 @@ create table garbage (
create table video_sample_entry (
id integer primary key,
-- A SHA-1 hash of |bytes|.
sha1 blob unique not null check (length(sha1) = 20),
-- The width and height in pixels; must match values within
-- |sample_entry_bytes|.
-- `sample_entry_bytes`.
width integer not null check (width > 0),
height integer not null check (height > 0),
@@ -354,10 +352,10 @@ create table user (
-- elsewhere), which holds the session id and an encrypted sequence number for
-- replay protection.
create table user_session (
-- The session id is a 48-byte blob. This is the unencoded, unsalted Blake2b-192
-- (24 bytes) of the unencoded session id. Much like `password_hash`, a
-- hash is used here so that a leaked database backup can't be trivially used
-- to steal credentials.
-- The session id is a 48-byte blob. This is the unsalted Blake3 (32 bytes)
-- of the unencoded session id. Much like `password_hash`, a hash is used here
-- so that a leaked database backup can't be trivially used to steal
-- credentials.
session_id_hash blob primary key not null,
user_id integer references user (id) not null,
@@ -395,6 +393,7 @@ create table user_session (
-- A value indicating the reason for revocation, with optional additional
-- text detail. Enumeration values:
-- 0: logout link clicked (i.e. from within the session itself)
-- 1: obsoleted by a change in hashing algorithm (eg schema 5->6 upgrade)
--
-- This might be extended for a variety of other reasons:
-- x: user revoked (while authenticated in another way)

15
db/upgrade/mod.rs
View File

@@ -220,19 +220,16 @@ mod tests {
"#)?;
upgraded.execute(r#"
insert into video_sample_entry (id, sha1, width, height, data)
values (1, ?, 1920, 1080, ?);
"#, params![&crate::sha1(testutil::TEST_VIDEO_SAMPLE_ENTRY_DATA).unwrap()[..],
testutil::TEST_VIDEO_SAMPLE_ENTRY_DATA])?;
values (1, X'0000000000000000000000000000000000000000', 1920, 1080, ?);
"#, params![testutil::TEST_VIDEO_SAMPLE_ENTRY_DATA])?;
upgraded.execute(r#"
insert into video_sample_entry (id, sha1, width, height, data)
values (2, ?, 320, 240, ?);
"#, params![&crate::sha1(BAD_ANAMORPHIC_VIDEO_SAMPLE_ENTRY).unwrap()[..],
BAD_ANAMORPHIC_VIDEO_SAMPLE_ENTRY])?;
values (2, X'0000000000000000000000000000000000000001', 320, 240, ?);
"#, params![BAD_ANAMORPHIC_VIDEO_SAMPLE_ENTRY])?;
upgraded.execute(r#"
insert into video_sample_entry (id, sha1, width, height, data)
values (3, ?, 704, 480, ?);
"#, params![&crate::sha1(GOOD_ANAMORPHIC_VIDEO_SAMPLE_ENTRY).unwrap()[..],
GOOD_ANAMORPHIC_VIDEO_SAMPLE_ENTRY])?;
values (3, X'0000000000000000000000000000000000000002', 704, 480, ?);
"#, params![GOOD_ANAMORPHIC_VIDEO_SAMPLE_ENTRY])?;
upgraded.execute_batch(r#"
insert into recording (id, camera_id, sample_file_bytes, start_time_90k, duration_90k,
local_time_delta_90k, video_samples, video_sync_samples,

107
db/upgrade/v5_to_v6.rs
View File

@@ -74,33 +74,42 @@ fn parse(data: &[u8]) -> Result<AvcDecoderConfigurationRecord, Error> {
pub fn run(_args: &super::Args, tx: &rusqlite::Transaction) -> Result<(), Error> {
// These create statements match the schema.sql when version 5 was the latest.
tx.execute_batch(r#"
alter table video_sample_entry add column pasp_h_spacing integer not null default 1 check (pasp_h_spacing > 0);
alter table video_sample_entry add column pasp_v_spacing integer not null default 1 check (pasp_v_spacing > 0);
alter table video_sample_entry rename to old_video_sample_entry;
create table video_sample_entry (
id integer primary key,
width integer not null check (width > 0),
height integer not null check (height > 0),
rfc6381_codec text not null,
data blob not null check (length(data) > 86),
pasp_h_spacing integer not null default 1 check (pasp_h_spacing > 0),
pasp_v_spacing integer not null default 1 check (pasp_v_spacing > 0)
);
"#)?;
let mut update = tx.prepare(r#"
update video_sample_entry
set data = :data,
sha1 = :sha1,
pasp_h_spacing = :pasp_h_spacing,
pasp_v_spacing = :pasp_v_spacing
where id = :id
let mut insert = tx.prepare(r#"
insert into video_sample_entry (id, width, height, rfc6381_codec, data,
pasp_h_spacing, pasp_v_spacing)
values (:id, :width, :height, :rfc6381_codec, :data,
:pasp_h_spacing, :pasp_v_spacing)
"#)?;
let mut stmt = tx.prepare(r#"
select
id,
width,
height,
rfc6381_codec,
data
from
video_sample_entry
old_video_sample_entry
"#)?;
let mut rows = stmt.query(params![])?;
while let Some(row) = rows.next()? {
let id: i32 = row.get(0)?;
let width: u16 = row.get::<_, i32>(1)?.try_into()?;
let height: u16 = row.get::<_, i32>(2)?.try_into()?;
let mut data: Vec<u8> = row.get(3)?;
let rfc6381_codec: &str = row.get_raw_checked(3)?.as_str()?;
let mut data: Vec<u8> = row.get(4)?;
let avcc = parse(&data)?;
if avcc.num_of_sequence_parameter_sets() != 1 {
bail!("Multiple SPSs!");
@@ -121,13 +130,85 @@ pub fn run(_args: &super::Args, tx: &rusqlite::Transaction) -> Result<(), Error>
BigEndian::write_u32(&mut data[0..4], u32::try_from(len)?);
}
update.execute_named(named_params!{
insert.execute_named(named_params!{
":id": id,
":width": width,
":height": height,
":rfc6381_codec": rfc6381_codec,
":data": &data,
":sha1": &crate::sha1(&data)?[..],
":pasp_h_spacing": pasp.0,
":pasp_v_spacing": pasp.1,
})?;
}
tx.execute_batch(r#"
alter table recording rename to old_recording;
create table recording (
composite_id integer primary key,
open_id integer not null,
stream_id integer not null references stream (id),
run_offset integer not null,
flags integer not null,
sample_file_bytes integer not null check (sample_file_bytes > 0),
start_time_90k integer not null check (start_time_90k > 0),
duration_90k integer not null
check (duration_90k >= 0 and duration_90k < 5*60*90000),
video_samples integer not null check (video_samples > 0),
video_sync_samples integer not null check (video_sync_samples > 0),
video_sample_entry_id integer references video_sample_entry (id),
check (composite_id >> 32 = stream_id)
);
insert into recording select * from old_recording;
drop index recording_cover;
create index recording_cover on recording (
stream_id,
start_time_90k,
open_id,
duration_90k,
video_samples,
video_sync_samples,
video_sample_entry_id,
sample_file_bytes,
run_offset,
flags
);
alter table recording_integrity rename to old_recording_integrity;
create table recording_integrity (
composite_id integer primary key references recording (composite_id),
local_time_delta_90k integer,
local_time_since_open_90k integer,
wall_time_delta_90k integer,
sample_file_blake3 blob check (length(sample_file_blake3) <= 32)
);
insert into recording_integrity
select
composite_id,
local_time_delta_90k,
local_time_since_open_90k,
wall_time_delta_90k,
null
from
old_recording_integrity;
alter table recording_playback rename to old_recording_playback;
create table recording_playback (
composite_id integer primary key references recording (composite_id),
video_index blob not null check (length(video_index) > 0)
);
insert into recording_playback select * from old_recording_playback;
drop table old_recording_playback;
drop table old_recording_integrity;
drop table old_recording;
drop table old_video_sample_entry;
update user_session
set
revocation_reason = 1,
revocation_reason_detail = 'Blake2b->Blake3 upgrade'
where
revocation_reason is null;
"#)?;
Ok(())
}

12
db/writer.rs
View File

@@ -40,7 +40,6 @@ use failure::{Error, bail, format_err};
use fnv::FnvHashMap;
use parking_lot::Mutex;
use log::{debug, trace, warn};
use openssl::hash;
use std::cmp::Ordering;
use std::cmp;
use std::io;
@@ -563,7 +562,7 @@ struct InnerWriter<F: FileWriter> {
/// segments have been sent out. Initially 0.
completed_live_segment_off_90k: i32,
hasher: hash::Hasher,
hasher: blake3::Hasher,
/// The start time of this segment, based solely on examining the local clock after frames in
/// this segment were received. Frames can suffer from various kinds of delay (initial
@@ -688,7 +687,7 @@ impl<'a, C: Clocks + Clone, D: DirWriter> Writer<'a, C, D> {
e: recording::SampleIndexEncoder::new(),
id,
completed_live_segment_off_90k: 0,
hasher: hash::Hasher::new(hash::MessageDigest::sha1())?,
hasher: blake3::Hasher::new(),
local_start: recording::Time(i64::max_value()),
adjuster: ClockAdjuster::new(prev.map(|p| p.local_time_delta.0)),
unflushed_sample: None,
@@ -757,7 +756,7 @@ impl<'a, C: Clocks + Clone, D: DirWriter> Writer<'a, C, D> {
len: pkt.len() as i32,
is_key,
});
w.hasher.update(pkt).unwrap();
w.hasher.update(pkt);
Ok(())
}
@@ -797,8 +796,7 @@ impl<F: FileWriter> InnerWriter<F> {
None => (self.adjuster.adjust(0), db::RecordingFlags::TrailingZero as i32),
Some(p) => (self.adjuster.adjust((p - unflushed.pts_90k) as i32), 0),
};
let mut sha1_bytes = [0u8; 20];
sha1_bytes.copy_from_slice(&self.hasher.finish().unwrap()[..]);
let blake3 = self.hasher.finalize();
let (local_time_delta, run_offset, end);
let d = self.add_sample(last_sample_duration, unflushed.len, unflushed.is_key,
unflushed.local_time)?;
@@ -814,7 +812,7 @@ impl<F: FileWriter> InnerWriter<F> {
l.flags = flags;
local_time_delta = self.local_start - l.start;
l.local_time_delta = local_time_delta;
l.sample_file_sha1 = sha1_bytes;
l.sample_file_blake3 = Some(blake3.as_bytes().clone());
total_duration = recording::Duration(l.duration_90k as i64);
run_offset = l.run_offset;
end = l.start + total_duration;