mirror of
https://github.com/scottlamb/moonfire-nvr.git
synced 2024-12-25 06:35:56 -05:00
upgrade rusqlite, linked-hash-map/hashlink
With Rust 1.48.0, I was getting panics like "attempted to leave type `linked_hash_map::Node<std::string::String, raw_statement::RawStatement>` uninitialized, which is invalid". Looks like this is due to a bug in linked-hash-map 0.5.2, fixed with 0.5.3. Along the way, I see that rusqlite no longer uses this crate; it switched to hashlink. Upgrade rusqlite and match that change for my own use (video index LRU cache). I'm still pulling in linked-hash-map via serde_yaml. I didn't even realize I was using serde_yaml, but libpasta wants it. Seems a little silly to me but that's something I might explore another day.
This commit is contained in:
Scott Lamb
parent
d83bb1bf4d
commit
16825537b9
53
Cargo.lock
generated
53
Cargo.lock
generated
@ -15,6 +15,12 @@ dependencies = [
|
||||
"const-random",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "ahash"
|
||||
version = "0.4.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f6789e291be47ace86a60303502173d84af8327e3627ecf334356ee0f87a164c"
|
||||
|
||||
[[package]]
|
||||
name = "ansi_term"
|
||||
version = "0.9.0"
|
||||
@ -390,7 +396,7 @@ version = "0.14.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "341b03eec276c30c6cdc640d8bd8c08eac9605064c3f9c4838f958dac06973bb"
|
||||
dependencies = [
|
||||
"ahash",
|
||||
"ahash 0.2.18",
|
||||
"cfg-if",
|
||||
"chrono",
|
||||
"crossbeam-channel",
|
||||
@ -791,6 +797,24 @@ dependencies = [
|
||||
"tokio-util",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "hashbrown"
|
||||
version = "0.9.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d7afe4a420e3fe79967a00898cc1f4db7c8a49a9333a29f8a4bd76a253d5cd04"
|
||||
dependencies = [
|
||||
"ahash 0.4.6",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "hashlink"
|
||||
version = "0.6.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d99cf782f0dc4372d26846bec3de7804ceb5df083c2d4462c0b8d2330e894fa8"
|
||||
dependencies = [
|
||||
"hashbrown",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "heck"
|
||||
version = "0.3.1"
|
||||
@ -1035,9 +1059,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "libsqlite3-sys"
|
||||
version = "0.17.3"
|
||||
version = "0.20.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "56d90181c2904c287e5390186be820e5ef311a3c62edebb7d6ca3d6a48ce041d"
|
||||
checksum = "64d31059f22935e6c31830db5249ba2b7ecd54fd73a9909286f0a67aa55c2fbd"
|
||||
dependencies = [
|
||||
"cc",
|
||||
"pkg-config",
|
||||
@ -1046,9 +1070,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "linked-hash-map"
|
||||
version = "0.5.2"
|
||||
version = "0.5.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ae91b68aebc4ddb91978b11a1b02ddd8602a05ec19002801c5666000e05e0f83"
|
||||
checksum = "8dd5a6d5999d9907cda8ed67bbd137d3af8085216c2ac62de5be860bd41f304a"
|
||||
|
||||
[[package]]
|
||||
name = "lock_api"
|
||||
@ -1068,15 +1092,6 @@ dependencies = [
|
||||
"cfg-if",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "lru-cache"
|
||||
version = "0.1.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "31e24f1ad8321ca0e8a1e0ac13f23cb668e6f5466c2c57319f6a5cf1cc8e3b1c"
|
||||
dependencies = [
|
||||
"linked-hash-map",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "maplit"
|
||||
version = "1.0.2"
|
||||
@ -1200,12 +1215,12 @@ dependencies = [
|
||||
"cstr",
|
||||
"failure",
|
||||
"fnv",
|
||||
"hashlink",
|
||||
"itertools",
|
||||
"lazy_static",
|
||||
"libc",
|
||||
"libpasta",
|
||||
"log",
|
||||
"lru-cache",
|
||||
"moonfire-base",
|
||||
"mylog",
|
||||
"nix",
|
||||
@ -1925,17 +1940,17 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rusqlite"
|
||||
version = "0.22.0"
|
||||
version = "0.24.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "57edf4c4cea4d7e0fab069acb5da9e8e8e5403c78abc81b1f37d83af02148ea5"
|
||||
checksum = "7e3d4791ab5517217f51216a84a688b53c1ebf7988736469c538d02f46ddba68"
|
||||
dependencies = [
|
||||
"bitflags",
|
||||
"fallible-iterator",
|
||||
"fallible-streaming-iterator",
|
||||
"hashlink",
|
||||
"libsqlite3-sys",
|
||||
"lru-cache",
|
||||
"memchr",
|
||||
"time 0.1.43",
|
||||
"smallvec",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
|
||||
@ -47,7 +47,7 @@ parking_lot = { version = "0.10", features = [] }
|
||||
protobuf = { git = "https://github.com/stepancheg/rust-protobuf" }
|
||||
reffers = "0.6.0"
|
||||
ring = "0.14.6"
|
||||
rusqlite = "0.22.0"
|
||||
rusqlite = "0.24.1"
|
||||
serde = { version = "1.0", features = ["derive"] }
|
||||
serde_json = "1.0"
|
||||
smallvec = "1.0"
|
||||
|
||||
@ -18,11 +18,11 @@ blake2-rfc = "0.2.18"
|
||||
cstr = "0.1.7"
|
||||
failure = "0.1.1"
|
||||
fnv = "1.0"
|
||||
hashlink = "0.6.0"
|
||||
lazy_static = "1.0"
|
||||
libc = "0.2"
|
||||
libpasta = "0.1.0-rc2"
|
||||
log = "0.4"
|
||||
lru-cache = "0.1"
|
||||
mylog = { git = "https://github.com/scottlamb/mylog" }
|
||||
nix = "0.17.0"
|
||||
odds = { version = "0.4.0", features = ["std-vec"] }
|
||||
@ -30,7 +30,7 @@ openssl = "0.10"
|
||||
parking_lot = { version = "0.10", features = [] }
|
||||
prettydiff = "0.3.1"
|
||||
protobuf = { git = "https://github.com/stepancheg/rust-protobuf" }
|
||||
rusqlite = "0.22.0"
|
||||
rusqlite = "0.24.1"
|
||||
smallvec = "1.0"
|
||||
tempdir = "0.3"
|
||||
time = "0.1"
|
||||
|
||||
48
db/db.rs
48
db/db.rs
@ -62,9 +62,9 @@ use crate::schema;
|
||||
use crate::signal;
|
||||
use failure::{Error, bail, format_err};
|
||||
use fnv::{FnvHashMap, FnvHashSet};
|
||||
use hashlink::LinkedHashMap;
|
||||
use itertools::Itertools;
|
||||
use log::{error, info, trace};
|
||||
use lru_cache::LruCache;
|
||||
use openssl::hash;
|
||||
use parking_lot::{Mutex,MutexGuard};
|
||||
use protobuf::prelude::MessageField;
|
||||
@ -87,6 +87,11 @@ use uuid::Uuid;
|
||||
/// Expected schema version. See `guide/schema.md` for more information.
|
||||
pub const EXPECTED_VERSION: i32 = 5;
|
||||
|
||||
/// Length of the video index cache.
|
||||
/// The actual data structure is one bigger than this because we insert before we remove.
|
||||
/// Make it one less than a power of two so that the data structure's size is efficient.
|
||||
const VIDEO_INDEX_CACHE_LEN: usize = 1023;
|
||||
|
||||
const GET_RECORDING_PLAYBACK_SQL: &'static str = r#"
|
||||
select
|
||||
video_index
|
||||
@ -652,7 +657,7 @@ pub struct LockedDatabase {
|
||||
streams_by_id: BTreeMap<i32, Stream>,
|
||||
cameras_by_uuid: BTreeMap<Uuid, i32>, // values are ids.
|
||||
video_sample_entries_by_id: BTreeMap<i32, Arc<VideoSampleEntry>>,
|
||||
video_index_cache: RefCell<LruCache<i64, Box<[u8]>, fnv::FnvBuildHasher>>,
|
||||
video_index_cache: RefCell<LinkedHashMap<i64, Box<[u8]>, fnv::FnvBuildHasher>>,
|
||||
on_flush: Vec<Box<dyn Fn() + Send>>,
|
||||
}
|
||||
|
||||
@ -1339,20 +1344,30 @@ impl LockedDatabase {
|
||||
|
||||
// Committed path.
|
||||
let mut cache = self.video_index_cache.borrow_mut();
|
||||
if let Some(video_index) = cache.get_mut(&id.0) {
|
||||
trace!("cache hit for recording {}", id);
|
||||
return f(&RecordingPlayback { video_index });
|
||||
use hashlink::linked_hash_map::RawEntryMut;
|
||||
match cache.raw_entry_mut().from_key(&id.0) {
|
||||
RawEntryMut::Occupied(mut occupied) => {
|
||||
trace!("cache hit for recording {}", id);
|
||||
occupied.to_back();
|
||||
let video_index = occupied.get();
|
||||
return f(&RecordingPlayback { video_index });
|
||||
},
|
||||
RawEntryMut::Vacant(vacant) => {
|
||||
trace!("cache miss for recording {}", id);
|
||||
let mut stmt = self.conn.prepare_cached(GET_RECORDING_PLAYBACK_SQL)?;
|
||||
let mut rows = stmt.query_named(named_params!{":composite_id": id.0})?;
|
||||
if let Some(row) = rows.next()? {
|
||||
let video_index: VideoIndex = row.get(0)?;
|
||||
let result = f(&RecordingPlayback { video_index: &video_index.0[..] });
|
||||
vacant.insert(id.0, video_index.0);
|
||||
if cache.len() > VIDEO_INDEX_CACHE_LEN {
|
||||
cache.pop_front();
|
||||
}
|
||||
return result;
|
||||
}
|
||||
Err(format_err!("no such recording {}", id))
|
||||
},
|
||||
}
|
||||
trace!("cache miss for recording {}", id);
|
||||
let mut stmt = self.conn.prepare_cached(GET_RECORDING_PLAYBACK_SQL)?;
|
||||
let mut rows = stmt.query_named(named_params!{":composite_id": id.0})?;
|
||||
if let Some(row) = rows.next()? {
|
||||
let video_index: VideoIndex = row.get(0)?;
|
||||
let result = f(&RecordingPlayback { video_index: &video_index.0[..] });
|
||||
cache.insert(id.0, video_index.0);
|
||||
return result;
|
||||
}
|
||||
Err(format_err!("no such recording {}", id))
|
||||
}
|
||||
|
||||
/// Queues for deletion the oldest recordings that aren't already queued.
|
||||
@ -2020,7 +2035,8 @@ impl<C: Clocks + Clone> Database<C> {
|
||||
cameras_by_uuid: BTreeMap::new(),
|
||||
streams_by_id: BTreeMap::new(),
|
||||
video_sample_entries_by_id: BTreeMap::new(),
|
||||
video_index_cache: RefCell::new(LruCache::with_hasher(1024, Default::default())),
|
||||
video_index_cache: RefCell::new(LinkedHashMap::with_capacity_and_hasher(
|
||||
VIDEO_INDEX_CACHE_LEN + 1, Default::default())),
|
||||
on_flush: Vec::new(),
|
||||
})),
|
||||
clocks,
|
||||
|
||||
Loading…
Reference in New Issue
Block a user