MyFavMirrors/moonfire-nvr
1
0
Fork 0
mirror of https://github.com/scottlamb/moonfire-nvr.git synced 2025-03-31 01:33:42 -04:00
Code Issues Packages Projects Releases Wiki Activity

clean up old garbage files on v5 upgrade

Browse Source
This commit is contained in:
Scott Lamb 2019-07-24 22:18:44 -07:00
parent fe575e1b63
commit 54e06a5326
2 changed files with 82 additions and 27 deletions

7
db/upgrade/mod.rs

@ -224,6 +224,13 @@ mod tests {
if let Some(f) = fresh_sql {
compare(&upgraded, *ver, f)?;
}
if *ver == 3 {
// Check that the garbage files is cleaned up properly, but also add it back
// to simulate a bug prior to 433be217. The v5 upgrade should take care of
// anything left over.
assert!(!garbage.exists());
std::fs::File::create(&garbage)?;
}
}
// Check that recording files get renamed.

124
db/upgrade/v4_to_v5.rs

@ -37,57 +37,28 @@ use crate::db::FromSqlUuid;
use crate::{dir, schema};
use cstr::*;
use failure::{Error, Fail, bail};
use log::info;
use nix::fcntl::{FlockArg, OFlag};
use nix::sys::stat::Mode;
use protobuf::{Message, prelude::MessageField};
use rusqlite::params;
use std::io::{Read, Write};
use std::os::unix::io::AsRawFd;
use uuid::Uuid;
const FIXED_DIR_META_LEN: usize = 512;
pub fn run(_args: &super::Args, tx: &rusqlite::Transaction) -> Result<(), Error> {
let db_uuid: FromSqlUuid =
tx.query_row_and_then(r"select uuid from meta", params![], |row| row.get(0))?;
let mut stmt = tx.prepare(r#"
select
d.path,
d.uuid,
d.last_complete_open_id,
o.uuid
from
sample_file_dir d
left join open o on (d.last_complete_open_id = o.id);
"#)?;
let mut rows = stmt.query(params![])?;
while let Some(row) = rows.next()? {
let path = row.get_raw_checked(0)?.as_str()?;
let dir_uuid: FromSqlUuid = row.get(1)?;
let open_id: Option<u32> = row.get(2)?;
let open_uuid: Option<FromSqlUuid> = row.get(3)?;
let mut db_meta = schema::DirMeta::new();
db_meta.db_uuid.extend_from_slice(&db_uuid.0.as_bytes()[..]);
db_meta.dir_uuid.extend_from_slice(&dir_uuid.0.as_bytes()[..]);
match (open_id, open_uuid) {
(Some(id), Some(uuid)) => {
let mut o = db_meta.last_complete_open.mut_message();
o.id = id;
o.uuid.extend_from_slice(&uuid.0.as_bytes()[..]);
},
(None, None) => {},
_ => bail!("open table missing id"),
}
let dir = dir::Fd::open(path, false)?;
dir.lock(FlockArg::LockExclusiveNonblock)?;
/// Maybe upgrades the `meta` file, returning if an upgrade happened (and thus a sync is needed).
fn maybe_upgrade_meta(dir: &dir::Fd, db_meta: &schema::DirMeta) -> Result<bool, Error> {
let tmp_path = cstr!("meta.tmp");
let path = cstr!("meta");
let mut f = crate::fs::openat(dir.as_raw_fd(), path, OFlag::O_RDONLY, Mode::empty())?;
let meta_path = cstr!("meta");
let mut f = crate::fs::openat(dir.as_raw_fd(), meta_path, OFlag::O_RDONLY, Mode::empty())?;
let mut data = Vec::new();
f.read_to_end(&mut data)?;
if data.len() == FIXED_DIR_META_LEN {
continue; // already upgraded.
return Ok(false);
}
let mut s = protobuf::CodedInputStream::from_bytes(&data);
let mut dir_meta = schema::DirMeta::new();
dir_meta.merge_from(&mut s)
@ -107,8 +78,85 @@ pub fn run(_args: &super::Args, tx: &rusqlite::Transaction) -> Result<(), Error>
data.resize(FIXED_DIR_META_LEN, 0); // pad to required length.
f.write_all(&data)?;
f.sync_all()?;
nix::fcntl::renameat(Some(dir.as_raw_fd()), tmp_path, Some(dir.as_raw_fd()), path)?;
nix::fcntl::renameat(Some(dir.as_raw_fd()), tmp_path, Some(dir.as_raw_fd()), meta_path)?;
Ok(true)
}
/// Looks for uuid-based filenames and deletes them.
///
/// The v1->v3 migration failed to remove garbage files prior to 433be217. Let's have a clean slate
/// at v5.
///
/// Returns true if something was done (and thus a sync is needed).
fn maybe_cleanup_garbage_uuids(dir: &dir::Fd) -> Result<bool, Error> {
let mut need_sync = false;
let mut dir2 = nix::dir::Dir::openat(dir.as_raw_fd(), ".",
OFlag::O_DIRECTORY | OFlag::O_RDONLY, Mode::empty())?;
for e in dir2.iter() {
let e = e?;
let f = e.file_name();
info!("file: {}", f.to_str().unwrap());
let f_str = match f.to_str() {
Ok(f) => f,
Err(_) => continue,
};
if Uuid::parse_str(f_str).is_ok() {
info!("removing leftover garbage file {}", f_str);
nix::unistd::unlinkat(Some(dir.as_raw_fd()), f,
nix::unistd::UnlinkatFlags::NoRemoveDir)?;
need_sync = true;
}
}
Ok(need_sync)
}
pub fn run(_args: &super::Args, tx: &rusqlite::Transaction) -> Result<(), Error> {
let db_uuid: FromSqlUuid =
tx.query_row_and_then(r"select uuid from meta", params![], |row| row.get(0))?;
let mut stmt = tx.prepare(r#"
select
d.path,
d.uuid,
d.last_complete_open_id,
o.uuid
from
sample_file_dir d
left join open o on (d.last_complete_open_id = o.id);
"#)?;
let mut rows = stmt.query(params![])?;
while let Some(row) = rows.next()? {
let path = row.get_raw_checked(0)?.as_str()?;
info!("path: {}", path);
let dir_uuid: FromSqlUuid = row.get(1)?;
let open_id: Option<u32> = row.get(2)?;
let open_uuid: Option<FromSqlUuid> = row.get(3)?;
let mut db_meta = schema::DirMeta::new();
db_meta.db_uuid.extend_from_slice(&db_uuid.0.as_bytes()[..]);
db_meta.dir_uuid.extend_from_slice(&dir_uuid.0.as_bytes()[..]);
match (open_id, open_uuid) {
(Some(id), Some(uuid)) => {
let mut o = db_meta.last_complete_open.mut_message();
o.id = id;
o.uuid.extend_from_slice(&uuid.0.as_bytes()[..]);
},
(None, None) => {},
_ => bail!("open table missing id"),
}
let dir = dir::Fd::open(path, false)?;
dir.lock(FlockArg::LockExclusiveNonblock)?;
let mut need_sync = maybe_upgrade_meta(&dir, &db_meta)?;
if maybe_cleanup_garbage_uuids(&dir)? {
need_sync = true;
}
if need_sync {
dir.sync()?;
}
info!("done with path: {}", path);
}
Ok(())
}