diff --git a/db/upgrade/mod.rs b/db/upgrade/mod.rs index e9b72f9..dbb0eaf 100644 --- a/db/upgrade/mod.rs +++ b/db/upgrade/mod.rs @@ -224,6 +224,13 @@ mod tests { if let Some(f) = fresh_sql { compare(&upgraded, *ver, f)?; } + if *ver == 3 { + // Check that the garbage files is cleaned up properly, but also add it back + // to simulate a bug prior to 433be217. The v5 upgrade should take care of + // anything left over. + assert!(!garbage.exists()); + std::fs::File::create(&garbage)?; + } } // Check that recording files get renamed. diff --git a/db/upgrade/v4_to_v5.rs b/db/upgrade/v4_to_v5.rs index b29d894..8ddfa13 100644 --- a/db/upgrade/v4_to_v5.rs +++ b/db/upgrade/v4_to_v5.rs @@ -37,15 +37,81 @@ use crate::db::FromSqlUuid; use crate::{dir, schema}; use cstr::*; use failure::{Error, Fail, bail}; +use log::info; use nix::fcntl::{FlockArg, OFlag}; use nix::sys::stat::Mode; use protobuf::{Message, prelude::MessageField}; use rusqlite::params; use std::io::{Read, Write}; use std::os::unix::io::AsRawFd; +use uuid::Uuid; const FIXED_DIR_META_LEN: usize = 512; +/// Maybe upgrades the `meta` file, returning if an upgrade happened (and thus a sync is needed). +fn maybe_upgrade_meta(dir: &dir::Fd, db_meta: &schema::DirMeta) -> Result { + let tmp_path = cstr!("meta.tmp"); + let meta_path = cstr!("meta"); + let mut f = crate::fs::openat(dir.as_raw_fd(), meta_path, OFlag::O_RDONLY, Mode::empty())?; + let mut data = Vec::new(); + f.read_to_end(&mut data)?; + if data.len() == FIXED_DIR_META_LEN { + return Ok(false); + } + + let mut s = protobuf::CodedInputStream::from_bytes(&data); + let mut dir_meta = schema::DirMeta::new(); + dir_meta.merge_from(&mut s) + .map_err(|e| e.context("Unable to parse metadata proto: {}"))?; + if !dir::SampleFileDir::consistent(&db_meta, &dir_meta) { + bail!("Inconsistent db_meta={:?} dir_meta={:?}", &db_meta, &dir_meta); + } + let mut f = crate::fs::openat(dir.as_raw_fd(), tmp_path, + OFlag::O_CREAT | OFlag::O_TRUNC | OFlag::O_WRONLY, + Mode::S_IRUSR | Mode::S_IWUSR)?; + let mut data = + dir_meta.write_length_delimited_to_bytes().expect("proto3->vec is infallible"); + if data.len() > FIXED_DIR_META_LEN { + bail!("Length-delimited DirMeta message requires {} bytes, over limit of {}", + data.len(), FIXED_DIR_META_LEN); + } + data.resize(FIXED_DIR_META_LEN, 0); // pad to required length. + f.write_all(&data)?; + f.sync_all()?; + + nix::fcntl::renameat(Some(dir.as_raw_fd()), tmp_path, Some(dir.as_raw_fd()), meta_path)?; + Ok(true) +} + +/// Looks for uuid-based filenames and deletes them. +/// +/// The v1->v3 migration failed to remove garbage files prior to 433be217. Let's have a clean slate +/// at v5. +/// +/// Returns true if something was done (and thus a sync is needed). +fn maybe_cleanup_garbage_uuids(dir: &dir::Fd) -> Result { + let mut need_sync = false; + let mut dir2 = nix::dir::Dir::openat(dir.as_raw_fd(), ".", + OFlag::O_DIRECTORY | OFlag::O_RDONLY, Mode::empty())?; + for e in dir2.iter() { + let e = e?; + let f = e.file_name(); + info!("file: {}", f.to_str().unwrap()); + let f_str = match f.to_str() { + Ok(f) => f, + Err(_) => continue, + }; + if Uuid::parse_str(f_str).is_ok() { + info!("removing leftover garbage file {}", f_str); + nix::unistd::unlinkat(Some(dir.as_raw_fd()), f, + nix::unistd::UnlinkatFlags::NoRemoveDir)?; + need_sync = true; + } + } + + Ok(need_sync) +} + pub fn run(_args: &super::Args, tx: &rusqlite::Transaction) -> Result<(), Error> { let db_uuid: FromSqlUuid = tx.query_row_and_then(r"select uuid from meta", params![], |row| row.get(0))?; @@ -62,6 +128,7 @@ pub fn run(_args: &super::Args, tx: &rusqlite::Transaction) -> Result<(), Error> let mut rows = stmt.query(params![])?; while let Some(row) = rows.next()? { let path = row.get_raw_checked(0)?.as_str()?; + info!("path: {}", path); let dir_uuid: FromSqlUuid = row.get(1)?; let open_id: Option = row.get(2)?; let open_uuid: Option = row.get(3)?; @@ -80,35 +147,16 @@ pub fn run(_args: &super::Args, tx: &rusqlite::Transaction) -> Result<(), Error> let dir = dir::Fd::open(path, false)?; dir.lock(FlockArg::LockExclusiveNonblock)?; - let tmp_path = cstr!("meta.tmp"); - let path = cstr!("meta"); - let mut f = crate::fs::openat(dir.as_raw_fd(), path, OFlag::O_RDONLY, Mode::empty())?; - let mut data = Vec::new(); - f.read_to_end(&mut data)?; - if data.len() == FIXED_DIR_META_LEN { - continue; // already upgraded. + + let mut need_sync = maybe_upgrade_meta(&dir, &db_meta)?; + if maybe_cleanup_garbage_uuids(&dir)? { + need_sync = true; } - let mut s = protobuf::CodedInputStream::from_bytes(&data); - let mut dir_meta = schema::DirMeta::new(); - dir_meta.merge_from(&mut s) - .map_err(|e| e.context("Unable to parse metadata proto: {}"))?; - if !dir::SampleFileDir::consistent(&db_meta, &dir_meta) { - bail!("Inconsistent db_meta={:?} dir_meta={:?}", &db_meta, &dir_meta); + + if need_sync { + dir.sync()?; } - let mut f = crate::fs::openat(dir.as_raw_fd(), tmp_path, - OFlag::O_CREAT | OFlag::O_TRUNC | OFlag::O_WRONLY, - Mode::S_IRUSR | Mode::S_IWUSR)?; - let mut data = - dir_meta.write_length_delimited_to_bytes().expect("proto3->vec is infallible"); - if data.len() > FIXED_DIR_META_LEN { - bail!("Length-delimited DirMeta message requires {} bytes, over limit of {}", - data.len(), FIXED_DIR_META_LEN); - } - data.resize(FIXED_DIR_META_LEN, 0); // pad to required length. - f.write_all(&data)?; - f.sync_all()?; - nix::fcntl::renameat(Some(dir.as_raw_fd()), tmp_path, Some(dir.as_raw_fd()), path)?; - dir.sync()?; + info!("done with path: {}", path); } Ok(()) }