clean up old garbage files on v5 upgrade

db/upgrade/mod.rs

@@ -224,6 +224,13 @@ mod tests {
             if let Some(f) = fresh_sql {
                 compare(&upgraded, *ver, f)?;
             }
+            if *ver == 3 {
+                // Check that the garbage files is cleaned up properly, but also add it back
+                // to simulate a bug prior to 433be217. The v5 upgrade should take care of
+                // anything left over.
+                assert!(!garbage.exists());
+                std::fs::File::create(&garbage)?;
+            }
         }
 
         // Check that recording files get renamed.

db/upgrade/v4_to_v5.rs

@@ -37,57 +37,28 @@ use crate::db::FromSqlUuid;
 use crate::{dir, schema};
 use cstr::*;
 use failure::{Error, Fail, bail};
+use log::info;
 use nix::fcntl::{FlockArg, OFlag};
 use nix::sys::stat::Mode;
 use protobuf::{Message, prelude::MessageField};
 use rusqlite::params;
 use std::io::{Read, Write};
 use std::os::unix::io::AsRawFd;
+use uuid::Uuid;
 
 const FIXED_DIR_META_LEN: usize = 512;
 
-pub fn run(_args: &super::Args, tx: &rusqlite::Transaction) -> Result<(), Error> {
+/// Maybe upgrades the `meta` file, returning if an upgrade happened (and thus a sync is needed).
-    let db_uuid: FromSqlUuid =
+fn maybe_upgrade_meta(dir: &dir::Fd, db_meta: &schema::DirMeta) -> Result<bool, Error> {
-        tx.query_row_and_then(r"select uuid from meta", params![], |row| row.get(0))?;
-    let mut stmt = tx.prepare(r#"
-        select
-          d.path,
-          d.uuid,
-          d.last_complete_open_id,
-          o.uuid
-        from
-          sample_file_dir d
-          left join open o on (d.last_complete_open_id = o.id);
-    "#)?;
-    let mut rows = stmt.query(params![])?;
-    while let Some(row) = rows.next()? {
-        let path = row.get_raw_checked(0)?.as_str()?;
-        let dir_uuid: FromSqlUuid = row.get(1)?;
-        let open_id: Option<u32> = row.get(2)?;
-        let open_uuid: Option<FromSqlUuid> = row.get(3)?;
-        let mut db_meta = schema::DirMeta::new();
-        db_meta.db_uuid.extend_from_slice(&db_uuid.0.as_bytes()[..]);
-        db_meta.dir_uuid.extend_from_slice(&dir_uuid.0.as_bytes()[..]);
-        match (open_id, open_uuid) {
-            (Some(id), Some(uuid)) => {
-                let mut o = db_meta.last_complete_open.mut_message();
-                o.id = id;
-                o.uuid.extend_from_slice(&uuid.0.as_bytes()[..]);
-            },
-            (None, None) => {},
-            _ => bail!("open table missing id"),
-        }
-
-        let dir = dir::Fd::open(path, false)?;
-        dir.lock(FlockArg::LockExclusiveNonblock)?;
     let tmp_path = cstr!("meta.tmp");
-        let path = cstr!("meta");
+    let meta_path = cstr!("meta");
-        let mut f = crate::fs::openat(dir.as_raw_fd(), path, OFlag::O_RDONLY, Mode::empty())?;
+    let mut f = crate::fs::openat(dir.as_raw_fd(), meta_path, OFlag::O_RDONLY, Mode::empty())?;
     let mut data = Vec::new();
     f.read_to_end(&mut data)?;
     if data.len() == FIXED_DIR_META_LEN {
-            continue;  // already upgraded.
+        return Ok(false);
     }
+
     let mut s = protobuf::CodedInputStream::from_bytes(&data);
     let mut dir_meta = schema::DirMeta::new();
     dir_meta.merge_from(&mut s)
@@ -107,8 +78,85 @@ pub fn run(_args: &super::Args, tx: &rusqlite::Transaction) -> Result<(), Error>
     data.resize(FIXED_DIR_META_LEN, 0);  // pad to required length.
     f.write_all(&data)?;
     f.sync_all()?;
-        nix::fcntl::renameat(Some(dir.as_raw_fd()), tmp_path, Some(dir.as_raw_fd()), path)?;
+
+    nix::fcntl::renameat(Some(dir.as_raw_fd()), tmp_path, Some(dir.as_raw_fd()), meta_path)?;
+    Ok(true)
+}
+
+/// Looks for uuid-based filenames and deletes them.
+///
+/// The v1->v3 migration failed to remove garbage files prior to 433be217. Let's have a clean slate
+/// at v5.
+///
+/// Returns true if something was done (and thus a sync is needed).
+fn maybe_cleanup_garbage_uuids(dir: &dir::Fd) -> Result<bool, Error> {
+    let mut need_sync = false;
+    let mut dir2 = nix::dir::Dir::openat(dir.as_raw_fd(), ".",
+                                         OFlag::O_DIRECTORY | OFlag::O_RDONLY, Mode::empty())?;
+    for e in dir2.iter() {
+        let e = e?;
+        let f = e.file_name();
+        info!("file: {}", f.to_str().unwrap());
+        let f_str = match f.to_str() {
+            Ok(f) => f,
+            Err(_) => continue,
+        };
+        if Uuid::parse_str(f_str).is_ok() {
+            info!("removing leftover garbage file {}", f_str);
+            nix::unistd::unlinkat(Some(dir.as_raw_fd()), f,
+                                  nix::unistd::UnlinkatFlags::NoRemoveDir)?;
+            need_sync = true;
+        }
+    }
+
+    Ok(need_sync)
+}
+
+pub fn run(_args: &super::Args, tx: &rusqlite::Transaction) -> Result<(), Error> {
+    let db_uuid: FromSqlUuid =
+        tx.query_row_and_then(r"select uuid from meta", params![], |row| row.get(0))?;
+    let mut stmt = tx.prepare(r#"
+        select
+          d.path,
+          d.uuid,
+          d.last_complete_open_id,
+          o.uuid
+        from
+          sample_file_dir d
+          left join open o on (d.last_complete_open_id = o.id);
+    "#)?;
+    let mut rows = stmt.query(params![])?;
+    while let Some(row) = rows.next()? {
+        let path = row.get_raw_checked(0)?.as_str()?;
+        info!("path: {}", path);
+        let dir_uuid: FromSqlUuid = row.get(1)?;
+        let open_id: Option<u32> = row.get(2)?;
+        let open_uuid: Option<FromSqlUuid> = row.get(3)?;
+        let mut db_meta = schema::DirMeta::new();
+        db_meta.db_uuid.extend_from_slice(&db_uuid.0.as_bytes()[..]);
+        db_meta.dir_uuid.extend_from_slice(&dir_uuid.0.as_bytes()[..]);
+        match (open_id, open_uuid) {
+            (Some(id), Some(uuid)) => {
+                let mut o = db_meta.last_complete_open.mut_message();
+                o.id = id;
+                o.uuid.extend_from_slice(&uuid.0.as_bytes()[..]);
+            },
+            (None, None) => {},
+            _ => bail!("open table missing id"),
+        }
+
+        let dir = dir::Fd::open(path, false)?;
+        dir.lock(FlockArg::LockExclusiveNonblock)?;
+
+        let mut need_sync = maybe_upgrade_meta(&dir, &db_meta)?;
+        if maybe_cleanup_garbage_uuids(&dir)? {
+            need_sync = true;
+        }
+
+        if need_sync {
             dir.sync()?;
         }
+        info!("done with path: {}", path);
+    }
     Ok(())
 }