flexible config for sample_file_dir

This commit is contained in:
Scott Lamb
2021-10-26 11:47:13 -07:00
parent dad349840d
commit 721141770f
13 changed files with 114 additions and 43 deletions

View File

@@ -27,6 +27,7 @@ use std::fs;
use std::io::{Read, Write};
use std::ops::Range;
use std::os::unix::io::{AsRawFd, RawFd};
use std::path::Path;
use std::sync::Arc;
/// The fixed length of a directory's `meta` file.
@@ -212,7 +213,7 @@ impl SampleFileDir {
///
/// `db_meta.in_progress_open` should be filled if the directory should be opened in read/write
/// mode; absent in read-only mode.
pub fn open(path: &str, expected_meta: &schema::DirMeta) -> Result<Arc<SampleFileDir>, Error> {
pub fn open(path: &Path, expected_meta: &schema::DirMeta) -> Result<Arc<SampleFileDir>, Error> {
let read_write = expected_meta.in_progress_open.is_some();
let s = SampleFileDir::open_self(path, false)?;
s.fd.lock(if read_write {
@@ -220,7 +221,7 @@ impl SampleFileDir {
} else {
FlockArg::LockSharedNonblock
})
.map_err(|e| e.context(format!("unable to lock dir {}", path)))?;
.map_err(|e| e.context(format!("unable to lock dir {}", path.display())))?;
let dir_meta = read_meta(&s.fd).map_err(|e| e.context("unable to read meta file"))?;
if let Err(e) = SampleFileDir::check_consistent(expected_meta, &dir_meta) {
bail!(
@@ -269,12 +270,12 @@ impl SampleFileDir {
}
pub(crate) fn create(
path: &str,
path: &Path,
db_meta: &schema::DirMeta,
) -> Result<Arc<SampleFileDir>, Error> {
let s = SampleFileDir::open_self(path, true)?;
s.fd.lock(FlockArg::LockExclusiveNonblock)
.map_err(|e| e.context(format!("unable to lock dir {}", path)))?;
.map_err(|e| e.context(format!("unable to lock dir {}", path.display())))?;
let old_meta = read_meta(&s.fd)?;
// Verify metadata. We only care that it hasn't been completely opened.
@@ -282,12 +283,15 @@ impl SampleFileDir {
if old_meta.last_complete_open.is_some() {
bail!(
"Can't create dir at path {}: is already in use:\n{:?}",
path,
path.display(),
old_meta
);
}
if !s.is_empty()? {
bail!("Can't create dir at path {} with existing files", path);
bail!(
"Can't create dir at path {} with existing files",
path.display()
);
}
s.write_meta(db_meta)?;
Ok(s)
@@ -316,7 +320,7 @@ impl SampleFileDir {
Ok(true)
}
fn open_self(path: &str, create: bool) -> Result<Arc<SampleFileDir>, Error> {
fn open_self(path: &Path, create: bool) -> Result<Arc<SampleFileDir>, Error> {
let fd = Arc::new(Fd::open(path, create)?);
let reader = reader::Reader::spawn(path, fd.clone());
Ok(Arc::new(SampleFileDir { fd, reader }))

View File

@@ -23,6 +23,7 @@
use std::convert::TryFrom;
use std::future::Future;
use std::os::unix::prelude::AsRawFd;
use std::path::Path;
use std::{
ops::Range,
pin::Pin,
@@ -44,7 +45,7 @@ use crate::CompositeId;
pub(super) struct Reader(tokio::sync::mpsc::UnboundedSender<ReaderCommand>);
impl Reader {
pub(super) fn spawn(path: &str, dir: Arc<super::Fd>) -> Self {
pub(super) fn spawn(path: &Path, dir: Arc<super::Fd>) -> Self {
let (tx, rx) = tokio::sync::mpsc::unbounded_channel();
let page_size = usize::try_from(
nix::unistd::sysconf(nix::unistd::SysconfVar::PAGE_SIZE)
@@ -54,7 +55,7 @@ impl Reader {
.expect("PAGE_SIZE fits in usize");
assert_eq!(page_size.count_ones(), 1, "invalid page size {}", page_size);
std::thread::Builder::new()
.name(format!("r-{}", path))
.name(format!("r-{}", path.display()))
.spawn(move || ReaderInt { dir, page_size }.run(rx))
.expect("unable to create reader thread");
Self(tx)
@@ -407,7 +408,7 @@ mod tests {
.tempdir()
.unwrap();
let fd = std::sync::Arc::new(super::super::Fd::open(tmpdir.path(), false).unwrap());
let reader = super::Reader::spawn("/path/goes/here", fd);
let reader = super::Reader::spawn(tmpdir.path(), fd);
std::fs::write(tmpdir.path().join("0123456789abcdef"), b"blah blah").unwrap();
let f = reader.open_file(crate::CompositeId(0x01234567_89abcdef), 1..8);
assert_eq!(f.try_concat().await.unwrap(), b"lah bla");