MyFavMirrors/moonfire-nvr
1
0
Fork 0
mirror of https://github.com/scottlamb/moonfire-nvr.git synced 2025-02-26 04:49:17 -05:00
Code Issues Packages Projects Releases Wiki Activity

Add a Filesystem interface for testability.

Browse Source 
Not immediately adding any tests that take advantage of it.
The new storage schema should handle ENOSPC correctly, and this will aid in
testing it.
This commit is contained in:
Scott Lamb 2016-01-02 10:48:58 -08:00
parent c9eda8ac15
commit 7968952295
7 changed files with 214 additions and 74 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

85
src/filesystem.cc
View File

@ -55,10 +55,55 @@
namespace moonfire_nvr {
bool DirForEach(const std::string &dir_path,
namespace {
class RealFile : public File {
public:
explicit RealFile(int fd) : fd_(fd) {}
RealFile(const RealFile &) = delete;
void operator=(const RealFile &) = delete;
~RealFile() final { Close(); }
int Close() final {
if (fd_ < 0) {
return 0;
}
int ret;
while ((ret = close(fd_)) != 0 && errno == EINTR)
;
if (ret != 0) {
return errno;
}
fd_ = -1;
return 0;
}
int Write(re2::StringPiece *data) final {
if (fd_ < 0) {
return EBADF;
}
ssize_t ret;
while ((ret = write(fd_, data->data(), data->size())) == -1 &&
errno == EINTR)
;
if (ret < 0) {
return errno;
}
data->remove_prefix(ret);
return 0;
}
private:
int fd_ = -1;
};
class RealFilesystem : public Filesystem {
public:
bool DirForEach(const char *dir_path,
std::function<IterationControl(const dirent *)> fn,
std::string *error_message) {
DIR *owned_dir = opendir(dir_path.c_str());
std::string *error_message) final {
DIR *owned_dir = opendir(dir_path);
if (owned_dir == nullptr) {
int err = errno;
*error_message =
@ -79,6 +124,40 @@ bool DirForEach(const std::string &dir_path,
return false;
}
return true;
}
int Open(const char *path, int flags, std::unique_ptr<File> *f) final {
return Open(path, flags, 0, f);
}
int Open(const char *path, int flags, mode_t mode,
std::unique_ptr<File> *f) final {
int ret = open(path, flags, mode);
if (ret < 0) {
return errno;
}
f->reset(new RealFile(ret));
return 0;
}
int Mkdir(const char *path, mode_t mode) final {
return (mkdir(path, mode) < 0) ? errno : 0;
}
int Rmdir(const char *path) final { return (rmdir(path) < 0) ? errno : 0; }
int Stat(const char *path, struct stat *buf) final {
return (stat(path, buf) < 0) ? errno : 0;
}
int Unlink(const char *path) final { return (unlink(path) < 0) ? errno : 0; }
};
} // namespace
Filesystem *GetRealFilesystem() {
static Filesystem *real_filesystem = new RealFilesystem;
return real_filesystem;
}
} // namespace moonfire_nvr

59
src/filesystem.h
View File

@ -38,8 +38,8 @@
#include <sys/stat.h>
#include <sys/types.h>
#include <memory>
#include <functional>
#include <iostream>
#include <string>
#include <event2/buffer.h>
@ -55,14 +55,57 @@ enum class IterationControl {
kBreak // indicates the caller should terminate the loop with success.
};
// Execute |fn| for each directory entry in |dir_path|, stopping early
// (successfully) if the callback returns IterationControl::kBreak.
//
// On success, returns true.
// On failure, returns false and updates |error_msg|.
bool DirForEach(const std::string &dir_path,
// Represents an open file. All methods but Close() are thread-safe.
class File {
public:
// Close the file, ignoring the result.
virtual ~File() {}
// Close the file, returning 0 on success or errno>0 on failure.
// Already closed is considered a success.
virtual int Close() = 0;
// Write to the file, returning 0 on success or errno>0 on failure.
// On success, data->remove_prefix will be called with the written bytes.
virtual int Write(re2::StringPiece *data) = 0;
};
// Interface to the local filesystem. There's typically one per program,
// but it's an abstract class for testability. Thread-safe.
class Filesystem {
public:
virtual ~Filesystem() {}
// Execute |fn| for each directory entry in |dir_path|, stopping early
// (successfully) if the callback returns IterationControl::kBreak.
//
// On success, returns true.
// On failure, returns false and updates |error_msg|.
virtual bool DirForEach(const char *dir_path,
std::function<IterationControl(const dirent *)> fn,
std::string *error_msg);
std::string *error_msg) = 0;
// open() the specified path, returning 0 on success or errno>0 on failure.
// On success, |f| is populated with an open file.
virtual int Open(const char *path, int flags, std::unique_ptr<File> *f) = 0;
virtual int Open(const char *path, int flags, mode_t mode,
std::unique_ptr<File> *f) = 0;
// mkdir() the specified path, returning 0 on success or errno>0 on failure.
virtual int Mkdir(const char *path, mode_t mode) = 0;
// rmdir() the specified path, returning 0 on success or errno>0 on failure.
virtual int Rmdir(const char *path) = 0;
// stat() the specified path, returning 0 on success or errno>0 on failure.
virtual int Stat(const char *path, struct stat *buf) = 0;
// unlink() the specified file, returning 0 on success or errno>0 on failure.
virtual int Unlink(const char *path) = 0;
};
// Get the (singleton) real filesystem, which is never deleted.
Filesystem *GetRealFilesystem();
} // namespace moonfire_nvr

7
src/moonfire-nvr-test.cc
View File

@ -73,6 +73,7 @@ class FileManagerTest : public testing::Test {
protected:
FileManagerTest() {
test_dir_ = PrepareTempDirOrDie("moonfire-nvr-file-manager");
env_.fs = GetRealFilesystem();
}
std::vector<std::string> GetFilenames(const FileManager &mgr) {
@ -83,12 +84,13 @@ class FileManagerTest : public testing::Test {
return out;
}
Environment env_;
std::string test_dir_;
};
TEST_F(FileManagerTest, InitWithNoDirectory) {
std::string subdir = test_dir_ + "/" + "subdir";
FileManager manager("foo", subdir, 0);
FileManager manager("foo", subdir, 0, &env_);
// Should succeed.
std::string error_message;
@ -124,7 +126,7 @@ TEST_F(FileManagerTest, InitAndRotateWithExistingFiles) {
WriteFileOrDie(test_dir_ + "/2.mp4", "123");
WriteFileOrDie(test_dir_ + "/3.mp4", "12345");
WriteFileOrDie(test_dir_ + "/other", "1234567");
FileManager manager("foo", test_dir_, 8);
FileManager manager("foo", test_dir_, 8, &env_);
// Should succeed.
std::string error_message;
@ -145,6 +147,7 @@ class StreamTest : public testing::Test {
test_dir_ = PrepareTempDirOrDie("moonfire-nvr-stream-copier");
env_.clock = &clock_;
env_.video_source = &video_source_;
env_.fs = GetRealFilesystem();
clock_.Sleep({1430006400, 0}); // 2016-04-26 00:00:00 UTC
config_.set_base_path(test_dir_);

30
src/moonfire-nvr.cc
View File

@ -61,15 +61,18 @@ const char kFilenameSuffix[] = ".mp4";
} // namespace
FileManager::FileManager(const std::string &short_name, const std::string &path,
uint64_t byte_limit)
: short_name_(short_name), path_(path), byte_limit_(byte_limit) {}
uint64_t byte_limit, Environment *env)
: short_name_(short_name),
path_(path),
byte_limit_(byte_limit),
env_(env) {}
bool FileManager::Init(std::string *error_message) {
// Create the directory if it doesn't exist.
// If the path exists, assume it is a valid directory.
if (mkdir(path_.c_str(), 0700) < 0 && errno != EEXIST) {
int err = errno;
*error_message = StrCat("Unable to create ", path_, ": ", strerror(err));
int ret = env_->fs->Mkdir(path_.c_str(), 0700);
if (ret != 0 && ret != EEXIST) {
*error_message = StrCat("Unable to create ", path_, ": ", strerror(ret));
return false;
}
@ -94,7 +97,7 @@ bool FileManager::Init(std::string *error_message) {
return IterationControl::kContinue;
};
if (!DirForEach(path_, file_fn, error_message)) {
if (!env_->fs->DirForEach(path_.c_str(), file_fn, error_message)) {
return false;
}
@ -115,18 +118,18 @@ bool FileManager::Rotate(std::string *error_message) {
// won't return prematurely.
mu_.unlock();
string fpath = StrCat(path_, "/", filename);
if (unlink(fpath.c_str()) == 0) {
int ret = env_->fs->Unlink(fpath.c_str());
if (ret == 0) {
LOG(INFO) << short_name_ << ": Deleted " << filename << " to reclaim "
<< size << " bytes.";
} else if (errno == ENOENT) {
} else if (ret == ENOENT) {
// This may have happened due to a racing Rotate() call.
// In any case, the file is gone, so proceed to mark it as such.
LOG(INFO) << short_name_ << ": File " << filename
<< " was already deleted.";
} else {
int err = errno;
*error_message =
StrCat("unlink failed on ", filename, ": ", strerror(err));
StrCat("unlink failed on ", filename, ": ", strerror(ret));
return false;
}
@ -154,9 +157,9 @@ bool FileManager::AddFile(const std::string &filename,
std::string *error_message) {
struct stat buf;
string fpath = StrCat(path_, "/", filename);
if (lstat(fpath.c_str(), &buf) != 0) {
int err = errno;
*error_message = StrCat("lstat on ", fpath, " failed: ", strerror(err));
int ret = env_->fs->Stat(fpath.c_str(), &buf);
if (ret != 0) {
*error_message = StrCat("stat on ", fpath, " failed: ", strerror(ret));
return false;
}
VLOG(1) << short_name_ << ": adding file " << filename << " size "
@ -466,6 +469,7 @@ void Stream::HttpCallbackForFile(evhttp_request *req, const string &filename) {
Nvr::Nvr() {
env_.clock = GetRealClock();
env_.video_source = GetRealVideoSource();
env_.fs = GetRealFilesystem();
}
Nvr::~Nvr() {

10
src/moonfire-nvr.h
View File

@ -46,6 +46,7 @@
#include <event2/http.h>
#include "config.pb.h"
#include "filesystem.h"
#include "ffmpeg.h"
#include "time.h"
@ -72,6 +73,7 @@ class ShutdownSignal {
struct Environment {
WallClock *clock = nullptr;
VideoSource *video_source = nullptr;
Filesystem *fs = nullptr;
};
// Delete old ".mp4" files within a specified directory, keeping them within a
@ -88,7 +90,7 @@ class FileManager {
// |short_name| will be prepended to log messages.
FileManager(const std::string &short_name, const std::string &path,
uint64_t byte_limit);
uint64_t byte_limit, Environment *env);
FileManager(const FileManager &) = delete;
FileManager &operator=(const FileManager &) = delete;
@ -120,6 +122,7 @@ class FileManager {
const std::string short_name_;
const std::string path_;
const uint64_t byte_limit_;
Environment *const env_;
mutable std::mutex mu_;
std::map<std::string, struct stat> files_;
@ -132,13 +135,14 @@ class FileManager {
class Stream {
public:
Stream(const ShutdownSignal *signal, const moonfire_nvr::Config &config,
const Environment *env, const moonfire_nvr::Camera &camera)
Environment *const env, const moonfire_nvr::Camera &camera)
: signal_(signal),
env_(env),
camera_path_(config.base_path() + "/" + camera.short_name()),
rotate_interval_(config.rotate_sec()),
camera_(camera),
manager_(camera_.short_name(), camera_path_, camera.retain_bytes()) {}
manager_(camera_.short_name(), camera_path_, camera.retain_bytes(),
env) {}
Stream(const Stream &) = delete;
Stream &operator=(const Stream &) = delete;

56
src/testutil.cc
View File

@ -34,56 +34,57 @@
#include <dirent.h>
#include <errno.h>
#include <fcntl.h>
#include <string.h>
#include <sys/stat.h>
#include <sys/types.h>
#include <fstream>
#include <glog/logging.h>
#include "filesystem.h"
#include "string.h"
namespace moonfire_nvr {
namespace {
bool DeleteChildrenRecursively(const std::string &dirname,
std::string *error_msg) {
bool DeleteChildrenRecursively(const char *dirname, std::string *error_msg) {
bool ok = true;
auto fn = [&dirname, &ok, error_msg](const struct dirent *ent) {
std::string name(ent->d_name);
std::string path = dirname + "/" + name;
std::string path = StrCat(dirname, "/", name);
if (name == "." || name == "..") {
return IterationControl::kContinue;
}
bool is_dir = (ent->d_type == DT_DIR);
if (ent->d_type == DT_UNKNOWN) {
struct stat buf;
PCHECK(stat(path.c_str(), &buf) == 0) << path;
int ret = GetRealFilesystem()->Stat(path.c_str(), &buf);
CHECK_EQ(ret, 0) << path << ": " << strerror(ret);
is_dir = S_ISDIR(buf.st_mode);
}
if (is_dir) {
ok = ok && DeleteChildrenRecursively(path, error_msg);
ok = ok && DeleteChildrenRecursively(path.c_str(), error_msg);
if (!ok) {
return IterationControl::kBreak;
}
if (rmdir(path.c_str()) != 0) {
*error_msg =
std::string("rmdir failed on ") + path + ": " + strerror(errno);
int ret = GetRealFilesystem()->Rmdir(path.c_str());
if (ret != 0) {
*error_msg = StrCat("rmdir failed on ", path, ": ", strerror(ret));
ok = false;
return IterationControl::kBreak;
}
} else {
if (unlink(path.c_str()) != 0) {
*error_msg =
std::string("unlink failed on ") + path + ": " + strerror(errno);
int ret = GetRealFilesystem()->Unlink(path.c_str());
if (ret != 0) {
*error_msg = StrCat("unlink failed on ", path, ": ", strerror(ret));
ok = false;
return IterationControl::kBreak;
}
}
return IterationControl::kContinue;
};
if (!DirForEach(dirname, fn, error_msg)) {
if (!GetRealFilesystem()->DirForEach(dirname, fn, error_msg)) {
return false;
}
return ok;
@ -92,22 +93,27 @@ bool DeleteChildrenRecursively(const std::string &dirname,
} // namespace
std::string PrepareTempDirOrDie(const std::string &test_name) {
std::string dirname = std::string("/tmp/test.") + test_name;
int res = mkdir(dirname.c_str(), 0700);
if (res != 0) {
int err = errno;
CHECK_EQ(err, EEXIST) << "mkdir failed: " << strerror(err);
std::string dirname = StrCat("/tmp/test.", test_name);
int ret = GetRealFilesystem()->Mkdir(dirname.c_str(), 0700);
if (ret != 0) {
CHECK_EQ(ret, EEXIST) << "mkdir failed: " << strerror(ret);
std::string error_msg;
CHECK(DeleteChildrenRecursively(dirname, &error_msg)) << error_msg;
CHECK(DeleteChildrenRecursively(dirname.c_str(), &error_msg)) << error_msg;
}
return dirname;
}
void WriteFileOrDie(const std::string &path, const std::string &contents) {
std::ofstream f(path);
f << contents;
f.close();
CHECK(!f.fail()) << "failed to write: " << path;
void WriteFileOrDie(const std::string &path, re2::StringPiece contents) {
std::unique_ptr<File> f;
int ret = GetRealFilesystem()->Open(path.c_str(),
O_WRONLY | O_CREAT | O_TRUNC, 0600, &f);
CHECK_EQ(ret, 0) << "open " << path << ": " << strerror(ret);
while (!contents.empty()) {
ret = f->Write(&contents);
CHECK_EQ(ret, 0) << "write " << path << ": " << strerror(ret);
}
ret = f->Close();
CHECK_EQ(ret, 0) << "close " << path << ": " << strerror(ret);
}
} // namespace moonfire_nvr

3
src/testutil.h
View File

@ -35,6 +35,7 @@
#include <glog/logging.h>
#include <gmock/gmock.h>
#include <re2/stringpiece.h>
namespace moonfire_nvr {
@ -43,7 +44,7 @@ namespace moonfire_nvr {
std::string PrepareTempDirOrDie(const std::string &test_name);
// Write the given file contents to the given path, or die.
void WriteFileOrDie(const std::string &path, const std::string &contents);
void WriteFileOrDie(const std::string &path, re2::StringPiece contents);
// A scoped log sink for testing that the right log messages are sent.
// Modelled after glog's "mock-log.h", which is not exported.