MyFavMirrors/moonfire-nvr
1
0
Fork 0
mirror of https://github.com/scottlamb/moonfire-nvr.git synced 2024-12-26 07:05:56 -05:00
Code Issues Packages Projects Releases Wiki Activity

Expanded, reasonably efficient SQL operations.

Browse Source 
* Schema revisions. The most dramatic is the addition of a covering index on
  (camera_id, start_time_90k) that avoids the need to make sparse accesses
  into the recording table (where the desired data is intermixed with both
  the large blobs and rows from other cameras). A query over a year's data
  previously took many seconds (6+ even in a form without the video_index)
  and now is roughly 10X faster. Queries for a couple weeks now should be
  unnoticeably fast.

  Other changes to shrink the rows, such as duration_90k instead of
  end_time_90k (more compact varint encoding) and video_sample_entry_id
  (typically 1 byte) instead of video_sample_entry_sha1 (20 bytes).
  And more CHECK constraints for good measure.

* Caching of expensive computations and logic to keep them up to date.
  The top-level web view previously went through the entire recording table,
  which was even slower. Now it is served from a small map in RAM.

* Expanded the scope of operations to cover (hopefully) everything needed for
  recording into the SQLite database.

* Added tests of MoonfireDatabase. These are basic tests that don't
  exercise a lot of error cases, but at least they exist.

The main MoonfireDatabase functionality still missing is support for quickly
seeing what calendar days have data over the full timespan of a camera. This
is more data to compute and cache.
This commit is contained in:
Scott Lamb 2016-01-24 17:57:46 -08:00
parent b9d6526492
commit 699ffe7777
14 changed files with 1293 additions and 280 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
src/CMakeLists.txt
View File

@ -71,7 +71,19 @@ install_programs(/bin FILES moonfire-nvr)
include_directories(${GTest_INCLUDE_DIR})
include_directories(${GMock_INCLUDE_DIR})
foreach(test coding crypto h264 http moonfire-nvr mp4 recording sqlite string)
set(MOONFIRE_NVR_TESTS
coding
crypto
h264
http
moonfire-db
moonfire-nvr
mp4
recording
sqlite
string)
foreach(test ${MOONFIRE_NVR_TESTS})
add_executable(${test}-test ${test}-test.cc testutil.cc)
target_link_libraries(${test}-test GTest GMock moonfire-nvr-lib)
add_test(NAME ${test}-test

345
src/moonfire-db-test.cc Normal file
View File

@ -0,0 +1,345 @@
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// In addition, as a special exception, the copyright holders give
// permission to link the code of portions of this program with the
// OpenSSL library under certain conditions as described in each
// individual source file, and distribute linked combinations including
// the two.
//
// You must obey the GNU General Public License in all respects for all
// of the code used other than OpenSSL. If you modify file(s) with this
// exception, you may extend this exception to your version of the
// file(s), but you are not obligated to do so. If you do not wish to do
// so, delete this exception statement from your version. If you delete
// this exception statement from all source files in the program, then
// also delete it here.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
//
// moonfire-db-test.cc: tests of the moonfire-db.h interface.
#include <string>
#include <gflags/gflags.h>
#include <gmock/gmock.h>
#include <gtest/gtest.h>
#include "moonfire-db.h"
#include "sqlite.h"
#include "string.h"
#include "testutil.h"
DECLARE_bool(alsologtostderr);
using testing::_;
using testing::HasSubstr;
using testing::DoAll;
using testing::Return;
using testing::SetArgPointee;
namespace moonfire_nvr {
namespace {
class MoonfireDbTest : public testing::Test {
protected:
MoonfireDbTest() {
tmpdir_ = PrepareTempDirOrDie("moonfire-db-test");
std::string error_message;
CHECK(db_.Open(StrCat(tmpdir_, "/db").c_str(),
SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE, &error_message))
<< error_message;
std::string create_sql = ReadFileOrDie("../src/schema.sql");
DatabaseContext ctx(&db_);
CHECK(RunStatements(&ctx, create_sql, &error_message)) << error_message;
}
int64_t AddCamera(Uuid uuid, re2::StringPiece short_name) {
DatabaseContext ctx(&db_);
auto run = ctx.UseOnce(
R"(
insert into camera (uuid, short_name, retain_bytes)
values (:uuid, :short_name, :retain_bytes);
)");
run.BindBlob(":uuid", uuid.binary_view());
run.BindText(":short_name", short_name);
run.BindInt64(":retain_bytes", 42);
if (run.Step() != SQLITE_DONE) {
ADD_FAILURE() << run.error_message();
return -1;
}
return ctx.last_insert_rowid();
}
void ExpectNoRecordings(Uuid camera_uuid) {
int rows = 0;
mdb_->ListCameras([&](const ListCamerasRow &row) {
++rows;
EXPECT_EQ(camera_uuid, row.uuid);
EXPECT_EQ(-1, row.min_start_time_90k);
EXPECT_EQ(-1, row.max_end_time_90k);
EXPECT_EQ(0, row.total_duration_90k);
EXPECT_EQ(0, row.total_sample_file_bytes);
return IterationControl::kContinue;
});
EXPECT_EQ(1, rows);
std::string error_message;
rows = 0;
EXPECT_TRUE(mdb_->ListCameraRecordings(
camera_uuid, 0, std::numeric_limits<int64_t>::max(),
[&](const ListCameraRecordingsRow &row) {
++rows;
return IterationControl::kBreak;
},
&error_message))
<< error_message;
EXPECT_EQ(0, rows);
rows = 0;
EXPECT_TRUE(mdb_->ListMp4Recordings(
camera_uuid, 0, std::numeric_limits<int64_t>::max(),
[&](Recording &recording, const VideoSampleEntry &entry) {
++rows;
return IterationControl::kBreak;
},
&error_message))
<< error_message;
EXPECT_EQ(0, rows);
}
void ExpectSingleRecording(Uuid camera_uuid, const Recording &recording,
const VideoSampleEntry &entry,
ListOldestSampleFilesRow *save_oldest_row) {
std::string error_message;
int rows = 0;
mdb_->ListCameras([&](const ListCamerasRow &row) {
++rows;
EXPECT_EQ(camera_uuid, row.uuid);
EXPECT_EQ(recording.start_time_90k, row.min_start_time_90k);
EXPECT_EQ(recording.end_time_90k, row.max_end_time_90k);
EXPECT_EQ(recording.end_time_90k - recording.start_time_90k,
row.total_duration_90k);
EXPECT_EQ(recording.sample_file_bytes, row.total_sample_file_bytes);
return IterationControl::kContinue;
});
EXPECT_EQ(1, rows);
GetCameraRow camera_row;
EXPECT_TRUE(mdb_->GetCamera(camera_uuid, &camera_row));
EXPECT_EQ(recording.start_time_90k, camera_row.min_start_time_90k);
EXPECT_EQ(recording.end_time_90k, camera_row.max_end_time_90k);
EXPECT_EQ(recording.end_time_90k - recording.start_time_90k,
camera_row.total_duration_90k);
EXPECT_EQ(recording.sample_file_bytes, camera_row.total_sample_file_bytes);
rows = 0;
EXPECT_TRUE(mdb_->ListCameraRecordings(
camera_uuid, 0, std::numeric_limits<int64_t>::max(),
[&](const ListCameraRecordingsRow &row) {
++rows;
EXPECT_EQ(recording.start_time_90k, row.start_time_90k);
EXPECT_EQ(recording.end_time_90k, row.end_time_90k);
EXPECT_EQ(recording.video_samples, row.video_samples);
EXPECT_EQ(recording.sample_file_bytes, row.sample_file_bytes);
EXPECT_EQ(entry.sha1, row.video_sample_entry_sha1);
EXPECT_EQ(entry.width, row.width);
EXPECT_EQ(entry.height, row.height);
return IterationControl::kContinue;
},
&error_message))
<< error_message;
EXPECT_EQ(1, rows);
rows = 0;
EXPECT_TRUE(mdb_->ListOldestSampleFiles(
camera_uuid,
[&](const ListOldestSampleFilesRow &row) {
++rows;
EXPECT_EQ(recording.id, row.recording_id);
EXPECT_EQ(recording.sample_file_uuid, row.sample_file_uuid);
EXPECT_EQ(recording.end_time_90k - recording.start_time_90k,
row.duration_90k);
EXPECT_EQ(recording.sample_file_bytes, row.sample_file_bytes);
*save_oldest_row = row;
return IterationControl::kContinue;
},
&error_message))
<< error_message;
EXPECT_EQ(1, rows);
rows = 0;
EXPECT_TRUE(mdb_->ListMp4Recordings(
camera_uuid, 0, std::numeric_limits<int64_t>::max(),
[&](Recording &some_recording, const VideoSampleEntry &some_entry) {
++rows;
EXPECT_EQ(recording.id, some_recording.id);
EXPECT_EQ(recording.camera_id, some_recording.camera_id);
EXPECT_EQ(recording.sample_file_sha1,
some_recording.sample_file_sha1);
EXPECT_EQ(recording.sample_file_uuid,
some_recording.sample_file_uuid);
EXPECT_EQ(recording.video_sample_entry_id,
some_recording.video_sample_entry_id);
EXPECT_EQ(recording.start_time_90k, some_recording.start_time_90k);
EXPECT_EQ(recording.end_time_90k, some_recording.end_time_90k);
EXPECT_EQ(recording.sample_file_bytes,
some_recording.sample_file_bytes);
EXPECT_EQ(recording.video_samples, some_recording.video_samples);
EXPECT_EQ(recording.video_sync_samples,
some_recording.video_sync_samples);
EXPECT_EQ(recording.video_index, some_recording.video_index);
EXPECT_EQ(entry.id, some_entry.id);
EXPECT_EQ(entry.sha1, some_entry.sha1);
EXPECT_EQ(entry.data, some_entry.data);
EXPECT_EQ(entry.width, some_entry.width);
EXPECT_EQ(entry.height, some_entry.height);
return IterationControl::kContinue;
},
&error_message))
<< error_message;
EXPECT_EQ(1, rows);
}
std::string tmpdir_;
Database db_;
std::unique_ptr<MoonfireDatabase> mdb_;
};
// Basic test of running some queries on an empty database.
TEST_F(MoonfireDbTest, EmptyDatabase) {
std::string error_message;
mdb_.reset(new MoonfireDatabase);
ASSERT_TRUE(mdb_->Init(&db_, &error_message)) << error_message;
mdb_->ListCameras([&](const ListCamerasRow &row) {
ADD_FAILURE() << "row unexpected";
return IterationControl::kBreak;
});
GetCameraRow get_camera_row;
EXPECT_FALSE(mdb_->GetCamera(Uuid(), &get_camera_row));
EXPECT_FALSE(
mdb_->ListCameraRecordings(Uuid(), 0, std::numeric_limits<int64_t>::max(),
[&](const ListCameraRecordingsRow &row) {
ADD_FAILURE() << "row unexpected";
return IterationControl::kBreak;
},
&error_message));
EXPECT_FALSE(mdb_->ListMp4Recordings(
Uuid(), 0, std::numeric_limits<int64_t>::max(),
[&](Recording &recording, const VideoSampleEntry &entry) {
ADD_FAILURE() << "row unexpected";
return IterationControl::kBreak;
},
&error_message));
}
// Basic test of the full lifecycle of recording.
// Does not exercise many error cases.
TEST_F(MoonfireDbTest, FullLifecycle) {
std::string error_message;
const char kCameraShortName[] = "testcam";
Uuid camera_uuid = GetRealUuidGenerator()->Generate();
int64_t camera_id = AddCamera(camera_uuid, kCameraShortName);
ASSERT_GT(camera_id, 0);
mdb_.reset(new MoonfireDatabase);
ASSERT_TRUE(mdb_->Init(&db_, &error_message)) << error_message;
ExpectNoRecordings(camera_uuid);
std::vector<Uuid> reserved;
EXPECT_TRUE(mdb_->ListReservedSampleFiles(&reserved, &error_message))
<< error_message;
EXPECT_THAT(reserved, testing::IsEmpty());
std::vector<Uuid> uuids = mdb_->ReserveSampleFiles(2, &error_message);
ASSERT_THAT(uuids, testing::SizeIs(2)) << error_message;
EXPECT_TRUE(mdb_->ListReservedSampleFiles(&reserved, &error_message))
<< error_message;
EXPECT_THAT(reserved, testing::UnorderedElementsAre(uuids[0], uuids[1]));
VideoSampleEntry entry;
entry.sha1.resize(20);
entry.width = 768;
entry.height = 512;
entry.data.resize(100);
ASSERT_TRUE(mdb_->InsertVideoSampleEntry(&entry, &error_message))
<< error_message;
ASSERT_GT(entry.id, 0);
Recording recording;
recording.camera_id = camera_id;
recording.sample_file_uuid = GetRealUuidGenerator()->Generate();
recording.video_sample_entry_id = entry.id;
SampleIndexEncoder encoder;
encoder.Init(&recording, UINT64_C(1430006400) * kTimeUnitsPerSecond);
encoder.AddSample(kTimeUnitsPerSecond, 42, true);
// Inserting a recording should succeed and remove its uuid from the
// reserved table.
ASSERT_FALSE(mdb_->InsertRecording(&recording, &error_message));
EXPECT_THAT(error_message, testing::HasSubstr("not reserved"));
recording.sample_file_uuid = uuids.back();
recording.sample_file_sha1.resize(20);
ASSERT_TRUE(mdb_->InsertRecording(&recording, &error_message))
<< error_message;
ASSERT_GT(recording.id, 0);
EXPECT_TRUE(mdb_->ListReservedSampleFiles(&reserved, &error_message))
<< error_message;
EXPECT_THAT(reserved, testing::ElementsAre(uuids[0]));
// Queries should return the correct result (with caches updated on insert).
ListOldestSampleFilesRow oldest;
ExpectSingleRecording(camera_uuid, recording, entry, &oldest);
// Queries on a fresh database should return the correct result (with caches
// populated from existing database contents).
mdb_.reset(new MoonfireDatabase);
ASSERT_TRUE(mdb_->Init(&db_, &error_message)) << error_message;
ExpectSingleRecording(camera_uuid, recording, entry, &oldest);
// Deleting a recording should succeed, update the min/max times, and mark
// the uuid as reserved.
std::vector<ListOldestSampleFilesRow> to_delete;
to_delete.push_back(oldest);
ASSERT_TRUE(mdb_->DeleteRecordings(to_delete, &error_message))
<< error_message;
EXPECT_TRUE(mdb_->ListReservedSampleFiles(&reserved, &error_message))
<< error_message;
EXPECT_THAT(reserved, testing::UnorderedElementsAre(uuids[0], uuids[1]));
LOG(INFO) << "after delete";
ExpectNoRecordings(camera_uuid);
EXPECT_TRUE(mdb_->MarkSampleFilesDeleted(uuids, &error_message))
<< error_message;
EXPECT_TRUE(mdb_->ListReservedSampleFiles(&reserved, &error_message))
<< error_message;
EXPECT_THAT(reserved, testing::IsEmpty());
}
} // namespace
} // namespace moonfire_nvr
int main(int argc, char **argv) {
FLAGS_alsologtostderr = true;
google::ParseCommandLineFlags(&argc, &argv, true);
testing::InitGoogleTest(&argc, argv);
google::InitGoogleLogging(argv[0]);
return RUN_ALL_TESTS();
}

791
src/moonfire-db.cc
View File

@ -29,6 +29,7 @@
// along with this program. If not, see <http://www.gnu.org/licenses/>.
//
// moonfire-db.cc: implementation of moonfire-db.h interface.
// see top-level comments there on performance & efficiency.
#include "moonfire-db.h"
@ -42,188 +43,336 @@
namespace moonfire_nvr {
bool MoonfireDatabase::Init(std::string *error_message) {
list_cameras_query_ = db_->Prepare(
R"(
select
camera.id,
camera.uuid,
camera.short_name,
camera.description,
camera.retain_bytes,
min(recording.start_time_90k),
max(recording.end_time_90k),
sum(recording.end_time_90k - recording.start_time_90k),
sum(recording.sample_file_bytes)
from
camera
left join recording on
(camera.id = recording.camera_id and
recording.status = 1)
group by
camera.id,
camera.uuid,
camera.short_name,
camera.description,
camera.retain_bytes;
)",
nullptr, error_message);
if (!list_cameras_query_.valid()) {
return false;
bool MoonfireDatabase::Init(Database *db, std::string *error_message) {
CHECK(db_ == nullptr);
db_ = db;
{
DatabaseContext ctx(db_);
// This query scans the entirety of the recording table's index.
// It is quite slow, so the results are cached.
auto list_cameras_run = ctx.UseOnce(
R"(
select
camera.id,
camera.uuid,
camera.short_name,
camera.description,
camera.retain_bytes,
min(recording.start_time_90k),
max(recording.start_time_90k + recording.duration_90k),
sum(recording.duration_90k),
sum(recording.sample_file_bytes)
from
camera
left join recording on (camera.id = recording.camera_id)
group by
camera.id,
camera.uuid,
camera.short_name,
camera.description,
camera.retain_bytes;
)");
while (list_cameras_run.Step() == SQLITE_ROW) {
CameraData data;
data.id = list_cameras_run.ColumnInt64(0);
Uuid uuid;
if (!uuid.ParseBinary(list_cameras_run.ColumnBlob(1))) {
*error_message =
StrCat("bad uuid ", ToHex(list_cameras_run.ColumnBlob(2)),
" for camera id ", data.id);
return false;
}
data.short_name = list_cameras_run.ColumnText(2).as_string();
data.description = list_cameras_run.ColumnText(3).as_string();
data.retain_bytes = list_cameras_run.ColumnInt64(4);
data.min_start_time_90k = list_cameras_run.ColumnType(5) == SQLITE_NULL
? -1
: list_cameras_run.ColumnInt64(5);
data.max_end_time_90k = list_cameras_run.ColumnType(6) == SQLITE_NULL
? -1
: list_cameras_run.ColumnInt64(6);
data.total_duration_90k = list_cameras_run.ColumnInt64(7);
data.total_sample_file_bytes = list_cameras_run.ColumnInt64(8);
auto ret = cameras_by_uuid_.insert(std::make_pair(uuid, data));
if (!ret.second) {
*error_message = StrCat("Duplicate camera uuid ", uuid.UnparseText());
return false;
}
CameraData *data_p = &ret.first->second;
if (!cameras_by_id_.insert(std::make_pair(data.id, data_p)).second) {
*error_message = StrCat("Duplicate camera id ", data.id);
return false;
}
}
if (list_cameras_run.status() != SQLITE_DONE) {
*error_message = StrCat("Camera list query failed: ",
list_cameras_run.error_message());
}
// It's simplest to just keep the video sample entries in RAM.
auto video_sample_entries_run = ctx.UseOnce(
R"(
select
id,
sha1,
width,
height,
data
from
video_sample_entry
)");
while (video_sample_entries_run.Step() == SQLITE_ROW) {
VideoSampleEntry entry;
entry.id = video_sample_entries_run.ColumnInt64(0);
entry.sha1 = video_sample_entries_run.ColumnBlob(1).as_string();
int64_t width_tmp = video_sample_entries_run.ColumnInt64(2);
int64_t height_tmp = video_sample_entries_run.ColumnInt64(3);
auto max = std::numeric_limits<uint16_t>::max();
if (width_tmp <= 0 || width_tmp > max || height_tmp <= 0 ||
height_tmp > max) {
*error_message =
StrCat("video_sample_entry id ", entry.id, " width ", width_tmp,
" / height ", height_tmp, " out of range.");
return false;
}
entry.width = width_tmp;
entry.height = height_tmp;
entry.data = video_sample_entries_run.ColumnBlob(4).as_string();
CHECK(
video_sample_entries_.insert(std::make_pair(entry.id, entry)).second)
<< "duplicate: " << entry.id;
}
}
get_camera_query_ = db_->Prepare(
R"(
select
uuid,
short_name,
description,
retain_bytes
from
camera
where
id = :camera_id;)",
nullptr, error_message);
if (!get_camera_query_.valid()) {
return false;
}
list_camera_recordings_query_ = db_->Prepare(
std::string list_camera_recordings_sql = StrCat(
R"(
select
recording.start_time_90k,
recording.end_time_90k,
recording.duration_90k,
recording.video_samples,
recording.sample_file_bytes,
recording.video_sample_entry_sha1,
video_sample_entry.width,
video_sample_entry.height
recording.video_sample_entry_id
from
recording
join video_sample_entry on
(recording.video_sample_entry_sha1 = video_sample_entry.sha1)
where
recording.status = 1 and
camera_id = :camera_id
camera_id = :camera_id and
recording.start_time_90k > :start_time_90k - )",
kMaxRecordingDuration, " and\n",
R"(
recording.start_time_90k < :end_time_90k and
recording.start_time_90k + recording.duration_90k > :start_time_90k
order by
recording.start_time_90k;)",
nullptr, error_message);
if (!list_camera_recordings_query_.valid()) {
recording.start_time_90k desc;)");
list_camera_recordings_stmt_ =
db_->Prepare(list_camera_recordings_sql, nullptr, error_message);
if (!list_camera_recordings_stmt_.valid()) {
return false;
}
std::string build_mp4_sql = StrCat(
R"(
select
recording.rowid,
recording.id,
recording.start_time_90k,
recording.end_time_90k,
recording.duration_90k,
recording.sample_file_bytes,
recording.sample_file_uuid,
recording.sample_file_sha1,
recording.video_sample_entry_sha1,
recording.video_index,
recording.video_samples,
recording.video_sync_samples,
video_sample_entry.bytes,
video_sample_entry.width,
video_sample_entry.height
recording.video_sample_entry_id
from
recording join video_sample_entry on
(recording.video_sample_entry_sha1 = video_sample_entry.sha1)
recording
where
recording.status = 1 and
camera_id = :camera_id and
recording.start_time_90k > :start_time_90k - )",
kMaxRecordingDuration, " and\n",
R"(
recording.start_time_90k < :end_time_90k and
recording.end_time_90k > :start_time_90k
recording.start_time_90k + recording.duration_90k > :start_time_90k
order by
recording.start_time_90k;)");
build_mp4_query_ = db_->Prepare(build_mp4_sql, nullptr, error_message);
if (!build_mp4_query_.valid()) {
build_mp4_stmt_ = db_->Prepare(build_mp4_sql, nullptr, error_message);
if (!build_mp4_stmt_.valid()) {
return false;
}
insert_reservation_stmt_ = db_->Prepare(
"insert into reserved_sample_files (uuid, state)\n"
" values (:uuid, :state);",
nullptr, error_message);
if (!insert_reservation_stmt_.valid()) {
return false;
}
delete_reservation_stmt_ =
db_->Prepare("delete from reserved_sample_files where uuid = :uuid;",
nullptr, error_message);
if (!delete_reservation_stmt_.valid()) {
return false;
}
insert_video_sample_entry_stmt_ = db_->Prepare(
R"(
insert into video_sample_entry (sha1, width, height, data)
values (:sha1, :width, :height, :data);
)",
nullptr, error_message);
if (!insert_video_sample_entry_stmt_.valid()) {
return false;
}
insert_recording_stmt_ = db_->Prepare(
R"(
insert into recording (camera_id, sample_file_bytes, start_time_90k,
duration_90k, video_samples, video_sync_samples,
video_sample_entry_id, sample_file_uuid,
sample_file_sha1, video_index)
values (:camera_id, :sample_file_bytes, :start_time_90k,
:duration_90k, :video_samples, :video_sync_samples,
:video_sample_entry_id, :sample_file_uuid,
:sample_file_sha1, :video_index);
)",
nullptr, error_message);
if (!insert_recording_stmt_.valid()) {
return false;
}
list_oldest_sample_files_stmt_ = db_->Prepare(
R"(
select
id,
sample_file_uuid,
duration_90k,
sample_file_bytes
from
recording
where
camera_id = :camera_id
order by
start_time_90k
)",
nullptr, error_message);
if (!list_oldest_sample_files_stmt_.valid()) {
return false;
}
delete_recording_stmt_ =
db_->Prepare("delete from recording where id = :recording_id;", nullptr,
error_message);
if (!delete_recording_stmt_.valid()) {
return false;
}
camera_min_start_stmt_ = db_->Prepare(
R"(
select
start_time_90k
from
recording
where
camera_id = :camera_id
order by start_time_90k limit 1;
)",
nullptr, error_message);
if (!camera_min_start_stmt_.valid()) {
return false;
}
camera_max_start_stmt_ = db_->Prepare(
R"(
select
start_time_90k,
duration_90k
from
recording
where
camera_id = :camera_id
order by start_time_90k desc;
)",
nullptr, error_message);
if (!camera_max_start_stmt_.valid()) {
return false;
}
return true;
}
bool MoonfireDatabase::ListCameras(
std::function<IterationControl(const ListCamerasRow &)> cb,
std::string *error_message) {
void MoonfireDatabase::ListCameras(
std::function<IterationControl(const ListCamerasRow &)> cb) {
DatabaseContext ctx(db_);
auto run = ctx.Borrow(&list_cameras_query_);
ListCamerasRow row;
while (run.Step() == SQLITE_ROW) {
row.id = run.ColumnInt64(0);
if (!row.uuid.ParseBinary(run.ColumnBlob(1))) {
*error_message = StrCat("invalid uuid in row id ", row.id);
return false;
}
row.short_name = run.ColumnText(2).as_string();
row.description = run.ColumnText(3).as_string();
row.retain_bytes = run.ColumnInt64(4);
row.min_recording_start_time_90k = run.ColumnInt64(5);
row.max_recording_end_time_90k = run.ColumnInt64(6);
row.total_recording_duration_90k = run.ColumnInt64(7);
row.total_sample_file_bytes = run.ColumnInt64(8);
for (const auto &entry : cameras_by_uuid_) {
row.uuid = entry.first;
row.short_name = entry.second.short_name;
row.description = entry.second.description;
row.retain_bytes = entry.second.retain_bytes;
row.min_start_time_90k = entry.second.min_start_time_90k;
row.max_end_time_90k = entry.second.max_end_time_90k;
row.total_duration_90k = entry.second.total_duration_90k;
row.total_sample_file_bytes = entry.second.total_sample_file_bytes;
if (cb(row) == IterationControl::kBreak) {
break;
return;
}
}
if (run.status() != SQLITE_DONE) {
*error_message = StrCat("sqlite query failed: ", run.error_message());
return false;
}
return true;
return;
}
bool MoonfireDatabase::GetCamera(int64_t camera_id, GetCameraRow *row,
std::string *error_message) {
bool MoonfireDatabase::GetCamera(Uuid camera_uuid, GetCameraRow *row) {
DatabaseContext ctx(db_);
auto run = ctx.Borrow(&get_camera_query_);
run.BindInt64(":camera_id", camera_id);
if (run.Step() == SQLITE_ROW) {
if (!row->uuid.ParseBinary(run.ColumnBlob(0))) {
*error_message =
StrCat("unable to parse uuid ", ToHex(run.ColumnBlob(0)));
return false;
}
row->short_name = run.ColumnText(1).as_string();
row->description = run.ColumnText(2).as_string();
row->retain_bytes = run.ColumnInt64(3);
} else if (run.status() == SQLITE_DONE) {
*error_message = "no such camera";
return false;
}
if (run.Step() == SQLITE_ROW) {
*error_message = "multiple rows returned unexpectedly";
const auto it = cameras_by_uuid_.find(camera_uuid);
if (it == cameras_by_uuid_.end()) {
return false;
}
const CameraData &data = it->second;
row->short_name = data.short_name;
row->description = data.description;
row->retain_bytes = data.retain_bytes;
row->min_start_time_90k = data.min_start_time_90k;
row->max_end_time_90k = data.max_end_time_90k;
row->total_duration_90k = data.total_duration_90k;
row->total_sample_file_bytes = data.total_sample_file_bytes;
return true;
}
bool MoonfireDatabase::ListCameraRecordings(
int64_t camera_id,
Uuid camera_uuid, int64_t start_time_90k, int64_t end_time_90k,
std::function<IterationControl(const ListCameraRecordingsRow &)> cb,
std::string *error_message) {
DatabaseContext ctx(db_);
auto run = ctx.Borrow(&list_camera_recordings_query_);
run.BindInt64(":camera_id", camera_id);
const auto camera_it = cameras_by_uuid_.find(camera_uuid);
if (camera_it == cameras_by_uuid_.end()) {
*error_message = StrCat("no such camera ", camera_uuid.UnparseText());
return false;
}
auto run = ctx.Borrow(&list_camera_recordings_stmt_);
run.BindInt64(":camera_id", camera_it->second.id);
run.BindInt64(":start_time_90k", start_time_90k);
run.BindInt64(":end_time_90k", end_time_90k);
ListCameraRecordingsRow row;
while (run.Step() == SQLITE_ROW) {
row.start_time_90k = run.ColumnInt64(0);
row.end_time_90k = run.ColumnInt64(1);
row.end_time_90k = row.start_time_90k + run.ColumnInt64(1);
row.video_samples = run.ColumnInt64(2);
row.sample_file_bytes = run.ColumnInt64(3);
auto video_sample_entry_sha1 = run.ColumnBlob(4);
row.video_sample_entry_sha1.assign(video_sample_entry_sha1.data(),
video_sample_entry_sha1.size());
row.width = run.ColumnInt64(5);
row.height = run.ColumnInt64(6);
int64_t video_sample_entry_id = run.ColumnInt64(4);
const auto it = video_sample_entries_.find(video_sample_entry_id);
if (it == video_sample_entries_.end()) {
*error_message =
StrCat("recording references invalid video sample entry ",
video_sample_entry_id);
return false;
}
const VideoSampleEntry &entry = it->second;
row.video_sample_entry_sha1 = entry.sha1;
row.width = entry.width;
row.height = entry.height;
if (cb(row) == IterationControl::kBreak) {
break;
return true;
}
}
if (run.status() != SQLITE_DONE) {
@ -234,27 +383,34 @@ bool MoonfireDatabase::ListCameraRecordings(
}
bool MoonfireDatabase::ListMp4Recordings(
int64_t camera_id, int64_t start_time_90k, int64_t end_time_90k,
Uuid camera_uuid, int64_t start_time_90k, int64_t end_time_90k,
std::function<IterationControl(Recording &, const VideoSampleEntry &)>
row_cb,
std::string *error_message) {
VLOG(1) << "...(1/4): Waiting for database lock";
DatabaseContext ctx(db_);
const auto it = cameras_by_uuid_.find(camera_uuid);
if (it == cameras_by_uuid_.end()) {
*error_message = StrCat("no such camera ", camera_uuid.UnparseText());
return false;
}
const CameraData &data = it->second;
VLOG(1) << "...(2/4): Querying database";
auto run = ctx.Borrow(&build_mp4_query_);
run.BindInt64(":camera_id", camera_id);
auto run = ctx.Borrow(&build_mp4_stmt_);
run.BindInt64(":camera_id", data.id);
run.BindInt64(":end_time_90k", end_time_90k);
run.BindInt64(":start_time_90k", start_time_90k);
Recording recording;
VideoSampleEntry sample_entry;
while (run.Step() == SQLITE_ROW) {
recording.rowid = run.ColumnInt64(0);
recording.id = run.ColumnInt64(0);
recording.camera_id = data.id;
recording.start_time_90k = run.ColumnInt64(1);
recording.end_time_90k = run.ColumnInt64(2);
recording.end_time_90k = recording.start_time_90k + run.ColumnInt64(2);
recording.sample_file_bytes = run.ColumnInt64(3);
if (!recording.sample_file_uuid.ParseBinary(run.ColumnBlob(4))) {
*error_message =
StrCat("recording ", recording.rowid, " has unparseable uuid ",
StrCat("recording ", recording.id, " has unparseable uuid ",
ToHex(run.ColumnBlob(4)));
return false;
}
@ -262,19 +418,23 @@ bool MoonfireDatabase::ListMp4Recordings(
StrCat("/home/slamb/new-moonfire/sample/",
recording.sample_file_uuid.UnparseText());
recording.sample_file_sha1 = run.ColumnBlob(5).as_string();
recording.video_sample_entry_sha1 = run.ColumnBlob(6).as_string();
recording.video_index = run.ColumnBlob(7).as_string();
recording.video_samples = run.ColumnInt64(8);
recording.video_sync_samples = run.ColumnInt64(9);
recording.video_index = run.ColumnBlob(6).as_string();
recording.video_samples = run.ColumnInt64(7);
recording.video_sync_samples = run.ColumnInt64(8);
recording.video_sample_entry_id = run.ColumnInt64(9);
if (recording.video_sample_entry_sha1 != sample_entry.sha1) {
sample_entry.sha1 = run.ColumnBlob(6).as_string();
sample_entry.data = run.ColumnBlob(10).as_string();
sample_entry.width = run.ColumnInt64(11);
sample_entry.height = run.ColumnInt64(12);
auto it = video_sample_entries_.find(recording.video_sample_entry_id);
if (it == video_sample_entries_.end()) {
*error_message = StrCat("recording ", recording.id,
" references unknown video sample entry ",
recording.video_sample_entry_id);
return false;
}
const VideoSampleEntry &entry = it->second;
row_cb(recording, sample_entry);
if (row_cb(recording, entry) == IterationControl::kBreak) {
return true;
}
}
if (run.status() != SQLITE_DONE && run.status() != SQLITE_ROW) {
*error_message = StrCat("sqlite query failed: ", run.error_message());
@ -283,10 +443,30 @@ bool MoonfireDatabase::ListMp4Recordings(
return true;
}
bool MoonfireDatabase::ListReservedSampleFiles(std::vector<Uuid> *reserved,
std::string *error_message) {
reserved->clear();
DatabaseContext ctx(db_);
auto run = ctx.UseOnce("select uuid from reserved_sample_files;");
while (run.Step() == SQLITE_ROW) {
Uuid uuid;
if (!uuid.ParseBinary(run.ColumnBlob(0))) {
*error_message = StrCat("unparseable uuid ", ToHex(run.ColumnBlob(0)));
return false;
}
reserved->push_back(uuid);
}
if (run.status() != SQLITE_DONE) {
*error_message = run.error_message();
return false;
}
return true;
}
std::shared_ptr<VirtualFile> MoonfireDatabase::BuildMp4(
int64_t camera_id, int64_t start_time_90k, int64_t end_time_90k,
Uuid camera_uuid, int64_t start_time_90k, int64_t end_time_90k,
std::string *error_message) {
LOG(INFO) << "Building mp4 for camera: " << camera_id
LOG(INFO) << "Building mp4 for camera: " << camera_uuid.UnparseText()
<< ", start_time_90k: " << start_time_90k
<< ", end_time_90k: " << end_time_90k;
@ -315,11 +495,11 @@ std::shared_ptr<VirtualFile> MoonfireDatabase::BuildMp4(
next_row_start_time_90k = recording.end_time_90k;
if (rows > 0 && recording.video_sample_entry_sha1 != sample_entry.sha1) {
if (rows > 0 && recording.video_sample_entry_id != sample_entry.id) {
*error_message =
StrCat("inconsistent video sample entries: this recording has ",
ToHex(recording.video_sample_entry_sha1), ", previous had ",
ToHex(sample_entry.sha1));
StrCat("inconsistent video sample entries: this recording has id ",
recording.video_sample_entry_id, " previous had ",
sample_entry.id, " (sha1 ", ToHex(sample_entry.sha1), ")");
ok = false;
return IterationControl::kBreak;
} else if (rows == 0) {
@ -334,7 +514,7 @@ std::shared_ptr<VirtualFile> MoonfireDatabase::BuildMp4(
return IterationControl::kContinue;
};
if (!ok ||
!ListMp4Recordings(camera_id, start_time_90k, end_time_90k, row_cb,
!ListMp4Recordings(camera_uuid, start_time_90k, end_time_90k, row_cb,
error_message)) {
return false;
}
@ -362,4 +542,329 @@ std::shared_ptr<VirtualFile> MoonfireDatabase::BuildMp4(
return file;
}
std::vector<Uuid> MoonfireDatabase::ReserveSampleFiles(
int n, std::string *error_message) {
if (n == 0) {
return std::vector<Uuid>();
}
auto *gen = GetRealUuidGenerator();
std::vector<Uuid> uuids;
uuids.reserve(n);
for (int i = 0; i < n; ++i) {
uuids.push_back(gen->Generate());
}
DatabaseContext ctx(db_);
if (!ctx.BeginTransaction(error_message)) {
return std::vector<Uuid>();
}
for (const auto &uuid : uuids) {
auto run = ctx.Borrow(&insert_reservation_stmt_);
run.BindBlob(":uuid", uuid.binary_view());
run.BindInt64(":state", static_cast<int64_t>(ReservationState::kWriting));
if (run.Step() != SQLITE_DONE) {
ctx.RollbackTransaction();
*error_message = run.error_message();
return std::vector<Uuid>();
}
}
if (!ctx.CommitTransaction(error_message)) {
return std::vector<Uuid>();
}
return uuids;
}
bool MoonfireDatabase::InsertVideoSampleEntry(VideoSampleEntry *entry,
std::string *error_message) {
if (entry->id != -1) {
*error_message = StrCat("video_sample_entry already has id ", entry->id);
return false;
}
DatabaseContext ctx(db_);
for (const auto &some_entry : video_sample_entries_) {
if (some_entry.second.sha1 == entry->sha1) {
if (entry->width != some_entry.second.width ||
entry->height != some_entry.second.height) {
*error_message =
StrCat("inconsistent entry for sha1 ", ToHex(entry->sha1),
": existing entry has ", some_entry.second.width, "x",
some_entry.second.height, ", new entry has ", entry->width,
"x", entry->height);
return false;
}
entry->id = some_entry.first;
return true;
}
}
auto insert_run = ctx.Borrow(&insert_video_sample_entry_stmt_);
insert_run.BindBlob(":sha1", entry->sha1);
insert_run.BindInt64(":width", entry->width);
insert_run.BindInt64(":height", entry->height);
insert_run.BindBlob(":data", entry->data);
if (insert_run.Step() != SQLITE_DONE) {
*error_message = insert_run.error_message();
return false;
}
entry->id = ctx.last_insert_rowid();
CHECK(video_sample_entries_.insert(std::make_pair(entry->id, *entry)).second)
<< "duplicate: " << entry->id;
return true;
}
bool MoonfireDatabase::InsertRecording(Recording *recording,
std::string *error_message) {
if (recording->id != -1) {
*error_message = StrCat("recording already has id ", recording->id);
return false;
}
if (recording->end_time_90k <= recording->start_time_90k) {
*error_message =
StrCat("end time ", recording->end_time_90k,
" must be greater than start time ", recording->start_time_90k);
return false;
}
DatabaseContext ctx(db_);
auto it = cameras_by_id_.find(recording->camera_id);
if (it == cameras_by_id_.end()) {
*error_message = StrCat("no camera with id ", recording->camera_id);
return false;
}
CameraData *camera_data = it->second;
if (!ctx.BeginTransaction(error_message)) {
return false;
}
auto delete_run = ctx.Borrow(&delete_reservation_stmt_);
delete_run.BindBlob(":uuid", recording->sample_file_uuid.binary_view());
if (delete_run.Step() != SQLITE_DONE) {
*error_message = delete_run.error_message();
ctx.RollbackTransaction();
return false;
}
if (ctx.changes() != 1) {
*error_message = StrCat("uuid ", recording->sample_file_uuid.UnparseText(),
" is not reserved");
ctx.RollbackTransaction();
return false;
}
auto insert_run = ctx.Borrow(&insert_recording_stmt_);
insert_run.BindInt64(":camera_id", recording->camera_id);
insert_run.BindInt64(":sample_file_bytes", recording->sample_file_bytes);
insert_run.BindInt64(":start_time_90k", recording->start_time_90k);
insert_run.BindInt64(":duration_90k",
recording->end_time_90k - recording->start_time_90k);
insert_run.BindInt64(":video_samples", recording->video_samples);
insert_run.BindInt64(":video_sync_samples", recording->video_sync_samples);
insert_run.BindInt64(":video_sample_entry_id",
recording->video_sample_entry_id);
insert_run.BindBlob(":sample_file_uuid",
recording->sample_file_uuid.binary_view());
insert_run.BindBlob(":sample_file_sha1", recording->sample_file_sha1);
insert_run.BindBlob(":video_index", recording->video_index);
if (insert_run.Step() != SQLITE_DONE) {
LOG(ERROR) << "insert_run failed: " << insert_run.error_message();
*error_message = insert_run.error_message();
ctx.RollbackTransaction();
return false;
}
if (!ctx.CommitTransaction(error_message)) {
LOG(ERROR) << "commit failed";
return false;
}
recording->id = ctx.last_insert_rowid();
if (camera_data->min_start_time_90k == -1 ||
camera_data->min_start_time_90k > recording->start_time_90k) {
camera_data->min_start_time_90k = recording->start_time_90k;
}
if (camera_data->max_end_time_90k == -1 ||
camera_data->max_end_time_90k < recording->end_time_90k) {
camera_data->max_end_time_90k = recording->end_time_90k;
}
camera_data->total_duration_90k +=
recording->end_time_90k - recording->start_time_90k;
camera_data->total_sample_file_bytes += recording->sample_file_bytes;
return true;
}
bool MoonfireDatabase::ListOldestSampleFiles(
Uuid camera_uuid,
std::function<IterationControl(const ListOldestSampleFilesRow &)> row_cb,
std::string *error_message) {
DatabaseContext ctx(db_);
auto it = cameras_by_uuid_.find(camera_uuid);
if (it == cameras_by_uuid_.end()) {
*error_message = StrCat("no such camera ", camera_uuid.UnparseText());
return false;
}
const CameraData &camera_data = it->second;
auto run = ctx.Borrow(&list_oldest_sample_files_stmt_);
run.BindInt64(":camera_id", camera_data.id);
ListOldestSampleFilesRow row;
while (run.Step() == SQLITE_ROW) {
row.camera_id = camera_data.id;
row.recording_id = run.ColumnInt64(0);
if (!row.sample_file_uuid.ParseBinary(run.ColumnBlob(1))) {
*error_message =
StrCat("recording ", row.recording_id, " has unparseable uuid ",
ToHex(run.ColumnBlob(1)));
return false;
}
row.duration_90k = run.ColumnInt64(2);
row.sample_file_bytes = run.ColumnInt64(3);
if (row_cb(row) == IterationControl::kBreak) {
return true;
}
}
if (run.status() != SQLITE_DONE) {
*error_message = run.error_message();
return false;
}
return true;
}
bool MoonfireDatabase::DeleteRecordings(
const std::vector<ListOldestSampleFilesRow> &recordings,
std::string *error_message) {
if (recordings.empty()) {
return true;
}
DatabaseContext ctx(db_);
if (!ctx.BeginTransaction(error_message)) {
return false;
}
struct State {
int64_t deleted_duration_90k = 0;
int64_t deleted_sample_file_bytes = 0;
int64_t min_start_time_90k = -1;
int64_t max_end_time_90k = -1;
CameraData *camera_data = nullptr;
};
std::map<int64_t, State> state_by_camera_id;
for (const auto &recording : recordings) {
State &state = state_by_camera_id[recording.camera_id];
state.deleted_duration_90k += recording.duration_90k;
state.deleted_sample_file_bytes += recording.sample_file_bytes;
auto delete_run = ctx.Borrow(&delete_recording_stmt_);
delete_run.BindInt64(":recording_id", recording.recording_id);
if (delete_run.Step() != SQLITE_DONE) {
ctx.RollbackTransaction();
*error_message = StrCat("delete: ", delete_run.error_message());
return false;
}
if (ctx.changes() != 1) {
ctx.RollbackTransaction();
*error_message = StrCat("no such recording ", recording.recording_id);
return false;
}
auto insert_run = ctx.Borrow(&insert_reservation_stmt_);
insert_run.BindBlob(":uuid", recording.sample_file_uuid.binary_view());
insert_run.BindInt64(":state",
static_cast<int64_t>(ReservationState::kDeleting));
if (insert_run.Step() != SQLITE_DONE) {
ctx.RollbackTransaction();
*error_message = StrCat("insert: ", insert_run.error_message());
return false;
}
}
// Recompute start and end times for each camera.
for (auto &state_entry : state_by_camera_id) {
int64_t camera_id = state_entry.first;
State &state = state_entry.second;
auto it = cameras_by_id_.find(camera_id);
if (it == cameras_by_id_.end()) {
*error_message =
StrCat("internal error; can't find camera id ", camera_id);
return false;
}
state.camera_data = it->second;
// The minimum is straightforward, taking advantage of the start_time_90k
// index for speed.
auto min_run = ctx.Borrow(&camera_min_start_stmt_);
min_run.BindInt64(":camera_id", camera_id);
if (min_run.Step() == SQLITE_ROW) {
state.min_start_time_90k = min_run.ColumnInt64(0);
} else if (min_run.Step() == SQLITE_DONE) {
// There are no recordings left.
state.min_start_time_90k = -1;
state.max_end_time_90k = -1;
continue; // skip additional query below to calculate max.
} else {
ctx.RollbackTransaction();
*error_message = StrCat("min: ", min_run.error_message());
return false;
}
// The maximum is less straightforward in the case of overlap - all
// recordings starting in the last kMaxRecordingDuration must be examined
// to take advantage of the start_time_90k index.
auto max_run = ctx.Borrow(&camera_max_start_stmt_);
max_run.BindInt64(":camera_id", camera_id);
if (max_run.Step() != SQLITE_ROW) {
// If there was a min row, there should be a max row too, so this is an
// error even in the SQLITE_DONE case.
ctx.RollbackTransaction();
*error_message = StrCat("max[0]: ", max_run.error_message());
return false;
}
int64_t max_start_90k = max_run.ColumnInt64(0);
do {
auto end_time_90k = max_run.ColumnInt64(0) + max_run.ColumnInt64(1);
state.max_end_time_90k = std::max(state.max_end_time_90k, end_time_90k);
} while (max_run.Step() == SQLITE_ROW &&
max_run.ColumnInt64(0) > max_start_90k - kMaxRecordingDuration);
if (max_run.status() != SQLITE_DONE && max_run.status() != SQLITE_ROW) {
*error_message = StrCat("max[1]: ", max_run.error_message());
ctx.RollbackTransaction();
return false;
}
}
if (!ctx.CommitTransaction(error_message)) {
*error_message = StrCat("commit: ", *error_message);
return false;
}
for (auto &state_entry : state_by_camera_id) {
State &state = state_entry.second;
state.camera_data->total_duration_90k -= state.deleted_duration_90k;
state.camera_data->total_sample_file_bytes -=
state.deleted_sample_file_bytes;
state.camera_data->min_start_time_90k = state.min_start_time_90k;
state.camera_data->max_end_time_90k = state.max_end_time_90k;
}
return true;
}
bool MoonfireDatabase::MarkSampleFilesDeleted(const std::vector<Uuid> &uuids,
std::string *error_message) {
if (uuids.empty()) {
return true;
}
DatabaseContext ctx(db_);
if (!ctx.BeginTransaction(error_message)) {
return false;
}
for (const auto &uuid : uuids) {
auto run = ctx.Borrow(&delete_reservation_stmt_);
run.BindBlob(":uuid", uuid.binary_view());
if (run.Step() != SQLITE_DONE) {
*error_message = run.error_message();
ctx.RollbackTransaction();
return false;
}
if (ctx.changes() != 1) {
*error_message = StrCat("no reservation for uuid ", uuid.UnparseText());
ctx.RollbackTransaction();
return false;
}
}
if (!ctx.CommitTransaction(error_message)) {
return false;
}
return true;
}
} // namespace moonfire_nvr

183
src/moonfire-db.h
View File

@ -32,18 +32,30 @@
// Currently focused on stuff needed by WebInterface to build a HTML or JSON
// interface.
//
// Performance note: camera-level operations do a sequential scan through
// essentially the entire database. This is unacceptable for full-sized
// databases; it will have to be measured and improved. Ideas:
// This caches data in RAM, making the assumption that only one process is
// accessing the database at a time. (TODO: enforce with flock or some such.)
// Performance and efficiency notes:
//
// * separate the video index blob from the rest of the recording row,
// as it's expected to be 10X-100X larger than everything else and not
// necessary for these operations.
// * paged results + SQL indexes (but this may only help so much, as it'd be
// useful to at least see what days have recordings in one go).
// * keep aggregates, either in-memory or as denormalized data in the camera
// table. Likely integrating with the recording system, although triggers
// may also be possible.
// * several query operations here feature row callbacks. The callback is
// invoked with the database lock. Thus, the caller mustn't perform database
// operations or other long-running operations.
//
// * startup may be slow, as it scans the entire index for the recording
// table. This seems acceptable.
//
// * the operations used for web file serving should return results with
// acceptable latency.
//
// * however, the database lock may be held for longer than is acceptable for
// the critical path of recording frames. It may be necessary to preallocate
// sample file uuids and such to avoid this.
//
// * the caller may need to perform several different types of write
// operations in a row. It might be worth creating an interface for batching
// these inside a transaction, to reduce latency and SSD write cycles. The
// pre-commit and post-commit logic of each operation would have to be
// pulled apart, with the latter being called by this wrapper class on
// commit of the overall transaction.
#ifndef MOONFIRE_NVR_MOONFIRE_DB_H
#define MOONFIRE_NVR_MOONFIRE_DB_H
@ -51,6 +63,7 @@
#include <functional>
#include <memory>
#include <string>
#include <vector>
#include "common.h"
#include "http.h"
@ -62,28 +75,32 @@ namespace moonfire_nvr {
// For use with MoonfireDatabase::ListCameras.
struct ListCamerasRow {
int64_t id = -1;
Uuid uuid;
std::string short_name;
std::string description;
int64_t retain_bytes = -1;
// Aggregates summarizing completed (status=1) recordings.
int64_t min_recording_start_time_90k = -1;
int64_t max_recording_end_time_90k = -1;
int64_t total_recording_duration_90k = -1;
// Aggregates summarizing completed recordings.
int64_t min_start_time_90k = -1;
int64_t max_end_time_90k = -1;
int64_t total_duration_90k = -1;
int64_t total_sample_file_bytes = -1;
};
// For use with MoonfireDatabase::GetCamera.
// This is the same information as in ListCamerasRow minus the stuff
// that's calculable from ListCameraRecordingsRow, which the camera details
// webpage also grabs.
// This includes everything in ListCamerasRow. In the future, it will include
// more data. Likely, that will mean a list of calendar days (in the system
// time zone) in which there is any data.
struct GetCameraRow {
int64_t retain_bytes = -1;
Uuid uuid;
std::string short_name;
std::string description;
int64_t retain_bytes = -1;
int64_t min_start_time_90k = -1;
int64_t max_end_time_90k = -1;
int64_t total_duration_90k = -1;
int64_t total_sample_file_bytes = -1;
// TODO: std::vector<std::string> days; // keys: YYYY-mm-dd.
};
// For use with MoonfireDatabase::ListCameraRecordings.
@ -93,53 +110,137 @@ struct ListCameraRecordingsRow {
int64_t end_time_90k = -1;
int64_t video_samples = -1;
int64_t sample_file_bytes = -1;
std::string video_sample_entry_sha1;
// Joined from the video_sample_entry table.
int64_t width = -1;
int64_t height = -1;
// |video_sample_entry_sha1| is valid as long as the MoonfireDatabase.
re2::StringPiece video_sample_entry_sha1;
uint16_t width = 0;
uint16_t height = 0;
};
// For use with MoonfireDatabase::ListOldestSampleFiles.
struct ListOldestSampleFilesRow {
int64_t camera_id = -1;
int64_t recording_id = -1;
Uuid sample_file_uuid;
int64_t duration_90k = -1;
int64_t sample_file_bytes = -1;
};
// Thread-safe after Init.
// (Uses a DatabaseContext for locking.)
class MoonfireDatabase {
public:
explicit MoonfireDatabase(Database *db) : db_(db) {}
MoonfireDatabase() {}
MoonfireDatabase(const MoonfireDatabase &) = delete;
void operator=(const MoonfireDatabase &) = delete;
bool Init(std::string *error_message);
// |db| must outlive the MoonfireDatabase.
bool Init(Database *db, std::string *error_message);
// List all cameras in the system, ordered by short name.
// Holds database lock; callback should be quick.
bool ListCameras(std::function<IterationControl(const ListCamerasRow &)> cb,
std::string *error_message);
void ListCameras(std::function<IterationControl(const ListCamerasRow &)> cb);
bool GetCamera(int64_t camera_id, GetCameraRow *row,
std::string *error_message);
// Get a single camera.
// Return true iff the camera exists.
bool GetCamera(Uuid camera_uuid, GetCameraRow *row);
// List all recordings associated with a camera, ordered by start time..
// Holds database lock; callback should be quick.
// List all recordings associated with a camera, descending by end time.
bool ListCameraRecordings(
int64_t camera_id,
Uuid camera_uuid, int64_t start_time_90k, int64_t end_time_90k,
std::function<IterationControl(const ListCameraRecordingsRow &)>,
std::string *error_message);
bool ListMp4Recordings(
int64_t camera_id, int64_t start_time_90k, int64_t end_time_90k,
Uuid camera_uuid, int64_t start_time_90k, int64_t end_time_90k,
std::function<IterationControl(Recording &, const VideoSampleEntry &)>
row_cb,
std::string *error_message);
std::shared_ptr<VirtualFile> BuildMp4(int64_t camera_id,
// TODO: more nuanced error code for HTTP.
// TODO: this should move somewhere that has access to the
// currently-writing Recording as well.
std::shared_ptr<VirtualFile> BuildMp4(Uuid camera_uuid,
int64_t start_time_90k,
int64_t end_time_90k,
std::string *error_message);
bool ListReservedSampleFiles(std::vector<Uuid> *reserved,
std::string *error_message);
// Reserve |n| new sample file uuids.
// Returns an empty vector on error.
std::vector<Uuid> ReserveSampleFiles(int n, std::string *error_message);
// Insert a video sample entry if not already inserted.
// On success, |entry->id| is filled in with the id of a freshly-created or
// existing row.
bool InsertVideoSampleEntry(VideoSampleEntry *entry,
std::string *error_message);
// Insert a new recording.
// The uuid must have been already reserved with ReserveSampleFileUuid above.
// On success, |recording->id| is filled in.
bool InsertRecording(Recording *recording, std::string *error_message);
// List sample files, starting from the oldest.
// The caller is expected to supply a |row_cb| that returns kBreak when
// enough have been listed.
bool ListOldestSampleFiles(
Uuid camera_uuid,
std::function<IterationControl(const ListOldestSampleFilesRow &)> row_cb,
std::string *error_message);
// Delete recording rows, moving their sample file uuids to the deleting
// state.
bool DeleteRecordings(const std::vector<ListOldestSampleFilesRow> &rows,
std::string *error_message);
// Mark a set of sample files as deleted.
// This shouldn't be called until the files have been unlinke()ed and the
// parent directory fsync()ed.
// Returns error if any sample files are not in the deleting state.
bool MarkSampleFilesDeleted(const std::vector<Uuid> &uuids,
std::string *error_message);
private:
Database *const db_;
Statement list_cameras_query_;
Statement get_camera_query_;
Statement list_camera_recordings_query_;
Statement build_mp4_query_;
struct CameraData {
// Cached values of the matching fields from the camera row.
int64_t id = -1;
std::string short_name;
std::string description;
int64_t retain_bytes = -1;
// Aggregates of all recordings associated with the camera.
int64_t min_start_time_90k = -1;
int64_t max_end_time_90k = -1;
int64_t total_sample_file_bytes = -1;
int64_t total_duration_90k = -1;
};
enum class ReservationState { kWriting = 0, kDeleting = 1 };
// Efficiently (re-)compute the bounds of recorded time for a given camera.
bool ComputeCameraRecordingBounds(DatabaseContext *ctx, int64_t camera_id,
int64_t *min_start_time_90k,
int64_t *max_end_time_90k,
std::string *error_message);
Database *db_ = nullptr;
Statement list_camera_recordings_stmt_;
Statement build_mp4_stmt_;
Statement insert_reservation_stmt_;
Statement delete_reservation_stmt_;
Statement insert_video_sample_entry_stmt_;
Statement insert_recording_stmt_;
Statement list_oldest_sample_files_stmt_;
Statement delete_recording_stmt_;
Statement camera_min_start_stmt_;
Statement camera_max_start_stmt_;
std::map<Uuid, CameraData> cameras_by_uuid_;
std::map<int64_t, CameraData *> cameras_by_id_;
std::map<int64_t, VideoSampleEntry> video_sample_entries_;
};
} // namespace moonfire_nvr

2
src/moonfire-nvr-test.cc
View File

@ -148,7 +148,7 @@ class StreamTest : public testing::Test {
env_.clock = &clock_;
env_.video_source = &video_source_;
env_.fs = GetRealFilesystem();
clock_.Sleep({1430006400, 0}); // 2016-04-26 00:00:00 UTC
clock_.Sleep({1430006400, 0}); // 2015-04-26 00:00:00 UTC
config_.set_base_path(test_dir_);
config_.set_rotate_sec(5);

11
src/mp4.cc
View File

@ -727,12 +727,11 @@ Mp4FileBuilder &Mp4FileBuilder::SetSampleEntry(const VideoSampleEntry &entry) {
std::shared_ptr<VirtualFile> Mp4FileBuilder::Build(std::string *error_message) {
int32_t sample_offset = 1;
for (auto &segment : segments_) {
if (segment->recording.video_sample_entry_sha1 !=
video_sample_entry_.sha1) {
*error_message =
StrCat("inconsistent video sample entries. builder has: ",
ToHex(video_sample_entry_.sha1), ", segment has: ",
ToHex(segment->recording.video_sample_entry_sha1));
if (segment->recording.video_sample_entry_id != video_sample_entry_.id) {
*error_message = StrCat(
"inconsistent video sample entries. builder has: ",
video_sample_entry_.id, " (sha1 ", ToHex(video_sample_entry_.sha1),
", segment has: ", segment->recording.video_sample_entry_id);
return std::shared_ptr<VirtualFile>();
}

9
src/recording.h
View File

@ -55,16 +55,19 @@ constexpr int64_t kTimeUnitsPerSecond = 90000;
// This limit should be more than the normal rotation time,
// as recording doesn't happen until the next key frame.
// 5 minutes is generously more than 1 minute, but still sufficient to
// allow the optimization to be useful.
// allow the optimization to be useful. This value must match the CHECK
// constraint on duration_90k in schema.sql.
constexpr int64_t kMaxRecordingDuration = 5 * 60 * kTimeUnitsPerSecond;
// Various fields from the "recording" table which are useful when viewing
// recordings.
struct Recording {
int64_t rowid = -1;
int64_t id = -1;
int64_t camera_id = -1;
std::string sample_file_path;
std::string sample_file_sha1;
Uuid sample_file_uuid;
int64_t video_sample_entry_id = -1;
// Fields populated by SampleIndexEncoder.
int64_t start_time_90k = -1;
@ -72,7 +75,6 @@ struct Recording {
int64_t sample_file_bytes = -1;
int64_t video_samples = -1;
int64_t video_sync_samples = -1;
std::string video_sample_entry_sha1;
std::string video_index;
};
@ -196,6 +198,7 @@ class SampleFileWriter {
};
struct VideoSampleEntry {
int64_t id = -1;
std::string sha1;
std::string data;
uint16_t width = 0;

73
src/schema.sql
View File

@ -31,14 +31,14 @@
-- schema.sql: SQLite3 database schema for Moonfire NVR.
-- See also design/schema.md.
pragma journal_mode = wal;
--pragma journal_mode = wal;
create table camera (
id integer primary key,
uuid blob unique not null,
uuid blob unique,-- not null check (length(uuid) = 16),
-- A short name of the camera, used in log messages.
short_name text not null,
short_name text,-- not null,
-- A short description of the camera.
description text,
@ -63,45 +63,70 @@ create table camera (
-- The number of bytes of video to retain, excluding the currently-recording
-- file. Older files will be deleted as necessary to stay within this limit.
retain_bytes integer
retain_bytes integer not null check (retain_bytes >= 0)
);
-- A single, typically 60-second, recorded segment of video.
-- Each row represents a single completed recorded segment of video.
-- Recordings are typically ~60 seconds; never more than 5 minutes.
create table recording (
id integer primary key,
camera_id integer references camera (id) not null,
status integer not null, -- 0 (WRITING), 1 (WRITTEN), or 2 (DELETING)
sample_file_bytes integer not null check (sample_file_bytes > 0),
sample_file_uuid blob unique not null,
sample_file_sha1 blob,
sample_file_bytes integer,
-- The starting and ending time of the recording, in 90 kHz units since
-- The starting time of the recording, in 90 kHz units since
-- 1970-01-01 00:00:00 UTC.
start_time_90k integer not null,
end_time_90k integer,
start_time_90k integer not null check (start_time_90k > 0),
video_samples integer,
video_sync_samples integer,
video_sample_entry_sha1 blob references video_sample_entry (sha1),
video_index blob
-- The duration of the recording, in 90 kHz units.
duration_90k integer not null
check (duration_90k >= 0 and duration_90k < 5*60*90000),
video_samples integer not null check (video_samples > 0),
video_sync_samples integer not null check (video_samples > 0),
video_sample_entry_id integer references video_sample_entry (id),
sample_file_uuid blob not null check (length(sample_file_uuid) = 16),
sample_file_sha1 blob not null check (length(sample_file_sha1) = 20),
video_index blob not null check (length(video_index) > 0)
);
create index recording_start_time_90k on recording (start_time_90k);
create index recording_cover on recording (
-- Typical queries use "where camera_id = ? order by start_time_90k (desc)?".
camera_id,
start_time_90k,
-- These fields are not used for ordering; they cover most queries so
-- that only database verification and actual viewing of recordings need
-- to consult the underlying row.
duration_90k,
video_samples,
video_sample_entry_id,
sample_file_bytes
);
-- Files in the sample file directory which may be present but should simply be
-- discarded on startup. (Recordings which were never completed or have been
-- marked for completion.)
create table reserved_sample_files (
uuid blob primary key check (length(uuid) = 16),
state integer not null -- 0 (writing) or 1 (deleted)
) without rowid;
-- A concrete box derived from a ISO/IEC 14496-12 section 8.5.2
-- VisualSampleEntry box. Describes the codec, width, height, etc.
create table video_sample_entry (
id integer primary key,
-- A SHA-1 hash of |bytes|.
sha1 blob primary key,
sha1 blob unique not null check (length(sha1) = 20),
-- The width and height in pixels; must match values within
-- |sample_entry_bytes|.
width integer,
height integer,
width integer not null check (width > 0),
height integer not null check (height > 0),
-- A serialized SampleEntry box, including the leading length and box
-- type (avcC in the case of H.264).
bytes blob
-- The serialized box, including the leading length and box type (avcC in
-- the case of H.264).
data blob not null check (length(data) > 86)
);

2
src/sqlite-test.cc
View File

@ -78,7 +78,7 @@ TEST_F(SqliteTest, BindAndColumn) {
0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f};
re2::StringPiece blob_piece = re2::StringPiece(kBlob, sizeof(kBlob));
const char kText[] = "foo";
const int64_t kInt64 = INT64_C(0xdeadbeeffeedface);
const int64_t kInt64 = INT64_C(0xbeeffeedface);
DatabaseContext ctx(&db_);
{

75
src/sqlite.cc
View File

@ -88,7 +88,8 @@ bool DatabaseContext::BeginTransaction(std::string *error_message) {
sqlite3_step(db_->begin_transaction_.me_);
int ret = sqlite3_reset(db_->begin_transaction_.me_);
if (ret != SQLITE_OK) {
*error_message = sqlite3_errstr(ret);
*error_message =
StrCat("begin transaction: ", sqlite3_errstr(ret), " (", ret, ")");
return false;
}
transaction_open_ = true;
@ -103,7 +104,8 @@ bool DatabaseContext::CommitTransaction(std::string *error_message) {
sqlite3_step(db_->commit_transaction_.me_);
int ret = sqlite3_reset(db_->commit_transaction_.me_);
if (ret != SQLITE_OK) {
*error_message = sqlite3_errstr(ret);
*error_message =
StrCat("commit transaction: ", sqlite3_errstr(ret), " (", ret, ")");
return false;
}
transaction_open_ = false;
@ -118,7 +120,8 @@ void DatabaseContext::RollbackTransaction() {
sqlite3_step(db_->rollback_transaction_.me_);
int ret = sqlite3_reset(db_->rollback_transaction_.me_);
if (ret != SQLITE_OK) {
LOG(WARNING) << this << ": rollback failed: " << sqlite3_errstr(ret);
LOG(WARNING) << this << ": rollback failed: " << sqlite3_errstr(ret) << " ("
<< ret << ")";
return;
}
transaction_open_ = false;
@ -137,26 +140,36 @@ RunningStatement DatabaseContext::UseOnce(re2::StringPiece sql) {
RunningStatement::RunningStatement(Statement *statement,
const std::string &deferred_error,
bool owns_statement)
: statement_(statement),
error_message_(deferred_error),
owns_statement_(owns_statement) {
CHECK(!statement->borrowed_) << "Statement already borrowed!";
statement->borrowed_ = true;
: error_message_(deferred_error), owns_statement_(owns_statement) {
if (statement != nullptr && statement->valid()) {
CHECK(!statement->borrowed_) << "Statement already borrowed!";
statement->borrowed_ = true;
statement_ = statement;
} else if (error_message_.empty()) {
error_message_ = "invalid statement";
}
if (!error_message_.empty()) {
status_ = SQLITE_MISUSE;
} else if (statement == nullptr) {
status_ = SQLITE_MISUSE;
error_message_ = "invalid statement";
}
}
RunningStatement::RunningStatement(RunningStatement &&o) {
statement_ = o.statement_;
status_ = o.status_;
owns_statement_ = o.owns_statement_;
o.statement_ = nullptr;
}
RunningStatement::~RunningStatement() {
CHECK(statement_->borrowed_) << "Statement no longer borrowed!";
sqlite3_reset(statement_->me_);
sqlite3_clear_bindings(statement_->me_);
statement_->borrowed_ = false;
if (owns_statement_) {
delete statement_;
if (statement_ != nullptr) {
CHECK(statement_->borrowed_) << "Statement no longer borrowed!";
sqlite3_clear_bindings(statement_->me_);
sqlite3_reset(statement_->me_);
statement_->borrowed_ = false;
if (owns_statement_) {
delete statement_;
}
}
}
@ -168,7 +181,7 @@ void RunningStatement::BindBlob(int param, re2::StringPiece value) {
value.size(), SQLITE_TRANSIENT);
if (status_ != SQLITE_OK) {
error_message_ = StrCat("Unable to bind parameter ", param, ": ",
sqlite3_errstr(status_));
sqlite3_errstr(status_), " (", status_, ")");
}
}
@ -186,7 +199,7 @@ void RunningStatement::BindBlob(const char *name, re2::StringPiece value) {
value.size(), SQLITE_TRANSIENT);
if (status_ != SQLITE_OK) {
error_message_ = StrCat("Unable to bind parameter ", name, ": ",
sqlite3_errstr(status_));
sqlite3_errstr(status_), " (", status_, ")");
}
}
@ -197,7 +210,7 @@ void RunningStatement::BindInt64(int param, int64_t value) {
status_ = sqlite3_bind_int64(statement_->me_, param, value);
if (status_ != SQLITE_OK) {
error_message_ = StrCat("Unable to bind parameter ", param, ": ",
sqlite3_errstr(status_));
sqlite3_errstr(status_), " (", status_, ")");
}
}
@ -214,7 +227,7 @@ void RunningStatement::BindInt64(const char *name, int64_t value) {
status_ = sqlite3_bind_int64(statement_->me_, param, value);
if (status_ != SQLITE_OK) {
error_message_ = StrCat("Unable to bind parameter ", name, ": ",
sqlite3_errstr(status_));
sqlite3_errstr(status_), " (", status_, ")");
}
}
@ -226,7 +239,7 @@ void RunningStatement::BindText(int param, re2::StringPiece value) {
value.size(), SQLITE_TRANSIENT, SQLITE_UTF8);
if (status_ != SQLITE_OK) {
error_message_ = StrCat("Unable to bind parameter ", param, ": ",
sqlite3_errstr(status_));
sqlite3_errstr(status_), " (", status_, ")");
}
}
@ -243,7 +256,7 @@ void RunningStatement::BindText(const char *name, re2::StringPiece value) {
value.size(), SQLITE_TRANSIENT, SQLITE_UTF8);
if (status_ != SQLITE_OK) {
error_message_ = StrCat("Unable to bind parameter ", name, ": ",
sqlite3_errstr(status_));
sqlite3_errstr(status_), " (", status_, ")");
}
}
@ -252,7 +265,8 @@ int RunningStatement::Step() {
return status_;
}
status_ = sqlite3_step(statement_->me_);
error_message_ = sqlite3_errstr(status_);
error_message_ =
StrCat("step: ", sqlite3_errstr(status_), " (", status_, ")");
return status_;
}
@ -291,7 +305,8 @@ bool Database::Open(const char *filename, int flags,
std::call_once(global_setup, &GlobalSetup);
int ret = sqlite3_open_v2(filename, &me_, flags, nullptr);
if (ret != SQLITE_OK) {
*error_message = sqlite3_errstr(ret);
*error_message =
StrCat("open ", filename, ": ", sqlite3_errstr(ret), " (", ret, ")");
return false;
}
@ -299,8 +314,8 @@ bool Database::Open(const char *filename, int flags,
if (ret != SQLITE_OK) {
sqlite3_close(me_);
me_ = nullptr;
*error_message =
StrCat("while enabling extended result codes: ", sqlite3_errstr(ret));
*error_message = StrCat("while enabling extended result codes: ",
sqlite3_errstr(ret), " (", ret, ")");
return false;
}
@ -331,8 +346,8 @@ bool Database::Open(const char *filename, int flags,
if (ret != SQLITE_DONE) {
sqlite3_close(me_);
me_ = nullptr;
*error_message =
StrCat("while enabling foreign keys: ", sqlite3_errstr(ret));
*error_message = StrCat("while enabling foreign keys: ",
sqlite3_errstr(ret), " (", ret, ")");
return false;
}
@ -346,7 +361,7 @@ Statement Database::Prepare(re2::StringPiece sql, size_t *used,
int err =
sqlite3_prepare_v2(me_, sql.data(), sql.size(), &statement.me_, &tail);
if (err != SQLITE_OK) {
*error_message = sqlite3_errstr(err);
*error_message = StrCat("prepare: ", sqlite3_errstr(err), " (", err, ")");
return statement;
}
if (used != nullptr) {

2
src/sqlite.h
View File

@ -141,7 +141,7 @@ class Database {
// }
class RunningStatement {
public:
RunningStatement(RunningStatement &&) = default;
RunningStatement(RunningStatement &&o);
// Reset/unbind/return the statement for the next use (in the case of
// Borrow) or delete it (in the case of UseOnce).

4
src/time.h
View File

@ -66,6 +66,10 @@ inline struct timespec SecToTimespec(double sec) {
return {static_cast<time_t>(intpart), static_cast<long>(fractpart * kNanos)};
}
inline double TimespecToSec(struct timespec t) {
return t.tv_sec + static_cast<double>(t.tv_nsec) / kNanos;
}
// Returns the real wall clock, which will never be deleted.
WallClock *GetRealClock();

2
src/uuid.h
View File

@ -61,6 +61,8 @@ class Uuid {
bool operator==(const Uuid &) const;
bool operator<(const Uuid &) const;
bool is_null() const { return uuid_is_null(me_); }
private:
friend class RealUuidGenerator;
uuid_t me_;

60
src/web.cc
View File

@ -62,34 +62,35 @@ void WebInterface::HandleCameraList(evhttp_request *req, void *arg) {
"<table>\n");
auto row_cb = [&](const ListCamerasRow &row) {
auto seconds =
(row.max_recording_end_time_90k - row.min_recording_start_time_90k) /
kTimeUnitsPerSecond;
(row.max_end_time_90k - row.min_start_time_90k) / kTimeUnitsPerSecond;
std::string min_start_time_90k =
row.min_start_time_90k == -1 ? std::string("n/a")
: PrettyTimestamp(row.min_start_time_90k);
std::string max_end_time_90k = row.max_end_time_90k == -1
? std::string("n/a")
: PrettyTimestamp(row.max_end_time_90k);
buf.AddPrintf(
"<tr class=header><td colspan=2><a href=\"/camera?id=%" PRId64
"\">%s</a></td></tr>\n"
"<tr class=header><td colspan=2><a href=\"/camera?uuid=%s\">%s</a>"
"</td></tr>\n"
"<tr><td>description</td><td>%s</td></tr>\n"
"<tr><td>space</td><td>%s / %s (%.1f%%)</td></tr>\n"
"<tr><td>uuid</td><td>%s</td></tr>\n"
"<tr><td>oldest recording</td><td>%s</td></tr>\n"
"<tr><td>newest recording</td><td>%s</td></tr>\n"
"<tr><td>total duration</td><td>%s</td></tr>\n",
row.id, EscapeHtml(row.short_name).c_str(),
row.uuid.UnparseText().c_str(), EscapeHtml(row.short_name).c_str(),
EscapeHtml(row.description).c_str(),
EscapeHtml(HumanizeWithBinaryPrefix(row.total_sample_file_bytes, "B"))
.c_str(),
EscapeHtml(HumanizeWithBinaryPrefix(row.retain_bytes, "B")).c_str(),
100.f * row.total_sample_file_bytes / row.retain_bytes,
EscapeHtml(row.uuid.UnparseText()).c_str(),
EscapeHtml(PrettyTimestamp(row.min_recording_start_time_90k)).c_str(),
EscapeHtml(PrettyTimestamp(row.max_recording_end_time_90k)).c_str(),
EscapeHtml(min_start_time_90k).c_str(),
EscapeHtml(max_end_time_90k).c_str(),
EscapeHtml(HumanizeDuration(seconds)).c_str());
return IterationControl::kContinue;
};
std::string error_message;
if (!this_->mdb_->ListCameras(row_cb, &error_message)) {
return evhttp_send_error(req, HTTP_INTERNAL,
EscapeHtml(error_message).c_str());
}
this_->mdb_->ListCameras(row_cb);
buf.Add(
"</table>\n"
"</body>\n"
@ -100,19 +101,15 @@ void WebInterface::HandleCameraList(evhttp_request *req, void *arg) {
void WebInterface::HandleCameraDetail(evhttp_request *req, void *arg) {
auto *this_ = reinterpret_cast<WebInterface *>(arg);
int64_t camera_id;
Uuid camera_uuid;
QueryParameters params(evhttp_request_get_uri(req));
if (!params.ok() || !Atoi64(params.Get("id"), 10, &camera_id)) {
if (!params.ok() || !camera_uuid.ParseText(params.Get("uuid"))) {
return evhttp_send_error(req, HTTP_BADREQUEST, "bad query parameters");
}
GetCameraRow camera_row;
std::string error_message;
if (!this_->mdb_->GetCamera(camera_id, &camera_row, &error_message)) {
// TODO: more nuanced error here, such as HTTP_NOTFOUND where appropriate.
return evhttp_send_error(
req, HTTP_INTERNAL,
StrCat("sqlite query failed: ", EscapeHtml(error_message)).c_str());
if (!this_->mdb_->GetCamera(camera_uuid, &camera_row)) {
return evhttp_send_error(req, HTTP_NOTFOUND, "no such camera");
}
EvBuffer buf;
@ -151,11 +148,12 @@ void WebInterface::HandleCameraDetail(evhttp_request *req, void *arg) {
aggregated.start_time_90k) /
kTimeUnitsPerSecond;
buf.AddPrintf(
"<tr><td><a href=\"/view.mp4?camera_id=%" PRId64
"&start_time_90k=%" PRId64 "&end_time_90k=%" PRId64
"<tr><td><a href=\"/view.mp4?camera_uuid=%s&start_time_90k=%" PRId64
"&end_time_90k=%" PRId64
"\">%s</a></td><td>%s</td><td>%dx%d</td>"
"<td>%.0f</td><td>%s</td><td>%s</td></tr>\n",
camera_id, aggregated.start_time_90k, aggregated.end_time_90k,
camera_uuid.UnparseText().c_str(), aggregated.start_time_90k,
aggregated.end_time_90k,
PrettyTimestamp(aggregated.start_time_90k).c_str(),
PrettyTimestamp(aggregated.end_time_90k).c_str(),
static_cast<int>(aggregated.width), static_cast<int>(aggregated.height),
@ -169,10 +167,10 @@ void WebInterface::HandleCameraDetail(evhttp_request *req, void *arg) {
auto handle_sql_row = [&](const ListCameraRecordingsRow &row) {
auto new_duration_90k = row.end_time_90k - aggregated.start_time_90k;
if (row.video_sample_entry_sha1 == aggregated.video_sample_entry_sha1 &&
row.start_time_90k == aggregated.end_time_90k &&
row.end_time_90k == aggregated.start_time_90k &&
new_duration_90k < kForceSplitDuration90k) {
// Append to current .mp4.
aggregated.end_time_90k = row.end_time_90k;
aggregated.start_time_90k = row.start_time_90k;
aggregated.video_samples += row.video_samples;
aggregated.sample_file_bytes += row.sample_file_bytes;
} else {
@ -182,7 +180,11 @@ void WebInterface::HandleCameraDetail(evhttp_request *req, void *arg) {
}
return IterationControl::kContinue;
};
if (!this_->mdb_->ListCameraRecordings(camera_id, handle_sql_row,
int64_t start_time_90k = 0;
int64_t end_time_90k = std::numeric_limits<int64_t>::max();
std::string error_message;
if (!this_->mdb_->ListCameraRecordings(camera_uuid, start_time_90k,
end_time_90k, handle_sql_row,
&error_message)) {
return evhttp_send_error(
req, HTTP_INTERNAL,
@ -198,11 +200,11 @@ void WebInterface::HandleCameraDetail(evhttp_request *req, void *arg) {
void WebInterface::HandleMp4View(evhttp_request *req, void *arg) {
auto *this_ = reinterpret_cast<WebInterface *>(arg);
int64_t camera_id;
Uuid camera_uuid;
int64_t start_time_90k;
int64_t end_time_90k;
QueryParameters params(evhttp_request_get_uri(req));
if (!params.ok() || !Atoi64(params.Get("camera_id"), 10, &camera_id) ||
if (!params.ok() || !camera_uuid.ParseText(params.Get("camera_uuid")) ||
!Atoi64(params.Get("start_time_90k"), 10, &start_time_90k) ||
!Atoi64(params.Get("end_time_90k"), 10, &end_time_90k) ||
start_time_90k < 0 || start_time_90k >= end_time_90k) {
@ -210,7 +212,7 @@ void WebInterface::HandleMp4View(evhttp_request *req, void *arg) {
}
std::string error_message;
auto file = this_->mdb_->BuildMp4(camera_id, start_time_90k, end_time_90k,
auto file = this_->mdb_->BuildMp4(camera_uuid, start_time_90k, end_time_90k,
&error_message);
if (file == nullptr) {
// TODO: more nuanced HTTP status codes.