Expanded, reasonably efficient SQL operations.

* Schema revisions. The most dramatic is the addition of a covering index on
  (camera_id, start_time_90k) that avoids the need to make sparse accesses
  into the recording table (where the desired data is intermixed with both
  the large blobs and rows from other cameras). A query over a year's data
  previously took many seconds (6+ even in a form without the video_index)
  and now is roughly 10X faster. Queries for a couple weeks now should be
  unnoticeably fast.

  Other changes to shrink the rows, such as duration_90k instead of
  end_time_90k (more compact varint encoding) and video_sample_entry_id
  (typically 1 byte) instead of video_sample_entry_sha1 (20 bytes).
  And more CHECK constraints for good measure.

* Caching of expensive computations and logic to keep them up to date.
  The top-level web view previously went through the entire recording table,
  which was even slower. Now it is served from a small map in RAM.

* Expanded the scope of operations to cover (hopefully) everything needed for
  recording into the SQLite database.

* Added tests of MoonfireDatabase. These are basic tests that don't
  exercise a lot of error cases, but at least they exist.

The main MoonfireDatabase functionality still missing is support for quickly
seeing what calendar days have data over the full timespan of a camera. This
is more data to compute and cache.

src/CMakeLists.txt

@@ -71,7 +71,19 @@ install_programs(/bin FILES moonfire-nvr)
 include_directories(${GTest_INCLUDE_DIR})
 include_directories(${GMock_INCLUDE_DIR})
 
-foreach(test coding crypto h264 http moonfire-nvr mp4 recording sqlite string)
+set(MOONFIRE_NVR_TESTS
+    coding
+    crypto
+    h264
+    http
+    moonfire-db
+    moonfire-nvr
+    mp4
+    recording
+    sqlite
+    string)
+
+foreach(test ${MOONFIRE_NVR_TESTS})
   add_executable(${test}-test ${test}-test.cc testutil.cc)
   target_link_libraries(${test}-test GTest GMock moonfire-nvr-lib)
   add_test(NAME ${test}-test

src/moonfire-db-test.cc

@@ -0,0 +1,345 @@
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// In addition, as a special exception, the copyright holders give
+// permission to link the code of portions of this program with the
+// OpenSSL library under certain conditions as described in each
+// individual source file, and distribute linked combinations including
+// the two.
+//
+// You must obey the GNU General Public License in all respects for all
+// of the code used other than OpenSSL. If you modify file(s) with this
+// exception, you may extend this exception to your version of the
+// file(s), but you are not obligated to do so. If you do not wish to do
+// so, delete this exception statement from your version. If you delete
+// this exception statement from all source files in the program, then
+// also delete it here.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+//
+// moonfire-db-test.cc: tests of the moonfire-db.h interface.
+
+#include <string>
+
+#include <gflags/gflags.h>
+#include <gmock/gmock.h>
+#include <gtest/gtest.h>
+
+#include "moonfire-db.h"
+#include "sqlite.h"
+#include "string.h"
+#include "testutil.h"
+
+DECLARE_bool(alsologtostderr);
+
+using testing::_;
+using testing::HasSubstr;
+using testing::DoAll;
+using testing::Return;
+using testing::SetArgPointee;
+
+namespace moonfire_nvr {
+namespace {
+
+class MoonfireDbTest : public testing::Test {
+ protected:
+  MoonfireDbTest() {
+    tmpdir_ = PrepareTempDirOrDie("moonfire-db-test");
+    std::string error_message;
+    CHECK(db_.Open(StrCat(tmpdir_, "/db").c_str(),
+                   SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE, &error_message))
+        << error_message;
+    std::string create_sql = ReadFileOrDie("../src/schema.sql");
+    DatabaseContext ctx(&db_);
+    CHECK(RunStatements(&ctx, create_sql, &error_message)) << error_message;
+  }
+
+  int64_t AddCamera(Uuid uuid, re2::StringPiece short_name) {
+    DatabaseContext ctx(&db_);
+    auto run = ctx.UseOnce(
+        R"(
+        insert into camera (uuid,  short_name,  retain_bytes)
+                    values (:uuid, :short_name, :retain_bytes);
+        )");
+    run.BindBlob(":uuid", uuid.binary_view());
+    run.BindText(":short_name", short_name);
+    run.BindInt64(":retain_bytes", 42);
+    if (run.Step() != SQLITE_DONE) {
+      ADD_FAILURE() << run.error_message();
+      return -1;
+    }
+    return ctx.last_insert_rowid();
+  }
+
+  void ExpectNoRecordings(Uuid camera_uuid) {
+    int rows = 0;
+    mdb_->ListCameras([&](const ListCamerasRow &row) {
+      ++rows;
+      EXPECT_EQ(camera_uuid, row.uuid);
+      EXPECT_EQ(-1, row.min_start_time_90k);
+      EXPECT_EQ(-1, row.max_end_time_90k);
+      EXPECT_EQ(0, row.total_duration_90k);
+      EXPECT_EQ(0, row.total_sample_file_bytes);
+      return IterationControl::kContinue;
+    });
+    EXPECT_EQ(1, rows);
+
+    std::string error_message;
+    rows = 0;
+    EXPECT_TRUE(mdb_->ListCameraRecordings(
+        camera_uuid, 0, std::numeric_limits<int64_t>::max(),
+        [&](const ListCameraRecordingsRow &row) {
+          ++rows;
+          return IterationControl::kBreak;
+        },
+        &error_message))
+        << error_message;
+    EXPECT_EQ(0, rows);
+
+    rows = 0;
+    EXPECT_TRUE(mdb_->ListMp4Recordings(
+        camera_uuid, 0, std::numeric_limits<int64_t>::max(),
+        [&](Recording &recording, const VideoSampleEntry &entry) {
+          ++rows;
+          return IterationControl::kBreak;
+        },
+        &error_message))
+        << error_message;
+    EXPECT_EQ(0, rows);
+  }
+
+  void ExpectSingleRecording(Uuid camera_uuid, const Recording &recording,
+                             const VideoSampleEntry &entry,
+                             ListOldestSampleFilesRow *save_oldest_row) {
+    std::string error_message;
+    int rows = 0;
+    mdb_->ListCameras([&](const ListCamerasRow &row) {
+      ++rows;
+      EXPECT_EQ(camera_uuid, row.uuid);
+      EXPECT_EQ(recording.start_time_90k, row.min_start_time_90k);
+      EXPECT_EQ(recording.end_time_90k, row.max_end_time_90k);
+      EXPECT_EQ(recording.end_time_90k - recording.start_time_90k,
+                row.total_duration_90k);
+      EXPECT_EQ(recording.sample_file_bytes, row.total_sample_file_bytes);
+      return IterationControl::kContinue;
+    });
+    EXPECT_EQ(1, rows);
+
+    GetCameraRow camera_row;
+    EXPECT_TRUE(mdb_->GetCamera(camera_uuid, &camera_row));
+    EXPECT_EQ(recording.start_time_90k, camera_row.min_start_time_90k);
+    EXPECT_EQ(recording.end_time_90k, camera_row.max_end_time_90k);
+    EXPECT_EQ(recording.end_time_90k - recording.start_time_90k,
+              camera_row.total_duration_90k);
+    EXPECT_EQ(recording.sample_file_bytes, camera_row.total_sample_file_bytes);
+
+    rows = 0;
+    EXPECT_TRUE(mdb_->ListCameraRecordings(
+        camera_uuid, 0, std::numeric_limits<int64_t>::max(),
+        [&](const ListCameraRecordingsRow &row) {
+          ++rows;
+          EXPECT_EQ(recording.start_time_90k, row.start_time_90k);
+          EXPECT_EQ(recording.end_time_90k, row.end_time_90k);
+          EXPECT_EQ(recording.video_samples, row.video_samples);
+          EXPECT_EQ(recording.sample_file_bytes, row.sample_file_bytes);
+          EXPECT_EQ(entry.sha1, row.video_sample_entry_sha1);
+          EXPECT_EQ(entry.width, row.width);
+          EXPECT_EQ(entry.height, row.height);
+          return IterationControl::kContinue;
+        },
+        &error_message))
+        << error_message;
+    EXPECT_EQ(1, rows);
+
+    rows = 0;
+    EXPECT_TRUE(mdb_->ListOldestSampleFiles(
+        camera_uuid,
+        [&](const ListOldestSampleFilesRow &row) {
+          ++rows;
+          EXPECT_EQ(recording.id, row.recording_id);
+          EXPECT_EQ(recording.sample_file_uuid, row.sample_file_uuid);
+          EXPECT_EQ(recording.end_time_90k - recording.start_time_90k,
+                    row.duration_90k);
+          EXPECT_EQ(recording.sample_file_bytes, row.sample_file_bytes);
+          *save_oldest_row = row;
+          return IterationControl::kContinue;
+        },
+        &error_message))
+        << error_message;
+    EXPECT_EQ(1, rows);
+
+    rows = 0;
+    EXPECT_TRUE(mdb_->ListMp4Recordings(
+        camera_uuid, 0, std::numeric_limits<int64_t>::max(),
+        [&](Recording &some_recording, const VideoSampleEntry &some_entry) {
+          ++rows;
+
+          EXPECT_EQ(recording.id, some_recording.id);
+          EXPECT_EQ(recording.camera_id, some_recording.camera_id);
+          EXPECT_EQ(recording.sample_file_sha1,
+                    some_recording.sample_file_sha1);
+          EXPECT_EQ(recording.sample_file_uuid,
+                    some_recording.sample_file_uuid);
+          EXPECT_EQ(recording.video_sample_entry_id,
+                    some_recording.video_sample_entry_id);
+          EXPECT_EQ(recording.start_time_90k, some_recording.start_time_90k);
+          EXPECT_EQ(recording.end_time_90k, some_recording.end_time_90k);
+          EXPECT_EQ(recording.sample_file_bytes,
+                    some_recording.sample_file_bytes);
+          EXPECT_EQ(recording.video_samples, some_recording.video_samples);
+          EXPECT_EQ(recording.video_sync_samples,
+                    some_recording.video_sync_samples);
+          EXPECT_EQ(recording.video_index, some_recording.video_index);
+
+          EXPECT_EQ(entry.id, some_entry.id);
+          EXPECT_EQ(entry.sha1, some_entry.sha1);
+          EXPECT_EQ(entry.data, some_entry.data);
+          EXPECT_EQ(entry.width, some_entry.width);
+          EXPECT_EQ(entry.height, some_entry.height);
+
+          return IterationControl::kContinue;
+        },
+        &error_message))
+        << error_message;
+    EXPECT_EQ(1, rows);
+  }
+
+  std::string tmpdir_;
+  Database db_;
+  std::unique_ptr<MoonfireDatabase> mdb_;
+};
+
+// Basic test of running some queries on an empty database.
+TEST_F(MoonfireDbTest, EmptyDatabase) {
+  std::string error_message;
+  mdb_.reset(new MoonfireDatabase);
+  ASSERT_TRUE(mdb_->Init(&db_, &error_message)) << error_message;
+
+  mdb_->ListCameras([&](const ListCamerasRow &row) {
+    ADD_FAILURE() << "row unexpected";
+    return IterationControl::kBreak;
+  });
+
+  GetCameraRow get_camera_row;
+  EXPECT_FALSE(mdb_->GetCamera(Uuid(), &get_camera_row));
+
+  EXPECT_FALSE(
+      mdb_->ListCameraRecordings(Uuid(), 0, std::numeric_limits<int64_t>::max(),
+                                 [&](const ListCameraRecordingsRow &row) {
+                                   ADD_FAILURE() << "row unexpected";
+                                   return IterationControl::kBreak;
+                                 },
+                                 &error_message));
+
+  EXPECT_FALSE(mdb_->ListMp4Recordings(
+      Uuid(), 0, std::numeric_limits<int64_t>::max(),
+      [&](Recording &recording, const VideoSampleEntry &entry) {
+        ADD_FAILURE() << "row unexpected";
+        return IterationControl::kBreak;
+      },
+      &error_message));
+}
+
+// Basic test of the full lifecycle of recording.
+// Does not exercise many error cases.
+TEST_F(MoonfireDbTest, FullLifecycle) {
+  std::string error_message;
+  const char kCameraShortName[] = "testcam";
+  Uuid camera_uuid = GetRealUuidGenerator()->Generate();
+  int64_t camera_id = AddCamera(camera_uuid, kCameraShortName);
+  ASSERT_GT(camera_id, 0);
+  mdb_.reset(new MoonfireDatabase);
+  ASSERT_TRUE(mdb_->Init(&db_, &error_message)) << error_message;
+
+  ExpectNoRecordings(camera_uuid);
+
+  std::vector<Uuid> reserved;
+  EXPECT_TRUE(mdb_->ListReservedSampleFiles(&reserved, &error_message))
+      << error_message;
+  EXPECT_THAT(reserved, testing::IsEmpty());
+
+  std::vector<Uuid> uuids = mdb_->ReserveSampleFiles(2, &error_message);
+  ASSERT_THAT(uuids, testing::SizeIs(2)) << error_message;
+
+  EXPECT_TRUE(mdb_->ListReservedSampleFiles(&reserved, &error_message))
+      << error_message;
+  EXPECT_THAT(reserved, testing::UnorderedElementsAre(uuids[0], uuids[1]));
+
+  VideoSampleEntry entry;
+  entry.sha1.resize(20);
+  entry.width = 768;
+  entry.height = 512;
+  entry.data.resize(100);
+  ASSERT_TRUE(mdb_->InsertVideoSampleEntry(&entry, &error_message))
+      << error_message;
+  ASSERT_GT(entry.id, 0);
+
+  Recording recording;
+  recording.camera_id = camera_id;
+  recording.sample_file_uuid = GetRealUuidGenerator()->Generate();
+  recording.video_sample_entry_id = entry.id;
+  SampleIndexEncoder encoder;
+  encoder.Init(&recording, UINT64_C(1430006400) * kTimeUnitsPerSecond);
+  encoder.AddSample(kTimeUnitsPerSecond, 42, true);
+
+  // Inserting a recording should succeed and remove its uuid from the
+  // reserved table.
+  ASSERT_FALSE(mdb_->InsertRecording(&recording, &error_message));
+  EXPECT_THAT(error_message, testing::HasSubstr("not reserved"));
+  recording.sample_file_uuid = uuids.back();
+  recording.sample_file_sha1.resize(20);
+  ASSERT_TRUE(mdb_->InsertRecording(&recording, &error_message))
+      << error_message;
+  ASSERT_GT(recording.id, 0);
+  EXPECT_TRUE(mdb_->ListReservedSampleFiles(&reserved, &error_message))
+      << error_message;
+  EXPECT_THAT(reserved, testing::ElementsAre(uuids[0]));
+
+  // Queries should return the correct result (with caches updated on insert).
+  ListOldestSampleFilesRow oldest;
+  ExpectSingleRecording(camera_uuid, recording, entry, &oldest);
+
+  // Queries on a fresh database should return the correct result (with caches
+  // populated from existing database contents).
+  mdb_.reset(new MoonfireDatabase);
+  ASSERT_TRUE(mdb_->Init(&db_, &error_message)) << error_message;
+  ExpectSingleRecording(camera_uuid, recording, entry, &oldest);
+
+  // Deleting a recording should succeed, update the min/max times, and mark
+  // the uuid as reserved.
+  std::vector<ListOldestSampleFilesRow> to_delete;
+  to_delete.push_back(oldest);
+  ASSERT_TRUE(mdb_->DeleteRecordings(to_delete, &error_message))
+      << error_message;
+  EXPECT_TRUE(mdb_->ListReservedSampleFiles(&reserved, &error_message))
+      << error_message;
+  EXPECT_THAT(reserved, testing::UnorderedElementsAre(uuids[0], uuids[1]));
+  LOG(INFO) << "after delete";
+  ExpectNoRecordings(camera_uuid);
+
+  EXPECT_TRUE(mdb_->MarkSampleFilesDeleted(uuids, &error_message))
+      << error_message;
+  EXPECT_TRUE(mdb_->ListReservedSampleFiles(&reserved, &error_message))
+      << error_message;
+  EXPECT_THAT(reserved, testing::IsEmpty());
+}
+
+}  // namespace
+}  // namespace moonfire_nvr
+
+int main(int argc, char **argv) {
+  FLAGS_alsologtostderr = true;
+  google::ParseCommandLineFlags(&argc, &argv, true);
+  testing::InitGoogleTest(&argc, argv);
+  google::InitGoogleLogging(argv[0]);
+  return RUN_ALL_TESTS();
+}

src/moonfire-db.cc

@@ -29,6 +29,7 @@
 // along with this program.  If not, see <http://www.gnu.org/licenses/>.
 //
 // moonfire-db.cc: implementation of moonfire-db.h interface.
+// see top-level comments there on performance & efficiency.
 
 #include "moonfire-db.h"
 
@@ -42,188 +43,336 @@
 
 namespace moonfire_nvr {
 
-bool MoonfireDatabase::Init(std::string *error_message) {
+bool MoonfireDatabase::Init(Database *db, std::string *error_message) {
-  list_cameras_query_ = db_->Prepare(
+  CHECK(db_ == nullptr);
-      R"(
+  db_ = db;
-      select
+
-        camera.id,
+  {
-        camera.uuid,
+    DatabaseContext ctx(db_);
-        camera.short_name,
+
-        camera.description,
+    // This query scans the entirety of the recording table's index.
-        camera.retain_bytes,
+    // It is quite slow, so the results are cached.
-        min(recording.start_time_90k),
+    auto list_cameras_run = ctx.UseOnce(
-        max(recording.end_time_90k),
+        R"(
-        sum(recording.end_time_90k - recording.start_time_90k),
+        select
-        sum(recording.sample_file_bytes)
+          camera.id,
-      from
+          camera.uuid,
-        camera
+          camera.short_name,
-        left join recording on
+          camera.description,
-            (camera.id = recording.camera_id and
+          camera.retain_bytes,
-             recording.status = 1)
+          min(recording.start_time_90k),
-      group by
+          max(recording.start_time_90k + recording.duration_90k),
-        camera.id,
+          sum(recording.duration_90k),
-        camera.uuid,
+          sum(recording.sample_file_bytes)
-        camera.short_name,
+        from
-        camera.description,
+          camera
-        camera.retain_bytes;
+          left join recording on (camera.id = recording.camera_id)
-  )",
+        group by
-      nullptr, error_message);
+          camera.id,
-  if (!list_cameras_query_.valid()) {
+          camera.uuid,
-    return false;
+          camera.short_name,
+          camera.description,
+          camera.retain_bytes;
+        )");
+    while (list_cameras_run.Step() == SQLITE_ROW) {
+      CameraData data;
+      data.id = list_cameras_run.ColumnInt64(0);
+      Uuid uuid;
+      if (!uuid.ParseBinary(list_cameras_run.ColumnBlob(1))) {
+        *error_message =
+            StrCat("bad uuid ", ToHex(list_cameras_run.ColumnBlob(2)),
+                   " for camera id ", data.id);
+        return false;
+      }
+      data.short_name = list_cameras_run.ColumnText(2).as_string();
+      data.description = list_cameras_run.ColumnText(3).as_string();
+      data.retain_bytes = list_cameras_run.ColumnInt64(4);
+      data.min_start_time_90k = list_cameras_run.ColumnType(5) == SQLITE_NULL
+                                    ? -1
+                                    : list_cameras_run.ColumnInt64(5);
+      data.max_end_time_90k = list_cameras_run.ColumnType(6) == SQLITE_NULL
+                                  ? -1
+                                  : list_cameras_run.ColumnInt64(6);
+      data.total_duration_90k = list_cameras_run.ColumnInt64(7);
+      data.total_sample_file_bytes = list_cameras_run.ColumnInt64(8);
+
+      auto ret = cameras_by_uuid_.insert(std::make_pair(uuid, data));
+      if (!ret.second) {
+        *error_message = StrCat("Duplicate camera uuid ", uuid.UnparseText());
+        return false;
+      }
+      CameraData *data_p = &ret.first->second;
+      if (!cameras_by_id_.insert(std::make_pair(data.id, data_p)).second) {
+        *error_message = StrCat("Duplicate camera id ", data.id);
+        return false;
+      }
+    }
+    if (list_cameras_run.status() != SQLITE_DONE) {
+      *error_message = StrCat("Camera list query failed: ",
+                              list_cameras_run.error_message());
+    }
+
+    // It's simplest to just keep the video sample entries in RAM.
+    auto video_sample_entries_run = ctx.UseOnce(
+        R"(
+        select
+          id,
+          sha1,
+          width,
+          height,
+          data
+        from
+          video_sample_entry
+        )");
+    while (video_sample_entries_run.Step() == SQLITE_ROW) {
+      VideoSampleEntry entry;
+      entry.id = video_sample_entries_run.ColumnInt64(0);
+      entry.sha1 = video_sample_entries_run.ColumnBlob(1).as_string();
+      int64_t width_tmp = video_sample_entries_run.ColumnInt64(2);
+      int64_t height_tmp = video_sample_entries_run.ColumnInt64(3);
+      auto max = std::numeric_limits<uint16_t>::max();
+      if (width_tmp <= 0 || width_tmp > max || height_tmp <= 0 ||
+          height_tmp > max) {
+        *error_message =
+            StrCat("video_sample_entry id ", entry.id, " width ", width_tmp,
+                   " / height ", height_tmp, " out of range.");
+        return false;
+      }
+      entry.width = width_tmp;
+      entry.height = height_tmp;
+      entry.data = video_sample_entries_run.ColumnBlob(4).as_string();
+      CHECK(
+          video_sample_entries_.insert(std::make_pair(entry.id, entry)).second)
+          << "duplicate: " << entry.id;
+    }
   }
 
-  get_camera_query_ = db_->Prepare(
+  std::string list_camera_recordings_sql = StrCat(
-      R"(
-      select
-        uuid,
-        short_name,
-        description,
-        retain_bytes
-      from
-        camera
-      where
-        id = :camera_id;)",
-      nullptr, error_message);
-  if (!get_camera_query_.valid()) {
-    return false;
-  }
-
-  list_camera_recordings_query_ = db_->Prepare(
       R"(
       select
         recording.start_time_90k,
-        recording.end_time_90k,
+        recording.duration_90k,
         recording.video_samples,
         recording.sample_file_bytes,
-        recording.video_sample_entry_sha1,
+        recording.video_sample_entry_id
-        video_sample_entry.width,
-        video_sample_entry.height
       from
         recording
-        join video_sample_entry on
-            (recording.video_sample_entry_sha1 = video_sample_entry.sha1)
       where
-        recording.status = 1 and
+        camera_id = :camera_id and
-        camera_id = :camera_id
+        recording.start_time_90k > :start_time_90k - )",
+      kMaxRecordingDuration, " and\n",
+      R"(
+        recording.start_time_90k < :end_time_90k and
+        recording.start_time_90k + recording.duration_90k > :start_time_90k
       order by
-        recording.start_time_90k;)",
+        recording.start_time_90k desc;)");
-      nullptr, error_message);
+  list_camera_recordings_stmt_ =
-  if (!list_camera_recordings_query_.valid()) {
+      db_->Prepare(list_camera_recordings_sql, nullptr, error_message);
+  if (!list_camera_recordings_stmt_.valid()) {
     return false;
   }
 
   std::string build_mp4_sql = StrCat(
       R"(
       select
-        recording.rowid,
+        recording.id,
         recording.start_time_90k,
-        recording.end_time_90k,
+        recording.duration_90k,
         recording.sample_file_bytes,
         recording.sample_file_uuid,
         recording.sample_file_sha1,
-        recording.video_sample_entry_sha1,
         recording.video_index,
         recording.video_samples,
         recording.video_sync_samples,
-        video_sample_entry.bytes,
+        recording.video_sample_entry_id
-        video_sample_entry.width,
-        video_sample_entry.height
       from
-        recording join video_sample_entry on
+        recording
-        (recording.video_sample_entry_sha1 = video_sample_entry.sha1)
       where
-        recording.status = 1 and
         camera_id = :camera_id and
         recording.start_time_90k > :start_time_90k - )",
       kMaxRecordingDuration, " and\n",
       R"(
         recording.start_time_90k < :end_time_90k and
-        recording.end_time_90k > :start_time_90k
+        recording.start_time_90k + recording.duration_90k > :start_time_90k
       order by
         recording.start_time_90k;)");
-  build_mp4_query_ = db_->Prepare(build_mp4_sql, nullptr, error_message);
+  build_mp4_stmt_ = db_->Prepare(build_mp4_sql, nullptr, error_message);
-  if (!build_mp4_query_.valid()) {
+  if (!build_mp4_stmt_.valid()) {
+    return false;
+  }
+
+  insert_reservation_stmt_ = db_->Prepare(
+      "insert into reserved_sample_files (uuid,  state)\n"
+      "                           values (:uuid, :state);",
+      nullptr, error_message);
+  if (!insert_reservation_stmt_.valid()) {
+    return false;
+  }
+
+  delete_reservation_stmt_ =
+      db_->Prepare("delete from reserved_sample_files where uuid = :uuid;",
+                   nullptr, error_message);
+  if (!delete_reservation_stmt_.valid()) {
+    return false;
+  }
+
+  insert_video_sample_entry_stmt_ = db_->Prepare(
+      R"(
+      insert into video_sample_entry (sha1,  width,  height,  data)
+                              values (:sha1, :width, :height, :data);
+      )",
+      nullptr, error_message);
+  if (!insert_video_sample_entry_stmt_.valid()) {
+    return false;
+  }
+
+  insert_recording_stmt_ = db_->Prepare(
+      R"(
+      insert into recording (camera_id, sample_file_bytes, start_time_90k,
+                             duration_90k, video_samples, video_sync_samples,
+                             video_sample_entry_id, sample_file_uuid,
+                             sample_file_sha1, video_index)
+                     values (:camera_id, :sample_file_bytes, :start_time_90k,
+                             :duration_90k, :video_samples, :video_sync_samples,
+                             :video_sample_entry_id, :sample_file_uuid,
+                             :sample_file_sha1, :video_index);
+      )",
+      nullptr, error_message);
+  if (!insert_recording_stmt_.valid()) {
+    return false;
+  }
+
+  list_oldest_sample_files_stmt_ = db_->Prepare(
+      R"(
+      select
+        id,
+        sample_file_uuid,
+        duration_90k,
+        sample_file_bytes
+      from
+        recording
+      where
+        camera_id = :camera_id
+      order by
+        start_time_90k
+      )",
+      nullptr, error_message);
+  if (!list_oldest_sample_files_stmt_.valid()) {
+    return false;
+  }
+
+  delete_recording_stmt_ =
+      db_->Prepare("delete from recording where id = :recording_id;", nullptr,
+                   error_message);
+  if (!delete_recording_stmt_.valid()) {
+    return false;
+  }
+
+  camera_min_start_stmt_ = db_->Prepare(
+      R"(
+      select
+        start_time_90k
+      from
+        recording
+      where
+        camera_id = :camera_id
+      order by start_time_90k limit 1;
+      )",
+      nullptr, error_message);
+  if (!camera_min_start_stmt_.valid()) {
+    return false;
+  }
+
+  camera_max_start_stmt_ = db_->Prepare(
+      R"(
+      select
+        start_time_90k,
+        duration_90k
+      from
+        recording
+      where
+        camera_id = :camera_id
+      order by start_time_90k desc;
+      )",
+      nullptr, error_message);
+  if (!camera_max_start_stmt_.valid()) {
     return false;
   }
 
   return true;
 }
 
-bool MoonfireDatabase::ListCameras(
+void MoonfireDatabase::ListCameras(
-    std::function<IterationControl(const ListCamerasRow &)> cb,
+    std::function<IterationControl(const ListCamerasRow &)> cb) {
-    std::string *error_message) {
   DatabaseContext ctx(db_);
-  auto run = ctx.Borrow(&list_cameras_query_);
   ListCamerasRow row;
-  while (run.Step() == SQLITE_ROW) {
+  for (const auto &entry : cameras_by_uuid_) {
-    row.id = run.ColumnInt64(0);
+    row.uuid = entry.first;
-    if (!row.uuid.ParseBinary(run.ColumnBlob(1))) {
+    row.short_name = entry.second.short_name;
-      *error_message = StrCat("invalid uuid in row id ", row.id);
+    row.description = entry.second.description;
-      return false;
+    row.retain_bytes = entry.second.retain_bytes;
-    }
+    row.min_start_time_90k = entry.second.min_start_time_90k;
-    row.short_name = run.ColumnText(2).as_string();
+    row.max_end_time_90k = entry.second.max_end_time_90k;
-    row.description = run.ColumnText(3).as_string();
+    row.total_duration_90k = entry.second.total_duration_90k;
-    row.retain_bytes = run.ColumnInt64(4);
+    row.total_sample_file_bytes = entry.second.total_sample_file_bytes;
-    row.min_recording_start_time_90k = run.ColumnInt64(5);
-    row.max_recording_end_time_90k = run.ColumnInt64(6);
-    row.total_recording_duration_90k = run.ColumnInt64(7);
-    row.total_sample_file_bytes = run.ColumnInt64(8);
     if (cb(row) == IterationControl::kBreak) {
-      break;
+      return;
     }
   }
-  if (run.status() != SQLITE_DONE) {
+  return;
-    *error_message = StrCat("sqlite query failed: ", run.error_message());
-    return false;
-  }
-  return true;
 }
 
-bool MoonfireDatabase::GetCamera(int64_t camera_id, GetCameraRow *row,
+bool MoonfireDatabase::GetCamera(Uuid camera_uuid, GetCameraRow *row) {
-                                 std::string *error_message) {
   DatabaseContext ctx(db_);
-  auto run = ctx.Borrow(&get_camera_query_);
+  const auto it = cameras_by_uuid_.find(camera_uuid);
-  run.BindInt64(":camera_id", camera_id);
+  if (it == cameras_by_uuid_.end()) {
-  if (run.Step() == SQLITE_ROW) {
-    if (!row->uuid.ParseBinary(run.ColumnBlob(0))) {
-      *error_message =
-          StrCat("unable to parse uuid ", ToHex(run.ColumnBlob(0)));
-      return false;
-    }
-    row->short_name = run.ColumnText(1).as_string();
-    row->description = run.ColumnText(2).as_string();
-    row->retain_bytes = run.ColumnInt64(3);
-  } else if (run.status() == SQLITE_DONE) {
-    *error_message = "no such camera";
-    return false;
-  }
-  if (run.Step() == SQLITE_ROW) {
-    *error_message = "multiple rows returned unexpectedly";
     return false;
   }
+  const CameraData &data = it->second;
+  row->short_name = data.short_name;
+  row->description = data.description;
+  row->retain_bytes = data.retain_bytes;
+  row->min_start_time_90k = data.min_start_time_90k;
+  row->max_end_time_90k = data.max_end_time_90k;
+  row->total_duration_90k = data.total_duration_90k;
+  row->total_sample_file_bytes = data.total_sample_file_bytes;
   return true;
 }
 
 bool MoonfireDatabase::ListCameraRecordings(
-    int64_t camera_id,
+    Uuid camera_uuid, int64_t start_time_90k, int64_t end_time_90k,
     std::function<IterationControl(const ListCameraRecordingsRow &)> cb,
     std::string *error_message) {
   DatabaseContext ctx(db_);
-  auto run = ctx.Borrow(&list_camera_recordings_query_);
+  const auto camera_it = cameras_by_uuid_.find(camera_uuid);
-  run.BindInt64(":camera_id", camera_id);
+  if (camera_it == cameras_by_uuid_.end()) {
+    *error_message = StrCat("no such camera ", camera_uuid.UnparseText());
+    return false;
+  }
+  auto run = ctx.Borrow(&list_camera_recordings_stmt_);
+  run.BindInt64(":camera_id", camera_it->second.id);
+  run.BindInt64(":start_time_90k", start_time_90k);
+  run.BindInt64(":end_time_90k", end_time_90k);
   ListCameraRecordingsRow row;
   while (run.Step() == SQLITE_ROW) {
     row.start_time_90k = run.ColumnInt64(0);
-    row.end_time_90k = run.ColumnInt64(1);
+    row.end_time_90k = row.start_time_90k + run.ColumnInt64(1);
     row.video_samples = run.ColumnInt64(2);
     row.sample_file_bytes = run.ColumnInt64(3);
-    auto video_sample_entry_sha1 = run.ColumnBlob(4);
+    int64_t video_sample_entry_id = run.ColumnInt64(4);
-    row.video_sample_entry_sha1.assign(video_sample_entry_sha1.data(),
+    const auto it = video_sample_entries_.find(video_sample_entry_id);
-                                       video_sample_entry_sha1.size());
+    if (it == video_sample_entries_.end()) {
-    row.width = run.ColumnInt64(5);
+      *error_message =
-    row.height = run.ColumnInt64(6);
+          StrCat("recording references invalid video sample entry ",
+                 video_sample_entry_id);
+      return false;
+    }
+    const VideoSampleEntry &entry = it->second;
+    row.video_sample_entry_sha1 = entry.sha1;
+    row.width = entry.width;
+    row.height = entry.height;
     if (cb(row) == IterationControl::kBreak) {
-      break;
+      return true;
     }
   }
   if (run.status() != SQLITE_DONE) {
@@ -234,27 +383,34 @@ bool MoonfireDatabase::ListCameraRecordings(
 }
 
 bool MoonfireDatabase::ListMp4Recordings(
-    int64_t camera_id, int64_t start_time_90k, int64_t end_time_90k,
+    Uuid camera_uuid, int64_t start_time_90k, int64_t end_time_90k,
     std::function<IterationControl(Recording &, const VideoSampleEntry &)>
         row_cb,
     std::string *error_message) {
   VLOG(1) << "...(1/4): Waiting for database lock";
   DatabaseContext ctx(db_);
+  const auto it = cameras_by_uuid_.find(camera_uuid);
+  if (it == cameras_by_uuid_.end()) {
+    *error_message = StrCat("no such camera ", camera_uuid.UnparseText());
+    return false;
+  }
+  const CameraData &data = it->second;
   VLOG(1) << "...(2/4): Querying database";
-  auto run = ctx.Borrow(&build_mp4_query_);
+  auto run = ctx.Borrow(&build_mp4_stmt_);
-  run.BindInt64(":camera_id", camera_id);
+  run.BindInt64(":camera_id", data.id);
   run.BindInt64(":end_time_90k", end_time_90k);
   run.BindInt64(":start_time_90k", start_time_90k);
   Recording recording;
   VideoSampleEntry sample_entry;
   while (run.Step() == SQLITE_ROW) {
-    recording.rowid = run.ColumnInt64(0);
+    recording.id = run.ColumnInt64(0);
+    recording.camera_id = data.id;
     recording.start_time_90k = run.ColumnInt64(1);
-    recording.end_time_90k = run.ColumnInt64(2);
+    recording.end_time_90k = recording.start_time_90k + run.ColumnInt64(2);
     recording.sample_file_bytes = run.ColumnInt64(3);
     if (!recording.sample_file_uuid.ParseBinary(run.ColumnBlob(4))) {
       *error_message =
-          StrCat("recording ", recording.rowid, " has unparseable uuid ",
+          StrCat("recording ", recording.id, " has unparseable uuid ",
                  ToHex(run.ColumnBlob(4)));
       return false;
     }
@@ -262,19 +418,23 @@ bool MoonfireDatabase::ListMp4Recordings(
         StrCat("/home/slamb/new-moonfire/sample/",
                recording.sample_file_uuid.UnparseText());
     recording.sample_file_sha1 = run.ColumnBlob(5).as_string();
-    recording.video_sample_entry_sha1 = run.ColumnBlob(6).as_string();
+    recording.video_index = run.ColumnBlob(6).as_string();
-    recording.video_index = run.ColumnBlob(7).as_string();
+    recording.video_samples = run.ColumnInt64(7);
-    recording.video_samples = run.ColumnInt64(8);
+    recording.video_sync_samples = run.ColumnInt64(8);
-    recording.video_sync_samples = run.ColumnInt64(9);
+    recording.video_sample_entry_id = run.ColumnInt64(9);
 
-    if (recording.video_sample_entry_sha1 != sample_entry.sha1) {
+    auto it = video_sample_entries_.find(recording.video_sample_entry_id);
-      sample_entry.sha1 = run.ColumnBlob(6).as_string();
+    if (it == video_sample_entries_.end()) {
-      sample_entry.data = run.ColumnBlob(10).as_string();
+      *error_message = StrCat("recording ", recording.id,
-      sample_entry.width = run.ColumnInt64(11);
+                              " references unknown video sample entry ",
-      sample_entry.height = run.ColumnInt64(12);
+                              recording.video_sample_entry_id);
+      return false;
     }
+    const VideoSampleEntry &entry = it->second;
 
-    row_cb(recording, sample_entry);
+    if (row_cb(recording, entry) == IterationControl::kBreak) {
+      return true;
+    }
   }
   if (run.status() != SQLITE_DONE && run.status() != SQLITE_ROW) {
     *error_message = StrCat("sqlite query failed: ", run.error_message());
@@ -283,10 +443,30 @@ bool MoonfireDatabase::ListMp4Recordings(
   return true;
 }
 
+bool MoonfireDatabase::ListReservedSampleFiles(std::vector<Uuid> *reserved,
+                                               std::string *error_message) {
+  reserved->clear();
+  DatabaseContext ctx(db_);
+  auto run = ctx.UseOnce("select uuid from reserved_sample_files;");
+  while (run.Step() == SQLITE_ROW) {
+    Uuid uuid;
+    if (!uuid.ParseBinary(run.ColumnBlob(0))) {
+      *error_message = StrCat("unparseable uuid ", ToHex(run.ColumnBlob(0)));
+      return false;
+    }
+    reserved->push_back(uuid);
+  }
+  if (run.status() != SQLITE_DONE) {
+    *error_message = run.error_message();
+    return false;
+  }
+  return true;
+}
+
 std::shared_ptr<VirtualFile> MoonfireDatabase::BuildMp4(
-    int64_t camera_id, int64_t start_time_90k, int64_t end_time_90k,
+    Uuid camera_uuid, int64_t start_time_90k, int64_t end_time_90k,
     std::string *error_message) {
-  LOG(INFO) << "Building mp4 for camera: " << camera_id
+  LOG(INFO) << "Building mp4 for camera: " << camera_uuid.UnparseText()
             << ", start_time_90k: " << start_time_90k
             << ", end_time_90k: " << end_time_90k;
 
@@ -315,11 +495,11 @@ std::shared_ptr<VirtualFile> MoonfireDatabase::BuildMp4(
 
     next_row_start_time_90k = recording.end_time_90k;
 
-    if (rows > 0 && recording.video_sample_entry_sha1 != sample_entry.sha1) {
+    if (rows > 0 && recording.video_sample_entry_id != sample_entry.id) {
       *error_message =
-          StrCat("inconsistent video sample entries: this recording has ",
+          StrCat("inconsistent video sample entries: this recording has id ",
-                 ToHex(recording.video_sample_entry_sha1), ", previous had ",
+                 recording.video_sample_entry_id, " previous had ",
-                 ToHex(sample_entry.sha1));
+                 sample_entry.id, " (sha1 ", ToHex(sample_entry.sha1), ")");
       ok = false;
       return IterationControl::kBreak;
     } else if (rows == 0) {
@@ -334,7 +514,7 @@ std::shared_ptr<VirtualFile> MoonfireDatabase::BuildMp4(
     return IterationControl::kContinue;
   };
   if (!ok ||
-      !ListMp4Recordings(camera_id, start_time_90k, end_time_90k, row_cb,
+      !ListMp4Recordings(camera_uuid, start_time_90k, end_time_90k, row_cb,
                          error_message)) {
     return false;
   }
@@ -362,4 +542,329 @@ std::shared_ptr<VirtualFile> MoonfireDatabase::BuildMp4(
   return file;
 }
 
+std::vector<Uuid> MoonfireDatabase::ReserveSampleFiles(
+    int n, std::string *error_message) {
+  if (n == 0) {
+    return std::vector<Uuid>();
+  }
+  auto *gen = GetRealUuidGenerator();
+  std::vector<Uuid> uuids;
+  uuids.reserve(n);
+  for (int i = 0; i < n; ++i) {
+    uuids.push_back(gen->Generate());
+  }
+  DatabaseContext ctx(db_);
+  if (!ctx.BeginTransaction(error_message)) {
+    return std::vector<Uuid>();
+  }
+  for (const auto &uuid : uuids) {
+    auto run = ctx.Borrow(&insert_reservation_stmt_);
+    run.BindBlob(":uuid", uuid.binary_view());
+    run.BindInt64(":state", static_cast<int64_t>(ReservationState::kWriting));
+    if (run.Step() != SQLITE_DONE) {
+      ctx.RollbackTransaction();
+      *error_message = run.error_message();
+      return std::vector<Uuid>();
+    }
+  }
+  if (!ctx.CommitTransaction(error_message)) {
+    return std::vector<Uuid>();
+  }
+  return uuids;
+}
+
+bool MoonfireDatabase::InsertVideoSampleEntry(VideoSampleEntry *entry,
+                                              std::string *error_message) {
+  if (entry->id != -1) {
+    *error_message = StrCat("video_sample_entry already has id ", entry->id);
+    return false;
+  }
+  DatabaseContext ctx(db_);
+  for (const auto &some_entry : video_sample_entries_) {
+    if (some_entry.second.sha1 == entry->sha1) {
+      if (entry->width != some_entry.second.width ||
+          entry->height != some_entry.second.height) {
+        *error_message =
+            StrCat("inconsistent entry for sha1 ", ToHex(entry->sha1),
+                   ": existing entry has ", some_entry.second.width, "x",
+                   some_entry.second.height, ", new entry has ", entry->width,
+                   "x", entry->height);
+        return false;
+      }
+      entry->id = some_entry.first;
+      return true;
+    }
+  }
+  auto insert_run = ctx.Borrow(&insert_video_sample_entry_stmt_);
+  insert_run.BindBlob(":sha1", entry->sha1);
+  insert_run.BindInt64(":width", entry->width);
+  insert_run.BindInt64(":height", entry->height);
+  insert_run.BindBlob(":data", entry->data);
+  if (insert_run.Step() != SQLITE_DONE) {
+    *error_message = insert_run.error_message();
+    return false;
+  }
+  entry->id = ctx.last_insert_rowid();
+  CHECK(video_sample_entries_.insert(std::make_pair(entry->id, *entry)).second)
+      << "duplicate: " << entry->id;
+  return true;
+}
+
+bool MoonfireDatabase::InsertRecording(Recording *recording,
+                                       std::string *error_message) {
+  if (recording->id != -1) {
+    *error_message = StrCat("recording already has id ", recording->id);
+    return false;
+  }
+  if (recording->end_time_90k <= recording->start_time_90k) {
+    *error_message =
+        StrCat("end time ", recording->end_time_90k,
+               " must be greater than start time ", recording->start_time_90k);
+    return false;
+  }
+  DatabaseContext ctx(db_);
+  auto it = cameras_by_id_.find(recording->camera_id);
+  if (it == cameras_by_id_.end()) {
+    *error_message = StrCat("no camera with id ", recording->camera_id);
+    return false;
+  }
+  CameraData *camera_data = it->second;
+  if (!ctx.BeginTransaction(error_message)) {
+    return false;
+  }
+  auto delete_run = ctx.Borrow(&delete_reservation_stmt_);
+  delete_run.BindBlob(":uuid", recording->sample_file_uuid.binary_view());
+  if (delete_run.Step() != SQLITE_DONE) {
+    *error_message = delete_run.error_message();
+    ctx.RollbackTransaction();
+    return false;
+  }
+  if (ctx.changes() != 1) {
+    *error_message = StrCat("uuid ", recording->sample_file_uuid.UnparseText(),
+                            " is not reserved");
+    ctx.RollbackTransaction();
+    return false;
+  }
+  auto insert_run = ctx.Borrow(&insert_recording_stmt_);
+  insert_run.BindInt64(":camera_id", recording->camera_id);
+  insert_run.BindInt64(":sample_file_bytes", recording->sample_file_bytes);
+  insert_run.BindInt64(":start_time_90k", recording->start_time_90k);
+  insert_run.BindInt64(":duration_90k",
+                       recording->end_time_90k - recording->start_time_90k);
+  insert_run.BindInt64(":video_samples", recording->video_samples);
+  insert_run.BindInt64(":video_sync_samples", recording->video_sync_samples);
+  insert_run.BindInt64(":video_sample_entry_id",
+                       recording->video_sample_entry_id);
+  insert_run.BindBlob(":sample_file_uuid",
+                      recording->sample_file_uuid.binary_view());
+  insert_run.BindBlob(":sample_file_sha1", recording->sample_file_sha1);
+  insert_run.BindBlob(":video_index", recording->video_index);
+  if (insert_run.Step() != SQLITE_DONE) {
+    LOG(ERROR) << "insert_run failed: " << insert_run.error_message();
+    *error_message = insert_run.error_message();
+    ctx.RollbackTransaction();
+    return false;
+  }
+  if (!ctx.CommitTransaction(error_message)) {
+    LOG(ERROR) << "commit failed";
+    return false;
+  }
+  recording->id = ctx.last_insert_rowid();
+  if (camera_data->min_start_time_90k == -1 ||
+      camera_data->min_start_time_90k > recording->start_time_90k) {
+    camera_data->min_start_time_90k = recording->start_time_90k;
+  }
+  if (camera_data->max_end_time_90k == -1 ||
+      camera_data->max_end_time_90k < recording->end_time_90k) {
+    camera_data->max_end_time_90k = recording->end_time_90k;
+  }
+  camera_data->total_duration_90k +=
+      recording->end_time_90k - recording->start_time_90k;
+  camera_data->total_sample_file_bytes += recording->sample_file_bytes;
+  return true;
+}
+
+bool MoonfireDatabase::ListOldestSampleFiles(
+    Uuid camera_uuid,
+    std::function<IterationControl(const ListOldestSampleFilesRow &)> row_cb,
+    std::string *error_message) {
+  DatabaseContext ctx(db_);
+  auto it = cameras_by_uuid_.find(camera_uuid);
+  if (it == cameras_by_uuid_.end()) {
+    *error_message = StrCat("no such camera ", camera_uuid.UnparseText());
+    return false;
+  }
+  const CameraData &camera_data = it->second;
+  auto run = ctx.Borrow(&list_oldest_sample_files_stmt_);
+  run.BindInt64(":camera_id", camera_data.id);
+  ListOldestSampleFilesRow row;
+  while (run.Step() == SQLITE_ROW) {
+    row.camera_id = camera_data.id;
+    row.recording_id = run.ColumnInt64(0);
+    if (!row.sample_file_uuid.ParseBinary(run.ColumnBlob(1))) {
+      *error_message =
+          StrCat("recording ", row.recording_id, " has unparseable uuid ",
+                 ToHex(run.ColumnBlob(1)));
+      return false;
+    }
+    row.duration_90k = run.ColumnInt64(2);
+    row.sample_file_bytes = run.ColumnInt64(3);
+    if (row_cb(row) == IterationControl::kBreak) {
+      return true;
+    }
+  }
+  if (run.status() != SQLITE_DONE) {
+    *error_message = run.error_message();
+    return false;
+  }
+  return true;
+}
+
+bool MoonfireDatabase::DeleteRecordings(
+    const std::vector<ListOldestSampleFilesRow> &recordings,
+    std::string *error_message) {
+  if (recordings.empty()) {
+    return true;
+  }
+
+  DatabaseContext ctx(db_);
+  if (!ctx.BeginTransaction(error_message)) {
+    return false;
+  }
+  struct State {
+    int64_t deleted_duration_90k = 0;
+    int64_t deleted_sample_file_bytes = 0;
+    int64_t min_start_time_90k = -1;
+    int64_t max_end_time_90k = -1;
+    CameraData *camera_data = nullptr;
+  };
+  std::map<int64_t, State> state_by_camera_id;
+  for (const auto &recording : recordings) {
+    State &state = state_by_camera_id[recording.camera_id];
+    state.deleted_duration_90k += recording.duration_90k;
+    state.deleted_sample_file_bytes += recording.sample_file_bytes;
+
+    auto delete_run = ctx.Borrow(&delete_recording_stmt_);
+    delete_run.BindInt64(":recording_id", recording.recording_id);
+    if (delete_run.Step() != SQLITE_DONE) {
+      ctx.RollbackTransaction();
+      *error_message = StrCat("delete: ", delete_run.error_message());
+      return false;
+    }
+    if (ctx.changes() != 1) {
+      ctx.RollbackTransaction();
+      *error_message = StrCat("no such recording ", recording.recording_id);
+      return false;
+    }
+
+    auto insert_run = ctx.Borrow(&insert_reservation_stmt_);
+    insert_run.BindBlob(":uuid", recording.sample_file_uuid.binary_view());
+    insert_run.BindInt64(":state",
+                         static_cast<int64_t>(ReservationState::kDeleting));
+    if (insert_run.Step() != SQLITE_DONE) {
+      ctx.RollbackTransaction();
+      *error_message = StrCat("insert: ", insert_run.error_message());
+      return false;
+    }
+  }
+
+  // Recompute start and end times for each camera.
+  for (auto &state_entry : state_by_camera_id) {
+    int64_t camera_id = state_entry.first;
+    State &state = state_entry.second;
+    auto it = cameras_by_id_.find(camera_id);
+    if (it == cameras_by_id_.end()) {
+      *error_message =
+          StrCat("internal error; can't find camera id ", camera_id);
+      return false;
+    }
+    state.camera_data = it->second;
+
+    // The minimum is straightforward, taking advantage of the start_time_90k
+    // index for speed.
+    auto min_run = ctx.Borrow(&camera_min_start_stmt_);
+    min_run.BindInt64(":camera_id", camera_id);
+    if (min_run.Step() == SQLITE_ROW) {
+      state.min_start_time_90k = min_run.ColumnInt64(0);
+    } else if (min_run.Step() == SQLITE_DONE) {
+      // There are no recordings left.
+      state.min_start_time_90k = -1;
+      state.max_end_time_90k = -1;
+      continue;  // skip additional query below to calculate max.
+    } else {
+      ctx.RollbackTransaction();
+      *error_message = StrCat("min: ", min_run.error_message());
+      return false;
+    }
+
+    // The maximum is less straightforward in the case of overlap - all
+    // recordings starting in the last kMaxRecordingDuration must be examined
+    // to take advantage of the start_time_90k index.
+    auto max_run = ctx.Borrow(&camera_max_start_stmt_);
+    max_run.BindInt64(":camera_id", camera_id);
+    if (max_run.Step() != SQLITE_ROW) {
+      // If there was a min row, there should be a max row too, so this is an
+      // error even in the SQLITE_DONE case.
+      ctx.RollbackTransaction();
+      *error_message = StrCat("max[0]: ", max_run.error_message());
+      return false;
+    }
+    int64_t max_start_90k = max_run.ColumnInt64(0);
+    do {
+      auto end_time_90k = max_run.ColumnInt64(0) + max_run.ColumnInt64(1);
+      state.max_end_time_90k = std::max(state.max_end_time_90k, end_time_90k);
+    } while (max_run.Step() == SQLITE_ROW &&
+             max_run.ColumnInt64(0) > max_start_90k - kMaxRecordingDuration);
+    if (max_run.status() != SQLITE_DONE && max_run.status() != SQLITE_ROW) {
+      *error_message = StrCat("max[1]: ", max_run.error_message());
+      ctx.RollbackTransaction();
+      return false;
+    }
+  }
+
+  if (!ctx.CommitTransaction(error_message)) {
+    *error_message = StrCat("commit: ", *error_message);
+    return false;
+  }
+
+  for (auto &state_entry : state_by_camera_id) {
+    State &state = state_entry.second;
+    state.camera_data->total_duration_90k -= state.deleted_duration_90k;
+    state.camera_data->total_sample_file_bytes -=
+        state.deleted_sample_file_bytes;
+    state.camera_data->min_start_time_90k = state.min_start_time_90k;
+    state.camera_data->max_end_time_90k = state.max_end_time_90k;
+  }
+  return true;
+}
+
+bool MoonfireDatabase::MarkSampleFilesDeleted(const std::vector<Uuid> &uuids,
+                                              std::string *error_message) {
+  if (uuids.empty()) {
+    return true;
+  }
+  DatabaseContext ctx(db_);
+  if (!ctx.BeginTransaction(error_message)) {
+    return false;
+  }
+  for (const auto &uuid : uuids) {
+    auto run = ctx.Borrow(&delete_reservation_stmt_);
+    run.BindBlob(":uuid", uuid.binary_view());
+    if (run.Step() != SQLITE_DONE) {
+      *error_message = run.error_message();
+      ctx.RollbackTransaction();
+      return false;
+    }
+    if (ctx.changes() != 1) {
+      *error_message = StrCat("no reservation for uuid ", uuid.UnparseText());
+      ctx.RollbackTransaction();
+      return false;
+    }
+  }
+  if (!ctx.CommitTransaction(error_message)) {
+    return false;
+  }
+  return true;
+}
+
 }  // namespace moonfire_nvr

src/moonfire-db.h

@@ -32,18 +32,30 @@
 // Currently focused on stuff needed by WebInterface to build a HTML or JSON
 // interface.
 //
-// Performance note: camera-level operations do a sequential scan through
+// This caches data in RAM, making the assumption that only one process is
-// essentially the entire database. This is unacceptable for full-sized
+// accessing the database at a time. (TODO: enforce with flock or some such.)
-// databases; it will have to be measured and improved. Ideas:
+// Performance and efficiency notes:
 //
-// * separate the video index blob from the rest of the recording row,
+// * several query operations here feature row callbacks. The callback is
-//   as it's expected to be 10X-100X larger than everything else and not
+//   invoked with the database lock. Thus, the caller mustn't perform database
-//   necessary for these operations.
+//   operations or other long-running operations.
-// * paged results + SQL indexes (but this may only help so much, as it'd be
+//
-//   useful to at least see what days have recordings in one go).
+// * startup may be slow, as it scans the entire index for the recording
-// * keep aggregates, either in-memory or as denormalized data in the camera
+//   table. This seems acceptable.
-//   table. Likely integrating with the recording system, although triggers
+//
-//   may also be possible.
+// * the operations used for web file serving should return results with
+//   acceptable latency.
+//
+// * however, the database lock may be held for longer than is acceptable for
+//   the critical path of recording frames. It may be necessary to preallocate
+//   sample file uuids and such to avoid this.
+//
+// * the caller may need to perform several different types of write
+//   operations in a row. It might be worth creating an interface for batching
+//   these inside a transaction, to reduce latency and SSD write cycles. The
+//   pre-commit and post-commit logic of each operation would have to be
+//   pulled apart, with the latter being called by this wrapper class on
+//   commit of the overall transaction.
 
 #ifndef MOONFIRE_NVR_MOONFIRE_DB_H
 #define MOONFIRE_NVR_MOONFIRE_DB_H
@@ -51,6 +63,7 @@
 #include <functional>
 #include <memory>
 #include <string>
+#include <vector>
 
 #include "common.h"
 #include "http.h"
@@ -62,28 +75,32 @@ namespace moonfire_nvr {
 
 // For use with MoonfireDatabase::ListCameras.
 struct ListCamerasRow {
-  int64_t id = -1;
   Uuid uuid;
   std::string short_name;
   std::string description;
   int64_t retain_bytes = -1;
 
-  // Aggregates summarizing completed (status=1) recordings.
+  // Aggregates summarizing completed recordings.
-  int64_t min_recording_start_time_90k = -1;
+  int64_t min_start_time_90k = -1;
-  int64_t max_recording_end_time_90k = -1;
+  int64_t max_end_time_90k = -1;
-  int64_t total_recording_duration_90k = -1;
+  int64_t total_duration_90k = -1;
   int64_t total_sample_file_bytes = -1;
 };
 
 // For use with MoonfireDatabase::GetCamera.
-// This is the same information as in ListCamerasRow minus the stuff
+// This includes everything in ListCamerasRow. In the future, it will include
-// that's calculable from ListCameraRecordingsRow, which the camera details
+// more data. Likely, that will mean a list of calendar days (in the system
-// webpage also grabs.
+// time zone) in which there is any data.
 struct GetCameraRow {
-  int64_t retain_bytes = -1;
-  Uuid uuid;
   std::string short_name;
   std::string description;
+  int64_t retain_bytes = -1;
+  int64_t min_start_time_90k = -1;
+  int64_t max_end_time_90k = -1;
+  int64_t total_duration_90k = -1;
+  int64_t total_sample_file_bytes = -1;
+
+  // TODO: std::vector<std::string> days;  // keys: YYYY-mm-dd.
 };
 
 // For use with MoonfireDatabase::ListCameraRecordings.
@@ -93,53 +110,137 @@ struct ListCameraRecordingsRow {
   int64_t end_time_90k = -1;
   int64_t video_samples = -1;
   int64_t sample_file_bytes = -1;
-  std::string video_sample_entry_sha1;
 
   // Joined from the video_sample_entry table.
-  int64_t width = -1;
+  // |video_sample_entry_sha1| is valid as long as the MoonfireDatabase.
-  int64_t height = -1;
+  re2::StringPiece video_sample_entry_sha1;
+  uint16_t width = 0;
+  uint16_t height = 0;
 };
 
+// For use with MoonfireDatabase::ListOldestSampleFiles.
+struct ListOldestSampleFilesRow {
+  int64_t camera_id = -1;
+  int64_t recording_id = -1;
+  Uuid sample_file_uuid;
+  int64_t duration_90k = -1;
+  int64_t sample_file_bytes = -1;
+};
+
+// Thread-safe after Init.
+// (Uses a DatabaseContext for locking.)
 class MoonfireDatabase {
  public:
-  explicit MoonfireDatabase(Database *db) : db_(db) {}
+  MoonfireDatabase() {}
   MoonfireDatabase(const MoonfireDatabase &) = delete;
   void operator=(const MoonfireDatabase &) = delete;
 
-  bool Init(std::string *error_message);
+  // |db| must outlive the MoonfireDatabase.
+  bool Init(Database *db, std::string *error_message);
 
   // List all cameras in the system, ordered by short name.
-  // Holds database lock; callback should be quick.
+  void ListCameras(std::function<IterationControl(const ListCamerasRow &)> cb);
-  bool ListCameras(std::function<IterationControl(const ListCamerasRow &)> cb,
-                   std::string *error_message);
 
-  bool GetCamera(int64_t camera_id, GetCameraRow *row,
+  // Get a single camera.
-                 std::string *error_message);
+  // Return true iff the camera exists.
+  bool GetCamera(Uuid camera_uuid, GetCameraRow *row);
 
-  // List all recordings associated with a camera, ordered by start time..
+  // List all recordings associated with a camera, descending by end time.
-  // Holds database lock; callback should be quick.
   bool ListCameraRecordings(
-      int64_t camera_id,
+      Uuid camera_uuid, int64_t start_time_90k, int64_t end_time_90k,
       std::function<IterationControl(const ListCameraRecordingsRow &)>,
       std::string *error_message);
 
   bool ListMp4Recordings(
-      int64_t camera_id, int64_t start_time_90k, int64_t end_time_90k,
+      Uuid camera_uuid, int64_t start_time_90k, int64_t end_time_90k,
       std::function<IterationControl(Recording &, const VideoSampleEntry &)>
           row_cb,
       std::string *error_message);
 
-  std::shared_ptr<VirtualFile> BuildMp4(int64_t camera_id,
+  // TODO: more nuanced error code for HTTP.
+  // TODO: this should move somewhere that has access to the
+  // currently-writing Recording as well.
+  std::shared_ptr<VirtualFile> BuildMp4(Uuid camera_uuid,
                                         int64_t start_time_90k,
                                         int64_t end_time_90k,
                                         std::string *error_message);
 
+  bool ListReservedSampleFiles(std::vector<Uuid> *reserved,
+                               std::string *error_message);
+
+  // Reserve |n| new sample file uuids.
+  // Returns an empty vector on error.
+  std::vector<Uuid> ReserveSampleFiles(int n, std::string *error_message);
+
+  // Insert a video sample entry if not already inserted.
+  // On success, |entry->id| is filled in with the id of a freshly-created or
+  // existing row.
+  bool InsertVideoSampleEntry(VideoSampleEntry *entry,
+                              std::string *error_message);
+
+  // Insert a new recording.
+  // The uuid must have been already reserved with ReserveSampleFileUuid above.
+  // On success, |recording->id| is filled in.
+  bool InsertRecording(Recording *recording, std::string *error_message);
+
+  // List sample files, starting from the oldest.
+  // The caller is expected to supply a |row_cb| that returns kBreak when
+  // enough have been listed.
+  bool ListOldestSampleFiles(
+      Uuid camera_uuid,
+      std::function<IterationControl(const ListOldestSampleFilesRow &)> row_cb,
+      std::string *error_message);
+
+  // Delete recording rows, moving their sample file uuids to the deleting
+  // state.
+  bool DeleteRecordings(const std::vector<ListOldestSampleFilesRow> &rows,
+                        std::string *error_message);
+
+  // Mark a set of sample files as deleted.
+  // This shouldn't be called until the files have been unlinke()ed and the
+  // parent directory fsync()ed.
+  // Returns error if any sample files are not in the deleting state.
+  bool MarkSampleFilesDeleted(const std::vector<Uuid> &uuids,
+                              std::string *error_message);
+
  private:
-  Database *const db_;
+  struct CameraData {
-  Statement list_cameras_query_;
+    // Cached values of the matching fields from the camera row.
-  Statement get_camera_query_;
+    int64_t id = -1;
-  Statement list_camera_recordings_query_;
+    std::string short_name;
-  Statement build_mp4_query_;
+    std::string description;
+    int64_t retain_bytes = -1;
+
+    // Aggregates of all recordings associated with the camera.
+    int64_t min_start_time_90k = -1;
+    int64_t max_end_time_90k = -1;
+    int64_t total_sample_file_bytes = -1;
+    int64_t total_duration_90k = -1;
+  };
+
+  enum class ReservationState { kWriting = 0, kDeleting = 1 };
+
+  // Efficiently (re-)compute the bounds of recorded time for a given camera.
+  bool ComputeCameraRecordingBounds(DatabaseContext *ctx, int64_t camera_id,
+                                    int64_t *min_start_time_90k,
+                                    int64_t *max_end_time_90k,
+                                    std::string *error_message);
+
+  Database *db_ = nullptr;
+  Statement list_camera_recordings_stmt_;
+  Statement build_mp4_stmt_;
+  Statement insert_reservation_stmt_;
+  Statement delete_reservation_stmt_;
+  Statement insert_video_sample_entry_stmt_;
+  Statement insert_recording_stmt_;
+  Statement list_oldest_sample_files_stmt_;
+  Statement delete_recording_stmt_;
+  Statement camera_min_start_stmt_;
+  Statement camera_max_start_stmt_;
+
+  std::map<Uuid, CameraData> cameras_by_uuid_;
+  std::map<int64_t, CameraData *> cameras_by_id_;
+  std::map<int64_t, VideoSampleEntry> video_sample_entries_;
 };
 
 }  // namespace moonfire_nvr

src/moonfire-nvr-test.cc

@@ -148,7 +148,7 @@ class StreamTest : public testing::Test {
     env_.clock = &clock_;
     env_.video_source = &video_source_;
     env_.fs = GetRealFilesystem();
-    clock_.Sleep({1430006400, 0});  // 2016-04-26 00:00:00 UTC
+    clock_.Sleep({1430006400, 0});  // 2015-04-26 00:00:00 UTC
 
     config_.set_base_path(test_dir_);
     config_.set_rotate_sec(5);

src/mp4.cc

@@ -727,12 +727,11 @@ Mp4FileBuilder &Mp4FileBuilder::SetSampleEntry(const VideoSampleEntry &entry) {
 std::shared_ptr<VirtualFile> Mp4FileBuilder::Build(std::string *error_message) {
   int32_t sample_offset = 1;
   for (auto &segment : segments_) {
-    if (segment->recording.video_sample_entry_sha1 !=
+    if (segment->recording.video_sample_entry_id != video_sample_entry_.id) {
-        video_sample_entry_.sha1) {
+      *error_message = StrCat(
-      *error_message =
+          "inconsistent video sample entries. builder has: ",
-          StrCat("inconsistent video sample entries. builder has: ",
+          video_sample_entry_.id, " (sha1 ", ToHex(video_sample_entry_.sha1),
-                 ToHex(video_sample_entry_.sha1), ", segment has: ",
+          ", segment has: ", segment->recording.video_sample_entry_id);
-                 ToHex(segment->recording.video_sample_entry_sha1));
       return std::shared_ptr<VirtualFile>();
     }
 

src/recording.h

@@ -55,16 +55,19 @@ constexpr int64_t kTimeUnitsPerSecond = 90000;
 // This limit should be more than the normal rotation time,
 // as recording doesn't happen until the next key frame.
 // 5 minutes is generously more than 1 minute, but still sufficient to
-// allow the optimization to be useful.
+// allow the optimization to be useful. This value must match the CHECK
+// constraint on duration_90k in schema.sql.
 constexpr int64_t kMaxRecordingDuration = 5 * 60 * kTimeUnitsPerSecond;
 
 // Various fields from the "recording" table which are useful when viewing
 // recordings.
 struct Recording {
-  int64_t rowid = -1;
+  int64_t id = -1;
+  int64_t camera_id = -1;
   std::string sample_file_path;
   std::string sample_file_sha1;
   Uuid sample_file_uuid;
+  int64_t video_sample_entry_id = -1;
 
   // Fields populated by SampleIndexEncoder.
   int64_t start_time_90k = -1;
@@ -72,7 +75,6 @@ struct Recording {
   int64_t sample_file_bytes = -1;
   int64_t video_samples = -1;
   int64_t video_sync_samples = -1;
-  std::string video_sample_entry_sha1;
   std::string video_index;
 };
 
@@ -196,6 +198,7 @@ class SampleFileWriter {
 };
 
 struct VideoSampleEntry {
+  int64_t id = -1;
   std::string sha1;
   std::string data;
   uint16_t width = 0;

src/schema.sql

@@ -31,14 +31,14 @@
 -- schema.sql: SQLite3 database schema for Moonfire NVR.
 -- See also design/schema.md.
 
-pragma journal_mode = wal;
+--pragma journal_mode = wal;
 
 create table camera (
   id integer primary key,
-  uuid blob unique not null,
+  uuid blob unique,-- not null check (length(uuid) = 16),
 
   -- A short name of the camera, used in log messages.
-  short_name text not null,
+  short_name text,-- not null,
 
   -- A short description of the camera.
   description text,
@@ -63,45 +63,70 @@ create table camera (
 
   -- The number of bytes of video to retain, excluding the currently-recording
   -- file. Older files will be deleted as necessary to stay within this limit.
-  retain_bytes integer
+  retain_bytes integer not null check (retain_bytes >= 0)
 );
 
--- A single, typically 60-second, recorded segment of video.
+-- Each row represents a single completed recorded segment of video.
+-- Recordings are typically ~60 seconds; never more than 5 minutes.
 create table recording (
   id integer primary key,
   camera_id integer references camera (id) not null,
 
-  status integer not null,  -- 0 (WRITING), 1 (WRITTEN), or 2 (DELETING)
+  sample_file_bytes integer not null check (sample_file_bytes > 0),
 
-  sample_file_uuid blob unique not null,
+  -- The starting time of the recording, in 90 kHz units since
-  sample_file_sha1 blob,
-  sample_file_bytes integer,
-
-  -- The starting and ending time of the recording, in 90 kHz units since
   -- 1970-01-01 00:00:00 UTC.
-  start_time_90k integer not null,
+  start_time_90k integer not null check (start_time_90k > 0),
-  end_time_90k integer,
 
-  video_samples integer,
+  -- The duration of the recording, in 90 kHz units.
-  video_sync_samples integer,
+  duration_90k integer not null
-  video_sample_entry_sha1 blob references video_sample_entry (sha1),
+      check (duration_90k >= 0 and duration_90k < 5*60*90000),
-  video_index blob
+
+  video_samples integer not null check (video_samples > 0),
+  video_sync_samples integer not null check (video_samples > 0),
+  video_sample_entry_id integer references video_sample_entry (id),
+
+  sample_file_uuid blob not null check (length(sample_file_uuid) = 16),
+  sample_file_sha1 blob not null check (length(sample_file_sha1) = 20),
+  video_index blob not null check (length(video_index) > 0)
 );
 
-create index recording_start_time_90k on recording (start_time_90k);
+create index recording_cover on recording (
+  -- Typical queries use "where camera_id = ? order by start_time_90k (desc)?".
+  camera_id,
+  start_time_90k,
+
+  -- These fields are not used for ordering; they cover most queries so
+  -- that only database verification and actual viewing of recordings need
+  -- to consult the underlying row.
+  duration_90k,
+  video_samples,
+  video_sample_entry_id,
+  sample_file_bytes
+);
+
+-- Files in the sample file directory which may be present but should simply be
+-- discarded on startup. (Recordings which were never completed or have been
+-- marked for completion.)
+create table reserved_sample_files (
+  uuid blob primary key check (length(uuid) = 16),
+  state integer not null  -- 0 (writing) or 1 (deleted)
+) without rowid;
 
 -- A concrete box derived from a ISO/IEC 14496-12 section 8.5.2
 -- VisualSampleEntry box. Describes the codec, width, height, etc.
 create table video_sample_entry (
+  id integer primary key,
+
   -- A SHA-1 hash of |bytes|.
-  sha1 blob primary key,
+  sha1 blob unique not null check (length(sha1) = 20),
 
   -- The width and height in pixels; must match values within
   -- |sample_entry_bytes|.
-  width integer,
+  width integer not null check (width > 0),
-  height integer,
+  height integer not null check (height > 0),
 
-  -- A serialized SampleEntry box, including the leading length and box
+  -- The serialized box, including the leading length and box type (avcC in
-  -- type (avcC in the case of H.264).
+  -- the case of H.264).
-  bytes blob
+  data blob not null check (length(data) > 86)
 );

src/sqlite-test.cc

@@ -78,7 +78,7 @@ TEST_F(SqliteTest, BindAndColumn) {
                         0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f};
   re2::StringPiece blob_piece = re2::StringPiece(kBlob, sizeof(kBlob));
   const char kText[] = "foo";
-  const int64_t kInt64 = INT64_C(0xdeadbeeffeedface);
+  const int64_t kInt64 = INT64_C(0xbeeffeedface);
 
   DatabaseContext ctx(&db_);
   {

src/sqlite.cc

@@ -88,7 +88,8 @@ bool DatabaseContext::BeginTransaction(std::string *error_message) {
   sqlite3_step(db_->begin_transaction_.me_);
   int ret = sqlite3_reset(db_->begin_transaction_.me_);
   if (ret != SQLITE_OK) {
-    *error_message = sqlite3_errstr(ret);
+    *error_message =
+        StrCat("begin transaction: ", sqlite3_errstr(ret), " (", ret, ")");
     return false;
   }
   transaction_open_ = true;
@@ -103,7 +104,8 @@ bool DatabaseContext::CommitTransaction(std::string *error_message) {
   sqlite3_step(db_->commit_transaction_.me_);
   int ret = sqlite3_reset(db_->commit_transaction_.me_);
   if (ret != SQLITE_OK) {
-    *error_message = sqlite3_errstr(ret);
+    *error_message =
+        StrCat("commit transaction: ", sqlite3_errstr(ret), " (", ret, ")");
     return false;
   }
   transaction_open_ = false;
@@ -118,7 +120,8 @@ void DatabaseContext::RollbackTransaction() {
   sqlite3_step(db_->rollback_transaction_.me_);
   int ret = sqlite3_reset(db_->rollback_transaction_.me_);
   if (ret != SQLITE_OK) {
-    LOG(WARNING) << this << ": rollback failed: " << sqlite3_errstr(ret);
+    LOG(WARNING) << this << ": rollback failed: " << sqlite3_errstr(ret) << " ("
+                 << ret << ")";
     return;
   }
   transaction_open_ = false;
@@ -137,26 +140,36 @@ RunningStatement DatabaseContext::UseOnce(re2::StringPiece sql) {
 RunningStatement::RunningStatement(Statement *statement,
                                    const std::string &deferred_error,
                                    bool owns_statement)
-    : statement_(statement),
+    : error_message_(deferred_error), owns_statement_(owns_statement) {
-      error_message_(deferred_error),
+  if (statement != nullptr && statement->valid()) {
-      owns_statement_(owns_statement) {
+    CHECK(!statement->borrowed_) << "Statement already borrowed!";
-  CHECK(!statement->borrowed_) << "Statement already borrowed!";
+    statement->borrowed_ = true;
-  statement->borrowed_ = true;
+    statement_ = statement;
+  } else if (error_message_.empty()) {
+    error_message_ = "invalid statement";
+  }
+
   if (!error_message_.empty()) {
     status_ = SQLITE_MISUSE;
-  } else if (statement == nullptr) {
-    status_ = SQLITE_MISUSE;
-    error_message_ = "invalid statement";
   }
 }
 
+RunningStatement::RunningStatement(RunningStatement &&o) {
+  statement_ = o.statement_;
+  status_ = o.status_;
+  owns_statement_ = o.owns_statement_;
+  o.statement_ = nullptr;
+}
+
 RunningStatement::~RunningStatement() {
-  CHECK(statement_->borrowed_) << "Statement no longer borrowed!";
+  if (statement_ != nullptr) {
-  sqlite3_reset(statement_->me_);
+    CHECK(statement_->borrowed_) << "Statement no longer borrowed!";
-  sqlite3_clear_bindings(statement_->me_);
+    sqlite3_clear_bindings(statement_->me_);
-  statement_->borrowed_ = false;
+    sqlite3_reset(statement_->me_);
-  if (owns_statement_) {
+    statement_->borrowed_ = false;
-    delete statement_;
+    if (owns_statement_) {
+      delete statement_;
+    }
   }
 }
 
@@ -168,7 +181,7 @@ void RunningStatement::BindBlob(int param, re2::StringPiece value) {
                                 value.size(), SQLITE_TRANSIENT);
   if (status_ != SQLITE_OK) {
     error_message_ = StrCat("Unable to bind parameter ", param, ": ",
-                            sqlite3_errstr(status_));
+                            sqlite3_errstr(status_), " (", status_, ")");
   }
 }
 
@@ -186,7 +199,7 @@ void RunningStatement::BindBlob(const char *name, re2::StringPiece value) {
                                 value.size(), SQLITE_TRANSIENT);
   if (status_ != SQLITE_OK) {
     error_message_ = StrCat("Unable to bind parameter ", name, ": ",
-                            sqlite3_errstr(status_));
+                            sqlite3_errstr(status_), " (", status_, ")");
   }
 }
 
@@ -197,7 +210,7 @@ void RunningStatement::BindInt64(int param, int64_t value) {
   status_ = sqlite3_bind_int64(statement_->me_, param, value);
   if (status_ != SQLITE_OK) {
     error_message_ = StrCat("Unable to bind parameter ", param, ": ",
-                            sqlite3_errstr(status_));
+                            sqlite3_errstr(status_), " (", status_, ")");
   }
 }
 
@@ -214,7 +227,7 @@ void RunningStatement::BindInt64(const char *name, int64_t value) {
   status_ = sqlite3_bind_int64(statement_->me_, param, value);
   if (status_ != SQLITE_OK) {
     error_message_ = StrCat("Unable to bind parameter ", name, ": ",
-                            sqlite3_errstr(status_));
+                            sqlite3_errstr(status_), " (", status_, ")");
   }
 }
 
@@ -226,7 +239,7 @@ void RunningStatement::BindText(int param, re2::StringPiece value) {
                                 value.size(), SQLITE_TRANSIENT, SQLITE_UTF8);
   if (status_ != SQLITE_OK) {
     error_message_ = StrCat("Unable to bind parameter ", param, ": ",
-                            sqlite3_errstr(status_));
+                            sqlite3_errstr(status_), " (", status_, ")");
   }
 }
 
@@ -243,7 +256,7 @@ void RunningStatement::BindText(const char *name, re2::StringPiece value) {
                                 value.size(), SQLITE_TRANSIENT, SQLITE_UTF8);
   if (status_ != SQLITE_OK) {
     error_message_ = StrCat("Unable to bind parameter ", name, ": ",
-                            sqlite3_errstr(status_));
+                            sqlite3_errstr(status_), " (", status_, ")");
   }
 }
 
@@ -252,7 +265,8 @@ int RunningStatement::Step() {
     return status_;
   }
   status_ = sqlite3_step(statement_->me_);
-  error_message_ = sqlite3_errstr(status_);
+  error_message_ =
+      StrCat("step: ", sqlite3_errstr(status_), " (", status_, ")");
   return status_;
 }
 
@@ -291,7 +305,8 @@ bool Database::Open(const char *filename, int flags,
   std::call_once(global_setup, &GlobalSetup);
   int ret = sqlite3_open_v2(filename, &me_, flags, nullptr);
   if (ret != SQLITE_OK) {
-    *error_message = sqlite3_errstr(ret);
+    *error_message =
+        StrCat("open ", filename, ": ", sqlite3_errstr(ret), " (", ret, ")");
     return false;
   }
 
@@ -299,8 +314,8 @@ bool Database::Open(const char *filename, int flags,
   if (ret != SQLITE_OK) {
     sqlite3_close(me_);
     me_ = nullptr;
-    *error_message =
+    *error_message = StrCat("while enabling extended result codes: ",
-        StrCat("while enabling extended result codes: ", sqlite3_errstr(ret));
+                            sqlite3_errstr(ret), " (", ret, ")");
     return false;
   }
 
@@ -331,8 +346,8 @@ bool Database::Open(const char *filename, int flags,
   if (ret != SQLITE_DONE) {
     sqlite3_close(me_);
     me_ = nullptr;
-    *error_message =
+    *error_message = StrCat("while enabling foreign keys: ",
-        StrCat("while enabling foreign keys: ", sqlite3_errstr(ret));
+                            sqlite3_errstr(ret), " (", ret, ")");
     return false;
   }
 
@@ -346,7 +361,7 @@ Statement Database::Prepare(re2::StringPiece sql, size_t *used,
   int err =
       sqlite3_prepare_v2(me_, sql.data(), sql.size(), &statement.me_, &tail);
   if (err != SQLITE_OK) {
-    *error_message = sqlite3_errstr(err);
+    *error_message = StrCat("prepare: ", sqlite3_errstr(err), " (", err, ")");
     return statement;
   }
   if (used != nullptr) {

src/sqlite.h

@@ -141,7 +141,7 @@ class Database {
 // }
 class RunningStatement {
  public:
-  RunningStatement(RunningStatement &&) = default;
+  RunningStatement(RunningStatement &&o);
 
   // Reset/unbind/return the statement for the next use (in the case of
   // Borrow) or delete it (in the case of UseOnce).

src/time.h

@@ -66,6 +66,10 @@ inline struct timespec SecToTimespec(double sec) {
   return {static_cast<time_t>(intpart), static_cast<long>(fractpart * kNanos)};
 }
 
+inline double TimespecToSec(struct timespec t) {
+  return t.tv_sec + static_cast<double>(t.tv_nsec) / kNanos;
+}
+
 // Returns the real wall clock, which will never be deleted.
 WallClock *GetRealClock();
 

src/uuid.h

@@ -61,6 +61,8 @@ class Uuid {
   bool operator==(const Uuid &) const;
   bool operator<(const Uuid &) const;
 
+  bool is_null() const { return uuid_is_null(me_); }
+
  private:
   friend class RealUuidGenerator;
   uuid_t me_;

src/web.cc

@@ -62,34 +62,35 @@ void WebInterface::HandleCameraList(evhttp_request *req, void *arg) {
       "<table>\n");
   auto row_cb = [&](const ListCamerasRow &row) {
     auto seconds =
-        (row.max_recording_end_time_90k - row.min_recording_start_time_90k) /
+        (row.max_end_time_90k - row.min_start_time_90k) / kTimeUnitsPerSecond;
-        kTimeUnitsPerSecond;
+    std::string min_start_time_90k =
+        row.min_start_time_90k == -1 ? std::string("n/a")
+                                     : PrettyTimestamp(row.min_start_time_90k);
+    std::string max_end_time_90k = row.max_end_time_90k == -1
+                                       ? std::string("n/a")
+                                       : PrettyTimestamp(row.max_end_time_90k);
     buf.AddPrintf(
-        "<tr class=header><td colspan=2><a href=\"/camera?id=%" PRId64
+        "<tr class=header><td colspan=2><a href=\"/camera?uuid=%s\">%s</a>"
-        "\">%s</a></td></tr>\n"
+        "</td></tr>\n"
         "<tr><td>description</td><td>%s</td></tr>\n"
         "<tr><td>space</td><td>%s / %s (%.1f%%)</td></tr>\n"
         "<tr><td>uuid</td><td>%s</td></tr>\n"
         "<tr><td>oldest recording</td><td>%s</td></tr>\n"
         "<tr><td>newest recording</td><td>%s</td></tr>\n"
         "<tr><td>total duration</td><td>%s</td></tr>\n",
-        row.id, EscapeHtml(row.short_name).c_str(),
+        row.uuid.UnparseText().c_str(), EscapeHtml(row.short_name).c_str(),
         EscapeHtml(row.description).c_str(),
         EscapeHtml(HumanizeWithBinaryPrefix(row.total_sample_file_bytes, "B"))
             .c_str(),
         EscapeHtml(HumanizeWithBinaryPrefix(row.retain_bytes, "B")).c_str(),
         100.f * row.total_sample_file_bytes / row.retain_bytes,
         EscapeHtml(row.uuid.UnparseText()).c_str(),
-        EscapeHtml(PrettyTimestamp(row.min_recording_start_time_90k)).c_str(),
+        EscapeHtml(min_start_time_90k).c_str(),
-        EscapeHtml(PrettyTimestamp(row.max_recording_end_time_90k)).c_str(),
+        EscapeHtml(max_end_time_90k).c_str(),
         EscapeHtml(HumanizeDuration(seconds)).c_str());
     return IterationControl::kContinue;
   };
-  std::string error_message;
+  this_->mdb_->ListCameras(row_cb);
-  if (!this_->mdb_->ListCameras(row_cb, &error_message)) {
-    return evhttp_send_error(req, HTTP_INTERNAL,
-                             EscapeHtml(error_message).c_str());
-  }
   buf.Add(
       "</table>\n"
       "</body>\n"
@@ -100,19 +101,15 @@ void WebInterface::HandleCameraList(evhttp_request *req, void *arg) {
 void WebInterface::HandleCameraDetail(evhttp_request *req, void *arg) {
   auto *this_ = reinterpret_cast<WebInterface *>(arg);
 
-  int64_t camera_id;
+  Uuid camera_uuid;
   QueryParameters params(evhttp_request_get_uri(req));
-  if (!params.ok() || !Atoi64(params.Get("id"), 10, &camera_id)) {
+  if (!params.ok() || !camera_uuid.ParseText(params.Get("uuid"))) {
     return evhttp_send_error(req, HTTP_BADREQUEST, "bad query parameters");
   }
 
   GetCameraRow camera_row;
-  std::string error_message;
+  if (!this_->mdb_->GetCamera(camera_uuid, &camera_row)) {
-  if (!this_->mdb_->GetCamera(camera_id, &camera_row, &error_message)) {
+    return evhttp_send_error(req, HTTP_NOTFOUND, "no such camera");
-    // TODO: more nuanced error here, such as HTTP_NOTFOUND where appropriate.
-    return evhttp_send_error(
-        req, HTTP_INTERNAL,
-        StrCat("sqlite query failed: ", EscapeHtml(error_message)).c_str());
   }
 
   EvBuffer buf;
@@ -151,11 +148,12 @@ void WebInterface::HandleCameraDetail(evhttp_request *req, void *arg) {
                                       aggregated.start_time_90k) /
                    kTimeUnitsPerSecond;
     buf.AddPrintf(
-        "<tr><td><a href=\"/view.mp4?camera_id=%" PRId64
+        "<tr><td><a href=\"/view.mp4?camera_uuid=%s&start_time_90k=%" PRId64
-        "&start_time_90k=%" PRId64 "&end_time_90k=%" PRId64
+        "&end_time_90k=%" PRId64
         "\">%s</a></td><td>%s</td><td>%dx%d</td>"
         "<td>%.0f</td><td>%s</td><td>%s</td></tr>\n",
-        camera_id, aggregated.start_time_90k, aggregated.end_time_90k,
+        camera_uuid.UnparseText().c_str(), aggregated.start_time_90k,
+        aggregated.end_time_90k,
         PrettyTimestamp(aggregated.start_time_90k).c_str(),
         PrettyTimestamp(aggregated.end_time_90k).c_str(),
         static_cast<int>(aggregated.width), static_cast<int>(aggregated.height),
@@ -169,10 +167,10 @@ void WebInterface::HandleCameraDetail(evhttp_request *req, void *arg) {
   auto handle_sql_row = [&](const ListCameraRecordingsRow &row) {
     auto new_duration_90k = row.end_time_90k - aggregated.start_time_90k;
     if (row.video_sample_entry_sha1 == aggregated.video_sample_entry_sha1 &&
-        row.start_time_90k == aggregated.end_time_90k &&
+        row.end_time_90k == aggregated.start_time_90k &&
         new_duration_90k < kForceSplitDuration90k) {
       // Append to current .mp4.
-      aggregated.end_time_90k = row.end_time_90k;
+      aggregated.start_time_90k = row.start_time_90k;
       aggregated.video_samples += row.video_samples;
       aggregated.sample_file_bytes += row.sample_file_bytes;
     } else {
@@ -182,7 +180,11 @@ void WebInterface::HandleCameraDetail(evhttp_request *req, void *arg) {
     }
     return IterationControl::kContinue;
   };
-  if (!this_->mdb_->ListCameraRecordings(camera_id, handle_sql_row,
+  int64_t start_time_90k = 0;
+  int64_t end_time_90k = std::numeric_limits<int64_t>::max();
+  std::string error_message;
+  if (!this_->mdb_->ListCameraRecordings(camera_uuid, start_time_90k,
+                                         end_time_90k, handle_sql_row,
                                          &error_message)) {
     return evhttp_send_error(
         req, HTTP_INTERNAL,
@@ -198,11 +200,11 @@ void WebInterface::HandleCameraDetail(evhttp_request *req, void *arg) {
 void WebInterface::HandleMp4View(evhttp_request *req, void *arg) {
   auto *this_ = reinterpret_cast<WebInterface *>(arg);
 
-  int64_t camera_id;
+  Uuid camera_uuid;
   int64_t start_time_90k;
   int64_t end_time_90k;
   QueryParameters params(evhttp_request_get_uri(req));
-  if (!params.ok() || !Atoi64(params.Get("camera_id"), 10, &camera_id) ||
+  if (!params.ok() || !camera_uuid.ParseText(params.Get("camera_uuid")) ||
       !Atoi64(params.Get("start_time_90k"), 10, &start_time_90k) ||
       !Atoi64(params.Get("end_time_90k"), 10, &end_time_90k) ||
       start_time_90k < 0 || start_time_90k >= end_time_90k) {
@@ -210,7 +212,7 @@ void WebInterface::HandleMp4View(evhttp_request *req, void *arg) {
   }
 
   std::string error_message;
-  auto file = this_->mdb_->BuildMp4(camera_id, start_time_90k, end_time_90k,
+  auto file = this_->mdb_->BuildMp4(camera_uuid, start_time_90k, end_time_90k,
                                     &error_message);
   if (file == nullptr) {
     // TODO: more nuanced HTTP status codes.