From b9d6526492c1456265dd68931223bdb30a1de916 Mon Sep 17 00:00:00 2001 From: Scott Lamb Date: Sun, 17 Jan 2016 01:14:29 -0800 Subject: [PATCH] Optimize the SQLite query for building .mp4s. On my laptop, with a month's data, a test query would take 0.1 to 0.2 seconds before. Now it takes 0.001 to 0.004 seconds. I improved this by creating and taking advantage of an index on start time. It's a little more complicated than that because the desired timespan is specified in terms of a recording's start and end time, not start time alone. I defined a maximum duration of a recording (5 minutes) and specified this with an extra condition in the query so that the end time can be used to narrow the valid range of start times. "explain query plan select ..." output confirms it's using the index with both > and < comparisons: 0|0|0|SEARCH TABLE recording USING INDEX recording_start_time_90k (start_time_90k>? AND start_time_90kPrepare( + std::string build_mp4_sql = StrCat( R"( select recording.rowid, @@ -134,11 +134,14 @@ bool MoonfireDatabase::Init(std::string *error_message) { where recording.status = 1 and camera_id = :camera_id and + recording.start_time_90k > :start_time_90k - )", + kMaxRecordingDuration, " and\n", + R"( recording.start_time_90k < :end_time_90k and recording.end_time_90k > :start_time_90k order by - recording.start_time_90k;)", - nullptr, error_message); + recording.start_time_90k;)"); + build_mp4_query_ = db_->Prepare(build_mp4_sql, nullptr, error_message); if (!build_mp4_query_.valid()) { return false; } @@ -230,6 +233,56 @@ bool MoonfireDatabase::ListCameraRecordings( return true; } +bool MoonfireDatabase::ListMp4Recordings( + int64_t camera_id, int64_t start_time_90k, int64_t end_time_90k, + std::function + row_cb, + std::string *error_message) { + VLOG(1) << "...(1/4): Waiting for database lock"; + DatabaseContext ctx(db_); + VLOG(1) << "...(2/4): Querying database"; + auto run = ctx.Borrow(&build_mp4_query_); + run.BindInt64(":camera_id", camera_id); + run.BindInt64(":end_time_90k", end_time_90k); + run.BindInt64(":start_time_90k", start_time_90k); + Recording recording; + VideoSampleEntry sample_entry; + while (run.Step() == SQLITE_ROW) { + recording.rowid = run.ColumnInt64(0); + recording.start_time_90k = run.ColumnInt64(1); + recording.end_time_90k = run.ColumnInt64(2); + recording.sample_file_bytes = run.ColumnInt64(3); + if (!recording.sample_file_uuid.ParseBinary(run.ColumnBlob(4))) { + *error_message = + StrCat("recording ", recording.rowid, " has unparseable uuid ", + ToHex(run.ColumnBlob(4))); + return false; + } + recording.sample_file_path = + StrCat("/home/slamb/new-moonfire/sample/", + recording.sample_file_uuid.UnparseText()); + recording.sample_file_sha1 = run.ColumnBlob(5).as_string(); + recording.video_sample_entry_sha1 = run.ColumnBlob(6).as_string(); + recording.video_index = run.ColumnBlob(7).as_string(); + recording.video_samples = run.ColumnInt64(8); + recording.video_sync_samples = run.ColumnInt64(9); + + if (recording.video_sample_entry_sha1 != sample_entry.sha1) { + sample_entry.sha1 = run.ColumnBlob(6).as_string(); + sample_entry.data = run.ColumnBlob(10).as_string(); + sample_entry.width = run.ColumnInt64(11); + sample_entry.height = run.ColumnInt64(12); + } + + row_cb(recording, sample_entry); + } + if (run.status() != SQLITE_DONE && run.status() != SQLITE_ROW) { + *error_message = StrCat("sqlite query failed: ", run.error_message()); + return false; + } + return true; +} + std::shared_ptr MoonfireDatabase::BuildMp4( int64_t camera_id, int64_t start_time_90k, int64_t end_time_90k, std::string *error_message) { @@ -239,81 +292,51 @@ std::shared_ptr MoonfireDatabase::BuildMp4( Mp4FileBuilder builder; int64_t next_row_start_time_90k = start_time_90k; - VideoSampleEntry sample_entry; int64_t rows = 0; - { - VLOG(1) << "...(1/4): Waiting for database lock"; - DatabaseContext ctx(db_); - VLOG(1) << "...(2/4): Querying database"; - auto run = ctx.Borrow(&build_mp4_query_); - run.BindInt64(":camera_id", camera_id); - run.BindInt64(":end_time_90k", end_time_90k); - run.BindInt64(":start_time_90k", start_time_90k); - Recording recording; - while (run.Step() == SQLITE_ROW) { - recording.rowid = run.ColumnInt64(0); - VLOG(2) << "row: " << recording.rowid; - recording.start_time_90k = run.ColumnInt64(1); - recording.end_time_90k = run.ColumnInt64(2); - recording.sample_file_bytes = run.ColumnInt64(3); - if (!recording.sample_file_uuid.ParseBinary(run.ColumnBlob(4))) { - *error_message = - StrCat("recording ", recording.rowid, " has unparseable uuid ", - ToHex(run.ColumnBlob(4))); - return false; - } - recording.sample_file_path = - StrCat("/home/slamb/new-moonfire/sample/", - recording.sample_file_uuid.UnparseText()); - recording.sample_file_sha1 = run.ColumnBlob(5).as_string(); - recording.video_sample_entry_sha1 = run.ColumnBlob(6).as_string(); - recording.video_index = run.ColumnBlob(7).as_string(); - recording.video_samples = run.ColumnInt64(8); - recording.video_sync_samples = run.ColumnInt64(9); - - if (rows == 0 && recording.start_time_90k != next_row_start_time_90k) { - *error_message = - StrCat("recording starts late: ", - PrettyTimestamp(recording.start_time_90k), " (", - recording.start_time_90k, ") rather than requested: ", - PrettyTimestamp(start_time_90k), " (", start_time_90k, ")"); - return false; - } else if (recording.start_time_90k != next_row_start_time_90k) { - *error_message = - StrCat("gap/overlap in recording: ", - PrettyTimestamp(next_row_start_time_90k), " (", - next_row_start_time_90k, ") to: ", - PrettyTimestamp(recording.start_time_90k), " (", - recording.start_time_90k, ") before row ", rows); - return false; - } - - next_row_start_time_90k = recording.end_time_90k; - - if (rows > 0 && recording.video_sample_entry_sha1 != sample_entry.sha1) { - *error_message = - StrCat("inconsistent video sample entries: this recording has ", - ToHex(recording.video_sample_entry_sha1), ", previous had ", - ToHex(sample_entry.sha1)); - return false; - } else if (rows == 0) { - sample_entry.sha1 = run.ColumnBlob(6).as_string(); - sample_entry.data = run.ColumnBlob(10).as_string(); - sample_entry.width = run.ColumnInt64(11); - sample_entry.height = run.ColumnInt64(12); - builder.SetSampleEntry(sample_entry); - } - - // TODO: correct bounds within recording. - // Currently this can return too much data. - builder.Append(std::move(recording), 0, - std::numeric_limits::max()); - ++rows; + bool ok = true; + auto row_cb = [&](Recording &recording, + const VideoSampleEntry &sample_entry) { + if (rows == 0 && recording.start_time_90k != next_row_start_time_90k) { + *error_message = StrCat( + "recording starts late: ", PrettyTimestamp(recording.start_time_90k), + " (", recording.start_time_90k, ") rather than requested: ", + PrettyTimestamp(start_time_90k), " (", start_time_90k, ")"); + ok = false; + return IterationControl::kBreak; + } else if (recording.start_time_90k != next_row_start_time_90k) { + *error_message = StrCat("gap/overlap in recording: ", + PrettyTimestamp(next_row_start_time_90k), " (", + next_row_start_time_90k, ") to: ", + PrettyTimestamp(recording.start_time_90k), " (", + recording.start_time_90k, ") before row ", rows); + ok = false; + return IterationControl::kBreak; } - if (run.status() != SQLITE_DONE) { - *error_message = StrCat("sqlite query failed: ", run.error_message()); - return false; + + next_row_start_time_90k = recording.end_time_90k; + + if (rows > 0 && recording.video_sample_entry_sha1 != sample_entry.sha1) { + *error_message = + StrCat("inconsistent video sample entries: this recording has ", + ToHex(recording.video_sample_entry_sha1), ", previous had ", + ToHex(sample_entry.sha1)); + ok = false; + return IterationControl::kBreak; + } else if (rows == 0) { + builder.SetSampleEntry(sample_entry); } + + // TODO: correct bounds within recording. + // Currently this can return too much data. + builder.Append(std::move(recording), 0, + std::numeric_limits::max()); + ++rows; + return IterationControl::kContinue; + }; + if (!ok || + !ListMp4Recordings(camera_id, start_time_90k, end_time_90k, row_cb, + error_message)) { + return false; } if (rows == 0) { *error_message = StrCat("no recordings in range"); diff --git a/src/moonfire-db.h b/src/moonfire-db.h index e15013a..5d57dd1 100644 --- a/src/moonfire-db.h +++ b/src/moonfire-db.h @@ -123,6 +123,12 @@ class MoonfireDatabase { std::function, std::string *error_message); + bool ListMp4Recordings( + int64_t camera_id, int64_t start_time_90k, int64_t end_time_90k, + std::function + row_cb, + std::string *error_message); + std::shared_ptr BuildMp4(int64_t camera_id, int64_t start_time_90k, int64_t end_time_90k, diff --git a/src/recording.h b/src/recording.h index 1b503a7..13fa8f3 100644 --- a/src/recording.h +++ b/src/recording.h @@ -50,6 +50,14 @@ namespace moonfire_nvr { constexpr int64_t kTimeUnitsPerSecond = 90000; +// Recordings are never longer than this (5 minutes). +// Having such a limit dramatically speeds up some SQL queries. +// This limit should be more than the normal rotation time, +// as recording doesn't happen until the next key frame. +// 5 minutes is generously more than 1 minute, but still sufficient to +// allow the optimization to be useful. +constexpr int64_t kMaxRecordingDuration = 5 * 60 * kTimeUnitsPerSecond; + // Various fields from the "recording" table which are useful when viewing // recordings. struct Recording { diff --git a/src/schema.sql b/src/schema.sql index 77d9372..d4672d5 100644 --- a/src/schema.sql +++ b/src/schema.sql @@ -88,6 +88,8 @@ create table recording ( video_index blob ); +create index recording_start_time_90k on recording (start_time_90k); + -- A concrete box derived from a ISO/IEC 14496-12 section 8.5.2 -- VisualSampleEntry box. Describes the codec, width, height, etc. create table video_sample_entry (