mirror of
https://github.com/scottlamb/moonfire-nvr.git
synced 2025-07-29 10:11:00 -04:00
Optimize the SQLite query for building .mp4s.
On my laptop, with a month's data, a test query would take 0.1 to 0.2 seconds before. Now it takes 0.001 to 0.004 seconds. I improved this by creating and taking advantage of an index on start time. It's a little more complicated than that because the desired timespan is specified in terms of a recording's start and end time, not start time alone. I defined a maximum duration of a recording (5 minutes) and specified this with an extra condition in the query so that the end time can be used to narrow the valid range of start times. "explain query plan select ..." output confirms it's using the index with both > and < comparisons: 0|0|0|SEARCH TABLE recording USING INDEX recording_start_time_90k (start_time_90k>? AND start_time_90k<?) 0|1|1|SEARCH TABLE video_sample_entry USING INDEX sqlite_autoindex_video_sample_entry_1 (sha1=?) I also refactored ListMp4Recordings out of BuildMp4File to make the measurement easier.
This commit is contained in:
parent
40cd983355
commit
b9d6526492
@ -112,7 +112,7 @@ bool MoonfireDatabase::Init(std::string *error_message) {
|
|||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
|
|
||||||
build_mp4_query_ = db_->Prepare(
|
std::string build_mp4_sql = StrCat(
|
||||||
R"(
|
R"(
|
||||||
select
|
select
|
||||||
recording.rowid,
|
recording.rowid,
|
||||||
@ -134,11 +134,14 @@ bool MoonfireDatabase::Init(std::string *error_message) {
|
|||||||
where
|
where
|
||||||
recording.status = 1 and
|
recording.status = 1 and
|
||||||
camera_id = :camera_id and
|
camera_id = :camera_id and
|
||||||
|
recording.start_time_90k > :start_time_90k - )",
|
||||||
|
kMaxRecordingDuration, " and\n",
|
||||||
|
R"(
|
||||||
recording.start_time_90k < :end_time_90k and
|
recording.start_time_90k < :end_time_90k and
|
||||||
recording.end_time_90k > :start_time_90k
|
recording.end_time_90k > :start_time_90k
|
||||||
order by
|
order by
|
||||||
recording.start_time_90k;)",
|
recording.start_time_90k;)");
|
||||||
nullptr, error_message);
|
build_mp4_query_ = db_->Prepare(build_mp4_sql, nullptr, error_message);
|
||||||
if (!build_mp4_query_.valid()) {
|
if (!build_mp4_query_.valid()) {
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
@ -230,6 +233,56 @@ bool MoonfireDatabase::ListCameraRecordings(
|
|||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
bool MoonfireDatabase::ListMp4Recordings(
|
||||||
|
int64_t camera_id, int64_t start_time_90k, int64_t end_time_90k,
|
||||||
|
std::function<IterationControl(Recording &, const VideoSampleEntry &)>
|
||||||
|
row_cb,
|
||||||
|
std::string *error_message) {
|
||||||
|
VLOG(1) << "...(1/4): Waiting for database lock";
|
||||||
|
DatabaseContext ctx(db_);
|
||||||
|
VLOG(1) << "...(2/4): Querying database";
|
||||||
|
auto run = ctx.Borrow(&build_mp4_query_);
|
||||||
|
run.BindInt64(":camera_id", camera_id);
|
||||||
|
run.BindInt64(":end_time_90k", end_time_90k);
|
||||||
|
run.BindInt64(":start_time_90k", start_time_90k);
|
||||||
|
Recording recording;
|
||||||
|
VideoSampleEntry sample_entry;
|
||||||
|
while (run.Step() == SQLITE_ROW) {
|
||||||
|
recording.rowid = run.ColumnInt64(0);
|
||||||
|
recording.start_time_90k = run.ColumnInt64(1);
|
||||||
|
recording.end_time_90k = run.ColumnInt64(2);
|
||||||
|
recording.sample_file_bytes = run.ColumnInt64(3);
|
||||||
|
if (!recording.sample_file_uuid.ParseBinary(run.ColumnBlob(4))) {
|
||||||
|
*error_message =
|
||||||
|
StrCat("recording ", recording.rowid, " has unparseable uuid ",
|
||||||
|
ToHex(run.ColumnBlob(4)));
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
recording.sample_file_path =
|
||||||
|
StrCat("/home/slamb/new-moonfire/sample/",
|
||||||
|
recording.sample_file_uuid.UnparseText());
|
||||||
|
recording.sample_file_sha1 = run.ColumnBlob(5).as_string();
|
||||||
|
recording.video_sample_entry_sha1 = run.ColumnBlob(6).as_string();
|
||||||
|
recording.video_index = run.ColumnBlob(7).as_string();
|
||||||
|
recording.video_samples = run.ColumnInt64(8);
|
||||||
|
recording.video_sync_samples = run.ColumnInt64(9);
|
||||||
|
|
||||||
|
if (recording.video_sample_entry_sha1 != sample_entry.sha1) {
|
||||||
|
sample_entry.sha1 = run.ColumnBlob(6).as_string();
|
||||||
|
sample_entry.data = run.ColumnBlob(10).as_string();
|
||||||
|
sample_entry.width = run.ColumnInt64(11);
|
||||||
|
sample_entry.height = run.ColumnInt64(12);
|
||||||
|
}
|
||||||
|
|
||||||
|
row_cb(recording, sample_entry);
|
||||||
|
}
|
||||||
|
if (run.status() != SQLITE_DONE && run.status() != SQLITE_ROW) {
|
||||||
|
*error_message = StrCat("sqlite query failed: ", run.error_message());
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
std::shared_ptr<VirtualFile> MoonfireDatabase::BuildMp4(
|
std::shared_ptr<VirtualFile> MoonfireDatabase::BuildMp4(
|
||||||
int64_t camera_id, int64_t start_time_90k, int64_t end_time_90k,
|
int64_t camera_id, int64_t start_time_90k, int64_t end_time_90k,
|
||||||
std::string *error_message) {
|
std::string *error_message) {
|
||||||
@ -239,81 +292,51 @@ std::shared_ptr<VirtualFile> MoonfireDatabase::BuildMp4(
|
|||||||
|
|
||||||
Mp4FileBuilder builder;
|
Mp4FileBuilder builder;
|
||||||
int64_t next_row_start_time_90k = start_time_90k;
|
int64_t next_row_start_time_90k = start_time_90k;
|
||||||
VideoSampleEntry sample_entry;
|
|
||||||
int64_t rows = 0;
|
int64_t rows = 0;
|
||||||
{
|
bool ok = true;
|
||||||
VLOG(1) << "...(1/4): Waiting for database lock";
|
auto row_cb = [&](Recording &recording,
|
||||||
DatabaseContext ctx(db_);
|
const VideoSampleEntry &sample_entry) {
|
||||||
VLOG(1) << "...(2/4): Querying database";
|
if (rows == 0 && recording.start_time_90k != next_row_start_time_90k) {
|
||||||
auto run = ctx.Borrow(&build_mp4_query_);
|
*error_message = StrCat(
|
||||||
run.BindInt64(":camera_id", camera_id);
|
"recording starts late: ", PrettyTimestamp(recording.start_time_90k),
|
||||||
run.BindInt64(":end_time_90k", end_time_90k);
|
" (", recording.start_time_90k, ") rather than requested: ",
|
||||||
run.BindInt64(":start_time_90k", start_time_90k);
|
PrettyTimestamp(start_time_90k), " (", start_time_90k, ")");
|
||||||
Recording recording;
|
ok = false;
|
||||||
while (run.Step() == SQLITE_ROW) {
|
return IterationControl::kBreak;
|
||||||
recording.rowid = run.ColumnInt64(0);
|
} else if (recording.start_time_90k != next_row_start_time_90k) {
|
||||||
VLOG(2) << "row: " << recording.rowid;
|
*error_message = StrCat("gap/overlap in recording: ",
|
||||||
recording.start_time_90k = run.ColumnInt64(1);
|
PrettyTimestamp(next_row_start_time_90k), " (",
|
||||||
recording.end_time_90k = run.ColumnInt64(2);
|
next_row_start_time_90k, ") to: ",
|
||||||
recording.sample_file_bytes = run.ColumnInt64(3);
|
PrettyTimestamp(recording.start_time_90k), " (",
|
||||||
if (!recording.sample_file_uuid.ParseBinary(run.ColumnBlob(4))) {
|
recording.start_time_90k, ") before row ", rows);
|
||||||
*error_message =
|
ok = false;
|
||||||
StrCat("recording ", recording.rowid, " has unparseable uuid ",
|
return IterationControl::kBreak;
|
||||||
ToHex(run.ColumnBlob(4)));
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
recording.sample_file_path =
|
|
||||||
StrCat("/home/slamb/new-moonfire/sample/",
|
|
||||||
recording.sample_file_uuid.UnparseText());
|
|
||||||
recording.sample_file_sha1 = run.ColumnBlob(5).as_string();
|
|
||||||
recording.video_sample_entry_sha1 = run.ColumnBlob(6).as_string();
|
|
||||||
recording.video_index = run.ColumnBlob(7).as_string();
|
|
||||||
recording.video_samples = run.ColumnInt64(8);
|
|
||||||
recording.video_sync_samples = run.ColumnInt64(9);
|
|
||||||
|
|
||||||
if (rows == 0 && recording.start_time_90k != next_row_start_time_90k) {
|
|
||||||
*error_message =
|
|
||||||
StrCat("recording starts late: ",
|
|
||||||
PrettyTimestamp(recording.start_time_90k), " (",
|
|
||||||
recording.start_time_90k, ") rather than requested: ",
|
|
||||||
PrettyTimestamp(start_time_90k), " (", start_time_90k, ")");
|
|
||||||
return false;
|
|
||||||
} else if (recording.start_time_90k != next_row_start_time_90k) {
|
|
||||||
*error_message =
|
|
||||||
StrCat("gap/overlap in recording: ",
|
|
||||||
PrettyTimestamp(next_row_start_time_90k), " (",
|
|
||||||
next_row_start_time_90k, ") to: ",
|
|
||||||
PrettyTimestamp(recording.start_time_90k), " (",
|
|
||||||
recording.start_time_90k, ") before row ", rows);
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
|
|
||||||
next_row_start_time_90k = recording.end_time_90k;
|
|
||||||
|
|
||||||
if (rows > 0 && recording.video_sample_entry_sha1 != sample_entry.sha1) {
|
|
||||||
*error_message =
|
|
||||||
StrCat("inconsistent video sample entries: this recording has ",
|
|
||||||
ToHex(recording.video_sample_entry_sha1), ", previous had ",
|
|
||||||
ToHex(sample_entry.sha1));
|
|
||||||
return false;
|
|
||||||
} else if (rows == 0) {
|
|
||||||
sample_entry.sha1 = run.ColumnBlob(6).as_string();
|
|
||||||
sample_entry.data = run.ColumnBlob(10).as_string();
|
|
||||||
sample_entry.width = run.ColumnInt64(11);
|
|
||||||
sample_entry.height = run.ColumnInt64(12);
|
|
||||||
builder.SetSampleEntry(sample_entry);
|
|
||||||
}
|
|
||||||
|
|
||||||
// TODO: correct bounds within recording.
|
|
||||||
// Currently this can return too much data.
|
|
||||||
builder.Append(std::move(recording), 0,
|
|
||||||
std::numeric_limits<int32_t>::max());
|
|
||||||
++rows;
|
|
||||||
}
|
}
|
||||||
if (run.status() != SQLITE_DONE) {
|
|
||||||
*error_message = StrCat("sqlite query failed: ", run.error_message());
|
next_row_start_time_90k = recording.end_time_90k;
|
||||||
return false;
|
|
||||||
|
if (rows > 0 && recording.video_sample_entry_sha1 != sample_entry.sha1) {
|
||||||
|
*error_message =
|
||||||
|
StrCat("inconsistent video sample entries: this recording has ",
|
||||||
|
ToHex(recording.video_sample_entry_sha1), ", previous had ",
|
||||||
|
ToHex(sample_entry.sha1));
|
||||||
|
ok = false;
|
||||||
|
return IterationControl::kBreak;
|
||||||
|
} else if (rows == 0) {
|
||||||
|
builder.SetSampleEntry(sample_entry);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// TODO: correct bounds within recording.
|
||||||
|
// Currently this can return too much data.
|
||||||
|
builder.Append(std::move(recording), 0,
|
||||||
|
std::numeric_limits<int32_t>::max());
|
||||||
|
++rows;
|
||||||
|
return IterationControl::kContinue;
|
||||||
|
};
|
||||||
|
if (!ok ||
|
||||||
|
!ListMp4Recordings(camera_id, start_time_90k, end_time_90k, row_cb,
|
||||||
|
error_message)) {
|
||||||
|
return false;
|
||||||
}
|
}
|
||||||
if (rows == 0) {
|
if (rows == 0) {
|
||||||
*error_message = StrCat("no recordings in range");
|
*error_message = StrCat("no recordings in range");
|
||||||
|
@ -123,6 +123,12 @@ class MoonfireDatabase {
|
|||||||
std::function<IterationControl(const ListCameraRecordingsRow &)>,
|
std::function<IterationControl(const ListCameraRecordingsRow &)>,
|
||||||
std::string *error_message);
|
std::string *error_message);
|
||||||
|
|
||||||
|
bool ListMp4Recordings(
|
||||||
|
int64_t camera_id, int64_t start_time_90k, int64_t end_time_90k,
|
||||||
|
std::function<IterationControl(Recording &, const VideoSampleEntry &)>
|
||||||
|
row_cb,
|
||||||
|
std::string *error_message);
|
||||||
|
|
||||||
std::shared_ptr<VirtualFile> BuildMp4(int64_t camera_id,
|
std::shared_ptr<VirtualFile> BuildMp4(int64_t camera_id,
|
||||||
int64_t start_time_90k,
|
int64_t start_time_90k,
|
||||||
int64_t end_time_90k,
|
int64_t end_time_90k,
|
||||||
|
@ -50,6 +50,14 @@ namespace moonfire_nvr {
|
|||||||
|
|
||||||
constexpr int64_t kTimeUnitsPerSecond = 90000;
|
constexpr int64_t kTimeUnitsPerSecond = 90000;
|
||||||
|
|
||||||
|
// Recordings are never longer than this (5 minutes).
|
||||||
|
// Having such a limit dramatically speeds up some SQL queries.
|
||||||
|
// This limit should be more than the normal rotation time,
|
||||||
|
// as recording doesn't happen until the next key frame.
|
||||||
|
// 5 minutes is generously more than 1 minute, but still sufficient to
|
||||||
|
// allow the optimization to be useful.
|
||||||
|
constexpr int64_t kMaxRecordingDuration = 5 * 60 * kTimeUnitsPerSecond;
|
||||||
|
|
||||||
// Various fields from the "recording" table which are useful when viewing
|
// Various fields from the "recording" table which are useful when viewing
|
||||||
// recordings.
|
// recordings.
|
||||||
struct Recording {
|
struct Recording {
|
||||||
|
@ -88,6 +88,8 @@ create table recording (
|
|||||||
video_index blob
|
video_index blob
|
||||||
);
|
);
|
||||||
|
|
||||||
|
create index recording_start_time_90k on recording (start_time_90k);
|
||||||
|
|
||||||
-- A concrete box derived from a ISO/IEC 14496-12 section 8.5.2
|
-- A concrete box derived from a ISO/IEC 14496-12 section 8.5.2
|
||||||
-- VisualSampleEntry box. Describes the codec, width, height, etc.
|
-- VisualSampleEntry box. Describes the codec, width, height, etc.
|
||||||
create table video_sample_entry (
|
create table video_sample_entry (
|
||||||
|
Loading…
x
Reference in New Issue
Block a user