MyFavMirrors/moonfire-nvr
1
0
Fork 0
mirror of https://github.com/scottlamb/moonfire-nvr.git synced 2025-01-13 16:03:22 -05:00
Code Issues Packages Projects Releases Wiki Activity

Optimize the SQLite query for building .mp4s.

Browse Source 
On my laptop, with a month's data, a test query would take 0.1 to 0.2 seconds
before. Now it takes 0.001 to 0.004 seconds.

I improved this by creating and taking advantage of an index on start time.
It's a little more complicated than that because the desired timespan is
specified in terms of a recording's start and end time, not start time alone.
I defined a maximum duration of a recording (5 minutes) and specified this
with an extra condition in the query so that the end time can be used to
narrow the valid range of start times.

"explain query plan select ..." output confirms it's using the index with
both > and < comparisons:

0|0|0|SEARCH TABLE recording USING INDEX recording_start_time_90k (start_time_90k>? AND start_time_90k<?)
0|1|1|SEARCH TABLE video_sample_entry USING INDEX sqlite_autoindex_video_sample_entry_1 (sha1=?)

I also refactored ListMp4Recordings out of BuildMp4File to make the measurement
easier.
This commit is contained in:
Scott Lamb 2016-01-17 01:14:29 -08:00
parent 40cd983355
commit b9d6526492
4 changed files with 114 additions and 75 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

173
src/moonfire-db.cc
View File

@ -112,7 +112,7 @@ bool MoonfireDatabase::Init(std::string *error_message) {
return false;
}
build_mp4_query_ = db_->Prepare(
std::string build_mp4_sql = StrCat(
R"(
select
recording.rowid,
@ -134,11 +134,14 @@ bool MoonfireDatabase::Init(std::string *error_message) {
where
recording.status = 1 and
camera_id = :camera_id and
recording.start_time_90k > :start_time_90k - )",
kMaxRecordingDuration, " and\n",
R"(
recording.start_time_90k < :end_time_90k and
recording.end_time_90k > :start_time_90k
order by
recording.start_time_90k;)",
nullptr, error_message);
recording.start_time_90k;)");
build_mp4_query_ = db_->Prepare(build_mp4_sql, nullptr, error_message);
if (!build_mp4_query_.valid()) {
return false;
}
@ -230,6 +233,56 @@ bool MoonfireDatabase::ListCameraRecordings(
return true;
}
bool MoonfireDatabase::ListMp4Recordings(
int64_t camera_id, int64_t start_time_90k, int64_t end_time_90k,
std::function<IterationControl(Recording &, const VideoSampleEntry &)>
row_cb,
std::string *error_message) {
VLOG(1) << "...(1/4): Waiting for database lock";
DatabaseContext ctx(db_);
VLOG(1) << "...(2/4): Querying database";
auto run = ctx.Borrow(&build_mp4_query_);
run.BindInt64(":camera_id", camera_id);
run.BindInt64(":end_time_90k", end_time_90k);
run.BindInt64(":start_time_90k", start_time_90k);
Recording recording;
VideoSampleEntry sample_entry;
while (run.Step() == SQLITE_ROW) {
recording.rowid = run.ColumnInt64(0);
recording.start_time_90k = run.ColumnInt64(1);
recording.end_time_90k = run.ColumnInt64(2);
recording.sample_file_bytes = run.ColumnInt64(3);
if (!recording.sample_file_uuid.ParseBinary(run.ColumnBlob(4))) {
*error_message =
StrCat("recording ", recording.rowid, " has unparseable uuid ",
ToHex(run.ColumnBlob(4)));
return false;
}
recording.sample_file_path =
StrCat("/home/slamb/new-moonfire/sample/",
recording.sample_file_uuid.UnparseText());
recording.sample_file_sha1 = run.ColumnBlob(5).as_string();
recording.video_sample_entry_sha1 = run.ColumnBlob(6).as_string();
recording.video_index = run.ColumnBlob(7).as_string();
recording.video_samples = run.ColumnInt64(8);
recording.video_sync_samples = run.ColumnInt64(9);
if (recording.video_sample_entry_sha1 != sample_entry.sha1) {
sample_entry.sha1 = run.ColumnBlob(6).as_string();
sample_entry.data = run.ColumnBlob(10).as_string();
sample_entry.width = run.ColumnInt64(11);
sample_entry.height = run.ColumnInt64(12);
}
row_cb(recording, sample_entry);
}
if (run.status() != SQLITE_DONE && run.status() != SQLITE_ROW) {
*error_message = StrCat("sqlite query failed: ", run.error_message());
return false;
}
return true;
}
std::shared_ptr<VirtualFile> MoonfireDatabase::BuildMp4(
int64_t camera_id, int64_t start_time_90k, int64_t end_time_90k,
std::string *error_message) {
@ -239,81 +292,51 @@ std::shared_ptr<VirtualFile> MoonfireDatabase::BuildMp4(
Mp4FileBuilder builder;
int64_t next_row_start_time_90k = start_time_90k;
VideoSampleEntry sample_entry;
int64_t rows = 0;
{
VLOG(1) << "...(1/4): Waiting for database lock";
DatabaseContext ctx(db_);
VLOG(1) << "...(2/4): Querying database";
auto run = ctx.Borrow(&build_mp4_query_);
run.BindInt64(":camera_id", camera_id);
run.BindInt64(":end_time_90k", end_time_90k);
run.BindInt64(":start_time_90k", start_time_90k);
Recording recording;
while (run.Step() == SQLITE_ROW) {
recording.rowid = run.ColumnInt64(0);
VLOG(2) << "row: " << recording.rowid;
recording.start_time_90k = run.ColumnInt64(1);
recording.end_time_90k = run.ColumnInt64(2);
recording.sample_file_bytes = run.ColumnInt64(3);
if (!recording.sample_file_uuid.ParseBinary(run.ColumnBlob(4))) {
*error_message =
StrCat("recording ", recording.rowid, " has unparseable uuid ",
ToHex(run.ColumnBlob(4)));
return false;
}
recording.sample_file_path =
StrCat("/home/slamb/new-moonfire/sample/",
recording.sample_file_uuid.UnparseText());
recording.sample_file_sha1 = run.ColumnBlob(5).as_string();
recording.video_sample_entry_sha1 = run.ColumnBlob(6).as_string();
recording.video_index = run.ColumnBlob(7).as_string();
recording.video_samples = run.ColumnInt64(8);
recording.video_sync_samples = run.ColumnInt64(9);
if (rows == 0 && recording.start_time_90k != next_row_start_time_90k) {
*error_message =
StrCat("recording starts late: ",
PrettyTimestamp(recording.start_time_90k), " (",
recording.start_time_90k, ") rather than requested: ",
PrettyTimestamp(start_time_90k), " (", start_time_90k, ")");
return false;
} else if (recording.start_time_90k != next_row_start_time_90k) {
*error_message =
StrCat("gap/overlap in recording: ",
PrettyTimestamp(next_row_start_time_90k), " (",
next_row_start_time_90k, ") to: ",
PrettyTimestamp(recording.start_time_90k), " (",
recording.start_time_90k, ") before row ", rows);
return false;
}
next_row_start_time_90k = recording.end_time_90k;
if (rows > 0 && recording.video_sample_entry_sha1 != sample_entry.sha1) {
*error_message =
StrCat("inconsistent video sample entries: this recording has ",
ToHex(recording.video_sample_entry_sha1), ", previous had ",
ToHex(sample_entry.sha1));
return false;
} else if (rows == 0) {
sample_entry.sha1 = run.ColumnBlob(6).as_string();
sample_entry.data = run.ColumnBlob(10).as_string();
sample_entry.width = run.ColumnInt64(11);
sample_entry.height = run.ColumnInt64(12);
builder.SetSampleEntry(sample_entry);
}
// TODO: correct bounds within recording.
// Currently this can return too much data.
builder.Append(std::move(recording), 0,
std::numeric_limits<int32_t>::max());
++rows;
bool ok = true;
auto row_cb = [&](Recording &recording,
const VideoSampleEntry &sample_entry) {
if (rows == 0 && recording.start_time_90k != next_row_start_time_90k) {
*error_message = StrCat(
"recording starts late: ", PrettyTimestamp(recording.start_time_90k),
" (", recording.start_time_90k, ") rather than requested: ",
PrettyTimestamp(start_time_90k), " (", start_time_90k, ")");
ok = false;
return IterationControl::kBreak;
} else if (recording.start_time_90k != next_row_start_time_90k) {
*error_message = StrCat("gap/overlap in recording: ",
PrettyTimestamp(next_row_start_time_90k), " (",
next_row_start_time_90k, ") to: ",
PrettyTimestamp(recording.start_time_90k), " (",
recording.start_time_90k, ") before row ", rows);
ok = false;
return IterationControl::kBreak;
}
if (run.status() != SQLITE_DONE) {
*error_message = StrCat("sqlite query failed: ", run.error_message());
return false;
next_row_start_time_90k = recording.end_time_90k;
if (rows > 0 && recording.video_sample_entry_sha1 != sample_entry.sha1) {
*error_message =
StrCat("inconsistent video sample entries: this recording has ",
ToHex(recording.video_sample_entry_sha1), ", previous had ",
ToHex(sample_entry.sha1));
ok = false;
return IterationControl::kBreak;
} else if (rows == 0) {
builder.SetSampleEntry(sample_entry);
}
// TODO: correct bounds within recording.
// Currently this can return too much data.
builder.Append(std::move(recording), 0,
std::numeric_limits<int32_t>::max());
++rows;
return IterationControl::kContinue;
};
if (!ok ||
!ListMp4Recordings(camera_id, start_time_90k, end_time_90k, row_cb,
error_message)) {
return false;
}
if (rows == 0) {
*error_message = StrCat("no recordings in range");

6
src/moonfire-db.h
View File

@ -123,6 +123,12 @@ class MoonfireDatabase {
std::function<IterationControl(const ListCameraRecordingsRow &)>,
std::string *error_message);
bool ListMp4Recordings(
int64_t camera_id, int64_t start_time_90k, int64_t end_time_90k,
std::function<IterationControl(Recording &, const VideoSampleEntry &)>
row_cb,
std::string *error_message);
std::shared_ptr<VirtualFile> BuildMp4(int64_t camera_id,
int64_t start_time_90k,
int64_t end_time_90k,

8
src/recording.h
View File

@ -50,6 +50,14 @@ namespace moonfire_nvr {
constexpr int64_t kTimeUnitsPerSecond = 90000;
// Recordings are never longer than this (5 minutes).
// Having such a limit dramatically speeds up some SQL queries.
// This limit should be more than the normal rotation time,
// as recording doesn't happen until the next key frame.
// 5 minutes is generously more than 1 minute, but still sufficient to
// allow the optimization to be useful.
constexpr int64_t kMaxRecordingDuration = 5 * 60 * kTimeUnitsPerSecond;
// Various fields from the "recording" table which are useful when viewing
// recordings.
struct Recording {

2
src/schema.sql
View File

@ -88,6 +88,8 @@ create table recording (
video_index blob
);
create index recording_start_time_90k on recording (start_time_90k);
-- A concrete box derived from a ISO/IEC 14496-12 section 8.5.2
-- VisualSampleEntry box. Describes the codec, width, height, etc.
create table video_sample_entry (