MyFavMirrors
/
moonfire-nvr
mirror of https://github.com/scottlamb/moonfire-nvr.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Add sample index codec; fix schema doc.

This commit is contained in:
Scott Lamb 2016-01-05 11:01:36 -08:00
parent 23ba5e0049
commit 60988f0646
5 changed files with 395 additions and 11 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

22
design/schema.md
View File

@ -70,13 +70,15 @@ The table below shows cost of processing a single stream, as a percentage of the
whole processor ((user+sys) time / video duration / CPU cores). **TODO:** try
different quality settings as well.
Decode:
$ time ffmpeg -y -threads 1 -i input.mp4 \
-f null /dev/null
Decode:
Combo (Decode + encode with libx264):
$ time ffmpeg -y -threads 1 -i input.mp4 \
-c:v libx264 -preset ultrafast -threads 1 -f mp4 /dev/null
$ time ffmpeg -y -threads 1 -i input.mp4 \
-f null /dev/null
Combo (Decode + encode with libx264):
$ time ffmpeg -y -threads 1 -i input.mp4 \
-c:v libx264 -preset ultrafast -threads 1 -f mp4 /dev/null
| Processor | 1080p30 decode | 1080p30 combo | 704x480p10 decode | 704x480p10 combo |
@ -483,11 +485,11 @@ See also the example below:
| bytes | 1000 | 10 | 15 | 12 | 1050 |
| duration\_delta | 10 | -1 | 2 | -1 | 0 |
| bytes\_delta | 1000 | 10 | 5 | -3 | 50 |
| varint1 | 42 | 3 | 8 | 3 | 1 |
| varint2 | 2000 | 20 | 10 | 5 | 2 |
| encoded | `2a d0 0f` | `03 14` | `08 0a` | `03 05` | `01 02` |
| varint1 | 41 | 2 | 8 | 3 | 1 |
| varint2 | 2000 | 20 | 10 | 5 | 100 |
| encoded | `29 d0 0f` | `02 14` | `08 0a` | `02 05` | `01 64` |
### <a href="on-demand"></a> On-demand `.mp4` construction
### <a href="on-demand"></a>On-demand `.mp4` construction
A major goal of this format is to support on-demand serving in various formats,
including two types of `.mp4` files:

3
src/CMakeLists.txt
View File

@ -46,6 +46,7 @@ set(MOONFIRE_NVR_SRCS
http.cc
moonfire-nvr.cc
profiler.cc
recording.cc
string.cc
time.cc)
@ -60,7 +61,7 @@ install_programs(/bin FILES moonfire-nvr)
include_directories(${GTest_INCLUDE_DIR})
include_directories(${GMock_INCLUDE_DIR})
foreach(test coding http moonfire-nvr string)
foreach(test coding http moonfire-nvr recording string)
add_executable(${test}-test ${test}-test.cc testutil.cc)
target_link_libraries(${test}-test GTest GMock moonfire-nvr-lib)
add_test(NAME ${test}-test

131
src/recording-test.cc Normal file
View File

@ -0,0 +1,131 @@
// This file is part of Moonfire NVR, a security camera network video recorder.
// Copyright (C) 2016 Scott Lamb <slamb@slamb.org>
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// In addition, as a special exception, the copyright holders give
// permission to link the code of portions of this program with the
// OpenSSL library under certain conditions as described in each
// individual source file, and distribute linked combinations including
// the two.
//
// You must obey the GNU General Public License in all respects for all
// of the code used other than OpenSSL. If you modify file(s) with this
// exception, you may extend this exception to your version of the
// file(s), but you are not obligated to do so. If you do not wish to do
// so, delete this exception statement from your version. If you delete
// this exception statement from all source files in the program, then
// also delete it here.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
//
// recording-test.cc: tests of the recording.h interface.
#include <gflags/gflags.h>
#include <gmock/gmock.h>
#include <gtest/gtest.h>
#include "recording.h"
#include "string.h"
DECLARE_bool(alsologtostderr);
using testing::HasSubstr;
namespace moonfire_nvr {
namespace {
// Example from design/schema.md.
TEST(SampleIndexTest, EncodeExample) {
SampleIndexEncoder encoder;
encoder.AddSample(10, 1000, true);
encoder.AddSample(9, 10, false);
encoder.AddSample(11, 15, false);
encoder.AddSample(10, 12, false);
encoder.AddSample(10, 1050, true);
ASSERT_EQ("29 d0 0f 02 14 08 0a 02 05 01 64", ToHex(encoder.data()));
}
TEST(SampleIndexTest, RoundTrip) {
SampleIndexEncoder encoder;
encoder.AddSample(10, 30000, true);
encoder.AddSample(9, 1000, false);
encoder.AddSample(11, 1100, false);
encoder.AddSample(18, 31000, true);
SampleIndexIterator it = SampleIndexIterator(encoder.data());
std::string error_message;
ASSERT_FALSE(it.done()) << it.error();
EXPECT_EQ(10, it.duration_90k());
EXPECT_EQ(30000, it.bytes());
EXPECT_TRUE(it.is_key());
it.Next();
ASSERT_FALSE(it.done()) << it.error();
EXPECT_EQ(9, it.duration_90k());
EXPECT_EQ(1000, it.bytes());
EXPECT_FALSE(it.is_key());
it.Next();
ASSERT_FALSE(it.done()) << it.error();
EXPECT_EQ(11, it.duration_90k());
EXPECT_EQ(1100, it.bytes());
EXPECT_FALSE(it.is_key());
it.Next();
ASSERT_FALSE(it.done()) << it.error();
EXPECT_EQ(18, it.duration_90k());
EXPECT_EQ(31000, it.bytes());
EXPECT_TRUE(it.is_key());
it.Next();
ASSERT_TRUE(it.done());
ASSERT_FALSE(it.has_error()) << it.error();
}
TEST(SampleIndexTest, IteratorErrors) {
std::string bad_first_varint("\x80");
SampleIndexIterator it(bad_first_varint);
EXPECT_TRUE(it.has_error());
EXPECT_EQ("buffer underrun", it.error());
std::string bad_second_varint("\x00\x80", 2);
it = SampleIndexIterator(bad_second_varint);
EXPECT_TRUE(it.has_error());
EXPECT_EQ("buffer underrun", it.error());
std::string zero_durations("\x00\x02\x00\x00", 4);
it = SampleIndexIterator(zero_durations);
EXPECT_TRUE(it.has_error());
EXPECT_THAT(it.error(), HasSubstr("zero duration"));
std::string negative_duration("\x02\x02", 2);
it = SampleIndexIterator(negative_duration);
EXPECT_TRUE(it.has_error());
EXPECT_THAT(it.error(), HasSubstr("negative duration"));
std::string non_positive_bytes("\x04\x00", 2);
it = SampleIndexIterator(non_positive_bytes);
EXPECT_TRUE(it.has_error());
EXPECT_THAT(it.error(), HasSubstr("non-positive bytes"));
}
} // namespace
} // namespace moonfire_nvr
int main(int argc, char **argv) {
FLAGS_alsologtostderr = true;
google::ParseCommandLineFlags(&argc, &argv, true);
testing::InitGoogleTest(&argc, argv);
google::InitGoogleLogging(argv[0]);
return RUN_ALL_TESTS();
}

120
src/recording.cc Normal file
View File

@ -0,0 +1,120 @@
// This file is part of Moonfire NVR, a security camera network video recorder.
// Copyright (C) 2015 Scott Lamb <slamb@slamb.org>
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// In addition, as a special exception, the copyright holders give
// permission to link the code of portions of this program with the
// OpenSSL library under certain conditions as described in each
// individual source file, and distribute linked combinations including
// the two.
//
// You must obey the GNU General Public License in all respects for all
// of the code used other than OpenSSL. If you modify file(s) with this
// exception, you may extend this exception to your version of the
// file(s), but you are not obligated to do so. If you do not wish to do
// so, delete this exception statement from your version. If you delete
// this exception statement from all source files in the program, then
// also delete it here.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
//
// recording.cc: see recording.h.
#include "recording.h"
#include "coding.h"
#include "string.h"
namespace moonfire_nvr {
void SampleIndexEncoder::AddSample(int32_t duration_90k, int32_t bytes,
bool is_key) {
CHECK_GE(duration_90k, 0);
CHECK_GT(bytes, 0);
int32_t duration_delta = duration_90k - prev_duration_90k_;
prev_duration_90k_ = duration_90k;
int32_t bytes_delta;
if (is_key) {
bytes_delta = bytes - prev_bytes_key_;
prev_bytes_key_ = bytes;
} else {
bytes_delta = bytes - prev_bytes_nonkey_;
prev_bytes_nonkey_ = bytes;
}
uint32_t zigzagged_bytes_delta = Zigzag32(bytes_delta);
AppendVar32((Zigzag32(duration_delta) << 1) | is_key, &data_);
AppendVar32(zigzagged_bytes_delta, &data_);
}
void SampleIndexEncoder::Clear() {
data_.clear();
prev_duration_90k_ = 0;
prev_bytes_key_ = 0;
prev_bytes_nonkey_ = 0;
}
void SampleIndexIterator::Next() {
uint32_t raw1;
uint32_t raw2;
pos_ += bytes_internal();
if (data_.empty() || !DecodeVar32(&data_, &raw1, &error_) ||
!DecodeVar32(&data_, &raw2, &error_)) {
done_ = true;
return;
}
start_90k_ += duration_90k_;
int32_t duration_90k_delta = Unzigzag32(raw1 >> 1);
duration_90k_ += duration_90k_delta;
if (duration_90k_ < 0) {
error_ = StrCat("negative duration ", duration_90k_,
" after applying delta ", duration_90k_delta);
done_ = true;
return;
}
if (duration_90k_ == 0 && !data_.empty()) {
error_ = StrCat("zero duration only allowed at end; have ", data_.size(),
"bytes left.");
done_ = true;
return;
}
is_key_ = raw1 & 0x01;
int32_t bytes_delta = Unzigzag32(raw2);
if (is_key_) {
bytes_key_ += bytes_delta;
} else {
bytes_nonkey_ += bytes_delta;
}
if (bytes_internal() <= 0) {
error_ = StrCat("non-positive bytes ", bytes_internal(),
" after applying delta ", bytes_delta, " to ",
(is_key_ ? "key" : "non-key"), " frame at ts ", start_90k_);
done_ = true;
return;
}
done_ = false;
return;
}
void SampleIndexIterator::Clear() {
data_.clear();
error_.clear();
pos_ = 0;
start_90k_ = 0;
duration_90k_ = 0;
bytes_key_ = 0;
bytes_nonkey_ = 0;
is_key_ = false;
done_ = true;
}
} // namespace moonfire_nvr

130
src/recording.h Normal file
View File

@ -0,0 +1,130 @@
// This file is part of Moonfire NVR, a security camera network video recorder.
// Copyright (C) 2016 Scott Lamb <slamb@slamb.org>
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// In addition, as a special exception, the copyright holders give
// permission to link the code of portions of this program with the
// OpenSSL library under certain conditions as described in each
// individual source file, and distribute linked combinations including
// the two.
//
// You must obey the GNU General Public License in all respects for all
// of the code used other than OpenSSL. If you modify file(s) with this
// exception, you may extend this exception to your version of the
// file(s), but you are not obligated to do so. If you do not wish to do
// so, delete this exception statement from your version. If you delete
// this exception statement from all source files in the program, then
// also delete it here.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
//
// recording.h: Write and read recordings. See design/schema.md for a
// description of the storage schema.
#ifndef MOONFIRE_NVR_RECORDING_H
#define MOONFIRE_NVR_RECORDING_H
#include <stdint.h>
#include <string>
#include <glog/logging.h>
#include <re2/stringpiece.h>
namespace moonfire_nvr {
// Encodes a sample index.
class SampleIndexEncoder {
public:
SampleIndexEncoder() { Clear(); }
void AddSample(int32_t duration_90k, int32_t bytes, bool is_key);
void Clear();
// Return the current data, which is invalidated by the next call to
// AddSample() or Clear().
re2::StringPiece data() { return data_; }
private:
std::string data_;
int32_t prev_duration_90k_;
int32_t prev_bytes_key_;
int32_t prev_bytes_nonkey_;
};
// Iterates through an encoded index, decoding on the fly. Copyable.
// Example usage:
//
// SampleIndexIterator it;
// for (it = index; !it.done(); it.Next()) {
// LOG(INFO) << "sample size: " << it.bytes();
// }
// if (it.has_error()) {
// LOG(ERROR) << "error: " << it.error();
// }
class SampleIndexIterator {
public:
SampleIndexIterator() { Clear(); }
// |index| must outlive the iterator.
explicit SampleIndexIterator(re2::StringPiece index) {
Clear();
data_ = index;
Next();
}
// Iteration control.
void Next();
bool done() const { return done_; }
bool has_error() const { return !error_.empty(); }
const std::string &error() const { return error_; }
// Return properties of the current sample.
// Note pos() and start_90k() are valid when done(); the others are not.
int64_t pos() const { return pos_; }
int32_t start_90k() const { return start_90k_; }
int32_t duration_90k() const {
DCHECK(!done_);
return duration_90k_;
}
int32_t end_90k() const { return start_90k_ + duration_90k(); }
int32_t bytes() const {
DCHECK(!done_);
return bytes_internal();
}
bool is_key() const {
DCHECK(!done_);
return is_key_;
}
private:
void Clear();
// Return the bytes taken by the current sample, or 0 after Clear().
int64_t bytes_internal() const {
return is_key_ ? bytes_key_ : bytes_nonkey_;
}
re2::StringPiece data_;
std::string error_;
int64_t pos_;
int32_t start_90k_;
int32_t duration_90k_;
int32_t bytes_key_;
int32_t bytes_nonkey_;
bool is_key_;
bool done_;
};
} // namespace moonfire_nvr
#endif // MOONFIRE_NVR_RECORDING_H