Add sample index codec; fix schema doc.

design/schema.md

@@ -71,10 +71,12 @@ whole processor ((user+sys) time / video duration / CPU cores). **TODO:** try
 different quality settings as well.
 
 Decode:
+
     $ time ffmpeg -y -threads 1 -i input.mp4 \
                   -f null /dev/null
 
 Combo (Decode + encode with libx264):
+
     $ time ffmpeg -y -threads 1 -i input.mp4 \
                   -c:v libx264 -preset ultrafast -threads 1 -f mp4 /dev/null
 
@@ -483,9 +485,9 @@ See also the example below:
 | bytes           |       1000 |      10 |      15 |      12 |    1050 |
 | duration\_delta |         10 |      -1 |       2 |      -1 |       0 |
 | bytes\_delta    |       1000 |      10 |       5 |      -3 |      50 |
-| varint1         |         42 |       3 |       8 |       3 |       1 |
+| varint1         |         41 |       2 |       8 |       3 |       1 |
-| varint2         |       2000 |      20 |      10 |       5 |       2 |
+| varint2         |       2000 |      20 |      10 |       5 |     100 |
-| encoded         | `2a d0 0f` | `03 14` | `08 0a` | `03 05` | `01 02` |
+| encoded         | `29 d0 0f` | `02 14` | `08 0a` | `02 05` | `01 64` |
 
 ### <a href="on-demand"></a>On-demand `.mp4` construction
 

src/CMakeLists.txt

@@ -46,6 +46,7 @@ set(MOONFIRE_NVR_SRCS
     http.cc
     moonfire-nvr.cc
     profiler.cc
+    recording.cc
     string.cc
     time.cc)
 
@@ -60,7 +61,7 @@ install_programs(/bin FILES moonfire-nvr)
 include_directories(${GTest_INCLUDE_DIR})
 include_directories(${GMock_INCLUDE_DIR})
 
-foreach(test coding http moonfire-nvr string)
+foreach(test coding http moonfire-nvr recording string)
   add_executable(${test}-test ${test}-test.cc testutil.cc)
   target_link_libraries(${test}-test GTest GMock moonfire-nvr-lib)
   add_test(NAME ${test}-test

src/recording-test.cc

@@ -0,0 +1,131 @@
+// This file is part of Moonfire NVR, a security camera network video recorder.
+// Copyright (C) 2016 Scott Lamb <slamb@slamb.org>
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// In addition, as a special exception, the copyright holders give
+// permission to link the code of portions of this program with the
+// OpenSSL library under certain conditions as described in each
+// individual source file, and distribute linked combinations including
+// the two.
+//
+// You must obey the GNU General Public License in all respects for all
+// of the code used other than OpenSSL. If you modify file(s) with this
+// exception, you may extend this exception to your version of the
+// file(s), but you are not obligated to do so. If you do not wish to do
+// so, delete this exception statement from your version. If you delete
+// this exception statement from all source files in the program, then
+// also delete it here.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+//
+// recording-test.cc: tests of the recording.h interface.
+
+#include <gflags/gflags.h>
+#include <gmock/gmock.h>
+#include <gtest/gtest.h>
+
+#include "recording.h"
+#include "string.h"
+
+DECLARE_bool(alsologtostderr);
+
+using testing::HasSubstr;
+
+namespace moonfire_nvr {
+namespace {
+
+// Example from design/schema.md.
+TEST(SampleIndexTest, EncodeExample) {
+  SampleIndexEncoder encoder;
+  encoder.AddSample(10, 1000, true);
+  encoder.AddSample(9, 10, false);
+  encoder.AddSample(11, 15, false);
+  encoder.AddSample(10, 12, false);
+  encoder.AddSample(10, 1050, true);
+  ASSERT_EQ("29 d0 0f 02 14 08 0a 02 05 01 64", ToHex(encoder.data()));
+}
+
+TEST(SampleIndexTest, RoundTrip) {
+  SampleIndexEncoder encoder;
+  encoder.AddSample(10, 30000, true);
+  encoder.AddSample(9, 1000, false);
+  encoder.AddSample(11, 1100, false);
+  encoder.AddSample(18, 31000, true);
+
+  SampleIndexIterator it = SampleIndexIterator(encoder.data());
+  std::string error_message;
+  ASSERT_FALSE(it.done()) << it.error();
+  EXPECT_EQ(10, it.duration_90k());
+  EXPECT_EQ(30000, it.bytes());
+  EXPECT_TRUE(it.is_key());
+
+  it.Next();
+  ASSERT_FALSE(it.done()) << it.error();
+  EXPECT_EQ(9, it.duration_90k());
+  EXPECT_EQ(1000, it.bytes());
+  EXPECT_FALSE(it.is_key());
+
+  it.Next();
+  ASSERT_FALSE(it.done()) << it.error();
+  EXPECT_EQ(11, it.duration_90k());
+  EXPECT_EQ(1100, it.bytes());
+  EXPECT_FALSE(it.is_key());
+
+  it.Next();
+  ASSERT_FALSE(it.done()) << it.error();
+  EXPECT_EQ(18, it.duration_90k());
+  EXPECT_EQ(31000, it.bytes());
+  EXPECT_TRUE(it.is_key());
+
+  it.Next();
+  ASSERT_TRUE(it.done());
+  ASSERT_FALSE(it.has_error()) << it.error();
+}
+
+TEST(SampleIndexTest, IteratorErrors) {
+  std::string bad_first_varint("\x80");
+  SampleIndexIterator it(bad_first_varint);
+  EXPECT_TRUE(it.has_error());
+  EXPECT_EQ("buffer underrun", it.error());
+
+  std::string bad_second_varint("\x00\x80", 2);
+  it = SampleIndexIterator(bad_second_varint);
+  EXPECT_TRUE(it.has_error());
+  EXPECT_EQ("buffer underrun", it.error());
+
+  std::string zero_durations("\x00\x02\x00\x00", 4);
+  it = SampleIndexIterator(zero_durations);
+  EXPECT_TRUE(it.has_error());
+  EXPECT_THAT(it.error(), HasSubstr("zero duration"));
+
+  std::string negative_duration("\x02\x02", 2);
+  it = SampleIndexIterator(negative_duration);
+  EXPECT_TRUE(it.has_error());
+  EXPECT_THAT(it.error(), HasSubstr("negative duration"));
+
+  std::string non_positive_bytes("\x04\x00", 2);
+  it = SampleIndexIterator(non_positive_bytes);
+  EXPECT_TRUE(it.has_error());
+  EXPECT_THAT(it.error(), HasSubstr("non-positive bytes"));
+}
+
+}  // namespace
+}  // namespace moonfire_nvr
+
+int main(int argc, char **argv) {
+  FLAGS_alsologtostderr = true;
+  google::ParseCommandLineFlags(&argc, &argv, true);
+  testing::InitGoogleTest(&argc, argv);
+  google::InitGoogleLogging(argv[0]);
+  return RUN_ALL_TESTS();
+}

src/recording.cc

@@ -0,0 +1,120 @@
+// This file is part of Moonfire NVR, a security camera network video recorder.
+// Copyright (C) 2015 Scott Lamb <slamb@slamb.org>
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// In addition, as a special exception, the copyright holders give
+// permission to link the code of portions of this program with the
+// OpenSSL library under certain conditions as described in each
+// individual source file, and distribute linked combinations including
+// the two.
+//
+// You must obey the GNU General Public License in all respects for all
+// of the code used other than OpenSSL. If you modify file(s) with this
+// exception, you may extend this exception to your version of the
+// file(s), but you are not obligated to do so. If you do not wish to do
+// so, delete this exception statement from your version. If you delete
+// this exception statement from all source files in the program, then
+// also delete it here.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+//
+// recording.cc: see recording.h.
+
+#include "recording.h"
+
+#include "coding.h"
+#include "string.h"
+
+namespace moonfire_nvr {
+
+void SampleIndexEncoder::AddSample(int32_t duration_90k, int32_t bytes,
+                                   bool is_key) {
+  CHECK_GE(duration_90k, 0);
+  CHECK_GT(bytes, 0);
+  int32_t duration_delta = duration_90k - prev_duration_90k_;
+  prev_duration_90k_ = duration_90k;
+  int32_t bytes_delta;
+  if (is_key) {
+    bytes_delta = bytes - prev_bytes_key_;
+    prev_bytes_key_ = bytes;
+  } else {
+    bytes_delta = bytes - prev_bytes_nonkey_;
+    prev_bytes_nonkey_ = bytes;
+  }
+  uint32_t zigzagged_bytes_delta = Zigzag32(bytes_delta);
+  AppendVar32((Zigzag32(duration_delta) << 1) | is_key, &data_);
+  AppendVar32(zigzagged_bytes_delta, &data_);
+}
+
+void SampleIndexEncoder::Clear() {
+  data_.clear();
+  prev_duration_90k_ = 0;
+  prev_bytes_key_ = 0;
+  prev_bytes_nonkey_ = 0;
+}
+
+void SampleIndexIterator::Next() {
+  uint32_t raw1;
+  uint32_t raw2;
+  pos_ += bytes_internal();
+  if (data_.empty() || !DecodeVar32(&data_, &raw1, &error_) ||
+      !DecodeVar32(&data_, &raw2, &error_)) {
+    done_ = true;
+    return;
+  }
+  start_90k_ += duration_90k_;
+  int32_t duration_90k_delta = Unzigzag32(raw1 >> 1);
+  duration_90k_ += duration_90k_delta;
+  if (duration_90k_ < 0) {
+    error_ = StrCat("negative duration ", duration_90k_,
+                    " after applying delta ", duration_90k_delta);
+    done_ = true;
+    return;
+  }
+  if (duration_90k_ == 0 && !data_.empty()) {
+    error_ = StrCat("zero duration only allowed at end; have ", data_.size(),
+                    "bytes left.");
+    done_ = true;
+    return;
+  }
+  is_key_ = raw1 & 0x01;
+  int32_t bytes_delta = Unzigzag32(raw2);
+  if (is_key_) {
+    bytes_key_ += bytes_delta;
+  } else {
+    bytes_nonkey_ += bytes_delta;
+  }
+  if (bytes_internal() <= 0) {
+    error_ = StrCat("non-positive bytes ", bytes_internal(),
+                    " after applying delta ", bytes_delta, " to ",
+                    (is_key_ ? "key" : "non-key"), " frame at ts ", start_90k_);
+    done_ = true;
+    return;
+  }
+  done_ = false;
+  return;
+}
+
+void SampleIndexIterator::Clear() {
+  data_.clear();
+  error_.clear();
+  pos_ = 0;
+  start_90k_ = 0;
+  duration_90k_ = 0;
+  bytes_key_ = 0;
+  bytes_nonkey_ = 0;
+  is_key_ = false;
+  done_ = true;
+}
+
+}  // namespace moonfire_nvr

src/recording.h

@@ -0,0 +1,130 @@
+// This file is part of Moonfire NVR, a security camera network video recorder.
+// Copyright (C) 2016 Scott Lamb <slamb@slamb.org>
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// In addition, as a special exception, the copyright holders give
+// permission to link the code of portions of this program with the
+// OpenSSL library under certain conditions as described in each
+// individual source file, and distribute linked combinations including
+// the two.
+//
+// You must obey the GNU General Public License in all respects for all
+// of the code used other than OpenSSL. If you modify file(s) with this
+// exception, you may extend this exception to your version of the
+// file(s), but you are not obligated to do so. If you do not wish to do
+// so, delete this exception statement from your version. If you delete
+// this exception statement from all source files in the program, then
+// also delete it here.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+//
+// recording.h: Write and read recordings. See design/schema.md for a
+// description of the storage schema.
+
+#ifndef MOONFIRE_NVR_RECORDING_H
+#define MOONFIRE_NVR_RECORDING_H
+
+#include <stdint.h>
+
+#include <string>
+
+#include <glog/logging.h>
+#include <re2/stringpiece.h>
+
+namespace moonfire_nvr {
+
+// Encodes a sample index.
+class SampleIndexEncoder {
+ public:
+  SampleIndexEncoder() { Clear(); }
+  void AddSample(int32_t duration_90k, int32_t bytes, bool is_key);
+  void Clear();
+
+  // Return the current data, which is invalidated by the next call to
+  // AddSample() or Clear().
+  re2::StringPiece data() { return data_; }
+
+ private:
+  std::string data_;
+  int32_t prev_duration_90k_;
+  int32_t prev_bytes_key_;
+  int32_t prev_bytes_nonkey_;
+};
+
+// Iterates through an encoded index, decoding on the fly. Copyable.
+// Example usage:
+//
+// SampleIndexIterator it;
+// for (it = index; !it.done(); it.Next()) {
+//   LOG(INFO) << "sample size: " << it.bytes();
+// }
+// if (it.has_error()) {
+//   LOG(ERROR) << "error: " << it.error();
+// }
+class SampleIndexIterator {
+ public:
+  SampleIndexIterator() { Clear(); }
+
+  // |index| must outlive the iterator.
+  explicit SampleIndexIterator(re2::StringPiece index) {
+    Clear();
+    data_ = index;
+    Next();
+  }
+
+  // Iteration control.
+  void Next();
+  bool done() const { return done_; }
+  bool has_error() const { return !error_.empty(); }
+  const std::string &error() const { return error_; }
+
+  // Return properties of the current sample.
+  // Note pos() and start_90k() are valid when done(); the others are not.
+  int64_t pos() const { return pos_; }
+  int32_t start_90k() const { return start_90k_; }
+  int32_t duration_90k() const {
+    DCHECK(!done_);
+    return duration_90k_;
+  }
+  int32_t end_90k() const { return start_90k_ + duration_90k(); }
+  int32_t bytes() const {
+    DCHECK(!done_);
+    return bytes_internal();
+  }
+  bool is_key() const {
+    DCHECK(!done_);
+    return is_key_;
+  }
+
+ private:
+  void Clear();
+
+  // Return the bytes taken by the current sample, or 0 after Clear().
+  int64_t bytes_internal() const {
+    return is_key_ ? bytes_key_ : bytes_nonkey_;
+  }
+
+  re2::StringPiece data_;
+  std::string error_;
+  int64_t pos_;
+  int32_t start_90k_;
+  int32_t duration_90k_;
+  int32_t bytes_key_;
+  int32_t bytes_nonkey_;
+  bool is_key_;
+  bool done_;
+};
+
+}  // namespace moonfire_nvr
+
+#endif  // MOONFIRE_NVR_RECORDING_H