Add util functions for binary encoding/decoding.

These will be used by the new sample index format.

src/CMakeLists.txt

@@ -39,7 +39,17 @@ set(MOONFIRE_DEPS
     ${PROTOBUF_LIBRARIES}
     ${RE2_LIBRARIES})
 
-add_library(moonfire-nvr-lib moonfire-nvr.cc ffmpeg.cc http.cc string.cc profiler.cc filesystem.cc time.cc ${PROTO_SRCS} ${PROTO_HDRS})
+set(MOONFIRE_NVR_SRCS
+    coding.cc
+    ffmpeg.cc
+    filesystem.cc
+    http.cc
+    moonfire-nvr.cc
+    profiler.cc
+    string.cc
+    time.cc)
+
+add_library(moonfire-nvr-lib ${MOONFIRE_NVR_SRCS} ${PROTO_SRCS} ${PROTO_HDRS})
 target_link_libraries(moonfire-nvr-lib ${MOONFIRE_DEPS})
 
 add_executable(moonfire-nvr moonfire-nvr-main.cc)
@@ -50,7 +60,7 @@ install_programs(/bin FILES moonfire-nvr)
 include_directories(${GTest_INCLUDE_DIR})
 include_directories(${GMock_INCLUDE_DIR})
 
-foreach(test http moonfire-nvr string)
+foreach(test coding http moonfire-nvr string)
   add_executable(${test}-test ${test}-test.cc testutil.cc)
   target_link_libraries(${test}-test GTest GMock moonfire-nvr-lib)
   add_test(NAME ${test}-test

src/coding-test.cc

@@ -0,0 +1,106 @@
+// This file is part of Moonfire NVR, a security camera network video recorder.
+// Copyright (C) 2016 Scott Lamb <slamb@slamb.org>
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// In addition, as a special exception, the copyright holders give
+// permission to link the code of portions of this program with the
+// OpenSSL library under certain conditions as described in each
+// individual source file, and distribute linked combinations including
+// the two.
+//
+// You must obey the GNU General Public License in all respects for all
+// of the code used other than OpenSSL. If you modify file(s) with this
+// exception, you may extend this exception to your version of the
+// file(s), but you are not obligated to do so. If you do not wish to do
+// so, delete this exception statement from your version. If you delete
+// this exception statement from all source files in the program, then
+// also delete it here.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+//
+// coding-test.cc: tests of the coding.h interface.
+
+#include <gflags/gflags.h>
+#include <glog/logging.h>
+#include <gmock/gmock.h>
+#include <gtest/gtest.h>
+
+#include "coding.h"
+
+DECLARE_bool(alsologtostderr);
+
+namespace moonfire_nvr {
+namespace {
+
+TEST(VarintTest, Simple) {
+  // Encode.
+  std::string foo;
+  AppendVar32(UINT32_C(1), &foo);
+  EXPECT_EQ("\x01", foo);
+  AppendVar32(UINT32_C(300), &foo);
+  EXPECT_EQ("\x01\xac\x02", foo);
+
+  // Decode.
+  re2::StringPiece p(foo);
+  uint32_t out;
+  std::string error_message;
+  EXPECT_TRUE(DecodeVar32(&p, &out, &error_message));
+  EXPECT_EQ(UINT32_C(1), out);
+  EXPECT_TRUE(DecodeVar32(&p, &out, &error_message));
+  EXPECT_EQ(UINT32_C(300), out);
+}
+
+TEST(VarintTest, DecodeErrors) {
+  re2::StringPiece empty;
+  uint32_t out;
+  std::string error_message;
+  EXPECT_FALSE(DecodeVar32(&empty, &out, &error_message));
+  EXPECT_EQ("buffer underrun", error_message);
+
+  re2::StringPiece partial("\x80", 1);
+  EXPECT_FALSE(DecodeVar32(&partial, &out, &error_message));
+  EXPECT_EQ("buffer underrun", error_message);
+
+  re2::StringPiece too_big("\x80\x80\x80\x80\x10", 5);
+  EXPECT_FALSE(DecodeVar32(&too_big, &out, &error_message));
+  EXPECT_EQ("integer overflow", error_message);
+}
+
+TEST(ZigzagTest, Encode) {
+  EXPECT_EQ(UINT32_C(0), Zigzag32(INT32_C(0)));
+  EXPECT_EQ(UINT32_C(1), Zigzag32(INT32_C(-1)));
+  EXPECT_EQ(UINT32_C(2), Zigzag32(INT32_C(1)));
+  EXPECT_EQ(UINT32_C(3), Zigzag32(INT32_C(-2)));
+  EXPECT_EQ(UINT32_C(4294967294), Zigzag32(INT32_C(2147483647)));
+  EXPECT_EQ(UINT32_C(4294967295), Zigzag32(INT32_C(-2147483648)));
+}
+
+TEST(ZigzagTest, Decode) {
+  EXPECT_EQ(INT32_C(0), Unzigzag32(UINT32_C(0)));
+  EXPECT_EQ(INT32_C(-1), Unzigzag32(UINT32_C(1)));
+  EXPECT_EQ(INT32_C(1), Unzigzag32(UINT32_C(2)));
+  EXPECT_EQ(INT32_C(-2), Unzigzag32(UINT32_C(3)));
+  EXPECT_EQ(INT32_C(2147483647), Unzigzag32(UINT32_C(4294967294)));
+  EXPECT_EQ(INT32_C(-2147483648), Unzigzag32(UINT32_C(4294967295)));
+}
+
+}  // namespace
+}  // namespace moonfire_nvr
+
+int main(int argc, char **argv) {
+  FLAGS_alsologtostderr = true;
+  google::ParseCommandLineFlags(&argc, &argv, true);
+  testing::InitGoogleTest(&argc, argv);
+  google::InitGoogleLogging(argv[0]);
+  return RUN_ALL_TESTS();
+}

src/coding.cc

@@ -0,0 +1,76 @@
+// This file is part of Moonfire NVR, a security camera network video recorder.
+// Copyright (C) 2016 Scott Lamb <slamb@slamb.org>
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// In addition, as a special exception, the copyright holders give
+// permission to link the code of portions of this program with the
+// OpenSSL library under certain conditions as described in each
+// individual source file, and distribute linked combinations including
+// the two.
+//
+// You must obey the GNU General Public License in all respects for all
+// of the code used other than OpenSSL. If you modify file(s) with this
+// exception, you may extend this exception to your version of the
+// file(s), but you are not obligated to do so. If you do not wish to do
+// so, delete this exception statement from your version. If you delete
+// this exception statement from all source files in the program, then
+// also delete it here.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+//
+// coding.cc: see coding.h.
+
+#include "coding.h"
+
+namespace moonfire_nvr {
+
+namespace internal {
+
+void AppendVar32Slow(uint32_t in, std::string *out) {
+  while (true) {
+    uint8_t next_byte = in & 0x7F;
+    in >>= 7;
+    if (in == 0) {
+      out->push_back(next_byte);
+      return;
+    }
+    out->push_back(next_byte | 0x80);
+  }
+}
+
+bool DecodeVar32Slow(re2::StringPiece *in, uint32_t *out_p,
+                     std::string *error_message) {
+  auto p = reinterpret_cast<uint8_t const *>(in->data());
+  auto end = p + in->size();
+  uint32_t out = 0;
+  int shift = 0;
+  do {
+    if (p == end) {
+      *error_message = "buffer underrun";
+      return false;
+    }
+    if (shift == 28 && *p & 0xf0) {
+      *error_message = "integer overflow";
+      return false;
+    }
+    out |= uint32_t(*p & 0x7f) << shift;
+    shift += 7;
+  } while ((*p++ & 0x80) != 0);
+  *out_p = out;
+  in->remove_prefix(reinterpret_cast<char const *>(p) - in->data());
+  return true;
+}
+
+}  // namespace internal
+
+}  // namespace moonfire_nvr

src/coding.h

@@ -0,0 +1,139 @@
+// This file is part of Moonfire NVR, a security camera network video recorder.
+// Copyright (C) 2016 Lamb <slamb@slamb.org>
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// In addition, as a special exception, the copyright holders give
+// permission to link the code of portions of this program with the
+// OpenSSL library under certain conditions as described in each
+// individual source file, and distribute linked combinations including
+// the two.
+//
+// You must obey the GNU General Public License in all respects for all
+// of the code used other than OpenSSL. If you modify file(s) with this
+// exception, you may extend this exception to your version of the
+// file(s), but you are not obligated to do so. If you do not wish to do
+// so, delete this exception statement from your version. If you delete
+// this exception statement from all source files in the program, then
+// also delete it here.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+//
+// coding.h: Binary encoding/decoding.
+
+#ifndef MOONFIRE_NVR_CODING_H
+#define MOONFIRE_NVR_CODING_H
+
+#include <endian.h>
+#include <stdint.h>
+
+#include <string>
+
+#include <re2/stringpiece.h>
+
+namespace moonfire_nvr {
+
+namespace internal {
+
+void AppendVar32Slow(uint32_t in, std::string *out);
+bool DecodeVar32Slow(re2::StringPiece *in, uint32_t *out,
+                     std::string *error_message);
+
+}  // namespace internal
+
+// Endianness conversion.
+
+#if __BYTE_ORDER == __LITTLE_ENDIAN
+// XXX: __builtin_bswap64 doesn't compile on gcc 5.2.1 with an error about a
+// narrowing conversion?!? Doing this by hand...
+constexpr uint64_t ToNetworkU64(uint64_t in) {
+  return ((in & UINT64_C(0xFF00000000000000)) >> 56) |
+         ((in & UINT64_C(0x00FF000000000000)) >> 40) |
+         ((in & UINT64_C(0x0000FF0000000000)) >> 24) |
+         ((in & UINT64_C(0x000000FF00000000)) >> 8) |
+         ((in & UINT64_C(0x00000000FF000000)) << 8) |
+         ((in & UINT64_C(0x0000000000FF0000)) << 24) |
+         ((in & UINT64_C(0x000000000000FF00)) << 40) |
+         ((in & UINT64_C(0x00000000000000FF)) << 56);
+}
+constexpr int64_t ToNetwork64(int64_t in) {
+  return static_cast<int64_t>(ToNetworkU64(static_cast<uint64_t>(in)));
+}
+constexpr uint32_t ToNetworkU32(uint32_t in) {
+  return ((in & UINT32_C(0xFF000000)) >> 24) |
+         ((in & UINT32_C(0x00FF0000)) >> 8) |
+         ((in & UINT32_C(0x0000FF00)) << 8) |
+         ((in & UINT32_C(0x000000FF)) << 24);
+}
+constexpr int32_t ToNetwork32(int32_t in) {
+  return static_cast<int32_t>(ToNetworkU32(static_cast<uint32_t>(in)));
+}
+constexpr uint16_t ToNetworkU16(uint16_t in) {
+  return ((in & UINT32_C(0xFF00)) >> 8) | ((in & UINT32_C(0x00FF)) << 8);
+}
+constexpr int16_t ToNetwork16(int16_t in) {
+  return static_cast<int16_t>(ToNetworkU16(static_cast<uint16_t>(in)));
+}
+#elif __BYTE_ORDER == __BIG_ENDIAN
+constexpr uint64_t ToNetworkU64(uint64_t in) { return in; }
+constexpr int64_t ToNetwork64(int64_t in) { return in; }
+constexpr uint32_t ToNetworkU32(uint32_t in) { return in; }
+constexpr int32_t ToNetwork32(int32_t in) { return in; }
+constexpr uint16_t ToNetworkU16(uint16_t in) { return in; }
+constexpr int16_t ToNetwork16(int16_t in) { return in; }
+#else
+#error Unknown byte order.
+#endif
+
+// Varint encoding, as in
+// https://developers.google.com/protocol-buffers/docs/encoding#varints
+
+inline void AppendVar32(uint32_t in, std::string *out) {
+  if (in < UINT32_C(1) << 7) {
+    out->push_back(static_cast<char>(in));
+  } else {
+    internal::AppendVar32Slow(in, out);
+  }
+}
+
+// Decode the first varint from |in|, saving it to |out| and advancing |in|.
+// Returns error if |in| does not hold a complete varint or on integer overflow.
+inline bool DecodeVar32(re2::StringPiece *in, uint32_t *out,
+                        std::string *error_message) {
+  if (in->size() == 0) {
+    *error_message = "buffer underrun";
+    return false;
+  }
+  auto first_byte = static_cast<uint8_t>(*in->data());
+  if (first_byte < 0x80) {
+    in->remove_prefix(1);
+    *out = first_byte;
+    return true;
+  } else {
+    return internal::DecodeVar32Slow(in, out, error_message);
+  }
+}
+
+// Zigzag encoding for signed integers, as in
+// https://developers.google.com/protocol-buffers/docs/encoding#types
+// Use the low bit to indicate signedness (1 = negative, 0 = non-negative).
+inline uint32_t Zigzag32(int32_t in) {
+  return static_cast<uint32_t>(in << 1) ^ (in >> 31);
+}
+
+inline int32_t Unzigzag32(uint32_t in) {
+  return (in >> 1) ^ -static_cast<int32_t>(in & 1);
+}
+
+}  // namespace moonfire_nvr
+
+#endif  // MOONFIRE_NVR_CODING_H

src/string-test.cc

@@ -92,6 +92,11 @@ TEST(EscapeTest, Simple) {
   EXPECT_EQ("&lt;tag&gt; &amp; text", moonfire_nvr::EscapeHtml("<tag> & text"));
 }
 
+TEST(ToHexTest, Simple) {
+  EXPECT_EQ("", ToHex(""));
+  EXPECT_EQ("12 34 de ad be ef", ToHex("\x12\x34\xde\xad\xbe\xef"));
+}
+
 }  // namespace
 }  // namespace moonfire_nvr
 

src/string.cc

@@ -38,6 +38,15 @@
 
 namespace moonfire_nvr {
 
+namespace {
+
+char HexDigit(unsigned int i) {
+  static char kHexadigits[] = "0123456789abcdef";
+  return (i < 16) ? kHexadigits[i] : 'x';
+}
+
+}  // namespace
+
 namespace internal {
 
 StrCatPiece::StrCatPiece(uint64_t p) {
@@ -103,6 +112,18 @@ std::string EscapeHtml(const std::string &input) {
   return output;
 }
 
+std::string ToHex(re2::StringPiece in) {
+  std::string out;
+  out.reserve(in.size() * 3 + 1);
+  for (int i = 0; i < in.size(); ++i) {
+    if (i > 0) out.push_back(' ');
+    uint8_t byte = in[i];
+    out.push_back(HexDigit(byte >> 4));
+    out.push_back(HexDigit(byte & 0x0F));
+  }
+  return out;
+}
+
 bool strto64(const char *str, int base, const char **endptr, int64_t *value) {
   static_assert(sizeof(int64_t) == sizeof(long long int),
                 "unknown memory model");

src/string.h

@@ -118,6 +118,10 @@ bool IsWord(const std::string &str);
 // HTML-escape the given UTF-8-encoded string.
 std::string EscapeHtml(const std::string &input);
 
+// Return a hex string for debugging.
+// For example, ToHex("\xde\xad\xbe\xef") returns "de ad be ef".
+std::string ToHex(re2::StringPiece in);
+
 // Wrapper around ::strtol that returns true iff valid and corrects
 // constness.
 bool strto64(const char *str, int base, const char **endptr, int64_t *value);