MyFavMirrors/moonfire-nvr
1
0
Fork 0
mirror of https://github.com/scottlamb/moonfire-nvr.git synced 2025-01-13 07:53:22 -05:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'rust' of https://github.com/scottlamb/moonfire-nvr into rust

Browse Source
This commit is contained in:
Dolf Starreveld 2016-11-30 21:48:24 -08:00
commit 7d15a54a47
75 changed files with 6629 additions and 11616 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

17
.gitignore vendored
View File

@ -1,13 +1,6 @@
*.swp
build
debug
release
obj-*
cameras.sql
debian/files
debian/moonfire-nvr.debhelper.log
debian/moonfire-nvr.postinst.debhelper
debian/moonfire-nvr.postrm.debhelper
debian/moonfire-nvr.prerm.debhelper
debian/moonfire-nvr.substvars
debian/moonfire-nvr/
Cargo.lock
.project
.settings
*.swp
target

154
CMakeLists.txt
View File

@ -1,154 +0,0 @@
# This file is part of Moonfire NVR, a security camera digital video recorder.
# Copyright (C) 2016 Scott Lamb <slamb@slamb.org>
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# In addition, as a special exception, the copyright holders give
# permission to link the code of portions of this program with the
# OpenSSL library under certain conditions as described in each
# individual source file, and distribute linked combinations including
# the two.
#
# You must obey the GNU General Public License in all respects for all
# of the code used other than OpenSSL. If you modify file(s) with this
# exception, you may extend this exception to your version of the
# file(s), but you are not obligated to do so. If you do not wish to do
# so, delete this exception statement from your version. If you delete
# this exception statement from all source files in the program, then
# also delete it here.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
# CMakeLists.txt: top-level definitions for building Moonfire NVR.
cmake_minimum_required(VERSION 3.0.2)
project(moonfire-nvr)
if(CMAKE_VERSION VERSION_LESS "3.1")
set(CMAKE_CXX_FLAGS "--std=c++11 ${CMAKE_CXX_FLAGS}")
else()
set(CMAKE_CXX_STANDARD 11)
endif()
set(CMAKE_CXX_FLAGS "-Wall -Werror -pedantic-errors -ggdb ${CMAKE_CXX_FLAGS}")
option(LTO "Use link-time optimization" ON)
option(FPROFILE_GENERATE "Compile executable to generate usage data" OFF)
option(FPROFILE_USE "Compile executable using generated usage data" OFF)
if(LTO)
set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} -flto")
set(CMAKE_AR "gcc-ar")
set(CMAKE_RANLIB "gcc-ranlib")
set(CMAKE_LD "gcc-ld")
endif()
if(PROFILE_GENERATE)
set(CMAKE_CXX_FLAGS "-fprofile-generate ${CMAKE_CXX_FLAGS}")
endif()
if(PROFILE_USE)
set(CMAKE_CXX_FLAGS "-fprofile-use -fprofile-correction -Wno-error=coverage-mismatch ${CMAKE_CXX_FLAGS}")
endif()
#
# Dependencies.
#
find_package(Threads REQUIRED)
# https://gflags.github.io/gflags/#cmake mentions a cmake module, but at
# least on Ubuntu 15.10, libgflags-dev does not include it. There's no
# pkgconfig either. Do this by hand.
find_library(GFLAGS_LIBRARIES gflags)
find_library(RE2_LIBRARIES re2)
find_library(PROFILER_LIBRARIES profiler)
# https://cmake.org/cmake/help/v3.0/module/FindPkgConfig.html
find_package(PkgConfig)
pkg_check_modules(FFMPEG REQUIRED libavutil libavcodec libavformat)
pkg_check_modules(LIBEVENT REQUIRED libevent>=2.1)
pkg_check_modules(JSONCPP REQUIRED jsoncpp)
pkg_check_modules(GLOG REQUIRED libglog)
pkg_check_modules(OPENSSL REQUIRED libcrypto)
pkg_check_modules(SQLITE REQUIRED sqlite3)
pkg_check_modules(UUID REQUIRED uuid)
# Check if ffmpeg support "stimeout".
set(CMAKE_REQUIRED_INCLUDES ${FFMPEG_INCLUDES})
set(CMAKE_REQUIRED_LIBRARIES ${FFMPEG_LIBRARIES})
include(CheckCSourceRuns)
check_c_source_runs([[
#include <stdlib.h>
#include <libavutil/opt.h>
#include <libavformat/avformat.h>
int main(int argc, char **argv) {
av_register_all();
AVInputFormat *input = av_find_input_format("rtsp");
const AVClass *klass = input->priv_class;
const AVOption *opt =
av_opt_find2(&klass, "stimeout", NULL, 0, AV_OPT_SEARCH_FAKE_OBJ, NULL);
return (opt != NULL) ? EXIT_SUCCESS : EXIT_FAILURE;
}
]] HAVE_STIMEOUT)
if(NOT HAVE_STIMEOUT)
message(WARNING [[
Your libavformat library lacks support for the "stimeout" rtsp option.
Moonfire NVR will not be able to detect network partitions or retry.
Consider installing a recent ffmpeg, from source if necessary.
]])
else()
message(STATUS "libavformat library has support for \"stimeout\" - good.")
endif()
enable_testing()
# http://www.kaizou.org/2014/11/gtest-cmake/
include(ExternalProject)
ExternalProject_Add(
GTestProject
URL "https://github.com/google/googletest/archive/release-1.8.0.tar.gz"
URL_HASH "SHA1=e7e646a6204638fe8e87e165292b8dd9cd4c36ed"
INSTALL_COMMAND "")
ExternalProject_Get_Property(GTestProject source_dir binary_dir)
set(GTest_INCLUDE_DIR ${source_dir}/googletest/include)
add_library(GTest STATIC IMPORTED)
add_dependencies(GTest GTestProject)
set_target_properties(GTest PROPERTIES
IMPORTED_LOCATION "${binary_dir}/googlemock/gtest/${CMAKE_STATIC_LIBRARY_PREFIX}gtest${CMAKE_STATIC_LIBRARY_SUFFIX}"
IMPORTED_LINK_INTERFACE_LIBRARIES "${CMAKE_THREAD_LIBS_INIT}")
set(GMock_INCLUDE_DIR ${source_dir}/googlemock/include)
add_library(GMock STATIC IMPORTED)
add_dependencies(GMock GTestProject)
set_target_properties(GMock PROPERTIES
IMPORTED_LOCATION "${binary_dir}/googlemock/${CMAKE_STATIC_LIBRARY_PREFIX}gmock${CMAKE_STATIC_LIBRARY_SUFFIX}"
IMPORTED_LINK_INTERFACE_LIBRARIES "${CMAKE_THREAD_LIBS_INIT}")
ExternalProject_Add(
GBenchmarkProject
URL "https://github.com/google/benchmark/archive/v1.0.0.tar.gz"
URL_HASH "SHA1=4f778985dce02d2e63262e6f388a24b595254a93"
CMAKE_ARGS
-DCMAKE_BUILD_TYPE:STRING=${CMAKE_BUILD_TYPE}
INSTALL_COMMAND "")
ExternalProject_Get_Property(GBenchmarkProject source_dir binary_dir)
set(GBenchmark_INCLUDE_DIR ${source_dir}/include)
add_library(GBenchmark STATIC IMPORTED)
add_dependencies(GBenchmark GBenchmarkProject)
set_target_properties(GBenchmark PROPERTIES
IMPORTED_LOCATION "${binary_dir}/src/${CMAKE_STATIC_LIBRARY_PREFIX}benchmark${CMAKE_STATIC_LIBRARY_SUFFIX}"
IMPORTED_LINK_INTERFACE_LIBRARIES "${CMAKE_THREAD_LIBS_INIT}")
#
# Subdirectories.
#
add_subdirectory(src)

55
Cargo.toml Normal file
View File

@ -0,0 +1,55 @@
[package]
name = "moonfire-nvr"
version = "0.1.0"
authors = ["Scott Lamb <slamb@slamb.org>"]
[dependencies]
byteorder = "0.5"
clippy = "0.0.103"
chan = "0.1"
chan-signal = "0.1"
docopt = "0.6"
fnv = "1.0"
hyper = "0.9"
lazy_static = "0.2"
libc = "0.2"
log = { version = "0.3", features = ["release_max_level_info"] }
lru-cache = "0.1"
memmap = "0.3"
mime = "0.2"
openssl = "0.8"
regex = "0.1"
rusqlite = "0.7"
rustc-serialize = "0.3"
serde = "0.8"
serde_json = "0.8"
serde_derive = "0.8"
slog = "1.2"
slog-envlogger = "0.5"
slog-stdlog = "1.1"
slog-term = "1.3"
smallvec = "0.2"
time = "0.1"
url = "1.2"
uuid = { version = "0.3", features = ["serde", "v4"] }
[dev-dependencies]
tempdir = "0.3"
[dependencies.ffmpeg]
version = "0.2.0-alpha.1"
default-features = false
features = ["codec", "format"]
[dependencies.ffmpeg-sys]
version = "2.8"
default-features = false
features = ["avcodec"]
[profile.release]
lto = true
debug = true
[profile.bench]
lto = true
debug = true

117
README.md
View File

@ -63,59 +63,51 @@ edge version from the command line via git:
# Building from source
There are no binary packages of Moonfire NVR available yet, so it must be built
from source. It requires several packages to build:
from source.
* [CMake](https://cmake.org/) version 3.1.0 or higher.
* a C++11 compiler, such as [gcc](https://gcc.gnu.org/) 4.7 or higher.
* [ffmpeg](http://ffmpeg.org/), including `libavutil`,
The `rust` branch contains a rewrite into the [Rust Programming
Language](https://www.rust-lang.org/en-US/). In the long term, I expect this
will result in a more secure, full-featured, easy-to-install software. In the
short term, there will be growing pains. Rust is a new programming language.
Moonfire NVR's primary author is new to Rust. And Moonfire NVR is a young
project.
You will need the following C libraries installed:
* [ffmpeg](http://ffmpeg.org/) version 2.x, including `libavutil`,
`libavcodec` (to inspect H.264 frames), and `libavformat` (to connect to RTSP
servers and write `.mp4` files). Note ffmpeg versions older than 55.1.101,
along with all versions of the competing project [libav](http://libav.org),
does not support socket timeouts for RTSP. For reliable reconnections on
error, it's strongly recommended to use ffmpeg >= 55.1.101.
* [libevent](http://libevent.org/) 2.1, for the built-in HTTP server.
(This might be replaced with the more full-featured
[nghttp2](https://github.com/tatsuhiro-t/nghttp2) in the future.)
Unfortunately, the libevent 2.0 bundled with current Debian releases is
unsuitable.
* [gflags](http://gflags.github.io/gflags/), for command line flag parsing.
* [glog](https://github.com/google/glog), for debug logging.
* [gperftools](https://github.com/gperftools/gperftools), for debugging.
* [googletest](https://github.com/google/googletest), for automated testing.
This will be automatically downloaded during the build process, so it's
not necessary to install it beforehand.
* [re2](https://github.com/google/re2), for parsing with regular expressions.
* libuuid from [util-linux](https://en.wikipedia.org/wiki/Util-linux).
servers and write `.mp4` files).
Note ffmpeg 3.x isn't supported yet by the Rust `ffmpeg` crate; see
[rust-ffmpeg/issues/64](https://github.com/meh/rust-ffmpeg/issues/64).
Additionally, ffmpeg library versions older than 55.1.101, along with
55.1.101, along with all versions of the competing project
[libav](http://libav.org), don't not support socket timeouts for RTSP. For
reliable reconnections on error, it's strongly recommended to use ffmpeg
library versions >= 55.1.101.
* [SQLite3](https://www.sqlite.org/).
On Ubuntu 15.10 or Raspbian Jessie, the following command will install most
pre-requisites (see also the `Build-Depends` field in `debian/control`):
On Ubuntu 16.04.1 LTS or Raspbian Jessie, the following command will install
all non-Rust dependencies:
$ sudo apt-get install \
build-essential \
cmake \
libavcodec-dev \
libavformat-dev \
libavutil-dev \
libgflags-dev \
libgoogle-glog-dev \
libgoogle-perftools-dev \
libjsoncpp-dev \
libre2-dev \
sqlite3 \
libsqlite3-dev \
pkgconf \
uuid-runtime \
uuid-dev
libevent 2.1 will have to be installed from source. In the future, this
dependency may be replaced or support may be added for automatically building
libevent in-tree to avoid the inconvenience.
uuid-runtime
uuid-runtime is only necessary if you wish to use the uuid command to generate
uuids for your cameras (see below). If you obtain them elsewhere, you can skip this
package.
Next, you need a nightly version of Rust and Cargo. The easiest way to install
them is by following the instructions at [rustup.rs](https://www.rustup.rs/).
You can continue to follow the build/install instructions below for a manual
build and install, or alternatively you can run the prep script called `prep.sh`.
@ -124,11 +116,7 @@ build and install, or alternatively you can run the prep script called `prep.sh`
The script will take the following command line options, should you need them:
* `-E`: Forcibly purge all existing libevent packages. You would only do this
if there is some apparent conflict (see remarks about building libevent
from source).
* `-f`: Force a build even if the binary appears to be installed. This can be useful
on repeat builds.
* `-D`: Skip database initialization.
* `-S`: Skip updating and installing dependencies through apt-get. This too can be
useful on repeated builds.
@ -151,17 +139,9 @@ For instructions, you can skip to "[Camera configuration and hard disk mounting]
Once prerequisites are installed, Moonfire NVR can be built as follows:
$ mkdir release
$ cd release
$ cmake -DCMAKE_BUILD_TYPE=Release ..
$ make
$ sudo make install
Alternatively, if you do have a sufficiently new apt-installed libevent
installed, you may be able to prepare a `.deb` package:
$ sudo apt-get install devscripts dh-systemd
$ debuild -us -uc
$ RUST_TEST_THREADS=1 cargo test
$ cargo build --release
$ sudo install -m 755 target/release/moonfire-nvr /usr/local/bin
# Further configuration
@ -282,9 +262,10 @@ been done for you. If not, Create
[Service]
ExecStart=/usr/local/bin/moonfire-nvr \
--sample_file_dir=/var/lib/moonfire-nvr/sample \
--db_dir=/var/lib/moonfire-nvr/db \
--http_port=8080
--sample-file-dir=/var/lib/moonfire-nvr/sample \
--db-dir=/var/lib/moonfire-nvr/db \
--http-addr=0.0.0.0:8080
Environment=RUST_LOG=info
Type=simple
User=moonfire-nvr
Nice=-20
@ -313,10 +294,16 @@ and `systemctl` may be of particular interest.
# Troubleshooting
While Moonfire NVR is running, logs will be written to `/tmp/moonfire-nvr.INFO`.
Also available will be `/tmp/moonfire-nvr.WARNING` and `/tmp/moonfire-nvr.ERROR`.
These latter to contain only warning or more serious messages, respectively only
error messages.
While Moonfire NVR is running, logs will be written to stdout. The `RUST_LOG`
environmental variable controls the log level; `RUST_LOG=info` is recommended.
If running through systemd, try `sudo journalctl --unit moonfire-nvr` to view
the logs.
If Moonfire NVR crashes with a `SIGSEGV`, the problem is likely an
incompatible version of the C `ffmpeg` libraries; use the latest 2.x release
instead. This is one of the Rust growing pains mentioned above. While most
code written in Rust is "safe", the foreign function interface is not only
unsafe but currently error-prone.
# <a name="help"></a> Getting help and getting involved
@ -325,17 +312,11 @@ Please email the
mailing list with questions, bug reports, feature requests, or just to say
you love/hate the software and why.
I'd welcome help with testing, development (in C++, JavaScript, and HTML), user
interface/graphic design, and documentation. Please email the mailing list
if interested. Patches are welcome, but I encourage you to discuss large
I'd welcome help with testing, development (in Rust, JavaScript, and HTML),
user interface/graphic design, and documentation. Please email the mailing
list if interested. Patches are welcome, but I encourage you to discuss large
changes on the mailing list first to save effort.
C++ code should be written using C++11 features, should follow the [Google C++
style guide](https://google.github.io/styleguide/cppguide.html) for
consistency, and should be automatically tested where practical. But don't
worry about this too much; I'm much happier to work with you to refine a rough
draft patch than never see your contribution at all!
# License
This file is part of Moonfire NVR, a security camera digital video recorder.

5
debian/changelog vendored
View File

@ -1,5 +0,0 @@
moonfire-nvr (0.1.0) UNRELEASED; urgency=medium
* Initial release.
-- Scott Lamb <slamb@slamb.org> Fri, 1 Jan 2016 21:00:00 -0800

1
debian/compat vendored
View File

@ -1 +0,0 @@
9

22
debian/control vendored
View File

@ -1,22 +0,0 @@
Source: moonfire-nvr
Maintainer: Scott Lamb <slamb@slamb.org>
Section: video
Priority: optional
Standards-Version: 3.9.6.1
Build-Depends: cmake,
libavcodec-dev,
libavformat-dev,
libavutil-dev,
libgflags-dev,
libgoogle-glog-dev,
libgoogle-perftools-dev,
libjsoncpp-dev,
libre2-dev,
libsqlite3-dev,
pkgconf,
uuid-dev
Package: moonfire-nvr
Architecture: any
Depends: ${shlibs:Depends}, ${misc:Depends}, adduser, sqlite3, uuid-runtime
Description: security camera network video recorder
moonfire-nvr records video files from IP security cameras.

43
debian/copyright vendored
View File

@ -1,43 +0,0 @@
Format: http://www.debian.org/doc/packaging-manuals/copyright-format/1.0/
Upstream-Name: Moonfire NVR
Upstream-Contact: Scott Lamb <slamb@slamb.org>
Source: http://github.com/scottlamb/moonfire-nvr
Files: *
Copyright: 2016 Scott Lamb
License: GPL-3+ with OpenSSL exception
License: GPL-3+ with OpenSSL exception
This program is free software: you can redistribute it
and/or modify it under the terms of the GNU General Public
License as published by the Free Software Foundation,
either version 3 of the License, or (at your option) any
later version.
.
In addition, as a special exception, the copyright holders give
permission to link the code of portions of this program with the
OpenSSL library under certain conditions as described in each
individual source file, and distribute linked combinations including
the two.
.
You must obey the GNU General Public License in all respects for all
of the code used other than OpenSSL. If you modify file(s) with this
exception, you may extend this exception to your version of the
file(s), but you are not obligated to do so. If you do not wish to do
so, delete this exception statement from your version. If you delete
this exception statement from all source files in the program, then
also delete it here.
.
This program is distributed in the hope that it will be
useful, but WITHOUT ANY WARRANTY; without even the implied
warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
PURPOSE. See the GNU General Public License for more
details.
.
You should have received a copy of the GNU General Public
License along with this program. If not, see
<http://www.gnu.org/licenses/>.
.
On Debian systems, the full text of the GNU General Public
License version 3 can be found in the file
`/usr/share/common-licenses/GPL-3'.

8
debian/moonfire-nvr.postinst vendored
View File

@ -1,8 +0,0 @@
#!/bin/sh
set -e
adduser --system moonfire-nvr
#DEBHELPER#
exit 0

16
debian/moonfire-nvr.service vendored
View File

@ -1,16 +0,0 @@
[Unit]
Description=Moonfire NVR
After=network-online.target
[Service]
ExecStart=/usr/bin/moonfire-nvr
Type=simple
User=moonfire-nvr
Nice=-20
Restart=on-abnormal
CPUAccounting=true
MemoryAccounting=true
BlockIOAccounting=true
[Install]
WantedBy=multi-user.target

3
debian/rules vendored
View File

@ -1,3 +0,0 @@
#!/usr/bin/make -f
%:
dh $@ --with=systemd

1
debian/source/format vendored
View File

@ -1 +0,0 @@
3.0 (native)

48
prep.sh
View File

@ -34,7 +34,7 @@
# Script to prepare for moonfire-nvr operations
#
# Command line options:
# -f: Force clean build, even if binary already installed
# -D: Skip database initialization
# -S: Skip apt-get update and install
#
@ -112,14 +112,10 @@ SERVICE_BIN="${SERVICE_BIN:-/usr/local/bin/moonfire-nvr}"
# Process command line options
#
while getopts ":DEfS" opt; do
while getopts ":DS" opt; do
case $opt in
D) SKIP_DB=1
;;
E) PURGE_LIBEVENT=1
;;
f) FORCE_BUILD=1
;;
S) SKIP_APT=1
;;
:)
@ -133,45 +129,34 @@ while getopts ":DEfS" opt; do
esac
done
# Setup all packages we need
# Setup all apt packages we need
#
echo 'Preparing and downloading packages we need...'; echo
if [ "${SKIP_APT:-0}" != 1 ]; then
sudo apt-get update
[ "${PURGE_LIBEVENT:-0}" == 1 ] && sudo apt-get --purge remove libevent-*
sudo apt-get install \
build-essential \
cmake \
libavcodec-dev \
libavformat-dev \
libavutil-dev \
libgflags-dev \
libgoogle-glog-dev \
libgoogle-perftools-dev \
libjsoncpp-dev \
libre2-dev \
libssl-dev \
sqlite3 \
libsqlite3-dev \
pkgconf \
uuid-runtime \
uuid-dev
uuid-runtime
fi
# Check if binary is installed. Setup for build if it is not
#
if [ ! -x "${SERVICE_BIN}" ]; then
echo "Binary not installed, building..."; echo
FORCE_BUILD=1
fi
# Build if requested
#
if [ "${FORCE_BUILD:-0}" -eq 1 ]; then
# Remove previous build, if any
[ -d release ] && rm -fr release 2>/dev/null
mkdir release; cd release
cmake -DCMAKE_BUILD_TYPE=Release .. && make && sudo make install
if ! cargo --version; then
echo "cargo not installed/working."
echo "Install a nightly Rust (see http://rustup.us) first."
echo
exit 1
fi
RUST_TEST_THREADS=1 cargo test
cargo build --release
sudo install -m 755 target/release/moonfire-nvr ${SERVICE_BIN}
if [ -x "${SERVICE_BIN}" ]; then
echo "Binary installed..."; echo
else
@ -233,9 +218,10 @@ After=network-online.target
[Service]
ExecStart=${SERVICE_BIN} \\
--sample_file_dir=${SAMPLES_PATH} \\
--db_dir=${DB_DIR} \\
--http_port=${NVR_PORT}
--sample-file-dir=${SAMPLES_PATH} \\
--db-dir=${DB_DIR} \\
--http-addr=0.0.0.0:${NVR_PORT}
Environment=RUST_LOG=info
Type=simple
User=${NVR_USER}
Nice=-20

101
src/CMakeLists.txt
View File

@ -1,101 +0,0 @@
# Copyright (C) 2016 Scott Lamb <slamb@slamb.org>
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# In addition, as a special exception, the copyright holders give
# permission to link the code of portions of this program with the
# OpenSSL library under certain conditions as described in each
# individual source file, and distribute linked combinations including
# the two.
#
# You must obey the GNU General Public License in all respects for all
# of the code used other than OpenSSL. If you modify file(s) with this
# exception, you may extend this exception to your version of the
# file(s), but you are not obligated to do so. If you do not wish to do
# so, delete this exception statement from your version. If you delete
# this exception statement from all source files in the program, then
# also delete it here.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
include_directories(${CMAKE_CURRENT_BINARY_DIR})
include_directories(${JSONCPP_INCLUDE_DIRS})
set(MOONFIRE_DEPS
${CMAKE_THREAD_LIBS_INIT}
${FFMPEG_LIBRARIES}
${GFLAGS_LIBRARIES}
${GLOG_LIBRARIES}
${JSONCPP_LIBRARIES}
${LIBEVENT_LIBRARIES}
${OPENSSL_LIBRARIES}
${PROFILER_LIBRARIES}
${RE2_LIBRARIES}
${SQLITE_LIBRARIES}
${UUID_LIBRARIES})
set(MOONFIRE_NVR_SRCS
coding.cc
crypto.cc
ffmpeg.cc
filesystem.cc
h264.cc
http.cc
moonfire-db.cc
moonfire-nvr.cc
mp4.cc
profiler.cc
recording.cc
sqlite.cc
string.cc
time.cc
uuid.cc
web.cc)
link_directories(${LIBEVENT_LIBRARY_DIRS})
add_library(moonfire-nvr-lib ${MOONFIRE_NVR_SRCS} ${PROTO_SRCS} ${PROTO_HDRS})
target_link_libraries(moonfire-nvr-lib ${MOONFIRE_DEPS})
add_executable(moonfire-nvr moonfire-nvr-main.cc)
target_link_libraries(moonfire-nvr moonfire-nvr-lib)
install_programs(/bin FILES moonfire-nvr)
# Tests.
include_directories(${GTest_INCLUDE_DIR})
include_directories(${GMock_INCLUDE_DIR})
include_directories(${GBenchmark_INCLUDE_DIR})
set(MOONFIRE_NVR_TESTS
coding
crypto
h264
http
moonfire-db
moonfire-nvr
mp4
recording
sqlite
string)
foreach(test ${MOONFIRE_NVR_TESTS})
add_executable(${test}-test ${test}-test.cc testutil.cc)
target_link_libraries(${test}-test GTest GMock moonfire-nvr-lib)
add_test(NAME ${test}-test
COMMAND ${test}-test
WORKING_DIRECTORY ${CMAKE_BINARY_DIR})
endforeach(test)
foreach(bench recording)
add_executable(${bench}-bench ${bench}-bench.cc testutil.cc)
target_link_libraries(${bench}-bench GTest GMock GBenchmark moonfire-nvr-lib)
endforeach(bench)

141
src/coding-test.cc
View File

@ -1,141 +0,0 @@
// This file is part of Moonfire NVR, a security camera network video recorder.
// Copyright (C) 2016 Scott Lamb <slamb@slamb.org>
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// In addition, as a special exception, the copyright holders give
// permission to link the code of portions of this program with the
// OpenSSL library under certain conditions as described in each
// individual source file, and distribute linked combinations including
// the two.
//
// You must obey the GNU General Public License in all respects for all
// of the code used other than OpenSSL. If you modify file(s) with this
// exception, you may extend this exception to your version of the
// file(s), but you are not obligated to do so. If you do not wish to do
// so, delete this exception statement from your version. If you delete
// this exception statement from all source files in the program, then
// also delete it here.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
//
// coding-test.cc: tests of the coding.h interface.
#include <gflags/gflags.h>
#include <glog/logging.h>
#include <gmock/gmock.h>
#include <gtest/gtest.h>
#include "coding.h"
DECLARE_bool(alsologtostderr);
namespace moonfire_nvr {
namespace {
TEST(VarintTest, Simple) {
// Encode.
std::string foo;
AppendVar32(UINT32_C(1), &foo);
EXPECT_EQ("\x01", foo);
AppendVar32(UINT32_C(300), &foo);
EXPECT_EQ("\x01\xac\x02", foo);
// Decode.
re2::StringPiece p(foo);
uint32_t out;
std::string error_message;
EXPECT_TRUE(DecodeVar32(&p, &out, &error_message));
EXPECT_EQ(UINT32_C(1), out);
EXPECT_TRUE(DecodeVar32(&p, &out, &error_message));
EXPECT_EQ(UINT32_C(300), out);
EXPECT_EQ(0, p.size());
}
TEST(VarintTest, AllDecodeSizes) {
std::string error_message;
const uint32_t kToDecode[]{
1,
1 | (2 << 7),
1 | (2 << 7) | (3 << 14),
1 | (2 << 7) | (3 << 14) | (4 << 21),
1 | (2 << 7) | (3 << 14) | (4 << 21) | (5 << 28),
};
for (size_t i = 0; i < sizeof(kToDecode) / sizeof(kToDecode[0]); ++i) {
auto in = kToDecode[i];
std::string foo;
AppendVar32(in, &foo);
ASSERT_EQ(i + 1, foo.size());
re2::StringPiece p(foo);
uint32_t out;
// Slow path: last bytes of the buffer.
DecodeVar32(&p, &out, &error_message);
EXPECT_EQ(in, out) << "i: " << i;
EXPECT_EQ(0, p.size()) << "i: " << i;
// Fast path: plenty of bytes in the buffer.
foo.append(4, 0);
p = foo;
DecodeVar32(&p, &out, &error_message);
EXPECT_EQ(in, out);
EXPECT_EQ(4, p.size());
}
}
TEST(VarintTest, DecodeErrors) {
re2::StringPiece empty;
uint32_t out;
std::string error_message;
for (auto input :
{re2::StringPiece("", 0), re2::StringPiece("\x80", 1),
re2::StringPiece("\x80\x80", 2), re2::StringPiece("\x80\x80\x80", 3),
re2::StringPiece("\x80\x80\x80\x80", 4)}) {
EXPECT_FALSE(DecodeVar32(&input, &out, &error_message)) << "input: "
<< input;
EXPECT_EQ("buffer underrun", error_message);
}
re2::StringPiece too_big("\x80\x80\x80\x80\x10", 5);
EXPECT_FALSE(DecodeVar32(&too_big, &out, &error_message));
EXPECT_EQ("integer overflow", error_message);
}
TEST(ZigzagTest, Encode) {
EXPECT_EQ(UINT32_C(0), Zigzag32(INT32_C(0)));
EXPECT_EQ(UINT32_C(1), Zigzag32(INT32_C(-1)));
EXPECT_EQ(UINT32_C(2), Zigzag32(INT32_C(1)));
EXPECT_EQ(UINT32_C(3), Zigzag32(INT32_C(-2)));
EXPECT_EQ(UINT32_C(4294967294), Zigzag32(INT32_C(2147483647)));
EXPECT_EQ(UINT32_C(4294967295), Zigzag32(INT32_C(-2147483648)));
}
TEST(ZigzagTest, Decode) {
EXPECT_EQ(INT32_C(0), Unzigzag32(UINT32_C(0)));
EXPECT_EQ(INT32_C(-1), Unzigzag32(UINT32_C(1)));
EXPECT_EQ(INT32_C(1), Unzigzag32(UINT32_C(2)));
EXPECT_EQ(INT32_C(-2), Unzigzag32(UINT32_C(3)));
EXPECT_EQ(INT32_C(2147483647), Unzigzag32(UINT32_C(4294967294)));
EXPECT_EQ(INT32_C(-2147483648), Unzigzag32(UINT32_C(4294967295)));
}
} // namespace
} // namespace moonfire_nvr
int main(int argc, char **argv) {
FLAGS_alsologtostderr = true;
google::ParseCommandLineFlags(&argc, &argv, true);
testing::InitGoogleTest(&argc, argv);
google::InitGoogleLogging(argv[0]);
return RUN_ALL_TESTS();
}

109
src/coding.cc
View File

@ -1,109 +0,0 @@
// This file is part of Moonfire NVR, a security camera network video recorder.
// Copyright (C) 2016 Scott Lamb <slamb@slamb.org>
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// In addition, as a special exception, the copyright holders give
// permission to link the code of portions of this program with the
// OpenSSL library under certain conditions as described in each
// individual source file, and distribute linked combinations including
// the two.
//
// You must obey the GNU General Public License in all respects for all
// of the code used other than OpenSSL. If you modify file(s) with this
// exception, you may extend this exception to your version of the
// file(s), but you are not obligated to do so. If you do not wish to do
// so, delete this exception statement from your version. If you delete
// this exception statement from all source files in the program, then
// also delete it here.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
//
// coding.cc: see coding.h.
#include "coding.h"
#include "common.h"
namespace moonfire_nvr {
namespace internal {
void AppendVar32Slow(uint32_t in, std::string *out) {
while (true) {
uint8_t next_byte = in & 0x7F;
in >>= 7;
if (in == 0) {
out->push_back(next_byte);
return;
}
out->push_back(next_byte | 0x80);
}
}
bool DecodeVar32Slow(re2::StringPiece *in, uint32_t *out_p,
std::string *error_message) {
// The fast path is inlined; this function is called only when
// byte 0 is present and >= 0x80.
size_t left = in->size() - 1;
auto p = reinterpret_cast<uint8_t const *>(in->data());
uint32_t v = uint32_t(p[0] & 0x7f);
size_t size = 1;
// Aid branch prediction in two ways:
// * have a faster path which doesn't check for buffer underrun on every
// byte if there's plenty of bytes left or the last byte is not continued.
// * fully unroll the loop
if (left >= 4 || (p[left] & 0x80) == 0) {
v |= uint32_t(p[size] & 0x7f) << 7;
if (p[size++] & 0x80) {
v |= uint32_t(p[size] & 0x7f) << 14;
if (p[size++] & 0x80) {
v |= uint32_t(p[size] & 0x7f) << 21;
if (p[size++] & 0x80) {
if (UNLIKELY(p[size] & 0xf0)) {
*error_message = "integer overflow";
return false;
}
v |= uint32_t(p[size++] & 0x7f) << 28;
}
}
}
*out_p = v;
in->remove_prefix(size);
return true;
}
// Slowest path.
if (LIKELY(left)) {
v |= uint32_t(p[size] & 0x7f) << 7;
if (p[size++] & 0x80 && --left > 0) {
v |= uint32_t(p[size] & 0x7f) << 14;
if (p[size++] & 0x80 && --left > 0) {
v |= uint32_t(p[size] & 0x7f) << 21;
if (p[size++] & 0x80) {
--left;
}
}
}
}
if (UNLIKELY(left == 0 && p[size - 1] & 0x80)) {
*error_message = "buffer underrun";
return false;
}
*out_p = v;
in->remove_prefix(size);
return true;
}
} // namespace internal
} // namespace moonfire_nvr

169
src/coding.h
View File

@ -1,169 +0,0 @@
// This file is part of Moonfire NVR, a security camera network video recorder.
// Copyright (C) 2016 Lamb <slamb@slamb.org>
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// In addition, as a special exception, the copyright holders give
// permission to link the code of portions of this program with the
// OpenSSL library under certain conditions as described in each
// individual source file, and distribute linked combinations including
// the two.
//
// You must obey the GNU General Public License in all respects for all
// of the code used other than OpenSSL. If you modify file(s) with this
// exception, you may extend this exception to your version of the
// file(s), but you are not obligated to do so. If you do not wish to do
// so, delete this exception statement from your version. If you delete
// this exception statement from all source files in the program, then
// also delete it here.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
//
// coding.h: Binary encoding/decoding.
#ifndef MOONFIRE_NVR_CODING_H
#define MOONFIRE_NVR_CODING_H
#include <endian.h>
#include <stdint.h>
#include <string>
#include <re2/stringpiece.h>
namespace moonfire_nvr {
namespace internal {
void AppendVar32Slow(uint32_t in, std::string *out);
bool DecodeVar32Slow(re2::StringPiece *in, uint32_t *out,
std::string *error_message);
} // namespace internal
// Endianness conversion.
#if __BYTE_ORDER == __LITTLE_ENDIAN
// XXX: __builtin_bswap64 doesn't compile on gcc 5.2.1 with an error about a
// narrowing conversion?!? Doing this by hand...
constexpr uint64_t ToNetworkU64(uint64_t in) {
return ((in & UINT64_C(0xFF00000000000000)) >> 56) |
((in & UINT64_C(0x00FF000000000000)) >> 40) |
((in & UINT64_C(0x0000FF0000000000)) >> 24) |
((in & UINT64_C(0x000000FF00000000)) >> 8) |
((in & UINT64_C(0x00000000FF000000)) << 8) |
((in & UINT64_C(0x0000000000FF0000)) << 24) |
((in & UINT64_C(0x000000000000FF00)) << 40) |
((in & UINT64_C(0x00000000000000FF)) << 56);
}
constexpr int64_t ToNetwork64(int64_t in) {
return static_cast<int64_t>(ToNetworkU64(static_cast<uint64_t>(in)));
}
constexpr uint32_t ToNetworkU32(uint32_t in) {
return ((in & UINT32_C(0xFF000000)) >> 24) |
((in & UINT32_C(0x00FF0000)) >> 8) |
((in & UINT32_C(0x0000FF00)) << 8) |
((in & UINT32_C(0x000000FF)) << 24);
}
constexpr int32_t ToNetwork32(int32_t in) {
return static_cast<int32_t>(ToNetworkU32(static_cast<uint32_t>(in)));
}
constexpr uint16_t ToNetworkU16(uint16_t in) {
return ((in & UINT32_C(0xFF00)) >> 8) | ((in & UINT32_C(0x00FF)) << 8);
}
constexpr int16_t ToNetwork16(int16_t in) {
return static_cast<int16_t>(ToNetworkU16(static_cast<uint16_t>(in)));
}
#elif __BYTE_ORDER == __BIG_ENDIAN
constexpr uint64_t ToNetworkU64(uint64_t in) { return in; }
constexpr int64_t ToNetwork64(int64_t in) { return in; }
constexpr uint32_t ToNetworkU32(uint32_t in) { return in; }
constexpr int32_t ToNetwork32(int32_t in) { return in; }
constexpr uint16_t ToNetworkU16(uint16_t in) { return in; }
constexpr int16_t ToNetwork16(int16_t in) { return in; }
#else
#error Unknown byte order.
#endif
// Varint encoding, as in
// https://developers.google.com/protocol-buffers/docs/encoding#varints
inline void AppendVar32(uint32_t in, std::string *out) {
if (in < UINT32_C(1) << 7) {
out->push_back(static_cast<char>(in));
} else {
internal::AppendVar32Slow(in, out);
}
}
// Decode the first varint from |in|, saving it to |out| and advancing |in|.
// Returns error if |in| does not hold a complete varint or on integer overflow.
inline bool DecodeVar32(re2::StringPiece *in, uint32_t *out,
std::string *error_message) {
if (in->size() == 0) {
*error_message = "buffer underrun";
return false;
}
auto first_byte = static_cast<uint8_t>(*in->data());
if (first_byte < 0x80) {
in->remove_prefix(1);
*out = first_byte;
return true;
} else {
return internal::DecodeVar32Slow(in, out, error_message);
}
}
// Zigzag encoding for signed integers, as in
// https://developers.google.com/protocol-buffers/docs/encoding#types
// Use the low bit to indicate signedness (1 = negative, 0 = non-negative).
inline uint32_t Zigzag32(int32_t in) {
return static_cast<uint32_t>(in << 1) ^ (in >> 31);
}
inline int32_t Unzigzag32(uint32_t in) {
return (in >> 1) ^ -static_cast<int32_t>(in & 1);
}
inline void AppendU16(uint16_t in, std::string *out) {
uint16_t net = ToNetworkU16(in);
out->append(reinterpret_cast<const char *>(&net), sizeof(uint16_t));
}
inline void Append16(int16_t in, std::string *out) {
int16_t net = ToNetwork16(in);
out->append(reinterpret_cast<const char *>(&net), sizeof(int16_t));
}
inline void AppendU32(uint32_t in, std::string *out) {
uint32_t net = ToNetworkU32(in);
out->append(reinterpret_cast<const char *>(&net), sizeof(uint32_t));
}
inline void Append32(int32_t in, std::string *out) {
int32_t net = ToNetwork32(in);
out->append(reinterpret_cast<const char *>(&net), sizeof(int32_t));
}
inline void AppendU64(uint64_t in, std::string *out) {
uint64_t net = ToNetworkU64(in);
out->append(reinterpret_cast<const char *>(&net), sizeof(uint64_t));
}
inline void Append64(int64_t in, std::string *out) {
int64_t net = ToNetwork64(in);
out->append(reinterpret_cast<const char *>(&net), sizeof(int64_t));
}
} // namespace moonfire_nvr
#endif // MOONFIRE_NVR_CODING_H

49
src/common.h
View File

@ -1,49 +0,0 @@
// This file is part of Moonfire NVR, a security camera network video recorder.
// Copyright (C) 2016 Scott Lamb <slamb@slamb.org>
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// In addition, as a special exception, the copyright holders give
// permission to link the code of portions of this program with the
// OpenSSL library under certain conditions as described in each
// individual source file, and distribute linked combinations including
// the two.
//
// You must obey the GNU General Public License in all respects for all
// of the code used other than OpenSSL. If you modify file(s) with this
// exception, you may extend this exception to your version of the
// file(s), but you are not obligated to do so. If you do not wish to do
// so, delete this exception statement from your version. If you delete
// this exception statement from all source files in the program, then
// also delete it here.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
//
// common.h: basic enums/defines/whatever.
#ifndef MOONFIRE_NVR_COMMON_H
#define MOONFIRE_NVR_COMMON_H
#define LIKELY(x) __builtin_expect((x), 1)
#define UNLIKELY(x) __builtin_expect((x), 0)
namespace moonfire_nvr {
// Return value for *ForEach callbacks.
enum class IterationControl {
kContinue, // indicates the caller should proceed with the loop.
kBreak // indicates the caller should terminate the loop with success.
};
} // namespace moonfire_nvr
#endif // MOONFIRE_NVR_COMMON_H

67
src/crypto-test.cc
View File

@ -1,67 +0,0 @@
// This file is part of Moonfire NVR, a security camera network video recorder.
// Copyright (C) 2016 Scott Lamb <slamb@slamb.org>
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// In addition, as a special exception, the copyright holders give
// permission to link the code of portions of this program with the
// OpenSSL library under certain conditions as described in each
// individual source file, and distribute linked combinations including
// the two.
//
// You must obey the GNU General Public License in all respects for all
// of the code used other than OpenSSL. If you modify file(s) with this
// exception, you may extend this exception to your version of the
// file(s), but you are not obligated to do so. If you do not wish to do
// so, delete this exception statement from your version. If you delete
// this exception statement from all source files in the program, then
// also delete it here.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
//
// crypto-test.cc: tests of the crypto.h interface.
#include <gflags/gflags.h>
#include <glog/logging.h>
#include <gmock/gmock.h>
#include <gtest/gtest.h>
#include "crypto.h"
#include "string.h"
DECLARE_bool(alsologtostderr);
namespace moonfire_nvr {
namespace {
TEST(DigestTest, Sha1) {
auto sha1 = Digest::SHA1();
EXPECT_EQ("da39a3ee5e6b4b0d3255bfef95601890afd80709",
ToHex(sha1->Finalize()));
sha1 = Digest::SHA1();
sha1->Update("hello");
sha1->Update(" world");
EXPECT_EQ("2aae6c35c94fcfb415dbe95f408b9ce91ee846ed",
ToHex(sha1->Finalize()));
}
} // namespace
} // namespace moonfire_nvr
int main(int argc, char **argv) {
FLAGS_alsologtostderr = true;
google::ParseCommandLineFlags(&argc, &argv, true);
testing::InitGoogleTest(&argc, argv);
google::InitGoogleLogging(argv[0]);
return RUN_ALL_TESTS();
}

61
src/crypto.cc
View File

@ -1,61 +0,0 @@
// This file is part of Moonfire NVR, a security camera network video recorder.
// Copyright (C) 2016 Scott Lamb <slamb@slamb.org>
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// In addition, as a special exception, the copyright holders give
// permission to link the code of portions of this program with the
// OpenSSL library under certain conditions as described in each
// individual source file, and distribute linked combinations including
// the two.
//
// You must obey the GNU General Public License in all respects for all
// of the code used other than OpenSSL. If you modify file(s) with this
// exception, you may extend this exception to your version of the
// file(s), but you are not obligated to do so. If you do not wish to do
// so, delete this exception statement from your version. If you delete
// this exception statement from all source files in the program, then
// also delete it here.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
//
// crypto.cc: see crypto.h.
#include "crypto.h"
#include <glog/logging.h>
namespace moonfire_nvr {
std::unique_ptr<Digest> Digest::SHA1() {
std::unique_ptr<Digest> d(new Digest);
CHECK_EQ(1, EVP_DigestInit_ex(d->ctx_, EVP_sha1(), nullptr));
return d;
}
Digest::Digest() { ctx_ = CHECK_NOTNULL(EVP_MD_CTX_create()); }
Digest::~Digest() { EVP_MD_CTX_destroy(ctx_); }
void Digest::Update(re2::StringPiece data) {
CHECK_EQ(1, EVP_DigestUpdate(ctx_, data.data(), data.size()));
}
std::string Digest::Finalize() {
std::string out;
out.resize(EVP_MD_CTX_size(ctx_));
auto *p = reinterpret_cast<unsigned char *>(&out[0]);
CHECK_EQ(1, EVP_DigestFinal_ex(ctx_, p, nullptr));
return out;
}
} // namespace moonfire_nvr

63
src/crypto.h
View File

@ -1,63 +0,0 @@
// This file is part of Moonfire NVR, a security camera network video recorder.
// Copyright (C) 2016 Scott Lamb <slamb@slamb.org>
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// In addition, as a special exception, the copyright holders give
// permission to link the code of portions of this program with the
// OpenSSL library under certain conditions as described in each
// individual source file, and distribute linked combinations including
// the two.
//
// You must obey the GNU General Public License in all respects for all
// of the code used other than OpenSSL. If you modify file(s) with this
// exception, you may extend this exception to your version of the
// file(s), but you are not obligated to do so. If you do not wish to do
// so, delete this exception statement from your version. If you delete
// this exception statement from all source files in the program, then
// also delete it here.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
//
// crypto.h: cryptographic functions.
#ifndef MOONFIRE_NVR_CRYPTO_H
#define MOONFIRE_NVR_CRYPTO_H
#include <memory>
#include <openssl/evp.h>
#include <re2/stringpiece.h>
namespace moonfire_nvr {
class Digest {
public:
static std::unique_ptr<Digest> SHA1();
~Digest();
// PRE: Finalize() has not been called.
void Update(re2::StringPiece data);
// PRE: Finalize() has not been called.
std::string Finalize();
private:
Digest();
Digest(const Digest &) = delete;
void operator=(const Digest &) = delete;
EVP_MD_CTX *ctx_ = nullptr;
};
} // namespace moonfire_nvr
#endif // MOONFIRE_NVR_CRYPTO_H

1314
src/db.rs Normal file
View File

File diff suppressed because it is too large Load Diff

400
src/dir.rs Normal file
View File

@ -0,0 +1,400 @@
// This file is part of Moonfire NVR, a security camera digital video recorder.
// Copyright (C) 2016 Scott Lamb <slamb@slamb.org>
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// In addition, as a special exception, the copyright holders give
// permission to link the code of portions of this program with the
// OpenSSL library under certain conditions as described in each
// individual source file, and distribute linked combinations including
// the two.
//
// You must obey the GNU General Public License in all respects for all
// of the code used other than OpenSSL. If you modify file(s) with this
// exception, you may extend this exception to your version of the
// file(s), but you are not obligated to do so. If you do not wish to do
// so, delete this exception statement from your version. If you delete
// this exception statement from all source files in the program, then
// also delete it here.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
//! Sample file directory management.
//!
//! This includes opening files for serving, rotating away old
//! files, and syncing new files to disk.
use db;
use libc;
use recording;
use error::Error;
use std::ffi;
use std::fs;
use std::io::{self, Write};
use std::mem;
use std::os::unix::io::FromRawFd;
use std::sync::{Arc, Mutex, MutexGuard};
use std::sync::mpsc;
use std::thread;
use uuid::Uuid;
/// A sample file directory. This is currently a singleton in production. (Maybe in the future
/// Moonfire will be extended to support multiple directories on different spindles.)
///
/// If the directory is used for writing, the `start_syncer` function should be called to start
/// a background thread. This thread manages deleting files and writing new files. It synces the
/// directory and commits these operations to the database in the correct order to maintain the
/// invariants described in `design/schema.md`.
pub struct SampleFileDir {
db: Arc<db::Database>,
/// The open file descriptor for the directory. The worker uses it to create files and sync the
/// directory. Other threads use it to open sample files for reading during video serving.
fd: Fd,
// Lock order: don't acquire mutable.lock() while holding db.lock().
mutable: Mutex<SharedMutableState>,
}
/// A file descriptor associated with a directory (not necessarily the sample file dir).
pub struct Fd(libc::c_int);
impl Drop for Fd {
fn drop(&mut self) {
if unsafe { libc::close(self.0) } < 0 {
let e = io::Error::last_os_error();
warn!("Unable to close sample file dir: {}", e);
}
}
}
impl Fd {
/// Opens the given path as a directory.
pub fn open(path: &str) -> Result<Fd, io::Error> {
let cstring = ffi::CString::new(path)
.map_err(|e| io::Error::new(io::ErrorKind::InvalidInput, e))?;
let fd = unsafe { libc::open(cstring.as_ptr(), libc::O_DIRECTORY | libc::O_RDONLY, 0) };
if fd < 0 {
return Err(io::Error::last_os_error().into());
}
Ok(Fd(fd))
}
/// Locks the directory with the specified `flock` operation.
pub fn lock(&self, operation: libc::c_int) -> Result<(), io::Error> {
let ret = unsafe { libc::flock(self.0, operation) };
if ret < 0 {
return Err(io::Error::last_os_error().into());
}
Ok(())
}
}
impl SampleFileDir {
pub fn new(path: &str, db: Arc<db::Database>) -> Result<Arc<SampleFileDir>, Error> {
let fd = Fd::open(path)?;
Ok(Arc::new(SampleFileDir{
db: db,
fd: fd,
mutable: Mutex::new(SharedMutableState{
next_uuid: None,
}),
}))
}
/// Opens the given sample file for reading.
pub fn open_sample_file(&self, uuid: Uuid) -> Result<fs::File, io::Error> {
self.open_int(uuid, libc::O_RDONLY, 0)
}
/// Creates a new writer.
/// Note this doesn't wait for previous rotation to complete; it's assumed the sample file
/// directory has sufficient space for a couple recordings per camera in addition to the
/// cameras' total `retain_bytes`.
pub fn create_writer(&self, start: recording::Time, local_start: recording::Time,
camera_id: i32, video_sample_entry_id: i32)
-> Result<recording::Writer, Error> {
// Grab the next uuid. Typically one is cached—a sync has usually completed since the last
// writer was created, and syncs ensure `next_uuid` is filled while performing their
// transaction. But if not, perform an extra database transaction to reserve a new one.
let uuid = match self.mutable.lock().unwrap().next_uuid.take() {
Some(u) => u,
None => {
info!("Committing extra transaction because there's no cached uuid");
let mut db = self.db.lock();
let mut tx = db.tx()?;
let u = tx.reserve_sample_file()?;
tx.commit()?;
u
},
};
let f = match self.open_int(uuid, libc::O_WRONLY | libc::O_EXCL | libc::O_CREAT, 0o600) {
Ok(f) => f,
Err(e) => {
self.mutable.lock().unwrap().next_uuid = Some(uuid);
return Err(e.into());
},
};
recording::Writer::open(f, uuid, start, local_start, camera_id, video_sample_entry_id)
}
/// Opens a sample file within this directory with the given flags and (if creating) mode.
fn open_int(&self, uuid: Uuid, flags: libc::c_int, mode: libc::c_int)
-> Result<fs::File, io::Error> {
let p = SampleFileDir::get_rel_pathname(uuid);
let fd = unsafe { libc::openat(self.fd.0, p.as_ptr(), flags, mode) };
if fd < 0 {
return Err(io::Error::last_os_error())
}
unsafe { Ok(fs::File::from_raw_fd(fd)) }
}
/// Gets a pathname for a sample file suitable for passing to open or unlink.
fn get_rel_pathname(uuid: Uuid) -> [libc::c_char; 37] {
let mut buf = [0u8; 37];
write!(&mut buf[..36], "{}", uuid.hyphenated()).expect("can't format uuid to pathname buf");
// libc::c_char seems to be i8 on some platforms (Linux/arm) and u8 on others (Linux/amd64).
// Transmute, suppressing the warning that happens on platforms in which it's already u8.
#[allow(useless_transmute)]
unsafe { mem::transmute::<[u8; 37], [libc::c_char; 37]>(buf) }
}
/// Unlinks the given sample file within this directory.
fn unlink(fd: &Fd, uuid: Uuid) -> Result<(), io::Error> {
let p = SampleFileDir::get_rel_pathname(uuid);
let res = unsafe { libc::unlinkat(fd.0, p.as_ptr(), 0) };
if res < 0 {
return Err(io::Error::last_os_error())
}
Ok(())
}
/// Syncs the directory itself.
fn sync(&self) -> Result<(), io::Error> {
let res = unsafe { libc::fsync(self.fd.0) };
if res < 0 {
return Err(io::Error::last_os_error())
};
Ok(())
}
}
/// State shared between users of the `SampleFileDirectory` struct and the syncer.
struct SharedMutableState {
next_uuid: Option<Uuid>,
}
/// A command sent to the syncer. These correspond to methods in the `SyncerChannel` struct.
enum SyncerCommand {
AsyncSaveWriter(db::RecordingToInsert, fs::File),
#[cfg(test)]
Flush(mpsc::SyncSender<()>),
}
/// A channel which can be used to send commands to the syncer.
/// Can be cloned to allow multiple threads to send commands.
#[derive(Clone)]
pub struct SyncerChannel(mpsc::Sender<SyncerCommand>);
/// State of the worker thread.
struct SyncerState {
dir: Arc<SampleFileDir>,
to_unlink: Vec<Uuid>,
to_mark_deleted: Vec<Uuid>,
cmds: mpsc::Receiver<SyncerCommand>,
}
/// Starts a syncer for the given sample file directory.
/// There should be only one syncer per directory, or 0 if operating in read-only mode.
/// This function will perform the initial rotation synchronously, so that it is finished before
/// file writing starts. Afterward the syncing happens in a background thread.
///
/// Returns a `SyncerChannel` which can be used to send commands (and can be cloned freely) and
/// a `JoinHandle` for the syncer thread. At program shutdown, all `SyncerChannel` clones should be
/// removed and then the handle joined to allow all recordings to be persisted.
pub fn start_syncer(dir: Arc<SampleFileDir>)
-> Result<(SyncerChannel, thread::JoinHandle<()>), Error> {
let to_unlink = dir.db.lock().list_reserved_sample_files()?;
let (snd, rcv) = mpsc::channel();
let mut state = SyncerState {
dir: dir,
to_unlink: to_unlink,
to_mark_deleted: Vec::new(),
cmds: rcv,
};
state.initial_rotation()?;
Ok((SyncerChannel(snd),
thread::Builder::new().name("syncer".into()).spawn(move || state.run()).unwrap()))
}
impl SyncerChannel {
/// Asynchronously syncs the given writer, closes it, records it into the database, and
/// starts rotation.
pub fn async_save_writer(&self, w: recording::Writer) -> Result<(), Error> {
let (recording, f) = w.close()?;
self.0.send(SyncerCommand::AsyncSaveWriter(recording, f)).unwrap();
Ok(())
}
/// For testing: flushes the syncer, waiting for all currently-queued commands to complete.
#[cfg(test)]
pub fn flush(&self) {
let (snd, rcv) = mpsc::sync_channel(0);
self.0.send(SyncerCommand::Flush(snd)).unwrap();
rcv.recv().unwrap_err(); // syncer should just drop the channel, closing it.
}
}
impl SyncerState {
fn run(&mut self) {
loop {
match self.cmds.recv() {
Err(_) => return, // all senders have closed the channel; shutdown
Ok(SyncerCommand::AsyncSaveWriter(recording, f)) => self.save_writer(recording, f),
#[cfg(test)]
Ok(SyncerCommand::Flush(_)) => {}, // just drop the supplied sender, closing it.
};
}
}
/// Rotates files for all cameras and deletes stale reserved uuids from previous runs.
fn initial_rotation(&mut self) -> Result<(), Error> {
let mut to_delete = Vec::new();
{
let mut db = self.dir.db.lock();
for (camera_id, camera) in db.cameras_by_id() {
self.get_rows_to_delete(&db, *camera_id, camera, 0, &mut to_delete)?;
}
let mut tx = db.tx()?;
tx.delete_recordings(&to_delete)?;
tx.commit()?;
}
for row in to_delete {
self.to_unlink.push(row.uuid);
}
self.try_unlink();
if !self.to_unlink.is_empty() {
return Err(Error::new(format!("failed to unlink {} sample files",
self.to_unlink.len())));
}
self.dir.sync()?;
{
let mut db = self.dir.db.lock();
let mut tx = db.tx()?;
tx.mark_sample_files_deleted(&self.to_mark_deleted)?;
tx.commit()?;
}
self.to_mark_deleted.clear();
Ok(())
}
/// Saves the given writer and causes rotation to happen.
/// Note that part of rotation is deferred for the next cycle (saved writing or program startup)
/// so that there can be only one dir sync and database transaction per save.
fn save_writer(&mut self, recording: db::RecordingToInsert, f: fs::File) {
if let Err(e) = self.save_writer_helper(&recording, f) {
error!("camera {}: will discard recording {} due to error while saving: {}",
recording.camera_id, recording.sample_file_uuid, e);
self.to_unlink.push(recording.sample_file_uuid);
return;
}
}
/// Internal helper for `save_writer`. This is separated out so that the question-mark operator
/// can be used in the many error paths.
fn save_writer_helper(&mut self, recording: &db::RecordingToInsert, f: fs::File)
-> Result<(), Error> {
self.try_unlink();
if !self.to_unlink.is_empty() {
return Err(Error::new(format!("failed to unlink {} files.", self.to_unlink.len())));
}
f.sync_all()?;
self.dir.sync()?;
let mut to_delete = Vec::new();
let mut l = self.dir.mutable.lock().unwrap();
let mut db = self.dir.db.lock();
let mut new_next_uuid = l.next_uuid;
{
let camera =
db.cameras_by_id().get(&recording.camera_id)
.ok_or_else(|| Error::new(format!("no such camera {}", recording.camera_id)))?;
self.get_rows_to_delete(&db, recording.camera_id, camera,
recording.sample_file_bytes as i64, &mut to_delete)?;
}
let mut tx = db.tx()?;
tx.mark_sample_files_deleted(&self.to_mark_deleted)?;
tx.delete_recordings(&to_delete)?;
if new_next_uuid.is_none() {
new_next_uuid = Some(tx.reserve_sample_file()?);
}
tx.insert_recording(recording)?;
tx.commit()?;
l.next_uuid = new_next_uuid;
self.to_mark_deleted.clear();
self.to_unlink.extend(to_delete.iter().map(|row| row.uuid));
Ok(())
}
/// Gets rows to delete to bring a camera's disk usage within bounds.
fn get_rows_to_delete(&self, db: &MutexGuard<db::LockedDatabase>, camera_id: i32,
camera: &db::Camera, extra_bytes_needed: i64,
to_delete: &mut Vec<db::ListOldestSampleFilesRow>) -> Result<(), Error> {
let bytes_needed = camera.sample_file_bytes + extra_bytes_needed - camera.retain_bytes;
let mut bytes_to_delete = 0;
if bytes_needed <= 0 {
debug!("{}: have remaining quota of {}", camera.short_name, -bytes_needed);
return Ok(());
}
let mut n = 0;
db.list_oldest_sample_files(camera_id, |row| {
bytes_to_delete += row.sample_file_bytes as i64;
to_delete.push(row);
n += 1;
bytes_needed > bytes_to_delete // continue as long as more deletions are needed.
})?;
if bytes_needed > bytes_to_delete {
return Err(Error::new(format!("{}: couldn't find enough files to delete: {} left.",
camera.short_name, bytes_needed)));
}
info!("{}: deleting {} bytes in {} recordings ({} bytes needed)",
camera.short_name, bytes_to_delete, n, bytes_needed);
Ok(())
}
/// Tries to unlink all the uuids in `self.to_unlink`. Any which can't be unlinked will
/// be retained in the vec.
fn try_unlink(&mut self) {
let to_mark_deleted = &mut self.to_mark_deleted;
let fd = &self.dir.fd;
self.to_unlink.retain(|uuid| {
if let Err(e) = SampleFileDir::unlink(fd, *uuid) {
if e.kind() == io::ErrorKind::NotFound {
warn!("dir: Sample file {} already deleted!", uuid.hyphenated());
to_mark_deleted.push(*uuid);
false
} else {
warn!("dir: Unable to unlink {}: {}", uuid.hyphenated(), e);
true
}
} else {
to_mark_deleted.push(*uuid);
false
}
});
}
}

145
src/error.rs Normal file
View File

@ -0,0 +1,145 @@
// This file is part of Moonfire NVR, a security camera digital video recorder.
// Copyright (C) 2016 Scott Lamb <slamb@slamb.org>
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// In addition, as a special exception, the copyright holders give
// permission to link the code of portions of this program with the
// OpenSSL library under certain conditions as described in each
// individual source file, and distribute linked combinations including
// the two.
//
// You must obey the GNU General Public License in all respects for all
// of the code used other than OpenSSL. If you modify file(s) with this
// exception, you may extend this exception to your version of the
// file(s), but you are not obligated to do so. If you do not wish to do
// so, delete this exception statement from your version. If you delete
// this exception statement from all source files in the program, then
// also delete it here.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
extern crate rusqlite;
extern crate time;
extern crate uuid;
use core::ops::Deref;
use core::num;
use ffmpeg;
use openssl::error::ErrorStack;
use serde_json;
use std::boxed::Box;
use std::convert::From;
use std::error;
use std::fmt;
use std::io;
use std::result;
use std::string::String;
#[derive(Debug)]
pub struct Error {
pub description: String,
pub cause: Option<Box<error::Error + Send + Sync>>,
}
impl Error {
pub fn new(description: String) -> Self {
Error{description: description, cause: None }
}
// Returns a function to pass to std::result::Result::map_err which annotates the given error
// with a prefix.
pub fn annotator(prefix: &'static str) -> impl Fn(Error) -> Error {
move |e| { Error{description: format!("{}: {}", prefix, e.description), cause: e.cause} }
}
}
impl error::Error for Error {
fn description(&self) -> &str { &self.description }
fn cause(&self) -> Option<&error::Error> {
match self.cause {
Some(ref b) => Some(b.deref()),
None => None
}
}
}
impl fmt::Display for Error {
fn fmt(&self, f: &mut fmt::Formatter) -> result::Result<(), fmt::Error> {
write!(f, "Error: {}", self.description)
}
}
// TODO(slamb): isn't there a "<? implements error::Error>" or some such?
impl From<rusqlite::Error> for Error {
fn from(err: rusqlite::Error) -> Self {
use std::error::{Error as E};
Error{description: String::from(err.description()),
cause: Some(Box::new(err))}
}
}
impl From<io::Error> for Error {
fn from(err: io::Error) -> Self {
use std::error::{Error as E};
Error{description: String::from(err.description()),
cause: Some(Box::new(err))}
}
}
impl From<time::ParseError> for Error {
fn from(err: time::ParseError) -> Self {
use std::error::{Error as E};
Error{description: String::from(err.description()),
cause: Some(Box::new(err))}
}
}
impl From<num::ParseIntError> for Error {
fn from(err: num::ParseIntError) -> Self {
use std::error::{Error as E};
Error{description: err.description().to_owned(),
cause: Some(Box::new(err))}
}
}
impl From<serde_json::Error> for Error {
fn from(err: serde_json::Error) -> Self {
use std::error::{Error as E};
Error{description: format!("{} ({})", err.description(), err),
cause: Some(Box::new(err))}
}
}
impl From<ffmpeg::Error> for Error {
fn from(err: ffmpeg::Error) -> Self {
use std::error::{Error as E};
Error{description: format!("{} ({})", err.description(), err),
cause: Some(Box::new(err))}
}
}
impl From<uuid::ParseError> for Error {
fn from(_: uuid::ParseError) -> Self {
Error{description: String::from("UUID parse error"),
cause: None}
}
}
impl From<ErrorStack> for Error {
fn from(_: ErrorStack) -> Self {
Error{description: String::from("openssl error"), cause: None}
}
}
pub type Result<T> = result::Result<T, Error>;

316
src/ffmpeg.cc
View File

@ -1,316 +0,0 @@
// This file is part of Moonfire NVR, a security camera digital video recorder.
// Copyright (C) 2016 Scott Lamb <slamb@slamb.org>
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// In addition, as a special exception, the copyright holders give
// permission to link the code of portions of this program with the
// OpenSSL library under certain conditions as described in each
// individual source file, and distribute linked combinations including
// the two.
//
// You must obey the GNU General Public License in all respects for all
// of the code used other than OpenSSL. If you modify file(s) with this
// exception, you may extend this exception to your version of the
// file(s), but you are not obligated to do so. If you do not wish to do
// so, delete this exception statement from your version. If you delete
// this exception statement from all source files in the program, then
// also delete it here.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
//
// ffmpeg.cc: See ffmpeg.h for description.
#include "ffmpeg.h"
#include <mutex>
extern "C" {
#include <libavutil/buffer.h>
#include <libavutil/mathematics.h>
#include <libavutil/version.h>
#include <libavcodec/avcodec.h>
#include <libavcodec/version.h>
#include <libavformat/version.h>
} // extern "C"
#include <gflags/gflags.h>
#include "string.h"
// libav lacks this ffmpeg constant.
#ifndef AV_ERROR_MAX_STRING_SIZE
#define AV_ERROR_MAX_STRING_SIZE 64
#endif
DEFINE_int32(avlevel, AV_LOG_INFO,
"maximum logging level for ffmpeg/libav; "
"higher levels will be ignored.");
namespace moonfire_nvr {
namespace {
std::string AvError2Str(re2::StringPiece function, int err) {
char str[AV_ERROR_MAX_STRING_SIZE];
if (av_strerror(err, str, sizeof(str)) == 0) {
return StrCat(function, ": ", str);
}
return StrCat(function, ": unknown error ", err);
}
struct Dictionary {
Dictionary() {}
Dictionary(const Dictionary &) = delete;
Dictionary &operator=(const Dictionary &) = delete;
~Dictionary() { av_dict_free(&dict); }
bool Set(const char *key, const char *value, std::string *error_message) {
int ret = av_dict_set(&dict, key, value, 0);
if (ret < 0) {
*error_message = AvError2Str("av_dict_set", ret);
return false;
}
return true;
}
bool size() const { return av_dict_count(dict); }
AVDictionary *dict = nullptr;
};
google::LogSeverity GlogLevelFromAvLevel(int avlevel) {
if (avlevel >= AV_LOG_INFO) {
return google::GLOG_INFO;
} else if (avlevel >= AV_LOG_WARNING) {
return google::GLOG_WARNING;
} else if (avlevel > AV_LOG_PANIC) {
return google::GLOG_ERROR;
} else {
return google::GLOG_FATAL;
}
}
void AvLogCallback(void *avcl, int avlevel, const char *fmt, va_list vl) {
if (avlevel > FLAGS_avlevel) {
return;
}
// google::LogMessage expects a "file" and "line" to be prefixed to the
// log message, like so:
//
// W1210 11:00:32.224936 28739 ffmpeg_rtsp:0] Estimating duration ...
// ^file ^line
//
// Normally this is filled in via the __FILE__ and __LINE__
// C preprocessor macros. In this case, try to fill in something useful
// based on the information ffmpeg supplies.
std::string file("ffmpeg");
if (avcl != nullptr) {
auto *avclass = *reinterpret_cast<AVClass **>(avcl);
file.push_back('_');
file.append(avclass->item_name(avcl));
}
char line[512];
vsnprintf(line, sizeof(line), fmt, vl);
google::LogSeverity glog_level = GlogLevelFromAvLevel(avlevel);
google::LogMessage(file.c_str(), 0, glog_level).stream() << line;
}
int AvLockCallback(void **mutex, enum AVLockOp op) {
auto typed_mutex = reinterpret_cast<std::mutex **>(mutex);
switch (op) {
case AV_LOCK_CREATE:
LOG_IF(DFATAL, *typed_mutex != nullptr)
<< "creating mutex over existing value.";
*typed_mutex = new std::mutex;
break;
case AV_LOCK_DESTROY:
delete *typed_mutex;
*typed_mutex = nullptr;
break;
case AV_LOCK_OBTAIN:
(*typed_mutex)->lock();
break;
case AV_LOCK_RELEASE:
(*typed_mutex)->unlock();
break;
}
return 0;
}
std::string StringifyVersion(int version_int) {
return StrCat((version_int >> 16) & 0xFF, ".", (version_int >> 8) & 0xFF, ".",
(version_int)&0xFF);
}
void LogVersion(const char *library_name, int compiled_version,
int running_version, const char *configuration) {
LOG(INFO) << library_name << ": compiled with version "
<< StringifyVersion(compiled_version) << ", running with version "
<< StringifyVersion(running_version)
<< ", configuration: " << configuration;
}
class RealInputVideoPacketStream : public InputVideoPacketStream {
public:
RealInputVideoPacketStream() {
ctx_ = CHECK_NOTNULL(avformat_alloc_context());
}
RealInputVideoPacketStream(const RealInputVideoPacketStream &) = delete;
RealInputVideoPacketStream &operator=(const RealInputVideoPacketStream &) =
delete;
~RealInputVideoPacketStream() final {
avformat_close_input(&ctx_);
avformat_free_context(ctx_);
}
bool GetNext(VideoPacket *pkt, std::string *error_message) final {
while (true) {
av_packet_unref(pkt->pkt());
int ret = av_read_frame(ctx_, pkt->pkt());
if (ret != 0) {
if (ret == AVERROR_EOF) {
error_message->clear();
} else {
*error_message = AvError2Str("av_read_frame", ret);
}
return false;
}
if (pkt->pkt()->stream_index != stream_index_) {
VLOG(3) << "Ignoring packet for stream " << pkt->pkt()->stream_index
<< "; only interested in " << stream_index_;
continue;
}
VLOG(3) << "Read packet with pts=" << pkt->pkt()->pts
<< ", dts=" << pkt->pkt()->dts << ", key=" << pkt->is_key();
return true;
}
}
const AVStream *stream() const final { return ctx_->streams[stream_index_]; }
private:
friend class RealVideoSource;
int64_t min_next_pts_ = std::numeric_limits<int64_t>::min();
int64_t min_next_dts_ = std::numeric_limits<int64_t>::min();
AVFormatContext *ctx_ = nullptr; // owned.
int stream_index_ = -1;
};
class RealVideoSource : public VideoSource {
public:
RealVideoSource() {
CHECK_GE(0, av_lockmgr_register(&AvLockCallback));
av_log_set_callback(&AvLogCallback);
av_register_all();
avformat_network_init();
LogVersion("avutil", LIBAVUTIL_VERSION_INT, avutil_version(),
avutil_configuration());
LogVersion("avformat", LIBAVFORMAT_VERSION_INT, avformat_version(),
avformat_configuration());
LogVersion("avcodec", LIBAVCODEC_VERSION_INT, avcodec_version(),
avcodec_configuration());
}
std::unique_ptr<InputVideoPacketStream> OpenRtsp(
const std::string &url, std::string *error_message) final {
std::unique_ptr<InputVideoPacketStream> stream;
Dictionary open_options;
if (!open_options.Set("rtsp_transport", "tcp", error_message) ||
// https://trac.ffmpeg.org/ticket/5018 workaround attempt.
!open_options.Set("probesize", "262144", error_message) ||
!open_options.Set("user-agent", "moonfire-nvr", error_message) ||
// 10-second socket timeout, in microseconds.
!open_options.Set("stimeout", "10000000", error_message)) {
return stream;
}
stream = OpenCommon(url, &open_options.dict, error_message);
if (stream == nullptr) {
return stream;
}
// Discard the first packet.
LOG(INFO) << "Discarding the first packet to work around "
"https://trac.ffmpeg.org/ticket/5018";
VideoPacket dummy;
if (!stream->GetNext(&dummy, error_message)) {
stream.reset();
}
return stream;
}
std::unique_ptr<InputVideoPacketStream> OpenFile(
const std::string &filename, std::string *error_message) final {
AVDictionary *open_options = nullptr;
return OpenCommon(filename, &open_options, error_message);
}
private:
std::unique_ptr<InputVideoPacketStream> OpenCommon(
const std::string &source, AVDictionary **dict,
std::string *error_message) {
std::unique_ptr<RealInputVideoPacketStream> stream(
new RealInputVideoPacketStream);
int ret = avformat_open_input(&stream->ctx_, source.c_str(), nullptr, dict);
if (ret != 0) {
*error_message = AvError2Str("avformat_open_input", ret);
return std::unique_ptr<InputVideoPacketStream>();
}
if (av_dict_count(*dict) != 0) {
std::vector<std::string> ignored;
AVDictionaryEntry *ent = nullptr;
while ((ent = av_dict_get(*dict, "", ent, AV_DICT_IGNORE_SUFFIX)) !=
nullptr) {
ignored.push_back(StrCat(ent->key, "=", ent->value));
}
LOG(WARNING) << "avformat_open_input ignored " << ignored.size()
<< " options: " << Join(ignored, ", ");
}
ret = avformat_find_stream_info(stream->ctx_, nullptr);
if (ret < 0) {
*error_message = AvError2Str("avformat_find_stream_info", ret);
return std::unique_ptr<InputVideoPacketStream>();
}
// Find the video stream.
for (unsigned int i = 0; i < stream->ctx_->nb_streams; ++i) {
if (stream->ctx_->streams[i]->codec->codec_type == AVMEDIA_TYPE_VIDEO) {
VLOG(1) << "Video stream index is " << i;
stream->stream_index_ = i;
break;
}
}
if (stream->stream() == nullptr) {
*error_message = StrCat("no video stream");
return std::unique_ptr<InputVideoPacketStream>();
}
return std::unique_ptr<InputVideoPacketStream>(stream.release());
}
};
} // namespace
VideoSource *GetRealVideoSource() {
static auto *real_video_source = new RealVideoSource; // never deleted.
return real_video_source;
}
} // namespace moonfire_nvr

149
src/ffmpeg.h
View File

@ -1,149 +0,0 @@
// This file is part of Moonfire NVR, a security camera digital video recorder.
// Copyright (C) 2016 Scott Lamb <slamb@slamb.org>
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// In addition, as a special exception, the copyright holders give
// permission to link the code of portions of this program with the
// OpenSSL library under certain conditions as described in each
// individual source file, and distribute linked combinations including
// the two.
//
// You must obey the GNU General Public License in all respects for all
// of the code used other than OpenSSL. If you modify file(s) with this
// exception, you may extend this exception to your version of the
// file(s), but you are not obligated to do so. If you do not wish to do
// so, delete this exception statement from your version. If you delete
// this exception statement from all source files in the program, then
// also delete it here.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
//
// ffmpeg.h: ffmpeg (or libav) wrappers for operations needed by moonfire_nvr.
// This is not a general-purpose wrapper. It makes assumptions about the
// data we will be operated on and the desired operations, such as:
//
// * The input should contain no "B" frames (bi-directionally predicted
// pictures) and thus input frames should be strictly in order of ascending
// PTS as well as DTS.
//
// * Only video frames are of interest.
#ifndef MOONFIRE_NVR_FFMPEG_H
#define MOONFIRE_NVR_FFMPEG_H
#include <limits>
#include <memory>
#include <string>
#include <glog/logging.h>
#include <re2/stringpiece.h>
extern "C" {
#include <libavformat/avformat.h>
} // extern "C"
namespace moonfire_nvr {
// An encoded video packet.
class VideoPacket {
public:
VideoPacket() { av_init_packet(&pkt_); }
VideoPacket(const VideoPacket &) = delete;
VideoPacket &operator=(const VideoPacket &) = delete;
~VideoPacket() { av_packet_unref(&pkt_); }
// Returns iff this packet represents a key frame.
//
// (A key frame is one that can be decoded without previous frames.)
//
// PRE: this packet is valid, as if it has been filled by
// InputVideoPacketStream::Next.
bool is_key() const { return (pkt_.flags & AV_PKT_FLAG_KEY) != 0; }
int64_t pts() const { return pkt_.pts; }
AVPacket *pkt() { return &pkt_; }
const AVPacket *pkt() const { return &pkt_; }
re2::StringPiece data() {
return re2::StringPiece(reinterpret_cast<const char *>(pkt_.data),
pkt_.size);
}
private:
AVPacket pkt_;
};
// An input stream of (still-encoded) video packets.
class InputVideoPacketStream {
public:
InputVideoPacketStream() {}
InputVideoPacketStream(const InputVideoPacketStream &) = delete;
InputVideoPacketStream &operator=(const InputVideoPacketStream &) = delete;
// Closes the stream.
virtual ~InputVideoPacketStream() {}
// Get the next packet.
//
// Returns true iff one is available, false on EOF or failure.
// |error_message| will be filled on failure, empty on EOF.
//
// PRE: the stream is healthy: there was no prior Close() call or GetNext()
// failure.
virtual bool GetNext(VideoPacket *pkt, std::string *error_message) = 0;
// Returns the video stream.
virtual const AVStream *stream() const = 0;
re2::StringPiece extradata() const {
return re2::StringPiece(
reinterpret_cast<const char *>(stream()->codec->extradata),
stream()->codec->extradata_size);
}
};
// A class which opens streams.
// There's one of these for proudction use; see GetRealVideoSource().
// It's an abstract class for testability.
class VideoSource {
public:
virtual ~VideoSource() {}
// Open the given RTSP URL, accessing the first video stream.
//
// The RTSP URL will be opened with TCP and a hardcoded socket timeout.
//
// The first frame will be automatically discarded as a bug workaround.
// https://trac.ffmpeg.org/ticket/5018
//
// Returns success, filling |error_message| on failure.
//
// PRE: closed.
virtual std::unique_ptr<InputVideoPacketStream> OpenRtsp(
const std::string &url, std::string *error_message) = 0;
// Open the given video file, accessing the first video stream.
//
// Returns the stream. On failure, returns nullptr and fills
// |error_message|.
virtual std::unique_ptr<InputVideoPacketStream> OpenFile(
const std::string &filename, std::string *error_message) = 0;
};
// Returns a VideoSource for production use, which will never be deleted.
VideoSource *GetRealVideoSource();
} // namespace moonfire_nvr
#endif // MOONFIRE_NVR_FFMPEG_H

222
src/filesystem.cc
View File

@ -1,222 +0,0 @@
// This file is part of Moonfire NVR, a security camera digital video recorder.
// Copyright (C) 2016 Scott Lamb <slamb@slamb.org>
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// In addition, as a special exception, the copyright holders give
// permission to link the code of portions of this program with the
// OpenSSL library under certain conditions as described in each
// individual source file, and distribute linked combinations including
// the two.
//
// You must obey the GNU General Public License in all respects for all
// of the code used other than OpenSSL. If you modify file(s) with this
// exception, you may extend this exception to your version of the
// file(s), but you are not obligated to do so. If you do not wish to do
// so, delete this exception statement from your version. If you delete
// this exception statement from all source files in the program, then
// also delete it here.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
//
// filesystem.cc: See filesystem.h.
#include "filesystem.h"
#include <errno.h>
#include <fcntl.h>
#include <stdlib.h>
#include <string.h>
#include <sys/file.h>
#include <sys/queue.h>
#include <sys/stat.h>
#include <sys/types.h>
#include <unistd.h>
#include <cstdlib>
#include <memory>
#include <event2/buffer.h>
#include <event2/event.h>
#include <event2/keyvalq_struct.h>
#include <event2/http.h>
#include <gperftools/profiler.h>
#include <glog/logging.h>
#include "string.h"
namespace moonfire_nvr {
namespace {
class RealFile : public File {
public:
RealFile(re2::StringPiece name, int fd)
: name_(name.data(), name.size()), fd_(fd) {}
RealFile(const RealFile &) = delete;
void operator=(const RealFile &) = delete;
~RealFile() final { Close(); }
const std::string &name() const { return name_; }
int Access(const char *path, int mode, int flags) final {
return faccessat(fd_, path, mode, flags) < 0 ? errno : 0;
}
int Close() final {
if (fd_ < 0) {
return 0;
}
int ret;
while ((ret = close(fd_)) != 0 && errno == EINTR)
;
if (ret != 0) {
return errno;
}
fd_ = -1;
return 0;
}
int Lock(int operation) final {
return (flock(fd_, operation) < 0) ? errno : 0;
}
int Open(const char *path, int flags, int *fd) final {
return Open(path, flags, 0, fd);
}
int Open(const char *path, int flags, std::unique_ptr<File> *f) final {
return Open(path, flags, 0, f);
}
int Open(const char *path, int flags, mode_t mode, int *fd) final {
int ret = openat(fd_, path, flags, mode);
if (ret < 0) {
return errno;
}
*fd = ret;
return 0;
}
int Open(const char *path, int flags, mode_t mode,
std::unique_ptr<File> *f) final {
int ret = openat(fd_, path, flags, mode);
if (ret < 0) {
return errno;
}
f->reset(new RealFile(StrCat(name_, "/", path), ret));
return 0;
}
int Read(void *buf, size_t size, size_t *bytes_read) final {
ssize_t ret;
while ((ret = read(fd_, buf, size)) == -1 && errno == EINTR)
;
if (ret < 0) {
return errno;
}
*bytes_read = static_cast<size_t>(ret);
return 0;
}
int Stat(struct stat *buf) final { return (fstat(fd_, buf) < 0) ? errno : 0; }
int Sync() final { return (fsync(fd_) < 0) ? errno : 0; }
int Truncate(off_t length) final {
return (ftruncate(fd_, length) < 0) ? errno : 0;
}
int Unlink(const char *pathname) {
return unlinkat(fd_, pathname, 0) != 0 ? errno : 0;
}
int Write(re2::StringPiece data, size_t *bytes_written) final {
ssize_t ret;
while ((ret = write(fd_, data.data(), data.size())) == -1 && errno == EINTR)
;
if (ret < 0) {
return errno;
}
*bytes_written = static_cast<size_t>(ret);
return 0;
}
private:
std::string name_;
int fd_ = -1;
};
class RealFilesystem : public Filesystem {
public:
bool DirForEach(const char *dir_path,
std::function<IterationControl(const dirent *)> fn,
std::string *error_message) final {
DIR *owned_dir = opendir(dir_path);
if (owned_dir == nullptr) {
int err = errno;
*error_message =
StrCat("Unable to examine ", dir_path, ": ", strerror(err));
return false;
}
struct dirent *ent;
while (errno = 0, (ent = readdir(owned_dir)) != nullptr) {
if (fn(ent) == IterationControl::kBreak) {
closedir(owned_dir);
return true;
}
}
int err = errno;
closedir(owned_dir);
if (err != 0) {
*error_message = StrCat("readdir failed: ", strerror(err));
return false;
}
return true;
}
int Open(const char *path, int flags, std::unique_ptr<File> *f) final {
return Open(path, flags, 0, f);
}
int Open(const char *path, int flags, mode_t mode,
std::unique_ptr<File> *f) final {
int ret = open(path, flags, mode);
if (ret < 0) {
return errno;
}
f->reset(new RealFile(path, ret));
return 0;
}
int Mkdir(const char *path, mode_t mode) final {
return (mkdir(path, mode) < 0) ? errno : 0;
}
int Rmdir(const char *path) final { return (rmdir(path) < 0) ? errno : 0; }
int Stat(const char *path, struct stat *buf) final {
return (stat(path, buf) < 0) ? errno : 0;
}
int Unlink(const char *path) final { return (unlink(path) < 0) ? errno : 0; }
};
} // namespace
Filesystem *GetRealFilesystem() {
static Filesystem *real_filesystem = new RealFilesystem;
return real_filesystem;
}
} // namespace moonfire_nvr

140
src/filesystem.h
View File

@ -1,140 +0,0 @@
// This file is part of Moonfire NVR, a security camera digital video recorder.
// Copyright (C) 2016 Scott Lamb <slamb@slamb.org>
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// In addition, as a special exception, the copyright holders give
// permission to link the code of portions of this program with the
// OpenSSL library under certain conditions as described in each
// individual source file, and distribute linked combinations including
// the two.
//
// You must obey the GNU General Public License in all respects for all
// of the code used other than OpenSSL. If you modify file(s) with this
// exception, you may extend this exception to your version of the
// file(s), but you are not obligated to do so. If you do not wish to do
// so, delete this exception statement from your version. If you delete
// this exception statement from all source files in the program, then
// also delete it here.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
//
// filesystem.h: helpers for dealing with the local filesystem.
#ifndef MOONFIRE_NVR_FILESYSTEM_H
#define MOONFIRE_NVR_FILESYSTEM_H
#include <dirent.h>
#include <stdarg.h>
#include <sys/stat.h>
#include <sys/types.h>
#include <memory>
#include <functional>
#include <string>
#include <event2/buffer.h>
#include <event2/http.h>
#include <glog/logging.h>
#include <re2/stringpiece.h>
#include "common.h"
namespace moonfire_nvr {
// Represents an open file descriptor. All methods but Close() are thread-safe.
class File {
public:
// Close the file, ignoring the result.
virtual ~File() {}
// A name for the file (typically assigned at open time).
virtual const std::string &name() const = 0;
// faccessat(), returning 0 on success or errno>0 on failure.
virtual int Access(const char *path, int mode, int flags) = 0;
// Close the file, returning 0 on success or errno>0 on failure.
// Already closed is considered a success.
virtual int Close() = 0;
// flock(), returning 0 on success or errno>0 on failure.
virtual int Lock(int operation) = 0;
// openat(), returning 0 on success or errno>0 on failure.
virtual int Open(const char *path, int flags, int *fd) = 0;
virtual int Open(const char *path, int flags, std::unique_ptr<File> *f) = 0;
virtual int Open(const char *path, int flags, mode_t mode, int *fd) = 0;
virtual int Open(const char *path, int flags, mode_t mode,
std::unique_ptr<File> *f) = 0;
// read(), returning 0 on success or errno>0 on failure.
// On success, |bytes_read| will be updated.
virtual int Read(void *buf, size_t count, size_t *bytes_read) = 0;
// fstat(), returning 0 on success or errno>0 on failure.
virtual int Stat(struct stat *buf) = 0;
// fsync(), returning 0 on success or errno>0 on failure.
virtual int Sync() = 0;
// ftruncate(), returning 0 on success or errno>0 on failure.
virtual int Truncate(off_t length) = 0;
// unlink() the specified file, returning 0 on success or errno>0 on failure.
virtual int Unlink(const char *path) = 0;
// Write to the file, returning 0 on success or errno>0 on failure.
// On success, |bytes_written| will be updated.
virtual int Write(re2::StringPiece data, size_t *bytes_written) = 0;
};
// Interface to the local filesystem. There's typically one per program,
// but it's an abstract class for testability. Thread-safe.
class Filesystem {
public:
virtual ~Filesystem() {}
// Execute |fn| for each directory entry in |dir_path|, stopping early
// (successfully) if the callback returns IterationControl::kBreak.
//
// On success, returns true.
// On failure, returns false and updates |error_msg|.
virtual bool DirForEach(const char *dir_path,
std::function<IterationControl(const dirent *)> fn,
std::string *error_msg) = 0;
// open() the specified path, returning 0 on success or errno>0 on failure.
// On success, |f| is populated with an open file.
virtual int Open(const char *path, int flags, std::unique_ptr<File> *f) = 0;
virtual int Open(const char *path, int flags, mode_t mode,
std::unique_ptr<File> *f) = 0;
// mkdir() the specified path, returning 0 on success or errno>0 on failure.
virtual int Mkdir(const char *path, mode_t mode) = 0;
// rmdir() the specified path, returning 0 on success or errno>0 on failure.
virtual int Rmdir(const char *path) = 0;
// stat() the specified path, returning 0 on success or errno>0 on failure.
virtual int Stat(const char *path, struct stat *buf) = 0;
// unlink() the specified file, returning 0 on success or errno>0 on failure.
virtual int Unlink(const char *path) = 0;
};
// Get the (singleton) real filesystem, which is never deleted.
Filesystem *GetRealFilesystem();
} // namespace moonfire_nvr
#endif // MOONFIRE_NVR_FILESYSTEM_H

153
src/h264-test.cc
View File

@ -1,153 +0,0 @@
// This file is part of Moonfire NVR, a security camera network video recorder.
// Copyright (C) 2016 Scott Lamb <slamb@slamb.org>
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// In addition, as a special exception, the copyright holders give
// permission to link the code of portions of this program with the
// OpenSSL library under certain conditions as described in each
// individual source file, and distribute linked combinations including
// the two.
//
// You must obey the GNU General Public License in all respects for all
// of the code used other than OpenSSL. If you modify file(s) with this
// exception, you may extend this exception to your version of the
// file(s), but you are not obligated to do so. If you do not wish to do
// so, delete this exception statement from your version. If you delete
// this exception statement from all source files in the program, then
// also delete it here.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
//
// h264-test.cc: tests of the h264.h interface.
#include <gflags/gflags.h>
#include <glog/logging.h>
#include <gmock/gmock.h>
#include <gtest/gtest.h>
#include "h264.h"
#include "string.h"
DECLARE_bool(alsologtostderr);
namespace moonfire_nvr {
namespace {
const uint8_t kAnnexBTestInput[] = {
0x00, 0x00, 0x00, 0x01, 0x67, 0x4d, 0x00, 0x1f, 0x9a, 0x66, 0x02, 0x80,
0x2d, 0xff, 0x35, 0x01, 0x01, 0x01, 0x40, 0x00, 0x00, 0xfa, 0x00, 0x00,
0x1d, 0x4c, 0x01, 0x00, 0x00, 0x00, 0x01, 0x68, 0xee, 0x3c, 0x80};
const uint8_t kAvcDecoderConfigTestInput[] = {
0x01, 0x4d, 0x00, 0x1f, 0xff, 0xe1, 0x00, 0x17, 0x67, 0x4d,
0x00, 0x1f, 0x9a, 0x66, 0x02, 0x80, 0x2d, 0xff, 0x35, 0x01,
0x01, 0x01, 0x40, 0x00, 0x00, 0xfa, 0x00, 0x00, 0x1d, 0x4c,
0x01, 0x01, 0x00, 0x04, 0x68, 0xee, 0x3c, 0x80};
const char kTestOutput[] =
"00 00 00 84 61 76 63 31 00 00 00 00 00 00 00 01 "
"00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 "
"05 00 02 d0 00 48 00 00 00 48 00 00 00 00 00 00 "
"00 01 00 00 00 00 00 00 00 00 00 00 00 00 00 00 "
"00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 "
"00 00 00 18 ff ff 00 00 00 2e 61 76 63 43 01 4d "
"00 1f ff e1 00 17 67 4d 00 1f 9a 66 02 80 2d ff "
"35 01 01 01 40 00 00 fa 00 00 1d 4c 01 01 00 04 "
"68 ee 3c 80";
TEST(H264Test, DecodeOnly) {
std::vector<std::string> nal_units_hexed;
re2::StringPiece test_input(reinterpret_cast<const char *>(kAnnexBTestInput),
sizeof(kAnnexBTestInput));
internal::NalUnitFunction fn = [&nal_units_hexed](re2::StringPiece nal_unit) {
nal_units_hexed.push_back(ToHex(nal_unit, true));
return IterationControl::kContinue;
};
std::string error_message;
ASSERT_TRUE(internal::DecodeH264AnnexB(test_input, fn, &error_message))
<< error_message;
EXPECT_THAT(nal_units_hexed,
testing::ElementsAre("67 4d 00 1f 9a 66 02 80 2d ff 35 01 01 01 "
"40 00 00 fa 00 00 1d 4c 01",
"68 ee 3c 80"));
}
TEST(H264Test, SampleEntryFromAnnexBExtraData) {
re2::StringPiece test_input(reinterpret_cast<const char *>(kAnnexBTestInput),
sizeof(kAnnexBTestInput));
std::string sample_entry;
std::string error_message;
bool need_transform;
ASSERT_TRUE(ParseExtraData(test_input, 1280, 720, &sample_entry,
&need_transform, &error_message))
<< error_message;
EXPECT_EQ(kTestOutput, ToHex(sample_entry, true));
EXPECT_TRUE(need_transform);
}
TEST(H264Test, SampleEntryFromAvcDecoderConfigExtraData) {
re2::StringPiece test_input(
reinterpret_cast<const char *>(kAvcDecoderConfigTestInput),
sizeof(kAvcDecoderConfigTestInput));
std::string sample_entry;
std::string error_message;
bool need_transform;
ASSERT_TRUE(ParseExtraData(test_input, 1280, 720, &sample_entry,
&need_transform, &error_message))
<< error_message;
EXPECT_EQ(kTestOutput, ToHex(sample_entry, true));
EXPECT_FALSE(need_transform);
}
TEST(H264Test, TransformSampleEntry) {
const uint8_t kInput[] = {
0x00, 0x00, 0x00, 0x01, 0x67, 0x4d, 0x00, 0x1f, 0x9a, 0x66,
0x02, 0x80, 0x2d, 0xff, 0x35, 0x01, 0x01, 0x01, 0x40, 0x00,
0x00, 0xfa, 0x00, 0x00, 0x1d, 0x4c, 0x01,
0x00, 0x00, 0x00, 0x01, 0x68, 0xee, 0x3c, 0x80,
0x00, 0x00, 0x00, 0x01, 0x06, 0x06, 0x01, 0xc4, 0x80,
0x00, 0x00, 0x00, 0x01, 0x65, 0x88, 0x80, 0x10, 0x00, 0x08,
0x7f, 0x00, 0x5d, 0x27, 0xb5, 0xc1, 0xff, 0x8c, 0xd6, 0x35,
// (truncated)
};
const char kExpectedOutput[] =
"00 00 00 17 "
"67 4d 00 1f 9a 66 02 80 2d ff 35 01 01 01 40 00 00 fa 00 00 1d 4c 01 "
"00 00 00 04 68 ee 3c 80 "
"00 00 00 05 06 06 01 c4 80 "
"00 00 00 10 "
"65 88 80 10 00 08 7f 00 5d 27 b5 c1 ff 8c d6 35";
re2::StringPiece input(reinterpret_cast<const char *>(kInput),
sizeof(kInput));
std::string out;
std::string error_message;
ASSERT_TRUE(TransformSampleData(input, &out, &error_message))
<< error_message;
EXPECT_EQ(kExpectedOutput, ToHex(out, true));
}
} // namespace
} // namespace moonfire_nvr
int main(int argc, char **argv) {
FLAGS_alsologtostderr = true;
google::ParseCommandLineFlags(&argc, &argv, true);
testing::InitGoogleTest(&argc, argv);
google::InitGoogleLogging(argv[0]);
return RUN_ALL_TESTS();
}

251
src/h264.cc
View File

@ -1,251 +0,0 @@
// This file is part of Moonfire NVR, a security camera network video recorder.
// Copyright (C) 2016 Scott Lamb <slamb@slamb.org>
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// In addition, as a special exception, the copyright holders give
// permission to link the code of portions of this program with the
// OpenSSL library under certain conditions as described in each
// individual source file, and distribute linked combinations including
// the two.
//
// You must obey the GNU General Public License in all respects for all
// of the code used other than OpenSSL. If you modify file(s) with this
// exception, you may extend this exception to your version of the
// file(s), but you are not obligated to do so. If you do not wish to do
// so, delete this exception statement from your version. If you delete
// this exception statement from all source files in the program, then
// also delete it here.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
//
// h264.cc: see h264.h.
#include "h264.h"
#include <re2/re2.h>
#include "coding.h"
#include "string.h"
namespace moonfire_nvr {
namespace {
// See ISO/IEC 14496-10 table 7-1 - NAL unit type codes, syntax element
// categories, and NAL unit type classes.
const int kNalUnitSeqParameterSet = 7;
const int kNalUnitPicParameterSet = 8;
const uint8_t kNalUnitTypeMask = 0x1F; // bottom 5 bits of first byte of unit.
// Parse sequence parameter set and picture parameter set from ffmpeg's
// "extra_data".
bool ParseAnnexBExtraData(re2::StringPiece extradata, re2::StringPiece *sps,
re2::StringPiece *pps, std::string *error_message) {
bool ok = true;
internal::NalUnitFunction fn = [&](re2::StringPiece nal_unit) {
// See ISO/IEC 14496-10 section 7.3.1, which defines nal_unit.
uint8_t nal_type = nal_unit[0] & kNalUnitTypeMask;
switch (nal_type) {
case kNalUnitSeqParameterSet:
*sps = nal_unit;
break;
case kNalUnitPicParameterSet:
*pps = nal_unit;
break;
default:
*error_message =
StrCat("Expected only SPS and PPS; got type ", nal_type);
ok = false;
return IterationControl::kBreak;
}
return IterationControl::kContinue;
};
if (!internal::DecodeH264AnnexB(extradata, fn, error_message) || !ok) {
return false;
}
if (sps->empty() || pps->empty()) {
*error_message = "SPS and PPS must be specified.";
return false;
}
return true;
}
} // namespace
namespace internal {
// See ISO/IEC 14496-10 section B.2: Byte stream NAL unit decoding process.
// This is a relatively simple, unoptimized implementation.
bool DecodeH264AnnexB(re2::StringPiece data, NalUnitFunction process_nal_unit,
std::string *error_message) {
static const RE2 kStartCode("(\\x00{2,}\\x01)");
if (!RE2::Consume(&data, kStartCode)) {
*error_message = StrCat("stream does not start with Annex B start code: ",
ToHex(data, true));
return false;
}
while (!data.empty()) {
// Now at the start of a NAL unit. Find the end.
re2::StringPiece next_start;
re2::StringPiece this_nal = data;
if (RE2::FindAndConsume(&data, kStartCode, &next_start)) {
// It ends where another start code is found.
this_nal = re2::StringPiece(this_nal.data(),
next_start.data() - this_nal.data());
} else {
// It ends at the end of |data|. |this_nal| is already correct.
// Set |data| to be empty so the while loop exits after this iteration.
data = re2::StringPiece();
}
if (this_nal.empty()) {
*error_message = "NAL unit can't be empty";
return false;
}
if (process_nal_unit(this_nal) == IterationControl::kBreak) {
break;
}
}
return true;
}
} // namespace internal
bool ParseExtraData(re2::StringPiece extradata, uint16_t width, uint16_t height,
std::string *sample_entry, bool *need_transform,
std::string *error_message) {
uint32_t avcc_len;
re2::StringPiece sps;
re2::StringPiece pps;
if (extradata.starts_with(re2::StringPiece("\x00\x00\x00\x01", 4)) ||
extradata.starts_with(re2::StringPiece("\x00\x00\x01", 3))) {
// ffmpeg supplied "extradata" in Annex B format.
if (!ParseAnnexBExtraData(extradata, &sps, &pps, error_message)) {
return false;
}
// This magic value is checked at the end.
avcc_len = 19 + sps.size() + pps.size();
*need_transform = true;
} else {
// Assume "extradata" holds an AVCDecoderConfiguration.
avcc_len = 8 + extradata.size();
*need_transform = false;
}
// This magic value is also checked at the end.
uint32_t avc1_len = 86 + avcc_len;
sample_entry->clear();
sample_entry->reserve(avc1_len);
// This is a concatenation of the following boxes/classes.
// SampleEntry, ISO/IEC 14496-10 section 8.5.2.
uint32_t avc1_len_pos = sample_entry->size();
AppendU32(avc1_len, sample_entry); // length
sample_entry->append("avc1"); // type
sample_entry->append(6, '\x00'); // reserved
AppendU16(1, sample_entry); // data_reference_index = 1
// VisualSampleEntry, ISO/IEC 14496-12 section 12.1.3.
sample_entry->append(16, '\x00'); // pre_defined + reserved
AppendU16(width, sample_entry);
AppendU16(height, sample_entry);
AppendU32(UINT32_C(0x00480000), sample_entry); // horizresolution
AppendU32(UINT32_C(0x00480000), sample_entry); // vertresolution
AppendU32(0, sample_entry); // reserved
AppendU16(1, sample_entry); // frame count
sample_entry->append(32, '\x00'); // compressorname
AppendU16(0x0018, sample_entry); // depth
Append16(-1, sample_entry); // pre_defined
// AVCSampleEntry, ISO/IEC 14496-15 section 5.3.4.1.
// AVCConfigurationBox, ISO/IEC 14496-15 section 5.3.4.1.
uint32_t avcc_len_pos = sample_entry->size();
AppendU32(avcc_len, sample_entry); // length
sample_entry->append("avcC"); // type
if (!sps.empty() && !pps.empty()) {
// Create the AVCDecoderConfiguration, ISO/IEC 14496-15 section 5.2.4.1.
// The beginning of the AVCDecoderConfiguration takes a few values from
// the SPS (ISO/IEC 14496-10 section 7.3.2.1.1). One caveat: that section
// defines the syntax in terms of RBSP, not NAL. The difference is the
// escaping of 00 00 01 and 00 00 02; see notes about
// "emulation_prevention_three_byte" in ISO/IEC 14496-10 section 7.4.
// It looks like 00 is not a valid value of profile_idc, so this distinction
// shouldn't be relevant here. And ffmpeg seems to ignore it.
sample_entry->push_back(1); // configurationVersion
sample_entry->push_back(sps[1]); // profile_idc -> AVCProfileIndication
sample_entry->push_back(
sps[2]); // ...misc bits... -> profile_compatibility
sample_entry->push_back(sps[3]); // level_idc -> AVCLevelIndication
// Hardcode lengthSizeMinusOne to 3, matching TransformSampleData's 4-byte
// lengths.
sample_entry->push_back(static_cast<char>(0xff));
// Only support one SPS and PPS.
// ffmpeg's ff_isom_write_avcc has the same limitation, so it's probably
// fine. This next byte is a reserved 0b111 + a 5-bit # of SPSs (1).
sample_entry->push_back(static_cast<char>(0xe1));
AppendU16(sps.size(), sample_entry);
sample_entry->append(sps.data(), sps.size());
sample_entry->push_back(1); // # of PPSs.
AppendU16(pps.size(), sample_entry);
sample_entry->append(pps.data(), pps.size());
if (sample_entry->size() - avcc_len_pos != avcc_len) {
*error_message = StrCat(
"internal error: anticipated AVCConfigurationBox length ", avcc_len,
", but was actually ", sample_entry->size() - avcc_len_pos,
"; sps length ", sps.size(), ", pps length ", pps.size());
return false;
}
} else {
sample_entry->append(extradata.data(), extradata.size());
}
if (sample_entry->size() - avc1_len_pos != avc1_len) {
*error_message =
StrCat("internal error: anticipated AVCSampleEntry length ", avc1_len,
", but was actually ", sample_entry->size() - avc1_len_pos,
"; sps length ", sps.size(), ", pps length ", pps.size());
return false;
}
return true;
}
bool TransformSampleData(re2::StringPiece annexb_sample,
std::string *avc_sample, std::string *error_message) {
// See AVCParameterSamples, ISO/IEC 14496-15 section 5.3.2.
avc_sample->clear();
auto fn = [&](re2::StringPiece nal_unit) {
// 4-byte length; this must be in sync with ParseExtraData's
// lengthSizeMinusOne == 3.
AppendU32(nal_unit.size(), avc_sample);
avc_sample->append(nal_unit.data(), nal_unit.size());
return IterationControl::kContinue;
};
if (!internal::DecodeH264AnnexB(annexb_sample, fn, error_message)) {
return false;
}
return true;
}
} // namespace moonfire_nvr

84
src/h264.h
View File

@ -1,84 +0,0 @@
// This file is part of Moonfire NVR, a security camera network video recorder.
// Copyright (C) 2016 Scott Lamb <slamb@slamb.org>
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// In addition, as a special exception, the copyright holders give
// permission to link the code of portions of this program with the
// OpenSSL library under certain conditions as described in each
// individual source file, and distribute linked combinations including
// the two.
//
// You must obey the GNU General Public License in all respects for all
// of the code used other than OpenSSL. If you modify file(s) with this
// exception, you may extend this exception to your version of the
// file(s), but you are not obligated to do so. If you do not wish to do
// so, delete this exception statement from your version. If you delete
// this exception statement from all source files in the program, then
// also delete it here.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
//
// h264.h: H.264 decoding. For the most part, Moonfire NVR does not try to
// understand the video codec. However, H.264 has two byte stream encodings:
// ISO/IEC 14496-10 Annex B, and ISO/IEC 14496-15 AVC access units.
// When streaming from RTSP, ffmpeg supplies the former. We need the latter
// to stick into .mp4 files. This file manages the conversion, both for
// the ffmpeg "extra data" (which should become the ISO/IEC 14496-15
// section 5.2.4.1 AVCDecoderConfigurationRecord) and the actual samples.
//
// ffmpeg of course has logic to do the same thing, but unfortunately it is
// not exposed except through ffmpeg's own generated .mp4 file. Extracting
// just this part of their .mp4 files would be more trouble than it's worth.
#ifndef MOONFIRE_NVR_H264_H
#define MOONFIRE_NVR_H264_H
#include <functional>
#include <string>
#include <re2/stringpiece.h>
#include "common.h"
namespace moonfire_nvr {
namespace internal {
using NalUnitFunction =
std::function<IterationControl(re2::StringPiece nal_unit)>;
// Decode a H.264 Annex B byte stream into NAL units.
// For GetH264SampleEntry; exposed for testing.
// Calls |process_nal_unit| for each NAL unit in the byte stream.
//
// Note: this won't spot all invalid byte streams. For example, several 0x00s
// not followed by a 0x01 will just be considered part of a NAL unit rather
// than proof of an invalid stream.
bool DecodeH264AnnexB(re2::StringPiece data, NalUnitFunction process_nal_unit,
std::string *error_message);
} // namespace
// Gets a H.264 sample entry (AVCSampleEntry, which extends
// VisualSampleEntry), given the "extradata", width, and height supplied by
// ffmpeg.
bool ParseExtraData(re2::StringPiece extradata, uint16_t width, uint16_t height,
std::string *sample_entry, bool *need_transform,
std::string *error_message);
bool TransformSampleData(re2::StringPiece annexb_sample,
std::string *avc_sample, std::string *error_message);
} // namespace moonfire_nvr
#endif // MOONFIRE_NVR_H264_H

349
src/h264.rs Normal file
View File

@ -0,0 +1,349 @@
// This file is part of Moonfire NVR, a security camera digital video recorder.
// Copyright (C) 2016 Scott Lamb <slamb@slamb.org>
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// In addition, as a special exception, the copyright holders give
// permission to link the code of portions of this program with the
// OpenSSL library under certain conditions as described in each
// individual source file, and distribute linked combinations including
// the two.
//
// You must obey the GNU General Public License in all respects for all
// of the code used other than OpenSSL. If you modify file(s) with this
// exception, you may extend this exception to your version of the
// file(s), but you are not obligated to do so. If you do not wish to do
// so, delete this exception statement from your version. If you delete
// this exception statement from all source files in the program, then
// also delete it here.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
//! H.264 decoding
//!
//! For the most part, Moonfire NVR does not try to understand the video codec. However, H.264 has
//! two byte stream encodings: ISO/IEC 14496-10 Annex B, and ISO/IEC 14496-15 AVC access units.
//! When streaming from RTSP, ffmpeg supplies the former. We need the latter to stick into `.mp4`
//! files. This file manages the conversion, both for the ffmpeg "extra data" (which should become
//! the ISO/IEC 14496-15 section 5.2.4.1 `AVCDecoderConfigurationRecord`) and the actual samples.
//!
//! ffmpeg of course has logic to do the same thing, but unfortunately it is not exposed except
//! through ffmpeg's own generated `.mp4` file. Extracting just this part of their `.mp4` files
//! would be more trouble than it's worth.
use byteorder::{BigEndian, WriteBytesExt};
use error::{Error, Result};
use regex::bytes::Regex;
// See ISO/IEC 14496-10 table 7-1 - NAL unit type codes, syntax element categories, and NAL unit
// type classes.
const NAL_UNIT_SEQ_PARAMETER_SET: u8 = 7;
const NAL_UNIT_PIC_PARAMETER_SET: u8 = 8;
const NAL_UNIT_TYPE_MASK: u8 = 0x1F; // bottom 5 bits of first byte of unit.
/// Decodes a H.264 Annex B byte stream into NAL units. Calls `f` for each NAL unit in the byte
/// stream. Aborts if `f` returns error.
///
/// See ISO/IEC 14496-10 section B.2: Byte stream NAL unit decoding process.
/// This is a relatively simple, unoptimized implementation.
///
/// TODO: detect invalid byte streams. For example, several 0x00s not followed by a 0x01, a stream
/// stream not starting with 0x00 0x00 0x00 0x01, or an empty NAL unit.
fn decode_h264_annex_b<'a, F>(data: &'a [u8], mut f: F) -> Result<()>
where F: FnMut(&'a [u8]) -> Result<()> {
lazy_static! {
static ref START_CODE: Regex = Regex::new(r"(\x00{2,}\x01)").unwrap();
}
for unit in START_CODE.split(data) {
if !unit.is_empty() {
f(unit)?;
}
}
Ok(())
}
/// Parses Annex B extra data, returning a tuple holding the `sps` and `pps` substrings.
fn parse_annex_b_extra_data(data: &[u8]) -> Result<(&[u8], &[u8])> {
let mut sps = None;
let mut pps = None;
decode_h264_annex_b(data, |unit| {
let nal_type = (unit[0] as u8) & NAL_UNIT_TYPE_MASK;
match nal_type {
NAL_UNIT_SEQ_PARAMETER_SET => { sps = Some(unit); },
NAL_UNIT_PIC_PARAMETER_SET => { pps = Some(unit); },
_ => { return Err(Error::new(format!("Expected SPS and PPS; got type {}", nal_type))); }
};
Ok(())
})?;
match (sps, pps) {
(Some(s), Some(p)) => Ok((s, p)),
_ => Err(Error::new("SPS and PPS must be specified".to_owned())),
}
}
/// Parsed representation of ffmpeg's "extradata".
#[derive(Debug, PartialEq, Eq)]
pub struct ExtraData {
pub sample_entry: Vec<u8>,
pub width: u16,
pub height: u16,
/// True iff sample data should be transformed from Annex B format to AVC format via a call to
/// `transform_sample_data`. (The assumption is that if the extra data was in Annex B format,
/// the sample data is also.)
pub need_transform: bool,
}
impl ExtraData {
/// Parses "extradata" from ffmpeg. This data may be in either Annex B format or AVC format.
pub fn parse(extradata: &[u8], width: u16, height: u16) -> Result<ExtraData> {
let mut sps_and_pps = None;
let need_transform;
let avcc_len = if extradata.starts_with(b"\x00\x00\x00\x01") ||
extradata.starts_with(b"\x00\x00\x01") {
// ffmpeg supplied "extradata" in Annex B format.
let (s, p) = parse_annex_b_extra_data(extradata)?;
sps_and_pps = Some((s, p));
need_transform = true;
// This magic value is checked at the end of the function;
// unit tests confirm its accuracy.
19 + s.len() + p.len()
} else {
// Assume "extradata" holds an AVCDecoderConfiguration.
need_transform = false;
8 + extradata.len()
};
let sps_and_pps = sps_and_pps;
let need_transform = need_transform;
// This magic value is also checked at the end.
let avc1_len = 86 + avcc_len;
let mut sample_entry = Vec::with_capacity(avc1_len);
// This is a concatenation of the following boxes/classes.
// SampleEntry, ISO/IEC 14496-10 section 8.5.2.
let avc1_len_pos = sample_entry.len();
sample_entry.write_u32::<BigEndian>(avc1_len as u32)?; // length
// type + reserved + data_reference_index = 1
sample_entry.extend_from_slice(b"avc1\x00\x00\x00\x00\x00\x00\x00\x01");
// VisualSampleEntry, ISO/IEC 14496-12 section 12.1.3.
sample_entry.extend_from_slice(&[0; 16]); // pre-defined + reserved
sample_entry.write_u16::<BigEndian>(width)?;
sample_entry.write_u16::<BigEndian>(height)?;
sample_entry.extend_from_slice(&[
0x00, 0x48, 0x00, 0x00, // horizresolution
0x00, 0x48, 0x00, 0x00, // vertresolution
0x00, 0x00, 0x00, 0x00, // reserved
0x00, 0x01, // frame count
0x00, 0x00, 0x00, 0x00, // compressorname
0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00,
0x00, 0x18, 0xff, 0xff, // depth + pre_defined
]);
// AVCSampleEntry, ISO/IEC 14496-15 section 5.3.4.1.
// AVCConfigurationBox, ISO/IEC 14496-15 section 5.3.4.1.
let avcc_len_pos = sample_entry.len();
sample_entry.write_u32::<BigEndian>(avcc_len as u32)?; // length
sample_entry.extend_from_slice(b"avcC");
let avc_decoder_config_len = if let Some((sps, pps)) = sps_and_pps {
let before = sample_entry.len();
// Create the AVCDecoderConfiguration, ISO/IEC 14496-15 section 5.2.4.1.
// The beginning of the AVCDecoderConfiguration takes a few values from
// the SPS (ISO/IEC 14496-10 section 7.3.2.1.1). One caveat: that section
// defines the syntax in terms of RBSP, not NAL. The difference is the
// escaping of 00 00 01 and 00 00 02; see notes about
// "emulation_prevention_three_byte" in ISO/IEC 14496-10 section 7.4.
// It looks like 00 is not a valid value of profile_idc, so this distinction
// shouldn't be relevant here. And ffmpeg seems to ignore it.
sample_entry.push(1); // configurationVersion
sample_entry.push(sps[1]); // profile_idc . AVCProfileIndication
sample_entry.push(sps[2]); // ...misc bits... . profile_compatibility
sample_entry.push(sps[3]); // level_idc . AVCLevelIndication
// Hardcode lengthSizeMinusOne to 3, matching TransformSampleData's 4-byte
// lengths.
sample_entry.push(0xff);
// Only support one SPS and PPS.
// ffmpeg's ff_isom_write_avcc has the same limitation, so it's probably
// fine. This next byte is a reserved 0b111 + a 5-bit # of SPSs (1).
sample_entry.push(0xe1);
sample_entry.write_u16::<BigEndian>(sps.len() as u16)?;
sample_entry.extend_from_slice(sps);
sample_entry.push(1); // # of PPSs.
sample_entry.write_u16::<BigEndian>(pps.len() as u16)?;
sample_entry.extend_from_slice(pps);
if sample_entry.len() - avcc_len_pos != avcc_len {
return Err(Error::new(format!("internal error: anticipated AVCConfigurationBox \
length {}, but was actually {}; sps length \
{}, pps length {}",
avcc_len, sample_entry.len() - avcc_len_pos,
sps.len(), pps.len())));
}
sample_entry.len() - before
} else {
sample_entry.extend_from_slice(extradata);
extradata.len()
};
if sample_entry.len() - avc1_len_pos != avc1_len {
return Err(Error::new(format!("internal error: anticipated AVCSampleEntry length \
{}, but was actually {}; AVCDecoderConfiguration \
length {}", avc1_len, sample_entry.len() - avc1_len_pos,
avc_decoder_config_len)));
}
Ok(ExtraData{
sample_entry: sample_entry,
width: width,
height: height,
need_transform: need_transform,
})
}
}
/// Transforms sample data from Annex B format to AVC format. Should be called on samples iff
/// `ExtraData::need_transform` is true. Uses an out parameter `avc_sample` rather than a return
/// so that memory allocations can be reused from sample to sample.
pub fn transform_sample_data(annexb_sample: &[u8], avc_sample: &mut Vec<u8>) -> Result<()> {
// See AVCParameterSamples, ISO/IEC 14496-15 section 5.3.2.
avc_sample.clear();
// The output will be about as long as the input. Annex B stop codes require at least three
// bytes; many seem to be four. The output lengths are exactly four.
avc_sample.reserve(annexb_sample.len() + 4);
decode_h264_annex_b(annexb_sample, |unit| {
// 4-byte length; this must match ParseExtraData's lengthSizeMinusOne == 3.
avc_sample.write_u32::<BigEndian>(unit.len() as u32)?; // length
avc_sample.extend_from_slice(unit);
Ok(())
})?;
Ok(())
}
#[cfg(test)]
mod tests {
const ANNEX_B_TEST_INPUT: [u8; 35] = [
0x00, 0x00, 0x00, 0x01, 0x67, 0x4d, 0x00, 0x1f,
0x9a, 0x66, 0x02, 0x80, 0x2d, 0xff, 0x35, 0x01,
0x01, 0x01, 0x40, 0x00, 0x00, 0xfa, 0x00, 0x00,
0x1d, 0x4c, 0x01, 0x00, 0x00, 0x00, 0x01, 0x68,
0xee, 0x3c, 0x80,
];
const AVC_DECODER_CONFIG_TEST_INPUT: [u8; 38] = [
0x01, 0x4d, 0x00, 0x1f, 0xff, 0xe1, 0x00, 0x17,
0x67, 0x4d, 0x00, 0x1f, 0x9a, 0x66, 0x02, 0x80,
0x2d, 0xff, 0x35, 0x01, 0x01, 0x01, 0x40, 0x00,
0x00, 0xfa, 0x00, 0x00, 0x1d, 0x4c, 0x01, 0x01,
0x00, 0x04, 0x68, 0xee, 0x3c, 0x80,
];
const TEST_OUTPUT: [u8; 132] = [
0x00, 0x00, 0x00, 0x84, 0x61, 0x76, 0x63, 0x31,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x05, 0x00, 0x02, 0xd0, 0x00, 0x48, 0x00, 0x00,
0x00, 0x48, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x18, 0xff, 0xff, 0x00, 0x00,
0x00, 0x2e, 0x61, 0x76, 0x63, 0x43, 0x01, 0x4d,
0x00, 0x1f, 0xff, 0xe1, 0x00, 0x17, 0x67, 0x4d,
0x00, 0x1f, 0x9a, 0x66, 0x02, 0x80, 0x2d, 0xff,
0x35, 0x01, 0x01, 0x01, 0x40, 0x00, 0x00, 0xfa,
0x00, 0x00, 0x1d, 0x4c, 0x01, 0x01, 0x00, 0x04,
0x68, 0xee, 0x3c, 0x80,
];
#[test]
fn test_decode() {
let data = &ANNEX_B_TEST_INPUT;
let mut pieces = Vec::new();
super::decode_h264_annex_b(data, |p| {
pieces.push(p);
Ok(())
}).unwrap();
assert_eq!(&pieces, &[&data[4 .. 27], &data[31 ..]]);
}
#[test]
fn test_sample_entry_from_avc_decoder_config() {
let e = super::ExtraData::parse(&AVC_DECODER_CONFIG_TEST_INPUT, 1280, 720).unwrap();
assert_eq!(&e.sample_entry[..], &TEST_OUTPUT[..]);
assert_eq!(e.width, 1280);
assert_eq!(e.height, 720);
assert_eq!(e.need_transform, false);
}
#[test]
fn test_sample_entry_from_annex_b() {
let e = super::ExtraData::parse(&ANNEX_B_TEST_INPUT, 1280, 720).unwrap();
assert_eq!(e.width, 1280);
assert_eq!(e.height, 720);
assert_eq!(e.need_transform, true);
}
#[test]
fn test_transform_sample_data() {
const INPUT: [u8; 64] = [
0x00, 0x00, 0x00, 0x01, 0x67, 0x4d, 0x00, 0x1f,
0x9a, 0x66, 0x02, 0x80, 0x2d, 0xff, 0x35, 0x01,
0x01, 0x01, 0x40, 0x00, 0x00, 0xfa, 0x00, 0x00,
0x1d, 0x4c, 0x01,
0x00, 0x00, 0x00, 0x01, 0x68, 0xee, 0x3c, 0x80,
0x00, 0x00, 0x00, 0x01, 0x06, 0x06, 0x01, 0xc4,
0x80,
0x00, 0x00, 0x00, 0x01, 0x65, 0x88, 0x80, 0x10,
0x00, 0x08, 0x7f, 0x00, 0x5d, 0x27, 0xb5, 0xc1,
0xff, 0x8c, 0xd6, 0x35,
// (truncated)
];
const EXPECTED_OUTPUT: [u8; 64] = [
0x00, 0x00, 0x00, 0x17, 0x67, 0x4d, 0x00, 0x1f,
0x9a, 0x66, 0x02, 0x80, 0x2d, 0xff, 0x35, 0x01,
0x01, 0x01, 0x40, 0x00, 0x00, 0xfa, 0x00, 0x00,
0x1d, 0x4c, 0x01,
0x00, 0x00, 0x00, 0x04, 0x68, 0xee, 0x3c, 0x80,
0x00, 0x00, 0x00, 0x05, 0x06, 0x06, 0x01, 0xc4,
0x80,
0x00, 0x00, 0x00, 0x10, 0x65, 0x88, 0x80, 0x10,
0x00, 0x08, 0x7f, 0x00, 0x5d, 0x27, 0xb5, 0xc1,
0xff, 0x8c, 0xd6, 0x35,
];
let mut out = Vec::new();
super::transform_sample_data(&INPUT, &mut out).unwrap();
assert_eq!(&out[..], &EXPECTED_OUTPUT[..]);
}
}

295
src/http-test.cc
View File

@ -1,295 +0,0 @@
// This file is part of Moonfire NVR, a security camera digital video recorder.
// Copyright (C) 2016 Scott Lamb <slamb@slamb.org>
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// In addition, as a special exception, the copyright holders give
// permission to link the code of portions of this program with the
// OpenSSL library under certain conditions as described in each
// individual source file, and distribute linked combinations including
// the two.
//
// You must obey the GNU General Public License in all respects for all
// of the code used other than OpenSSL. If you modify file(s) with this
// exception, you may extend this exception to your version of the
// file(s), but you are not obligated to do so. If you do not wish to do
// so, delete this exception statement from your version. If you delete
// this exception statement from all source files in the program, then
// also delete it here.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
//
// util_test.cc: tests of the util.h interface.
#include <gflags/gflags.h>
#include <glog/logging.h>
#include <gmock/gmock.h>
#include <gtest/gtest.h>
#include <fcntl.h>
#include <string.h>
#include <sys/stat.h>
#include <sys/types.h>
#include "http.h"
#include "string.h"
#include "testutil.h"
DECLARE_bool(alsologtostderr);
using moonfire_nvr::internal::ParseRangeHeader;
using moonfire_nvr::internal::RangeHeaderType;
using testing::_;
using testing::AnyNumber;
using testing::DoAll;
using testing::Return;
using testing::SetArgPointee;
namespace moonfire_nvr {
namespace {
class MockFileSlice : public FileSlice {
public:
MOCK_CONST_METHOD0(size, int64_t());
MOCK_CONST_METHOD3(AddRange, int64_t(ByteRange, EvBuffer *, std::string *));
};
TEST(EvBufferTest, AddFileTest) {
std::string dir = PrepareTempDirOrDie("http");
std::string foo_filename = StrCat(dir, "/foo");
WriteFileOrDie(foo_filename, "foo");
int in_fd = open(foo_filename.c_str(), O_RDONLY);
PCHECK(in_fd >= 0) << "open: " << foo_filename;
std::string error_message;
// Ensure adding the whole file succeeds.
EvBuffer buf1;
ASSERT_TRUE(buf1.AddFile(in_fd, 0, 3, &error_message)) << error_message;
in_fd = -1;
EXPECT_EQ(3u, evbuffer_get_length(buf1.get()));
// Ensure adding an empty region succeeds.
EvBuffer buf2;
ASSERT_TRUE(buf2.AddFile(in_fd, 0, 0, &error_message)) << error_message;
EXPECT_EQ(0u, evbuffer_get_length(buf2.get()));
// Ensure adding part of a file after another string succeeds.
in_fd = open(foo_filename.c_str(), O_RDONLY);
EvBuffer buf3;
buf3.Add("1234");
ASSERT_TRUE(buf3.AddFile(in_fd, 1, 2, &error_message)) << error_message;
auto size3 = evbuffer_get_length(buf3.get());
EXPECT_EQ(6u, size3);
std::string buf3_contents = std::string(
reinterpret_cast<const char *>(evbuffer_pullup(buf3.get(), size3)),
size3);
EXPECT_EQ("1234oo", buf3_contents);
}
class FileSlicesTest : public testing::Test {
protected:
void Init(int flags) {
EXPECT_CALL(a_, size()).Times(AnyNumber()).WillRepeatedly(Return(5));
EXPECT_CALL(b_, size()).Times(AnyNumber()).WillRepeatedly(Return(13));
EXPECT_CALL(c_, size()).Times(AnyNumber()).WillRepeatedly(Return(7));
EXPECT_CALL(d_, size()).Times(AnyNumber()).WillRepeatedly(Return(17));
EXPECT_CALL(e_, size()).Times(AnyNumber()).WillRepeatedly(Return(19));
slices_.Append(&a_, flags);
slices_.Append(&b_, flags);
slices_.Append(&c_, flags);
slices_.Append(&d_, flags);
slices_.Append(&e_, flags);
}
FileSlices slices_;
testing::StrictMock<MockFileSlice> a_;
testing::StrictMock<MockFileSlice> b_;
testing::StrictMock<MockFileSlice> c_;
testing::StrictMock<MockFileSlice> d_;
testing::StrictMock<MockFileSlice> e_;
};
TEST_F(FileSlicesTest, Size) {
Init(0);
EXPECT_EQ(5 + 13 + 7 + 17 + 19, slices_.size());
}
TEST_F(FileSlicesTest, ExactSlice) {
// Exactly slice b.
Init(0);
std::string error_message;
EXPECT_CALL(b_, AddRange(ByteRange(0, 13), _, _)).WillOnce(Return(13));
EXPECT_EQ(13, slices_.AddRange(ByteRange(5, 18), nullptr, &error_message))
<< error_message;
}
TEST_F(FileSlicesTest, Offset) {
// Part of slice b, all of slice c, and part of slice d.
Init(0);
std::string error_message;
EXPECT_CALL(b_, AddRange(ByteRange(12, 13), _, _)).WillOnce(Return(1));
EXPECT_CALL(c_, AddRange(ByteRange(0, 7), _, _)).WillOnce(Return(7));
EXPECT_CALL(d_, AddRange(ByteRange(0, 1), _, _)).WillOnce(Return(1));
EXPECT_EQ(9, slices_.AddRange(ByteRange(17, 26), nullptr, &error_message))
<< error_message;
}
TEST_F(FileSlicesTest, Everything) {
Init(0);
std::string error_message;
EXPECT_CALL(a_, AddRange(ByteRange(0, 5), _, _)).WillOnce(Return(5));
EXPECT_CALL(b_, AddRange(ByteRange(0, 13), _, _)).WillOnce(Return(13));
EXPECT_CALL(c_, AddRange(ByteRange(0, 7), _, _)).WillOnce(Return(7));
EXPECT_CALL(d_, AddRange(ByteRange(0, 17), _, _)).WillOnce(Return(17));
EXPECT_CALL(e_, AddRange(ByteRange(0, 19), _, _)).WillOnce(Return(19));
EXPECT_EQ(61, slices_.AddRange(ByteRange(0, 61), nullptr, &error_message))
<< error_message;
}
TEST_F(FileSlicesTest, Lazy) {
Init(FileSlices::kLazy);
std::string error_message;
EXPECT_CALL(a_, AddRange(ByteRange(0, 5), _, _)).WillOnce(Return(5));
EXPECT_EQ(5, slices_.AddRange(ByteRange(0, 61), nullptr, &error_message))
<< error_message;
EXPECT_CALL(b_, AddRange(ByteRange(0, 13), _, _)).WillOnce(Return(13));
EXPECT_EQ(13, slices_.AddRange(ByteRange(5, 61), nullptr, &error_message))
<< error_message;
EXPECT_CALL(c_, AddRange(ByteRange(0, 7), _, _)).WillOnce(Return(7));
EXPECT_EQ(7, slices_.AddRange(ByteRange(18, 61), nullptr, &error_message))
<< error_message;
EXPECT_CALL(d_, AddRange(ByteRange(0, 17), _, _)).WillOnce(Return(17));
EXPECT_EQ(17, slices_.AddRange(ByteRange(25, 61), nullptr, &error_message))
<< error_message;
EXPECT_CALL(e_, AddRange(ByteRange(0, 19), _, _)).WillOnce(Return(19));
EXPECT_EQ(19, slices_.AddRange(ByteRange(42, 61), nullptr, &error_message))
<< error_message;
}
TEST_F(FileSlicesTest, SliceWithPartialReturn) {
Init(0);
std::string error_message;
EXPECT_CALL(a_, AddRange(ByteRange(0, 5), _, _)).WillOnce(Return(5));
EXPECT_CALL(b_, AddRange(ByteRange(0, 13), _, _)).WillOnce(Return(1));
EXPECT_EQ(6, slices_.AddRange(ByteRange(0, 61), nullptr, &error_message))
<< error_message;
}
TEST_F(FileSlicesTest, PropagateError) {
Init(0);
std::string error_message;
EXPECT_CALL(a_, AddRange(ByteRange(0, 5), _, _)).WillOnce(Return(5));
EXPECT_CALL(b_, AddRange(ByteRange(0, 13), _, _))
.WillRepeatedly(DoAll(SetArgPointee<2>("asdf"), Return(-1)));
EXPECT_EQ(5, slices_.AddRange(ByteRange(0, 61), nullptr, &error_message));
EXPECT_EQ(-1, slices_.AddRange(ByteRange(5, 61), nullptr, &error_message));
EXPECT_EQ("asdf", error_message);
}
// Test the specific examples enumerated in RFC 2616 section 14.35.1.
TEST(RangeHeaderTest, Rfc_2616_Section_14_35_1) {
std::vector<ByteRange> ranges;
EXPECT_EQ(RangeHeaderType::kSatisfiable,
ParseRangeHeader("bytes=0-499", 10000, &ranges));
EXPECT_THAT(ranges, testing::ElementsAre(ByteRange(0, 500)));
EXPECT_EQ(RangeHeaderType::kSatisfiable,
ParseRangeHeader("bytes=500-999", 10000, &ranges));
EXPECT_THAT(ranges, testing::ElementsAre(ByteRange(500, 1000)));
EXPECT_EQ(RangeHeaderType::kSatisfiable,
ParseRangeHeader("bytes=-500", 10000, &ranges));
EXPECT_THAT(ranges, testing::ElementsAre(ByteRange(9500, 10000)));
EXPECT_EQ(RangeHeaderType::kSatisfiable,
ParseRangeHeader("bytes=9500-", 10000, &ranges));
EXPECT_THAT(ranges, testing::ElementsAre(ByteRange(9500, 10000)));
EXPECT_EQ(RangeHeaderType::kSatisfiable,
ParseRangeHeader("bytes=0-0,-1", 10000, &ranges));
EXPECT_THAT(ranges,
testing::ElementsAre(ByteRange(0, 1), ByteRange(9999, 10000)));
// Non-canonical ranges. Possibly the point of these is that the adjacent
// and overlapping ranges are supposed to be coalesced into one? I'm not
// going to do that for now...just trying to get something working...
EXPECT_EQ(RangeHeaderType::kSatisfiable,
ParseRangeHeader("bytes=500-600,601-999", 10000, &ranges));
EXPECT_THAT(ranges,
testing::ElementsAre(ByteRange(500, 601), ByteRange(601, 1000)));
EXPECT_EQ(RangeHeaderType::kSatisfiable,
ParseRangeHeader("bytes=500-700,601-999", 10000, &ranges));
EXPECT_THAT(ranges,
testing::ElementsAre(ByteRange(500, 701), ByteRange(601, 1000)));
}
TEST(RangeHeaderTest, Satisfiability) {
std::vector<ByteRange> ranges;
EXPECT_EQ(RangeHeaderType::kNotSatisfiable,
ParseRangeHeader("bytes=10000-", 10000, &ranges));
EXPECT_EQ(RangeHeaderType::kSatisfiable,
ParseRangeHeader("bytes=0-499,10000-", 10000, &ranges));
EXPECT_THAT(ranges, testing::ElementsAre(ByteRange(0, 500)));
EXPECT_EQ(RangeHeaderType::kNotSatisfiable,
ParseRangeHeader("bytes=-1", 0, &ranges));
EXPECT_EQ(RangeHeaderType::kNotSatisfiable,
ParseRangeHeader("bytes=0-0", 0, &ranges));
EXPECT_EQ(RangeHeaderType::kNotSatisfiable,
ParseRangeHeader("bytes=0-", 0, &ranges));
EXPECT_EQ(RangeHeaderType::kSatisfiable,
ParseRangeHeader("bytes=0-0", 1, &ranges));
EXPECT_THAT(ranges, testing::ElementsAre(ByteRange(0, 1)));
EXPECT_EQ(RangeHeaderType::kSatisfiable,
ParseRangeHeader("bytes=0-", 1, &ranges));
EXPECT_THAT(ranges, testing::ElementsAre(ByteRange(0, 1)));
}
TEST(RangeHeaderTest, AbsentOrInvalid) {
std::vector<ByteRange> ranges;
EXPECT_EQ(RangeHeaderType::kAbsentOrInvalid,
ParseRangeHeader(nullptr, 10000, &ranges));
EXPECT_EQ(RangeHeaderType::kAbsentOrInvalid,
ParseRangeHeader("", 10000, &ranges));
EXPECT_EQ(RangeHeaderType::kAbsentOrInvalid,
ParseRangeHeader("foo=0-499", 10000, &ranges));
EXPECT_EQ(RangeHeaderType::kAbsentOrInvalid,
ParseRangeHeader("foo=0-499", 10000, &ranges));
EXPECT_EQ(RangeHeaderType::kAbsentOrInvalid,
ParseRangeHeader("bytes=499-0", 10000, &ranges));
EXPECT_EQ(RangeHeaderType::kAbsentOrInvalid,
ParseRangeHeader("bytes=", 10000, &ranges));
EXPECT_EQ(RangeHeaderType::kAbsentOrInvalid,
ParseRangeHeader("bytes=,", 10000, &ranges));
EXPECT_EQ(RangeHeaderType::kAbsentOrInvalid,
ParseRangeHeader("bytes=-", 10000, &ranges));
}
// TODO: test HttpServe itself!
// Currently the testing is manual. Three important cases:
// * HTTP request succeeds
// * client aborts (as in hitting ctrl-C in curl during a long request)
// * the VirtualFile returns error (say, by chmod u-r on the file backing
// a RealFileSlice)
} // namespace
} // namespace moonfire_nvr
int main(int argc, char **argv) {
FLAGS_alsologtostderr = true;
google::ParseCommandLineFlags(&argc, &argv, true);
testing::InitGoogleTest(&argc, argv);
google::InitGoogleLogging(argv[0]);
return RUN_ALL_TESTS();
}

414
src/http.cc
View File

@ -1,414 +0,0 @@
// This file is part of Moonfire NVR, a security camera digital video recorder.
// Copyright (C) 2016 Scott Lamb <slamb@slamb.org>
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// In addition, as a special exception, the copyright holders give
// permission to link the code of portions of this program with the
// OpenSSL library under certain conditions as described in each
// individual source file, and distribute linked combinations including
// the two.
//
// You must obey the GNU General Public License in all respects for all
// of the code used other than OpenSSL. If you modify file(s) with this
// exception, you may extend this exception to your version of the
// file(s), but you are not obligated to do so. If you do not wish to do
// so, delete this exception statement from your version. If you delete
// this exception statement from all source files in the program, then
// also delete it here.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
//
// http.cc: See http.h.
#include "http.h"
#include <errno.h>
#include <fcntl.h>
#include <stdlib.h>
#include <string.h>
#include <sys/stat.h>
#include <sys/types.h>
#include <algorithm>
#include <event2/buffer.h>
#include <event2/event.h>
#include <event2/keyvalq_struct.h>
#include <event2/http.h>
#include <glog/logging.h>
#include "string.h"
namespace moonfire_nvr {
namespace {
// An HttpServe call still in progress.
struct ServeInProgress {
ByteRange left;
int64_t sent_bytes = 0;
std::shared_ptr<VirtualFile> file;
evhttp_request *req = nullptr;
};
void ServeCloseCallback(evhttp_connection *con, void *arg) {
std::unique_ptr<ServeInProgress> serve(
reinterpret_cast<ServeInProgress *>(arg));
LOG(INFO) << serve->req << ": received client abort after sending "
<< serve->sent_bytes << " bytes; there were " << serve->left.size()
<< " bytes left.";
// The call to cancel will guarantee ServeChunkCallback is not called again.
evhttp_cancel_request(serve->req);
}
void ServeChunkCallback(evhttp_connection *con, void *arg) {
std::unique_ptr<ServeInProgress> serve(
reinterpret_cast<ServeInProgress *>(arg));
if (serve->left.size() == 0) {
LOG(INFO) << serve->req << ": done; sent " << serve->sent_bytes
<< " bytes.";
evhttp_connection_set_closecb(con, nullptr, nullptr);
evhttp_send_reply_end(serve->req);
return;
}
// Serve more data.
EvBuffer buf;
std::string error_message;
int64_t added = serve->file->AddRange(serve->left, &buf, &error_message);
if (added <= 0) {
// Order is important here: evhttp_cancel_request immediately calls the
// close callback, so remove it first to avoid double-freeing |serve|.
evhttp_connection_set_closecb(con, nullptr, nullptr);
evhttp_cancel_request(serve->req);
LOG(ERROR) << serve->req << ": Failed to serve request after sending "
<< serve->sent_bytes << " bytes (" << serve->left.size()
<< " bytes left): " << error_message;
return;
}
serve->sent_bytes += added;
serve->left.begin += added;
VLOG(1) << serve->req << ": sending " << added << " bytes (more) data; still "
<< serve->left.size() << " bytes left";
evhttp_send_reply_chunk_with_cb(serve->req, buf.get(), &ServeChunkCallback,
serve.get());
evhttp_send_reply_chunk(serve->req, buf.get());
serve.release();
}
} // namespace
namespace internal {
RangeHeaderType ParseRangeHeader(const char *inptr, int64_t size,
std::vector<ByteRange> *ranges) {
if (inptr == nullptr) {
return RangeHeaderType::kAbsentOrInvalid; // absent.
}
if (strncmp(inptr, "bytes=", strlen("bytes=")) != 0) {
return RangeHeaderType::kAbsentOrInvalid; // invalid syntax.
}
inptr += strlen("bytes=");
ranges->clear();
int n_ranges = 0;
while (*inptr != 0) { // have more byte-range-sets.
++n_ranges;
ByteRange r;
// Parse a number.
const char *endptr;
int64_t value;
if (!strto64(inptr, 10, &endptr, &value)) {
return RangeHeaderType::kAbsentOrInvalid; // invalid syntax.
}
if (value < 0) { // just parsed suffix-byte-range-spec.
r.begin = std::max(size + value, INT64_C(0));
r.end = size;
if (r.begin < r.end) { // satisfiable.
ranges->emplace_back(std::move(r));
}
inptr = endptr;
} else { // just parsed start of byte-range-spec.
if (*endptr != '-') {
return RangeHeaderType::kAbsentOrInvalid;
}
r.begin = value;
inptr = endptr + 1; // move past the '-'.
if (*inptr == ',' || *inptr == 0) { // no end specified; use EOF.
r.end = size;
} else { // explicit range.
if (!strto64(inptr, 10, &endptr, &value) || value < r.begin) {
return RangeHeaderType::kAbsentOrInvalid; // invalid syntax.
}
inptr = endptr;
r.end = std::min(size, value + 1); // note inclusive->exclusive.
}
if (r.begin < size) {
ranges->emplace_back(std::move(r)); // satisfiable.
}
}
if (*inptr == ',') {
inptr++;
if (*inptr == 0) {
return RangeHeaderType::kAbsentOrInvalid; // invalid syntax.
}
}
}
if (n_ranges == 0) { // must be at least one range.
return RangeHeaderType::kAbsentOrInvalid;
}
return ranges->empty() ? RangeHeaderType::kNotSatisfiable
: RangeHeaderType::kSatisfiable;
}
} // namespace internal
bool EvBuffer::AddFile(int fd, ev_off_t offset, ev_off_t length,
std::string *error_message) {
if (length == 0) {
// evbuffer_add_file fails in this trivial case, at least when using mmap.
// Just return true since there's nothing to be done.
return true;
}
if (evbuffer_get_length(buf_) > 0) {
// Work around https://github.com/libevent/libevent/issues/306 by using a
// fresh buffer for evbuffer_add_file.
EvBuffer fresh_buffer;
if (!fresh_buffer.AddFile(fd, offset, length, error_message)) {
return false;
}
// Crash if evbuffer_add_buffer fails, because the ownership of |fd| has
// already been transferred, and it's too confusing to support some
// failures in which the caller still owns |fd| and some in which it does
// not.
CHECK_EQ(0, evbuffer_add_buffer(buf_, fresh_buffer.buf_))
<< "evbuffer_add_buffer failed: " << strerror(errno);
return true;
}
if (evbuffer_add_file(buf_, fd, offset, length) != 0) {
int err = errno;
*error_message = StrCat("evbuffer_add_file failed with offset ", offset,
", length ", length, ": ", strerror(err));
return false;
}
return true;
}
void RealFileSlice::Init(File *dir, re2::StringPiece filename,
ByteRange range) {
dir_ = dir;
filename_ = filename.as_string();
range_ = range;
}
int64_t RealFileSlice::AddRange(ByteRange range, EvBuffer *buf,
std::string *error_message) const {
int fd;
int ret = dir_->Open(filename_.c_str(), O_RDONLY, &fd);
if (ret != 0) {
*error_message = StrCat("open ", filename_, ": ", strerror(ret));
return -1;
}
if (!buf->AddFile(fd, range_.begin + range.begin, range.size(),
error_message)) {
close(fd);
return -1;
}
// |buf| now owns |fd|.
return range.size();
}
int64_t FillerFileSlice::AddRange(ByteRange range, EvBuffer *buf,
std::string *error_message) const {
std::unique_ptr<std::string> s(new std::string);
s->reserve(size_);
if (!fn_(s.get(), error_message)) {
return 0;
}
if (s->size() != size_) {
*error_message = StrCat("Expected filled slice to be ", size_,
" bytes; got ", s->size(), " bytes.");
return 0;
}
std::string *unowned_s = s.release();
buf->AddReference(unowned_s->data() + range.begin,
range.size(), [](const void *, size_t, void *s) {
delete reinterpret_cast<std::string *>(s);
}, unowned_s);
return range.size();
}
int64_t StringPieceSlice::AddRange(ByteRange range, EvBuffer *buf,
std::string *error_message) const {
buf->AddReference(piece_.data() + range.begin, range.size(), nullptr,
nullptr);
return range.size();
}
int64_t FileSlices::AddRange(ByteRange range, EvBuffer *buf,
std::string *error_message) const {
if (range.begin < 0 || range.begin > range.end || range.end > size_) {
*error_message = StrCat("Range ", range.DebugString(),
" not valid for file of size ", size_);
return false;
}
int64_t total_bytes_added = 0;
auto it = std::upper_bound(slices_.begin(), slices_.end(), range.begin,
[](int64_t begin, const SliceInfo &info) {
return begin < info.range.end;
});
for (; it != slices_.end() && range.end > it->range.begin; ++it) {
if (total_bytes_added > 0 && (it->flags & kLazy) != 0) {
VLOG(1) << "early return of " << total_bytes_added << "/" << range.size()
<< " bytes from FileSlices " << this << " because slice "
<< it->slice << " is lazy.";
break;
}
ByteRange mapped(
std::max(INT64_C(0), range.begin - it->range.begin),
std::min(range.end - it->range.begin, it->range.end - it->range.begin));
int64_t slice_bytes_added = it->slice->AddRange(mapped, buf, error_message);
total_bytes_added += slice_bytes_added > 0 ? slice_bytes_added : 0;
if (slice_bytes_added < 0 && total_bytes_added == 0) {
LOG(WARNING) << "early return of " << total_bytes_added << "/"
<< range.size() << " bytes from FileSlices " << this
<< " due to slice " << it->slice
<< " returning error: " << *error_message;
return -1;
} else if (slice_bytes_added < mapped.size()) {
LOG(INFO) << "early return of " << total_bytes_added << "/"
<< range.size() << " bytes from FileSlices " << this
<< " due to slice " << it->slice << " returning "
<< slice_bytes_added << "/" << mapped.size()
<< " bytes. error_message (maybe populated): "
<< *error_message;
break;
}
}
return total_bytes_added;
}
void HttpSendError(evhttp_request *req, int http_err, const std::string &prefix,
int posix_err) {
evhttp_send_error(req, http_err,
EscapeHtml(prefix + strerror(posix_err)).c_str());
}
void HttpServe(const std::shared_ptr<VirtualFile> &file, evhttp_request *req) {
// We could support HEAD, but there's probably no need.
if (evhttp_request_get_command(req) != EVHTTP_REQ_GET) {
return evhttp_send_error(req, HTTP_BADMETHOD, "only GET allowed");
}
const struct evkeyvalq *in_hdrs = evhttp_request_get_input_headers(req);
struct evkeyvalq *out_hdrs = evhttp_request_get_output_headers(req);
// Construct a Last-Modified: header.
time_t last_modified = file->last_modified();
struct tm last_modified_tm;
if (gmtime_r(&last_modified, &last_modified_tm) == 0) {
return HttpSendError(req, HTTP_INTERNAL, "gmtime_r failed: ", errno);
}
char last_modified_str[50];
if (strftime(last_modified_str, sizeof(last_modified_str),
"%a, %d %b %Y %H:%M:%S GMT", &last_modified_tm) == 0) {
return HttpSendError(req, HTTP_INTERNAL, "strftime failed: ", errno);
}
std::string etag = file->etag();
// Ignore the "Range:" header if "If-Range:" specifies an incorrect etag.
const char *if_range = evhttp_find_header(in_hdrs, "If-Range");
const char *range_hdr = evhttp_find_header(in_hdrs, "Range");
if (if_range != nullptr && etag != if_range) {
LOG(INFO) << req << ": Ignoring Range: because If-Range: is stale.";
range_hdr = nullptr;
}
EvBuffer buf;
std::vector<ByteRange> ranges;
auto range_type =
internal::ParseRangeHeader(range_hdr, file->size(), &ranges);
std::string error_message;
int http_status;
const char *http_status_str;
ByteRange left;
switch (range_type) {
case internal::RangeHeaderType::kNotSatisfiable: {
std::string range_hdr = StrCat("bytes */", file->size());
evhttp_add_header(out_hdrs, "Content-Range", range_hdr.c_str());
http_status = 416;
http_status_str = "Range Not Satisfiable";
LOG(INFO) << req
<< ": Replying to non-satisfiable range request: " << range_hdr;
break;
}
case internal::RangeHeaderType::kSatisfiable:
// We only support the simpler single-range case for now.
// A multi-range request just serves the whole file via the fallthrough.
if (ranges.size() == 1) {
std::string range_hdr = StrCat("bytes ", ranges[0].begin, "-",
ranges[0].end - 1, "/", file->size());
left = ranges[0];
evhttp_add_header(out_hdrs, "Content-Range", range_hdr.c_str());
http_status = 206;
http_status_str = "Partial Content";
LOG(INFO) << req << ": URI " << evhttp_request_get_uri(req)
<< ": client requested byte range " << left
<< " (total file size " << file->size() << ")";
break;
}
// FALLTHROUGH
case internal::RangeHeaderType::kAbsentOrInvalid: {
left = ByteRange(0, file->size());
LOG(INFO) << req << ": URI " << evhttp_request_get_uri(req)
<< ": Client requested whole file of size " << file->size();
http_status = HTTP_OK;
http_status_str = "OK";
break;
}
default:
LOG(FATAL) << "unexpected range_type: " << static_cast<int>(range_type);
}
// Successful reply started; add common headers and send.
evhttp_add_header(out_hdrs, "Content-Length", StrCat(left.size()).c_str());
evhttp_add_header(out_hdrs, "Content-Type", file->mime_type().c_str());
evhttp_add_header(out_hdrs, "Accept-Ranges", "bytes");
evhttp_add_header(out_hdrs, "Last-Modified", last_modified_str);
evhttp_add_header(out_hdrs, "ETag", etag.c_str());
evhttp_send_reply_start(req, http_status, http_status_str);
ServeInProgress *serve = new ServeInProgress;
serve->file = file;
serve->left = left;
serve->req = req;
evhttp_connection *con = evhttp_request_get_connection(req);
evhttp_connection_set_closecb(con, &ServeCloseCallback, serve);
return ServeChunkCallback(con, serve);
}
} // namespace moonfire_nvr

306
src/http.h
View File

@ -1,306 +0,0 @@
// This file is part of Moonfire NVR, a security camera digital video recorder.
// Copyright (C) 2016 Scott Lamb <slamb@slamb.org>
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// In addition, as a special exception, the copyright holders give
// permission to link the code of portions of this program with the
// OpenSSL library under certain conditions as described in each
// individual source file, and distribute linked combinations including
// the two.
//
// You must obey the GNU General Public License in all respects for all
// of the code used other than OpenSSL. If you modify file(s) with this
// exception, you may extend this exception to your version of the
// file(s), but you are not obligated to do so. If you do not wish to do
// so, delete this exception statement from your version. If you delete
// this exception statement from all source files in the program, then
// also delete it here.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
//
// http.h: classes for HTTP serving. In particular, there are helpers for
// serving HTTP byte range requests with libevent.
#ifndef MOONFIRE_NVR_HTTP_H
#define MOONFIRE_NVR_HTTP_H
#include <dirent.h>
#include <stdarg.h>
#include <sys/queue.h>
#include <sys/stat.h>
#include <sys/types.h>
#include <functional>
#include <iostream>
#include <memory>
#include <string>
#include <event2/buffer.h>
#include <event2/keyvalq_struct.h>
#include <event2/http.h>
#include <glog/logging.h>
#include <re2/stringpiece.h>
#include "filesystem.h"
#include "string.h"
namespace moonfire_nvr {
// Single-use object to represent a set of HTTP query parameters.
class QueryParameters {
public:
// Parse parameters from the given URI.
// Caller should check ok() afterward.
QueryParameters(const char *uri) {
TAILQ_INIT(&me_);
ok_ = evhttp_parse_query(uri, &me_) == 0;
}
QueryParameters(const QueryParameters &) = delete;
void operator=(const QueryParameters &) = delete;
~QueryParameters() { evhttp_clear_headers(&me_); }
bool ok() const { return ok_; }
const char *Get(const char *param) const {
return evhttp_find_header(&me_, param);
}
private:
struct evkeyvalq me_;
bool ok_ = false;
};
// Wrapped version of libevent's "struct evbuffer" which uses RAII and simply
// aborts the process if allocations fail. (Moonfire NVR is intended to run on
// Linux systems with the default vm.overcommit_memory=0, so there's probably
// no point in trying to gracefully recover from a condition that's unlikely
// to ever happen.)
class EvBuffer {
public:
EvBuffer() { buf_ = CHECK_NOTNULL(evbuffer_new()); }
EvBuffer(const EvBuffer &) = delete;
EvBuffer &operator=(const EvBuffer &) = delete;
~EvBuffer() { evbuffer_free(buf_); }
struct evbuffer *get() {
return buf_;
}
void Add(const re2::StringPiece &s) {
CHECK_EQ(0, evbuffer_add(buf_, s.data(), s.size()));
}
void AddPrintf(const char *fmt, ...) __attribute__((format(printf, 2, 3))) {
va_list argp;
va_start(argp, fmt);
CHECK_LE(0, evbuffer_add_vprintf(buf_, fmt, argp));
va_end(argp);
}
// Delegates to evbuffer_add_file.
// On success, |fd| will be closed by libevent. On failure, it remains open.
bool AddFile(int fd, ev_off_t offset, ev_off_t length,
std::string *error_message);
void AddReference(const void *data, size_t datlen,
evbuffer_ref_cleanup_cb cleanupfn, void *cleanupfn_arg) {
CHECK_EQ(
0, evbuffer_add_reference(buf_, data, datlen, cleanupfn, cleanupfn_arg))
<< strerror(errno);
}
private:
struct evbuffer *buf_;
};
struct ByteRange {
ByteRange() {}
ByteRange(int64_t begin, int64_t end) : begin(begin), end(end) {}
int64_t begin = 0;
int64_t end = 0; // exclusive.
int64_t size() const { return end - begin; }
bool operator==(const ByteRange &o) const {
return begin == o.begin && end == o.end;
}
std::string DebugString() const { return StrCat("[", begin, ", ", end, ")"); }
};
inline std::ostream &operator<<(std::ostream &out, const ByteRange &range) {
return out << range.DebugString();
}
// Helper for sending HTTP errors based on POSIX error returns.
void HttpSendError(evhttp_request *req, int http_err, const std::string &prefix,
int posix_errno);
class FileSlice {
public:
virtual ~FileSlice() {}
virtual int64_t size() const = 0;
// Add some to all of the given non-empty |range| to |buf|.
// Returns the number of bytes added, or < 0 on error.
// On error, |error_message| should be populated. (|error_message| may also be
// populated if 0 <= return value < range.size(), such as if one of a
// FileSlices object's failed. However, it's safe to simply retry such
// partial failures later.)
virtual int64_t AddRange(ByteRange range, EvBuffer *buf,
std::string *error_message) const = 0;
};
class VirtualFile : public FileSlice {
public:
virtual ~VirtualFile() {}
// Return the given property of the file.
virtual time_t last_modified() const = 0;
virtual std::string etag() const = 0;
virtual std::string mime_type() const = 0;
};
class RealFileSlice : public FileSlice {
public:
// |dir| must outlive the RealFileSlice.
void Init(File *dir, re2::StringPiece filename, ByteRange range);
int64_t size() const final { return range_.size(); }
int64_t AddRange(ByteRange range, EvBuffer *buf,
std::string *error_message) const final;
private:
File *dir_;
std::string filename_;
ByteRange range_;
};
// A FileSlice of a pre-defined length which calls a function which fills the
// slice on demand. The FillerFileSlice is responsible for subsetting.
class FillerFileSlice : public FileSlice {
public:
using FillFunction =
std::function<bool(std::string *slice, std::string *error_message)>;
void Init(size_t size, FillFunction fn) {
fn_ = fn;
size_ = size;
}
int64_t size() const final { return size_; }
int64_t AddRange(ByteRange range, EvBuffer *buf,
std::string *error_message) const final;
private:
FillFunction fn_;
size_t size_;
};
// A FileSlice backed by in-memory data which outlives this object.
class StringPieceSlice : public FileSlice {
public:
StringPieceSlice() = default;
explicit StringPieceSlice(re2::StringPiece piece) : piece_(piece) {}
void Init(re2::StringPiece piece) { piece_ = piece; }
int64_t size() const final { return piece_.size(); }
int64_t AddRange(ByteRange range, EvBuffer *buf,
std::string *error_message) const final;
private:
re2::StringPiece piece_;
};
// A slice composed of other slices.
class FileSlices : public FileSlice {
public:
FileSlices() {}
FileSlices(const FileSlices &) = delete;
FileSlices &operator=(const FileSlices &) = delete;
// |slice| must outlive the FileSlices.
// |slice->size()| should not change after this call.
// |flags| should be a bitmask of Flags values below.
void Append(const FileSlice *slice, int flags = 0) {
int64_t new_size = size_ + slice->size();
slices_.emplace_back(ByteRange(size_, new_size), slice, flags);
size_ = new_size;
}
size_t num_slices() const { return slices_.size(); }
int64_t size() const final { return size_; }
int64_t AddRange(ByteRange range, EvBuffer *buf,
std::string *error_message) const final;
enum Flags {
// kLazy, as an argument to Append, instructs the FileSlices to append
// this slice in AddRange only if it is the first slice in the requested
// range. Otherwise it returns early, expecting HttpServe to call AddRange
// again after the earlier ranges have been sent. This is useful if it is
// expensive to have the given slice pending. In particular, it is useful
// when serving many file slices on 32-bit machines to avoid exhausting
// the address space with too many memory mappings.
kLazy = 1
};
private:
struct SliceInfo {
SliceInfo(ByteRange range, const FileSlice *slice, int flags)
: range(range), slice(slice), flags(flags) {}
ByteRange range;
const FileSlice *slice = nullptr;
int flags;
};
int64_t size_ = 0;
std::vector<SliceInfo> slices_;
};
// Serve an HTTP request |req| from |file|, handling byte range and
// conditional serving. (Similar to golang's http.ServeContent.)
//
// |file| will be retained as long as the request is being served.
void HttpServe(const std::shared_ptr<VirtualFile> &file, evhttp_request *req);
namespace internal {
// Value to represent result of parsing HTTP 1.1 "Range:" header.
enum class RangeHeaderType {
// Ignore the header, serving all bytes in the file.
kAbsentOrInvalid,
// The server SHOULD return a response with status 416 (Requested range not
// satisfiable).
kNotSatisfiable,
// The server SHOULD return a response with status 406 (Partial Content).
kSatisfiable
};
// Parse an HTTP 1.1 "Range:" header value, following RFC 2616 section 14.35.
// This function is for use by HttpServe; it is exposed for testing only.
//
// |value| on entry should be the header value (after the ": "), or nullptr.
// |size| on entry should be the number of bytes available to serve.
// On kSatisfiable return, |ranges| will be filled with the satisfiable ranges.
// Otherwise, its contents are undefined.
RangeHeaderType ParseRangeHeader(const char *value, int64_t size,
std::vector<ByteRange> *ranges);
} // namespace internal
} // namespace moonfire_nvr
#endif // MOONFIRE_NVR_HTTP_H

195
src/main.rs Normal file
View File

@ -0,0 +1,195 @@
// This file is part of Moonfire NVR, a security camera digital video recorder.
// Copyright (C) 2016 Scott Lamb <slamb@slamb.org>
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// In addition, as a special exception, the copyright holders give
// permission to link the code of portions of this program with the
// OpenSSL library under certain conditions as described in each
// individual source file, and distribute linked combinations including
// the two.
//
// You must obey the GNU General Public License in all respects for all
// of the code used other than OpenSSL. If you modify file(s) with this
// exception, you may extend this exception to your version of the
// file(s), but you are not obligated to do so. If you do not wish to do
// so, delete this exception statement from your version. If you delete
// this exception statement from all source files in the program, then
// also delete it here.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
#![cfg_attr(test, feature(test))]
#![feature(alloc, box_syntax, conservative_impl_trait, plugin, proc_macro)]
#![plugin(clippy)]
extern crate alloc;
extern crate byteorder;
extern crate core;
#[macro_use] extern crate chan;
extern crate chan_signal;
extern crate docopt;
#[macro_use] extern crate ffmpeg;
extern crate ffmpeg_sys;
extern crate fnv;
extern crate hyper;
#[macro_use] extern crate lazy_static;
extern crate libc;
#[macro_use] extern crate log;
extern crate lru_cache;
extern crate rusqlite;
extern crate memmap;
#[macro_use] extern crate mime;
extern crate openssl;
extern crate regex;
extern crate rustc_serialize;
extern crate serde;
#[macro_use] extern crate serde_derive;
extern crate serde_json;
extern crate slog;
extern crate slog_envlogger;
extern crate slog_stdlog;
extern crate slog_term;
extern crate smallvec;
extern crate time;
extern crate url;
extern crate uuid;
use hyper::server::Server;
use slog::DrainExt;
use std::path::Path;
use std::sync::Arc;
use std::sync::atomic::{AtomicBool, Ordering};
use std::thread;
mod db;
mod dir;
mod error;
mod h264;
mod mmapfile;
mod mp4;
mod pieces;
mod recording;
mod resource;
mod stream;
mod streamer;
#[cfg(test)] mod testutil;
mod web;
/// Commandline usage string. This is in the particular format expected by the `docopt` crate.
/// Besides being printed on --help or argument parsing error, it's actually parsed to define the
/// allowed commandline arguments and their defaults.
const USAGE: &'static str = "
Usage: moonfire-nvr [options]
moonfire-nvr (--help | --version)
Options:
-h, --help Show this message.
--version Show the version of moonfire-nvr.
--db-dir DIR Set the directory holding the SQLite3 index database.
This is typically on a flash device.
[default: /var/lib/moonfire-nvr/db]
--sample-file-dir DIR Set the directory holding video data.
This is typically on a hard drive.
[default: /var/lib/moonfire-nvr/sample]
--http-addr ADDR Set the bind address for the unencrypted HTTP server.
[default: 0.0.0.0:8080]
--read-only Forces read-only mode / disables recording.
";
/// Commandline arguments corresponding to `USAGE`; automatically filled by the `docopt` crate.
#[derive(RustcDecodable)]
struct Args {
flag_db_dir: String,
flag_sample_file_dir: String,
flag_http_addr: String,
flag_read_only: bool,
}
fn main() {
// Watch for termination signals.
// This must be started before any threads are spawned (such as the async logger thread) so
// that signals will be blocked in all threads.
let signal = chan_signal::notify(&[chan_signal::Signal::INT, chan_signal::Signal::TERM]);
// Initialize logging.
let drain = slog_term::StreamerBuilder::new().async().full().build();
let drain = slog_envlogger::new(drain);
slog_stdlog::set_logger(slog::Logger::root(drain.ignore_err(), None)).unwrap();
// Parse commandline arguments.
let version = "Moonfire NVR 0.1.0".to_owned();
let args: Args = docopt::Docopt::new(USAGE)
.and_then(|d| d.version(Some(version)).decode())
.unwrap_or_else(|e| e.exit());
// Open the database and populate cached state.
let db_dir = dir::Fd::open(&args.flag_db_dir).unwrap();
db_dir.lock(if args.flag_read_only { libc::LOCK_SH } else { libc::LOCK_EX } | libc::LOCK_NB)
.unwrap();
let conn = rusqlite::Connection::open_with_flags(
Path::new(&args.flag_db_dir).join("db"),
if args.flag_read_only {
rusqlite::SQLITE_OPEN_READ_ONLY
} else {
rusqlite::SQLITE_OPEN_READ_WRITE
} |
// rusqlite::Connection is not Sync, so there's no reason to tell SQLite3 to use the
// serialized threading mode.
rusqlite::SQLITE_OPEN_NO_MUTEX).unwrap();
let db = Arc::new(db::Database::new(conn).unwrap());
let dir = dir::SampleFileDir::new(&args.flag_sample_file_dir, db.clone()).unwrap();
info!("Database is loaded.");
// Start a streamer for each camera.
let shutdown = Arc::new(AtomicBool::new(false));
let mut streamers = Vec::new();
let syncer = if !args.flag_read_only {
let (syncer_channel, syncer_join) = dir::start_syncer(dir.clone()).unwrap();
let l = db.lock();
let cameras = l.cameras_by_id().len();
for (i, (id, camera)) in l.cameras_by_id().iter().enumerate() {
let rotate_offset_sec = streamer::ROTATE_INTERVAL_SEC * i as i64 / cameras as i64;
let mut streamer = streamer::Streamer::new(
db.clone(), dir.clone(), syncer_channel.clone(), shutdown.clone(), *id, camera,
rotate_offset_sec);
let name = format!("stream-{}", streamer.short_name());
streamers.push(thread::Builder::new().name(name).spawn(move|| {
streamer.run();
}).expect("can't create thread"));
}
Some((syncer_channel, syncer_join))
} else { None };
// Start the web interface.
let server = Server::http(args.flag_http_addr.as_str()).unwrap();
let h = web::Handler::new(db.clone(), dir.clone());
let _guard = server.handle(h);
info!("Ready to serve HTTP requests");
// Wait for a signal and shut down.
chan_select! {
signal.recv() -> signal => info!("Received signal {:?}; shutting down streamers.", signal),
}
shutdown.store(true, Ordering::SeqCst);
for streamer in streamers.drain(..) {
streamer.join().unwrap();
}
if let Some((syncer_channel, syncer_join)) = syncer {
info!("Shutting down syncer.");
drop(syncer_channel);
syncer_join.join().unwrap();
}
info!("Exiting.");
// TODO: drain the logger.
std::process::exit(0);
}

71
src/mmapfile.rs Normal file
View File

@ -0,0 +1,71 @@
// This file is part of Moonfire NVR, a security camera digital video recorder.
// Copyright (C) 2016 Scott Lamb <slamb@slamb.org>
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// In addition, as a special exception, the copyright holders give
// permission to link the code of portions of this program with the
// OpenSSL library under certain conditions as described in each
// individual source file, and distribute linked combinations including
// the two.
//
// You must obey the GNU General Public License in all respects for all
// of the code used other than OpenSSL. If you modify file(s) with this
// exception, you may extend this exception to your version of the
// file(s), but you are not obligated to do so. If you do not wish to do
// so, delete this exception statement from your version. If you delete
// this exception statement from all source files in the program, then
// also delete it here.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
//! Memory-mapped file serving.
extern crate memmap;
use error::Result;
use std::fs::File;
use std::io;
use std::ops::Range;
/// Memory-mapped file slice.
/// This struct is meant to be used in constructing an implementation of the `resource::Resource`
/// or `pieces::ContextWriter` traits. The file in question should be immutable, as files shrinking
/// during `mmap` will cause the process to fail with `SIGBUS`. Moonfire NVR sample files satisfy
/// this requirement:
///
/// * They should only be modified by Moonfire NVR itself. Installation instructions encourage
/// creating a dedicated user/group for Moonfire NVR and ensuring only this group has
/// permissions to Moonfire NVR's directories.
/// * Moonfire NVR never modifies sample files after inserting their matching recording entries
/// into the database. They are kept as-is until they are deleted.
pub struct MmapFileSlice {
f: File,
range: Range<u64>,
}
impl MmapFileSlice {
pub fn new(f: File, range: Range<u64>) -> MmapFileSlice {
MmapFileSlice{f: f, range: range}
}
pub fn write_to(&self, range: Range<u64>, out: &mut io::Write) -> Result<()> {
// TODO: overflow check (in case u64 is larger than usize).
let r = self.range.start + range.start .. self.range.start + range.end;
assert!(r.end <= self.range.end,
"requested={:?} within={:?}", range, self.range);
let mmap = memmap::Mmap::open_with_offset(
&self.f, memmap::Protection::Read, r.start as usize, (r.end - r.start) as usize)?;
unsafe { out.write_all(mmap.as_slice())?; }
Ok(())
}
}

458
src/moonfire-db-test.cc
View File

@ -1,458 +0,0 @@
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// In addition, as a special exception, the copyright holders give
// permission to link the code of portions of this program with the
// OpenSSL library under certain conditions as described in each
// individual source file, and distribute linked combinations including
// the two.
//
// You must obey the GNU General Public License in all respects for all
// of the code used other than OpenSSL. If you modify file(s) with this
// exception, you may extend this exception to your version of the
// file(s), but you are not obligated to do so. If you do not wish to do
// so, delete this exception statement from your version. If you delete
// this exception statement from all source files in the program, then
// also delete it here.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
//
// moonfire-db-test.cc: tests of the moonfire-db.h interface.
#include <time.h>
#include <string>
#include <gflags/gflags.h>
#include <gmock/gmock.h>
#include <gtest/gtest.h>
#include "moonfire-db.h"
#include "sqlite.h"
#include "string.h"
#include "testutil.h"
DECLARE_bool(alsologtostderr);
using testing::_;
using testing::HasSubstr;
using testing::DoAll;
using testing::Return;
using testing::SetArgPointee;
namespace moonfire_nvr {
namespace {
class MoonfireDbTest : public testing::Test {
protected:
MoonfireDbTest() {
tmpdir_ = PrepareTempDirOrDie("moonfire-db-test");
std::string error_message;
CHECK(db_.Open(StrCat(tmpdir_, "/db").c_str(),
SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE, &error_message))
<< error_message;
std::string create_sql = ReadFileOrDie("../src/schema.sql");
DatabaseContext ctx(&db_);
CHECK(RunStatements(&ctx, create_sql, &error_message)) << error_message;
}
int64_t AddCamera(Uuid uuid, re2::StringPiece short_name) {
DatabaseContext ctx(&db_);
auto run = ctx.UseOnce(
R"(
insert into camera (uuid, short_name, host, username, password,
main_rtsp_path, sub_rtsp_path, retain_bytes)
values (:uuid, :short_name, :host, :username, :password,
:main_rtsp_path, :sub_rtsp_path, :retain_bytes);
)");
run.BindBlob(":uuid", uuid.binary_view());
run.BindText(":short_name", short_name);
run.BindText(":host", "test-camera");
run.BindText(":username", "foo");
run.BindText(":password", "bar");
run.BindText(":main_rtsp_path", "/main");
run.BindText(":sub_rtsp_path", "/sub");
run.BindInt64(":retain_bytes", 42);
CHECK_EQ(SQLITE_DONE, run.Step()) << run.error_message();
if (run.Step() != SQLITE_DONE) {
ADD_FAILURE() << run.error_message();
return -1;
}
return ctx.last_insert_rowid();
}
void ExpectNoRecordings(Uuid camera_uuid) {
int rows = 0;
mdb_->ListCameras([&](const ListCamerasRow &row) {
++rows;
EXPECT_EQ(camera_uuid, row.uuid);
EXPECT_EQ("test-camera", row.host);
EXPECT_EQ("foo", row.username);
EXPECT_EQ("bar", row.password);
EXPECT_EQ("/main", row.main_rtsp_path);
EXPECT_EQ("/sub", row.sub_rtsp_path);
EXPECT_EQ(42, row.retain_bytes);
EXPECT_EQ(std::numeric_limits<int64_t>::max(), row.min_start_time_90k);
EXPECT_EQ(std::numeric_limits<int64_t>::min(), row.max_end_time_90k);
EXPECT_EQ(0, row.total_duration_90k);
EXPECT_EQ(0, row.total_sample_file_bytes);
return IterationControl::kContinue;
});
EXPECT_EQ(1, rows);
GetCameraRow row;
EXPECT_TRUE(mdb_->GetCamera(camera_uuid, &row));
EXPECT_THAT(row.days, testing::ElementsAre());
std::string error_message;
rows = 0;
EXPECT_TRUE(mdb_->ListCameraRecordings(
camera_uuid, 0, std::numeric_limits<int64_t>::max(),
[&](const ListCameraRecordingsRow &row) {
++rows;
return IterationControl::kBreak;
},
&error_message))
<< error_message;
EXPECT_EQ(0, rows);
rows = 0;
EXPECT_TRUE(mdb_->ListMp4Recordings(
camera_uuid, 0, std::numeric_limits<int64_t>::max(),
[&](Recording &recording, const VideoSampleEntry &entry) {
++rows;
return IterationControl::kBreak;
},
&error_message))
<< error_message;
EXPECT_EQ(0, rows);
}
void ExpectSingleRecording(Uuid camera_uuid, const Recording &recording,
const VideoSampleEntry &entry,
ListOldestSampleFilesRow *save_oldest_row) {
std::string error_message;
int rows = 0;
mdb_->ListCameras([&](const ListCamerasRow &row) {
++rows;
EXPECT_EQ(camera_uuid, row.uuid);
EXPECT_EQ(recording.start_time_90k, row.min_start_time_90k);
EXPECT_EQ(recording.end_time_90k, row.max_end_time_90k);
EXPECT_EQ(recording.end_time_90k - recording.start_time_90k,
row.total_duration_90k);
EXPECT_EQ(recording.sample_file_bytes, row.total_sample_file_bytes);
return IterationControl::kContinue;
});
EXPECT_EQ(1, rows);
std::map<std::string, int64_t> expected_days;
internal::AdjustDaysMap(recording.start_time_90k, recording.end_time_90k, 1,
&expected_days);
GetCameraRow row;
EXPECT_TRUE(mdb_->GetCamera(camera_uuid, &row));
EXPECT_THAT(row.days, testing::Eq(expected_days));
GetCameraRow camera_row;
EXPECT_TRUE(mdb_->GetCamera(camera_uuid, &camera_row));
EXPECT_EQ(recording.start_time_90k, camera_row.min_start_time_90k);
EXPECT_EQ(recording.end_time_90k, camera_row.max_end_time_90k);
EXPECT_EQ(recording.end_time_90k - recording.start_time_90k,
camera_row.total_duration_90k);
EXPECT_EQ(recording.sample_file_bytes, camera_row.total_sample_file_bytes);
rows = 0;
EXPECT_TRUE(mdb_->ListCameraRecordings(
camera_uuid, 0, std::numeric_limits<int64_t>::max(),
[&](const ListCameraRecordingsRow &row) {
++rows;
EXPECT_EQ(recording.start_time_90k, row.start_time_90k);
EXPECT_EQ(recording.end_time_90k, row.end_time_90k);
EXPECT_EQ(recording.video_samples, row.video_samples);
EXPECT_EQ(recording.sample_file_bytes, row.sample_file_bytes);
EXPECT_EQ(entry.sha1, row.video_sample_entry_sha1);
EXPECT_EQ(entry.width, row.width);
EXPECT_EQ(entry.height, row.height);
return IterationControl::kContinue;
},
&error_message))
<< error_message;
EXPECT_EQ(1, rows);
rows = 0;
EXPECT_TRUE(mdb_->ListOldestSampleFiles(
camera_uuid,
[&](const ListOldestSampleFilesRow &row) {
++rows;
EXPECT_EQ(recording.id, row.recording_id);
EXPECT_EQ(recording.sample_file_uuid, row.sample_file_uuid);
EXPECT_EQ(recording.end_time_90k - recording.start_time_90k,
row.duration_90k);
EXPECT_EQ(recording.sample_file_bytes, row.sample_file_bytes);
*save_oldest_row = row;
return IterationControl::kContinue;
},
&error_message))
<< error_message;
EXPECT_EQ(1, rows);
rows = 0;
EXPECT_TRUE(mdb_->ListMp4Recordings(
camera_uuid, 0, std::numeric_limits<int64_t>::max(),
[&](Recording &some_recording, const VideoSampleEntry &some_entry) {
++rows;
EXPECT_EQ(recording.id, some_recording.id);
EXPECT_EQ(recording.camera_id, some_recording.camera_id);
EXPECT_EQ(recording.sample_file_sha1,
some_recording.sample_file_sha1);
EXPECT_EQ(recording.sample_file_uuid,
some_recording.sample_file_uuid);
EXPECT_EQ(recording.video_sample_entry_id,
some_recording.video_sample_entry_id);
EXPECT_EQ(recording.start_time_90k, some_recording.start_time_90k);
EXPECT_EQ(recording.end_time_90k, some_recording.end_time_90k);
EXPECT_EQ(recording.sample_file_bytes,
some_recording.sample_file_bytes);
EXPECT_EQ(recording.video_samples, some_recording.video_samples);
EXPECT_EQ(recording.video_sync_samples,
some_recording.video_sync_samples);
EXPECT_EQ(recording.video_index, some_recording.video_index);
EXPECT_EQ(entry.id, some_entry.id);
EXPECT_EQ(entry.sha1, some_entry.sha1);
EXPECT_EQ(entry.data, some_entry.data);
EXPECT_EQ(entry.width, some_entry.width);
EXPECT_EQ(entry.height, some_entry.height);
return IterationControl::kContinue;
},
&error_message))
<< error_message;
EXPECT_EQ(1, rows);
}
std::string tmpdir_;
Database db_;
std::unique_ptr<MoonfireDatabase> mdb_;
};
TEST(AdjustDaysMapTest, Basic) {
std::map<std::string, int64_t> days;
// Create a day.
const int64_t kTestTime = INT64_C(130647162600000); // 2015-12-31 23:59:00
moonfire_nvr::internal::AdjustDaysMap(
kTestTime, kTestTime + 60 * kTimeUnitsPerSecond, 1, &days);
EXPECT_THAT(days, testing::ElementsAre(std::make_pair(
"2015-12-31", 60 * kTimeUnitsPerSecond)));
// Add to a day.
moonfire_nvr::internal::AdjustDaysMap(
kTestTime, kTestTime + 60 * kTimeUnitsPerSecond, 1, &days);
EXPECT_THAT(days, testing::ElementsAre(std::make_pair(
"2015-12-31", 120 * kTimeUnitsPerSecond)));
// Subtract from a day.
moonfire_nvr::internal::AdjustDaysMap(
kTestTime, kTestTime + 60 * kTimeUnitsPerSecond, -1, &days);
EXPECT_THAT(days, testing::ElementsAre(std::make_pair(
"2015-12-31", 60 * kTimeUnitsPerSecond)));
// Remove a day.
moonfire_nvr::internal::AdjustDaysMap(
kTestTime, kTestTime + 60 * kTimeUnitsPerSecond, -1, &days);
EXPECT_THAT(days, testing::ElementsAre());
// Create two days.
moonfire_nvr::internal::AdjustDaysMap(
kTestTime, kTestTime + 3 * 60 * kTimeUnitsPerSecond, 1, &days);
EXPECT_THAT(days,
testing::ElementsAre(
std::make_pair("2015-12-31", 1 * 60 * kTimeUnitsPerSecond),
std::make_pair("2016-01-01", 2 * 60 * kTimeUnitsPerSecond)));
// Add to two days.
moonfire_nvr::internal::AdjustDaysMap(
kTestTime, kTestTime + 3 * 60 * kTimeUnitsPerSecond, 1, &days);
EXPECT_THAT(days,
testing::ElementsAre(
std::make_pair("2015-12-31", 2 * 60 * kTimeUnitsPerSecond),
std::make_pair("2016-01-01", 4 * 60 * kTimeUnitsPerSecond)));
// Subtract from two days.
moonfire_nvr::internal::AdjustDaysMap(
kTestTime, kTestTime + 3 * 60 * kTimeUnitsPerSecond, -1, &days);
EXPECT_THAT(days,
testing::ElementsAre(
std::make_pair("2015-12-31", 1 * 60 * kTimeUnitsPerSecond),
std::make_pair("2016-01-01", 2 * 60 * kTimeUnitsPerSecond)));
// Remove two days.
moonfire_nvr::internal::AdjustDaysMap(
kTestTime, kTestTime + 3 * 60 * kTimeUnitsPerSecond, -1, &days);
EXPECT_THAT(days, testing::ElementsAre());
}
TEST(GetDayBoundsTest, Basic) {
int64_t start_90k;
int64_t end_90k;
std::string error_msg;
// Normal day.
EXPECT_TRUE(GetDayBounds("2015-12-31", &start_90k, &end_90k, &error_msg))
<< error_msg;
EXPECT_EQ(INT64_C(130639392000000), start_90k);
EXPECT_EQ(INT64_C(130647168000000), end_90k);
// Spring forward (23-hour day).
EXPECT_TRUE(GetDayBounds("2016-03-13", &start_90k, &end_90k, &error_msg));
EXPECT_EQ(INT64_C(131207040000000), start_90k);
EXPECT_EQ(INT64_C(131214492000000), end_90k);
// Fall back (25-hour day).
EXPECT_TRUE(GetDayBounds("2016-11-06", &start_90k, &end_90k, &error_msg));
EXPECT_EQ(INT64_C(133057404000000), start_90k);
EXPECT_EQ(INT64_C(133065504000000), end_90k);
// Unparseable day.
EXPECT_FALSE(GetDayBounds("xxxx-xx-xx", &start_90k, &end_90k, &error_msg));
}
// Basic test of running some queries on an empty database.
TEST_F(MoonfireDbTest, EmptyDatabase) {
std::string error_message;
mdb_.reset(new MoonfireDatabase);
ASSERT_TRUE(mdb_->Init(&db_, &error_message)) << error_message;
mdb_->ListCameras([&](const ListCamerasRow &row) {
ADD_FAILURE() << "row unexpected";
return IterationControl::kBreak;
});
GetCameraRow get_camera_row;
EXPECT_FALSE(mdb_->GetCamera(Uuid(), &get_camera_row));
EXPECT_FALSE(
mdb_->ListCameraRecordings(Uuid(), 0, std::numeric_limits<int64_t>::max(),
[&](const ListCameraRecordingsRow &row) {
ADD_FAILURE() << "row unexpected";
return IterationControl::kBreak;
},
&error_message));
EXPECT_FALSE(mdb_->ListMp4Recordings(
Uuid(), 0, std::numeric_limits<int64_t>::max(),
[&](Recording &recording, const VideoSampleEntry &entry) {
ADD_FAILURE() << "row unexpected";
return IterationControl::kBreak;
},
&error_message));
}
// Basic test of the full lifecycle of recording.
// Does not exercise many error cases.
TEST_F(MoonfireDbTest, FullLifecycle) {
std::string error_message;
const char kCameraShortName[] = "testcam";
Uuid camera_uuid = GetRealUuidGenerator()->Generate();
int64_t camera_id = AddCamera(camera_uuid, kCameraShortName);
ASSERT_GT(camera_id, 0);
mdb_.reset(new MoonfireDatabase);
ASSERT_TRUE(mdb_->Init(&db_, &error_message)) << error_message;
ExpectNoRecordings(camera_uuid);
std::vector<Uuid> reserved;
EXPECT_TRUE(mdb_->ListReservedSampleFiles(&reserved, &error_message))
<< error_message;
EXPECT_THAT(reserved, testing::IsEmpty());
std::vector<Uuid> uuids = mdb_->ReserveSampleFiles(2, &error_message);
ASSERT_THAT(uuids, testing::SizeIs(2)) << error_message;
EXPECT_TRUE(mdb_->ListReservedSampleFiles(&reserved, &error_message))
<< error_message;
EXPECT_THAT(reserved, testing::UnorderedElementsAre(uuids[0], uuids[1]));
VideoSampleEntry entry;
entry.sha1.resize(20);
entry.width = 768;
entry.height = 512;
entry.data.resize(100);
ASSERT_TRUE(mdb_->InsertVideoSampleEntry(&entry, &error_message))
<< error_message;
ASSERT_GT(entry.id, 0);
Recording recording;
recording.camera_id = camera_id;
recording.sample_file_uuid = GetRealUuidGenerator()->Generate();
recording.video_sample_entry_id = entry.id;
SampleIndexEncoder encoder;
encoder.Init(&recording, UINT64_C(1430006400) * kTimeUnitsPerSecond);
encoder.AddSample(kTimeUnitsPerSecond, 42, true);
// Inserting a recording should succeed and remove its uuid from the
// reserved table.
ASSERT_FALSE(mdb_->InsertRecording(&recording, &error_message));
EXPECT_THAT(error_message, testing::HasSubstr("not reserved"));
recording.sample_file_uuid = uuids.back();
recording.sample_file_sha1.resize(20);
ASSERT_TRUE(mdb_->InsertRecording(&recording, &error_message))
<< error_message;
ASSERT_GT(recording.id, 0);
EXPECT_TRUE(mdb_->ListReservedSampleFiles(&reserved, &error_message))
<< error_message;
EXPECT_THAT(reserved, testing::ElementsAre(uuids[0]));
// Queries should return the correct result (with caches updated on insert).
ListOldestSampleFilesRow oldest;
ExpectSingleRecording(camera_uuid, recording, entry, &oldest);
// Queries on a fresh database should return the correct result (with caches
// populated from existing database contents).
mdb_.reset(new MoonfireDatabase);
ASSERT_TRUE(mdb_->Init(&db_, &error_message)) << error_message;
ExpectSingleRecording(camera_uuid, recording, entry, &oldest);
// Deleting a recording should succeed, update the min/max times, and mark
// the uuid as reserved.
std::vector<ListOldestSampleFilesRow> to_delete;
to_delete.push_back(oldest);
ASSERT_TRUE(mdb_->DeleteRecordings(to_delete, &error_message))
<< error_message;
EXPECT_TRUE(mdb_->ListReservedSampleFiles(&reserved, &error_message))
<< error_message;
EXPECT_THAT(reserved, testing::UnorderedElementsAre(uuids[0], uuids[1]));
ExpectNoRecordings(camera_uuid);
EXPECT_TRUE(mdb_->MarkSampleFilesDeleted(uuids, &error_message))
<< error_message;
EXPECT_TRUE(mdb_->ListReservedSampleFiles(&reserved, &error_message))
<< error_message;
EXPECT_THAT(reserved, testing::IsEmpty());
}
} // namespace
} // namespace moonfire_nvr
int main(int argc, char **argv) {
FLAGS_alsologtostderr = true;
google::ParseCommandLineFlags(&argc, &argv, true);
testing::InitGoogleTest(&argc, argv);
google::InitGoogleLogging(argv[0]);
// The calendar day math assumes this timezone.
CHECK_EQ(0, setenv("TZ", "America/Los_Angeles", 1)) << strerror(errno);
tzset();
return RUN_ALL_TESTS();
}

954
src/moonfire-db.cc
View File

@ -1,954 +0,0 @@
// This file is part of Moonfire NVR, a security camera network video recorder.
// Copyright (C) 2016 Scott Lamb <slamb@slamb.org>
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// In addition, as a special exception, the copyright holders give
// permission to link the code of portions of this program with the
// OpenSSL library under certain conditions as described in each
// individual source file, and distribute linked combinations including
// the two.
//
// You must obey the GNU General Public License in all respects for all
// of the code used other than OpenSSL. If you modify file(s) with this
// exception, you may extend this exception to your version of the
// file(s), but you are not obligated to do so. If you do not wish to do
// so, delete this exception statement from your version. If you delete
// this exception statement from all source files in the program, then
// also delete it here.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
//
// moonfire-db.cc: implementation of moonfire-db.h interface.
// see top-level comments there on performance & efficiency.
#include "moonfire-db.h"
#include <time.h>
#include <string>
#include <glog/logging.h>
#include "http.h"
#include "mp4.h"
#include "recording.h"
namespace moonfire_nvr {
namespace {
const char kDayFmt[] = "%Y-%m-%d";
constexpr size_t kDayFmtBufSize = sizeof("YYYY-mm-DD");
// Helper for AdjustDaysMap.
void AdjustDay(const std::string &day, int64_t delta,
std::map<std::string, int64_t> *days) {
auto it = days->find(day);
if (it != days->end()) {
it->second += delta;
DCHECK_GE(it->second, 0) << day << ", " << delta;
if (it->second == 0) {
days->erase(it);
}
} else {
days->insert(it, std::make_pair(day, delta));
}
}
} // namespace
namespace internal {
void AdjustDaysMap(int64_t start_time_90k, int64_t end_time_90k, int sign,
std::map<std::string, int64_t> *days) {
// There will always be at most two days adjusted, because
// kMaxRecordingDuration is less than a day (even during spring forward).
DCHECK_LE(start_time_90k, end_time_90k);
DCHECK_LE(end_time_90k - start_time_90k, kMaxRecordingDuration);
static_assert(kMaxRecordingDuration <= 23 * 60 * kTimeUnitsPerSecond,
"max duration should be less than a (spring-forward) day");
if (start_time_90k == end_time_90k) {
return;
}
// Fill |buf| with the first day.
struct tm mytm;
memset(&mytm, 0, sizeof(mytm));
time_t start_ts = start_time_90k / kTimeUnitsPerSecond;
localtime_r(&start_ts, &mytm);
char buf[kDayFmtBufSize];
strftime(buf, sizeof(buf), kDayFmt, &mytm);
// Determine the start of the next day.
// Note that mktime(3) normalizes tm_mday, so this should work on the last
// day of the month/year.
mytm.tm_isdst = -1;
mytm.tm_hour = 0;
mytm.tm_min = 0;
mytm.tm_sec = 0;
++mytm.tm_mday;
auto boundary_90k = kTimeUnitsPerSecond * static_cast<int64_t>(mktime(&mytm));
// Adjust the first day.
auto first_day_delta =
sign * (std::min(end_time_90k, boundary_90k) - start_time_90k);
DCHECK_NE(first_day_delta, 0) << "start=" << start_time_90k
<< ", end=" << end_time_90k;
AdjustDay(buf, first_day_delta, days);
if (end_time_90k <= boundary_90k) {
return; // no second day.
}
// Fill |buf| with the second day.
strftime(buf, sizeof(buf), kDayFmt, &mytm);
// Adjust the second day.
auto second_day_delta = sign * (end_time_90k - boundary_90k);
DCHECK_NE(second_day_delta, 0) << "start=" << start_time_90k
<< ", end=" << end_time_90k;
AdjustDay(buf, second_day_delta, days);
}
} // namespace internal
bool GetDayBounds(const std::string &day, int64_t *start_time_90k,
int64_t *end_time_90k, std::string *error_message) {
struct tm mytm;
memset(&mytm, 0, sizeof(mytm));
if (strptime(day.c_str(), kDayFmt, &mytm) != day.c_str() + day.size()) {
*error_message = StrCat("Unparseable day: ", day);
return false;
}
mytm.tm_isdst = -1;
mytm.tm_hour = 0;
mytm.tm_min = 0;
mytm.tm_sec = 0;
*start_time_90k = kTimeUnitsPerSecond * static_cast<int64_t>(mktime(&mytm));
mytm.tm_isdst = -1;
mytm.tm_hour = 0;
mytm.tm_min = 0;
mytm.tm_sec = 0;
++mytm.tm_mday;
*end_time_90k = kTimeUnitsPerSecond * static_cast<int64_t>(mktime(&mytm));
return true;
}
bool MoonfireDatabase::Init(Database *db, std::string *error_message) {
CHECK(db_ == nullptr);
db_ = db;
{
DatabaseContext ctx(db_);
auto list_cameras_run = ctx.UseOnce(
R"(
select
camera.id,
camera.uuid,
camera.short_name,
camera.description,
camera.host,
camera.username,
camera.password,
camera.main_rtsp_path,
camera.sub_rtsp_path,
camera.retain_bytes
from
camera;
)");
while (list_cameras_run.Step() == SQLITE_ROW) {
CameraData data;
data.id = list_cameras_run.ColumnInt64(0);
Uuid uuid;
if (!uuid.ParseBinary(list_cameras_run.ColumnBlob(1))) {
*error_message =
StrCat("bad uuid ", ToHex(list_cameras_run.ColumnBlob(1)),
" for camera id ", data.id);
return false;
}
data.short_name = list_cameras_run.ColumnText(2).as_string();
data.description = list_cameras_run.ColumnText(3).as_string();
data.host = list_cameras_run.ColumnText(4).as_string();
data.username = list_cameras_run.ColumnText(5).as_string();
data.password = list_cameras_run.ColumnText(6).as_string();
data.main_rtsp_path = list_cameras_run.ColumnText(7).as_string();
data.sub_rtsp_path = list_cameras_run.ColumnText(8).as_string();
data.retain_bytes = list_cameras_run.ColumnInt64(9);
auto ret = cameras_by_uuid_.insert(std::make_pair(uuid, data));
if (!ret.second) {
*error_message = StrCat("Duplicate camera uuid ", uuid.UnparseText());
return false;
}
CameraData *data_p = &ret.first->second;
if (!cameras_by_id_.insert(std::make_pair(data.id, data_p)).second) {
*error_message = StrCat("Duplicate camera id ", data.id);
return false;
}
}
if (list_cameras_run.status() != SQLITE_DONE) {
*error_message = StrCat("Camera list query failed: ",
list_cameras_run.error_message());
return false;
}
// This query scans the entirety of the recording table's index.
// It is quite slow, so the results are cached.
auto list_recordings_run = ctx.UseOnce(
R"(
select
recording.start_time_90k,
recording.duration_90k,
recording.sample_file_bytes,
recording.camera_id
from
recording
)");
while (list_recordings_run.Step() == SQLITE_ROW) {
int64_t start_time_90k = list_recordings_run.ColumnInt64(0);
int64_t duration_90k = list_recordings_run.ColumnInt64(1);
int64_t end_time_90k = start_time_90k + duration_90k;
int64_t sample_file_bytes = list_recordings_run.ColumnInt64(2);
int64_t camera_id = list_recordings_run.ColumnInt64(3);
auto it = cameras_by_id_.find(camera_id);
if (it == cameras_by_id_.end()) {
*error_message =
StrCat("Recording refers to unknown camera ", camera_id);
return false;
}
CameraData *data = it->second;
data->min_start_time_90k =
std::min(data->min_start_time_90k, start_time_90k);
data->max_end_time_90k = std::max(data->max_end_time_90k, end_time_90k);
data->total_sample_file_bytes += sample_file_bytes;
data->total_duration_90k += duration_90k;
internal::AdjustDaysMap(start_time_90k, end_time_90k, 1, &data->days);
}
if (list_cameras_run.status() != SQLITE_DONE) {
*error_message = StrCat("Recording list query failed: ",
list_recordings_run.error_message());
return false;
}
// It's simplest to just keep the video sample entries in RAM.
auto video_sample_entries_run = ctx.UseOnce(
R"(
select
id,
sha1,
width,
height,
data
from
video_sample_entry
)");
while (video_sample_entries_run.Step() == SQLITE_ROW) {
VideoSampleEntry entry;
entry.id = video_sample_entries_run.ColumnInt64(0);
entry.sha1 = video_sample_entries_run.ColumnBlob(1).as_string();
int64_t width_tmp = video_sample_entries_run.ColumnInt64(2);
int64_t height_tmp = video_sample_entries_run.ColumnInt64(3);
auto max = std::numeric_limits<uint16_t>::max();
if (width_tmp <= 0 || width_tmp > max || height_tmp <= 0 ||
height_tmp > max) {
*error_message =
StrCat("video_sample_entry id ", entry.id, " width ", width_tmp,
" / height ", height_tmp, " out of range.");
return false;
}
entry.width = width_tmp;
entry.height = height_tmp;
entry.data = video_sample_entries_run.ColumnBlob(4).as_string();
CHECK(
video_sample_entries_.insert(std::make_pair(entry.id, entry)).second)
<< "duplicate: " << entry.id;
}
}
std::string list_camera_recordings_sql = StrCat(
R"(
select
recording.start_time_90k,
recording.duration_90k,
recording.video_samples,
recording.sample_file_bytes,
recording.video_sample_entry_id
from
recording
where
camera_id = :camera_id and
recording.start_time_90k > :start_time_90k - )",
kMaxRecordingDuration, " and\n",
R"(
recording.start_time_90k < :end_time_90k and
recording.start_time_90k + recording.duration_90k > :start_time_90k
order by
recording.start_time_90k desc;)");
list_camera_recordings_stmt_ =
db_->Prepare(list_camera_recordings_sql, nullptr, error_message);
if (!list_camera_recordings_stmt_.valid()) {
return false;
}
std::string build_mp4_sql = StrCat(
R"(
select
recording.id,
recording.start_time_90k,
recording.duration_90k,
recording.sample_file_bytes,
recording.sample_file_uuid,
recording.sample_file_sha1,
recording.video_index,
recording.video_samples,
recording.video_sync_samples,
recording.video_sample_entry_id
from
recording
where
camera_id = :camera_id and
recording.start_time_90k > :start_time_90k - )",
kMaxRecordingDuration, " and\n",
R"(
recording.start_time_90k < :end_time_90k and
recording.start_time_90k + recording.duration_90k > :start_time_90k
order by
recording.start_time_90k;)");
build_mp4_stmt_ = db_->Prepare(build_mp4_sql, nullptr, error_message);
if (!build_mp4_stmt_.valid()) {
return false;
}
insert_reservation_stmt_ = db_->Prepare(
"insert into reserved_sample_files (uuid, state)\n"
" values (:uuid, :state);",
nullptr, error_message);
if (!insert_reservation_stmt_.valid()) {
return false;
}
delete_reservation_stmt_ =
db_->Prepare("delete from reserved_sample_files where uuid = :uuid;",
nullptr, error_message);
if (!delete_reservation_stmt_.valid()) {
return false;
}
insert_video_sample_entry_stmt_ = db_->Prepare(
R"(
insert into video_sample_entry (sha1, width, height, data)
values (:sha1, :width, :height, :data);
)",
nullptr, error_message);
if (!insert_video_sample_entry_stmt_.valid()) {
return false;
}
insert_recording_stmt_ = db_->Prepare(
R"(
insert into recording (camera_id, sample_file_bytes, start_time_90k,
duration_90k, local_time_delta_90k, video_samples,
video_sync_samples, video_sample_entry_id,
sample_file_uuid, sample_file_sha1, video_index)
values (:camera_id, :sample_file_bytes, :start_time_90k,
:duration_90k, :local_time_delta_90k,
:video_samples, :video_sync_samples,
:video_sample_entry_id, :sample_file_uuid,
:sample_file_sha1, :video_index);
)",
nullptr, error_message);
if (!insert_recording_stmt_.valid()) {
return false;
}
list_oldest_sample_files_stmt_ = db_->Prepare(
R"(
select
id,
sample_file_uuid,
start_time_90k,
duration_90k,
sample_file_bytes
from
recording
where
camera_id = :camera_id
order by
start_time_90k
)",
nullptr, error_message);
if (!list_oldest_sample_files_stmt_.valid()) {
return false;
}
delete_recording_stmt_ =
db_->Prepare("delete from recording where id = :recording_id;", nullptr,
error_message);
if (!delete_recording_stmt_.valid()) {
return false;
}
camera_min_start_stmt_ = db_->Prepare(
R"(
select
start_time_90k
from
recording
where
camera_id = :camera_id
order by start_time_90k limit 1;
)",
nullptr, error_message);
if (!camera_min_start_stmt_.valid()) {
return false;
}
camera_max_start_stmt_ = db_->Prepare(
R"(
select
start_time_90k,
duration_90k
from
recording
where
camera_id = :camera_id
order by start_time_90k desc;
)",
nullptr, error_message);
if (!camera_max_start_stmt_.valid()) {
return false;
}
return true;
}
void MoonfireDatabase::ListCameras(
std::function<IterationControl(const ListCamerasRow &)> cb) {
DatabaseContext ctx(db_);
ListCamerasRow row;
for (const auto &entry : cameras_by_uuid_) {
row.id = entry.second.id;
row.uuid = entry.first;
row.short_name = entry.second.short_name;
row.description = entry.second.description;
row.host = entry.second.host;
row.username = entry.second.username;
row.password = entry.second.password;
row.main_rtsp_path = entry.second.main_rtsp_path;
row.sub_rtsp_path = entry.second.sub_rtsp_path;
row.retain_bytes = entry.second.retain_bytes;
row.min_start_time_90k = entry.second.min_start_time_90k;
row.max_end_time_90k = entry.second.max_end_time_90k;
row.total_duration_90k = entry.second.total_duration_90k;
row.total_sample_file_bytes = entry.second.total_sample_file_bytes;
if (cb(row) == IterationControl::kBreak) {
return;
}
}
return;
}
bool MoonfireDatabase::GetCamera(Uuid camera_uuid, GetCameraRow *row) {
DatabaseContext ctx(db_);
const auto it = cameras_by_uuid_.find(camera_uuid);
if (it == cameras_by_uuid_.end()) {
return false;
}
const CameraData &data = it->second;
row->short_name = data.short_name;
row->description = data.description;
row->retain_bytes = data.retain_bytes;
row->min_start_time_90k = data.min_start_time_90k;
row->max_end_time_90k = data.max_end_time_90k;
row->total_duration_90k = data.total_duration_90k;
row->total_sample_file_bytes = data.total_sample_file_bytes;
row->days = data.days;
return true;
}
bool MoonfireDatabase::ListCameraRecordings(
Uuid camera_uuid, int64_t start_time_90k, int64_t end_time_90k,
std::function<IterationControl(const ListCameraRecordingsRow &)> cb,
std::string *error_message) {
DatabaseContext ctx(db_);
const auto camera_it = cameras_by_uuid_.find(camera_uuid);
if (camera_it == cameras_by_uuid_.end()) {
*error_message = StrCat("no such camera ", camera_uuid.UnparseText());
return false;
}
auto run = ctx.Borrow(&list_camera_recordings_stmt_);
run.BindInt64(":camera_id", camera_it->second.id);
run.BindInt64(":start_time_90k", start_time_90k);
run.BindInt64(":end_time_90k", end_time_90k);
ListCameraRecordingsRow row;
while (run.Step() == SQLITE_ROW) {
row.start_time_90k = run.ColumnInt64(0);
row.end_time_90k = row.start_time_90k + run.ColumnInt64(1);
row.video_samples = run.ColumnInt64(2);
row.sample_file_bytes = run.ColumnInt64(3);
int64_t video_sample_entry_id = run.ColumnInt64(4);
const auto it = video_sample_entries_.find(video_sample_entry_id);
if (it == video_sample_entries_.end()) {
*error_message =
StrCat("recording references invalid video sample entry ",
video_sample_entry_id);
return false;
}
const VideoSampleEntry &entry = it->second;
row.video_sample_entry_sha1 = entry.sha1;
row.width = entry.width;
row.height = entry.height;
if (cb(row) == IterationControl::kBreak) {
return true;
}
}
if (run.status() != SQLITE_DONE) {
*error_message = StrCat("sqlite query failed: ", run.error_message());
return false;
}
return true;
}
bool MoonfireDatabase::ListMp4Recordings(
Uuid camera_uuid, int64_t start_time_90k, int64_t end_time_90k,
std::function<IterationControl(Recording &, const VideoSampleEntry &)>
row_cb,
std::string *error_message) {
VLOG(1) << "...(1/4): Waiting for database lock";
DatabaseContext ctx(db_);
const auto it = cameras_by_uuid_.find(camera_uuid);
if (it == cameras_by_uuid_.end()) {
*error_message = StrCat("no such camera ", camera_uuid.UnparseText());
return false;
}
const CameraData &data = it->second;
VLOG(1) << "...(2/4): Querying database";
auto run = ctx.Borrow(&build_mp4_stmt_);
run.BindInt64(":camera_id", data.id);
run.BindInt64(":end_time_90k", end_time_90k);
run.BindInt64(":start_time_90k", start_time_90k);
Recording recording;
VideoSampleEntry sample_entry;
while (run.Step() == SQLITE_ROW) {
recording.id = run.ColumnInt64(0);
recording.camera_id = data.id;
recording.start_time_90k = run.ColumnInt64(1);
recording.end_time_90k = recording.start_time_90k + run.ColumnInt64(2);
recording.sample_file_bytes = run.ColumnInt64(3);
if (!recording.sample_file_uuid.ParseBinary(run.ColumnBlob(4))) {
*error_message =
StrCat("recording ", recording.id, " has unparseable uuid ",
ToHex(run.ColumnBlob(4)));
return false;
}
recording.sample_file_sha1 = run.ColumnBlob(5).as_string();
recording.video_index = run.ColumnBlob(6).as_string();
recording.video_samples = run.ColumnInt64(7);
recording.video_sync_samples = run.ColumnInt64(8);
recording.video_sample_entry_id = run.ColumnInt64(9);
auto it = video_sample_entries_.find(recording.video_sample_entry_id);
if (it == video_sample_entries_.end()) {
*error_message = StrCat("recording ", recording.id,
" references unknown video sample entry ",
recording.video_sample_entry_id);
return false;
}
const VideoSampleEntry &entry = it->second;
if (row_cb(recording, entry) == IterationControl::kBreak) {
return true;
}
}
if (run.status() != SQLITE_DONE && run.status() != SQLITE_ROW) {
*error_message = StrCat("sqlite query failed: ", run.error_message());
return false;
}
return true;
}
bool MoonfireDatabase::ListReservedSampleFiles(std::vector<Uuid> *reserved,
std::string *error_message) {
reserved->clear();
DatabaseContext ctx(db_);
auto run = ctx.UseOnce("select uuid from reserved_sample_files;");
while (run.Step() == SQLITE_ROW) {
Uuid uuid;
if (!uuid.ParseBinary(run.ColumnBlob(0))) {
*error_message = StrCat("unparseable uuid ", ToHex(run.ColumnBlob(0)));
return false;
}
reserved->push_back(uuid);
}
if (run.status() != SQLITE_DONE) {
*error_message = run.error_message();
return false;
}
return true;
}
std::vector<Uuid> MoonfireDatabase::ReserveSampleFiles(
int n, std::string *error_message) {
if (n == 0) {
return std::vector<Uuid>();
}
std::vector<Uuid> uuids;
uuids.reserve(n);
for (int i = 0; i < n; ++i) {
uuids.push_back(uuidgen_->Generate());
}
DatabaseContext ctx(db_);
if (!ctx.BeginTransaction(error_message)) {
return std::vector<Uuid>();
}
for (const auto &uuid : uuids) {
auto run = ctx.Borrow(&insert_reservation_stmt_);
run.BindBlob(":uuid", uuid.binary_view());
run.BindInt64(":state", static_cast<int64_t>(ReservationState::kWriting));
if (run.Step() != SQLITE_DONE) {
ctx.RollbackTransaction();
*error_message = run.error_message();
return std::vector<Uuid>();
}
}
if (!ctx.CommitTransaction(error_message)) {
return std::vector<Uuid>();
}
return uuids;
}
bool MoonfireDatabase::InsertVideoSampleEntry(VideoSampleEntry *entry,
std::string *error_message) {
if (entry->id != -1) {
*error_message = StrCat("video_sample_entry already has id ", entry->id);
return false;
}
DatabaseContext ctx(db_);
for (const auto &some_entry : video_sample_entries_) {
if (some_entry.second.sha1 == entry->sha1) {
if (entry->width != some_entry.second.width ||
entry->height != some_entry.second.height) {
*error_message =
StrCat("inconsistent entry for sha1 ", ToHex(entry->sha1),
": existing entry has ", some_entry.second.width, "x",
some_entry.second.height, ", new entry has ", entry->width,
"x", entry->height);
return false;
}
entry->id = some_entry.first;
return true;
}
}
auto insert_run = ctx.Borrow(&insert_video_sample_entry_stmt_);
insert_run.BindBlob(":sha1", entry->sha1);
insert_run.BindInt64(":width", entry->width);
insert_run.BindInt64(":height", entry->height);
insert_run.BindBlob(":data", entry->data);
if (insert_run.Step() != SQLITE_DONE) {
*error_message =
StrCat("insert video sample entry: ", insert_run.error_message(),
": sha1=", ToHex(entry->sha1), ", dimensions=", entry->width,
"x", entry->height, ", data=", ToHex(entry->data));
return false;
}
entry->id = ctx.last_insert_rowid();
CHECK(video_sample_entries_.insert(std::make_pair(entry->id, *entry)).second)
<< "duplicate: " << entry->id;
return true;
}
bool MoonfireDatabase::InsertRecording(Recording *recording,
std::string *error_message) {
if (recording->id != -1) {
*error_message = StrCat("recording already has id ", recording->id);
return false;
}
if (recording->end_time_90k < recording->start_time_90k) {
*error_message =
StrCat("end time ", recording->end_time_90k, " must be >= start time ",
recording->start_time_90k);
return false;
}
DatabaseContext ctx(db_);
auto it = cameras_by_id_.find(recording->camera_id);
if (it == cameras_by_id_.end()) {
*error_message = StrCat("no camera with id ", recording->camera_id);
return false;
}
CameraData *camera_data = it->second;
if (!ctx.BeginTransaction(error_message)) {
return false;
}
auto delete_run = ctx.Borrow(&delete_reservation_stmt_);
delete_run.BindBlob(":uuid", recording->sample_file_uuid.binary_view());
if (delete_run.Step() != SQLITE_DONE) {
*error_message = delete_run.error_message();
ctx.RollbackTransaction();
return false;
}
if (ctx.changes() != 1) {
*error_message = StrCat("uuid ", recording->sample_file_uuid.UnparseText(),
" is not reserved");
ctx.RollbackTransaction();
return false;
}
auto insert_run = ctx.Borrow(&insert_recording_stmt_);
insert_run.BindInt64(":camera_id", recording->camera_id);
insert_run.BindInt64(":sample_file_bytes", recording->sample_file_bytes);
insert_run.BindInt64(":start_time_90k", recording->start_time_90k);
insert_run.BindInt64(":duration_90k",
recording->end_time_90k - recording->start_time_90k);
insert_run.BindInt64(":local_time_delta_90k",
recording->local_time_90k - recording->start_time_90k);
insert_run.BindInt64(":video_samples", recording->video_samples);
insert_run.BindInt64(":video_sync_samples", recording->video_sync_samples);
insert_run.BindInt64(":video_sample_entry_id",
recording->video_sample_entry_id);
insert_run.BindBlob(":sample_file_uuid",
recording->sample_file_uuid.binary_view());
insert_run.BindBlob(":sample_file_sha1", recording->sample_file_sha1);
insert_run.BindBlob(":video_index", recording->video_index);
if (insert_run.Step() != SQLITE_DONE) {
*error_message =
StrCat("insert failed: ", insert_run.error_message(), ", camera_id=",
recording->camera_id, ", sample_file_bytes=",
recording->sample_file_bytes, ", start_time_90k=",
recording->start_time_90k, ", duration_90k=",
recording->end_time_90k - recording->start_time_90k,
", local_time_delta_90k=",
recording->local_time_90k - recording->start_time_90k,
", video_samples=", recording->video_samples,
", video_sync_samples=", recording->video_sync_samples,
", video_sample_entry_id=", recording->video_sample_entry_id,
", sample_file_uuid=", recording->sample_file_uuid.UnparseText(),
", sample_file_sha1=", ToHex(recording->sample_file_sha1),
", video_index length ", recording->video_index.size());
ctx.RollbackTransaction();
return false;
}
if (!ctx.CommitTransaction(error_message)) {
LOG(ERROR) << "commit failed";
return false;
}
recording->id = ctx.last_insert_rowid();
if (camera_data->min_start_time_90k == -1 ||
camera_data->min_start_time_90k > recording->start_time_90k) {
camera_data->min_start_time_90k = recording->start_time_90k;
}
if (camera_data->max_end_time_90k == -1 ||
camera_data->max_end_time_90k < recording->end_time_90k) {
camera_data->max_end_time_90k = recording->end_time_90k;
}
camera_data->total_duration_90k +=
recording->end_time_90k - recording->start_time_90k;
camera_data->total_sample_file_bytes += recording->sample_file_bytes;
internal::AdjustDaysMap(recording->start_time_90k, recording->end_time_90k, 1,
&camera_data->days);
return true;
}
bool MoonfireDatabase::ListOldestSampleFiles(
Uuid camera_uuid,
std::function<IterationControl(const ListOldestSampleFilesRow &)> row_cb,
std::string *error_message) {
DatabaseContext ctx(db_);
auto it = cameras_by_uuid_.find(camera_uuid);
if (it == cameras_by_uuid_.end()) {
*error_message = StrCat("no such camera ", camera_uuid.UnparseText());
return false;
}
const CameraData &camera_data = it->second;
auto run = ctx.Borrow(&list_oldest_sample_files_stmt_);
run.BindInt64(":camera_id", camera_data.id);
ListOldestSampleFilesRow row;
while (run.Step() == SQLITE_ROW) {
row.camera_id = camera_data.id;
row.recording_id = run.ColumnInt64(0);
if (!row.sample_file_uuid.ParseBinary(run.ColumnBlob(1))) {
*error_message =
StrCat("recording ", row.recording_id, " has unparseable uuid ",
ToHex(run.ColumnBlob(1)));
return false;
}
row.start_time_90k = run.ColumnInt64(2);
row.duration_90k = run.ColumnInt64(3);
row.sample_file_bytes = run.ColumnInt64(4);
if (row_cb(row) == IterationControl::kBreak) {
return true;
}
}
if (run.status() != SQLITE_DONE) {
*error_message = run.error_message();
return false;
}
return true;
}
bool MoonfireDatabase::DeleteRecordings(
const std::vector<ListOldestSampleFilesRow> &recordings,
std::string *error_message) {
if (recordings.empty()) {
return true;
}
DatabaseContext ctx(db_);
if (!ctx.BeginTransaction(error_message)) {
return false;
}
struct State {
int64_t deleted_duration_90k = 0;
int64_t deleted_sample_file_bytes = 0;
int64_t min_start_time_90k = -1;
int64_t max_end_time_90k = -1;
std::map<std::string, int64_t> days;
CameraData *camera_data = nullptr;
};
std::map<int64_t, State> state_by_camera_id;
for (const auto &recording : recordings) {
State &state = state_by_camera_id[recording.camera_id];
state.deleted_duration_90k += recording.duration_90k;
state.deleted_sample_file_bytes += recording.sample_file_bytes;
internal::AdjustDaysMap(recording.start_time_90k,
recording.start_time_90k + recording.duration_90k,
1, &state.days);
auto delete_run = ctx.Borrow(&delete_recording_stmt_);
delete_run.BindInt64(":recording_id", recording.recording_id);
if (delete_run.Step() != SQLITE_DONE) {
ctx.RollbackTransaction();
*error_message = StrCat("delete: ", delete_run.error_message());
return false;
}
if (ctx.changes() != 1) {
ctx.RollbackTransaction();
*error_message = StrCat("no such recording ", recording.recording_id);
return false;
}
auto insert_run = ctx.Borrow(&insert_reservation_stmt_);
insert_run.BindBlob(":uuid", recording.sample_file_uuid.binary_view());
insert_run.BindInt64(":state",
static_cast<int64_t>(ReservationState::kDeleting));
if (insert_run.Step() != SQLITE_DONE) {
ctx.RollbackTransaction();
*error_message = StrCat("insert: ", insert_run.error_message());
return false;
}
}
// Recompute start and end times for each camera.
for (auto &state_entry : state_by_camera_id) {
int64_t camera_id = state_entry.first;
State &state = state_entry.second;
auto it = cameras_by_id_.find(camera_id);
if (it == cameras_by_id_.end()) {
*error_message =
StrCat("internal error; can't find camera id ", camera_id);
return false;
}
state.camera_data = it->second;
// The minimum is straightforward, taking advantage of the start_time_90k
// index for speed.
auto min_run = ctx.Borrow(&camera_min_start_stmt_);
min_run.BindInt64(":camera_id", camera_id);
if (min_run.Step() == SQLITE_ROW) {
state.min_start_time_90k = min_run.ColumnInt64(0);
} else if (min_run.Step() == SQLITE_DONE) {
// There are no recordings left.
state.min_start_time_90k = std::numeric_limits<int64_t>::max();
state.max_end_time_90k = std::numeric_limits<int64_t>::min();
continue; // skip additional query below to calculate max.
} else {
ctx.RollbackTransaction();
*error_message = StrCat("min: ", min_run.error_message());
return false;
}
// The maximum is less straightforward in the case of overlap - all
// recordings starting in the last kMaxRecordingDuration must be examined
// to take advantage of the start_time_90k index.
auto max_run = ctx.Borrow(&camera_max_start_stmt_);
max_run.BindInt64(":camera_id", camera_id);
if (max_run.Step() != SQLITE_ROW) {
// If there was a min row, there should be a max row too, so this is an
// error even in the SQLITE_DONE case.
ctx.RollbackTransaction();
*error_message = StrCat("max[0]: ", max_run.error_message());
return false;
}
int64_t max_start_90k = max_run.ColumnInt64(0);
do {
auto end_time_90k = max_run.ColumnInt64(0) + max_run.ColumnInt64(1);
state.max_end_time_90k = std::max(state.max_end_time_90k, end_time_90k);
} while (max_run.Step() == SQLITE_ROW &&
max_run.ColumnInt64(0) > max_start_90k - kMaxRecordingDuration);
if (max_run.status() != SQLITE_DONE && max_run.status() != SQLITE_ROW) {
*error_message = StrCat("max[1]: ", max_run.error_message());
ctx.RollbackTransaction();
return false;
}
}
if (!ctx.CommitTransaction(error_message)) {
*error_message = StrCat("commit: ", *error_message);
return false;
}
for (auto &state_entry : state_by_camera_id) {
State &state = state_entry.second;
state.camera_data->total_duration_90k -= state.deleted_duration_90k;
state.camera_data->total_sample_file_bytes -=
state.deleted_sample_file_bytes;
state.camera_data->min_start_time_90k = state.min_start_time_90k;
state.camera_data->max_end_time_90k = state.max_end_time_90k;
for (const auto &day : state.days) {
AdjustDay(day.first, -day.second, &state.camera_data->days);
}
}
return true;
}
bool MoonfireDatabase::MarkSampleFilesDeleted(const std::vector<Uuid> &uuids,
std::string *error_message) {
if (uuids.empty()) {
return true;
}
DatabaseContext ctx(db_);
if (!ctx.BeginTransaction(error_message)) {
return false;
}
for (const auto &uuid : uuids) {
auto run = ctx.Borrow(&delete_reservation_stmt_);
run.BindBlob(":uuid", uuid.binary_view());
if (run.Step() != SQLITE_DONE) {
*error_message = run.error_message();
ctx.RollbackTransaction();
return false;
}
if (ctx.changes() != 1) {
*error_message = StrCat("no reservation for uuid ", uuid.UnparseText());
ctx.RollbackTransaction();
return false;
}
}
if (!ctx.CommitTransaction(error_message)) {
return false;
}
return true;
}
} // namespace moonfire_nvr

279
src/moonfire-db.h
View File

@ -1,279 +0,0 @@
// This file is part of Moonfire NVR, a security camera network video recorder.
// Copyright (C) 2016 Scott Lamb <slamb@slamb.org>
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// In addition, as a special exception, the copyright holders give
// permission to link the code of portions of this program with the
// OpenSSL library under certain conditions as described in each
// individual source file, and distribute linked combinations including
// the two.
//
// You must obey the GNU General Public License in all respects for all
// of the code used other than OpenSSL. If you modify file(s) with this
// exception, you may extend this exception to your version of the
// file(s), but you are not obligated to do so. If you do not wish to do
// so, delete this exception statement from your version. If you delete
// this exception statement from all source files in the program, then
// also delete it here.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
//
// moonfire-db.h: database access logic for the Moonfire NVR SQLite schema.
// Currently focused on stuff needed by WebInterface to build a HTML or JSON
// interface.
//
// This caches data in RAM, making the assumption that only one process is
// accessing the database at a time. (TODO: enforce with flock or some such.)
// Performance and efficiency notes:
//
// * several query operations here feature row callbacks. The callback is
// invoked with the database lock. Thus, the caller mustn't perform database
// operations or other long-running operations.
//
// * startup may be slow, as it scans the entire index for the recording
// table. This seems acceptable.
//
// * the operations used for web file serving should return results with
// acceptable latency.
//
// * however, the database lock may be held for longer than is acceptable for
// the critical path of recording frames. It may be necessary to preallocate
// sample file uuids and such to avoid this.
//
// * the caller may need to perform several different types of write
// operations in a row. It might be worth creating an interface for batching
// these inside a transaction, to reduce latency and SSD write cycles. The
// pre-commit and post-commit logic of each operation would have to be
// pulled apart, with the latter being called by this wrapper class on
// commit of the overall transaction.
#ifndef MOONFIRE_NVR_MOONFIRE_DB_H
#define MOONFIRE_NVR_MOONFIRE_DB_H
#include <functional>
#include <map>
#include <memory>
#include <string>
#include <vector>
#include "common.h"
#include "http.h"
#include "mp4.h"
#include "sqlite.h"
#include "uuid.h"
namespace moonfire_nvr {
// For use with MoonfireDatabase::ListCameras.
struct ListCamerasRow {
int64_t id = -1;
Uuid uuid;
std::string short_name;
std::string description;
std::string host;
std::string username;
std::string password;
std::string main_rtsp_path;
std::string sub_rtsp_path;
int64_t retain_bytes = -1;
// Aggregates summarizing completed recordings.
int64_t min_start_time_90k = -1;
int64_t max_end_time_90k = -1;
int64_t total_duration_90k = -1;
int64_t total_sample_file_bytes = -1;
};
// For use with MoonfireDatabase::GetCamera.
// This includes everything in ListCamerasRow. In the future, it will include
// more data. Likely, that will mean a list of calendar days (in the system
// time zone) in which there is any data.
struct GetCameraRow {
std::string short_name;
std::string description;
int64_t retain_bytes = -1;
int64_t min_start_time_90k = -1;
int64_t max_end_time_90k = -1;
int64_t total_duration_90k = -1;
int64_t total_sample_file_bytes = -1;
std::map<std::string, int64_t> days; // YYYY-mm-dd -> duration_90k.
};
// For use with MoonfireDatabase::ListCameraRecordings.
struct ListCameraRecordingsRow {
// From the recording table.
int64_t start_time_90k = -1;
int64_t end_time_90k = -1;
int64_t video_samples = -1;
int64_t sample_file_bytes = -1;
// Joined from the video_sample_entry table.
// |video_sample_entry_sha1| is valid as long as the MoonfireDatabase.
re2::StringPiece video_sample_entry_sha1;
uint16_t width = 0;
uint16_t height = 0;
};
// For use with MoonfireDatabase::ListOldestSampleFiles.
struct ListOldestSampleFilesRow {
int64_t camera_id = -1;
int64_t recording_id = -1;
Uuid sample_file_uuid;
int64_t start_time_90k = -1;
int64_t duration_90k = -1;
int64_t sample_file_bytes = -1;
};
// Thread-safe after Init.
// (Uses a DatabaseContext for locking.)
class MoonfireDatabase {
public:
MoonfireDatabase() {}
MoonfireDatabase(const MoonfireDatabase &) = delete;
void operator=(const MoonfireDatabase &) = delete;
// |db| must outlive the MoonfireDatabase.
bool Init(Database *db, std::string *error_message);
// List all cameras in the system, ordered by short name.
void ListCameras(std::function<IterationControl(const ListCamerasRow &)> cb);
// Get a single camera.
// Return true iff the camera exists.
bool GetCamera(Uuid camera_uuid, GetCameraRow *row);
// List all recordings associated with a camera, descending by end time.
bool ListCameraRecordings(
Uuid camera_uuid, int64_t start_time_90k, int64_t end_time_90k,
std::function<IterationControl(const ListCameraRecordingsRow &)>,
std::string *error_message);
bool ListMp4Recordings(
Uuid camera_uuid, int64_t start_time_90k, int64_t end_time_90k,
std::function<IterationControl(Recording &, const VideoSampleEntry &)>
row_cb,
std::string *error_message);
bool ListReservedSampleFiles(std::vector<Uuid> *reserved,
std::string *error_message);
// Reserve |n| new sample file uuids.
// Returns an empty vector on error.
std::vector<Uuid> ReserveSampleFiles(int n, std::string *error_message);
// Insert a video sample entry if not already inserted.
// On success, |entry->id| is filled in with the id of a freshly-created or
// existing row.
bool InsertVideoSampleEntry(VideoSampleEntry *entry,
std::string *error_message);
// Insert a new recording.
// The uuid must have been already reserved with ReserveSampleFileUuid above.
// On success, |recording->id| is filled in.
bool InsertRecording(Recording *recording, std::string *error_message);
// List sample files, starting from the oldest.
// The caller is expected to supply a |row_cb| that returns kBreak when
// enough have been listed.
bool ListOldestSampleFiles(
Uuid camera_uuid,
std::function<IterationControl(const ListOldestSampleFilesRow &)> row_cb,
std::string *error_message);
// Delete recording rows, moving their sample file uuids to the deleting
// state.
bool DeleteRecordings(const std::vector<ListOldestSampleFilesRow> &rows,
std::string *error_message);
// Mark a set of sample files as deleted.
// This shouldn't be called until the files have been unlinke()ed and the
// parent directory fsync()ed.
// Returns error if any sample files are not in the deleting state.
bool MarkSampleFilesDeleted(const std::vector<Uuid> &uuids,
std::string *error_message);
// Replace the default real UUID generator with the supplied one.
// Exposed only for testing; not thread-safe.
void SetUuidGeneratorForTesting(UuidGenerator *uuidgen) {
uuidgen_ = uuidgen;
}
private:
struct CameraData {
// Cached values of the matching fields from the camera row.
int64_t id = -1;
std::string short_name;
std::string description;
std::string host;
std::string username;
std::string password;
std::string main_rtsp_path;
std::string sub_rtsp_path;
int64_t retain_bytes = -1;
// Aggregates of all recordings associated with the camera.
int64_t min_start_time_90k = std::numeric_limits<int64_t>::max();
int64_t max_end_time_90k = std::numeric_limits<int64_t>::min();
int64_t total_sample_file_bytes = 0;
int64_t total_duration_90k = 0;
// A map of calendar days (in the local timezone, "YYYY-mm-DD") to the
// total duration (in 90k units) of recorded data in the day. A day is
// present in the map ff the value is non-zero.
std::map<std::string, int64_t> days;
};
enum class ReservationState { kWriting = 0, kDeleting = 1 };
// Efficiently (re-)compute the bounds of recorded time for a given camera.
bool ComputeCameraRecordingBounds(DatabaseContext *ctx, int64_t camera_id,
int64_t *min_start_time_90k,
int64_t *max_end_time_90k,
std::string *error_message);
Database *db_ = nullptr;
UuidGenerator *uuidgen_ = GetRealUuidGenerator();
Statement list_camera_recordings_stmt_;
Statement build_mp4_stmt_;
Statement insert_reservation_stmt_;
Statement delete_reservation_stmt_;
Statement insert_video_sample_entry_stmt_;
Statement insert_recording_stmt_;
Statement list_oldest_sample_files_stmt_;
Statement delete_recording_stmt_;
Statement camera_min_start_stmt_;
Statement camera_max_start_stmt_;
std::map<Uuid, CameraData> cameras_by_uuid_;
std::map<int64_t, CameraData *> cameras_by_id_;
std::map<int64_t, VideoSampleEntry> video_sample_entries_;
};
// Given a key in the day-to-duration map, produce the start and end times of
// the day. (Typically the end time is 24 hours later than the start; but it's
// 23 or 25 hours for the days of spring forward and fall back, respectively.)
bool GetDayBounds(const std::string &day, int64_t *start_time_90k,
int64_t *end_time_90k, std::string *error_message);
namespace internal {
// Adjust a day-to-duration map (see MoonfireDatabase::CameraData::days_)
// to reflect a recording.
void AdjustDaysMap(int64_t start_time_90k, int64_t end_time_90k, int sign,
std::map<std::string, int64_t> *days);
} // namespace internal
} // namespace moonfire_nvr
#endif // MOONFIRE_NVR_MOONFIRE_DB_H

237
src/moonfire-nvr-main.cc
View File

@ -1,237 +0,0 @@
// This file is part of Moonfire NVR, a security camera digital video recorder.
// Copyright (C) 2016 Scott Lamb <slamb@slamb.org>
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// In addition, as a special exception, the copyright holders give
// permission to link the code of portions of this program with the
// OpenSSL library under certain conditions as described in each
// individual source file, and distribute linked combinations including
// the two.
//
// You must obey the GNU General Public License in all respects for all
// of the code used other than OpenSSL. If you modify file(s) with this
// exception, you may extend this exception to your version of the
// file(s), but you are not obligated to do so. If you do not wish to do
// so, delete this exception statement from your version. If you delete
// this exception statement from all source files in the program, then
// also delete it here.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
//
// moonfire-nvr-main.cc: main program. This should be kept as short as
// practical, so that individual parts of the program can be tested with the
// googletest framework.
#include <fcntl.h>
#include <signal.h>
#include <sys/types.h>
#include <unistd.h>
#include <string>
#include <event2/buffer.h>
#include <event2/event.h>
#include <event2/event_struct.h>
#include <event2/http.h>
#include <gflags/gflags.h>
#include <glog/logging.h>
#include "ffmpeg.h"
#include "profiler.h"
#include "moonfire-db.h"
#include "moonfire-nvr.h"
#include "sqlite.h"
#include "string.h"
#include "web.h"
using moonfire_nvr::StrCat;
DEFINE_int32(http_port, 0, "");
DEFINE_string(db_dir, "", "");
DEFINE_string(sample_file_dir, "", "");
DEFINE_bool(read_only, false, "");
namespace {
const struct timeval kLogFlushInterval = {1, 0};
struct event_base* base;
void EventLogCallback(int severity, const char* msg) {
int vlog_level = 0;
google::LogSeverity glog_level;
if (severity <= EVENT_LOG_DEBUG) {
vlog_level = 1;
glog_level = google::GLOG_INFO;
} else if (severity <= EVENT_LOG_MSG) {
glog_level = google::GLOG_INFO;
} else if (severity <= EVENT_LOG_WARN) {
glog_level = google::GLOG_WARNING;
} else {
glog_level = google::GLOG_ERROR;
}
if (vlog_level > 0 && !VLOG_IS_ON(vlog_level)) {
return;
}
google::LogMessage("libevent", 0, glog_level).stream() << msg;
}
// Called on SIGTERM or SIGINT.
void SignalCallback(evutil_socket_t, short, void*) {
event_base_loopexit(base, nullptr);
}
void FlushLogsCallback(evutil_socket_t, short, void* ev) {
google::FlushLogFiles(google::GLOG_INFO);
CHECK_EQ(0,
event_add(reinterpret_cast<struct event*>(ev), &kLogFlushInterval));
}
} // namespace
// Note that main never returns; it calls exit on either success or failure.
// This avoids the need to design an orderly shutdown for all dependencies,
// instead letting the OS clean up memory allocations en masse. State may be
// allocated in whatever way is most convenient: on the stack, in a unique_ptr
// (that may never go out of scope), or as a bare pointer that is never
// deleted.
int main(int argc, char** argv) {
google::ParseCommandLineFlags(&argc, &argv, true);
google::InitGoogleLogging(argv[0]);
google::InstallFailureSignalHandler();
signal(SIGPIPE, SIG_IGN);
if (FLAGS_sample_file_dir.empty()) {
LOG(ERROR) << "--sample_file_dir must be specified; exiting.";
exit(1);
}
if (FLAGS_db_dir.empty()) {
LOG(ERROR) << "--db_dir must be specified; exiting.";
exit(1);
}
if (FLAGS_http_port == 0) {
LOG(ERROR) << "--http_port must be specified; exiting.";
exit(1);
}
moonfire_nvr::Environment env;
env.clock = moonfire_nvr::GetRealClock();
env.video_source = moonfire_nvr::GetRealVideoSource();
std::unique_ptr<moonfire_nvr::File> sample_file_dir;
std::string sample_file_dirname = FLAGS_sample_file_dir;
int ret = moonfire_nvr::GetRealFilesystem()->Open(
sample_file_dirname.c_str(), O_DIRECTORY | O_RDONLY, &sample_file_dir);
if (ret != 0) {
LOG(ERROR) << "Unable to open --sample_file_dir=" << sample_file_dirname
<< ": " << strerror(ret) << "; exiting.";
exit(1);
}
// Separately, ensure the sample file directory is writable.
// (Opening the directory above with O_DIRECTORY|O_RDWR doesn't work even
// when the directory is writable; it fails with EISDIR.)
ret = sample_file_dir->Access(".", W_OK, 0);
if (ret != 0) {
LOG(ERROR) << "--sample_file_dir=" << sample_file_dirname
<< " is not writable: " << strerror(ret) << "; exiting.";
exit(1);
}
env.sample_file_dir = sample_file_dir.release();
std::unique_ptr<moonfire_nvr::File> db_dir;
std::string db_dirname = FLAGS_db_dir;
ret = moonfire_nvr::GetRealFilesystem()->Open(
db_dirname.c_str(), O_DIRECTORY | O_RDONLY, &db_dir);
if (ret != 0) {
LOG(ERROR) << "Unable to open --db_dir=" << db_dirname << ": "
<< strerror(ret) << "; exiting.";
exit(1);
}
bool read_only = FLAGS_read_only;
ret = db_dir->Lock((read_only ? LOCK_SH : LOCK_EX) | LOCK_NB);
if (ret != 0) {
LOG(ERROR) << "Unable to lock --db_dir=" << db_dirname << ": "
<< strerror(ret) << "; exiting.";
exit(1);
}
moonfire_nvr::Database db;
std::string error_msg;
std::string db_path = StrCat(FLAGS_db_dir, "/db");
if (!db.Open(db_path.c_str(),
read_only ? SQLITE_OPEN_READONLY : SQLITE_OPEN_READWRITE,
&error_msg)) {
LOG(ERROR) << error_msg << "; exiting.";
exit(1);
}
moonfire_nvr::MoonfireDatabase mdb;
CHECK(mdb.Init(&db, &error_msg)) << error_msg;
env.mdb = &mdb;
moonfire_nvr::WebInterface web(&env);
event_set_log_callback(&EventLogCallback);
LOG(INFO) << "libevent: compiled with version " << LIBEVENT_VERSION
<< ", running with version " << event_get_version();
base = CHECK_NOTNULL(event_base_new());
std::unique_ptr<moonfire_nvr::Nvr> nvr;
if (!read_only) {
nvr.reset(new moonfire_nvr::Nvr(&env));
if (!nvr->Init(&error_msg)) {
LOG(ERROR) << "Unable to initialize: " << error_msg << "; exiting.";
exit(1);
}
}
evhttp* http = CHECK_NOTNULL(evhttp_new(base));
moonfire_nvr::RegisterProfiler(base, http);
web.Register(http);
if (evhttp_bind_socket(http, "0.0.0.0", FLAGS_http_port) != 0) {
LOG(ERROR) << "Unable to bind to --http_port=" << FLAGS_http_port
<< "; exiting.";
exit(1);
}
// Register for termination signals.
struct event ev_sigterm;
struct event ev_sigint;
CHECK_EQ(0, event_assign(&ev_sigterm, base, SIGTERM, EV_SIGNAL | EV_PERSIST,
&SignalCallback, nullptr));
CHECK_EQ(0, event_assign(&ev_sigint, base, SIGINT, EV_SIGNAL | EV_PERSIST,
&SignalCallback, nullptr));
CHECK_EQ(0, event_add(&ev_sigterm, nullptr));
CHECK_EQ(0, event_add(&ev_sigint, nullptr));
// Flush the logfiles regularly for debuggability.
struct event ev_flushlogs;
CHECK_EQ(0, event_assign(&ev_flushlogs, base, 0, 0, &FlushLogsCallback,
&ev_flushlogs));
CHECK_EQ(0, event_add(&ev_flushlogs, &kLogFlushInterval));
// Wait for events.
LOG(INFO) << "Main thread entering event loop.";
CHECK_EQ(0, event_base_loop(base, 0));
LOG(INFO) << "Shutting down.";
google::FlushLogFiles(google::GLOG_INFO);
nvr.reset();
LOG(INFO) << "Done.";
google::ShutdownGoogleLogging();
exit(0);
}

423
src/moonfire-nvr-test.cc
View File

@ -1,423 +0,0 @@
// This file is part of Moonfire NVR, a security camera digital video recorder.
// Copyright (C) 2016 Scott Lamb <slamb@slamb.org>
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// In addition, as a special exception, the copyright holders give
// permission to link the code of portions of this program with the
// OpenSSL library under certain conditions as described in each
// individual source file, and distribute linked combinations including
// the two.
//
// You must obey the GNU General Public License in all respects for all
// of the code used other than OpenSSL. If you modify file(s) with this
// exception, you may extend this exception to your version of the
// file(s), but you are not obligated to do so. If you do not wish to do
// so, delete this exception statement from your version. If you delete
// this exception statement from all source files in the program, then
// also delete it here.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
//
// moonfire-nvr-test.cc: tests of the moonfire-nvr.cc interface.
#include <fcntl.h>
#include <sys/stat.h>
#include <sys/types.h>
#include <gflags/gflags.h>
#include <gmock/gmock.h>
#include <gtest/gtest.h>
#include "moonfire-nvr.h"
#include "string.h"
#include "testutil.h"
DECLARE_bool(alsologtostderr);
using testing::_;
using testing::AnyNumber;
using testing::HasSubstr;
using testing::Invoke;
using testing::Return;
namespace moonfire_nvr {
namespace {
class MockVideoSource : public VideoSource {
public:
// Proxy, as gmock doesn't support non-copyable return values.
std::unique_ptr<InputVideoPacketStream> OpenRtsp(
const std::string &url, std::string *error_message) final {
return std::unique_ptr<InputVideoPacketStream>(
OpenRtspRaw(url, error_message));
}
std::unique_ptr<InputVideoPacketStream> OpenFile(
const std::string &file, std::string *error_message) final {
return std::unique_ptr<InputVideoPacketStream>(
OpenFileRaw(file, error_message));
}
MOCK_METHOD2(OpenRtspRaw,
InputVideoPacketStream *(const std::string &, std::string *));
MOCK_METHOD2(OpenFileRaw,
InputVideoPacketStream *(const std::string &, std::string *));
};
class StreamTest : public testing::Test {
public:
StreamTest() {
std::string error_message;
test_dir_ = PrepareTempDirOrDie("moonfire-nvr-stream-copier");
env_.clock = &clock_;
env_.video_source = &video_source_;
int ret = moonfire_nvr::GetRealFilesystem()->Open(
test_dir_.c_str(), O_DIRECTORY | O_RDONLY, &sample_file_dir_);
CHECK_EQ(0, ret) << "open: " << strerror(ret);
env_.sample_file_dir = sample_file_dir_.get();
CHECK(db_.Open(StrCat(test_dir_, "/db").c_str(),
SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE, &error_message))
<< error_message;
std::string create_sql = ReadFileOrDie("../src/schema.sql");
{
DatabaseContext ctx(&db_);
CHECK(RunStatements(&ctx, create_sql, &error_message)) << error_message;
auto run = ctx.UseOnce(
R"(
insert into camera (uuid, short_name, host, username, password,
main_rtsp_path, sub_rtsp_path, retain_bytes)
values (:uuid, :short_name, :host, :username, :password,
:main_rtsp_path, :sub_rtsp_path, :retain_bytes);
)");
run.BindBlob(":uuid", GetRealUuidGenerator()->Generate().binary_view());
run.BindText(":short_name", "test");
run.BindText(":host", "test-camera");
run.BindText(":username", "foo");
run.BindText(":password", "bar");
run.BindText(":main_rtsp_path", "/main");
run.BindText(":sub_rtsp_path", "/sub");
run.BindInt64(":retain_bytes", 1000000);
CHECK_EQ(SQLITE_DONE, run.Step()) << run.error_message();
}
mdb_.SetUuidGeneratorForTesting(&uuidgen_);
CHECK(mdb_.Init(&db_, &error_message)) << error_message;
env_.mdb = &mdb_;
ListCamerasRow row;
int n_rows = 0;
mdb_.ListCameras([&row, &n_rows](const ListCamerasRow &some_row) {
++n_rows;
row = some_row;
return IterationControl::kContinue;
});
CHECK_EQ(1, n_rows);
clock_.Sleep({1430006400, 0}); // 2015-04-26 00:00:00 UTC
stream_.reset(new Stream(&signal_, &env_, row, 0, 5));
}
// A function to use in OpenRtspRaw invocations which shuts down the stream
// and indicates that the input video source can't be opened.
InputVideoPacketStream *Shutdown(const std::string &url,
std::string *error_message) {
*error_message = "(shutting down)";
signal_.Shutdown();
return nullptr;
}
struct Frame {
Frame(bool is_key, int64_t pts, int64_t duration)
: is_key(is_key), pts(pts), duration(duration) {}
bool is_key;
int64_t pts;
int64_t duration;
bool operator==(const Frame &o) const {
return is_key == o.is_key && pts == o.pts && duration == o.duration;
}
friend std::ostream &operator<<(std::ostream &os, const Frame &f) {
return os << "Frame(" << f.is_key << ", " << f.pts << ", " << f.duration
<< ")";
}
};
#if 0
std::vector<Frame> GetFrames(const std::string &path) {
std::vector<Frame> frames;
std::string error_message;
std::string full_path = StrCat(test_dir_, "/test/", path);
auto f = GetRealVideoSource()->OpenFile(full_path, &error_message);
if (f == nullptr) {
ADD_FAILURE() << full_path << ": " << error_message;
return frames;
}
VideoPacket pkt;
while (f->GetNext(&pkt, &error_message)) {
frames.push_back(Frame(pkt.is_key(), pkt.pts(), pkt.pkt()->duration));
}
EXPECT_EQ("", error_message);
return frames;
}
#else
std::vector<Frame> GetFrames(const re2::StringPiece uuid_text) {
std::vector<Frame> frames;
Uuid uuid;
if (!uuid.ParseText(uuid_text)) {
ADD_FAILURE() << "unparseable: " << uuid_text;
return frames;
}
DatabaseContext ctx(&db_);
auto run = ctx.UseOnce(
"select video_index from recording where sample_file_uuid = :uuid;");
run.BindBlob(":uuid", uuid.binary_view());
if (run.Step() != SQLITE_ROW) {
ADD_FAILURE() << run.error_message();
return frames;
}
for (SampleIndexIterator it(run.ColumnBlob(0)); !it.done(); it.Next()) {
frames.push_back(Frame(it.is_key(), it.start_90k(), it.duration_90k()));
}
return frames;
}
#endif
MockUuidGenerator uuidgen_;
ShutdownSignal signal_;
SimulatedClock clock_;
testing::StrictMock<MockVideoSource> video_source_;
Database db_;
MoonfireDatabase mdb_;
std::unique_ptr<moonfire_nvr::File> sample_file_dir_;
Environment env_;
std::string test_dir_;
std::unique_ptr<Stream> stream_;
};
class ProxyingInputVideoPacketStream : public InputVideoPacketStream {
public:
explicit ProxyingInputVideoPacketStream(
std::unique_ptr<InputVideoPacketStream> base, SimulatedClock *clock)
: base_(std::move(base)), clock_(clock) {}
bool GetNext(VideoPacket *pkt, std::string *error_message) final {
if (pkts_left_-- == 0) {
*error_message = "(pkt limit reached)";
return false;
}
// Advance time to when this packet starts.
clock_->Sleep(SecToTimespec(last_duration_sec_));
if (!base_->GetNext(pkt, error_message)) {
return false;
}
last_duration_sec_ =
pkt->pkt()->duration * av_q2d(base_->stream()->time_base);
// Adjust timestamps.
if (ts_offset_pkts_left_ > 0) {
pkt->pkt()->pts += ts_offset_;
pkt->pkt()->dts += ts_offset_;
--ts_offset_pkts_left_;
}
// Use a fixed duration, as the duration from a real RTSP stream is only
// an estimate. Our test video is 1 fps, 90 kHz time base.
pkt->pkt()->duration = 90000;
return true;
}
const AVStream *stream() const final { return base_->stream(); }
void set_ts_offset(int64_t offset, int pkts) {
ts_offset_ = offset;
ts_offset_pkts_left_ = pkts;
}
void set_pkts(int num) { pkts_left_ = num; }
private:
std::unique_ptr<InputVideoPacketStream> base_;
SimulatedClock *clock_ = nullptr;
double last_duration_sec_ = 0.;
int64_t ts_offset_ = 0;
int ts_offset_pkts_left_ = 0;
int pkts_left_ = std::numeric_limits<int>::max();
};
TEST_F(StreamTest, Basic) {
std::string error_message;
// This is a ~1 fps test video with a timebase of 90 kHz.
auto in_stream = GetRealVideoSource()->OpenFile("../src/testdata/clip.mp4",
&error_message);
ASSERT_TRUE(in_stream != nullptr) << error_message;
auto *proxy_stream =
new ProxyingInputVideoPacketStream(std::move(in_stream), &clock_);
// The starting pts of the input should be irrelevant.
proxy_stream->set_ts_offset(180000, std::numeric_limits<int>::max());
Uuid uuid1;
ASSERT_TRUE(uuid1.ParseText("00000000-0000-0000-0000-000000000001"));
Uuid uuid2;
ASSERT_TRUE(uuid2.ParseText("00000000-0000-0000-0000-000000000002"));
EXPECT_CALL(uuidgen_, Generate())
.WillOnce(Return(uuid1))
.WillOnce(Return(uuid2));
EXPECT_CALL(video_source_, OpenRtspRaw("rtsp://foo:bar@test-camera/main", _))
.WillOnce(Return(proxy_stream))
.WillOnce(Invoke(this, &StreamTest::Shutdown));
stream_->Run();
// Compare frame-by-frame.
// Note below that while the rotation is scheduled to happen near 5-second
// boundaries (such as 2016-04-26 00:00:05), it gets deferred until the next
// key frame, which in this case is 00:00:07.
EXPECT_THAT(GetFrames("00000000-0000-0000-0000-000000000001"),
testing::ElementsAre(
Frame(true, 0, 90379), Frame(false, 90379, 89884),
Frame(false, 180263, 89749), Frame(false, 270012, 89981),
Frame(true, 359993, 90055),
Frame(false, 450048,
89967), // pts_time 5.000533, past rotation time.
Frame(false, 540015, 90021),
Frame(false, 630036, 89958)));
EXPECT_THAT(
GetFrames("00000000-0000-0000-0000-000000000002"),
testing::ElementsAre(Frame(true, 0, 90011), Frame(false, 90011, 0)));
}
TEST_F(StreamTest, NonIncreasingTimestamp) {
std::string error_message;
auto in_stream = GetRealVideoSource()->OpenFile("../src/testdata/clip.mp4",
&error_message);
ASSERT_TRUE(in_stream != nullptr) << error_message;
auto *proxy_stream =
new ProxyingInputVideoPacketStream(std::move(in_stream), &clock_);
proxy_stream->set_ts_offset(12345678, 1);
EXPECT_CALL(video_source_, OpenRtspRaw("rtsp://foo:bar@test-camera/main", _))
.WillOnce(Return(proxy_stream))
.WillOnce(Invoke(this, &StreamTest::Shutdown));
Uuid uuid1;
ASSERT_TRUE(uuid1.ParseText("00000000-0000-0000-0000-000000000001"));
EXPECT_CALL(uuidgen_, Generate()).WillOnce(Return(uuid1));
{
ScopedMockLog log;
EXPECT_CALL(log, Log(_, _, _)).Times(AnyNumber());
EXPECT_CALL(log,
Log(_, _, HasSubstr("Rejecting non-increasing pts=90379")));
log.Start();
stream_->Run();
}
// The output file should still be added to the file manager, with the one
// packet that made it. The final packet on input error will have 0
// duration.
EXPECT_THAT(GetFrames("00000000-0000-0000-0000-000000000001"),
testing::ElementsAre(Frame(true, 0, 0)));
}
TEST_F(StreamTest, RetryOnInputError) {
std::string error_message;
auto in_stream_1 = GetRealVideoSource()->OpenFile("../src/testdata/clip.mp4",
&error_message);
ASSERT_TRUE(in_stream_1 != nullptr) << error_message;
auto *proxy_stream_1 =
new ProxyingInputVideoPacketStream(std::move(in_stream_1), &clock_);
proxy_stream_1->set_pkts(1);
auto in_stream_2 = GetRealVideoSource()->OpenFile("../src/testdata/clip.mp4",
&error_message);
ASSERT_TRUE(in_stream_2 != nullptr) << error_message;
auto *proxy_stream_2 =
new ProxyingInputVideoPacketStream(std::move(in_stream_2), &clock_);
proxy_stream_2->set_pkts(1);
EXPECT_CALL(video_source_, OpenRtspRaw("rtsp://foo:bar@test-camera/main", _))
.WillOnce(Return(proxy_stream_1))
.WillOnce(Return(proxy_stream_2))
.WillOnce(Invoke(this, &StreamTest::Shutdown));
Uuid uuid1;
ASSERT_TRUE(uuid1.ParseText("00000000-0000-0000-0000-000000000001"));
Uuid uuid2;
ASSERT_TRUE(uuid2.ParseText("00000000-0000-0000-0000-000000000002"));
EXPECT_CALL(uuidgen_, Generate())
.WillOnce(Return(uuid1))
.WillOnce(Return(uuid2));
stream_->Run();
// Each attempt should have resulted in a file with one packet.
EXPECT_THAT(GetFrames("00000000-0000-0000-0000-000000000001"),
testing::ElementsAre(Frame(true, 0, 0)));
EXPECT_THAT(GetFrames("00000000-0000-0000-0000-000000000002"),
testing::ElementsAre(Frame(true, 0, 0)));
}
TEST_F(StreamTest, DiscardInitialNonKeyFrames) {
std::string error_message;
auto in_stream = GetRealVideoSource()->OpenFile("../src/testdata/clip.mp4",
&error_message);
ASSERT_TRUE(in_stream != nullptr) << error_message;
// Discard the initial key frame packet.
VideoPacket dummy;
ASSERT_TRUE(in_stream->GetNext(&dummy, &error_message)) << error_message;
auto *proxy_stream =
new ProxyingInputVideoPacketStream(std::move(in_stream), &clock_);
EXPECT_CALL(video_source_, OpenRtspRaw("rtsp://foo:bar@test-camera/main", _))
.WillOnce(Return(proxy_stream))
.WillOnce(Invoke(this, &StreamTest::Shutdown));
Uuid uuid1;
ASSERT_TRUE(uuid1.ParseText("00000000-0000-0000-0000-000000000001"));
Uuid uuid2;
ASSERT_TRUE(uuid2.ParseText("00000000-0000-0000-0000-000000000002"));
EXPECT_CALL(uuidgen_, Generate())
.WillOnce(Return(uuid1))
.WillOnce(Return(uuid2));
stream_->Run();
// Skipped: initial key frame packet (duration 90379)
// Ignored: duration 89884, 89749, 89981 (total pts time: 2.99571... sec)
// Thus, the first output file should start at 00:00:02.
EXPECT_THAT(
GetFrames("00000000-0000-0000-0000-000000000001"),
testing::ElementsAre(
Frame(true, 0, 90055),
Frame(false, 90055, 89967), // pts_time 5.000533, past rotation time.
Frame(false, 180022, 90021), Frame(false, 270043, 89958)));
EXPECT_THAT(
GetFrames("00000000-0000-0000-0000-000000000002"),
testing::ElementsAre(Frame(true, 0, 90011), Frame(false, 90011, 0)));
}
// TODO: test output stream error (on open, writing packet, closing).
// TODO: test rotation!
} // namespace
} // namespace moonfire_nvr
int main(int argc, char **argv) {
FLAGS_alsologtostderr = true;
google::ParseCommandLineFlags(&argc, &argv, true);
testing::InitGoogleTest(&argc, argv);
google::InitGoogleLogging(argv[0]);
return RUN_ALL_TESTS();
}

462
src/moonfire-nvr.cc
View File

@ -1,462 +0,0 @@
// This file is part of Moonfire NVR, a security camera network video recorder.
// Copyright (C) 2016 Scott Lamb <slamb@slamb.org>
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// In addition, as a special exception, the copyright holders give
// permission to link the code of portions of this program with the
// OpenSSL library under certain conditions as described in each
// individual source file, and distribute linked combinations including
// the two.
//
// You must obey the GNU General Public License in all respects for all
// of the code used other than OpenSSL. If you modify file(s) with this
// exception, you may extend this exception to your version of the
// file(s), but you are not obligated to do so. If you do not wish to do
// so, delete this exception statement from your version. If you delete
// this exception statement from all source files in the program, then
// also delete it here.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
//
// moonfire-nvr.cc: implementation of moonfire-nvr.h.
//
// Caveats:
//
// Currently the recording thread blocks while a just-finished recording
// is synced to disk and written to the database, which can be 250+ ms.
// Likewise when recordings are being deleted. It would be better to hand
// off to a separate syncer thread, only blocking the recording when there
// would otherwise be insufficient disk space.
//
// This also commits to the SQLite database potentially several times per
// minute per camera:
//
// 1. (rarely) to get a new video sample entry id
// 2. to reserve a new uuid
// 3. to move uuids planned for deletion from "recording" to
// "reserved_sample_Files"
// 4. to mark those uuids as deleted
// 5. to insert the new recording
//
// These could be combined into a single batch per minute per camera or even
// per minute by doing some operations sooner (such as reserving the next
// minute's uuid when inserting the previous minute's recording) and some
// later (such as marking uuids as deleted).
#define _BSD_SOURCE // for timegm(3).
#include "moonfire-nvr.h"
#include <dirent.h>
#include <errno.h>
#include <fcntl.h>
#include <signal.h>
#include <sys/time.h>
#include <sys/types.h>
#include <unistd.h>
#include <event2/http.h>
#include <gflags/gflags.h>
#include <glog/logging.h>
#include <re2/re2.h>
#include "filesystem.h"
#include "h264.h"
#include "http.h"
#include "recording.h"
#include "string.h"
#include "time.h"
using std::string;
namespace moonfire_nvr {
namespace {
const int kRotateIntervalSec = 60;
} // namespace
// Call from dedicated thread. Runs until shutdown requested.
void Stream::Run() {
std::string error_message;
// Do an initial rotation so that if retain_bytes has been reduced, the
// bulk deletion happens now, rather than while an input stream is open.
if (!RotateFiles(&error_message)) {
LOG(WARNING) << row_.short_name
<< ": initial rotation failed: " << error_message;
}
while (!signal_->ShouldShutdown()) {
if (in_ == nullptr && !OpenInput(&error_message)) {
LOG(WARNING) << row_.short_name
<< ": Failed to open input; sleeping before retrying: "
<< error_message;
env_->clock->Sleep({1, 0});
continue;
}
LOG(INFO) << row_.short_name << ": Calling ProcessPackets.";
ProcessPacketsResult res = ProcessPackets(&error_message);
if (res == kInputError) {
CloseOutput(-1);
in_.reset();
start_localtime_90k_ = -1;
LOG(WARNING) << row_.short_name
<< ": Input error; sleeping before retrying: "
<< error_message;
env_->clock->Sleep({1, 0});
continue;
} else if (res == kOutputError) {
CloseOutput(-1);
LOG(WARNING) << row_.short_name
<< ": Output error; sleeping before retrying: "
<< error_message;
env_->clock->Sleep({1, 0});
continue;
}
}
CloseOutput(-1);
}
Stream::ProcessPacketsResult Stream::ProcessPackets(
std::string *error_message) {
moonfire_nvr::VideoPacket pkt;
CHECK(in_ != nullptr);
CHECK(!writer_.is_open());
while (!signal_->ShouldShutdown()) {
if (!in_->GetNext(&pkt, error_message)) {
if (error_message->empty()) {
*error_message = "unexpected end of stream";
}
return kInputError;
}
// With gcc 4.9 (Raspbian Jessie),
// #define AV_NOPTS_VALUE INT64_C(0x8000000000000000)
// produces an unsigned value. Argh. Work around.
static const int64_t kAvNoptsValue = AV_NOPTS_VALUE;
if (pkt.pkt()->pts == kAvNoptsValue || pkt.pkt()->dts == kAvNoptsValue) {
*error_message = "Rejecting packet with missing pts/dts";
return kInputError;
}
if (pkt.pkt()->pts != pkt.pkt()->dts) {
*error_message =
StrCat("Rejecting packet with pts=", pkt.pkt()->pts, " != dts=",
pkt.pkt()->dts, "; expecting only I or P frames.");
return kInputError;
}
if (pkt.pkt()->pts < min_next_pts_) {
*error_message = StrCat("Rejecting non-increasing pts=", pkt.pkt()->pts,
"; expected at least ", min_next_pts_);
return kInputError;
}
min_next_pts_ = pkt.pkt()->pts + 1;
frame_realtime_ = env_->clock->Now();
if (writer_.is_open() && frame_realtime_.tv_sec >= rotate_time_ &&
pkt.is_key()) {
LOG(INFO) << row_.short_name << ": Reached rotation time; closing "
<< recording_.sample_file_uuid.UnparseText() << ".";
CloseOutput(pkt.pkt()->pts - start_pts_);
} else if (writer_.is_open()) {
VLOG(3) << row_.short_name << ": Rotation time=" << rotate_time_
<< " vs current time=" << frame_realtime_.tv_sec;
}
// Discard the initial, non-key frames from the input.
if (!seen_key_frame_ && !pkt.is_key()) {
continue;
} else if (!seen_key_frame_) {
seen_key_frame_ = true;
}
if (!writer_.is_open()) {
start_pts_ = pkt.pts();
if (!OpenOutput(error_message)) {
return kOutputError;
}
rotate_time_ = frame_realtime_.tv_sec -
(frame_realtime_.tv_sec % rotate_interval_sec_) +
rotate_offset_sec_;
if (rotate_time_ <= frame_realtime_.tv_sec) {
rotate_time_ += rotate_interval_sec_;
}
}
auto start_time_90k = pkt.pkt()->pts - start_pts_;
if (prev_pkt_start_time_90k_ != -1) {
index_.AddSample(start_time_90k - prev_pkt_start_time_90k_,
prev_pkt_bytes_, prev_pkt_key_);
}
re2::StringPiece data = pkt.data();
if (need_transform_) {
if (!TransformSampleData(data, &transform_tmp_, error_message)) {
return kInputError;
}
data = transform_tmp_;
}
if (!writer_.Write(data, error_message)) {
return kOutputError;
}
prev_pkt_start_time_90k_ = start_time_90k;
prev_pkt_bytes_ = data.size();
prev_pkt_key_ = pkt.is_key();
}
return kStopped;
}
bool Stream::OpenInput(std::string *error_message) {
CHECK(in_ == nullptr);
string url = StrCat("rtsp://", row_.username, ":", row_.password, "@",
row_.host, row_.main_rtsp_path);
string redacted_url = StrCat("rtsp://", row_.username, ":redacted@",
row_.host, row_.main_rtsp_path);
LOG(INFO) << row_.short_name << ": Opening input: " << redacted_url;
in_ = env_->video_source->OpenRtsp(url, error_message);
min_next_pts_ = std::numeric_limits<int64_t>::min();
seen_key_frame_ = false;
if (in_ == nullptr) {
return false;
}
// The time base should match the 90kHz frequency specified in RFC 3551
// section 5.
if (in_->stream()->time_base.num != 1 ||
in_->stream()->time_base.den != kTimeUnitsPerSecond) {
*error_message =
StrCat("unexpected time base ", in_->stream()->time_base.num, "/",
in_->stream()->time_base.den);
return false;
}
// width and height must fix into 16-bit ints for MP4 encoding.
int max_dimension = std::numeric_limits<uint16_t>::max();
if (in_->stream()->codec->width > max_dimension ||
in_->stream()->codec->height > max_dimension) {
*error_message =
StrCat("input dimensions ", in_->stream()->codec->width, "x",
in_->stream()->codec->height, " are too large.");
return false;
}
entry_.id = -1;
entry_.width = in_->stream()->codec->width;
entry_.height = in_->stream()->codec->height;
re2::StringPiece extradata = in_->extradata();
if (!ParseExtraData(extradata, entry_.width, entry_.height, &entry_.data,
&need_transform_, error_message)) {
in_.reset();
return false;
}
auto sha1 = Digest::SHA1();
sha1->Update(entry_.data);
entry_.sha1 = sha1->Finalize();
if (!env_->mdb->InsertVideoSampleEntry(&entry_, error_message)) {
in_.reset();
return false;
}
return true;
}
void Stream::CloseOutput(int64_t pts) {
if (!writer_.is_open()) {
return;
}
std::string error_message;
if (prev_pkt_start_time_90k_ != -1) {
int64_t duration_90k = pts - prev_pkt_start_time_90k_;
index_.AddSample(duration_90k > 0 ? duration_90k : 0, prev_pkt_bytes_,
prev_pkt_key_);
}
if (!writer_.Close(&recording_.sample_file_sha1, &error_message)) {
LOG(ERROR) << row_.short_name << ": Closing output "
<< recording_.sample_file_uuid.UnparseText()
<< " failed with error: " << error_message;
uuids_to_unlink_.push_back(recording_.sample_file_uuid);
TryUnlink();
return;
}
int ret = env_->sample_file_dir->Sync();
if (ret != 0) {
LOG(ERROR) << row_.short_name
<< ": Unable to sync sample file dir after writing "
<< recording_.sample_file_uuid.UnparseText() << ": "
<< strerror(ret);
uuids_to_unlink_.push_back(recording_.sample_file_uuid);
TryUnlink();
return;
}
if (!env_->mdb->InsertRecording(&recording_, &error_message)) {
LOG(ERROR) << row_.short_name << ": Unable to insert recording "
<< recording_.sample_file_uuid.UnparseText() << ": "
<< error_message;
uuids_to_unlink_.push_back(recording_.sample_file_uuid);
TryUnlink();
return;
}
row_.total_sample_file_bytes += recording_.sample_file_bytes;
VLOG(1) << row_.short_name << ": ...wrote "
<< recording_.sample_file_uuid.UnparseText() << "; usage now "
<< HumanizeWithBinaryPrefix(row_.total_sample_file_bytes, "B");
}
void Stream::TryUnlink() {
std::vector<Uuid> still_not_unlinked;
for (const auto &uuid : uuids_to_unlink_) {
std::string text = uuid.UnparseText();
int ret = env_->sample_file_dir->Unlink(text.c_str());
if (ret == ENOENT) {
LOG(WARNING) << row_.short_name << ": Sample file " << text
<< " already deleted!";
} else if (ret != 0) {
LOG(WARNING) << row_.short_name << ": Unable to unlink " << text << ": "
<< strerror(ret);
still_not_unlinked.push_back(uuid);
continue;
}
uuids_to_mark_deleted_.push_back(uuid);
}
uuids_to_unlink_ = std::move(still_not_unlinked);
}
bool Stream::OpenOutput(std::string *error_message) {
int64_t frame_localtime_90k = To90k(frame_realtime_);
if (start_localtime_90k_ == -1) {
start_localtime_90k_ = frame_localtime_90k - start_pts_;
}
if (!RotateFiles(error_message)) {
return false;
}
std::vector<Uuid> reserved = env_->mdb->ReserveSampleFiles(1, error_message);
if (reserved.size() != 1) {
return false;
}
CHECK(!writer_.is_open());
string filename = reserved[0].UnparseText();
recording_.id = -1;
recording_.camera_id = row_.id;
recording_.sample_file_uuid = reserved[0];
recording_.video_sample_entry_id = entry_.id;
recording_.local_time_90k = frame_localtime_90k;
index_.Init(&recording_, start_localtime_90k_ + start_pts_);
if (!writer_.Open(filename.c_str(), error_message)) {
return false;
}
prev_pkt_start_time_90k_ = -1;
prev_pkt_bytes_ = -1;
prev_pkt_key_ = false;
LOG(INFO) << row_.short_name << ": Opened output " << filename
<< ", using start_pts=" << start_pts_
<< ", input timebase=" << in_->stream()->time_base.num << "/"
<< in_->stream()->time_base.den;
return true;
}
bool Stream::RotateFiles(std::string *error_message) {
int64_t bytes_needed = row_.total_sample_file_bytes - row_.retain_bytes;
int64_t bytes_to_delete = 0;
if (bytes_needed <= 0) {
VLOG(1) << row_.short_name << ": have remaining quota of "
<< HumanizeWithBinaryPrefix(-bytes_needed, "B");
return true;
}
LOG(INFO) << row_.short_name << ": need to delete "
<< HumanizeWithBinaryPrefix(bytes_needed, "B");
std::vector<ListOldestSampleFilesRow> to_delete;
auto row_cb = [&](const ListOldestSampleFilesRow &row) {
bytes_needed -= row.sample_file_bytes;
bytes_to_delete += row.sample_file_bytes;
to_delete.push_back(row);
return bytes_needed < 0 ? IterationControl::kBreak
: IterationControl::kContinue;
};
if (!env_->mdb->ListOldestSampleFiles(row_.uuid, row_cb, error_message)) {
return false;
}
if (bytes_needed > 0) {
*error_message =
StrCat("couldn't find enough files to delete; ",
HumanizeWithBinaryPrefix(bytes_needed, "B"), " left.");
return false;
}
if (!env_->mdb->DeleteRecordings(to_delete, error_message)) {
return false;
}
for (const auto &to_delete_row : to_delete) {
uuids_to_unlink_.push_back(to_delete_row.sample_file_uuid);
}
row_.total_sample_file_bytes -= bytes_to_delete;
TryUnlink();
if (!uuids_to_unlink_.empty()) {
*error_message =
StrCat("failed to unlink ", uuids_to_unlink_.size(), " files.");
return false;
}
int ret = env_->sample_file_dir->Sync();
if (ret != 0) {
*error_message = StrCat("fsync sample directory: ", strerror(ret));
return false;
}
if (!env_->mdb->MarkSampleFilesDeleted(uuids_to_mark_deleted_,
error_message)) {
*error_message = StrCat("unable to mark ", uuids_to_mark_deleted_.size(),
" sample files as deleted");
return false;
}
uuids_to_mark_deleted_.clear();
VLOG(1) << row_.short_name << ": ...deleted successfully; usage now "
<< HumanizeWithBinaryPrefix(row_.total_sample_file_bytes, "B");
return true;
}
Nvr::~Nvr() {
signal_.Shutdown();
for (auto &thread : stream_threads_) {
thread.join();
}
// TODO: cleanup reservations?
}
bool Nvr::Init(std::string *error_msg) {
std::vector<Uuid> all_reserved;
if (!env_->mdb->ListReservedSampleFiles(&all_reserved, error_msg)) {
return false;
}
for (const auto &reserved : all_reserved) {
int ret = env_->sample_file_dir->Unlink(reserved.UnparseText().c_str());
if (ret != 0 && ret != ENOENT) {
LOG(WARNING) << "Unable to remove reserved sample file: "
<< reserved.UnparseText();
}
}
std::vector<ListCamerasRow> cameras;
env_->mdb->ListCameras([&](const ListCamerasRow &row) {
cameras.push_back(row);
return IterationControl::kContinue;
});
for (size_t i = 0; i < cameras.size(); ++i) {
int rotate_offset_sec = kRotateIntervalSec * i / cameras.size();
auto *stream = new Stream(&signal_, env_, cameras[i], rotate_offset_sec,
kRotateIntervalSec);
streams_.emplace_back(stream);
stream_threads_.emplace_back([stream]() { stream->Run(); });
};
return true;
}
} // namespace moonfire_nvr

187
src/moonfire-nvr.h
View File

@ -1,187 +0,0 @@
// This file is part of Moonfire NVR, a security camera digital video recorder.
// Copyright (C) 2016 Scott Lamb <slamb@slamb.org>
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// In addition, as a special exception, the copyright holders give
// permission to link the code of portions of this program with the
// OpenSSL library under certain conditions as described in each
// individual source file, and distribute linked combinations including
// the two.
//
// You must obey the GNU General Public License in all respects for all
// of the code used other than OpenSSL. If you modify file(s) with this
// exception, you may extend this exception to your version of the
// file(s), but you are not obligated to do so. If you do not wish to do
// so, delete this exception statement from your version. If you delete
// this exception statement from all source files in the program, then
// also delete it here.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
//
// moonfire-nvr.h: main digital video recorder components.
#ifndef MOONFIRE_NVR_NVR_H
#define MOONFIRE_NVR_NVR_H
#include <sys/stat.h>
#include <time.h>
#include <atomic>
#include <map>
#include <mutex>
#include <string>
#include <thread>
#include <vector>
#include <event2/http.h>
#include "filesystem.h"
#include "moonfire-db.h"
#include "ffmpeg.h"
#include "time.h"
namespace moonfire_nvr {
// A signal that all streams associated with an Nvr should shut down.
class ShutdownSignal {
public:
ShutdownSignal() {}
ShutdownSignal(const ShutdownSignal &) = delete;
ShutdownSignal &operator=(const ShutdownSignal &) = delete;
void Shutdown() { shutdown_.store(true, std::memory_order_relaxed); }
bool ShouldShutdown() const {
return shutdown_.load(std::memory_order_relaxed);
}
private:
std::atomic_bool shutdown_{false};
};
// The Nvr's environment. This is supplied for testability.
struct Environment {
WallClock *clock = nullptr;
VideoSource *video_source = nullptr;
File *sample_file_dir = nullptr;
MoonfireDatabase *mdb = nullptr;
};
// A single video stream, currently always a camera's "main" (as opposed to
// "sub") stream. Methods are thread-compatible rather than thread-safe; the
// Nvr should call Run in a dedicated thread.
class Stream {
public:
Stream(const ShutdownSignal *signal, Environment *const env,
const moonfire_nvr::ListCamerasRow &row, int rotate_offset_sec,
int rotate_interval_sec)
: signal_(signal),
env_(env),
row_(row),
rotate_offset_sec_(rotate_offset_sec),
rotate_interval_sec_(rotate_interval_sec),
writer_(env->sample_file_dir) {}
Stream(const Stream &) = delete;
Stream &operator=(const Stream &) = delete;
// Call from dedicated thread. Runs until shutdown requested.
void Run();
private:
enum ProcessPacketsResult { kInputError, kOutputError, kStopped };
ProcessPacketsResult ProcessPackets(std::string *error_message);
bool OpenInput(std::string *error_message);
// |pts| should be the relative pts within this output segment if closing
// due to normal rotation, or -1 if closing abruptly.
void CloseOutput(int64_t pts);
bool OpenOutput(std::string *error_message);
bool RotateFiles(std::string *error_message);
void TryUnlink();
const ShutdownSignal *signal_;
const Environment *env_;
ListCamerasRow row_;
const int rotate_offset_sec_;
const int rotate_interval_sec_;
//
// State below is used only by the thread in Run().
//
std::unique_ptr<moonfire_nvr::InputVideoPacketStream> in_;
int64_t min_next_pts_ = std::numeric_limits<int64_t>::min();
bool seen_key_frame_ = false;
// need_transform_ indicates if TransformSampleData will need to be called
// on each video sample.
bool need_transform_ = false;
VideoSampleEntry entry_;
std::string transform_tmp_;
std::vector<Uuid> uuids_to_unlink_;
std::vector<Uuid> uuids_to_mark_deleted_;
// Current output segment.
Recording recording_;
moonfire_nvr::SampleFileWriter writer_;
SampleIndexEncoder index_;
time_t rotate_time_ = 0; // rotate when frame_realtime_ >= rotate_time_.
// start_pts_ is the pts of the first frame included in the current output.
int64_t start_pts_ = -1;
// start_localtime_90k_ is the local system's time since epoch (in 90k units)
// to match start_pts_.
int64_t start_localtime_90k_ = -1;
// These fields describe a packet which has been written to the
// sample file but (because the duration is not yet known) has not been
// added to the index.
int32_t prev_pkt_start_time_90k_ = -1;
int32_t prev_pkt_bytes_ = -1;
bool prev_pkt_key_ = false;
struct timespec frame_realtime_ = {0, 0};
};
// The main network video recorder, which manages a collection of streams.
class Nvr {
public:
explicit Nvr(Environment *env) : env_(env) {}
Nvr(const Nvr &) = delete;
Nvr &operator=(const Nvr &) = delete;
// Shut down, blocking for outstanding streams.
// Caller only has to guarantee that HttpCallback is not being called / will
// not be called again, likely by having already shut down the event loop.
~Nvr();
// Initialize the NVR. Call before any other operation.
// Verifies configuration and starts background threads to capture/rotate
// streams.
bool Init(std::string *error_msg);
private:
void HttpCallbackForTopLevel(evhttp_request *req);
Environment *const env_;
std::vector<std::unique_ptr<Stream>> streams_;
std::vector<std::thread> stream_threads_;
ShutdownSignal signal_;
};
} // namespace moonfire_nvr
#endif // MOONFIRE_NVR_NVR_H

404
src/mp4-test.cc
View File

@ -1,404 +0,0 @@
// This file is part of Moonfire DVR, a security camera digital video recorder.
// Copyright (C) 2015 Scott Lamb <slamb@slamb.org>
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// In addition, as a special exception, the copyright holders give
// permission to link the code of portions of this program with the
// OpenSSL library under certain conditions as described in each
// individual source file, and distribute linked combinations including
// the two.
//
// You must obey the GNU General Public License in all respects for all
// of the code used other than OpenSSL. If you modify file(s) with this
// exception, you may extend this exception to your version of the
// file(s), but you are not obligated to do so. If you do not wish to do
// so, delete this exception statement from your version. If you delete
// this exception statement from all source files in the program, then
// also delete it here.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
//
// mp4-test.cc: tests of the mp4.h interface.
#include <fcntl.h>
#include <sys/stat.h>
#include <sys/types.h>
#include <event2/buffer.h>
#include <gflags/gflags.h>
#include <gmock/gmock.h>
#include <gtest/gtest.h>
#include "ffmpeg.h"
#include "h264.h"
#include "http.h"
#include "mp4.h"
#include "string.h"
#include "testutil.h"
DECLARE_bool(alsologtostderr);
using moonfire_nvr::internal::Mp4SampleTablePieces;
namespace moonfire_nvr {
namespace {
std::string ToHex(const FileSlice *slice, bool pad) {
EvBuffer buf;
std::string error_message;
size_t size = slice->size();
CHECK(slice->AddRange(ByteRange(0, size), &buf, &error_message))
<< error_message;
CHECK_EQ(size, evbuffer_get_length(buf.get()));
return ::moonfire_nvr::ToHex(
re2::StringPiece(
reinterpret_cast<const char *>(evbuffer_pullup(buf.get(), size)),
size),
pad);
}
std::string Digest(const FileSlice *slice) {
EvBuffer buf;
std::string error_message;
ByteRange left(0, slice->size());
while (left.size() > 0) {
auto ret = slice->AddRange(left, &buf, &error_message);
CHECK_GT(ret, 0) << error_message;
left.begin += ret;
}
evbuffer_iovec vec;
auto digest = Digest::SHA1();
while (evbuffer_peek(buf.get(), -1, nullptr, &vec, 1) > 0) {
digest->Update(re2::StringPiece(
reinterpret_cast<const char *>(vec.iov_base), vec.iov_len));
evbuffer_drain(buf.get(), vec.iov_len);
}
return ::moonfire_nvr::ToHex(digest->Finalize());
}
TEST(Mp4SampleTablePiecesTest, AllSyncFrames) {
Recording recording;
SampleIndexEncoder encoder;
encoder.Init(&recording, 42);
for (int i = 1; i <= 5; ++i) {
int64_t sample_duration_90k = 2 * i;
int64_t sample_bytes = 3 * i;
encoder.AddSample(sample_duration_90k, sample_bytes, true);
}
Mp4SampleTablePieces pieces;
std::string error_message;
// Time range [2, 2 + 4 + 6 + 8) means the 2nd, 3rd, 4th samples should be
// included.
ASSERT_TRUE(pieces.Init(&recording, 2, 10, 2, 2 + 4 + 6 + 8, &error_message))
<< error_message;
EXPECT_EQ(3, pieces.stts_entry_count());
const char kExpectedStts[] =
"00 00 00 01 00 00 00 04 " // run length / timestamps.
"00 00 00 01 00 00 00 06 "
"00 00 00 01 00 00 00 08";
EXPECT_EQ(kExpectedStts, ToHex(pieces.stts_entries(), true));
// Initial index "10" as given above.
EXPECT_EQ(3, pieces.stss_entry_count());
const char kExpectedStss[] = "00 00 00 0a 00 00 00 0b 00 00 00 0c";
EXPECT_EQ(kExpectedStss, ToHex(pieces.stss_entries(), true));
EXPECT_EQ(3, pieces.stsz_entry_count());
const char kExpectedStsz[] = "00 00 00 06 00 00 00 09 00 00 00 0c";
EXPECT_EQ(kExpectedStsz, ToHex(pieces.stsz_entries(), true));
}
TEST(Mp4SampleTablePiecesTest, HalfSyncFrames) {
Recording recording;
SampleIndexEncoder encoder;
encoder.Init(&recording, 42);
for (int i = 1; i <= 5; ++i) {
int64_t sample_duration_90k = 2 * i;
int64_t sample_bytes = 3 * i;
encoder.AddSample(sample_duration_90k, sample_bytes, (i % 2) == 1);
}
Mp4SampleTablePieces pieces;
std::string error_message;
// Time range [2 + 4 + 6, 2 + 4 + 6 + 8) means the 4th samples should be
// included. The 3rd gets pulled in also because it is a sync frame and the
// 4th is not.
ASSERT_TRUE(
pieces.Init(&recording, 2, 10, 2 + 4 + 6, 2 + 4 + 6 + 8, &error_message))
<< error_message;
EXPECT_EQ(2, pieces.stts_entry_count());
const char kExpectedStts[] =
"00 00 00 01 00 00 00 06 "
"00 00 00 01 00 00 00 08";
EXPECT_EQ(kExpectedStts, ToHex(pieces.stts_entries(), true));
EXPECT_EQ(1, pieces.stss_entry_count());
const char kExpectedStss[] = "00 00 00 0a";
EXPECT_EQ(kExpectedStss, ToHex(pieces.stss_entries(), true));
EXPECT_EQ(2, pieces.stsz_entry_count());
const char kExpectedStsz[] = "00 00 00 09 00 00 00 0c";
EXPECT_EQ(kExpectedStsz, ToHex(pieces.stsz_entries(), true));
}
TEST(Mp4SampleTablePiecesTest, FastPath) {
Recording recording;
SampleIndexEncoder encoder;
encoder.Init(&recording, 42);
for (int i = 1; i <= 5; ++i) {
int64_t sample_duration_90k = 2 * i;
int64_t sample_bytes = 3 * i;
encoder.AddSample(sample_duration_90k, sample_bytes, (i % 2) == 1);
}
auto total_duration_90k = recording.end_time_90k - recording.start_time_90k;
Mp4SampleTablePieces pieces;
std::string error_message;
// Time range [0, end - start) means to pull in everything.
// This uses a fast path which can determine the size without examining the
// index.
ASSERT_TRUE(
pieces.Init(&recording, 2, 10, 0, total_duration_90k, &error_message))
<< error_message;
EXPECT_EQ(5, pieces.stts_entry_count());
const char kExpectedStts[] =
"00 00 00 01 00 00 00 02 "
"00 00 00 01 00 00 00 04 "
"00 00 00 01 00 00 00 06 "
"00 00 00 01 00 00 00 08 "
"00 00 00 01 00 00 00 0a";
EXPECT_EQ(kExpectedStts, ToHex(pieces.stts_entries(), true));
EXPECT_EQ(3, pieces.stss_entry_count());
const char kExpectedStss[] = "00 00 00 0a 00 00 00 0c 00 00 00 0e";
EXPECT_EQ(kExpectedStss, ToHex(pieces.stss_entries(), true));
EXPECT_EQ(5, pieces.stsz_entry_count());
const char kExpectedStsz[] =
"00 00 00 03 00 00 00 06 00 00 00 09 00 00 00 0c 00 00 00 0f";
EXPECT_EQ(kExpectedStsz, ToHex(pieces.stsz_entries(), true));
}
class IntegrationTest : public testing::Test {
protected:
IntegrationTest() {
tmpdir_path_ = PrepareTempDirOrDie("mp4-integration-test");
int ret = GetRealFilesystem()->Open(tmpdir_path_.c_str(),
O_RDONLY | O_DIRECTORY, &tmpdir_);
CHECK_EQ(0, ret) << strerror(ret);
}
Recording CopyMp4ToSingleRecording() {
std::string error_message;
Recording recording;
SampleIndexEncoder index;
// Set start time to 2015-04-26 00:00:00 UTC.
index.Init(&recording, UINT64_C(1430006400) * kTimeUnitsPerSecond);
SampleFileWriter writer(tmpdir_.get());
std::string filename = recording.sample_file_uuid.UnparseText();
if (!writer.Open(filename.c_str(), &error_message)) {
ADD_FAILURE() << "open " << filename << ": " << error_message;
return recording;
}
auto in = GetRealVideoSource()->OpenFile("../src/testdata/clip.mp4",
&error_message);
if (in == nullptr) {
ADD_FAILURE() << "open clip.mp4" << error_message;
return recording;
}
video_sample_entry_.width = in->stream()->codec->width;
video_sample_entry_.height = in->stream()->codec->height;
bool need_transform;
if (!ParseExtraData(in->extradata(), in->stream()->codec->width,
in->stream()->codec->height, &video_sample_entry_.data,
&need_transform, &error_message)) {
ADD_FAILURE() << "GetH264SampleEntry: " << error_message;
return recording;
}
EXPECT_FALSE(need_transform);
while (true) {
VideoPacket pkt;
if (!in->GetNext(&pkt, &error_message)) {
if (!error_message.empty()) {
ADD_FAILURE() << "GetNext: " << error_message;
return recording;
}
break;
}
if (!writer.Write(GetData(pkt), &error_message)) {
ADD_FAILURE() << "Write: " << error_message;
return recording;
}
index.AddSample(pkt.pkt()->duration, pkt.pkt()->size, pkt.is_key());
}
if (!writer.Close(&recording.sample_file_sha1, &error_message)) {
ADD_FAILURE() << "Close: " << error_message;
}
return recording;
}
std::shared_ptr<VirtualFile> CreateMp4FromSingleRecording(
const Recording &recording, int32_t rel_start_90k, int32_t rel_end_90k,
bool include_ts) {
Mp4FileBuilder builder(tmpdir_.get());
builder.SetSampleEntry(video_sample_entry_);
builder.Append(Recording(recording), rel_start_90k, rel_end_90k);
builder.include_timestamp_subtitle_track(include_ts);
std::string error_message;
auto mp4 = builder.Build(&error_message);
EXPECT_TRUE(mp4 != nullptr) << error_message;
return mp4;
}
void WriteMp4(VirtualFile *f) {
EvBuffer buf;
std::string error_message;
ByteRange left(0, f->size());
while (left.size() > 0) {
auto ret = f->AddRange(left, &buf, &error_message);
ASSERT_GT(ret, 0) << error_message;
left.begin += ret;
}
WriteFileOrDie(StrCat(tmpdir_path_, "/clip.new.mp4"), &buf);
}
void CompareMp4s(int64_t pts_offset) {
std::string error_message;
auto original = GetRealVideoSource()->OpenFile("../src/testdata/clip.mp4",
&error_message);
ASSERT_TRUE(original != nullptr) << error_message;
auto copied = GetRealVideoSource()->OpenFile(
StrCat(tmpdir_path_, "/clip.new.mp4"), &error_message);
ASSERT_TRUE(copied != nullptr) << error_message;
EXPECT_EQ(original->extradata(), copied->extradata());
EXPECT_EQ(original->stream()->codec->width, copied->stream()->codec->width);
EXPECT_EQ(original->stream()->codec->height,
copied->stream()->codec->height);
int pkt = 0;
while (true) {
VideoPacket original_pkt;
VideoPacket copied_pkt;
bool original_has_next = original->GetNext(&original_pkt, &error_message);
ASSERT_TRUE(original_has_next || error_message.empty())
<< "pkt " << pkt << ": " << error_message;
bool copied_has_next = copied->GetNext(&copied_pkt, &error_message);
ASSERT_TRUE(copied_has_next || error_message.empty())
<< "pkt " << pkt << ": " << error_message;
if (!original_has_next && !copied_has_next) {
break;
}
ASSERT_TRUE(original_has_next) << "pkt " << pkt;
ASSERT_TRUE(copied_has_next) << "pkt " << pkt;
EXPECT_EQ(original_pkt.pkt()->pts + pts_offset, copied_pkt.pkt()->pts)
<< "pkt " << pkt;
// One would normally expect the duration to be exactly the same, but
// when using an edit list, ffmpeg appears to extend the last packet's
// duration by the amount skipped at the beginning. I think this is a
// bug on their side.
EXPECT_LE(original_pkt.pkt()->duration, copied_pkt.pkt()->duration)
<< "pkt " << pkt;
EXPECT_EQ(GetData(original_pkt), GetData(copied_pkt)) << "pkt " << pkt;
++pkt;
}
}
re2::StringPiece GetData(const VideoPacket &pkt) {
return re2::StringPiece(reinterpret_cast<const char *>(pkt.pkt()->data),
pkt.pkt()->size);
}
std::string tmpdir_path_;
std::unique_ptr<File> tmpdir_;
std::string etag_;
VideoSampleEntry video_sample_entry_;
};
TEST_F(IntegrationTest, RoundTrip) {
Recording recording = CopyMp4ToSingleRecording();
if (HasFailure()) {
return;
}
auto f = CreateMp4FromSingleRecording(
recording, 0, std::numeric_limits<int32_t>::max(), false);
WriteMp4(f.get());
CompareMp4s(0);
// This test is brittle, which is the point. Any time the digest comparison
// here fails, it can be updated, but the etag must change as well!
// Otherwise clients may combine ranges from the new format with ranges
// from the old format!
EXPECT_EQ("1e5331e8371bd97ac3158b3a86494abc87cdc70e", Digest(f.get()));
EXPECT_EQ("\"268db2cd6e4814676d38832f1f9340c7555e4e71\"", f->etag());
// 10 seconds later than the segment's start time.
EXPECT_EQ(1430006410, f->last_modified());
}
TEST_F(IntegrationTest, RoundTripWithSubtitle) {
Recording recording = CopyMp4ToSingleRecording();
if (HasFailure()) {
return;
}
auto f = CreateMp4FromSingleRecording(
recording, 0, std::numeric_limits<int32_t>::max(), true);
WriteMp4(f.get());
CompareMp4s(0);
// This test is brittle, which is the point. Any time the digest comparison
// here fails, it can be updated, but the etag must change as well!
// Otherwise clients may combine ranges from the new format with ranges
// from the old format!
EXPECT_EQ("0081a442ba73092027fc580eeac2ebf25cb1ef50", Digest(f.get()));
EXPECT_EQ("\"8a29042355e1e28c10fbba328d1ddc9d54e450cd\"", f->etag());
}
TEST_F(IntegrationTest, RoundTripWithEditList) {
Recording recording = CopyMp4ToSingleRecording();
if (HasFailure()) {
return;
}
auto f = CreateMp4FromSingleRecording(
recording, 1, std::numeric_limits<int32_t>::max(), false);
WriteMp4(f.get());
CompareMp4s(-1);
// This test is brittle, which is the point. Any time the digest comparison
// here fails, it can be updated, but the etag must change as well!
// Otherwise clients may combine ranges from the new format with ranges
// from the old format!
EXPECT_EQ("685e026af44204bc9cc52115c5e17058e9fb7c70", Digest(f.get()));
EXPECT_EQ("\"1373289ddc7c05580deeeb1f1624e2d6cac7ddd3\"", f->etag());
}
} // namespace
} // namespace moonfire_nvr
int main(int argc, char **argv) {
FLAGS_alsologtostderr = true;
google::ParseCommandLineFlags(&argc, &argv, true);
testing::InitGoogleTest(&argc, argv);
google::InitGoogleLogging(argv[0]);
return RUN_ALL_TESTS();
}

1147
src/mp4.cc
View File

File diff suppressed because it is too large Load Diff

188
src/mp4.h
View File

@ -1,188 +0,0 @@
// This file is part of Moonfire NVR, a security camera network video recorder.
// Copyright (C) 2016 Scott Lamb <slamb@slamb.org>
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// In addition, as a special exception, the copyright holders give
// permission to link the code of portions of this program with the
// OpenSSL library under certain conditions as described in each
// individual source file, and distribute linked combinations including
// the two.
//
// You must obey the GNU General Public License in all respects for all
// of the code used other than OpenSSL. If you modify file(s) with this
// exception, you may extend this exception to your version of the
// file(s), but you are not obligated to do so. If you do not wish to do
// so, delete this exception statement from your version. If you delete
// this exception statement from all source files in the program, then
// also delete it here.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
//
// mp4.h: interface for building VirtualFiles representing ISO/IEC 14496-12
// (ISO base media format / MPEG-4 / .mp4) video. These can be constructed
// from one or more recordings and are suitable for HTTP range serving or
// download.
#ifndef MOONFIRE_NVR_MP4_H
#define MOONFIRE_NVR_MP4_H
#include <limits>
#include <memory>
#include <vector>
#include "recording.h"
#include "http.h"
namespace moonfire_nvr {
namespace internal {
// Represents pieces of .mp4 sample tables for one recording. Many recordings,
// and thus many of these objects, may be spliced together into a single
// virtual .mp4 file. For internal use by Mp4FileBuilder. Exposed for testing.
class Mp4SampleTablePieces {
public:
Mp4SampleTablePieces() {}
Mp4SampleTablePieces(const Mp4SampleTablePieces &) = delete;
void operator=(const Mp4SampleTablePieces &) = delete;
// |recording| must outlive the Mp4SampleTablePieces.
//
// |sample_entry_index| should be the (1-based) index into the "stsd" box
// of an entry matching this recording's video_sample_entry_sha1. It may
// be shared with other recordings.
//
// |sample_offset| should be the (1-based) index of the first sample in
// this file. It should be 1 + the sum of all previous Mp4SampleTablePieces'
// samples() values.
//
// |start_90k| and |end_90k| should be relative to the start of the recording.
// They indicate the *desired* time range. The *actual* time range will
// start at the last sync sample <= |start_90k|. (The caller is responsible
// for creating an edit list to skip the undesired portion.) It will end at
// the desired range, or the end of the recording, whichever is sooner.
bool Init(const Recording *recording, int sample_entry_index,
int32_t sample_offset, int32_t start_90k, int32_t end_90k,
std::string *error_message);
int32_t stts_entry_count() const { return frames_; }
const FileSlice *stts_entries() const { return &stts_entries_; }
int32_t stss_entry_count() const { return key_frames_; }
const FileSlice *stss_entries() const { return &stss_entries_; }
int32_t stsz_entry_count() const { return frames_; }
const FileSlice *stsz_entries() const { return &stsz_entries_; }
int32_t samples() const { return frames_; }
// Return the byte range in the sample file of the frames represented here.
ByteRange sample_pos() const { return sample_pos_; }
// As described above, these may differ from the desired range.
uint64_t duration_90k() const { return actual_end_90k_ - begin_.start_90k(); }
int32_t start_90k() const { return begin_.start_90k(); }
int32_t end_90k() const { return actual_end_90k_; }
private:
bool FillSttsEntries(std::string *s, std::string *error_message) const;
bool FillStssEntries(std::string *s, std::string *error_message) const;
bool FillStscEntries(std::string *s, std::string *error_message) const;
bool FillStszEntries(std::string *s, std::string *error_message) const;
// After Init(), |begin_| will be on the first sample after the start of the
// range (or it will be done()).
SampleIndexIterator begin_;
ByteRange sample_pos_;
FillerFileSlice stts_entries_;
FillerFileSlice stss_entries_;
FillerFileSlice stsz_entries_;
int sample_entry_index_ = -1;
int32_t sample_offset_ = -1;
int32_t desired_end_90k_ = -1;
int32_t actual_end_90k_ = -1;
int32_t frames_ = 0;
int32_t key_frames_ = 0;
};
struct Mp4FileSegment {
Recording recording;
Mp4SampleTablePieces pieces;
RealFileSlice sample_file_slice;
// Requested start time, relative to recording.start_90k.
// If there is no key frame at exactly this position, |pieces| will actually
// start sooner, and an edit list should be used to skip the undesired
// prefix.
int32_t rel_start_90k = 0;
// Requested end time, relative to recording.end_90k.
// This will be clamped to the actual duration of the recording.
int32_t rel_end_90k = std::numeric_limits<int32_t>::max();
};
} // namespace internal
// Builder for a virtual .mp4 file.
class Mp4FileBuilder {
public:
// |sample_file_dir| must outlive the Mp4FileBuilder and the returned
// VirtualFile.
explicit Mp4FileBuilder(File *sample_file_dir)
: sample_file_dir_(sample_file_dir) {}
Mp4FileBuilder(const Mp4FileBuilder &) = delete;
void operator=(const Mp4FileBuilder &) = delete;
// Append part or all of a recording.
// Note that |recording.video_sample_entry_sha1| must be added via
// AddSampleEntry.
Mp4FileBuilder &Append(Recording &&recording, int32_t rel_start_300ths,
int32_t rel_end_300ths);
// TODO: support multiple sample entries?
Mp4FileBuilder &SetSampleEntry(const VideoSampleEntry &entry);
// Set if a subtitle track should be added with timestamps.
Mp4FileBuilder &include_timestamp_subtitle_track(bool v) {
include_timestamp_subtitle_track_ = v;
return *this;
}
// TODO: MPEG-DASH / ISO BMFF Byte Stream Format support.
// Build the .mp4 file, returning it to the caller.
// The Mp4FileBuilder is left in an undefined state; it should not
// be used afterward. On error, nullptr is returned, with |error_message|
// populated.
//
// Errors include:
// * TODO: width/height mismatch? or is this okay?
// * No segments.
// * Non-final segment has zero duration of last sample.
// * Data error in one of the recording sample indexes.
// * Invalid start/end.
std::shared_ptr<VirtualFile> Build(std::string *error_message);
private:
File *sample_file_dir_;
std::vector<std::unique_ptr<internal::Mp4FileSegment>> segments_;
VideoSampleEntry video_sample_entry_;
bool include_timestamp_subtitle_track_ = false;
};
} // namespace moonfire_nvr
#endif // MOONFIRE_NVR_MP4_H

1534
src/mp4.rs Normal file
View File

File diff suppressed because it is too large Load Diff

295
src/pieces.rs Normal file
View File

@ -0,0 +1,295 @@
// This file is part of Moonfire NVR, a security camera digital video recorder.
// Copyright (C) 2016 Scott Lamb <slamb@slamb.org>
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// In addition, as a special exception, the copyright holders give
// permission to link the code of portions of this program with the
// OpenSSL library under certain conditions as described in each
// individual source file, and distribute linked combinations including
// the two.
//
// You must obey the GNU General Public License in all respects for all
// of the code used other than OpenSSL. If you modify file(s) with this
// exception, you may extend this exception to your version of the
// file(s), but you are not obligated to do so. If you do not wish to do
// so, delete this exception statement from your version. If you delete
// this exception statement from all source files in the program, then
// also delete it here.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
use error::{Error, Result};
use std::fmt;
use std::io;
use std::marker::PhantomData;
use std::ops::Range;
#[derive(Debug)]
struct SliceInfo<W> {
end: u64,
writer: W,
}
pub trait ContextWriter<Ctx> {
fn write_to(&self, ctx: &Ctx, r: Range<u64>, l: u64, out: &mut io::Write) -> Result<()>;
}
/// Calls `f` with an `io::Write` which delegates to `inner` only for the section defined by `r`.
/// This is useful for easily implementing the `ContextWriter` interface for pieces that generate
/// data on-the-fly rather than simply copying a buffer.
pub fn clip_to_range<F>(r: Range<u64>, l: u64, inner: &mut io::Write, mut f: F) -> Result<()>
where F: FnMut(&mut Vec<u8>) -> Result<()> {
// Just create a buffer for the whole slice and copy out the relevant portion.
// One might expect it to be faster to avoid this memory allocation and extra copying, but
// benchmarks show when making many 4-byte writes it's better to be able to inline many
// Vec::write_all calls then make one call through traits to hyper's write logic.
let mut buf = Vec::with_capacity(l as usize);
f(&mut buf)?;
inner.write_all(&buf[r.start as usize .. r.end as usize])?;
Ok(())
}
pub struct Slices<W, C> {
len: u64,
slices: Vec<SliceInfo<W>>,
phantom: PhantomData<C>,
}
impl<W, C> fmt::Debug for Slices<W, C> where W: fmt::Debug {
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
write!(f, "{} slices with overall length {}:", self.slices.len(), self.len)?;
let mut start = 0;
for (i, s) in self.slices.iter().enumerate() {
write!(f, "\n{:7}: [{:12}, {:12}): {:?}", i, start, s.end, s.writer)?;
start = s.end;
}
Ok(())
}
}
impl<W, C> Slices<W, C> where W: ContextWriter<C> {
pub fn new() -> Slices<W, C> {
Slices{len: 0, slices: Vec::new(), phantom: PhantomData}
}
pub fn reserve(&mut self, additional: usize) {
self.slices.reserve(additional)
}
pub fn append(&mut self, len: u64, writer: W) {
self.len += len;
self.slices.push(SliceInfo{end: self.len, writer: writer});
}
/// Returns the total byte length of all slices.
pub fn len(&self) -> u64 { self.len }
/// Returns the number of slices.
pub fn num(&self) -> usize { self.slices.len() }
pub fn write_to(&self, ctx: &C, range: Range<u64>, out: &mut io::Write)
-> Result<()> {
if range.start > range.end || range.end > self.len {
return Err(Error{
description: format!("Bad range {:?} for slice of length {}", range, self.len),
cause: None});
}
// Binary search for the first slice of the range to write, determining its index and
// (from the preceding slice) the start of its range.
let (mut i, mut slice_start) = match self.slices.binary_search_by_key(&range.start,
|s| s.end) {
Ok(i) if i == self.slices.len() - 1 => return Ok(()), // at end.
Ok(i) => (i+1, self.slices[i].end), // desired start == slice i's end; first is i+1!
Err(i) if i == 0 => (i, 0), // desired start < slice 0's end; first is 0.
Err(i) => (i, self.slices[i-1].end), // desired start < slice i's end; first is i.
};
// Iterate through and write each slice until the end.
let mut start_pos = range.start - slice_start;
loop {
let s = &self.slices[i];
let l = s.end - slice_start;
if range.end <= s.end { // last slice.
return s.writer.write_to(ctx, start_pos .. range.end - slice_start, l, out);
}
s.writer.write_to(ctx, start_pos .. s.end - slice_start, l, out)?;
// setup next iteration.
start_pos = 0;
slice_start = s.end;
i += 1;
}
}
}
#[cfg(test)]
mod tests {
use error::{Error, Result};
use std::cell::RefCell;
use std::error::Error as E;
use std::io::Write;
use std::ops::Range;
use std::vec::Vec;
use super::{ContextWriter, Slices, clip_to_range};
#[derive(Debug, Eq, PartialEq)]
pub struct FakeWrite {
writer: &'static str,
range: Range<u64>,
}
pub struct FakeWriter {
name: &'static str,
}
impl ContextWriter<RefCell<Vec<FakeWrite>>> for FakeWriter {
fn write_to(&self, ctx: &RefCell<Vec<FakeWrite>>, r: Range<u64>, _l: u64, _out: &mut Write)
-> Result<()> {
ctx.borrow_mut().push(FakeWrite{writer: self.name, range: r});
Ok(())
}
}
pub fn new_slices() -> Slices<FakeWriter, RefCell<Vec<FakeWrite>>> {
let mut s = Slices::new();
s.append(5, FakeWriter{name: "a"});
s.append(13, FakeWriter{name: "b"});
s.append(7, FakeWriter{name: "c"});
s.append(17, FakeWriter{name: "d"});
s.append(19, FakeWriter{name: "e"});
s
}
#[test]
pub fn size() {
assert_eq!(5 + 13 + 7 + 17 + 19, new_slices().len());
}
#[test]
pub fn exact_slice() {
// Test writing exactly slice b.
let s = new_slices();
let w = RefCell::new(Vec::new());
let mut dummy = Vec::new();
s.write_to(&w, 5 .. 18, &mut dummy).unwrap();
assert_eq!(&[FakeWrite{writer: "b", range: 0 .. 13}], &w.borrow()[..]);
}
#[test]
pub fn offset_first() {
// Test writing part of slice a.
let s = new_slices();
let w = RefCell::new(Vec::new());
let mut dummy = Vec::new();
s.write_to(&w, 1 .. 3, &mut dummy).unwrap();
assert_eq!(&[FakeWrite{writer: "a", range: 1 .. 3}], &w.borrow()[..]);
}
#[test]
pub fn offset_mid() {
// Test writing part of slice b, all of slice c, and part of slice d.
let s = new_slices();
let w = RefCell::new(Vec::new());
let mut dummy = Vec::new();
s.write_to(&w, 17 .. 26, &mut dummy).unwrap();
assert_eq!(&[
FakeWrite{writer: "b", range: 12 .. 13},
FakeWrite{writer: "c", range: 0 .. 7},
FakeWrite{writer: "d", range: 0 .. 1},
], &w.borrow()[..]);
}
#[test]
pub fn everything() {
// Test writing the whole Slices.
let s = new_slices();
let w = RefCell::new(Vec::new());
let mut dummy = Vec::new();
s.write_to(&w, 0 .. 61, &mut dummy).unwrap();
assert_eq!(&[
FakeWrite{writer: "a", range: 0 .. 5},
FakeWrite{writer: "b", range: 0 .. 13},
FakeWrite{writer: "c", range: 0 .. 7},
FakeWrite{writer: "d", range: 0 .. 17},
FakeWrite{writer: "e", range: 0 .. 19},
], &w.borrow()[..]);
}
#[test]
pub fn at_end() {
let s = new_slices();
let w = RefCell::new(Vec::new());
let mut dummy = Vec::new();
s.write_to(&w, 61 .. 61, &mut dummy).unwrap();
let empty: &[FakeWrite] = &[];
assert_eq!(empty, &w.borrow()[..]);
}
#[test]
pub fn test_clip_to_range() {
let mut out = Vec::new();
// Simple case: one write with everything.
clip_to_range(0 .. 5, 5, &mut out, |w| {
w.write_all(b"01234").unwrap();
Ok(())
}).unwrap();
assert_eq!(b"01234", &out[..]);
// Same in a few writes.
out.clear();
clip_to_range(0 .. 5, 5, &mut out, |w| {
w.write_all(b"0").unwrap();
w.write_all(b"123").unwrap();
w.write_all(b"4").unwrap();
Ok(())
}).unwrap();
assert_eq!(b"01234", &out[..]);
// Limiting to a prefix.
out.clear();
clip_to_range(0 .. 2, 5, &mut out, |w| {
w.write_all(b"0").unwrap(); // all of this write
w.write_all(b"123").unwrap(); // some of this write
w.write_all(b"4").unwrap(); // none of this write
Ok(())
}).unwrap();
assert_eq!(b"01", &out[..]);
// Limiting to part in the middle.
out.clear();
clip_to_range(2 .. 4, 5, &mut out, |w| {
w.write_all(b"0").unwrap(); // none of this write
w.write_all(b"1234").unwrap(); // middle of this write
w.write_all(b"5678").unwrap(); // none of this write
Ok(())
}).unwrap();
assert_eq!(b"23", &out[..]);
// If the callback returns an error, it should be propagated (fast path or not).
out.clear();
assert_eq!(
clip_to_range(0 .. 4, 4, &mut out, |_| Err(Error::new("some error".to_owned())))
.unwrap_err().description(),
"some error");
out.clear();
assert_eq!(
clip_to_range(0 .. 1, 4, &mut out, |_| Err(Error::new("some error".to_owned())))
.unwrap_err().description(),
"some error");
// TODO: if inner.write does a partial write, the next try should start at the correct
// position.
}
}

145
src/profiler.cc
View File

@ -1,145 +0,0 @@
// This file is part of Moonfire NVR, a security camera digital video recorder.
// Copyright (C) 2016 Scott Lamb <slamb@slamb.org>
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// In addition, as a special exception, the copyright holders give
// permission to link the code of portions of this program with the
// OpenSSL library under certain conditions as described in each
// individual source file, and distribute linked combinations including
// the two.
//
// You must obey the GNU General Public License in all respects for all
// of the code used other than OpenSSL. If you modify file(s) with this
// exception, you may extend this exception to your version of the
// file(s), but you are not obligated to do so. If you do not wish to do
// so, delete this exception statement from your version. If you delete
// this exception statement from all source files in the program, then
// also delete it here.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
//
// profiler.cc: See profiler.h.
#include "profiler.h"
#include <errno.h>
#include <fcntl.h>
#include <stdlib.h>
#include <string.h>
#include <sys/stat.h>
#include <sys/types.h>
#include <unistd.h>
#include <cstdlib>
#include <memory>
#include <event2/buffer.h>
#include <event2/event.h>
#include <event2/http.h>
#include <gperftools/profiler.h>
#include <glog/logging.h>
#include "http.h"
#include "string.h"
namespace moonfire_nvr {
namespace {
const int kDefaultProfileSeconds = 30;
// Only a single CPU profile may be active at once. Track if it is active now.
bool profiling;
struct ProfileRequestContext {
#define TEMPLATE "/tmp/moonfire-nvr.profile.XXXXXX"
char filename[sizeof(TEMPLATE)] = TEMPLATE;
#undef TEMPLATE
evhttp_request *req = nullptr;
event *timer = nullptr;
int fd = -1;
};
// End a CPU profile. Serve the result from the temporary file and delete it.
void EndProfileCallback(evutil_socket_t, short, void *arg) {
CHECK(profiling);
ProfilerStop();
profiling = false;
std::unique_ptr<ProfileRequestContext> ctx(
reinterpret_cast<ProfileRequestContext *>(arg));
if (unlink(ctx->filename) < 0) {
int err = errno;
LOG(WARNING) << "Unable to unlink temporary profile file: " << ctx->filename
<< ": " << strerror(err);
}
event_free(ctx->timer);
struct stat statbuf;
if (fstat(ctx->fd, &statbuf) < 0) {
close(ctx->fd);
return HttpSendError(ctx->req, HTTP_INTERNAL, "fstat: ", errno);
}
EvBuffer buf;
std::string error_message;
if (!buf.AddFile(ctx->fd, 0, statbuf.st_size, &error_message)) {
evhttp_send_error(ctx->req, HTTP_INTERNAL,
EscapeHtml(error_message).c_str());
close(ctx->fd);
return;
}
evhttp_send_reply(ctx->req, HTTP_OK, "OK", buf.get());
}
// Start a CPU profile. Creates a temporary file for the profiler library
// to use and schedules a call to EndProfileCallback.
void StartProfileCallback(struct evhttp_request *req, void *arg) {
auto *base = reinterpret_cast<event_base *>(arg);
if (evhttp_request_get_command(req) != EVHTTP_REQ_GET) {
return evhttp_send_error(req, HTTP_BADMETHOD, "only GET allowed");
}
if (profiling) {
return evhttp_send_error(req, HTTP_SERVUNAVAIL,
"Profiling already in progress");
}
struct timeval timeout = {0, 0};
QueryParameters params(evhttp_request_get_uri(req));
const char *seconds_value = params.Get("seconds");
timeout.tv_sec =
seconds_value == nullptr ? kDefaultProfileSeconds : atoi(seconds_value);
if (timeout.tv_sec <= 0) {
return evhttp_send_error(req, HTTP_BADREQUEST, "invalid seconds");
}
auto *ctx = new ProfileRequestContext;
ctx->fd = mkstemp(ctx->filename);
if (ctx->fd < 0) {
delete ctx;
return HttpSendError(req, HTTP_INTERNAL, "mkstemp: ", errno);
}
if (ProfilerStart(ctx->filename) == 0) {
delete ctx;
return evhttp_send_error(req, HTTP_INTERNAL, "ProfilerStart failed");
}
profiling = true;
ctx->req = req;
ctx->timer = evtimer_new(base, &EndProfileCallback, ctx);
evtimer_add(ctx->timer, &timeout);
}
} // namespace
void RegisterProfiler(event_base *base, evhttp *http) {
evhttp_set_cb(http, "/pprof/profile", &StartProfileCallback, base);
}
} // namespace moonfire_nvr

66
src/recording-bench.cc
View File

@ -1,66 +0,0 @@
// This file is part of Moonfire NVR, a security camera network video recorder.
// Copyright (C) 2016 Scott Lamb <slamb@slamb.org>
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// In addition, as a special exception, the copyright holders give
// permission to link the code of portions of this program with the
// OpenSSL library under certain conditions as described in each
// individual source file, and distribute linked combinations including
// the two.
//
// You must obey the GNU General Public License in all respects for all
// of the code used other than OpenSSL. If you modify file(s) with this
// exception, you may extend this exception to your version of the
// file(s), but you are not obligated to do so. If you do not wish to do
// so, delete this exception statement from your version. If you delete
// this exception statement from all source files in the program, then
// also delete it here.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
//
// recording-bench.cc: benchmarks of the recording.h interface.
#include <benchmark/benchmark.h>
#include <gflags/gflags.h>
#include "recording.h"
#include "testutil.h"
DECLARE_bool(alsologtostderr);
static void BM_Iterator(benchmark::State &state) {
using moonfire_nvr::ReadFileOrDie;
using moonfire_nvr::SampleIndexIterator;
// state.PauseTiming();
std::string index = ReadFileOrDie("../src/testdata/video_sample_index.bin");
// state.ResumeTiming();
while (state.KeepRunning()) {
SampleIndexIterator it(index);
while (!it.done()) it.Next();
CHECK(!it.has_error()) << it.error();
}
state.SetBytesProcessed(int64_t(state.iterations()) * int64_t(index.size()));
}
BENCHMARK(BM_Iterator);
int main(int argc, char **argv) {
FLAGS_alsologtostderr = true;
// Sadly, these two flag-parsing libraries don't appear to get along.
// google::ParseCommandLineFlags(&argc, &argv, true);
benchmark::Initialize(&argc, argv);
google::InitGoogleLogging(argv[0]);
benchmark::RunSpecifiedBenchmarks();
return 0;
}

264
src/recording-test.cc
View File

@ -1,264 +0,0 @@
// This file is part of Moonfire NVR, a security camera network video recorder.
// Copyright (C) 2016 Scott Lamb <slamb@slamb.org>
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// In addition, as a special exception, the copyright holders give
// permission to link the code of portions of this program with the
// OpenSSL library under certain conditions as described in each
// individual source file, and distribute linked combinations including
// the two.
//
// You must obey the GNU General Public License in all respects for all
// of the code used other than OpenSSL. If you modify file(s) with this
// exception, you may extend this exception to your version of the
// file(s), but you are not obligated to do so. If you do not wish to do
// so, delete this exception statement from your version. If you delete
// this exception statement from all source files in the program, then
// also delete it here.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
//
// recording-test.cc: tests of the recording.h interface.
#include <fcntl.h>
#include <sys/stat.h>
#include <sys/types.h>
#include <gflags/gflags.h>
#include <gmock/gmock.h>
#include <gtest/gtest.h>
#include "recording.h"
#include "string.h"
#include "testutil.h"
DECLARE_bool(alsologtostderr);
using testing::_;
using testing::HasSubstr;
using testing::DoAll;
using testing::Return;
using testing::SetArgPointee;
namespace moonfire_nvr {
namespace {
// Example from design/schema.md.
TEST(SampleIndexTest, EncodeExample) {
Recording recording;
SampleIndexEncoder encoder;
encoder.Init(&recording, 1000);
encoder.AddSample(10, 1000, true);
encoder.AddSample(9, 10, false);
encoder.AddSample(11, 15, false);
encoder.AddSample(10, 12, false);
encoder.AddSample(10, 1050, true);
EXPECT_EQ("29 d0 0f 02 14 08 0a 02 05 01 64",
ToHex(recording.video_index, true));
EXPECT_EQ(1000, recording.start_time_90k);
EXPECT_EQ(1000 + 10 + 9 + 11 + 10 + 10, recording.end_time_90k);
EXPECT_EQ(1000 + 10 + 15 + 12 + 1050, recording.sample_file_bytes);
EXPECT_EQ(5, recording.video_samples);
EXPECT_EQ(2, recording.video_sync_samples);
}
TEST(SampleIndexTest, RoundTrip) {
Recording recording;
SampleIndexEncoder encoder;
encoder.Init(&recording, 1000);
encoder.AddSample(10, 30000, true);
encoder.AddSample(9, 1000, false);
encoder.AddSample(11, 1100, false);
encoder.AddSample(18, 31000, true);
SampleIndexIterator it = SampleIndexIterator(recording.video_index);
std::string error_message;
ASSERT_FALSE(it.done()) << it.error();
EXPECT_EQ(10, it.duration_90k());
EXPECT_EQ(30000, it.bytes());
EXPECT_TRUE(it.is_key());
it.Next();
ASSERT_FALSE(it.done()) << it.error();
EXPECT_EQ(9, it.duration_90k());
EXPECT_EQ(1000, it.bytes());
EXPECT_FALSE(it.is_key());
it.Next();
ASSERT_FALSE(it.done()) << it.error();
EXPECT_EQ(11, it.duration_90k());
EXPECT_EQ(1100, it.bytes());
EXPECT_FALSE(it.is_key());
it.Next();
ASSERT_FALSE(it.done()) << it.error();
EXPECT_EQ(18, it.duration_90k());
EXPECT_EQ(31000, it.bytes());
EXPECT_TRUE(it.is_key());
it.Next();
ASSERT_TRUE(it.done());
ASSERT_FALSE(it.has_error()) << it.error();
}
TEST(SampleIndexTest, IteratorErrors) {
std::string bad_first_varint("\x80");
SampleIndexIterator it(bad_first_varint);
EXPECT_TRUE(it.has_error());
EXPECT_EQ("buffer underrun", it.error());
std::string bad_second_varint("\x00\x80", 2);
it = SampleIndexIterator(bad_second_varint);
EXPECT_TRUE(it.has_error());
EXPECT_EQ("buffer underrun", it.error());
std::string zero_durations("\x00\x02\x00\x00", 4);
it = SampleIndexIterator(zero_durations);
EXPECT_TRUE(it.has_error());
EXPECT_THAT(it.error(), HasSubstr("zero duration"));
std::string negative_duration("\x02\x02", 2);
it = SampleIndexIterator(negative_duration);
EXPECT_TRUE(it.has_error());
EXPECT_THAT(it.error(), HasSubstr("negative duration"));
std::string non_positive_bytes("\x04\x00", 2);
it = SampleIndexIterator(non_positive_bytes);
EXPECT_TRUE(it.has_error());
EXPECT_THAT(it.error(), HasSubstr("non-positive bytes"));
}
TEST(SampleFileWriterTest, Simple) {
testing::StrictMock<MockFile> parent;
auto *f = new testing::StrictMock<MockFile>;
re2::StringPiece write_1("write 1");
re2::StringPiece write_2("write 2");
EXPECT_CALL(parent, OpenRaw("foo", O_WRONLY | O_EXCL | O_CREAT, 0600, _))
.WillOnce(DoAll(SetArgPointee<3>(f), Return(0)));
EXPECT_CALL(*f, Write(write_1, _))
.WillOnce(DoAll(SetArgPointee<1>(7), Return(0)));
EXPECT_CALL(*f, Write(write_2, _))
.WillOnce(DoAll(SetArgPointee<1>(7), Return(0)));
EXPECT_CALL(*f, Sync()).WillOnce(Return(0));
EXPECT_CALL(*f, Close()).WillOnce(Return(0));
SampleFileWriter writer(&parent);
std::string error_message;
std::string sha1;
ASSERT_TRUE(writer.Open("foo", &error_message)) << error_message;
EXPECT_TRUE(writer.Write(write_1, &error_message)) << error_message;
EXPECT_TRUE(writer.Write(write_2, &error_message)) << error_message;
EXPECT_TRUE(writer.Close(&sha1, &error_message)) << error_message;
EXPECT_EQ("6bc37325b36fb5fd205e57284429e75764338618", ToHex(sha1));
}
TEST(SampleFileWriterTest, PartialWriteIsRetried) {
testing::StrictMock<MockFile> parent;
auto *f = new testing::StrictMock<MockFile>;
re2::StringPiece write_1("write 1");
re2::StringPiece write_2("write 2");
re2::StringPiece write_2b(write_2);
write_2b.remove_prefix(3);
EXPECT_CALL(parent, OpenRaw("foo", O_WRONLY | O_EXCL | O_CREAT, 0600, _))
.WillOnce(DoAll(SetArgPointee<3>(f), Return(0)));
EXPECT_CALL(*f, Write(write_1, _))
.WillOnce(DoAll(SetArgPointee<1>(7), Return(0)));
EXPECT_CALL(*f, Write(write_2, _))
.WillOnce(DoAll(SetArgPointee<1>(3), Return(0)));
EXPECT_CALL(*f, Write(write_2b, _))
.WillOnce(DoAll(SetArgPointee<1>(4), Return(0)));
EXPECT_CALL(*f, Sync()).WillOnce(Return(0));
EXPECT_CALL(*f, Close()).WillOnce(Return(0));
SampleFileWriter writer(&parent);
std::string error_message;
std::string sha1;
ASSERT_TRUE(writer.Open("foo", &error_message)) << error_message;
EXPECT_TRUE(writer.Write(write_1, &error_message)) << error_message;
EXPECT_TRUE(writer.Write(write_2, &error_message)) << error_message;
EXPECT_TRUE(writer.Close(&sha1, &error_message)) << error_message;
EXPECT_EQ("6bc37325b36fb5fd205e57284429e75764338618", ToHex(sha1));
}
TEST(SampleFileWriterTest, PartialWriteIsTruncated) {
testing::StrictMock<MockFile> parent;
auto *f = new testing::StrictMock<MockFile>;
re2::StringPiece write_1("write 1");
re2::StringPiece write_2("write 2");
re2::StringPiece write_2b(write_2);
write_2b.remove_prefix(3);
EXPECT_CALL(parent, OpenRaw("foo", O_WRONLY | O_EXCL | O_CREAT, 0600, _))
.WillOnce(DoAll(SetArgPointee<3>(f), Return(0)));
EXPECT_CALL(*f, Write(write_1, _))
.WillOnce(DoAll(SetArgPointee<1>(7), Return(0)));
EXPECT_CALL(*f, Write(write_2, _))
.WillOnce(DoAll(SetArgPointee<1>(3), Return(0)));
EXPECT_CALL(*f, Write(write_2b, _)).WillOnce(Return(ENOSPC));
EXPECT_CALL(*f, Truncate(7)).WillOnce(Return(0));
EXPECT_CALL(*f, Sync()).WillOnce(Return(0));
EXPECT_CALL(*f, Close()).WillOnce(Return(0));
SampleFileWriter writer(&parent);
std::string error_message;
std::string sha1;
ASSERT_TRUE(writer.Open("foo", &error_message)) << error_message;
EXPECT_TRUE(writer.Write(write_1, &error_message)) << error_message;
EXPECT_FALSE(writer.Write(write_2, &error_message)) << error_message;
EXPECT_TRUE(writer.Close(&sha1, &error_message)) << error_message;
EXPECT_EQ("b1ccee339b935587c09997a9ec8bb2374e02b5d0", ToHex(sha1));
}
TEST(SampleFileWriterTest, PartialWriteTruncateFailureCausesCloseToFail) {
testing::StrictMock<MockFile> parent;
auto *f = new testing::StrictMock<MockFile>;
re2::StringPiece write_1("write 1");
re2::StringPiece write_2("write 2");
re2::StringPiece write_2b(write_2);
write_2b.remove_prefix(3);
EXPECT_CALL(parent, OpenRaw("foo", O_WRONLY | O_EXCL | O_CREAT, 0600, _))
.WillOnce(DoAll(SetArgPointee<3>(f), Return(0)));
EXPECT_CALL(*f, Write(write_1, _))
.WillOnce(DoAll(SetArgPointee<1>(7), Return(0)));
EXPECT_CALL(*f, Write(write_2, _))
.WillOnce(DoAll(SetArgPointee<1>(3), Return(0)));
EXPECT_CALL(*f, Write(write_2b, _)).WillOnce(Return(EIO));
EXPECT_CALL(*f, Truncate(7)).WillOnce(Return(EIO));
EXPECT_CALL(*f, Close()).WillOnce(Return(0));
SampleFileWriter writer(&parent);
std::string error_message;
std::string sha1;
ASSERT_TRUE(writer.Open("foo", &error_message)) << error_message;
EXPECT_TRUE(writer.Write(write_1, &error_message)) << error_message;
EXPECT_FALSE(writer.Write(write_2, &error_message)) << error_message;
EXPECT_FALSE(writer.Close(&sha1, &error_message)) << error_message;
}
} // namespace
} // namespace moonfire_nvr
int main(int argc, char **argv) {
FLAGS_alsologtostderr = true;
google::ParseCommandLineFlags(&argc, &argv, true);
testing::InitGoogleTest(&argc, argv);
google::InitGoogleLogging(argv[0]);
return RUN_ALL_TESTS();
}

229
src/recording.cc
View File

@ -1,229 +0,0 @@
// This file is part of Moonfire NVR, a security camera network video recorder.
// Copyright (C) 2015 Scott Lamb <slamb@slamb.org>
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// In addition, as a special exception, the copyright holders give
// permission to link the code of portions of this program with the
// OpenSSL library under certain conditions as described in each
// individual source file, and distribute linked combinations including
// the two.
//
// You must obey the GNU General Public License in all respects for all
// of the code used other than OpenSSL. If you modify file(s) with this
// exception, you may extend this exception to your version of the
// file(s), but you are not obligated to do so. If you do not wish to do
// so, delete this exception statement from your version. If you delete
// this exception statement from all source files in the program, then
// also delete it here.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
//
// recording.cc: see recording.h.
#include "recording.h"
#include <fcntl.h>
#include <sys/stat.h>
#include <sys/types.h>
#include "common.h"
#include "coding.h"
#include "string.h"
namespace moonfire_nvr {
void SampleIndexEncoder::Init(Recording *recording, int64_t start_time_90k) {
recording_ = recording;
recording_->start_time_90k = start_time_90k;
recording_->end_time_90k = start_time_90k;
recording_->sample_file_bytes = 0;
recording_->video_samples = 0;
recording_->video_sync_samples = 0;
recording_->video_index.clear();
prev_duration_90k_ = 0;
prev_bytes_key_ = 0;
prev_bytes_nonkey_ = 0;
}
void SampleIndexEncoder::AddSample(int32_t duration_90k, int32_t bytes,
bool is_key) {
CHECK_GE(duration_90k, 0);
CHECK_GT(bytes, 0);
int32_t duration_delta = duration_90k - prev_duration_90k_;
prev_duration_90k_ = duration_90k;
int32_t bytes_delta;
recording_->end_time_90k += duration_90k;
recording_->sample_file_bytes += bytes;
++recording_->video_samples;
if (is_key) {
bytes_delta = bytes - prev_bytes_key_;
prev_bytes_key_ = bytes;
++recording_->video_sync_samples;
} else {
bytes_delta = bytes - prev_bytes_nonkey_;
prev_bytes_nonkey_ = bytes;
}
uint32_t zigzagged_bytes_delta = Zigzag32(bytes_delta);
AppendVar32((Zigzag32(duration_delta) << 1) | is_key,
&recording_->video_index);
AppendVar32(zigzagged_bytes_delta, &recording_->video_index);
}
void SampleIndexIterator::Next() {
uint32_t raw1;
uint32_t raw2;
pos_ += bytes_;
if (UNLIKELY(data_.empty()) ||
UNLIKELY(!DecodeVar32(&data_, &raw1, &error_)) ||
UNLIKELY(!DecodeVar32(&data_, &raw2, &error_))) {
done_ = true;
return;
}
start_90k_ += duration_90k_;
int32_t duration_90k_delta = Unzigzag32(raw1 >> 1);
duration_90k_ += duration_90k_delta;
if (UNLIKELY(duration_90k_ < 0)) {
error_ = StrCat("negative duration ", duration_90k_,
" after applying delta ", duration_90k_delta);
done_ = true;
return;
}
if (UNLIKELY(duration_90k_ == 0 && !data_.empty())) {
error_ = StrCat("zero duration only allowed at end; have ", data_.size(),
"bytes left.");
done_ = true;
return;
}
is_key_ = raw1 & 0x01;
int32_t bytes_delta = Unzigzag32(raw2);
if (UNLIKELY(is_key_)) {
bytes_ = bytes_key_ += bytes_delta;
} else {
bytes_ = bytes_nonkey_ += bytes_delta;
}
if (UNLIKELY(bytes_ <= 0)) {
error_ = StrCat("non-positive bytes ", bytes_, " after applying delta ",
bytes_delta, " to ", (is_key_ ? "key" : "non-key"),
" frame at ts ", start_90k_);
done_ = true;
return;
}
done_ = false;
return;
}
void SampleIndexIterator::Clear() {
data_.clear();
error_.clear();
pos_ = 0;
start_90k_ = 0;
duration_90k_ = 0;
bytes_key_ = 0;
bytes_nonkey_ = 0;
bytes_ = 0;
is_key_ = false;
done_ = true;
}
SampleFileWriter::SampleFileWriter(File *parent_dir)
: parent_dir_(parent_dir), sha1_(Digest::SHA1()) {}
bool SampleFileWriter::Open(const char *filename, std::string *error_message) {
if (is_open()) {
*error_message = "already open!";
return false;
}
int ret =
parent_dir_->Open(filename, O_WRONLY | O_CREAT | O_EXCL, 0600, &file_);
if (ret != 0) {
*error_message = StrCat("open ", filename, " (within dir ",
parent_dir_->name(), "): ", strerror(ret));
return false;
}
return true;
}
bool SampleFileWriter::Write(re2::StringPiece pkt, std::string *error_message) {
if (!is_open()) {
*error_message = "not open!";
return false;
}
auto old_pos = pos_;
re2::StringPiece remaining(pkt);
while (!remaining.empty()) {
size_t written;
int write_ret = file_->Write(remaining, &written);
if (write_ret != 0) {
if (pos_ > old_pos) {
int truncate_ret = file_->Truncate(old_pos);
if (truncate_ret != 0) {
*error_message =
StrCat("write failed with: ", strerror(write_ret),
" and ftruncate failed with: ", strerror(truncate_ret));
corrupt_ = true;
return false;
}
}
*error_message = StrCat("write: ", strerror(write_ret));
return false;
}
remaining.remove_prefix(written);
pos_ += written;
}
sha1_->Update(pkt);
return true;
}
bool SampleFileWriter::Close(std::string *sha1, std::string *error_message) {
if (!is_open()) {
*error_message = "not open!";
return false;
}
if (corrupt_) {
*error_message = "File already corrupted.";
} else {
int ret = file_->Sync();
if (ret != 0) {
*error_message = StrCat("fsync failed with: ", strerror(ret));
corrupt_ = true;
}
}
int ret = file_->Close();
if (ret != 0 && !corrupt_) {
corrupt_ = true;
*error_message = StrCat("close failed with: ", strerror(ret));
}
bool ok = !corrupt_;
file_.reset();
*sha1 = sha1_->Finalize();
sha1_ = Digest::SHA1();
pos_ = 0;
corrupt_ = false;
return ok;
}
std::string PrettyTimestamp(int64_t ts_90k) {
struct tm mytm;
memset(&mytm, 0, sizeof(mytm));
time_t ts = ts_90k / kTimeUnitsPerSecond;
localtime_r(&ts, &mytm);
const size_t kTimeBufLen = 50;
char tmbuf[kTimeBufLen];
strftime(tmbuf, kTimeBufLen, "%a, %d %b %Y %H:%M:%S %Z", &mytm);
return tmbuf;
}
} // namespace moonfire_nvr

214
src/recording.h
View File

@ -1,214 +0,0 @@
// This file is part of Moonfire NVR, a security camera network video recorder.
// Copyright (C) 2016 Scott Lamb <slamb@slamb.org>
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// In addition, as a special exception, the copyright holders give
// permission to link the code of portions of this program with the
// OpenSSL library under certain conditions as described in each
// individual source file, and distribute linked combinations including
// the two.
//
// You must obey the GNU General Public License in all respects for all
// of the code used other than OpenSSL. If you modify file(s) with this
// exception, you may extend this exception to your version of the
// file(s), but you are not obligated to do so. If you do not wish to do
// so, delete this exception statement from your version. If you delete
// this exception statement from all source files in the program, then
// also delete it here.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
//
// recording.h: Write and read recordings. See design/schema.md for a
// description of the storage schema.
#ifndef MOONFIRE_NVR_RECORDING_H
#define MOONFIRE_NVR_RECORDING_H
#include <stdint.h>
#include <memory>
#include <string>
#include <glog/logging.h>
#include <re2/stringpiece.h>
#include "crypto.h"
#include "filesystem.h"
#include "uuid.h"
namespace moonfire_nvr {
constexpr int64_t kTimeUnitsPerSecond = 90000;
// Recordings are never longer than this (5 minutes).
// Having such a limit dramatically speeds up some SQL queries.
// This limit should be more than the normal rotation time,
// as recording doesn't happen until the next key frame.
// 5 minutes is generously more than 1 minute, but still sufficient to
// allow the optimization to be useful. This value must match the CHECK
// constraint on duration_90k in schema.sql.
constexpr int64_t kMaxRecordingDuration = 5 * 60 * kTimeUnitsPerSecond;
// Various fields from the "recording" table which are useful when viewing
// recordings.
struct Recording {
int64_t id = -1;
int64_t camera_id = -1;
std::string sample_file_sha1;
std::string sample_file_path;
Uuid sample_file_uuid;
int64_t video_sample_entry_id = -1;
int64_t local_time_90k = -1;
// Fields populated by SampleIndexEncoder.
int64_t start_time_90k = -1;
int64_t end_time_90k = -1;
int64_t sample_file_bytes = -1;
int64_t video_samples = -1;
int64_t video_sync_samples = -1;
std::string video_index;
};
// Reusable object to encode sample index data to a Recording object.
class SampleIndexEncoder {
public:
SampleIndexEncoder() {}
SampleIndexEncoder(const SampleIndexEncoder &) = delete;
void operator=(const SampleIndexEncoder &) = delete;
void Init(Recording *recording, int64_t start_time_90k);
void AddSample(int32_t duration_90k, int32_t bytes, bool is_key);
private:
Recording *recording_;
int32_t prev_duration_90k_ = 0;
int32_t prev_bytes_key_ = 0;
int32_t prev_bytes_nonkey_ = 0;
};
// Iterates through an encoded index, decoding on the fly. Copyable.
// Example usage:
//
// SampleIndexIterator it;
// for (it = index; !it.done(); it.Next()) {
// LOG(INFO) << "sample size: " << it.bytes();
// }
// if (it.has_error()) {
// LOG(ERROR) << "error: " << it.error();
// }
class SampleIndexIterator {
public:
SampleIndexIterator() { Clear(); }
// |index| must outlive the iterator.
explicit SampleIndexIterator(re2::StringPiece index) {
Clear();
data_ = index;
Next();
}
// Iteration control.
void Next();
bool done() const { return done_; }
bool has_error() const { return !error_.empty(); }
const std::string &error() const { return error_; }
// Return properties of the current sample.
// Note pos() and start_90k() are valid when done(); the others are not.
int64_t pos() const { return pos_; }
int32_t start_90k() const { return start_90k_; }
int32_t duration_90k() const {
DCHECK(!done_);
return duration_90k_;
}
int32_t end_90k() const { return start_90k_ + duration_90k(); }
int32_t bytes() const {
DCHECK(!done_);
return bytes_;
}
bool is_key() const {
DCHECK(!done_);
return is_key_;
}
private:
void Clear();
re2::StringPiece data_;
std::string error_;
int64_t pos_;
int32_t start_90k_;
int32_t duration_90k_;
int32_t bytes_; // bytes taken by the current sample, or 0 after Clear().
int32_t bytes_key_;
int32_t bytes_nonkey_;
bool is_key_;
bool done_;
};
// Writes a sample file. Can be used repeatedly. Thread-compatible.
class SampleFileWriter {
public:
// |parent_dir| must outlive the writer.
SampleFileWriter(File *parent_dir);
SampleFileWriter(const SampleFileWriter &) = delete;
void operator=(const SampleFileWriter &) = delete;
// PRE: !is_open().
bool Open(const char *filename, std::string *error_message);
// Writes a single packet, returning success.
// On failure, the stream should be closed. If Close() returns true, the
// file contains the results of all packets up to (but not including) this
// one.
//
// PRE: is_open().
bool Write(re2::StringPiece pkt, std::string *error_message);
// fsync() and close() the stream.
// Note the caller is still responsible for fsync()ing the parent stream,
// so that operations can be batched.
// On success, |sha1| will be filled with the raw SHA-1 hash of the file.
// On failure, the file should be considered corrupt and discarded.
//
// PRE: is_open().
bool Close(std::string *sha1, std::string *error_message);
bool is_open() const { return file_ != nullptr; }
private:
File *parent_dir_;
std::unique_ptr<File> file_;
std::unique_ptr<Digest> sha1_;
int64_t pos_ = 0;
bool corrupt_ = false;
};
struct VideoSampleEntry {
int64_t id = -1;
std::string sha1;
std::string data;
uint16_t width = 0;
uint16_t height = 0;
};
std::string PrettyTimestamp(int64_t ts_90k);
inline int64_t To90k(const struct timespec &ts) {
return (ts.tv_sec * kTimeUnitsPerSecond) +
(ts.tv_nsec * kTimeUnitsPerSecond / 1000000000);
}
} // namespace moonfire_nvr
#endif // MOONFIRE_NVR_RECORDING_H

726
src/recording.rs Normal file
View File

@ -0,0 +1,726 @@
// This file is part of Moonfire NVR, a security camera digital video recorder.
// Copyright (C) 2016 Scott Lamb <slamb@slamb.org>
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// In addition, as a special exception, the copyright holders give
// permission to link the code of portions of this program with the
// OpenSSL library under certain conditions as described in each
// individual source file, and distribute linked combinations including
// the two.
//
// You must obey the GNU General Public License in all respects for all
// of the code used other than OpenSSL. If you modify file(s) with this
// exception, you may extend this exception to your version of the
// file(s), but you are not obligated to do so. If you do not wish to do
// so, delete this exception statement from your version. If you delete
// this exception statement from all source files in the program, then
// also delete it here.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
#![allow(inline_always)]
extern crate uuid;
use db;
use std::ops;
use error::Error;
use openssl::crypto::hash;
use std::fmt;
use std::fs;
use std::io::Write;
use std::ops::Range;
use std::string::String;
use time;
use uuid::Uuid;
pub const TIME_UNITS_PER_SEC: i64 = 90000;
pub const DESIRED_RECORDING_DURATION: i64 = 60 * TIME_UNITS_PER_SEC;
pub const MAX_RECORDING_DURATION: i64 = 5 * 60 * TIME_UNITS_PER_SEC;
/// A time specified as 90,000ths of a second since 1970-01-01 00:00:00 UTC.
#[derive(Clone, Copy, Debug, Eq, Ord, PartialEq, PartialOrd, Serialize)]
pub struct Time(pub i64);
impl Time {
pub fn new(tm: time::Timespec) -> Self {
Time(tm.sec * TIME_UNITS_PER_SEC + tm.nsec as i64 * TIME_UNITS_PER_SEC / 1_000_000_000)
}
pub fn unix_seconds(&self) -> i64 { self.0 / TIME_UNITS_PER_SEC }
}
impl ops::Sub for Time {
type Output = Duration;
fn sub(self, rhs: Time) -> Duration { Duration(self.0 - rhs.0) }
}
impl ops::AddAssign<Duration> for Time {
fn add_assign(&mut self, rhs: Duration) { self.0 += rhs.0 }
}
impl ops::Add<Duration> for Time {
type Output = Time;
fn add(self, rhs: Duration) -> Time { Time(self.0 + rhs.0) }
}
impl ops::Sub<Duration> for Time {
type Output = Time;
fn sub(self, rhs: Duration) -> Time { Time(self.0 - rhs.0) }
}
impl fmt::Display for Time {
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
let tm = time::at(time::Timespec{sec: self.0 / TIME_UNITS_PER_SEC, nsec: 0});
write!(f, "{}:{:05}", tm.strftime("%FT%T%Z").or_else(|_| Err(fmt::Error))?,
self.0 % TIME_UNITS_PER_SEC)
}
}
/// A duration specified in 1/90,000ths of a second.
/// Durations are typically non-negative, but a `db::CameraDayValue::duration` may be negative.
#[derive(Clone, Copy, Debug, Eq, Ord, PartialEq, PartialOrd, Serialize)]
pub struct Duration(pub i64);
impl fmt::Display for Duration {
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
let mut seconds = self.0 / TIME_UNITS_PER_SEC;
const MINUTE_IN_SECONDS: i64 = 60;
const HOUR_IN_SECONDS: i64 = 60 * MINUTE_IN_SECONDS;
const DAY_IN_SECONDS: i64 = 24 * HOUR_IN_SECONDS;
let days = seconds / DAY_IN_SECONDS;
seconds %= DAY_IN_SECONDS;
let hours = seconds / HOUR_IN_SECONDS;
seconds %= HOUR_IN_SECONDS;
let minutes = seconds / MINUTE_IN_SECONDS;
seconds %= MINUTE_IN_SECONDS;
let mut have_written = if days > 0 {
write!(f, "{} day{}", days, if days == 1 { "" } else { "s" })?;
true
} else {
false
};
if hours > 0 {
write!(f, "{}{} hour{}", if have_written { " " } else { "" },
hours, if hours == 1 { "" } else { "s" })?;
have_written = true;
}
if minutes > 0 {
write!(f, "{}{} minute{}", if have_written { " " } else { "" },
minutes, if minutes == 1 { "" } else { "s" })?;
have_written = true;
}
if seconds > 0 || !have_written {
write!(f, "{}{} second{}", if have_written { " " } else { "" },
seconds, if seconds == 1 { "" } else { "s" })?;
}
Ok(())
}
}
impl ops::Add for Duration {
type Output = Duration;
fn add(self, rhs: Duration) -> Duration { Duration(self.0 + rhs.0) }
}
impl ops::AddAssign for Duration {
fn add_assign(&mut self, rhs: Duration) { self.0 += rhs.0 }
}
impl ops::SubAssign for Duration {
fn sub_assign(&mut self, rhs: Duration) { self.0 -= rhs.0 }
}
#[derive(Clone, Copy, Debug)]
pub struct SampleIndexIterator {
i: usize,
pub pos: i32,
pub start_90k: i32,
pub duration_90k: i32,
pub bytes: i32,
bytes_key: i32,
bytes_nonkey: i32,
pub is_key: bool
}
#[derive(Debug)]
pub struct SampleIndexEncoder {
// Internal state.
prev_duration_90k: i32,
prev_bytes_key: i32,
prev_bytes_nonkey: i32,
// Eventual output.
// TODO: move to another struct?
pub sample_file_bytes: i32,
pub total_duration_90k: i32,
pub video_samples: i32,
pub video_sync_samples: i32,
pub video_index: Vec<u8>,
}
pub struct Writer {
f: fs::File,
index: SampleIndexEncoder,
uuid: Uuid,
corrupt: bool,
hasher: hash::Hasher,
start_time: Time,
local_time: Time,
camera_id: i32,
video_sample_entry_id: i32,
}
/// Zigzag-encodes a signed integer, as in [protocol buffer
/// encoding](https://developers.google.com/protocol-buffers/docs/encoding#types). Uses the low bit
/// to indicate signedness (1 = negative, 0 = non-negative).
#[inline(always)]
fn zigzag32(i: i32) -> u32 { ((i << 1) as u32) ^ ((i >> 31) as u32) }
/// Zigzag-decodes to a signed integer.
/// See `zigzag`.
#[inline(always)]
fn unzigzag32(i: u32) -> i32 { ((i >> 1) as i32) ^ -((i & 1) as i32) }
#[inline(always)]
fn decode_varint32(data: &[u8], i: usize) -> Result<(u32, usize), ()> {
// Unroll a few likely possibilities before going into the robust out-of-line loop.
// This aids branch prediction.
if data.len() > i && (data[i] & 0x80) == 0 {
return Ok((data[i] as u32, i+1))
} else if data.len() > i + 1 && (data[i+1] & 0x80) == 0 {
return Ok((( (data[i] & 0x7f) as u32) |
(( data[i+1] as u32) << 7),
i+2))
} else if data.len() > i + 2 && (data[i+2] & 0x80) == 0 {
return Ok((( (data[i] & 0x7f) as u32) |
(((data[i+1] & 0x7f) as u32) << 7) |
(( data[i+2] as u32) << 14),
i+3))
}
decode_varint32_slow(data, i)
}
#[cold]
fn decode_varint32_slow(data: &[u8], mut i: usize) -> Result<(u32, usize), ()> {
let l = data.len();
let mut out = 0;
let mut shift = 0;
loop {
if i == l {
return Err(())
}
let b = data[i];
if shift == 28 && (b & 0xf0) != 0 {
return Err(())
}
out |= ((b & 0x7f) as u32) << shift;
shift += 7;
i += 1;
if (b & 0x80) == 0 {
break;
}
}
Ok((out, i))
}
fn append_varint32(i: u32, data: &mut Vec<u8>) {
if i < 1u32 << 7 {
data.push(i as u8);
} else if i < 1u32 << 14 {
data.extend_from_slice(&[(( i & 0x7F) | 0x80) as u8,
(i >> 7) as u8]);
} else if i < 1u32 << 21 {
data.extend_from_slice(&[(( i & 0x7F) | 0x80) as u8,
(((i >> 7) & 0x7F) | 0x80) as u8,
(i >> 14) as u8]);
} else if i < 1u32 << 28 {
data.extend_from_slice(&[(( i & 0x7F) | 0x80) as u8,
(((i >> 7) & 0x7F) | 0x80) as u8,
(((i >> 14) & 0x7F) | 0x80) as u8,
(i >> 21) as u8]);
} else {
data.extend_from_slice(&[(( i & 0x7F) | 0x80) as u8,
(((i >> 7) & 0x7F) | 0x80) as u8,
(((i >> 14) & 0x7F) | 0x80) as u8,
(((i >> 21) & 0x7F) | 0x80) as u8,
(i >> 28) as u8]);
}
}
impl SampleIndexIterator {
pub fn new() -> SampleIndexIterator {
SampleIndexIterator{i: 0,
pos: 0,
start_90k: 0,
duration_90k: 0,
bytes: 0,
bytes_key: 0,
bytes_nonkey: 0,
is_key: false}
}
pub fn next(&mut self, data: &[u8]) -> Result<bool, Error> {
self.pos += self.bytes;
self.start_90k += self.duration_90k;
if self.i == data.len() {
return Ok(false)
}
let (raw1, i1) = match decode_varint32(data, self.i) {
Ok(tuple) => tuple,
Err(()) => return Err(Error::new(format!("bad varint 1 at offset {}", self.i))),
};
let (raw2, i2) = match decode_varint32(data, i1) {
Ok(tuple) => tuple,
Err(()) => return Err(Error::new(format!("bad varint 2 at offset {}", i1))),
};
self.i = i2;
let duration_90k_delta = unzigzag32(raw1 >> 1);
self.duration_90k += duration_90k_delta;
if self.duration_90k < 0 {
return Err(Error{
description: format!("negative duration {} after applying delta {}",
self.duration_90k, duration_90k_delta),
cause: None});
}
if self.duration_90k == 0 && data.len() > self.i {
return Err(Error{
description: format!("zero duration only allowed at end; have {} bytes left",
data.len() - self.i),
cause: None});
}
self.is_key = (raw1 & 1) == 1;
let bytes_delta = unzigzag32(raw2);
self.bytes = if self.is_key {
self.bytes_key += bytes_delta;
self.bytes_key
} else {
self.bytes_nonkey += bytes_delta;
self.bytes_nonkey
};
if self.bytes <= 0 {
return Err(Error{
description: format!("non-positive bytes {} after applying delta {} to key={} frame at ts {}",
self.bytes, bytes_delta, self.is_key,
self.start_90k),
cause: None});
}
Ok(true)
}
}
impl SampleIndexEncoder {
pub fn new() -> Self {
SampleIndexEncoder{
prev_duration_90k: 0,
prev_bytes_key: 0,
prev_bytes_nonkey: 0,
total_duration_90k: 0,
sample_file_bytes: 0,
video_samples: 0,
video_sync_samples: 0,
video_index: Vec::new(),
}
}
pub fn add_sample(&mut self, duration_90k: i32, bytes: i32, is_key: bool) {
let duration_delta = duration_90k - self.prev_duration_90k;
self.prev_duration_90k = duration_90k;
self.total_duration_90k += duration_90k;
self.sample_file_bytes += bytes;
self.video_samples += 1;
let bytes_delta = bytes - if is_key {
let prev = self.prev_bytes_key;
self.video_sync_samples += 1;
self.prev_bytes_key = bytes;
prev
} else {
let prev = self.prev_bytes_nonkey;
self.prev_bytes_nonkey = bytes;
prev
};
append_varint32((zigzag32(duration_delta) << 1) | (is_key as u32), &mut self.video_index);
append_varint32(zigzag32(bytes_delta), &mut self.video_index);
}
}
impl Writer {
pub fn open(f: fs::File, uuid: Uuid, start_time: Time, local_time: Time,
camera_id: i32, video_sample_entry_id: i32) -> Result<Self, Error> {
Ok(Writer{
f: f,
index: SampleIndexEncoder::new(),
uuid: uuid,
corrupt: false,
hasher: hash::Hasher::new(hash::Type::SHA1)?,
start_time: start_time,
local_time: local_time,
camera_id: camera_id,
video_sample_entry_id: video_sample_entry_id,
})
}
pub fn write(&mut self, pkt: &[u8], duration_90k: i32, is_key: bool) -> Result<(), Error> {
let mut remaining = pkt;
while !remaining.is_empty() {
let written = match self.f.write(remaining) {
Ok(b) => b,
Err(e) => {
if remaining.len() < pkt.len() {
// Partially written packet. Truncate if possible.
if let Err(e2) = self.f.set_len(self.index.sample_file_bytes as u64) {
error!("After write to {} failed with {}, truncate failed with {}; \
sample file is corrupt.", self.uuid.hyphenated(), e, e2);
self.corrupt = true;
}
}
return Err(Error::from(e));
},
};
remaining = &remaining[written..];
}
self.index.add_sample(duration_90k, pkt.len() as i32, is_key);
self.hasher.update(pkt)?;
Ok(())
}
pub fn end(&self) -> Time {
self.start_time + Duration(self.index.total_duration_90k as i64)
}
// TODO: clean up this interface.
pub fn close(mut self) -> Result<(db::RecordingToInsert, fs::File), Error> {
if self.corrupt {
return Err(Error::new(format!("recording {} is corrupt", self.uuid)));
}
let mut sha1_bytes = [0u8; 20];
sha1_bytes.copy_from_slice(&self.hasher.finish()?[..]);
Ok((db::RecordingToInsert{
camera_id: self.camera_id,
sample_file_bytes: self.index.sample_file_bytes,
time: self.start_time .. self.end(),
local_time: self.local_time,
video_samples: self.index.video_samples,
video_sync_samples: self.index.video_sync_samples,
video_sample_entry_id: self.video_sample_entry_id,
sample_file_uuid: self.uuid,
video_index: self.index.video_index,
sample_file_sha1: sha1_bytes,
}, self.f))
}
}
/// A segment represents a view of some or all of a single recording, starting from a key frame.
/// Used by the `Mp4FileBuilder` class to splice together recordings into a single virtual .mp4.
pub struct Segment {
pub id: i64,
pub start: Time,
begin: SampleIndexIterator,
pub file_end: i32,
pub desired_range_90k: Range<i32>,
actual_end_90k: i32,
pub frames: i32,
pub key_frames: i32,
pub video_sample_entry_id: i32,
}
impl Segment {
/// Creates a segment in a semi-initialized state. This is very light initialization because
/// it is called with the database lock held. `init` must be called before usage, and the
/// Segment should not be used if `init` fails.
///
/// `desired_range_90k` represents the desired range of the segment relative to the start of
/// the recording. The actual range will start at the first key frame at or before the
/// desired start time. (The caller is responsible for creating an edit list to skip the
/// undesired portion.) It will end at the first frame after the desired range (unless the
/// desired range extends beyond the recording).
pub fn new(recording: &db::ListCameraRecordingsRow,
desired_range_90k: Range<i32>) -> Segment {
Segment{
id: recording.id,
start: recording.start,
begin: SampleIndexIterator::new(),
file_end: recording.sample_file_bytes,
desired_range_90k: desired_range_90k,
actual_end_90k: recording.duration_90k,
frames: recording.video_samples,
key_frames: recording.video_sync_samples,
video_sample_entry_id: recording.video_sample_entry.id,
}
}
/// Completes initialization of the segment. Must be called without the database lock held;
/// this will use the database to retrieve the video index for partial recordings.
pub fn init(&mut self, db: &db::Database) -> Result<(), Error> {
if self.desired_range_90k.start > self.desired_range_90k.end ||
self.desired_range_90k.end > self.actual_end_90k {
return Err(Error::new(format!(
"desired range [{}, {}) invalid for recording of length {}",
self.desired_range_90k.start, self.desired_range_90k.end, self.actual_end_90k)));
}
if self.desired_range_90k.start == 0 &&
self.desired_range_90k.end == self.actual_end_90k {
// Fast path. Existing entry is fine.
return Ok(())
}
// Slow path. Need to iterate through the index.
let extra = db.lock().get_recording(self.id)?;
let data = &(&extra).video_index;
let mut it = SampleIndexIterator::new();
if !it.next(data)? {
return Err(Error{description: String::from("no index"),
cause: None});
}
if !it.is_key {
return Err(Error{description: String::from("not key frame"),
cause: None});
}
loop {
if it.start_90k <= self.desired_range_90k.start && it.is_key {
// new start candidate.
self.begin = it;
self.frames = 0;
self.key_frames = 0;
}
if it.start_90k >= self.desired_range_90k.end {
break;
}
self.frames += 1;
self.key_frames += it.is_key as i32;
if !it.next(data)? {
break;
}
}
self.file_end = it.pos;
self.actual_end_90k = it.start_90k;
Ok(())
}
/// Returns the byte range within the sample file of data associated with this segment.
pub fn sample_file_range(&self) -> Range<u64> {
Range{start: self.begin.pos as u64, end: self.file_end as u64}
}
/// Returns the actual time range as described in `new`.
pub fn actual_time_90k(&self) -> Range<i32> {
Range{start: self.begin.start_90k, end: self.actual_end_90k}
}
/// Iterates through each frame in the segment.
/// Must be called without the database lock held; retrieves video index from the cache.
pub fn foreach<F>(&self, db: &db::Database, mut f: F) -> Result<(), Error>
where F: FnMut(&SampleIndexIterator) -> Result<(), Error>
{
let extra = db.lock().get_recording(self.id)?;
let data = &(&extra).video_index;
let mut it = self.begin;
if it.i == 0 {
assert!(it.next(data)?);
assert!(it.is_key);
}
loop {
f(&it)?;
if !it.next(data)? {
return Ok(());
}
}
}
}
#[cfg(test)]
mod tests {
extern crate test;
use super::{append_varint32, decode_varint32, unzigzag32, zigzag32};
use super::*;
use self::test::Bencher;
#[test]
fn test_zigzag() {
struct Test {
decoded: i32,
encoded: u32,
}
let tests = [
Test{decoded: 0, encoded: 0},
Test{decoded: -1, encoded: 1},
Test{decoded: 1, encoded: 2},
Test{decoded: -2, encoded: 3},
Test{decoded: 2147483647, encoded: 4294967294},
Test{decoded: -2147483648, encoded: 4294967295},
];
for test in &tests {
assert_eq!(test.encoded, zigzag32(test.decoded));
assert_eq!(test.decoded, unzigzag32(test.encoded));
}
}
#[test]
fn test_correct_varints() {
struct Test {
decoded: u32,
encoded: &'static [u8],
}
let tests = [
Test{decoded: 1, encoded: b"\x01"},
Test{decoded: 257, encoded: b"\x81\x02"},
Test{decoded: 49409, encoded: b"\x81\x82\x03"},
Test{decoded: 8438017, encoded: b"\x81\x82\x83\x04"},
Test{decoded: 1350615297, encoded: b"\x81\x82\x83\x84\x05"},
];
for test in &tests {
// Test encoding to an empty buffer.
let mut out = Vec::new();
append_varint32(test.decoded, &mut out);
assert_eq!(&out[..], test.encoded);
// ...and to a non-empty buffer.
let mut buf = Vec::new();
out.clear();
out.push(b'x');
buf.push(b'x');
buf.extend_from_slice(test.encoded);
append_varint32(test.decoded, &mut out);
assert_eq!(out, buf);
// Test decoding from the beginning of the string.
assert_eq!((test.decoded, test.encoded.len()),
decode_varint32(test.encoded, 0).unwrap());
// ...and from the middle of a buffer.
buf.push(b'x');
assert_eq!((test.decoded, test.encoded.len() + 1),
decode_varint32(&buf, 1).unwrap());
}
}
#[test]
fn test_display_duration() {
let tests = &[
// (output, seconds)
("0 seconds", 0),
("1 second", 1),
("1 minute", 60),
("1 minute 1 second", 61),
("2 minutes", 120),
("1 hour", 3600),
("1 hour 1 minute", 3660),
("2 hours", 7200),
("1 day", 86400),
("1 day 1 hour", 86400 + 3600),
("2 days", 2 * 86400),
];
for test in tests {
assert_eq!(test.0, format!("{}", Duration(test.1 * TIME_UNITS_PER_SEC)));
}
}
#[test]
fn test_bad_varints() {
let tests: &[&[u8]] = &[
// buffer underruns
b"",
b"\x80",
b"\x80\x80",
b"\x80\x80\x80",
b"\x80\x80\x80\x80",
// int32 overflows
b"\x80\x80\x80\x80\x80",
b"\x80\x80\x80\x80\x80\x00",
];
for (i, encoded) in tests.iter().enumerate() {
assert!(decode_varint32(encoded, 0).is_err(), "while on test {}", i);
}
}
/// Tests the example from design/schema.md.
#[test]
fn test_encode_example() {
let mut e = SampleIndexEncoder::new();
e.add_sample(10, 1000, true);
e.add_sample(9, 10, false);
e.add_sample(11, 15, false);
e.add_sample(10, 12, false);
e.add_sample(10, 1050, true);
assert_eq!(e.video_index, b"\x29\xd0\x0f\x02\x14\x08\x0a\x02\x05\x01\x64");
assert_eq!(10 + 9 + 11 + 10 + 10, e.total_duration_90k);
assert_eq!(5, e.video_samples);
assert_eq!(2, e.video_sync_samples);
}
#[test]
fn test_round_trip() {
#[derive(Debug, PartialEq, Eq)]
struct Sample {
duration_90k: i32,
bytes: i32,
is_key: bool,
}
let samples = [
Sample{duration_90k: 10, bytes: 30000, is_key: true},
Sample{duration_90k: 9, bytes: 1000, is_key: false},
Sample{duration_90k: 11, bytes: 1100, is_key: false},
Sample{duration_90k: 18, bytes: 31000, is_key: true},
Sample{duration_90k: 0, bytes: 1000, is_key: false},
];
let mut e = SampleIndexEncoder::new();
for sample in &samples {
e.add_sample(sample.duration_90k, sample.bytes, sample.is_key);
}
let mut it = SampleIndexIterator::new();
for sample in &samples {
assert!(it.next(&e.video_index).unwrap());
assert_eq!(sample,
&Sample{duration_90k: it.duration_90k, bytes: it.bytes, is_key: it.is_key});
}
assert!(!it.next(&e.video_index).unwrap());
}
#[test]
fn test_iterator_errors() {
struct Test {
encoded: &'static [u8],
err: &'static str,
}
let tests = [
Test{encoded: b"\x80", err: "bad varint 1 at offset 0"},
Test{encoded: b"\x00\x80", err: "bad varint 2 at offset 1"},
Test{encoded: b"\x00\x02\x00\x00",
err: "zero duration only allowed at end; have 2 bytes left"},
Test{encoded: b"\x02\x02",
err: "negative duration -1 after applying delta -1"},
Test{encoded: b"\x04\x00",
err: "non-positive bytes 0 after applying delta 0 to key=false frame at ts 0"},
];
for test in &tests {
let mut it = SampleIndexIterator::new();
assert_eq!(it.next(test.encoded).unwrap_err().description, test.err);
}
}
/// Benchmarks the decoder, which is performance-critical for .mp4 serving.
#[bench]
fn bench_decoder(b: &mut Bencher) {
let data = include_bytes!("testdata/video_sample_index.bin");
b.bytes = data.len() as u64;
b.iter(|| {
let mut it = SampleIndexIterator::new();
while it.next(data).unwrap() {}
assert_eq!(30104460, it.pos);
assert_eq!(5399985, it.start_90k);
});
}
}

682
src/resource.rs Normal file
View File

@ -0,0 +1,682 @@
// This file is part of Moonfire NVR, a security camera digital video recorder.
// Copyright (C) 2016 Scott Lamb <slamb@slamb.org>
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// In addition, as a special exception, the copyright holders give
// permission to link the code of portions of this program with the
// OpenSSL library under certain conditions as described in each
// individual source file, and distribute linked combinations including
// the two.
//
// You must obey the GNU General Public License in all respects for all
// of the code used other than OpenSSL. If you modify file(s) with this
// exception, you may extend this exception to your version of the
// file(s), but you are not obligated to do so. If you do not wish to do
// so, delete this exception statement from your version. If you delete
// this exception statement from all source files in the program, then
// also delete it here.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
extern crate core;
extern crate hyper;
extern crate time;
use error::Result;
use hyper::server::{Request, Response};
use hyper::header;
use hyper::method::Method;
use hyper::net::Fresh;
use mime;
use smallvec::SmallVec;
use std::cmp;
use std::io;
use std::ops::Range;
/// An HTTP resource for GET and HEAD serving.
pub trait Resource {
/// Returns the length of the slice in bytes.
fn len(&self) -> u64;
/// Writes bytes within this slice indicated by `range` to `out.`
/// TODO: different result type?
fn write_to(&self, range: Range<u64>, out: &mut io::Write) -> Result<()>;
fn content_type(&self) -> mime::Mime;
fn etag(&self) -> Option<&header::EntityTag>;
fn last_modified(&self) -> &header::HttpDate;
}
#[derive(Debug, Eq, PartialEq)]
enum ResolvedRanges {
AbsentOrInvalid,
NotSatisfiable,
Satisfiable(SmallVec<[Range<u64>; 1]>)
}
fn parse_range_header(range: Option<&header::Range>, resource_len: u64) -> ResolvedRanges {
if let Some(&header::Range::Bytes(ref byte_ranges)) = range {
let mut ranges: SmallVec<[Range<u64>; 1]> = SmallVec::new();
for range in byte_ranges {
match *range {
header::ByteRangeSpec::FromTo(range_from, range_to) => {
let end = cmp::min(range_to + 1, resource_len);
if range_from >= end {
debug!("Range {:?} not satisfiable with length {:?}", range, resource_len);
continue;
}
ranges.push(Range{start: range_from, end: end});
},
header::ByteRangeSpec::AllFrom(range_from) => {
if range_from >= resource_len {
debug!("Range {:?} not satisfiable with length {:?}", range, resource_len);
continue;
}
ranges.push(Range{start: range_from, end: resource_len});
},
header::ByteRangeSpec::Last(last) => {
if last >= resource_len {
debug!("Range {:?} not satisfiable with length {:?}", range, resource_len);
continue;
}
ranges.push(Range{start: resource_len - last,
end: resource_len});
},
}
}
if !ranges.is_empty() {
debug!("Ranges {:?} all satisfiable with length {:?}", range, resource_len);
return ResolvedRanges::Satisfiable(ranges);
}
return ResolvedRanges::NotSatisfiable;
}
ResolvedRanges::AbsentOrInvalid
}
/// Returns true if `req` doesn't have an `If-None-Match` header matching `req`.
fn none_match(etag: Option<&header::EntityTag>, req: &Request) -> bool {
match req.headers.get::<header::IfNoneMatch>() {
Some(&header::IfNoneMatch::Any) => false,
Some(&header::IfNoneMatch::Items(ref items)) => {
if let Some(some_etag) = etag {
for item in items {
if item.weak_eq(some_etag) {
return false;
}
}
}
true
},
None => true,
}
}
/// Returns true if `req` has no `If-Match` header or one which matches `etag`.
fn any_match(etag: Option<&header::EntityTag>, req: &Request) -> bool {
match req.headers.get::<header::IfMatch>() {
Some(&header::IfMatch::Any) => true,
Some(&header::IfMatch::Items(ref items)) => {
if let Some(some_etag) = etag {
for item in items {
if item.strong_eq(some_etag) {
return true;
}
}
}
false
},
None => false,
}
}
/// Serves GET and HEAD requests for a given byte-ranged resource.
/// Handles conditional & subrange requests.
/// The caller is expected to have already determined the correct resource and appended
/// Expires, Cache-Control, and Vary headers.
///
/// TODO: is it appropriate to include those headers on all response codes used in this function?
///
/// TODO: check HTTP rules about weak vs strong comparisons with range requests. I don't think I'm
/// doing this correctly.
pub fn serve(rsrc: &Resource, req: &Request, mut res: Response<Fresh>) -> Result<()> {
if req.method != Method::Get && req.method != Method::Head {
*res.status_mut() = hyper::status::StatusCode::MethodNotAllowed;
res.headers_mut().set(header::ContentType(mime!(Text/Plain)));
res.headers_mut().set(header::Allow(vec![Method::Get, Method::Head]));
res.send(b"This resource only supports GET and HEAD.")?;
return Ok(());
}
let last_modified = rsrc.last_modified();
let etag = rsrc.etag();
res.headers_mut().set(header::AcceptRanges(vec![header::RangeUnit::Bytes]));
res.headers_mut().set(header::LastModified(*last_modified));
if let Some(some_etag) = etag {
res.headers_mut().set(header::ETag(some_etag.clone()));
}
if let Some(&header::IfUnmodifiedSince(ref since)) = req.headers.get() {
if last_modified.0.to_timespec() > since.0.to_timespec() {
*res.status_mut() = hyper::status::StatusCode::PreconditionFailed;
res.send(b"Precondition failed")?;
return Ok(());
}
}
if any_match(etag, req) {
*res.status_mut() = hyper::status::StatusCode::PreconditionFailed;
res.send(b"Precondition failed")?;
return Ok(());
}
if !none_match(etag, req) {
*res.status_mut() = hyper::status::StatusCode::NotModified;
res.send(b"")?;
return Ok(());
}
if let Some(&header::IfModifiedSince(ref since)) = req.headers.get() {
if last_modified <= since {
*res.status_mut() = hyper::status::StatusCode::NotModified;
res.send(b"")?;
return Ok(());
}
}
let mut range_hdr = req.headers.get::<header::Range>();
// See RFC 2616 section 10.2.7: a Partial Content response should include certain
// entity-headers or not based on the If-Range response.
let include_entity_headers_on_range = match req.headers.get::<header::IfRange>() {
Some(&header::IfRange::EntityTag(ref if_etag)) => {
if let Some(some_etag) = etag {
if if_etag.strong_eq(some_etag) {
false
} else {
range_hdr = None;
true
}
} else {
range_hdr = None;
true
}
},
Some(&header::IfRange::Date(ref if_date)) => {
// The to_timespec conversion appears necessary because in the If-Range off the wire,
// fields such as tm_yday are absent, causing strict equality to spuriously fail.
if if_date.0.to_timespec() != last_modified.0.to_timespec() {
range_hdr = None;
true
} else {
false
}
},
None => true,
};
let len = rsrc.len();
let (range, include_entity_headers) = match parse_range_header(range_hdr, len) {
ResolvedRanges::AbsentOrInvalid => (0 .. len, true),
ResolvedRanges::Satisfiable(rs) => {
if rs.len() == 1 {
res.headers_mut().set(header::ContentRange(
header::ContentRangeSpec::Bytes{
range: Some((rs[0].start, rs[0].end-1)),
instance_length: Some(len)}));
*res.status_mut() = hyper::status::StatusCode::PartialContent;
(rs[0].clone(), include_entity_headers_on_range)
} else {
// Ignore multi-part range headers for now. They require additional complexity, and
// I don't see clients sending them in the wild.
(0 .. len, true)
}
},
ResolvedRanges::NotSatisfiable => {
res.headers_mut().set(header::ContentRange(
header::ContentRangeSpec::Bytes{
range: None,
instance_length: Some(len)}));
*res.status_mut() = hyper::status::StatusCode::RangeNotSatisfiable;
res.send(b"")?;
return Ok(());
}
};
if include_entity_headers {
res.headers_mut().set(header::ContentType(rsrc.content_type()));
}
res.headers_mut().set(header::ContentLength(range.end - range.start));
let mut stream = res.start()?;
if req.method == Method::Get {
rsrc.write_to(range, &mut stream)?;
}
stream.end()?;
Ok(())
}
#[cfg(test)]
mod tests {
use error::Result;
use hyper;
use hyper::header::{self, ByteRangeSpec, ContentRangeSpec, EntityTag};
use hyper::header::Range::Bytes;
use mime;
use smallvec::SmallVec;
use std::io::{Read, Write};
use std::ops::Range;
use std::sync::Mutex;
use super::{ResolvedRanges, parse_range_header};
use super::*;
use testutil;
use time;
/// Tests the specific examples enumerated in RFC 2616 section 14.35.1.
#[test]
fn test_resolve_ranges_rfc() {
let mut v = SmallVec::new();
v.push(0 .. 500);
assert_eq!(ResolvedRanges::Satisfiable(v.clone()),
parse_range_header(Some(&Bytes(vec![ByteRangeSpec::FromTo(0, 499)])),
10000));
v.clear();
v.push(500 .. 1000);
assert_eq!(ResolvedRanges::Satisfiable(v.clone()),
parse_range_header(Some(&Bytes(vec![ByteRangeSpec::FromTo(500, 999)])),
10000));
v.clear();
v.push(9500 .. 10000);
assert_eq!(ResolvedRanges::Satisfiable(v.clone()),
parse_range_header(Some(&Bytes(vec![ByteRangeSpec::Last(500)])),
10000));
v.clear();
v.push(9500 .. 10000);
assert_eq!(ResolvedRanges::Satisfiable(v.clone()),
parse_range_header(Some(&Bytes(vec![ByteRangeSpec::AllFrom(9500)])),
10000));
v.clear();
v.push(0 .. 1);
v.push(9999 .. 10000);
assert_eq!(ResolvedRanges::Satisfiable(v.clone()),
parse_range_header(Some(&Bytes(vec![ByteRangeSpec::FromTo(0, 0),
ByteRangeSpec::Last(1)])),
10000));
// Non-canonical ranges. Possibly the point of these is that the adjacent and overlapping
// ranges are supposed to be coalesced into one? I'm not going to do that for now.
v.clear();
v.push(500 .. 601);
v.push(601 .. 1000);
assert_eq!(ResolvedRanges::Satisfiable(v.clone()),
parse_range_header(Some(&Bytes(vec![ByteRangeSpec::FromTo(500, 600),
ByteRangeSpec::FromTo(601, 999)])),
10000));
v.clear();
v.push(500 .. 701);
v.push(601 .. 1000);
assert_eq!(ResolvedRanges::Satisfiable(v.clone()),
parse_range_header(Some(&Bytes(vec![ByteRangeSpec::FromTo(500, 700),
ByteRangeSpec::FromTo(601, 999)])),
10000));
}
#[test]
fn test_resolve_ranges_satisfiability() {
assert_eq!(ResolvedRanges::NotSatisfiable,
parse_range_header(Some(&Bytes(vec![ByteRangeSpec::AllFrom(10000)])),
10000));
let mut v = SmallVec::new();
v.push(0 .. 500);
assert_eq!(ResolvedRanges::Satisfiable(v.clone()),
parse_range_header(Some(&Bytes(vec![ByteRangeSpec::FromTo(0, 499),
ByteRangeSpec::AllFrom(10000)])),
10000));
assert_eq!(ResolvedRanges::NotSatisfiable,
parse_range_header(Some(&Bytes(vec![ByteRangeSpec::Last(1)])), 0));
assert_eq!(ResolvedRanges::NotSatisfiable,
parse_range_header(Some(&Bytes(vec![ByteRangeSpec::FromTo(0, 0)])), 0));
assert_eq!(ResolvedRanges::NotSatisfiable,
parse_range_header(Some(&Bytes(vec![ByteRangeSpec::AllFrom(0)])), 0));
v.clear();
v.push(0 .. 1);
assert_eq!(ResolvedRanges::Satisfiable(v.clone()),
parse_range_header(Some(&Bytes(vec![ByteRangeSpec::FromTo(0, 0)])), 1));
v.clear();
v.push(0 .. 500);
assert_eq!(ResolvedRanges::Satisfiable(v.clone()),
parse_range_header(Some(&Bytes(vec![ByteRangeSpec::FromTo(0, 10000)])),
500));
}
#[test]
fn test_resolve_ranges_absent_or_invalid() {
assert_eq!(ResolvedRanges::AbsentOrInvalid, parse_range_header(None, 10000));
}
struct FakeResource {
etag: Option<EntityTag>,
mime: mime::Mime,
last_modified: header::HttpDate,
body: &'static [u8],
}
impl Resource for FakeResource {
fn len(&self) -> u64 { self.body.len() as u64 }
fn write_to(&self, range: Range<u64>, out: &mut Write) -> Result<()> {
Ok(out.write_all(&self.body[range.start as usize .. range.end as usize])?)
}
fn content_type(&self) -> mime::Mime { self.mime.clone() }
fn etag(&self) -> Option<&EntityTag> { self.etag.as_ref() }
fn last_modified(&self) -> &header::HttpDate { &self.last_modified }
}
fn new_server() -> String {
let mut listener = hyper::net::HttpListener::new("127.0.0.1:0").unwrap();
use hyper::net::NetworkListener;
let addr = listener.local_addr().unwrap();
let server = hyper::Server::new(listener);
use std::thread::spawn;
spawn(move || {
use hyper::server::{Request, Response, Fresh};
let _ = server.handle(move |req: Request, res: Response<Fresh>| {
let l = RESOURCE.lock().unwrap();
let resource = l.as_ref().unwrap();
serve(resource, &req, res).unwrap();
});
});
format!("http://{}:{}/", addr.ip(), addr.port())
}
lazy_static! {
static ref RESOURCE: Mutex<Option<FakeResource>> = { Mutex::new(None) };
static ref SERVER: String = { new_server() };
static ref SOME_DATE: header::HttpDate = {
header::HttpDate(time::at_utc(time::Timespec::new(1430006400i64, 0)))
};
static ref LATER_DATE: header::HttpDate = {
header::HttpDate(time::at_utc(time::Timespec::new(1430092800i64, 0)))
};
}
#[test]
fn serve_without_etag() {
testutil::init();
*RESOURCE.lock().unwrap() = Some(FakeResource{
etag: None,
mime: mime!(Application/OctetStream),
last_modified: *SOME_DATE,
body: b"01234",
});
let client = hyper::Client::new();
let mut buf = Vec::new();
// Full body.
let mut resp = client.get(&*SERVER).send().unwrap();
assert_eq!(hyper::status::StatusCode::Ok, resp.status);
assert_eq!(Some(&header::ContentType(mime!(Application/OctetStream))),
resp.headers.get::<header::ContentType>());
assert_eq!(None, resp.headers.get::<header::ContentRange>());
buf.clear();
resp.read_to_end(&mut buf).unwrap();
assert_eq!(b"01234", &buf[..]);
// If-None-Match any.
let mut resp = client.get(&*SERVER)
.header(header::IfNoneMatch::Any)
.send()
.unwrap();
assert_eq!(hyper::status::StatusCode::NotModified, resp.status);
assert_eq!(None, resp.headers.get::<header::ContentRange>());
buf.clear();
resp.read_to_end(&mut buf).unwrap();
assert_eq!(b"", &buf[..]);
// If-None-Match by etag doesn't match (as this request has no etag).
let mut resp =
client.get(&*SERVER)
.header(header::IfNoneMatch::Items(vec![EntityTag::strong("foo".to_owned())]))
.send()
.unwrap();
assert_eq!(hyper::status::StatusCode::Ok, resp.status);
assert_eq!(Some(&header::ContentType(mime!(Application/OctetStream))),
resp.headers.get::<header::ContentType>());
assert_eq!(None, resp.headers.get::<header::ContentRange>());
buf.clear();
resp.read_to_end(&mut buf).unwrap();
assert_eq!(b"01234", &buf[..]);
// Unmodified since supplied date.
let mut resp = client.get(&*SERVER)
.header(header::IfModifiedSince(*SOME_DATE))
.send()
.unwrap();
assert_eq!(hyper::status::StatusCode::NotModified, resp.status);
assert_eq!(None, resp.headers.get::<header::ContentRange>());
buf.clear();
resp.read_to_end(&mut buf).unwrap();
assert_eq!(b"", &buf[..]);
// Range serving - basic case.
let mut resp = client.get(&*SERVER)
.header(Bytes(vec![ByteRangeSpec::FromTo(1, 3)]))
.send()
.unwrap();
assert_eq!(hyper::status::StatusCode::PartialContent, resp.status);
assert_eq!(Some(&header::ContentRange(ContentRangeSpec::Bytes{
range: Some((1, 3)),
instance_length: Some(5),
})), resp.headers.get());
buf.clear();
resp.read_to_end(&mut buf).unwrap();
assert_eq!(b"123", &buf[..]);
// Range serving - multiple ranges. Currently falls back to whole range.
let mut resp = client.get(&*SERVER)
.header(Bytes(vec![ByteRangeSpec::FromTo(0, 1),
ByteRangeSpec::FromTo(3, 4)]))
.send()
.unwrap();
assert_eq!(None, resp.headers.get::<header::ContentRange>());
assert_eq!(hyper::status::StatusCode::Ok, resp.status);
buf.clear();
resp.read_to_end(&mut buf).unwrap();
assert_eq!(b"01234", &buf[..]);
// Range serving - not satisfiable.
let mut resp = client.get(&*SERVER)
.header(Bytes(vec![ByteRangeSpec::AllFrom(500)]))
.send()
.unwrap();
assert_eq!(hyper::status::StatusCode::RangeNotSatisfiable, resp.status);
assert_eq!(Some(&header::ContentRange(ContentRangeSpec::Bytes{
range: None,
instance_length: Some(5),
})), resp.headers.get());
buf.clear();
resp.read_to_end(&mut buf).unwrap();
assert_eq!(b"", &buf[..]);
// Range serving - matching If-Range by date honors the range.
let mut resp = client.get(&*SERVER)
.header(Bytes(vec![ByteRangeSpec::FromTo(1, 3)]))
.header(header::IfRange::Date(*SOME_DATE))
.send()
.unwrap();
assert_eq!(hyper::status::StatusCode::PartialContent, resp.status);
assert_eq!(Some(&header::ContentRange(ContentRangeSpec::Bytes{
range: Some((1, 3)),
instance_length: Some(5),
})), resp.headers.get());
buf.clear();
resp.read_to_end(&mut buf).unwrap();
assert_eq!(b"123", &buf[..]);
// Range serving - non-matching If-Range by date ignores the range.
let mut resp = client.get(&*SERVER)
.header(Bytes(vec![ByteRangeSpec::FromTo(1, 3)]))
.header(header::IfRange::Date(*LATER_DATE))
.send()
.unwrap();
assert_eq!(hyper::status::StatusCode::Ok, resp.status);
assert_eq!(Some(&header::ContentType(mime!(Application/OctetStream))),
resp.headers.get::<header::ContentType>());
assert_eq!(None, resp.headers.get::<header::ContentRange>());
buf.clear();
resp.read_to_end(&mut buf).unwrap();
assert_eq!(b"01234", &buf[..]);
// Range serving - this resource has no etag, so any If-Range by etag ignores the range.
let mut resp =
client.get(&*SERVER)
.header(Bytes(vec![ByteRangeSpec::FromTo(1, 3)]))
.header(header::IfRange::EntityTag(EntityTag::strong("foo".to_owned())))
.send()
.unwrap();
assert_eq!(hyper::status::StatusCode::Ok, resp.status);
assert_eq!(Some(&header::ContentType(mime!(Application/OctetStream))),
resp.headers.get::<header::ContentType>());
assert_eq!(None, resp.headers.get::<header::ContentRange>());
buf.clear();
resp.read_to_end(&mut buf).unwrap();
assert_eq!(b"01234", &buf[..]);
}
#[test]
fn serve_with_strong_etag() {
testutil::init();
*RESOURCE.lock().unwrap() = Some(FakeResource{
etag: Some(EntityTag::strong("foo".to_owned())),
mime: mime!(Application/OctetStream),
last_modified: *SOME_DATE,
body: b"01234",
});
let client = hyper::Client::new();
let mut buf = Vec::new();
// If-None-Match by etag which matches.
let mut resp =
client.get(&*SERVER)
.header(header::IfNoneMatch::Items(vec![EntityTag::strong("foo".to_owned())]))
.send()
.unwrap();
assert_eq!(hyper::status::StatusCode::NotModified, resp.status);
assert_eq!(None, resp.headers.get::<header::ContentRange>());
buf.clear();
resp.read_to_end(&mut buf).unwrap();
assert_eq!(b"", &buf[..]);
// If-None-Match by etag which doesn't match.
let mut resp =
client.get(&*SERVER)
.header(header::IfNoneMatch::Items(vec![EntityTag::strong("bar".to_owned())]))
.send()
.unwrap();
assert_eq!(hyper::status::StatusCode::Ok, resp.status);
assert_eq!(None, resp.headers.get::<header::ContentRange>());
buf.clear();
resp.read_to_end(&mut buf).unwrap();
assert_eq!(b"01234", &buf[..]);
// Range serving - If-Range matching by etag.
let mut resp =
client.get(&*SERVER)
.header(Bytes(vec![ByteRangeSpec::FromTo(1, 3)]))
.header(header::IfRange::EntityTag(EntityTag::strong("foo".to_owned())))
.send()
.unwrap();
assert_eq!(hyper::status::StatusCode::PartialContent, resp.status);
assert_eq!(None, resp.headers.get::<header::ContentType>());
assert_eq!(Some(&header::ContentRange(ContentRangeSpec::Bytes{
range: Some((1, 3)),
instance_length: Some(5),
})), resp.headers.get());
buf.clear();
resp.read_to_end(&mut buf).unwrap();
assert_eq!(b"123", &buf[..]);
// Range serving - If-Range not matching by etag.
let mut resp =
client.get(&*SERVER)
.header(Bytes(vec![ByteRangeSpec::FromTo(1, 3)]))
.header(header::IfRange::EntityTag(EntityTag::strong("bar".to_owned())))
.send()
.unwrap();
assert_eq!(hyper::status::StatusCode::Ok, resp.status);
assert_eq!(Some(&header::ContentType(mime!(Application/OctetStream))),
resp.headers.get::<header::ContentType>());
assert_eq!(None, resp.headers.get::<header::ContentRange>());
buf.clear();
resp.read_to_end(&mut buf).unwrap();
assert_eq!(b"01234", &buf[..]);
}
#[test]
fn serve_with_weak_etag() {
testutil::init();
*RESOURCE.lock().unwrap() = Some(FakeResource{
etag: Some(EntityTag::weak("foo".to_owned())),
mime: mime!(Application/OctetStream),
last_modified: *SOME_DATE,
body: b"01234",
});
let client = hyper::Client::new();
let mut buf = Vec::new();
// If-None-Match by identical weak etag is sufficient.
let mut resp =
client.get(&*SERVER)
.header(header::IfNoneMatch::Items(vec![EntityTag::weak("foo".to_owned())]))
.send()
.unwrap();
assert_eq!(hyper::status::StatusCode::NotModified, resp.status);
assert_eq!(None, resp.headers.get::<header::ContentRange>());
buf.clear();
resp.read_to_end(&mut buf).unwrap();
assert_eq!(b"", &buf[..]);
// If-None-Match by etag which doesn't match.
let mut resp =
client.get(&*SERVER)
.header(header::IfNoneMatch::Items(vec![EntityTag::weak("bar".to_owned())]))
.send()
.unwrap();
assert_eq!(hyper::status::StatusCode::Ok, resp.status);
assert_eq!(None, resp.headers.get::<header::ContentRange>());
buf.clear();
resp.read_to_end(&mut buf).unwrap();
assert_eq!(b"01234", &buf[..]);
// Range serving - If-Range matching by weak etag isn't sufficient.
let mut resp =
client.get(&*SERVER)
.header(Bytes(vec![ByteRangeSpec::FromTo(1, 3)]))
.header(header::IfRange::EntityTag(EntityTag::weak("foo".to_owned())))
.send()
.unwrap();
assert_eq!(hyper::status::StatusCode::Ok, resp.status);
assert_eq!(Some(&header::ContentType(mime!(Application/OctetStream))),
resp.headers.get::<header::ContentType>());
assert_eq!(None, resp.headers.get::<header::ContentRange>());
buf.clear();
resp.read_to_end(&mut buf).unwrap();
assert_eq!(b"01234", &buf[..]);
}
}

1
src/schema.sql
View File

@ -110,6 +110,7 @@ create index recording_cover on recording (
-- to consult the underlying row.
duration_90k,
video_samples,
video_sync_samples,
video_sample_entry_id,
sample_file_bytes
);

111
src/sqlite-test.cc
View File

@ -1,111 +0,0 @@
// This file is part of Moonfire NVR, a security camera network video recorder.
// Copyright (C) 2016 Scott Lamb <slamb@slamb.org>
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// In addition, as a special exception, the copyright holders give
// permission to link the code of portions of this program with the
// OpenSSL library under certain conditions as described in each
// individual source file, and distribute linked combinations including
// the two.
//
// You must obey the GNU General Public License in all respects for all
// of the code used other than OpenSSL. If you modify file(s) with this
// exception, you may extend this exception to your version of the
// file(s), but you are not obligated to do so. If you do not wish to do
// so, delete this exception statement from your version. If you delete
// this exception statement from all source files in the program, then
// also delete it here.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
//
// sqlite-test.cc: tests of the sqlite.h interface.
#include <string>
#include <gflags/gflags.h>
#include <glog/logging.h>
#include <gmock/gmock.h>
#include <gtest/gtest.h>
#include "sqlite.h"
#include "string.h"
#include "testutil.h"
DECLARE_bool(alsologtostderr);
namespace moonfire_nvr {
namespace {
class SqliteTest : public testing::Test {
protected:
SqliteTest() {
tmpdir_ = PrepareTempDirOrDie("sqlite-test");
std::string error_message;
CHECK(db_.Open(StrCat(tmpdir_, "/db").c_str(),
SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE, &error_message))
<< error_message;
std::string create_sql = ReadFileOrDie("../src/schema.sql");
DatabaseContext ctx(&db_);
CHECK(RunStatements(&ctx, create_sql, &error_message)) << error_message;
}
std::string tmpdir_;
Database db_;
};
TEST_F(SqliteTest, JustCreate) {}
TEST_F(SqliteTest, BindAndColumn) {
std::string error_message;
auto insert_stmt = db_.Prepare(
"insert into camera (uuid, short_name, retain_bytes) "
" values (:uuid, :short_name, :retain_bytes)",
nullptr, &error_message);
ASSERT_TRUE(insert_stmt.valid()) << error_message;
const char kBlob[] = {0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07,
0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f};
re2::StringPiece blob_piece = re2::StringPiece(kBlob, sizeof(kBlob));
const char kText[] = "foo";
const int64_t kInt64 = INT64_C(0xbeeffeedface);
DatabaseContext ctx(&db_);
{
auto run = ctx.Borrow(&insert_stmt);
run.BindBlob(1, blob_piece);
run.BindText(2, kText);
run.BindInt64(3, kInt64);
ASSERT_EQ(SQLITE_DONE, run.Step()) << run.error_message();
}
{
auto run = ctx.UseOnce("select uuid, short_name, retain_bytes from camera");
ASSERT_EQ(SQLITE_ROW, run.Step()) << run.error_message();
EXPECT_EQ(ToHex(blob_piece, true), ToHex(run.ColumnBlob(0), true));
EXPECT_EQ(kText, run.ColumnText(1).as_string());
EXPECT_EQ(kInt64, run.ColumnInt64(2));
ASSERT_EQ(SQLITE_DONE, run.Step()) << run.error_message();
}
}
} // namespace
} // namespace moonfire_nvr
int main(int argc, char **argv) {
FLAGS_alsologtostderr = true;
google::ParseCommandLineFlags(&argc, &argv, true);
testing::InitGoogleTest(&argc, argv);
google::InitGoogleLogging(argv[0]);
return RUN_ALL_TESTS();
}

408
src/sqlite.cc
View File

@ -1,408 +0,0 @@
// This file is part of Moonfire NVR, a security camera digital video recorder.
// Copyright (C) 2016 Scott Lamb <slamb@slamb.org>
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// In addition, as a special exception, the copyright holders give
// permission to link the code of portions of this program with the
// OpenSSL library under certain conditions as described in each
// individual source file, and distribute linked combinations including
// the two.
//
// You must obey the GNU General Public License in all respects for all
// of the code used other than OpenSSL. If you modify file(s) with this
// exception, you may extend this exception to your version of the
// file(s), but you are not obligated to do so. If you do not wish to do
// so, delete this exception statement from your version. If you delete
// this exception statement from all source files in the program, then
// also delete it here.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
//
// sqlite.cc: implementation of the sqlite.h interface.
#include "sqlite.h"
#include <mutex>
#include <glog/logging.h>
#include "string.h"
namespace moonfire_nvr {
namespace {
void LogCallback(void *, int err_code, const char *msg) {
LOG(ERROR) << "(" << err_code << ") " << msg;
}
void GlobalSetup() {
VLOG(1) << "Installing sqlite3 log callback";
sqlite3_config(SQLITE_CONFIG_LOG, &LogCallback, nullptr);
}
std::once_flag global_setup;
} // namespace
Statement::Statement(Statement &&other) { *this = std::move(other); }
void Statement::operator=(Statement &&other) {
Clear();
memcpy(this, &other, sizeof(Statement));
other.me_ = nullptr;
other.borrowed_ = false;
}
Statement::~Statement() { Clear(); }
void Statement::Clear() {
CHECK(!borrowed_) << "can't delete statement while still borrowed!";
sqlite3_finalize(me_);
}
DatabaseContext::DatabaseContext(Database *db) : db_(db), lock_(db->ctx_mu_) {}
DatabaseContext::~DatabaseContext() {
if (transaction_open_) {
LOG(WARNING) << this << ": transaction left open! closing in destructor.";
RollbackTransaction();
}
}
bool DatabaseContext::BeginTransaction(std::string *error_message) {
if (transaction_open_) {
*error_message = "transaction already open!";
return false;
}
sqlite3_step(db_->begin_transaction_.me_);
int ret = sqlite3_reset(db_->begin_transaction_.me_);
if (ret != SQLITE_OK) {
*error_message =
StrCat("begin transaction: ", sqlite3_errstr(ret), " (", ret, ")");
return false;
}
transaction_open_ = true;
return true;
}
bool DatabaseContext::CommitTransaction(std::string *error_message) {
if (!transaction_open_) {
*error_message = "transaction not open!";
return false;
}
sqlite3_step(db_->commit_transaction_.me_);
int ret = sqlite3_reset(db_->commit_transaction_.me_);
if (ret != SQLITE_OK) {
*error_message =
StrCat("commit transaction: ", sqlite3_errstr(ret), " (", ret, ")");
return false;
}
transaction_open_ = false;
return true;
}
void DatabaseContext::RollbackTransaction() {
if (!transaction_open_) {
LOG(WARNING) << this << ": rollback failed: transaction not open!";
return;
}
sqlite3_step(db_->rollback_transaction_.me_);
int ret = sqlite3_reset(db_->rollback_transaction_.me_);
if (ret != SQLITE_OK) {
LOG(WARNING) << this << ": rollback failed: " << sqlite3_errstr(ret) << " ("
<< ret << ")";
return;
}
transaction_open_ = false;
}
RunningStatement DatabaseContext::Borrow(Statement *statement) {
return RunningStatement(statement, std::string(), false);
}
RunningStatement DatabaseContext::UseOnce(re2::StringPiece sql) {
std::string error_message;
auto *statement = new Statement(db_->Prepare(sql, nullptr, &error_message));
return RunningStatement(statement, error_message, true);
}
RunningStatement::RunningStatement(Statement *statement,
const std::string &deferred_error,
bool owns_statement)
: error_message_(deferred_error), owns_statement_(owns_statement) {
if (statement != nullptr && statement->valid()) {
CHECK(!statement->borrowed_) << "Statement already borrowed!";
statement->borrowed_ = true;
statement_ = statement;
} else if (error_message_.empty()) {
error_message_ = "invalid statement";
}
if (!error_message_.empty()) {
status_ = SQLITE_MISUSE;
}
}
RunningStatement::RunningStatement(RunningStatement &&o) {
statement_ = o.statement_;
status_ = o.status_;
owns_statement_ = o.owns_statement_;
o.statement_ = nullptr;
}
RunningStatement::~RunningStatement() {
if (statement_ != nullptr) {
CHECK(statement_->borrowed_) << "Statement no longer borrowed!";
sqlite3_clear_bindings(statement_->me_);
sqlite3_reset(statement_->me_);
statement_->borrowed_ = false;
if (owns_statement_) {
delete statement_;
}
}
}
void RunningStatement::BindBlob(int param, re2::StringPiece value) {
if (status_ != SQLITE_OK) {
return;
}
status_ = sqlite3_bind_blob64(statement_->me_, param, value.data(),
value.size(), SQLITE_TRANSIENT);
if (status_ != SQLITE_OK) {
error_message_ = StrCat("Unable to bind parameter ", param, ": ",
sqlite3_errstr(status_), " (", status_, ")");
}
}
void RunningStatement::BindBlob(const char *name, re2::StringPiece value) {
if (status_ != SQLITE_OK) {
return;
}
int param = sqlite3_bind_parameter_index(statement_->me_, name);
if (param == 0) {
status_ = SQLITE_MISUSE;
error_message_ = StrCat("Unable to bind parameter ", name, ": not found.");
return;
}
status_ = sqlite3_bind_blob64(statement_->me_, param, value.data(),
value.size(), SQLITE_TRANSIENT);
if (status_ != SQLITE_OK) {
error_message_ = StrCat("Unable to bind parameter ", name, ": ",
sqlite3_errstr(status_), " (", status_, ")");
}
}
void RunningStatement::BindInt64(int param, int64_t value) {
if (status_ != SQLITE_OK) {
return;
}
status_ = sqlite3_bind_int64(statement_->me_, param, value);
if (status_ != SQLITE_OK) {
error_message_ = StrCat("Unable to bind parameter ", param, ": ",
sqlite3_errstr(status_), " (", status_, ")");
}
}
void RunningStatement::BindInt64(const char *name, int64_t value) {
if (status_ != SQLITE_OK) {
return;
}
int param = sqlite3_bind_parameter_index(statement_->me_, name);
if (param == 0) {
status_ = SQLITE_MISUSE;
error_message_ = StrCat("Unable to bind parameter ", name, ": not found.");
return;
}
status_ = sqlite3_bind_int64(statement_->me_, param, value);
if (status_ != SQLITE_OK) {
error_message_ = StrCat("Unable to bind parameter ", name, ": ",
sqlite3_errstr(status_), " (", status_, ")");
}
}
void RunningStatement::BindText(int param, re2::StringPiece value) {
if (status_ != SQLITE_OK) {
return;
}
status_ = sqlite3_bind_text64(statement_->me_, param, value.data(),
value.size(), SQLITE_TRANSIENT, SQLITE_UTF8);
if (status_ != SQLITE_OK) {
error_message_ = StrCat("Unable to bind parameter ", param, ": ",
sqlite3_errstr(status_), " (", status_, ")");
}
}
void RunningStatement::BindText(const char *name, re2::StringPiece value) {
if (status_ != SQLITE_OK) {
return;
}
int param = sqlite3_bind_parameter_index(statement_->me_, name);
if (param == 0) {
error_message_ = StrCat("Unable to bind parameter ", name, ": not found.");
return;
}
status_ = sqlite3_bind_text64(statement_->me_, param, value.data(),
value.size(), SQLITE_TRANSIENT, SQLITE_UTF8);
if (status_ != SQLITE_OK) {
error_message_ = StrCat("Unable to bind parameter ", name, ": ",
sqlite3_errstr(status_), " (", status_, ")");
}
}
int RunningStatement::Step() {
if (status_ != SQLITE_OK && status_ != SQLITE_ROW) {
return status_;
}
status_ = sqlite3_step(statement_->me_);
error_message_ =
StrCat("step: ", sqlite3_errstr(status_), " (", status_, ")");
return status_;
}
int RunningStatement::ColumnType(int col) {
return sqlite3_column_type(statement_->me_, col);
}
re2::StringPiece RunningStatement::ColumnBlob(int col) {
// Order matters: call _blob first, then _bytes.
const void *data = sqlite3_column_blob(statement_->me_, col);
size_t len = sqlite3_column_bytes(statement_->me_, col);
return re2::StringPiece(reinterpret_cast<const char *>(data), len);
}
int64_t RunningStatement::ColumnInt64(int col) {
return sqlite3_column_int64(statement_->me_, col);
}
re2::StringPiece RunningStatement::ColumnText(int col) {
// Order matters: call _text first, then _bytes.
const unsigned char *data = sqlite3_column_text(statement_->me_, col);
size_t len = sqlite3_column_bytes(statement_->me_, col);
return re2::StringPiece(reinterpret_cast<const char *>(data), len);
}
Database::~Database() {
begin_transaction_ = Statement();
commit_transaction_ = Statement();
rollback_transaction_ = Statement();
int err = sqlite3_close(me_);
CHECK_EQ(SQLITE_OK, err) << "sqlite3_close: " << sqlite3_errstr(err);
}
bool Database::Open(const char *filename, int flags,
std::string *error_message) {
std::call_once(global_setup, &GlobalSetup);
int ret = sqlite3_open_v2(filename, &me_, flags, nullptr);
if (ret != SQLITE_OK) {
*error_message =
StrCat("open ", filename, ": ", sqlite3_errstr(ret), " (", ret, ")");
return false;
}
ret = sqlite3_extended_result_codes(me_, 1);
if (ret != SQLITE_OK) {
sqlite3_close(me_);
me_ = nullptr;
*error_message = StrCat("while enabling extended result codes: ",
sqlite3_errstr(ret), " (", ret, ")");
return false;
}
Statement pragma_foreignkeys;
struct StatementToInitialize {
Statement *p;
re2::StringPiece sql;
};
StatementToInitialize stmts[] = {
{&begin_transaction_, "begin transaction;"},
{&commit_transaction_, "commit transaction;"},
{&rollback_transaction_, "rollback transaction;"},
{&pragma_foreignkeys, "pragma foreign_keys = true;"}};
for (const auto &stmt : stmts) {
*stmt.p = Prepare(stmt.sql, nullptr, error_message);
if (!stmt.p->valid()) {
sqlite3_close(me_);
me_ = nullptr;
*error_message = StrCat("while preparing SQL for \"", stmt.sql, "\": ",
*error_message);
return false;
}
}
ret = sqlite3_step(pragma_foreignkeys.me_);
sqlite3_reset(pragma_foreignkeys.me_);
if (ret != SQLITE_DONE) {
sqlite3_close(me_);
me_ = nullptr;
*error_message = StrCat("while enabling foreign keys: ",
sqlite3_errstr(ret), " (", ret, ")");
return false;
}
return true;
}
Statement Database::Prepare(re2::StringPiece sql, size_t *used,
std::string *error_message) {
Statement statement;
const char *tail;
int err =
sqlite3_prepare_v2(me_, sql.data(), sql.size(), &statement.me_, &tail);
if (err != SQLITE_OK) {
*error_message = StrCat("prepare: ", sqlite3_errstr(err), " (", err, ")");
return statement;
}
if (used != nullptr) {
*used = tail - sql.data();
}
if (statement.me_ == nullptr) {
error_message->clear();
}
return statement;
}
bool RunStatements(DatabaseContext *ctx, re2::StringPiece stmts,
std::string *error_message) {
while (true) {
size_t used = 0;
auto stmt = ctx->db()->Prepare(stmts, &used, error_message);
if (!stmt.valid()) {
// Statement didn't parse. If |error_message| is empty, there are just no
// more statements. Otherwise this is due to an error. Either way, return.
return error_message->empty();
}
VLOG(1) << "Running statement:\n" << stmts.substr(0, used).as_string();
int64_t rows = 0;
auto run = ctx->Borrow(&stmt);
while (run.Step() == SQLITE_ROW) {
++rows;
}
if (rows > 0) {
VLOG(1) << "Statement returned " << rows << " row(s).";
}
if (run.status() != SQLITE_DONE) {
VLOG(1) << "Statement failed with " << run.status() << ": "
<< run.error_message();
*error_message =
StrCat("Unexpected error ", run.error_message(),
" from statement: \"", stmts.substr(0, used), "\"");
return false;
}
VLOG(1) << "Statement succeeded.";
stmts.remove_prefix(used);
}
}
} // namespace moonfire_nvr

276
src/sqlite.h
View File

@ -1,276 +0,0 @@
// This file is part of Moonfire NVR, a security camera digital video recorder.
// Copyright (C) 2016 Scott Lamb <slamb@slamb.org>
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// In addition, as a special exception, the copyright holders give
// permission to link the code of portions of this program with the
// OpenSSL library under certain conditions as described in each
// individual source file, and distribute linked combinations including
// the two.
//
// You must obey the GNU General Public License in all respects for all
// of the code used other than OpenSSL. If you modify file(s) with this
// exception, you may extend this exception to your version of the
// file(s), but you are not obligated to do so. If you do not wish to do
// so, delete this exception statement from your version. If you delete
// this exception statement from all source files in the program, then
// also delete it here.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
//
// sqlite.h: a quick C++ wrapper interface around the SQLite3 C API.
// This provides RAII and takes advantage of some types like re2::StringPiece.
// It makes no attempt to hide how the underlying API works, so read
// alongside: <https://www.sqlite.org/capi3ref.html>
#ifndef MOONFIRE_NVR_SQLITE_H
#define MOONFIRE_NVR_SQLITE_H
#include <functional>
#include <mutex>
#include <string>
#include <glog/logging.h>
#include <re2/stringpiece.h>
#include <sqlite3.h>
#include "common.h"
namespace moonfire_nvr {
// Prepared statement. Movable, not copyable.
// The caller can obtain a Statement via Database::Prepare
// and use one via DatabaseContext::Borrow.
class Statement {
public:
Statement() {}
Statement(Statement &&);
void operator=(Statement &&);
~Statement();
bool valid() const { return me_ != nullptr; }
private:
friend class Database;
friend class DatabaseContext;
friend class RunningStatement;
Statement(const Statement &) = delete;
void operator=(const Statement &) = delete;
void Clear();
sqlite3_stmt *me_ = nullptr; // owned.
bool borrowed_ = false;
};
class Database {
public:
Database() {}
Database(const Database &) = delete;
Database &operator=(const Database &) = delete;
// PRE: all DatabaseContext and Statement objects have been deleted.
~Database();
// Open the database and do initial setup.
//
// Foreign keys will always be enabled via "pragma foreign_keys = true;".
//
// extended result codes will always be enabled via
// sqlite3_extended_result_codes.
bool Open(const char *filename, int flags, std::string *error_message);
// Prepare a statement. Thread-safe.
//
// |used|, if non-null, will be updated with the number of bytes used from
// |sql| on success. (Only the first statement is parsed.)
//
// Returns a statement, which may or may not be valid().
// |error_message| will be empty if there is simply no statement to parse.
Statement Prepare(re2::StringPiece sql, size_t *used,
std::string *error_message);
private:
friend class DatabaseContext;
friend class RunningStatement;
sqlite3 *me_ = nullptr;
Statement begin_transaction_;
Statement commit_transaction_;
Statement rollback_transaction_;
std::mutex ctx_mu_; // used by DatabaseContext.
};
// A running statement; get via DatabaseContext::Borrow or
// DatabaseContext::UseOnce. Example uses:
//
// {
// DatabaseContext ctx(&db);
// auto run = ctx.UseOnce("insert into table (column) values (:column)");
// run.BindText(":column", "value");
// if (run.Step() != SQLITE_DONE) {
// LOG(ERROR) << "Operation failed: " << run.error_message();
// return;
// }
// int64_t rowid = ctx.last_insert_rowid();
// }
//
// Statement select_stmt = db.Prepare(
// "select rowid from table;", nullptr, &error_message);
// ...
// {
// auto run = ctx.Borrow(&select_stmt);
// while (run.Step() == SQLITE_ROW) {
// int64_t rowid = op.ColumnInt64(0);
// ...
// }
// if (run.status() != SQLITE_DONE) {
// LOG(ERROR) << "Operation failed: " << run.error_message();
// }
// }
class RunningStatement {
public:
RunningStatement(RunningStatement &&o);
// Reset/unbind/return the statement for the next use (in the case of
// Borrow) or delete it (in the case of UseOnce).
~RunningStatement();
// Bind a value to a parameter. Call before the first Step.
// |param| is indexed from 1 (unlike columns!).
//
// StringPiece |value|s will be copied; they do not need to outlive the
// Bind{Blob,Text} call. Text values are assumed to be UTF-8.
//
// Errors are deferred until Step() for simplicity of caller code.
void BindBlob(int param, re2::StringPiece value);
void BindBlob(const char *param, re2::StringPiece value);
void BindText(int param, re2::StringPiece value);
void BindText(const char *param, re2::StringPiece value);
void BindInt64(int param, int64_t value);
void BindInt64(const char *param, int64_t value);
// Advance the statement, returning SQLITE_ROW, SQLITE_DONE, or an error.
// Note that this may return a "deferred error" if UseOnce failed to parse
// the SQL or if a bind failed.
int Step();
// Convenience function; re-return the last status from Step().
int status() { return status_; }
// Return a stringified version of the last status.
// This may have more information than sqlite3_errstr(status()),
// in the case of "deferred errors".
std::string error_message() { return error_message_; }
// Column accessors, to be called after Step() returns SQLITE_ROW.
// Columns are indexed from 0 (unlike bind parameters!).
// StringPiece values are valid only until a type conversion, the following
// NextRow() call, or destruction of the RunningStatement, whichever
// happens first.
//
// Note there is no useful way to report error here. In particular, the
// underlying SQLite functions return a default value on error, which can't
// be distinguished from a legitimate value. The error code is set on the
// database, but it's not guaranteed to *not* be set if there's no error.
// Return the type of a given column; if SQLITE_NULL, the value is null.
// As noted in sqlite3_column_type() documentation, this is only meaningful
// if other Column* calls have not forced a type conversion.
int ColumnType(int col);
re2::StringPiece ColumnBlob(int col);
int64_t ColumnInt64(int col);
re2::StringPiece ColumnText(int col);
private:
friend class DatabaseContext;
RunningStatement(Statement *stmt, const std::string &deferred_error,
bool own_statement);
RunningStatement(const RunningStatement &) = delete;
void operator=(const RunningStatement &) = delete;
Statement *statement_ = nullptr; // maybe owned; see owns_statement_.
std::string error_message_;
int status_ = SQLITE_OK;
bool owns_statement_ = false;
};
// A scoped database lock and transaction manager.
//
// Moonfire NVR does all SQLite operations under a lock, to avoid SQLITE_BUSY
// and so that calls such as sqlite3_last_insert_rowid return useful values.
// This class implicitly acquires the lock on entry / releases it on exit.
// In the future, it may have instrumentation to track slow operations.
class DatabaseContext {
public:
// Acquire a lock on |db|, which must already be opened.
explicit DatabaseContext(Database *db);
DatabaseContext(const DatabaseContext &) = delete;
void operator=(const DatabaseContext &) = delete;
// Release the lock and, if an explicit transaction is active, roll it
// back with a logged warning.
~DatabaseContext();
// Begin a transaction, or return false and fill |error_message|.
// If successful, the caller should explicitly call CommitTransaction or
// RollbackTransaction before the DatabaseContext goes out of scope.
bool BeginTransaction(std::string *error_message);
// Commit the transaction, or return false and fill |error_message|.
bool CommitTransaction(std::string *error_message);
// Roll back the transaction, logging error on failure.
// The error code is not returned; there's nothing useful the caller can do.
void RollbackTransaction();
// Borrow a prepared statement to run.
// |statement| should outlive the RunningStatement. It can't be borrowed
// twice simultaneously, but two similar statements can be run side-by-side
// (in the same context).
RunningStatement Borrow(Statement *statement);
// Use the given |sql| once.
// Note that parse errors are "deferred" until RunningStatement::Step().
RunningStatement UseOnce(re2::StringPiece sql);
// Return the number of changes for the last DML statement (insert, update, or
// delete), as with sqlite3_changes.
int64_t changes() { return sqlite3_changes(db_->me_); }
// Return the last rowid inserted into a table that does not specify "WITHOUT
// ROWID", as with sqlite3_last_insert_rowid.
int64_t last_insert_rowid() { return sqlite3_last_insert_rowid(db_->me_); }
Database *db() { return db_; }
private:
Database *db_;
std::lock_guard<std::mutex> lock_;
bool transaction_open_ = false;
};
// Convenience routines below.
// Run through all the statements in |stmts|.
// Return error if any do not parse or return something other than SQLITE_DONE
// when stepped. (SQLITE_ROW returns are skipped over, though. This is useful
// for "pragma journal_mode = wal;" which returns a row.)
bool RunStatements(DatabaseContext *ctx, re2::StringPiece stmts,
std::string *error_message);
} // namespace moonfire_nvr
#endif // MOONFIRE_NVR_SQLITE_H

162
src/stream.rs Normal file
View File

@ -0,0 +1,162 @@
// This file is part of Moonfire NVR, a security camera digital video recorder.
// Copyright (C) 2016 Scott Lamb <slamb@slamb.org>
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// In addition, as a special exception, the copyright holders give
// permission to link the code of portions of this program with the
// OpenSSL library under certain conditions as described in each
// individual source file, and distribute linked combinations including
// the two.
//
// You must obey the GNU General Public License in all respects for all
// of the code used other than OpenSSL. If you modify file(s) with this
// exception, you may extend this exception to your version of the
// file(s), but you are not obligated to do so. If you do not wish to do
// so, delete this exception statement from your version. If you delete
// this exception statement from all source files in the program, then
// also delete it here.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
use error::Error;
use ffmpeg::{self, format, media};
use ffmpeg_sys::{self, AVLockOp};
use h264;
use libc::{self, c_int, c_void};
use std::mem;
use std::ptr;
use std::result::Result;
use std::slice;
use std::sync;
static START: sync::Once = sync::ONCE_INIT;
pub enum StreamSource<'a> {
#[cfg(test)]
File(&'a str), // filename, for testing.
Rtsp(&'a str), // url, for production use.
}
// TODO: I think this should be provided by ffmpeg-sys. Otherwise, ffmpeg-sys is thread-hostile,
// which I believe is not allowed at all in Rust. (Also, this method's signature should include
// unsafe.)
extern "C" fn lock_callback(untyped_ptr: *mut *mut c_void, op: AVLockOp) -> c_int {
unsafe {
let ptr = mem::transmute::<*mut *mut c_void, *mut *mut libc::pthread_mutex_t>(untyped_ptr);
match op {
AVLockOp::AV_LOCK_CREATE => {
let m = Box::<libc::pthread_mutex_t>::new(mem::uninitialized());
*ptr = Box::into_raw(m);
libc::pthread_mutex_init(*ptr, ptr::null());
},
AVLockOp::AV_LOCK_DESTROY => {
libc::pthread_mutex_destroy(*ptr);
Box::from_raw(*ptr); // delete.
*ptr = ptr::null_mut();
},
AVLockOp::AV_LOCK_OBTAIN => {
libc::pthread_mutex_lock(*ptr);
},
AVLockOp::AV_LOCK_RELEASE => {
libc::pthread_mutex_unlock(*ptr);
},
};
};
0
}
impl<'a> StreamSource<'a> {
pub fn open(&self) -> Result<Stream, Error> {
START.call_once(|| {
unsafe { ffmpeg_sys::av_lockmgr_register(lock_callback); };
ffmpeg::init().unwrap();
ffmpeg::format::network::init();
});
let (input, discard_first) = match *self {
#[cfg(test)]
StreamSource::File(filename) =>
(format::input_with(&format!("file:{}", filename), ffmpeg::Dictionary::new())?,
false),
StreamSource::Rtsp(url) => {
let open_options = dict![
"rtsp_transport" => "tcp",
// https://trac.ffmpeg.org/ticket/5018 workaround attempt.
"probesize" => "262144",
"user-agent" => "moonfire-nvr",
// 10-second socket timeout, in microseconds.
"stimeout" => "10000000"
];
(format::input_with(&url, open_options)?, true)
},
};
// Find the video stream.
let mut video_i = None;
for (i, stream) in input.streams().enumerate() {
if stream.codec().medium() == media::Type::Video {
debug!("Video stream index is {}", i);
video_i = Some(i);
break;
}
}
let video_i = match video_i {
Some(i) => i,
None => { return Err(Error::new("no video stream".to_owned())) },
};
let mut stream = Stream{
input: input,
video_i: video_i,
};
if discard_first {
info!("Discarding the first packet to work around https://trac.ffmpeg.org/ticket/5018");
stream.get_next()?;
}
Ok(stream)
}
}
pub struct Stream {
input: format::context::Input,
video_i: usize,
}
impl Stream {
pub fn get_extra_data(&self) -> Result<h264::ExtraData, Error> {
let video = self.input.stream(self.video_i).expect("can't get video stream known to exist");
let codec = video.codec();
let (extradata, width, height) = unsafe {
let ptr = codec.as_ptr();
(slice::from_raw_parts((*ptr).extradata, (*ptr).extradata_size as usize),
(*ptr).width as u16,
(*ptr).height as u16)
};
// TODO: verify video stream is h264.
h264::ExtraData::parse(extradata, width, height)
}
pub fn get_next(&mut self) -> Result<ffmpeg::Packet, ffmpeg::Error> {
let mut pkt = ffmpeg::Packet::empty();
loop {
pkt.read(&mut self.input)?;
if pkt.stream() == self.video_i {
return Ok(pkt);
}
}
}
}

158
src/streamer.rs Normal file
View File

@ -0,0 +1,158 @@
// This file is part of Moonfire NVR, a security camera digital video recorder.
// Copyright (C) 2016 Scott Lamb <slamb@slamb.org>
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// In addition, as a special exception, the copyright holders give
// permission to link the code of portions of this program with the
// OpenSSL library under certain conditions as described in each
// individual source file, and distribute linked combinations including
// the two.
//
// You must obey the GNU General Public License in all respects for all
// of the code used other than OpenSSL. If you modify file(s) with this
// exception, you may extend this exception to your version of the
// file(s), but you are not obligated to do so. If you do not wish to do
// so, delete this exception statement from your version. If you delete
// this exception statement from all source files in the program, then
// also delete it here.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
use db::{Camera, Database};
use dir;
use error::Error;
use h264;
use recording;
use std::result::Result;
use std::sync::atomic::{AtomicBool, Ordering};
use std::sync::Arc;
use std::thread;
use std::time::Duration;
use stream::StreamSource;
use time;
pub static ROTATE_INTERVAL_SEC: i64 = 60;
pub struct Streamer {
shutdown: Arc<AtomicBool>,
// State below is only used by the thread in Run.
rotate_offset_sec: i64,
db: Arc<Database>,
dir: Arc<dir::SampleFileDir>,
syncer_channel: dir::SyncerChannel,
camera_id: i32,
short_name: String,
url: String,
redacted_url: String,
}
impl Streamer {
pub fn new(db: Arc<Database>, dir: Arc<dir::SampleFileDir>, syncer_channel: dir::SyncerChannel,
shutdown: Arc<AtomicBool>, camera_id: i32, c: &Camera, rotate_offset_sec: i64)
-> Self {
Streamer{
shutdown: shutdown,
rotate_offset_sec: rotate_offset_sec,
db: db,
dir: dir,
syncer_channel: syncer_channel,
camera_id: camera_id,
short_name: c.short_name.to_owned(),
url: format!("rtsp://{}:{}@{}{}", c.username, c.password, c.host, c.main_rtsp_path),
redacted_url: format!("rtsp://{}:redacted@{}{}", c.username, c.host, c.main_rtsp_path),
}
}
pub fn short_name(&self) -> &str { &self.short_name }
pub fn run(&mut self) {
while !self.shutdown.load(Ordering::SeqCst) {
if let Err(e) = self.run_once() {
let sleep_time = Duration::from_secs(1);
warn!("{}: sleeping for {:?} after error: {}", self.short_name, sleep_time, e);
thread::sleep(sleep_time);
}
}
info!("{}: shutting down", self.short_name);
}
fn run_once(&mut self) -> Result<(), Error> {
info!("{}: Opening input: {}", self.short_name, self.redacted_url);
// TODO: mockability?
let mut stream = StreamSource::Rtsp(&self.url).open()?;
// TODO: verify time base.
// TODO: verify width/height.
let extra_data = stream.get_extra_data()?;
let video_sample_entry_id =
self.db.lock().insert_video_sample_entry(extra_data.width, extra_data.height,
&extra_data.sample_entry)?;
debug!("{}: video_sample_entry_id={}", self.short_name, video_sample_entry_id);
let mut seen_key_frame = false;
let mut rotate = None;
let mut writer: Option<recording::Writer> = None;
let mut transformed = Vec::new();
let mut next_start = None;
while !self.shutdown.load(Ordering::SeqCst) {
let pkt = stream.get_next()?;
if !seen_key_frame && !pkt.is_key() {
continue;
} else if !seen_key_frame {
debug!("{}: have first key frame", self.short_name);
seen_key_frame = true;
}
let frame_realtime = time::get_time();
if let Some(r) = rotate {
if frame_realtime.sec > r && pkt.is_key() {
let w = writer.take().expect("rotate set implies writer is set");
next_start = Some(w.end());
// TODO: restore this log message.
// info!("{}: wrote {}: [{}, {})", self.short_name, r.sample_file_uuid,
// r.time.start, r.time.end);
self.syncer_channel.async_save_writer(w)?;
}
};
let mut w = match writer {
Some(w) => w,
None => {
let r = frame_realtime.sec -
(frame_realtime.sec % ROTATE_INTERVAL_SEC) +
self.rotate_offset_sec;
rotate = Some(
if r <= frame_realtime.sec { r + ROTATE_INTERVAL_SEC } else { r });
let local_realtime = recording::Time::new(frame_realtime);
self.dir.create_writer(next_start.unwrap_or(local_realtime), local_realtime,
self.camera_id, video_sample_entry_id)?
},
};
let orig_data = match pkt.data() {
Some(d) => d,
None => return Err(Error::new("packet has no data".to_owned())),
};
let transformed_data = if extra_data.need_transform {
h264::transform_sample_data(orig_data, &mut transformed)?;
transformed.as_slice()
} else {
orig_data
};
w.write(transformed_data, pkt.duration() as i32, pkt.is_key())?;
writer = Some(w);
}
if let Some(w) = writer {
self.syncer_channel.async_save_writer(w)?;
}
Ok(())
}
}

116
src/string-test.cc
View File

@ -1,116 +0,0 @@
// This file is part of Moonfire NVR, a security camera digital video recorder.
// Copyright (C) 2016 Scott Lamb <slamb@slamb.org>
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// In addition, as a special exception, the copyright holders give
// permission to link the code of portions of this program with the
// OpenSSL library under certain conditions as described in each
// individual source file, and distribute linked combinations including
// the two.
//
// You must obey the GNU General Public License in all respects for all
// of the code used other than OpenSSL. If you modify file(s) with this
// exception, you may extend this exception to your version of the
// file(s), but you are not obligated to do so. If you do not wish to do
// so, delete this exception statement from your version. If you delete
// this exception statement from all source files in the program, then
// also delete it here.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
//
// string-test.cc: tests of the string.h interface.
#include <gflags/gflags.h>
#include <glog/logging.h>
#include <gmock/gmock.h>
#include <gtest/gtest.h>
#include "string.h"
DECLARE_bool(alsologtostderr);
namespace moonfire_nvr {
namespace {
TEST(StrCatTest, Simple) {
EXPECT_EQ("foo", StrCat("foo"));
EXPECT_EQ("foobar", StrCat("foo", "bar"));
EXPECT_EQ("foo", StrCat(std::string("foo")));
EXPECT_EQ("42", StrCat(uint64_t(42)));
EXPECT_EQ("0", StrCat(uint64_t(0)));
EXPECT_EQ("18446744073709551615",
StrCat(std::numeric_limits<uint64_t>::max()));
EXPECT_EQ("42", StrCat(int64_t(42)));
EXPECT_EQ("0", StrCat(int64_t(0)));
EXPECT_EQ("-9223372036854775808",
StrCat(std::numeric_limits<int64_t>::min()));
EXPECT_EQ("9223372036854775807", StrCat(std::numeric_limits<int64_t>::max()));
}
TEST(JoinTest, Simple) {
EXPECT_EQ("", Join(std::initializer_list<std::string>(), ","));
EXPECT_EQ("a", Join(std::initializer_list<std::string>({"a"}), ","));
EXPECT_EQ("a,b", Join(std::initializer_list<const char *>({"a", "b"}), ","));
EXPECT_EQ(
"a,b,c",
Join(std::initializer_list<re2::StringPiece>({"a", "b", "c"}), ","));
}
TEST(EscapeTest, Simple) {
EXPECT_EQ("", moonfire_nvr::EscapeHtml(""));
EXPECT_EQ("no special chars", moonfire_nvr::EscapeHtml("no special chars"));
EXPECT_EQ("&lt;tag&gt; &amp; text", moonfire_nvr::EscapeHtml("<tag> & text"));
}
TEST(ToHexTest, Simple) {
EXPECT_EQ("", ToHex("", false));
EXPECT_EQ("", ToHex("", true));
EXPECT_EQ("1234deadbeef", ToHex("\x12\x34\xde\xad\xbe\xef", false));
EXPECT_EQ("12 34 de ad be ef", ToHex("\x12\x34\xde\xad\xbe\xef", true));
}
TEST(HumanizeTest, Simple) {
EXPECT_EQ("1.0 B", HumanizeWithBinaryPrefix(1.f, "B"));
EXPECT_EQ("1.0 KiB", HumanizeWithBinaryPrefix(UINT64_C(1) << 10, "B"));
EXPECT_EQ("1.0 EiB", HumanizeWithBinaryPrefix(UINT64_C(1) << 60, "B"));
EXPECT_EQ("1.5 EiB", HumanizeWithBinaryPrefix(
(UINT64_C(1) << 60) + (UINT64_C(1) << 59), "B"));
EXPECT_EQ("16.0 EiB", HumanizeWithBinaryPrefix(
std::numeric_limits<uint64_t>::max(), "B"));
EXPECT_EQ("1.0 Mbps", HumanizeWithDecimalPrefix(1e6f, "bps"));
EXPECT_EQ("1000.0 Ebps", HumanizeWithDecimalPrefix(1e21, "bps"));
}
TEST(AtoiTest, Simple) {
int64_t out;
EXPECT_TRUE(Atoi64("1234", 10, &out));
EXPECT_EQ(1234, out);
EXPECT_FALSE(Atoi64(nullptr, 10, &out));
EXPECT_FALSE(Atoi64("", 10, &out));
EXPECT_FALSE(Atoi64("asdf", 10, &out));
EXPECT_FALSE(Atoi64("1234asdf", 10, &out));
}
} // namespace
} // namespace moonfire_nvr
int main(int argc, char **argv) {
FLAGS_alsologtostderr = true;
google::ParseCommandLineFlags(&argc, &argv, true);
testing::InitGoogleTest(&argc, argv);
google::InitGoogleLogging(argv[0]);
return RUN_ALL_TESTS();
}

189
src/string.cc
View File

@ -1,189 +0,0 @@
// This file is part of Moonfire NVR, a security camera digital video recorder.
// Copyright (C) 2016 Scott Lamb <slamb@slamb.org>
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// In addition, as a special exception, the copyright holders give
// permission to link the code of portions of this program with the
// OpenSSL library under certain conditions as described in each
// individual source file, and distribute linked combinations including
// the two.
//
// You must obey the GNU General Public License in all respects for all
// of the code used other than OpenSSL. If you modify file(s) with this
// exception, you may extend this exception to your version of the
// file(s), but you are not obligated to do so. If you do not wish to do
// so, delete this exception statement from your version. If you delete
// this exception statement from all source files in the program, then
// also delete it here.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
//
// string.cc: See string.h.
#include "string.h"
#include <string.h>
#include <glog/logging.h>
namespace moonfire_nvr {
namespace {
char HexDigit(unsigned int i) {
static char kHexadigits[] = "0123456789abcdef";
return (i < 16) ? kHexadigits[i] : 'x';
}
std::string Humanize(std::initializer_list<const re2::StringPiece> prefixes,
float f, float n, re2::StringPiece suffix) {
size_t i;
for (i = 0; i < prefixes.size() - 1 && n >= f; ++i) n /= f;
char buf[64];
snprintf(buf, sizeof(buf), "%.1f", n);
return StrCat(buf, *(prefixes.begin() + i), suffix);
}
} // namespace
namespace internal {
StrCatPiece::StrCatPiece(uint64_t p) {
if (p == 0) {
piece_ = "0";
} else {
size_t i = sizeof(buf_);
while (p != 0) {
buf_[--i] = '0' + (p % 10);
p /= 10;
}
piece_.set(buf_ + i, sizeof(buf_) - i);
}
}
StrCatPiece::StrCatPiece(int64_t p) {
if (p == 0) {
piece_ = "0";
} else {
bool negative = p < 0;
size_t i = sizeof(buf_);
while (p != 0) {
buf_[--i] = '0' + std::abs(p % 10);
p /= 10;
}
if (negative) {
buf_[--i] = '-';
}
piece_.set(buf_ + i, sizeof(buf_) - i);
}
}
} // namespace internal
std::string EscapeHtml(const std::string &input) {
std::string output;
output.reserve(input.size());
for (char c : input) {
switch (c) {
case '&':
output.append("&amp;");
break;
case '<':
output.append("&lt;");
break;
case '>':
output.append("&gt;");
break;
default:
output.push_back(c);
}
}
return output;
}
std::string ToHex(re2::StringPiece in, bool pad) {
std::string out;
out.reserve(in.size() * (2 + pad) + pad);
for (int i = 0; i < in.size(); ++i) {
if (pad && i > 0) out.push_back(' ');
uint8_t byte = in[i];
out.push_back(HexDigit(byte >> 4));
out.push_back(HexDigit(byte & 0x0F));
}
return out;
}
std::string HumanizeWithDecimalPrefix(float n, re2::StringPiece suffix) {
static const std::initializer_list<const re2::StringPiece> kPrefixes = {
" ", " k", " M", " G", " T", " P", " E"};
return Humanize(kPrefixes, 1000., n, suffix);
}
std::string HumanizeWithBinaryPrefix(float n, re2::StringPiece suffix) {
static const std::initializer_list<const re2::StringPiece> kPrefixes = {
" ", " Ki", " Mi", " Gi", " Ti", " Pi", " Ei"};
return Humanize(kPrefixes, 1024., n, suffix);
}
std::string HumanizeDuration(int64_t seconds) {
static const int64_t kMinuteInSeconds = 60;
static const int64_t kHourInSeconds = 60 * kMinuteInSeconds;
static const int64_t kDayInSeconds = 24 * kHourInSeconds;
int64_t days = seconds / kDayInSeconds;
seconds %= kDayInSeconds;
int64_t hours = seconds / kHourInSeconds;
seconds %= kHourInSeconds;
int64_t minutes = seconds / kMinuteInSeconds;
seconds %= kMinuteInSeconds;
std::string out;
if (days > 0) {
out.append(StrCat(days, days == 1 ? " day" : " days"));
}
if (hours > 0) {
if (!out.empty()) {
out.append(" ");
}
out.append(StrCat(hours, hours == 1 ? " hour" : " hours"));
}
if (minutes > 0) {
if (!out.empty()) {
out.append(" ");
}
out.append(StrCat(minutes, minutes == 1 ? " minute" : " minutes"));
}
if (seconds > 0 || out.empty()) {
if (!out.empty()) {
out.append(" ");
}
out.append(StrCat(seconds, seconds == 1 ? " second" : " seconds"));
}
return out;
}
bool strto64(const char *str, int base, const char **endptr, int64_t *value) {
static_assert(sizeof(int64_t) == sizeof(long long int),
"unknown memory model");
if (str == nullptr) {
return false;
}
errno = 0;
*value = ::strtoll(str, const_cast<char **>(endptr), base);
return *endptr != str && errno == 0;
}
bool Atoi64(const char *str, int base, int64_t *value) {
const char *endptr;
return strto64(str, base, &endptr, value) && *endptr == '\0';
}
} // namespace moonfire_nvr

138
src/string.h
View File

@ -1,138 +0,0 @@
// This file is part of Moonfire NVR, a security camera digital video recorder.
// Copyright (C) 2016 Scott Lamb <slamb@slamb.org>
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// In addition, as a special exception, the copyright holders give
// permission to link the code of portions of this program with the
// OpenSSL library under certain conditions as described in each
// individual source file, and distribute linked combinations including
// the two.
//
// You must obey the GNU General Public License in all respects for all
// of the code used other than OpenSSL. If you modify file(s) with this
// exception, you may extend this exception to your version of the
// file(s), but you are not obligated to do so. If you do not wish to do
// so, delete this exception statement from your version. If you delete
// this exception statement from all source files in the program, then
// also delete it here.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
//
// string.h: convenience methods for dealing with strings.
#ifndef MOONFIRE_NVR_STRING_H
#define MOONFIRE_NVR_STRING_H
#include <string>
#include <re2/stringpiece.h>
namespace moonfire_nvr {
namespace internal {
// Only used within StrCat() and Join().
// Note implicit constructor, which is necessary to avoid a copy,
// though it could be wrapped in another type.
// http://stackoverflow.com/questions/34112755/can-i-avoid-a-c11-move-when-initializing-an-array/34113744
class StrCatPiece {
public:
StrCatPiece(uint64_t p);
StrCatPiece(int64_t p);
StrCatPiece(uint32_t p) : StrCatPiece(static_cast<uint64_t>(p)) {}
StrCatPiece(int32_t p) : StrCatPiece(static_cast<int64_t>(p)) {}
#ifndef __LP64__ // if sizeof(long) == sizeof(int32_t)
// Need to resolve ambiguity.
StrCatPiece(long p) : StrCatPiece(static_cast<int32_t>(p)) {}
StrCatPiece(unsigned long p) : StrCatPiece(static_cast<uint32_t>(p)) {}
#endif
StrCatPiece(re2::StringPiece p) : piece_(p) {}
StrCatPiece(const StrCatPiece &) = delete;
StrCatPiece &operator=(const StrCatPiece &) = delete;
const char *data() const { return piece_.data(); }
size_t size() const { return piece_.size(); }
private:
// Not allowed: ambiguous meaning.
StrCatPiece(char);
// |piece_| points either to within buf_ (numeric constructors) or to unowned
// string data (StringPiece constructor).
re2::StringPiece piece_;
char buf_[20]; // length of maximum uint64 (no terminator needed).
};
} // namespace internal
// Concatenate any number of strings, StringPieces, and numeric values into a
// single string.
template <typename... Types>
std::string StrCat(Types... args) {
internal::StrCatPiece pieces[] = {{args}...};
size_t size = 0;
for (const auto &p : pieces) {
size += p.size();
}
std::string out;
out.reserve(size);
for (const auto &p : pieces) {
out.append(p.data(), p.size());
}
return out;
}
// Join any number of string fragments (of like type) together into a single
// string, with a separator.
template <typename Container>
std::string Join(const Container &pieces, re2::StringPiece separator) {
std::string out;
bool first = true;
for (const auto &p : pieces) {
if (!first) {
out.append(separator.data(), separator.size());
}
first = false;
internal::StrCatPiece piece(p);
out.append(piece.data(), piece.size());
}
return out;
}
// HTML-escape the given UTF-8-encoded string.
std::string EscapeHtml(const std::string &input);
// Return a hex-encoded version of |in|, optionally padding between bytes.
// For example, ToHex("\xde\xad\xbe\xef", true) returns "de ad be ef".
std::string ToHex(re2::StringPiece in, bool pad = false);
// Return a human-friendly approximation of the given non-negative value, using
// SI (base-10) or IEC (base-2) prefixes.
std::string HumanizeWithDecimalPrefix(float n, re2::StringPiece suffix);
std::string HumanizeWithBinaryPrefix(float n, re2::StringPiece suffix);
std::string HumanizeDuration(int64_t sec);
// Wrapper around ::strtoll that returns true iff valid and corrects
// constness. Returns false if |str| is null.
bool strto64(const char *str, int base, const char **endptr, int64_t *value);
// Simpler form that expects the entire string to be a single integer.
bool Atoi64(const char *str, int base, int64_t *value);
} // namespace moonfire_nvr
#endif // MOONFIRE_NVR_STRING_H

153
src/testutil.cc
View File

@ -1,153 +0,0 @@
// This file is part of Moonfire NVR, a security camera digital video recorder.
// Copyright (C) 2016 Scott Lamb <slamb@slamb.org>
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// In addition, as a special exception, the copyright holders give
// permission to link the code of portions of this program with the
// OpenSSL library under certain conditions as described in each
// individual source file, and distribute linked combinations including
// the two.
//
// You must obey the GNU General Public License in all respects for all
// of the code used other than OpenSSL. If you modify file(s) with this
// exception, you may extend this exception to your version of the
// file(s), but you are not obligated to do so. If you do not wish to do
// so, delete this exception statement from your version. If you delete
// this exception statement from all source files in the program, then
// also delete it here.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
//
// testutil.cc: implementation of testutil.h interface.
#include "testutil.h"
#include <dirent.h>
#include <errno.h>
#include <fcntl.h>
#include <string.h>
#include <sys/stat.h>
#include <sys/types.h>
#include <event2/buffer.h>
#include <glog/logging.h>
#include "filesystem.h"
#include "string.h"
namespace moonfire_nvr {
namespace {
bool DeleteChildrenRecursively(const char *dirname, std::string *error_msg) {
bool ok = true;
auto fn = [&dirname, &ok, error_msg](const struct dirent *ent) {
std::string name(ent->d_name);
std::string path = StrCat(dirname, "/", name);
if (name == "." || name == "..") {
return IterationControl::kContinue;
}
bool is_dir = (ent->d_type == DT_DIR);
if (ent->d_type == DT_UNKNOWN) {
struct stat buf;
int ret = GetRealFilesystem()->Stat(path.c_str(), &buf);
CHECK_EQ(ret, 0) << path << ": " << strerror(ret);
is_dir = S_ISDIR(buf.st_mode);
}
if (is_dir) {
ok = ok && DeleteChildrenRecursively(path.c_str(), error_msg);
if (!ok) {
return IterationControl::kBreak;
}
int ret = GetRealFilesystem()->Rmdir(path.c_str());
if (ret != 0) {
*error_msg = StrCat("rmdir failed on ", path, ": ", strerror(ret));
ok = false;
return IterationControl::kBreak;
}
} else {
int ret = GetRealFilesystem()->Unlink(path.c_str());
if (ret != 0) {
*error_msg = StrCat("unlink failed on ", path, ": ", strerror(ret));
ok = false;
return IterationControl::kBreak;
}
}
return IterationControl::kContinue;
};
if (!GetRealFilesystem()->DirForEach(dirname, fn, error_msg)) {
return false;
}
return ok;
}
} // namespace
std::string PrepareTempDirOrDie(const std::string &test_name) {
std::string dirname = StrCat("/tmp/test.", test_name);
int ret = GetRealFilesystem()->Mkdir(dirname.c_str(), 0700);
if (ret != 0) {
CHECK_EQ(ret, EEXIST) << "mkdir failed: " << strerror(ret);
std::string error_msg;
CHECK(DeleteChildrenRecursively(dirname.c_str(), &error_msg)) << error_msg;
}
return dirname;
}
void WriteFileOrDie(const std::string &path, re2::StringPiece contents) {
std::unique_ptr<File> f;
int ret = GetRealFilesystem()->Open(path.c_str(),
O_WRONLY | O_CREAT | O_TRUNC, 0600, &f);
CHECK_EQ(ret, 0) << "open " << path << ": " << strerror(ret);
while (!contents.empty()) {
size_t written;
ret = f->Write(contents, &written);
CHECK_EQ(ret, 0) << "write " << path << ": " << strerror(ret);
contents.remove_prefix(written);
}
ret = f->Close();
CHECK_EQ(ret, 0) << "close " << path << ": " << strerror(ret);
}
void WriteFileOrDie(const std::string &path, EvBuffer *buf) {
int fd = open(path.c_str(), O_CREAT | O_WRONLY | O_TRUNC, 0600);
PCHECK(fd >= 0) << "open: " << path;
while (evbuffer_get_length(buf->get()) > 0) {
size_t buf_len = evbuffer_get_length(buf->get());
int written = evbuffer_write(buf->get(), fd);
PCHECK(written >= 0) << "buf_len: " << buf_len << ", written: " << written;
}
PCHECK(close(fd) == 0) << "close";
}
std::string ReadFileOrDie(const std::string &path) {
std::unique_ptr<File> f;
int ret = GetRealFilesystem()->Open(path.c_str(), O_RDONLY, &f);
CHECK_EQ(ret, 0) << "open " << path << ": " << strerror(ret);
struct stat statbuf;
ret = f->Stat(&statbuf);
CHECK_EQ(ret, 0) << "fstat " << path << ": " << strerror(ret);
std::string out(statbuf.st_size, '0');
off_t bytes_read_total = 0;
size_t bytes_read;
while (bytes_read_total < statbuf.st_size) {
ret = f->Read(&out[bytes_read_total], statbuf.st_size - bytes_read_total,
&bytes_read);
CHECK_EQ(ret, 0) << "read " << path << ": " << strerror(ret);
CHECK_GT(bytes_read, 0);
bytes_read_total += bytes_read;
}
return out;
}
} // namespace moonfire_nvr

155
src/testutil.h
View File

@ -1,155 +0,0 @@
// This file is part of Moonfire NVR, a security camera digital video recorder.
// Copyright (C) 2016 Scott Lamb <slamb@slamb.org>
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// In addition, as a special exception, the copyright holders give
// permission to link the code of portions of this program with the
// OpenSSL library under certain conditions as described in each
// individual source file, and distribute linked combinations including
// the two.
//
// You must obey the GNU General Public License in all respects for all
// of the code used other than OpenSSL. If you modify file(s) with this
// exception, you may extend this exception to your version of the
// file(s), but you are not obligated to do so. If you do not wish to do
// so, delete this exception statement from your version. If you delete
// this exception statement from all source files in the program, then
// also delete it here.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
//
// testutil.h: utilities for testing.
#ifndef MOONFIRE_NVR_TESTUTIL_H
#define MOONFIRE_NVR_TESTUTIL_H
#include <glog/logging.h>
#include <gmock/gmock.h>
#include <re2/stringpiece.h>
#include "filesystem.h"
#include "http.h"
#include "uuid.h"
namespace moonfire_nvr {
// Create or empty the given test directory, or die.
// Returns the full path.
std::string PrepareTempDirOrDie(const std::string &test_name);
// Write the given file contents to the given path, or die.
void WriteFileOrDie(const std::string &path, re2::StringPiece contents);
void WriteFileOrDie(const std::string &path, EvBuffer *buf);
// Read the contents of the given path, or die.
std::string ReadFileOrDie(const std::string &path);
// A scoped log sink for testing that the right log messages are sent.
// Modelled after glog's "mock-log.h", which is not exported.
// Use as follows:
//
// {
// ScopedMockLog log;
// EXPECT_CALL(log, Log(ERROR, _, HasSubstr("blah blah")));
// log.Start();
// ThingThatLogs();
// }
class ScopedMockLog : public google::LogSink {
public:
~ScopedMockLog() final { google::RemoveLogSink(this); }
// Start logging to this sink.
// This is not done at construction time so that it's possible to set
// expectations first, which is important if some background thread is
// already logging.
void Start() { google::AddLogSink(this); }
// Set expectations here.
MOCK_METHOD3(Log, void(google::LogSeverity severity,
const std::string &full_filename,
const std::string &message));
private:
struct LogEntry {
google::LogSeverity severity = -1;
std::string full_filename;
std::string message;
};
// This method is called with locks held and thus shouldn't call Log.
// It just stashes away the log entry for later.
void send(google::LogSeverity severity, const char *full_filename,
const char *base_filename, int line, const tm *tm_time,
const char *message, size_t message_len) final {
pending_.severity = severity;
pending_.full_filename = full_filename;
pending_.message.assign(message, message_len);
}
// This method is always called after send() without locks.
// It does the actual work of calling Log. It moves data away from
// pending_ in case Log() logs itself (causing a nested call to send() and
// WaitTillSent()).
void WaitTillSent() final {
LogEntry entry = std::move(pending_);
Log(entry.severity, entry.full_filename, entry.message);
}
LogEntry pending_;
};
class MockUuidGenerator : public UuidGenerator {
public:
MOCK_METHOD0(Generate, Uuid());
};
class MockFile : public File {
public:
MOCK_CONST_METHOD0(name, const std::string &());
MOCK_METHOD3(Access, int(const char *, int, int));
MOCK_METHOD0(Close, int());
// The std::unique_ptr<File> variants of Open are wrapped here because gmock's
// SetArgPointee doesn't work well with std::unique_ptr.
int Open(const char *path, int flags, std::unique_ptr<File> *f) final {
File *f_tmp = nullptr;
int ret = OpenRaw(path, flags, &f_tmp);
f->reset(f_tmp);
return ret;
}
int Open(const char *path, int flags, mode_t mode,
std::unique_ptr<File> *f) final {
File *f_tmp = nullptr;
int ret = OpenRaw(path, flags, mode, &f_tmp);
f->reset(f_tmp);
return ret;
}
MOCK_METHOD1(Lock, int(int));
MOCK_METHOD3(Open, int(const char *, int, int *));
MOCK_METHOD4(Open, int(const char *, int, mode_t, int *));
MOCK_METHOD3(OpenRaw, int(const char *, int, File **));
MOCK_METHOD4(OpenRaw, int(const char *, int, mode_t, File **));
MOCK_METHOD3(Read, int(void *, size_t, size_t *));
MOCK_METHOD1(Stat, int(struct stat *));
MOCK_METHOD0(Sync, int());
MOCK_METHOD1(Truncate, int(off_t));
MOCK_METHOD1(Unlink, int(const char *));
MOCK_METHOD2(Write, int(re2::StringPiece, size_t *));
};
} // namespace moonfire_nvr
#endif // MOONFIRE_NVR_TESTUTIL_H

41
src/profiler.h → src/testutil.rs
View File

@ -27,25 +27,28 @@
//
// You should have received a copy of the GNU General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
//
// profiler.h: support for on-demand profiling. The interface is described here:
// <https://github.com/gperftools/gperftools/blob/master/doc/pprof_remote_servers.html>
// Currently this only supports CPU profiling; heap profiling may be added
// later.
#ifndef MOONFIRE_NVR_PROFILER_H
#define MOONFIRE_NVR_PROFILER_H
use std::env;
use std::sync;
use slog::{self, DrainExt};
use slog_envlogger;
use slog_stdlog;
use slog_term;
use time;
#include <event2/buffer.h>
#include <event2/http.h>
static INIT: sync::Once = sync::ONCE_INIT;
namespace moonfire_nvr {
// Register a HTTP URI for a CPU profile with google-perftools.
// Not thread-safe, so |http| must be running on |base|, and |base| must be
// using a single thread.
void RegisterProfiler(event_base *base, evhttp *http);
} // namespace moonfire_nvr
#endif // MOONFIRE_NVR_PROFILER_H
/// Performs global initialization for tests.
/// * set up logging. (Note the output can be confusing unless `RUST_TEST_THREADS=1` is set in
/// the program's environment prior to running.)
/// * set `TZ=America/Los_Angeles` so that tests that care about calendar time get the expected
/// results regardless of machine setup.)
pub fn init() {
INIT.call_once(|| {
let drain = slog_term::StreamerBuilder::new().async().full().build();
let drain = slog_envlogger::new(drain);
slog_stdlog::set_logger(slog::Logger::root(drain.ignore_err(), None)).unwrap();
env::set_var("TZ", "America/Los_Angeles");
time::tzset();
});
}

90
src/time.cc
View File

@ -1,90 +0,0 @@
// This file is part of Moonfire NVR, a security camera digital video recorder.
// Copyright (C) 2016 Scott Lamb <slamb@slamb.org>
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// In addition, as a special exception, the copyright holders give
// permission to link the code of portions of this program with the
// OpenSSL library under certain conditions as described in each
// individual source file, and distribute linked combinations including
// the two.
//
// You must obey the GNU General Public License in all respects for all
// of the code used other than OpenSSL. If you modify file(s) with this
// exception, you may extend this exception to your version of the
// file(s), but you are not obligated to do so. If you do not wish to do
// so, delete this exception statement from your version. If you delete
// this exception statement from all source files in the program, then
// also delete it here.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
//
// time.cc: implementation of time.h interface.
#include "time.h"
#include <errno.h>
#include <string.h>
#include <glog/logging.h>
namespace moonfire_nvr {
namespace {
class RealClock : public WallClock {
public:
struct timespec Now() const final {
struct timespec now;
CHECK_EQ(0, clock_gettime(CLOCK_REALTIME, &now)) << strerror(errno);
return now;
}
void Sleep(struct timespec req) final {
struct timespec rem;
while (true) {
int ret = nanosleep(&req, &rem);
if (ret != 0 && errno != EINTR) {
PLOG(FATAL) << "nanosleep";
}
if (ret == 0) {
return;
}
req = rem;
}
}
};
} // namespace
// Returns the real wall clock, which will never be deleted.
WallClock *GetRealClock() {
static RealClock *real_clock = new RealClock; // never deleted.
return real_clock;
}
struct timespec SimulatedClock::Now() const {
std::lock_guard<std::mutex> l(mu_);
return now_;
}
void SimulatedClock::Sleep(struct timespec req) {
std::lock_guard<std::mutex> l(mu_);
now_.tv_sec += req.tv_sec;
now_.tv_nsec += req.tv_nsec;
if (now_.tv_nsec > kNanos) {
now_.tv_nsec -= kNanos;
now_.tv_sec++;
}
}
} // namespace moonfire_nvr

78
src/time.h
View File

@ -1,78 +0,0 @@
// This file is part of Moonfire NVR, a security camera digital video recorder.
// Copyright (C) 2016 Scott Lamb <slamb@slamb.org>
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// In addition, as a special exception, the copyright holders give
// permission to link the code of portions of this program with the
// OpenSSL library under certain conditions as described in each
// individual source file, and distribute linked combinations including
// the two.
//
// You must obey the GNU General Public License in all respects for all
// of the code used other than OpenSSL. If you modify file(s) with this
// exception, you may extend this exception to your version of the
// file(s), but you are not obligated to do so. If you do not wish to do
// so, delete this exception statement from your version. If you delete
// this exception statement from all source files in the program, then
// also delete it here.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
//
// time.h: functions dealing with (wall) time.
#ifndef MOONFIRE_NVR_TIME_H
#define MOONFIRE_NVR_TIME_H
#include <math.h>
#include <time.h>
#include <mutex>
namespace moonfire_nvr {
constexpr long kNanos = 1000000000;
class WallClock {
public:
virtual ~WallClock() {}
virtual struct timespec Now() const = 0;
virtual void Sleep(struct timespec) = 0;
};
class SimulatedClock : public WallClock {
public:
SimulatedClock() : now_({0, 0}) {}
struct timespec Now() const final;
void Sleep(struct timespec req) final;
private:
mutable std::mutex mu_;
struct timespec now_;
};
inline struct timespec SecToTimespec(double sec) {
double intpart;
double fractpart = modf(sec, &intpart);
return {static_cast<time_t>(intpart), static_cast<long>(fractpart * kNanos)};
}
inline double TimespecToSec(struct timespec t) {
return t.tv_sec + static_cast<double>(t.tv_nsec) / kNanos;
}
// Returns the real wall clock, which will never be deleted.
WallClock *GetRealClock();
} // namespace moonfire_nvr
#endif // MOONFIRE_NVR_TIME_H

94
src/uuid.cc
View File

@ -1,94 +0,0 @@
// This file is part of Moonfire NVR, a security camera network video recorder.
// Copyright (C) 2016 Lamb <slamb@slamb.org>
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// In addition, as a special exception, the copyright holders give
// permission to link the code of portions of this program with the
// OpenSSL library under certain conditions as described in each
// individual source file, and distribute linked combinations including
// the two.
//
// You must obey the GNU General Public License in all respects for all
// of the code used other than OpenSSL. If you modify file(s) with this
// exception, you may extend this exception to your version of the
// file(s), but you are not obligated to do so. If you do not wish to do
// so, delete this exception statement from your version. If you delete
// this exception statement from all source files in the program, then
// also delete it here.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
//
// uuid.cc: implementation of uuid.h interface.
#include "uuid.h"
namespace moonfire_nvr {
namespace {
const size_t kTextFormatLength =
sizeof("xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx") - 1;
} // namespace
bool Uuid::ParseText(re2::StringPiece input) {
if (input.size() != kTextFormatLength) {
return false;
}
char tmp[kTextFormatLength + 1];
memcpy(tmp, input.data(), kTextFormatLength);
tmp[kTextFormatLength] = 0;
return uuid_parse(tmp, me_) == 0;
}
bool Uuid::ParseBinary(re2::StringPiece input) {
if (input.size() != sizeof(uuid_t)) {
return false;
}
memcpy(me_, input.data(), sizeof(uuid_t));
return true;
}
std::string Uuid::UnparseText() const {
char tmp[kTextFormatLength + 1];
uuid_unparse_lower(me_, tmp);
return tmp;
}
re2::StringPiece Uuid::binary_view() const {
return re2::StringPiece(reinterpret_cast<const char *>(me_), sizeof(me_));
}
bool Uuid::operator==(const Uuid &other) const {
return uuid_compare(me_, other.me_) == 0;
}
bool Uuid::operator<(const Uuid &other) const {
return uuid_compare(me_, other.me_) < 0;
}
class RealUuidGenerator : public UuidGenerator {
public:
Uuid Generate() final {
Uuid out;
uuid_generate(out.me_);
return out;
}
};
UuidGenerator *GetRealUuidGenerator() {
static RealUuidGenerator *gen = new RealUuidGenerator; // never freed.
return gen;
}
} // namespace moonfire_nvr

81
src/uuid.h
View File

@ -1,81 +0,0 @@
// This file is part of Moonfire NVR, a security camera network video recorder.
// Copyright (C) 2016 Lamb <slamb@slamb.org>
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// In addition, as a special exception, the copyright holders give
// permission to link the code of portions of this program with the
// OpenSSL library under certain conditions as described in each
// individual source file, and distribute linked combinations including
// the two.
//
// You must obey the GNU General Public License in all respects for all
// of the code used other than OpenSSL. If you modify file(s) with this
// exception, you may extend this exception to your version of the
// file(s), but you are not obligated to do so. If you do not wish to do
// so, delete this exception statement from your version. If you delete
// this exception statement from all source files in the program, then
// also delete it here.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
//
// uuid.h: small wrapper around the C UUID library for generating/parsing
// RFC 4122 UUIDs.
#ifndef MOONFIRE_NVR_UUID_H
#define MOONFIRE_NVR_UUID_H
#include <re2/stringpiece.h>
#include <uuid/uuid.h>
namespace moonfire_nvr {
class Uuid {
public:
// Create a null uuid.
Uuid() { uuid_clear(me_); }
// Parse the text UUID. Returns success.
bool ParseText(re2::StringPiece input);
// Parse a binary UUID. In practice any 16-byte string is considered valid.
bool ParseBinary(re2::StringPiece input);
// Return a 36-byte lowercase text representation, such as
// 1b4e28ba-2fa1-11d2-883f-0016d3cca427.
std::string UnparseText() const;
// Return a reference to the 16-byte binary form.
// Invalidated by any change to the Uuid object.
re2::StringPiece binary_view() const;
bool operator==(const Uuid &) const;
bool operator<(const Uuid &) const;
bool is_null() const { return uuid_is_null(me_); }
private:
friend class RealUuidGenerator;
uuid_t me_;
};
class UuidGenerator {
public:
virtual ~UuidGenerator() {}
virtual Uuid Generate() = 0;
};
UuidGenerator *GetRealUuidGenerator();
} // namespace moonfire_nvr
#endif // MOONFIRE_NVR_CODING_H

505
src/web.cc
View File

@ -1,505 +0,0 @@
// This file is part of Moonfire NVR, a security camera network video recorder.
// Copyright (C) 2016 Scott Lamb <slamb@slamb.org>
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// In addition, as a special exception, the copyright holders give
// permission to link the code of portions of this program with the
// OpenSSL library under certain conditions as described in each
// individual source file, and distribute linked combinations including
// the two.
//
// You must obey the GNU General Public License in all respects for all
// of the code used other than OpenSSL. If you modify file(s) with this
// exception, you may extend this exception to your version of the
// file(s), but you are not obligated to do so. If you do not wish to do
// so, delete this exception statement from your version. If you delete
// this exception statement from all source files in the program, then
// also delete it here.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
//
// web.cc: implementation of web.h interface.
#include "web.h"
#include <glog/logging.h>
#include <json/value.h>
#include <json/writer.h>
#include <re2/re2.h>
#include "recording.h"
#include "string.h"
namespace moonfire_nvr {
namespace {
static const char kJsonMimeType[] = "application/json";
bool ParseOptionalStartAndEnd(const QueryParameters &params,
int64_t *start_time_90k, int64_t *end_time_90k) {
*start_time_90k = std::numeric_limits<int64_t>::min();
*end_time_90k = std::numeric_limits<int64_t>::max();
if (!params.ok() ||
(params.Get("start_time_90k") != nullptr &&
!Atoi64(params.Get("start_time_90k"), 10, start_time_90k)) ||
(params.Get("end_time_90k") != nullptr &&
!Atoi64(params.Get("end_time_90k"), 10, end_time_90k))) {
return false;
}
return true;
}
void ReplyWithJson(evhttp_request *req, const Json::Value &value) {
EvBuffer buf;
buf.Add(Json::FastWriter().write(value));
evhttp_add_header(evhttp_request_get_output_headers(req), "Content-Type",
kJsonMimeType);
evhttp_send_reply(req, HTTP_OK, "OK", buf.get());
}
} // namespace
void WebInterface::Register(evhttp *http) {
evhttp_set_gencb(http, &WebInterface::DispatchHttpRequest, this);
}
void WebInterface::DispatchHttpRequest(evhttp_request *req, void *arg) {
static const RE2 kCameraUri("/cameras/([^/]+)/");
static const RE2 kCameraRecordingsUri("/cameras/([^/]+)/recordings");
static const RE2 kCameraViewUri("/cameras/([^/]+)/view.mp4");
re2::StringPiece accept =
evhttp_find_header(evhttp_request_get_input_headers(req), "Accept");
bool json = accept == kJsonMimeType;
auto *this_ = reinterpret_cast<WebInterface *>(arg);
const evhttp_uri *uri = evhttp_request_get_evhttp_uri(req);
re2::StringPiece path = evhttp_uri_get_path(uri);
re2::StringPiece camera_uuid_str;
Uuid camera_uuid;
if (path == "/" || path == "/cameras/") {
if (json) {
this_->HandleJsonCameraList(req);
} else {
this_->HandleHtmlCameraList(req);
}
} else if (RE2::FullMatch(path, kCameraUri, &camera_uuid_str) &&
camera_uuid.ParseText(camera_uuid_str)) {
if (json) {
this_->HandleJsonCameraDetail(req, camera_uuid);
} else {
this_->HandleHtmlCameraDetail(req, camera_uuid);
}
} else if (RE2::FullMatch(path, kCameraRecordingsUri, &camera_uuid_str) &&
camera_uuid.ParseText(camera_uuid_str)) {
// The HTML version includes this in the top-level camera view.
// So only support JSON at this URI.
this_->HandleJsonCameraRecordings(req, camera_uuid);
} else if (RE2::FullMatch(path, kCameraViewUri, &camera_uuid_str) &&
camera_uuid.ParseText(camera_uuid_str)) {
this_->HandleMp4View(req, camera_uuid);
} else {
evhttp_send_error(req, HTTP_NOTFOUND, "path not understood");
}
}
void WebInterface::HandleHtmlCameraList(evhttp_request *req) {
EvBuffer buf;
buf.Add(
"<!DOCTYPE html>\n"
"<html>\n"
"<head>\n"
"<title>Camera list</title>\n"
"<meta http-equiv=\"Content-Language\" content=\"en\">\n"
"<style type=\"text/css\">\n"
".header { background-color: #ddd; }\n"
"td { padding-right: 3em; }\n"
"</style>\n"
"</head>\n"
"<body>\n"
"<table>\n");
auto row_cb = [&](const ListCamerasRow &row) {
auto seconds = row.total_duration_90k / kTimeUnitsPerSecond;
std::string min_start_time_90k =
row.min_start_time_90k == -1 ? std::string("n/a")
: PrettyTimestamp(row.min_start_time_90k);
std::string max_end_time_90k = row.max_end_time_90k == -1
? std::string("n/a")
: PrettyTimestamp(row.max_end_time_90k);
buf.AddPrintf(
"<tr class=header><td colspan=2><a href=\"/cameras/%s/\">%s</a>"
"</td></tr>\n"
"<tr><td>description</td><td>%s</td></tr>\n"
"<tr><td>space</td><td>%s / %s (%.1f%%)</td></tr>\n"
"<tr><td>uuid</td><td>%s</td></tr>\n"
"<tr><td>oldest recording</td><td>%s</td></tr>\n"
"<tr><td>newest recording</td><td>%s</td></tr>\n"
"<tr><td>total duration</td><td>%s</td></tr>\n",
row.uuid.UnparseText().c_str(), EscapeHtml(row.short_name).c_str(),
EscapeHtml(row.description).c_str(),
EscapeHtml(HumanizeWithBinaryPrefix(row.total_sample_file_bytes, "B"))
.c_str(),
EscapeHtml(HumanizeWithBinaryPrefix(row.retain_bytes, "B")).c_str(),
100.f * row.total_sample_file_bytes / row.retain_bytes,
EscapeHtml(row.uuid.UnparseText()).c_str(),
EscapeHtml(min_start_time_90k).c_str(),
EscapeHtml(max_end_time_90k).c_str(),
EscapeHtml(HumanizeDuration(seconds)).c_str());
return IterationControl::kContinue;
};
env_->mdb->ListCameras(row_cb);
buf.Add(
"</table>\n"
"</body>\n"
"<html>\n");
evhttp_send_reply(req, HTTP_OK, "OK", buf.get());
}
void WebInterface::HandleJsonCameraList(evhttp_request *req) {
Json::Value cameras(Json::arrayValue);
auto row_cb = [&](const ListCamerasRow &row) {
Json::Value camera(Json::objectValue);
camera["uuid"] = row.uuid.UnparseText();
camera["short_name"] = row.short_name;
camera["description"] = row.description;
camera["retain_bytes"] = static_cast<Json::Int64>(row.retain_bytes);
camera["total_duration_90k"] =
static_cast<Json::Int64>(row.total_duration_90k);
camera["total_sample_file_bytes"] =
static_cast<Json::Int64>(row.total_sample_file_bytes);
if (row.min_start_time_90k != -1) {
camera["min_start_time_90k"] =
static_cast<Json::Int64>(row.min_start_time_90k);
}
if (row.max_end_time_90k != -1) {
camera["max_end_time_90k"] =
static_cast<Json::Int64>(row.max_end_time_90k);
}
cameras.append(camera);
return IterationControl::kContinue;
};
env_->mdb->ListCameras(row_cb);
ReplyWithJson(req, cameras);
}
bool WebInterface::ListAggregatedCameraRecordings(
Uuid camera_uuid, int64_t start_time_90k, int64_t end_time_90k,
int64_t forced_split_duration_90k,
const std::function<void(const ListCameraRecordingsRow &)> &fn,
std::string *error_message) {
ListCameraRecordingsRow aggregated;
auto handle_sql_row = [&](const ListCameraRecordingsRow &row) {
auto new_duration_90k = aggregated.end_time_90k - row.start_time_90k;
if (row.video_sample_entry_sha1 == aggregated.video_sample_entry_sha1 &&
row.end_time_90k == aggregated.start_time_90k &&
new_duration_90k < forced_split_duration_90k) {
// Append to current .mp4.
aggregated.start_time_90k = row.start_time_90k;
aggregated.video_samples += row.video_samples;
aggregated.sample_file_bytes += row.sample_file_bytes;
} else {
// Start a new .mp4.
if (aggregated.start_time_90k != -1) { fn(aggregated); }
aggregated = row;
}
return IterationControl::kContinue;
};
if (!env_->mdb->ListCameraRecordings(camera_uuid, start_time_90k,
end_time_90k, handle_sql_row,
error_message)) {
return false;
}
if (aggregated.start_time_90k != -1) { fn(aggregated); }
return true;
}
void WebInterface::HandleHtmlCameraDetail(evhttp_request *req,
Uuid camera_uuid) {
GetCameraRow camera_row;
if (!env_->mdb->GetCamera(camera_uuid, &camera_row)) {
return evhttp_send_error(req, HTTP_NOTFOUND, "no such camera");
}
int64_t start_time_90k;
int64_t end_time_90k;
QueryParameters params(evhttp_request_get_uri(req));
if (!ParseOptionalStartAndEnd(params, &start_time_90k, &end_time_90k)) {
return evhttp_send_error(req, HTTP_BADREQUEST, "bad query parameters");
}
EvBuffer buf;
buf.AddPrintf(
"<!DOCTYPE html>\n"
"<html>\n"
"<head>\n"
"<title>%s recordings</title>\n"
"<meta http-equiv=\"Content-Language\" content=\"en\">\n"
"<style type=\"text/css\">\n"
"tr:not(:first-child):hover { background-color: #ddd; }\n"
"th, td { padding: 0.5ex 1.5em; text-align: right; }\n"
"</style>\n"
"</head>\n"
"<body>\n"
"<h1>%s</h1>\n"
"<p>%s</p>\n"
"<table>\n"
"<tr><th>start</th><th>end</th><th>resolution</th>"
"<th>fps</th><th>size</th><th>bitrate</th>"
"</tr>\n",
EscapeHtml(camera_row.short_name).c_str(),
EscapeHtml(camera_row.short_name).c_str(),
EscapeHtml(camera_row.description).c_str());
// Rather than listing each 60-second recording, generate a HTML row for
// aggregated .mp4 files of up to kForceSplitDuration90k each, provided
// there is no gap or change in video parameters between recordings.
static const int64_t kForceSplitDuration90k = 60 * 60 * kTimeUnitsPerSecond;
auto finish_html_row = [&](const ListCameraRecordingsRow &aggregated) {
auto seconds = static_cast<float>(aggregated.end_time_90k -
aggregated.start_time_90k) /
kTimeUnitsPerSecond;
buf.AddPrintf(
"<tr><td><a href=\"view.mp4?start_time_90k=%" PRId64
"&end_time_90k=%" PRId64
"\">%s</a></td><td>%s</td><td>%dx%d</td>"
"<td>%.0f</td><td>%s</td><td>%s</td></tr>\n",
aggregated.start_time_90k, aggregated.end_time_90k,
PrettyTimestamp(aggregated.start_time_90k).c_str(),
PrettyTimestamp(aggregated.end_time_90k).c_str(),
static_cast<int>(aggregated.width), static_cast<int>(aggregated.height),
static_cast<float>(aggregated.video_samples) / seconds,
HumanizeWithBinaryPrefix(aggregated.sample_file_bytes, "B").c_str(),
HumanizeWithDecimalPrefix(
static_cast<float>(aggregated.sample_file_bytes) * 8 / seconds,
"bps")
.c_str());
};
std::string error_message;
if (!ListAggregatedCameraRecordings(camera_uuid, start_time_90k,
end_time_90k, kForceSplitDuration90k,
finish_html_row, &error_message)) {
return evhttp_send_error(
req, HTTP_INTERNAL,
StrCat("sqlite query failed: ", EscapeHtml(error_message)).c_str());
}
buf.Add(
"</table>\n"
"</html>\n");
evhttp_send_reply(req, HTTP_OK, "OK", buf.get());
}
void WebInterface::HandleJsonCameraDetail(evhttp_request *req,
Uuid camera_uuid) {
GetCameraRow camera_row;
if (!env_->mdb->GetCamera(camera_uuid, &camera_row)) {
return evhttp_send_error(req, HTTP_NOTFOUND, "no such camera");
}
Json::Value camera(Json::objectValue);
camera["short_name"] = camera_row.short_name;
camera["description"] = camera_row.description;
camera["retain_bytes"] = static_cast<Json::Int64>(camera_row.retain_bytes);
camera["total_duration_90k"] =
static_cast<Json::Int64>(camera_row.total_duration_90k);
camera["total_sample_file_bytes"] =
static_cast<Json::Int64>(camera_row.total_sample_file_bytes);
if (camera_row.min_start_time_90k != std::numeric_limits<int64_t>::max()) {
camera["min_start_time_90k"] =
static_cast<Json::Int64>(camera_row.min_start_time_90k);
}
if (camera_row.max_end_time_90k != std::numeric_limits<int64_t>::min()) {
camera["max_end_time_90k"] =
static_cast<Json::Int64>(camera_row.max_end_time_90k);
}
Json::Value days(Json::objectValue);
std::string error_message;
for (const auto &day : camera_row.days) {
int64_t start_time_90k;
int64_t end_time_90k;
if (!GetDayBounds(day.first, &start_time_90k, &end_time_90k,
&error_message)) {
return evhttp_send_error(
req, HTTP_INTERNAL,
StrCat("internal error: ", EscapeHtml(error_message)).c_str());
}
Json::Value day_val(Json::objectValue);
day_val["start_time_90k"] = static_cast<Json::Int64>(start_time_90k);
day_val["end_time_90k"] = static_cast<Json::Int64>(end_time_90k);
day_val["total_duration_90k"] = static_cast<Json::Int64>(day.second);
days[day.first] = day_val;
}
camera["days"] = days;
ReplyWithJson(req, camera);
}
void WebInterface::HandleJsonCameraRecordings(evhttp_request *req,
Uuid camera_uuid) {
int64_t start_time_90k;
int64_t end_time_90k;
QueryParameters params(evhttp_request_get_uri(req));
if (!ParseOptionalStartAndEnd(params, &start_time_90k, &end_time_90k)) {
return evhttp_send_error(req, HTTP_BADREQUEST, "bad query parameters");
}
GetCameraRow camera_row;
if (!env_->mdb->GetCamera(camera_uuid, &camera_row)) {
return evhttp_send_error(req, HTTP_NOTFOUND, "no such camera");
}
// TODO(slamb): paging support.
Json::Value recordings(Json::arrayValue);
auto handle_row = [&](const ListCameraRecordingsRow &row) {
Json::Value recording(Json::objectValue);
recording["end_time_90k"] = static_cast<Json::Int64>(row.end_time_90k);
recording["start_time_90k"] = static_cast<Json::Int64>(row.start_time_90k);
recording["video_samples"] = static_cast<Json::Int64>(row.video_samples);
recording["sample_file_bytes"] =
static_cast<Json::Int64>(row.sample_file_bytes);
recording["video_sample_entry_sha1"] = ToHex(row.video_sample_entry_sha1);
recording["video_sample_entry_width"] = row.width;
recording["video_sample_entry_height"] = row.height;
recordings.append(recording);
return IterationControl::kContinue;
};
std::string error_message;
const auto kForceSplitDuration90k = std::numeric_limits<int64_t>::max();
if (!ListAggregatedCameraRecordings(camera_uuid, start_time_90k,
end_time_90k, kForceSplitDuration90k,
handle_row, &error_message)) {
return evhttp_send_error(
req, HTTP_INTERNAL,
StrCat("sqlite query failed: ", EscapeHtml(error_message)).c_str());
}
Json::Value response(Json::objectValue);
response["recordings"] = recordings;
ReplyWithJson(req, response);
}
void WebInterface::HandleMp4View(evhttp_request *req, Uuid camera_uuid) {
int64_t start_time_90k;
int64_t end_time_90k;
QueryParameters params(evhttp_request_get_uri(req));
if (!params.ok() ||
!Atoi64(params.Get("start_time_90k"), 10, &start_time_90k) ||
!Atoi64(params.Get("end_time_90k"), 10, &end_time_90k) ||
start_time_90k < 0 || start_time_90k >= end_time_90k) {
return evhttp_send_error(req, HTTP_BADREQUEST, "bad query parameters");
}
bool include_ts = re2::StringPiece(params.Get("ts")) == "true";
std::string error_message;
auto file = BuildMp4(camera_uuid, start_time_90k, end_time_90k, include_ts,
&error_message);
if (file == nullptr) {
// TODO: more nuanced HTTP status codes.
LOG(WARNING) << "BuildMp4 failed: " << error_message;
return evhttp_send_error(req, HTTP_INTERNAL,
EscapeHtml(error_message).c_str());
}
return HttpServe(file, req);
}
std::shared_ptr<VirtualFile> WebInterface::BuildMp4(
Uuid camera_uuid, int64_t start_time_90k, int64_t end_time_90k,
bool include_ts, std::string *error_message) {
LOG(INFO) << "Building mp4 for camera: " << camera_uuid.UnparseText()
<< ", start_time_90k: " << start_time_90k
<< ", end_time_90k: " << end_time_90k;
Mp4FileBuilder builder(env_->sample_file_dir);
int64_t next_row_start_time_90k = start_time_90k;
int64_t rows = 0;
bool ok = true;
auto row_cb = [&](Recording &recording,
const VideoSampleEntry &sample_entry) {
if (rows == 0 && recording.start_time_90k > next_row_start_time_90k) {
*error_message = StrCat(
"recording starts late: ", PrettyTimestamp(recording.start_time_90k),
" (", recording.start_time_90k, ") rather than requested: ",
PrettyTimestamp(start_time_90k), " (", start_time_90k, ")");
ok = false;
return IterationControl::kBreak;
} else if (rows > 0 &&
recording.start_time_90k != next_row_start_time_90k) {
*error_message = StrCat("gap/overlap in recording: ",
PrettyTimestamp(next_row_start_time_90k), " (",
next_row_start_time_90k, ") to: ",
PrettyTimestamp(recording.start_time_90k), " (",
recording.start_time_90k, ") before row ", rows);
ok = false;
return IterationControl::kBreak;
}
next_row_start_time_90k = recording.end_time_90k;
if (rows > 0 && recording.video_sample_entry_id != sample_entry.id) {
*error_message =
StrCat("inconsistent video sample entries: this recording has id ",
recording.video_sample_entry_id, " previous had ",
sample_entry.id, " (sha1 ", ToHex(sample_entry.sha1), ")");
ok = false;
return IterationControl::kBreak;
} else if (rows == 0) {
builder.SetSampleEntry(sample_entry);
}
int32_t rel_start_90k = 0;
int32_t rel_end_90k = std::numeric_limits<int32_t>::max();
if (recording.start_time_90k < start_time_90k) {
rel_start_90k = start_time_90k - recording.start_time_90k;
}
if (recording.end_time_90k > end_time_90k) {
rel_end_90k = end_time_90k - recording.start_time_90k;
}
builder.Append(std::move(recording), rel_start_90k, rel_end_90k);
++rows;
return IterationControl::kContinue;
};
if (!env_->mdb->ListMp4Recordings(camera_uuid, start_time_90k, end_time_90k,
row_cb, error_message) ||
!ok) {
return false;
}
if (rows == 0) {
*error_message = StrCat("no recordings in range");
return false;
}
if (next_row_start_time_90k < end_time_90k) {
*error_message = StrCat("recording ends early: ",
PrettyTimestamp(next_row_start_time_90k), " (",
next_row_start_time_90k, "), not requested: ",
PrettyTimestamp(end_time_90k), " (", end_time_90k,
") after ", rows, " rows");
return false;
}
builder.include_timestamp_subtitle_track(include_ts);
VLOG(1) << "...(3/4) building VirtualFile from " << rows << " recordings.";
auto file = builder.Build(error_message);
if (file == nullptr) {
return false;
}
VLOG(1) << "...(4/4) success, " << file->size() << " bytes, etag "
<< file->etag();
return file;
}
} // namespace moonfire_nvr

92
src/web.h
View File

@ -1,92 +0,0 @@
// This file is part of Moonfire NVR, a security camera network video recorder.
// Copyright (C) 2016 Scott Lamb <slamb@slamb.org>
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// In addition, as a special exception, the copyright holders give
// permission to link the code of portions of this program with the
// OpenSSL library under certain conditions as described in each
// individual source file, and distribute linked combinations including
// the two.
//
// You must obey the GNU General Public License in all respects for all
// of the code used other than OpenSSL. If you modify file(s) with this
// exception, you may extend this exception to your version of the
// file(s), but you are not obligated to do so. If you do not wish to do
// so, delete this exception statement from your version. If you delete
// this exception statement from all source files in the program, then
// also delete it here.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
//
// web.h: web (HTTP/HTML/JSON) interface to the SQLite-based recording schema.
// See design/api.md for a description of the JSON API.
//
// In the future, the interface will be reworked for tighter integration with
// the recording system to support more features:
//
// * including the recording currently being written in the web interface
// * subscribing to changes
// * reconfiguring the recording system, such as
// adding/removing/starting/stopping/editing cameras
// * showing thumbnails of the latest key frame from each camera
// * ...
#ifndef MOONFIRE_NVR_WEB_H
#define MOONFIRE_NVR_WEB_H
#include <string>
#include <event2/http.h>
#include "moonfire-db.h"
#include "moonfire-nvr.h"
#include "http.h"
namespace moonfire_nvr {
class WebInterface {
public:
explicit WebInterface(Environment *env) : env_(env) {}
WebInterface(const WebInterface &) = delete;
void operator=(const WebInterface &) = delete;
void Register(evhttp *http);
private:
static void DispatchHttpRequest(evhttp_request *req, void *arg);
void HandleHtmlCameraList(evhttp_request *req);
void HandleJsonCameraList(evhttp_request *req);
void HandleHtmlCameraDetail(evhttp_request *req, Uuid camera_uuid);
void HandleJsonCameraDetail(evhttp_request *req, Uuid camera_uuid);
void HandleJsonCameraRecordings(evhttp_request *req, Uuid camera_uuid);
void HandleMp4View(evhttp_request *req, Uuid camera_uuid);
bool ListAggregatedCameraRecordings(
Uuid camera_uuid, int64_t start_time_90k, int64_t end_time_90k,
int64_t forced_split_duration_90k,
const std::function<void (const ListCameraRecordingsRow &)> &fn,
std::string *error_message);
// TODO: more nuanced error code for HTTP.
std::shared_ptr<VirtualFile> BuildMp4(Uuid camera_uuid,
int64_t start_time_90k,
int64_t end_time_90k, bool include_ts,
std::string *error_message);
Environment *const env_;
};
} // namespace moonfire_nvr
#endif // MOONFIRE_NVR_WEB_H

449
src/web.rs Normal file
View File

@ -0,0 +1,449 @@
// This file is part of Moonfire NVR, a security camera digital video recorder.
// Copyright (C) 2016 Scott Lamb <slamb@slamb.org>
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// In addition, as a special exception, the copyright holders give
// permission to link the code of portions of this program with the
// OpenSSL library under certain conditions as described in each
// individual source file, and distribute linked combinations including
// the two.
//
// You must obey the GNU General Public License in all respects for all
// of the code used other than OpenSSL. If you modify file(s) with this
// exception, you may extend this exception to your version of the
// file(s), but you are not obligated to do so. If you do not wish to do
// so, delete this exception statement from your version. If you delete
// this exception statement from all source files in the program, then
// also delete it here.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
extern crate hyper;
use core::borrow::Borrow;
use core::str::FromStr;
use db;
use dir::SampleFileDir;
use error::{Error, Result};
use hyper::{header,server,status};
use hyper::uri::RequestUri;
use mime;
use mp4;
use recording;
use resource;
use serde_json;
use serde::ser::Serializer;
use std::collections::BTreeMap;
use std::fmt;
use std::io::Write;
use std::result;
use std::sync::{Arc,MutexGuard};
use time;
use url::form_urlencoded;
use uuid::Uuid;
const BINARY_PREFIXES: &'static [&'static str] = &[" ", " Ki", " Mi", " Gi", " Ti", " Pi", " Ei"];
const DECIMAL_PREFIXES: &'static [&'static str] =&[" ", " k", " M", " G", " T", " P", " E"];
lazy_static! {
static ref JSON: mime::Mime = mime!(Application/Json);
static ref HTML: mime::Mime = mime!(Text/Html);
}
enum Path {
CamerasList, // "/" or "/cameras/"
Camera(Uuid), // "/cameras/<uuid>/"
CameraRecordings(Uuid), // "/cameras/<uuid>/recordings"
CameraViewMp4(Uuid), // "/cameras/<uuid>/view.mp4"
NotFound,
}
fn get_path_and_query(uri: &RequestUri) -> (&str, &str) {
match *uri {
RequestUri::AbsolutePath(ref both) => match both.find('?') {
Some(split) => (&both[..split], &both[split+1..]),
None => (both, ""),
},
RequestUri::AbsoluteUri(ref u) => (u.path(), u.query().unwrap_or("")),
_ => ("", ""),
}
}
fn decode_path(path: &str) -> Path {
if path == "/" {
return Path::CamerasList;
}
if !path.starts_with("/cameras/") {
return Path::NotFound;
}
let path = &path["/cameras/".len()..];
if path == "" {
return Path::CamerasList;
}
let slash = match path.find('/') {
None => { return Path::NotFound; },
Some(s) => s,
};
let (uuid, path) = path.split_at(slash);
// TODO(slamb): require uuid to be in canonical format.
let uuid = match Uuid::parse_str(uuid) {
Ok(u) => u,
Err(_) => { return Path::NotFound },
};
match path {
"/" => Path::Camera(uuid),
"/recordings" => Path::CameraRecordings(uuid),
"/view.mp4" => Path::CameraViewMp4(uuid),
_ => Path::NotFound,
}
}
fn is_json(req: &server::Request) -> bool {
if let Some(accept) = req.headers.get::<header::Accept>() {
return accept.len() == 1 && accept[0].item == *JSON &&
accept[0].quality == header::Quality(1000);
}
false
}
pub struct HtmlEscaped<'a>(&'a str);
impl<'a> fmt::Display for HtmlEscaped<'a> {
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
let mut last_end = 0;
for (start, part) in self.0.match_indices(|c| c == '<' || c == '&') {
try!(f.write_str(&self.0[last_end..start]));
try!(f.write_str(if part == "<" { "&lt;" } else { "&amp;" }));
last_end = start + 1;
}
f.write_str(&self.0[last_end..])
}
}
pub struct Humanized(i64);
impl Humanized {
fn do_fmt(&self, base: f32, prefixes: &[&str], f: &mut fmt::Formatter) -> fmt::Result {
let mut n = self.0 as f32;
let mut i = 0;
loop {
if n < base || i >= prefixes.len() - 1 {
break;
}
n /= base;
i += 1;
}
write!(f, "{:.1}{}", n, prefixes[i])
}
}
impl fmt::Display for Humanized {
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
self.do_fmt(1000., DECIMAL_PREFIXES, f)
}
}
impl fmt::Binary for Humanized {
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
self.do_fmt(1024., BINARY_PREFIXES, f)
}
}
pub struct HumanizedTimestamp(Option<recording::Time>);
impl fmt::Display for HumanizedTimestamp {
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
match self.0 {
None => f.write_str("n/a"),
Some(t) => {
let tm = time::at(time::Timespec{sec: t.unix_seconds(), nsec: 0});
write!(f, "{}",
try!(tm.strftime("%a, %d %b %Y %H:%M:%S %Z").or_else(|_| Err(fmt::Error))))
}
}
}
}
pub struct Handler {
db: Arc<db::Database>,
dir: Arc<SampleFileDir>,
}
#[derive(Serialize)]
struct ListCameras<'a> {
// Use a custom serializer which presents the map's values as a sequence.
#[serde(serialize_with = "ListCameras::serialize_cameras")]
cameras: &'a BTreeMap<i32, db::Camera>,
}
impl<'a> ListCameras<'a> {
fn serialize_cameras<S>(cameras: &BTreeMap<i32, db::Camera>,
serializer: &mut S) -> result::Result<(), S::Error>
where S: Serializer {
let mut state = try!(serializer.serialize_seq(Some(cameras.len())));
for c in cameras.values() {
try!(serializer.serialize_seq_elt(&mut state, c));
}
serializer.serialize_seq_end(state)
}
}
impl Handler {
pub fn new(db: Arc<db::Database>, dir: Arc<SampleFileDir>) -> Self {
Handler{db: db, dir: dir}
}
fn not_found(&self, mut res: server::Response) -> Result<()> {
*res.status_mut() = status::StatusCode::NotFound;
try!(res.send(b"not found"));
Ok(())
}
fn list_cameras(&self, req: &server::Request, mut res: server::Response) -> Result<()> {
let json = is_json(req);
let buf = {
let db = self.db.lock();
if json {
try!(serde_json::to_vec(&ListCameras{cameras: db.cameras_by_id()}))
} else {
try!(self.list_cameras_html(db))
}
};
res.headers_mut().set(header::ContentType(if json { JSON.clone() } else { HTML.clone() }));
try!(res.send(&buf));
Ok(())
}
fn list_cameras_html(&self, db: MutexGuard<db::LockedDatabase>) -> Result<Vec<u8>> {
let mut buf = Vec::new();
buf.extend_from_slice(b"\
<!DOCTYPE html>\n\
<html>\n\
<head>\n\
<title>Camera list</title>\n\
<meta http-equiv=\"Content-Language\" content=\"en\">\n\
<style type=\"text/css\">\n\
.header { background-color: #ddd; }\n\
td { padding-right: 3em; }\n\
</style>\n\
</head>\n\
<body>\n\
<table>\n");
for row in db.cameras_by_id().values() {
try!(write!(&mut buf, "\
<tr class=header><td colspan=2><a href=\"/cameras/{}/\">{}</a></td></tr>\n\
<tr><td>description</td><td>{}</td></tr>\n\
<tr><td>space</td><td>{:b}B / {:b}B ({:.1}%)</td></tr>\n\
<tr><td>uuid</td><td>{}</td></tr>\n\
<tr><td>oldest recording</td><td>{}</td></tr>\n\
<tr><td>newest recording</td><td>{}</td></tr>\n\
<tr><td>total duration</td><td>{}</td></tr>\n",
row.uuid, HtmlEscaped(&row.short_name), HtmlEscaped(&row.description),
Humanized(row.sample_file_bytes), Humanized(row.retain_bytes),
100. * row.sample_file_bytes as f32 / row.retain_bytes as f32,
row.uuid, HumanizedTimestamp(row.range.as_ref().map(|r| r.start)),
HumanizedTimestamp(row.range.as_ref().map(|r| r.end)),
row.duration));
}
Ok(buf)
}
fn camera(&self, uuid: Uuid, req: &server::Request, mut res: server::Response) -> Result<()> {
let json = is_json(req);
let buf = {
let db = self.db.lock();
if json {
let camera = try!(db.get_camera(uuid)
.ok_or_else(|| Error::new("no such camera".to_owned())));
try!(serde_json::to_vec(&camera))
} else {
try!(self.camera_html(db, uuid))
}
};
res.headers_mut().set(header::ContentType(if json { JSON.clone() } else { HTML.clone() }));
try!(res.send(&buf));
Ok(())
}
fn camera_html(&self, db: MutexGuard<db::LockedDatabase>, uuid: Uuid) -> Result<Vec<u8>> {
let camera = try!(db.get_camera(uuid)
.ok_or_else(|| Error::new("no such camera".to_owned())));
let mut buf = Vec::new();
try!(write!(&mut buf, "\
<!DOCTYPE html>\n\
<html>\n\
<head>\n\
<title>{0} recordings</title>\n\
<meta http-equiv=\"Content-Language\" content=\"en\">\n\
<style type=\"text/css\">\n\
tr:not(:first-child):hover {{ background-color: #ddd; }}\n\
th, td {{ padding: 0.5ex 1.5em; text-align: right; }}\n\
</style>\n\
</head>\n\
<body>\n\
<h1>{0}</h1>\n\
<p>{1}</p>\n\
<table>\n\
<tr><th>start</th><th>end</th><th>resolution</th>\
<th>fps</th><th>size</th><th>bitrate</th>\
</tr>\n",
HtmlEscaped(&camera.short_name), HtmlEscaped(&camera.description)));
let r = recording::Time(i64::min_value()) .. recording::Time(i64::max_value());
// Rather than listing each 60-second recording, generate a HTML row for aggregated .mp4
// files of up to FORCE_SPLIT_DURATION each, provided there is no gap or change in video
// parameters between recordings.
static FORCE_SPLIT_DURATION: recording::Duration =
recording::Duration(60 * 60 * recording::TIME_UNITS_PER_SEC);
try!(db.list_aggregated_recordings(camera.id, &r, FORCE_SPLIT_DURATION, |row| {
let seconds = (row.range.end.0 - row.range.start.0) / recording::TIME_UNITS_PER_SEC;
try!(write!(&mut buf, "\
<tr><td><a href=\"view.mp4?start_time_90k={}&end_time_90k={}\">{}</a></td>\
<td>{}</td><td>{}x{}</td><td>{:.0}</td><td>{:b}B</td><td>{}bps</td></tr>\n",
row.range.start.0, row.range.end.0,
HumanizedTimestamp(Some(row.range.start)),
HumanizedTimestamp(Some(row.range.end)), row.video_sample_entry.width,
row.video_sample_entry.height,
if seconds == 0 { 0. } else { row.video_samples as f32 / seconds as f32 },
Humanized(row.sample_file_bytes),
Humanized(if seconds == 0 { 0 } else { row.sample_file_bytes * 8 / seconds })));
Ok(())
}));
buf.extend_from_slice(b"</table>\n</html>\n");
Ok(buf)
}
fn camera_recordings(&self, _uuid: Uuid, _req: &server::Request,
mut res: server::Response) -> Result<()> {
*res.status_mut() = status::StatusCode::NotImplemented;
try!(res.send(b"not implemented"));
Ok(())
}
fn camera_view_mp4(&self, uuid: Uuid, query: &str, req: &server::Request,
res: server::Response) -> Result<()> {
let camera_id = {
let db = self.db.lock();
let camera = try!(db.get_camera(uuid)
.ok_or_else(|| Error::new("no such camera".to_owned())));
camera.id
};
let mut start = None;
let mut end = None;
let mut include_ts = false;
for (key, value) in form_urlencoded::parse(query.as_bytes()) {
let (key, value) = (key.borrow(), value.borrow());
match key {
"start_time_90k" => start = Some(recording::Time(try!(i64::from_str(value)))),
"end_time_90k" => end = Some(recording::Time(try!(i64::from_str(value)))),
"ts" => { include_ts = value == "true"; },
_ => {},
}
};
let start = try!(start.ok_or_else(|| Error::new("start_time_90k missing".to_owned())));
let end = try!(end.ok_or_else(|| Error::new("end_time_90k missing".to_owned())));
let desired_range = start .. end;
let mut builder = mp4::Mp4FileBuilder::new();
// There should be roughly ceil((end - start) / desired_recording_duration) recordings
// in the desired timespan if there are no gaps or overlap. Add a couple more to be safe:
// one for misalignment of the requested timespan with the rotate offset, another because
// rotation only happens at key frames.
let ceil_durations = ((end - start).0 + recording::DESIRED_RECORDING_DURATION - 1) /
recording::DESIRED_RECORDING_DURATION;
let est_records = (ceil_durations + 2) as usize;
let mut next_start = start;
builder.reserve(est_records);
{
let db = self.db.lock();
try!(db.list_recordings(camera_id, &desired_range, |r| {
if builder.len() == 0 && r.start > next_start {
return Err(Error::new(format!("recording started late ({} vs requested {})",
r.start, start)));
} else if builder.len() != 0 && r.start != next_start {
return Err(Error::new(format!("gap/overlap in recording: {} to {} after row {}",
next_start, r.start, builder.len())));
}
next_start = r.start + recording::Duration(r.duration_90k as i64);
// TODO: check for inconsistent video sample entries.
let rel_start = if r.start < start {
(start - r.start).0 as i32
} else {
0
};
let rel_end = if r.start + recording::Duration(r.duration_90k as i64) > end {
(end - r.start).0 as i32
} else {
r.duration_90k
};
builder.append(r, rel_start .. rel_end);
Ok(())
}));
}
if next_start < end {
return Err(Error::new(format!(
"recording ends early: {}, not requested: {} after {} rows.",
next_start, end, builder.len())))
}
if builder.len() > est_records {
warn!("Estimated {} records for time [{}, {}); actually were {}",
est_records, start, end, builder.len());
} else {
debug!("Estimated {} records for time [{}, {}); actually were {}",
est_records, start, end, builder.len());
}
builder.include_timestamp_subtitle_track(include_ts);
let mp4 = try!(builder.build(self.db.clone(), self.dir.clone()));
try!(resource::serve(&mp4, req, res));
Ok(())
}
}
impl server::Handler for Handler {
fn handle(&self, req: server::Request, res: server::Response) {
let (path, query) = get_path_and_query(&req.uri);
let res = match decode_path(path) {
Path::CamerasList => self.list_cameras(&req, res),
Path::Camera(uuid) => self.camera(uuid, &req, res),
Path::CameraRecordings(uuid) => self.camera_recordings(uuid, &req, res),
Path::CameraViewMp4(uuid) => self.camera_view_mp4(uuid, query, &req, res),
Path::NotFound => self.not_found(res),
};
if let Err(ref e) = res {
warn!("Error handling request: {}", e);
}
}
}
#[cfg(test)]
mod tests {
use super::{HtmlEscaped, Humanized};
#[test]
fn test_humanize() {
assert_eq!("1.0 B", format!("{:b}B", Humanized(1)));
assert_eq!("1.0 EiB", format!("{:b}B", Humanized(1i64 << 60)));
assert_eq!("1.5 EiB", format!("{:b}B", Humanized((1i64 << 60) + (1i64 << 59))));
assert_eq!("8.0 EiB", format!("{:b}B", Humanized(i64::max_value())));
assert_eq!("1.0 Mbps", format!("{}bps", Humanized(1_000_000)));
}
#[test]
fn test_html_escaped() {
assert_eq!("", format!("{}", HtmlEscaped("")));
assert_eq!("no special chars", format!("{}", HtmlEscaped("no special chars")));
assert_eq!("a &lt;tag> &amp; text", format!("{}", HtmlEscaped("a <tag> & text")));
}
}