diff --git a/.gitignore b/.gitignore index 5bb3549..1263479 100644 --- a/.gitignore +++ b/.gitignore @@ -1,13 +1,6 @@ -*.swp -build -debug -release -obj-* cameras.sql -debian/files -debian/moonfire-nvr.debhelper.log -debian/moonfire-nvr.postinst.debhelper -debian/moonfire-nvr.postrm.debhelper -debian/moonfire-nvr.prerm.debhelper -debian/moonfire-nvr.substvars -debian/moonfire-nvr/ +Cargo.lock +.project +.settings +*.swp +target diff --git a/CMakeLists.txt b/CMakeLists.txt deleted file mode 100644 index 0b16b2d..0000000 --- a/CMakeLists.txt +++ /dev/null @@ -1,154 +0,0 @@ -# This file is part of Moonfire NVR, a security camera digital video recorder. -# Copyright (C) 2016 Scott Lamb -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# In addition, as a special exception, the copyright holders give -# permission to link the code of portions of this program with the -# OpenSSL library under certain conditions as described in each -# individual source file, and distribute linked combinations including -# the two. -# -# You must obey the GNU General Public License in all respects for all -# of the code used other than OpenSSL. If you modify file(s) with this -# exception, you may extend this exception to your version of the -# file(s), but you are not obligated to do so. If you do not wish to do -# so, delete this exception statement from your version. If you delete -# this exception statement from all source files in the program, then -# also delete it here. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . -# -# CMakeLists.txt: top-level definitions for building Moonfire NVR. - -cmake_minimum_required(VERSION 3.0.2) -project(moonfire-nvr) - -if(CMAKE_VERSION VERSION_LESS "3.1") - set(CMAKE_CXX_FLAGS "--std=c++11 ${CMAKE_CXX_FLAGS}") -else() - set(CMAKE_CXX_STANDARD 11) -endif() - -set(CMAKE_CXX_FLAGS "-Wall -Werror -pedantic-errors -ggdb ${CMAKE_CXX_FLAGS}") - -option(LTO "Use link-time optimization" ON) -option(FPROFILE_GENERATE "Compile executable to generate usage data" OFF) -option(FPROFILE_USE "Compile executable using generated usage data" OFF) - -if(LTO) - set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} -flto") - set(CMAKE_AR "gcc-ar") - set(CMAKE_RANLIB "gcc-ranlib") - set(CMAKE_LD "gcc-ld") -endif() - -if(PROFILE_GENERATE) - set(CMAKE_CXX_FLAGS "-fprofile-generate ${CMAKE_CXX_FLAGS}") -endif() -if(PROFILE_USE) - set(CMAKE_CXX_FLAGS "-fprofile-use -fprofile-correction -Wno-error=coverage-mismatch ${CMAKE_CXX_FLAGS}") -endif() - -# -# Dependencies. -# - -find_package(Threads REQUIRED) - -# https://gflags.github.io/gflags/#cmake mentions a cmake module, but at -# least on Ubuntu 15.10, libgflags-dev does not include it. There's no -# pkgconfig either. Do this by hand. -find_library(GFLAGS_LIBRARIES gflags) -find_library(RE2_LIBRARIES re2) -find_library(PROFILER_LIBRARIES profiler) - -# https://cmake.org/cmake/help/v3.0/module/FindPkgConfig.html -find_package(PkgConfig) -pkg_check_modules(FFMPEG REQUIRED libavutil libavcodec libavformat) -pkg_check_modules(LIBEVENT REQUIRED libevent>=2.1) -pkg_check_modules(JSONCPP REQUIRED jsoncpp) -pkg_check_modules(GLOG REQUIRED libglog) -pkg_check_modules(OPENSSL REQUIRED libcrypto) -pkg_check_modules(SQLITE REQUIRED sqlite3) -pkg_check_modules(UUID REQUIRED uuid) - -# Check if ffmpeg support "stimeout". -set(CMAKE_REQUIRED_INCLUDES ${FFMPEG_INCLUDES}) -set(CMAKE_REQUIRED_LIBRARIES ${FFMPEG_LIBRARIES}) -include(CheckCSourceRuns) -check_c_source_runs([[ -#include -#include -#include -int main(int argc, char **argv) { - av_register_all(); - AVInputFormat *input = av_find_input_format("rtsp"); - const AVClass *klass = input->priv_class; - const AVOption *opt = - av_opt_find2(&klass, "stimeout", NULL, 0, AV_OPT_SEARCH_FAKE_OBJ, NULL); - return (opt != NULL) ? EXIT_SUCCESS : EXIT_FAILURE; -} -]] HAVE_STIMEOUT) -if(NOT HAVE_STIMEOUT) - message(WARNING [[ -Your libavformat library lacks support for the "stimeout" rtsp option. -Moonfire NVR will not be able to detect network partitions or retry. -Consider installing a recent ffmpeg, from source if necessary. -]]) -else() - message(STATUS "libavformat library has support for \"stimeout\" - good.") -endif() - -enable_testing() - -# http://www.kaizou.org/2014/11/gtest-cmake/ -include(ExternalProject) -ExternalProject_Add( - GTestProject - URL "https://github.com/google/googletest/archive/release-1.8.0.tar.gz" - URL_HASH "SHA1=e7e646a6204638fe8e87e165292b8dd9cd4c36ed" - INSTALL_COMMAND "") -ExternalProject_Get_Property(GTestProject source_dir binary_dir) -set(GTest_INCLUDE_DIR ${source_dir}/googletest/include) -add_library(GTest STATIC IMPORTED) -add_dependencies(GTest GTestProject) -set_target_properties(GTest PROPERTIES - IMPORTED_LOCATION "${binary_dir}/googlemock/gtest/${CMAKE_STATIC_LIBRARY_PREFIX}gtest${CMAKE_STATIC_LIBRARY_SUFFIX}" - IMPORTED_LINK_INTERFACE_LIBRARIES "${CMAKE_THREAD_LIBS_INIT}") -set(GMock_INCLUDE_DIR ${source_dir}/googlemock/include) -add_library(GMock STATIC IMPORTED) -add_dependencies(GMock GTestProject) -set_target_properties(GMock PROPERTIES - IMPORTED_LOCATION "${binary_dir}/googlemock/${CMAKE_STATIC_LIBRARY_PREFIX}gmock${CMAKE_STATIC_LIBRARY_SUFFIX}" - IMPORTED_LINK_INTERFACE_LIBRARIES "${CMAKE_THREAD_LIBS_INIT}") - -ExternalProject_Add( - GBenchmarkProject - URL "https://github.com/google/benchmark/archive/v1.0.0.tar.gz" - URL_HASH "SHA1=4f778985dce02d2e63262e6f388a24b595254a93" - CMAKE_ARGS - -DCMAKE_BUILD_TYPE:STRING=${CMAKE_BUILD_TYPE} - INSTALL_COMMAND "") -ExternalProject_Get_Property(GBenchmarkProject source_dir binary_dir) -set(GBenchmark_INCLUDE_DIR ${source_dir}/include) -add_library(GBenchmark STATIC IMPORTED) -add_dependencies(GBenchmark GBenchmarkProject) -set_target_properties(GBenchmark PROPERTIES - IMPORTED_LOCATION "${binary_dir}/src/${CMAKE_STATIC_LIBRARY_PREFIX}benchmark${CMAKE_STATIC_LIBRARY_SUFFIX}" - IMPORTED_LINK_INTERFACE_LIBRARIES "${CMAKE_THREAD_LIBS_INIT}") - -# -# Subdirectories. -# - -add_subdirectory(src) diff --git a/Cargo.toml b/Cargo.toml new file mode 100644 index 0000000..c138e6b --- /dev/null +++ b/Cargo.toml @@ -0,0 +1,55 @@ +[package] +name = "moonfire-nvr" +version = "0.1.0" +authors = ["Scott Lamb "] + +[dependencies] +byteorder = "0.5" +clippy = "0.0.103" +chan = "0.1" +chan-signal = "0.1" +docopt = "0.6" +fnv = "1.0" +hyper = "0.9" +lazy_static = "0.2" +libc = "0.2" +log = { version = "0.3", features = ["release_max_level_info"] } +lru-cache = "0.1" +memmap = "0.3" +mime = "0.2" +openssl = "0.8" +regex = "0.1" +rusqlite = "0.7" +rustc-serialize = "0.3" +serde = "0.8" +serde_json = "0.8" +serde_derive = "0.8" +slog = "1.2" +slog-envlogger = "0.5" +slog-stdlog = "1.1" +slog-term = "1.3" +smallvec = "0.2" +time = "0.1" +url = "1.2" +uuid = { version = "0.3", features = ["serde", "v4"] } + +[dev-dependencies] +tempdir = "0.3" + +[dependencies.ffmpeg] +version = "0.2.0-alpha.1" +default-features = false +features = ["codec", "format"] + +[dependencies.ffmpeg-sys] +version = "2.8" +default-features = false +features = ["avcodec"] + +[profile.release] +lto = true +debug = true + +[profile.bench] +lto = true +debug = true diff --git a/README.md b/README.md index cbbd3ff..f0ddcfd 100644 --- a/README.md +++ b/README.md @@ -63,59 +63,51 @@ edge version from the command line via git: # Building from source There are no binary packages of Moonfire NVR available yet, so it must be built -from source. It requires several packages to build: +from source. -* [CMake](https://cmake.org/) version 3.1.0 or higher. -* a C++11 compiler, such as [gcc](https://gcc.gnu.org/) 4.7 or higher. -* [ffmpeg](http://ffmpeg.org/), including `libavutil`, +The `rust` branch contains a rewrite into the [Rust Programming +Language](https://www.rust-lang.org/en-US/). In the long term, I expect this +will result in a more secure, full-featured, easy-to-install software. In the +short term, there will be growing pains. Rust is a new programming language. +Moonfire NVR's primary author is new to Rust. And Moonfire NVR is a young +project. + +You will need the following C libraries installed: + +* [ffmpeg](http://ffmpeg.org/) version 2.x, including `libavutil`, `libavcodec` (to inspect H.264 frames), and `libavformat` (to connect to RTSP - servers and write `.mp4` files). Note ffmpeg versions older than 55.1.101, - along with all versions of the competing project [libav](http://libav.org), - does not support socket timeouts for RTSP. For reliable reconnections on - error, it's strongly recommended to use ffmpeg >= 55.1.101. -* [libevent](http://libevent.org/) 2.1, for the built-in HTTP server. - (This might be replaced with the more full-featured - [nghttp2](https://github.com/tatsuhiro-t/nghttp2) in the future.) - Unfortunately, the libevent 2.0 bundled with current Debian releases is - unsuitable. -* [gflags](http://gflags.github.io/gflags/), for command line flag parsing. -* [glog](https://github.com/google/glog), for debug logging. -* [gperftools](https://github.com/gperftools/gperftools), for debugging. -* [googletest](https://github.com/google/googletest), for automated testing. - This will be automatically downloaded during the build process, so it's - not necessary to install it beforehand. -* [re2](https://github.com/google/re2), for parsing with regular expressions. -* libuuid from [util-linux](https://en.wikipedia.org/wiki/Util-linux). + servers and write `.mp4` files). + + Note ffmpeg 3.x isn't supported yet by the Rust `ffmpeg` crate; see + [rust-ffmpeg/issues/64](https://github.com/meh/rust-ffmpeg/issues/64). + + Additionally, ffmpeg library versions older than 55.1.101, along with + 55.1.101, along with all versions of the competing project + [libav](http://libav.org), don't not support socket timeouts for RTSP. For + reliable reconnections on error, it's strongly recommended to use ffmpeg + library versions >= 55.1.101. + * [SQLite3](https://www.sqlite.org/). -On Ubuntu 15.10 or Raspbian Jessie, the following command will install most -pre-requisites (see also the `Build-Depends` field in `debian/control`): +On Ubuntu 16.04.1 LTS or Raspbian Jessie, the following command will install +all non-Rust dependencies: $ sudo apt-get install \ build-essential \ - cmake \ libavcodec-dev \ libavformat-dev \ libavutil-dev \ - libgflags-dev \ - libgoogle-glog-dev \ - libgoogle-perftools-dev \ - libjsoncpp-dev \ - libre2-dev \ sqlite3 \ libsqlite3-dev \ - pkgconf \ - uuid-runtime \ - uuid-dev - -libevent 2.1 will have to be installed from source. In the future, this -dependency may be replaced or support may be added for automatically building -libevent in-tree to avoid the inconvenience. + uuid-runtime uuid-runtime is only necessary if you wish to use the uuid command to generate uuids for your cameras (see below). If you obtain them elsewhere, you can skip this package. +Next, you need a nightly version of Rust and Cargo. The easiest way to install +them is by following the instructions at [rustup.rs](https://www.rustup.rs/). + You can continue to follow the build/install instructions below for a manual build and install, or alternatively you can run the prep script called `prep.sh`. @@ -124,11 +116,7 @@ build and install, or alternatively you can run the prep script called `prep.sh` The script will take the following command line options, should you need them: -* `-E`: Forcibly purge all existing libevent packages. You would only do this - if there is some apparent conflict (see remarks about building libevent - from source). -* `-f`: Force a build even if the binary appears to be installed. This can be useful - on repeat builds. +* `-D`: Skip database initialization. * `-S`: Skip updating and installing dependencies through apt-get. This too can be useful on repeated builds. @@ -151,17 +139,9 @@ For instructions, you can skip to "[Camera configuration and hard disk mounting] Once prerequisites are installed, Moonfire NVR can be built as follows: - $ mkdir release - $ cd release - $ cmake -DCMAKE_BUILD_TYPE=Release .. - $ make - $ sudo make install - -Alternatively, if you do have a sufficiently new apt-installed libevent -installed, you may be able to prepare a `.deb` package: - - $ sudo apt-get install devscripts dh-systemd - $ debuild -us -uc + $ RUST_TEST_THREADS=1 cargo test + $ cargo build --release + $ sudo install -m 755 target/release/moonfire-nvr /usr/local/bin # Further configuration @@ -282,9 +262,10 @@ been done for you. If not, Create [Service] ExecStart=/usr/local/bin/moonfire-nvr \ - --sample_file_dir=/var/lib/moonfire-nvr/sample \ - --db_dir=/var/lib/moonfire-nvr/db \ - --http_port=8080 + --sample-file-dir=/var/lib/moonfire-nvr/sample \ + --db-dir=/var/lib/moonfire-nvr/db \ + --http-addr=0.0.0.0:8080 + Environment=RUST_LOG=info Type=simple User=moonfire-nvr Nice=-20 @@ -313,10 +294,16 @@ and `systemctl` may be of particular interest. # Troubleshooting -While Moonfire NVR is running, logs will be written to `/tmp/moonfire-nvr.INFO`. -Also available will be `/tmp/moonfire-nvr.WARNING` and `/tmp/moonfire-nvr.ERROR`. -These latter to contain only warning or more serious messages, respectively only -error messages. +While Moonfire NVR is running, logs will be written to stdout. The `RUST_LOG` +environmental variable controls the log level; `RUST_LOG=info` is recommended. +If running through systemd, try `sudo journalctl --unit moonfire-nvr` to view +the logs. + +If Moonfire NVR crashes with a `SIGSEGV`, the problem is likely an +incompatible version of the C `ffmpeg` libraries; use the latest 2.x release +instead. This is one of the Rust growing pains mentioned above. While most +code written in Rust is "safe", the foreign function interface is not only +unsafe but currently error-prone. # Getting help and getting involved @@ -325,17 +312,11 @@ Please email the mailing list with questions, bug reports, feature requests, or just to say you love/hate the software and why. -I'd welcome help with testing, development (in C++, JavaScript, and HTML), user -interface/graphic design, and documentation. Please email the mailing list -if interested. Patches are welcome, but I encourage you to discuss large +I'd welcome help with testing, development (in Rust, JavaScript, and HTML), +user interface/graphic design, and documentation. Please email the mailing +list if interested. Patches are welcome, but I encourage you to discuss large changes on the mailing list first to save effort. -C++ code should be written using C++11 features, should follow the [Google C++ -style guide](https://google.github.io/styleguide/cppguide.html) for -consistency, and should be automatically tested where practical. But don't -worry about this too much; I'm much happier to work with you to refine a rough -draft patch than never see your contribution at all! - # License This file is part of Moonfire NVR, a security camera digital video recorder. diff --git a/debian/changelog b/debian/changelog deleted file mode 100644 index 68f9a05..0000000 --- a/debian/changelog +++ /dev/null @@ -1,5 +0,0 @@ -moonfire-nvr (0.1.0) UNRELEASED; urgency=medium - - * Initial release. - - -- Scott Lamb Fri, 1 Jan 2016 21:00:00 -0800 diff --git a/debian/compat b/debian/compat deleted file mode 100644 index ec63514..0000000 --- a/debian/compat +++ /dev/null @@ -1 +0,0 @@ -9 diff --git a/debian/control b/debian/control deleted file mode 100644 index ab54b61..0000000 --- a/debian/control +++ /dev/null @@ -1,22 +0,0 @@ -Source: moonfire-nvr -Maintainer: Scott Lamb -Section: video -Priority: optional -Standards-Version: 3.9.6.1 -Build-Depends: cmake, - libavcodec-dev, - libavformat-dev, - libavutil-dev, - libgflags-dev, - libgoogle-glog-dev, - libgoogle-perftools-dev, - libjsoncpp-dev, - libre2-dev, - libsqlite3-dev, - pkgconf, - uuid-dev -Package: moonfire-nvr -Architecture: any -Depends: ${shlibs:Depends}, ${misc:Depends}, adduser, sqlite3, uuid-runtime -Description: security camera network video recorder - moonfire-nvr records video files from IP security cameras. diff --git a/debian/copyright b/debian/copyright deleted file mode 100644 index 5f67c00..0000000 --- a/debian/copyright +++ /dev/null @@ -1,43 +0,0 @@ -Format: http://www.debian.org/doc/packaging-manuals/copyright-format/1.0/ -Upstream-Name: Moonfire NVR -Upstream-Contact: Scott Lamb -Source: http://github.com/scottlamb/moonfire-nvr - -Files: * -Copyright: 2016 Scott Lamb -License: GPL-3+ with OpenSSL exception - -License: GPL-3+ with OpenSSL exception - This program is free software: you can redistribute it - and/or modify it under the terms of the GNU General Public - License as published by the Free Software Foundation, - either version 3 of the License, or (at your option) any - later version. - . - In addition, as a special exception, the copyright holders give - permission to link the code of portions of this program with the - OpenSSL library under certain conditions as described in each - individual source file, and distribute linked combinations including - the two. - . - You must obey the GNU General Public License in all respects for all - of the code used other than OpenSSL. If you modify file(s) with this - exception, you may extend this exception to your version of the - file(s), but you are not obligated to do so. If you do not wish to do - so, delete this exception statement from your version. If you delete - this exception statement from all source files in the program, then - also delete it here. - . - This program is distributed in the hope that it will be - useful, but WITHOUT ANY WARRANTY; without even the implied - warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR - PURPOSE. See the GNU General Public License for more - details. - . - You should have received a copy of the GNU General Public - License along with this program. If not, see - . - . - On Debian systems, the full text of the GNU General Public - License version 3 can be found in the file - `/usr/share/common-licenses/GPL-3'. diff --git a/debian/moonfire-nvr.postinst b/debian/moonfire-nvr.postinst deleted file mode 100644 index a9d7f4e..0000000 --- a/debian/moonfire-nvr.postinst +++ /dev/null @@ -1,8 +0,0 @@ -#!/bin/sh - -set -e -adduser --system moonfire-nvr - -#DEBHELPER# - -exit 0 diff --git a/debian/moonfire-nvr.service b/debian/moonfire-nvr.service deleted file mode 100644 index c36fb93..0000000 --- a/debian/moonfire-nvr.service +++ /dev/null @@ -1,16 +0,0 @@ -[Unit] -Description=Moonfire NVR -After=network-online.target - -[Service] -ExecStart=/usr/bin/moonfire-nvr -Type=simple -User=moonfire-nvr -Nice=-20 -Restart=on-abnormal -CPUAccounting=true -MemoryAccounting=true -BlockIOAccounting=true - -[Install] -WantedBy=multi-user.target diff --git a/debian/rules b/debian/rules deleted file mode 100755 index 93646c7..0000000 --- a/debian/rules +++ /dev/null @@ -1,3 +0,0 @@ -#!/usr/bin/make -f -%: - dh $@ --with=systemd diff --git a/debian/source/format b/debian/source/format deleted file mode 100644 index 89ae9db..0000000 --- a/debian/source/format +++ /dev/null @@ -1 +0,0 @@ -3.0 (native) diff --git a/prep.sh b/prep.sh index d1c73e9..537575f 100755 --- a/prep.sh +++ b/prep.sh @@ -34,7 +34,7 @@ # Script to prepare for moonfire-nvr operations # # Command line options: -# -f: Force clean build, even if binary already installed +# -D: Skip database initialization # -S: Skip apt-get update and install # @@ -112,14 +112,10 @@ SERVICE_BIN="${SERVICE_BIN:-/usr/local/bin/moonfire-nvr}" # Process command line options # -while getopts ":DEfS" opt; do +while getopts ":DS" opt; do case $opt in D) SKIP_DB=1 ;; - E) PURGE_LIBEVENT=1 - ;; - f) FORCE_BUILD=1 - ;; S) SKIP_APT=1 ;; :) @@ -133,45 +129,34 @@ while getopts ":DEfS" opt; do esac done -# Setup all packages we need +# Setup all apt packages we need # echo 'Preparing and downloading packages we need...'; echo if [ "${SKIP_APT:-0}" != 1 ]; then sudo apt-get update - [ "${PURGE_LIBEVENT:-0}" == 1 ] && sudo apt-get --purge remove libevent-* sudo apt-get install \ build-essential \ - cmake \ libavcodec-dev \ libavformat-dev \ libavutil-dev \ - libgflags-dev \ - libgoogle-glog-dev \ - libgoogle-perftools-dev \ - libjsoncpp-dev \ - libre2-dev \ - libssl-dev \ sqlite3 \ libsqlite3-dev \ - pkgconf \ - uuid-runtime \ - uuid-dev + uuid-runtime fi # Check if binary is installed. Setup for build if it is not # if [ ! -x "${SERVICE_BIN}" ]; then echo "Binary not installed, building..."; echo - FORCE_BUILD=1 -fi - -# Build if requested -# -if [ "${FORCE_BUILD:-0}" -eq 1 ]; then - # Remove previous build, if any - [ -d release ] && rm -fr release 2>/dev/null - mkdir release; cd release - cmake -DCMAKE_BUILD_TYPE=Release .. && make && sudo make install + if ! cargo --version; then + echo "cargo not installed/working." + echo "Install a nightly Rust (see http://rustup.us) first." + echo + exit 1 + fi + RUST_TEST_THREADS=1 cargo test + cargo build --release + sudo install -m 755 target/release/moonfire-nvr ${SERVICE_BIN} if [ -x "${SERVICE_BIN}" ]; then echo "Binary installed..."; echo else @@ -233,9 +218,10 @@ After=network-online.target [Service] ExecStart=${SERVICE_BIN} \\ - --sample_file_dir=${SAMPLES_PATH} \\ - --db_dir=${DB_DIR} \\ - --http_port=${NVR_PORT} + --sample-file-dir=${SAMPLES_PATH} \\ + --db-dir=${DB_DIR} \\ + --http-addr=0.0.0.0:${NVR_PORT} +Environment=RUST_LOG=info Type=simple User=${NVR_USER} Nice=-20 diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt deleted file mode 100644 index fceca19..0000000 --- a/src/CMakeLists.txt +++ /dev/null @@ -1,101 +0,0 @@ -# Copyright (C) 2016 Scott Lamb -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# In addition, as a special exception, the copyright holders give -# permission to link the code of portions of this program with the -# OpenSSL library under certain conditions as described in each -# individual source file, and distribute linked combinations including -# the two. -# -# You must obey the GNU General Public License in all respects for all -# of the code used other than OpenSSL. If you modify file(s) with this -# exception, you may extend this exception to your version of the -# file(s), but you are not obligated to do so. If you do not wish to do -# so, delete this exception statement from your version. If you delete -# this exception statement from all source files in the program, then -# also delete it here. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -include_directories(${CMAKE_CURRENT_BINARY_DIR}) -include_directories(${JSONCPP_INCLUDE_DIRS}) - -set(MOONFIRE_DEPS - ${CMAKE_THREAD_LIBS_INIT} - ${FFMPEG_LIBRARIES} - ${GFLAGS_LIBRARIES} - ${GLOG_LIBRARIES} - ${JSONCPP_LIBRARIES} - ${LIBEVENT_LIBRARIES} - ${OPENSSL_LIBRARIES} - ${PROFILER_LIBRARIES} - ${RE2_LIBRARIES} - ${SQLITE_LIBRARIES} - ${UUID_LIBRARIES}) - -set(MOONFIRE_NVR_SRCS - coding.cc - crypto.cc - ffmpeg.cc - filesystem.cc - h264.cc - http.cc - moonfire-db.cc - moonfire-nvr.cc - mp4.cc - profiler.cc - recording.cc - sqlite.cc - string.cc - time.cc - uuid.cc - web.cc) - -link_directories(${LIBEVENT_LIBRARY_DIRS}) - -add_library(moonfire-nvr-lib ${MOONFIRE_NVR_SRCS} ${PROTO_SRCS} ${PROTO_HDRS}) -target_link_libraries(moonfire-nvr-lib ${MOONFIRE_DEPS}) - -add_executable(moonfire-nvr moonfire-nvr-main.cc) -target_link_libraries(moonfire-nvr moonfire-nvr-lib) -install_programs(/bin FILES moonfire-nvr) - -# Tests. -include_directories(${GTest_INCLUDE_DIR}) -include_directories(${GMock_INCLUDE_DIR}) -include_directories(${GBenchmark_INCLUDE_DIR}) - -set(MOONFIRE_NVR_TESTS - coding - crypto - h264 - http - moonfire-db - moonfire-nvr - mp4 - recording - sqlite - string) - -foreach(test ${MOONFIRE_NVR_TESTS}) - add_executable(${test}-test ${test}-test.cc testutil.cc) - target_link_libraries(${test}-test GTest GMock moonfire-nvr-lib) - add_test(NAME ${test}-test - COMMAND ${test}-test - WORKING_DIRECTORY ${CMAKE_BINARY_DIR}) -endforeach(test) - -foreach(bench recording) - add_executable(${bench}-bench ${bench}-bench.cc testutil.cc) - target_link_libraries(${bench}-bench GTest GMock GBenchmark moonfire-nvr-lib) -endforeach(bench) diff --git a/src/coding-test.cc b/src/coding-test.cc deleted file mode 100644 index d358d7a..0000000 --- a/src/coding-test.cc +++ /dev/null @@ -1,141 +0,0 @@ -// This file is part of Moonfire NVR, a security camera network video recorder. -// Copyright (C) 2016 Scott Lamb -// -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// In addition, as a special exception, the copyright holders give -// permission to link the code of portions of this program with the -// OpenSSL library under certain conditions as described in each -// individual source file, and distribute linked combinations including -// the two. -// -// You must obey the GNU General Public License in all respects for all -// of the code used other than OpenSSL. If you modify file(s) with this -// exception, you may extend this exception to your version of the -// file(s), but you are not obligated to do so. If you do not wish to do -// so, delete this exception statement from your version. If you delete -// this exception statement from all source files in the program, then -// also delete it here. -// -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. -// -// You should have received a copy of the GNU General Public License -// along with this program. If not, see . -// -// coding-test.cc: tests of the coding.h interface. - -#include -#include -#include -#include - -#include "coding.h" - -DECLARE_bool(alsologtostderr); - -namespace moonfire_nvr { -namespace { - -TEST(VarintTest, Simple) { - // Encode. - std::string foo; - AppendVar32(UINT32_C(1), &foo); - EXPECT_EQ("\x01", foo); - AppendVar32(UINT32_C(300), &foo); - EXPECT_EQ("\x01\xac\x02", foo); - - // Decode. - re2::StringPiece p(foo); - uint32_t out; - std::string error_message; - EXPECT_TRUE(DecodeVar32(&p, &out, &error_message)); - EXPECT_EQ(UINT32_C(1), out); - EXPECT_TRUE(DecodeVar32(&p, &out, &error_message)); - EXPECT_EQ(UINT32_C(300), out); - EXPECT_EQ(0, p.size()); -} - -TEST(VarintTest, AllDecodeSizes) { - std::string error_message; - const uint32_t kToDecode[]{ - 1, - 1 | (2 << 7), - 1 | (2 << 7) | (3 << 14), - 1 | (2 << 7) | (3 << 14) | (4 << 21), - 1 | (2 << 7) | (3 << 14) | (4 << 21) | (5 << 28), - }; - for (size_t i = 0; i < sizeof(kToDecode) / sizeof(kToDecode[0]); ++i) { - auto in = kToDecode[i]; - std::string foo; - AppendVar32(in, &foo); - ASSERT_EQ(i + 1, foo.size()); - re2::StringPiece p(foo); - uint32_t out; - - // Slow path: last bytes of the buffer. - DecodeVar32(&p, &out, &error_message); - EXPECT_EQ(in, out) << "i: " << i; - EXPECT_EQ(0, p.size()) << "i: " << i; - - // Fast path: plenty of bytes in the buffer. - foo.append(4, 0); - p = foo; - DecodeVar32(&p, &out, &error_message); - EXPECT_EQ(in, out); - EXPECT_EQ(4, p.size()); - } -} - -TEST(VarintTest, DecodeErrors) { - re2::StringPiece empty; - uint32_t out; - std::string error_message; - - for (auto input : - {re2::StringPiece("", 0), re2::StringPiece("\x80", 1), - re2::StringPiece("\x80\x80", 2), re2::StringPiece("\x80\x80\x80", 3), - re2::StringPiece("\x80\x80\x80\x80", 4)}) { - EXPECT_FALSE(DecodeVar32(&input, &out, &error_message)) << "input: " - << input; - EXPECT_EQ("buffer underrun", error_message); - } - - re2::StringPiece too_big("\x80\x80\x80\x80\x10", 5); - EXPECT_FALSE(DecodeVar32(&too_big, &out, &error_message)); - EXPECT_EQ("integer overflow", error_message); -} - -TEST(ZigzagTest, Encode) { - EXPECT_EQ(UINT32_C(0), Zigzag32(INT32_C(0))); - EXPECT_EQ(UINT32_C(1), Zigzag32(INT32_C(-1))); - EXPECT_EQ(UINT32_C(2), Zigzag32(INT32_C(1))); - EXPECT_EQ(UINT32_C(3), Zigzag32(INT32_C(-2))); - EXPECT_EQ(UINT32_C(4294967294), Zigzag32(INT32_C(2147483647))); - EXPECT_EQ(UINT32_C(4294967295), Zigzag32(INT32_C(-2147483648))); -} - -TEST(ZigzagTest, Decode) { - EXPECT_EQ(INT32_C(0), Unzigzag32(UINT32_C(0))); - EXPECT_EQ(INT32_C(-1), Unzigzag32(UINT32_C(1))); - EXPECT_EQ(INT32_C(1), Unzigzag32(UINT32_C(2))); - EXPECT_EQ(INT32_C(-2), Unzigzag32(UINT32_C(3))); - EXPECT_EQ(INT32_C(2147483647), Unzigzag32(UINT32_C(4294967294))); - EXPECT_EQ(INT32_C(-2147483648), Unzigzag32(UINT32_C(4294967295))); -} - -} // namespace -} // namespace moonfire_nvr - -int main(int argc, char **argv) { - FLAGS_alsologtostderr = true; - google::ParseCommandLineFlags(&argc, &argv, true); - testing::InitGoogleTest(&argc, argv); - google::InitGoogleLogging(argv[0]); - return RUN_ALL_TESTS(); -} diff --git a/src/coding.cc b/src/coding.cc deleted file mode 100644 index 0c024a7..0000000 --- a/src/coding.cc +++ /dev/null @@ -1,109 +0,0 @@ -// This file is part of Moonfire NVR, a security camera network video recorder. -// Copyright (C) 2016 Scott Lamb -// -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// In addition, as a special exception, the copyright holders give -// permission to link the code of portions of this program with the -// OpenSSL library under certain conditions as described in each -// individual source file, and distribute linked combinations including -// the two. -// -// You must obey the GNU General Public License in all respects for all -// of the code used other than OpenSSL. If you modify file(s) with this -// exception, you may extend this exception to your version of the -// file(s), but you are not obligated to do so. If you do not wish to do -// so, delete this exception statement from your version. If you delete -// this exception statement from all source files in the program, then -// also delete it here. -// -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. -// -// You should have received a copy of the GNU General Public License -// along with this program. If not, see . -// -// coding.cc: see coding.h. - -#include "coding.h" -#include "common.h" - -namespace moonfire_nvr { - -namespace internal { - -void AppendVar32Slow(uint32_t in, std::string *out) { - while (true) { - uint8_t next_byte = in & 0x7F; - in >>= 7; - if (in == 0) { - out->push_back(next_byte); - return; - } - out->push_back(next_byte | 0x80); - } -} - -bool DecodeVar32Slow(re2::StringPiece *in, uint32_t *out_p, - std::string *error_message) { - // The fast path is inlined; this function is called only when - // byte 0 is present and >= 0x80. - size_t left = in->size() - 1; - auto p = reinterpret_cast(in->data()); - uint32_t v = uint32_t(p[0] & 0x7f); - size_t size = 1; - - // Aid branch prediction in two ways: - // * have a faster path which doesn't check for buffer underrun on every - // byte if there's plenty of bytes left or the last byte is not continued. - // * fully unroll the loop - if (left >= 4 || (p[left] & 0x80) == 0) { - v |= uint32_t(p[size] & 0x7f) << 7; - if (p[size++] & 0x80) { - v |= uint32_t(p[size] & 0x7f) << 14; - if (p[size++] & 0x80) { - v |= uint32_t(p[size] & 0x7f) << 21; - if (p[size++] & 0x80) { - if (UNLIKELY(p[size] & 0xf0)) { - *error_message = "integer overflow"; - return false; - } - v |= uint32_t(p[size++] & 0x7f) << 28; - } - } - } - *out_p = v; - in->remove_prefix(size); - return true; - } - - // Slowest path. - if (LIKELY(left)) { - v |= uint32_t(p[size] & 0x7f) << 7; - if (p[size++] & 0x80 && --left > 0) { - v |= uint32_t(p[size] & 0x7f) << 14; - if (p[size++] & 0x80 && --left > 0) { - v |= uint32_t(p[size] & 0x7f) << 21; - if (p[size++] & 0x80) { - --left; - } - } - } - } - if (UNLIKELY(left == 0 && p[size - 1] & 0x80)) { - *error_message = "buffer underrun"; - return false; - } - *out_p = v; - in->remove_prefix(size); - return true; -} - -} // namespace internal - -} // namespace moonfire_nvr diff --git a/src/coding.h b/src/coding.h deleted file mode 100644 index ff983c7..0000000 --- a/src/coding.h +++ /dev/null @@ -1,169 +0,0 @@ -// This file is part of Moonfire NVR, a security camera network video recorder. -// Copyright (C) 2016 Lamb -// -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// In addition, as a special exception, the copyright holders give -// permission to link the code of portions of this program with the -// OpenSSL library under certain conditions as described in each -// individual source file, and distribute linked combinations including -// the two. -// -// You must obey the GNU General Public License in all respects for all -// of the code used other than OpenSSL. If you modify file(s) with this -// exception, you may extend this exception to your version of the -// file(s), but you are not obligated to do so. If you do not wish to do -// so, delete this exception statement from your version. If you delete -// this exception statement from all source files in the program, then -// also delete it here. -// -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. -// -// You should have received a copy of the GNU General Public License -// along with this program. If not, see . -// -// coding.h: Binary encoding/decoding. - -#ifndef MOONFIRE_NVR_CODING_H -#define MOONFIRE_NVR_CODING_H - -#include -#include - -#include - -#include - -namespace moonfire_nvr { - -namespace internal { - -void AppendVar32Slow(uint32_t in, std::string *out); -bool DecodeVar32Slow(re2::StringPiece *in, uint32_t *out, - std::string *error_message); - -} // namespace internal - -// Endianness conversion. - -#if __BYTE_ORDER == __LITTLE_ENDIAN -// XXX: __builtin_bswap64 doesn't compile on gcc 5.2.1 with an error about a -// narrowing conversion?!? Doing this by hand... -constexpr uint64_t ToNetworkU64(uint64_t in) { - return ((in & UINT64_C(0xFF00000000000000)) >> 56) | - ((in & UINT64_C(0x00FF000000000000)) >> 40) | - ((in & UINT64_C(0x0000FF0000000000)) >> 24) | - ((in & UINT64_C(0x000000FF00000000)) >> 8) | - ((in & UINT64_C(0x00000000FF000000)) << 8) | - ((in & UINT64_C(0x0000000000FF0000)) << 24) | - ((in & UINT64_C(0x000000000000FF00)) << 40) | - ((in & UINT64_C(0x00000000000000FF)) << 56); -} -constexpr int64_t ToNetwork64(int64_t in) { - return static_cast(ToNetworkU64(static_cast(in))); -} -constexpr uint32_t ToNetworkU32(uint32_t in) { - return ((in & UINT32_C(0xFF000000)) >> 24) | - ((in & UINT32_C(0x00FF0000)) >> 8) | - ((in & UINT32_C(0x0000FF00)) << 8) | - ((in & UINT32_C(0x000000FF)) << 24); -} -constexpr int32_t ToNetwork32(int32_t in) { - return static_cast(ToNetworkU32(static_cast(in))); -} -constexpr uint16_t ToNetworkU16(uint16_t in) { - return ((in & UINT32_C(0xFF00)) >> 8) | ((in & UINT32_C(0x00FF)) << 8); -} -constexpr int16_t ToNetwork16(int16_t in) { - return static_cast(ToNetworkU16(static_cast(in))); -} -#elif __BYTE_ORDER == __BIG_ENDIAN -constexpr uint64_t ToNetworkU64(uint64_t in) { return in; } -constexpr int64_t ToNetwork64(int64_t in) { return in; } -constexpr uint32_t ToNetworkU32(uint32_t in) { return in; } -constexpr int32_t ToNetwork32(int32_t in) { return in; } -constexpr uint16_t ToNetworkU16(uint16_t in) { return in; } -constexpr int16_t ToNetwork16(int16_t in) { return in; } -#else -#error Unknown byte order. -#endif - -// Varint encoding, as in -// https://developers.google.com/protocol-buffers/docs/encoding#varints - -inline void AppendVar32(uint32_t in, std::string *out) { - if (in < UINT32_C(1) << 7) { - out->push_back(static_cast(in)); - } else { - internal::AppendVar32Slow(in, out); - } -} - -// Decode the first varint from |in|, saving it to |out| and advancing |in|. -// Returns error if |in| does not hold a complete varint or on integer overflow. -inline bool DecodeVar32(re2::StringPiece *in, uint32_t *out, - std::string *error_message) { - if (in->size() == 0) { - *error_message = "buffer underrun"; - return false; - } - auto first_byte = static_cast(*in->data()); - if (first_byte < 0x80) { - in->remove_prefix(1); - *out = first_byte; - return true; - } else { - return internal::DecodeVar32Slow(in, out, error_message); - } -} - -// Zigzag encoding for signed integers, as in -// https://developers.google.com/protocol-buffers/docs/encoding#types -// Use the low bit to indicate signedness (1 = negative, 0 = non-negative). -inline uint32_t Zigzag32(int32_t in) { - return static_cast(in << 1) ^ (in >> 31); -} - -inline int32_t Unzigzag32(uint32_t in) { - return (in >> 1) ^ -static_cast(in & 1); -} - -inline void AppendU16(uint16_t in, std::string *out) { - uint16_t net = ToNetworkU16(in); - out->append(reinterpret_cast(&net), sizeof(uint16_t)); -} - -inline void Append16(int16_t in, std::string *out) { - int16_t net = ToNetwork16(in); - out->append(reinterpret_cast(&net), sizeof(int16_t)); -} - -inline void AppendU32(uint32_t in, std::string *out) { - uint32_t net = ToNetworkU32(in); - out->append(reinterpret_cast(&net), sizeof(uint32_t)); -} - -inline void Append32(int32_t in, std::string *out) { - int32_t net = ToNetwork32(in); - out->append(reinterpret_cast(&net), sizeof(int32_t)); -} - -inline void AppendU64(uint64_t in, std::string *out) { - uint64_t net = ToNetworkU64(in); - out->append(reinterpret_cast(&net), sizeof(uint64_t)); -} - -inline void Append64(int64_t in, std::string *out) { - int64_t net = ToNetwork64(in); - out->append(reinterpret_cast(&net), sizeof(int64_t)); -} - -} // namespace moonfire_nvr - -#endif // MOONFIRE_NVR_CODING_H diff --git a/src/common.h b/src/common.h deleted file mode 100644 index e5a149f..0000000 --- a/src/common.h +++ /dev/null @@ -1,49 +0,0 @@ -// This file is part of Moonfire NVR, a security camera network video recorder. -// Copyright (C) 2016 Scott Lamb -// -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// In addition, as a special exception, the copyright holders give -// permission to link the code of portions of this program with the -// OpenSSL library under certain conditions as described in each -// individual source file, and distribute linked combinations including -// the two. -// -// You must obey the GNU General Public License in all respects for all -// of the code used other than OpenSSL. If you modify file(s) with this -// exception, you may extend this exception to your version of the -// file(s), but you are not obligated to do so. If you do not wish to do -// so, delete this exception statement from your version. If you delete -// this exception statement from all source files in the program, then -// also delete it here. -// -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. -// -// You should have received a copy of the GNU General Public License -// along with this program. If not, see . -// -// common.h: basic enums/defines/whatever. - -#ifndef MOONFIRE_NVR_COMMON_H -#define MOONFIRE_NVR_COMMON_H - -#define LIKELY(x) __builtin_expect((x), 1) -#define UNLIKELY(x) __builtin_expect((x), 0) - -namespace moonfire_nvr { - -// Return value for *ForEach callbacks. -enum class IterationControl { - kContinue, // indicates the caller should proceed with the loop. - kBreak // indicates the caller should terminate the loop with success. -}; - -} // namespace moonfire_nvr - -#endif // MOONFIRE_NVR_COMMON_H diff --git a/src/crypto-test.cc b/src/crypto-test.cc deleted file mode 100644 index b80f5b6..0000000 --- a/src/crypto-test.cc +++ /dev/null @@ -1,67 +0,0 @@ -// This file is part of Moonfire NVR, a security camera network video recorder. -// Copyright (C) 2016 Scott Lamb -// -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// In addition, as a special exception, the copyright holders give -// permission to link the code of portions of this program with the -// OpenSSL library under certain conditions as described in each -// individual source file, and distribute linked combinations including -// the two. -// -// You must obey the GNU General Public License in all respects for all -// of the code used other than OpenSSL. If you modify file(s) with this -// exception, you may extend this exception to your version of the -// file(s), but you are not obligated to do so. If you do not wish to do -// so, delete this exception statement from your version. If you delete -// this exception statement from all source files in the program, then -// also delete it here. -// -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. -// -// You should have received a copy of the GNU General Public License -// along with this program. If not, see . -// -// crypto-test.cc: tests of the crypto.h interface. - -#include -#include -#include -#include - -#include "crypto.h" -#include "string.h" - -DECLARE_bool(alsologtostderr); - -namespace moonfire_nvr { -namespace { - -TEST(DigestTest, Sha1) { - auto sha1 = Digest::SHA1(); - EXPECT_EQ("da39a3ee5e6b4b0d3255bfef95601890afd80709", - ToHex(sha1->Finalize())); - - sha1 = Digest::SHA1(); - sha1->Update("hello"); - sha1->Update(" world"); - EXPECT_EQ("2aae6c35c94fcfb415dbe95f408b9ce91ee846ed", - ToHex(sha1->Finalize())); -} - -} // namespace -} // namespace moonfire_nvr - -int main(int argc, char **argv) { - FLAGS_alsologtostderr = true; - google::ParseCommandLineFlags(&argc, &argv, true); - testing::InitGoogleTest(&argc, argv); - google::InitGoogleLogging(argv[0]); - return RUN_ALL_TESTS(); -} diff --git a/src/crypto.cc b/src/crypto.cc deleted file mode 100644 index 9b7b255..0000000 --- a/src/crypto.cc +++ /dev/null @@ -1,61 +0,0 @@ -// This file is part of Moonfire NVR, a security camera network video recorder. -// Copyright (C) 2016 Scott Lamb -// -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// In addition, as a special exception, the copyright holders give -// permission to link the code of portions of this program with the -// OpenSSL library under certain conditions as described in each -// individual source file, and distribute linked combinations including -// the two. -// -// You must obey the GNU General Public License in all respects for all -// of the code used other than OpenSSL. If you modify file(s) with this -// exception, you may extend this exception to your version of the -// file(s), but you are not obligated to do so. If you do not wish to do -// so, delete this exception statement from your version. If you delete -// this exception statement from all source files in the program, then -// also delete it here. -// -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. -// -// You should have received a copy of the GNU General Public License -// along with this program. If not, see . -// -// crypto.cc: see crypto.h. - -#include "crypto.h" - -#include - -namespace moonfire_nvr { - -std::unique_ptr Digest::SHA1() { - std::unique_ptr d(new Digest); - CHECK_EQ(1, EVP_DigestInit_ex(d->ctx_, EVP_sha1(), nullptr)); - return d; -} - -Digest::Digest() { ctx_ = CHECK_NOTNULL(EVP_MD_CTX_create()); } - -Digest::~Digest() { EVP_MD_CTX_destroy(ctx_); } - -void Digest::Update(re2::StringPiece data) { - CHECK_EQ(1, EVP_DigestUpdate(ctx_, data.data(), data.size())); -} - -std::string Digest::Finalize() { - std::string out; - out.resize(EVP_MD_CTX_size(ctx_)); - auto *p = reinterpret_cast(&out[0]); - CHECK_EQ(1, EVP_DigestFinal_ex(ctx_, p, nullptr)); - return out; -} - -} // namespace moonfire_nvr diff --git a/src/crypto.h b/src/crypto.h deleted file mode 100644 index 9b1033a..0000000 --- a/src/crypto.h +++ /dev/null @@ -1,63 +0,0 @@ -// This file is part of Moonfire NVR, a security camera network video recorder. -// Copyright (C) 2016 Scott Lamb -// -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// In addition, as a special exception, the copyright holders give -// permission to link the code of portions of this program with the -// OpenSSL library under certain conditions as described in each -// individual source file, and distribute linked combinations including -// the two. -// -// You must obey the GNU General Public License in all respects for all -// of the code used other than OpenSSL. If you modify file(s) with this -// exception, you may extend this exception to your version of the -// file(s), but you are not obligated to do so. If you do not wish to do -// so, delete this exception statement from your version. If you delete -// this exception statement from all source files in the program, then -// also delete it here. -// -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. -// -// You should have received a copy of the GNU General Public License -// along with this program. If not, see . -// -// crypto.h: cryptographic functions. - -#ifndef MOONFIRE_NVR_CRYPTO_H -#define MOONFIRE_NVR_CRYPTO_H - -#include - -#include -#include - -namespace moonfire_nvr { - -class Digest { - public: - static std::unique_ptr SHA1(); - ~Digest(); - - // PRE: Finalize() has not been called. - void Update(re2::StringPiece data); - - // PRE: Finalize() has not been called. - std::string Finalize(); - - private: - Digest(); - Digest(const Digest &) = delete; - void operator=(const Digest &) = delete; - EVP_MD_CTX *ctx_ = nullptr; -}; - -} // namespace moonfire_nvr - -#endif // MOONFIRE_NVR_CRYPTO_H diff --git a/src/db.rs b/src/db.rs new file mode 100644 index 0000000..b879256 --- /dev/null +++ b/src/db.rs @@ -0,0 +1,1314 @@ +// This file is part of Moonfire NVR, a security camera digital video recorder. +// Copyright (C) 2016 Scott Lamb +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// In addition, as a special exception, the copyright holders give +// permission to link the code of portions of this program with the +// OpenSSL library under certain conditions as described in each +// individual source file, and distribute linked combinations including +// the two. +// +// You must obey the GNU General Public License in all respects for all +// of the code used other than OpenSSL. If you modify file(s) with this +// exception, you may extend this exception to your version of the +// file(s), but you are not obligated to do so. If you do not wish to do +// so, delete this exception statement from your version. If you delete +// this exception statement from all source files in the program, then +// also delete it here. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program. If not, see . + +//! Database access logic for the Moonfire NVR SQLite schema. +//! +//! The SQLite schema includes everything except the actual video samples (see the `dir` module +//! for management of those). See `schema.sql` for a more detailed description. +//! +//! The `Database` struct caches data in RAM, making the assumption that only one process is +//! accessing the database at a time. Performance and efficiency notes: +//! +//! * several query operations here feature row callbacks. The callback is invoked with +//! the database lock. Thus, the callback shouldn't perform long-running operations. +//! +//! * startup may be slow, as it scans the entire index for the recording table. This seems +//! acceptable. +//! +//! * the operations used for web file serving should return results with acceptable latency. +//! +//! * however, the database lock may be held for longer than is acceptable for +//! the critical path of recording frames. The caller should preallocate sample file uuids +//! and such to avoid database operations in these paths. +//! +//! * the `Transaction` interface allows callers to batch write operations to reduce latency and +//! SSD write samples. + +// Suppress false positive warnings caused by using the word SQLite in a docstring. +// clippy thinks this is an identifier which should be enclosed in backticks. +#![allow(doc_markdown)] + +use error::Error; +use fnv; +use lru_cache::LruCache; +use openssl::crypto::hash; +use recording::{self, TIME_UNITS_PER_SEC}; +use rusqlite; +use serde::ser::{Serialize, Serializer}; +use std::collections::BTreeMap; +use std::cell::RefCell; +use std::cmp; +use std::io::Write; +use std::ops::Range; +use std::str; +use std::string::String; +use std::sync::{Arc,Mutex,MutexGuard}; +use std::vec::Vec; +use time; +use uuid::Uuid; + +const GET_RECORDING_SQL: &'static str = + "select sample_file_uuid, video_index from recording where id = :id"; + +const DELETE_RESERVATION_SQL: &'static str = + "delete from reserved_sample_files where uuid = :uuid"; + +const INSERT_RESERVATION_SQL: &'static str = r#" + insert into reserved_sample_files (uuid, state) + values (:uuid, :state); +"#; + +/// Valid values for the `state` column in the `reserved_sample_files` table. +enum ReservationState { + /// This uuid has not yet been added to the `recording` table. The file may be unwritten, + /// partially written, or fully written. + Writing = 0, + + /// This uuid was previously in the `recording` table. The file may be fully written or + /// unlinked. + Deleting = 1, +} + +const INSERT_VIDEO_SAMPLE_ENTRY_SQL: &'static str = r#" + insert into video_sample_entry (sha1, width, height, data) + values (:sha1, :width, :height, :data); +"#; + +const INSERT_RECORDING_SQL: &'static str = r#" + insert into recording (camera_id, sample_file_bytes, start_time_90k, + duration_90k, local_time_delta_90k, video_samples, + video_sync_samples, video_sample_entry_id, + sample_file_uuid, sample_file_sha1, video_index) + values (:camera_id, :sample_file_bytes, :start_time_90k, + :duration_90k, :local_time_delta_90k, + :video_samples, :video_sync_samples, + :video_sample_entry_id, :sample_file_uuid, + :sample_file_sha1, :video_index); +"#; + +const LIST_OLDEST_SAMPLE_FILES_SQL: &'static str = r#" + select + id, + sample_file_uuid, + start_time_90k, + duration_90k, + sample_file_bytes + from + recording + where + camera_id = :camera_id + order by + start_time_90k +"#; + +const DELETE_RECORDING_SQL: &'static str = r#" + delete from recording where id = :recording_id; +"#; + +const CAMERA_MIN_START_SQL: &'static str = r#" + select + start_time_90k + from + recording + where + camera_id = :camera_id + order by start_time_90k limit 1; +"#; + +const CAMERA_MAX_START_SQL: &'static str = r#" + select + start_time_90k, + duration_90k + from + recording + where + camera_id = :camera_id + order by start_time_90k desc; +"#; + +/// A concrete box derived from a ISO/IEC 14496-12 section 8.5.2 VisualSampleEntry box. Describes +/// the codec, width, height, etc. +#[derive(Debug)] +pub struct VideoSampleEntry { + pub id: i32, + pub width: u16, + pub height: u16, + pub sha1: [u8; 20], + pub data: Vec, +} + +/// A row used in `list_recordings`. +#[derive(Debug)] +pub struct ListCameraRecordingsRow { + pub id: i64, + pub start: recording::Time, + + /// This is a recording::Duration, but a single recording's duration fits into an i32. + pub duration_90k: i32, + pub video_samples: i32, + pub video_sync_samples: i32, + pub sample_file_bytes: i32, + pub video_sample_entry: Arc, +} + +/// A row used in `list_aggregated_recordings`. +#[derive(Debug)] +pub struct ListAggregatedRecordingsRow { + pub range: Range, + pub video_samples: i64, + pub video_sync_samples: i64, + pub sample_file_bytes: i64, + pub video_sample_entry: Arc, +} + +/// Extra data about a recording, beyond what is returned by ListCameraRecordingsRow. +/// Retrieve with `get_recording`. +#[derive(Debug)] +pub struct ExtraRecording { + pub sample_file_uuid: Uuid, + pub video_index: Vec +} + +/// A recording to pass to `insert_recording`. +#[derive(Debug)] +pub struct RecordingToInsert { + pub camera_id: i32, + pub sample_file_bytes: i32, + pub time: Range, + pub local_time: recording::Time, + pub video_samples: i32, + pub video_sync_samples: i32, + pub video_sample_entry_id: i32, + pub sample_file_uuid: Uuid, + pub video_index: Vec, + pub sample_file_sha1: [u8; 20], +} + +/// A row used in `list_oldest_sample_files`. +#[derive(Debug)] +pub struct ListOldestSampleFilesRow { + pub uuid: Uuid, + pub camera_id: i32, + pub recording_id: i64, + pub time: Range, + pub sample_file_bytes: i32, +} + +/// A calendar day in `YYYY-mm-dd` format. +#[derive(Copy, Clone, Debug, Eq, Ord, PartialEq, PartialOrd)] +pub struct CameraDayKey([u8; 10]); + +impl CameraDayKey { + fn new(tm: time::Tm) -> Result { + let mut s = CameraDayKey([0u8; 10]); + write!(&mut s.0[..], "{}", tm.strftime("%Y-%m-%d")?)?; + Ok(s) + } +} + +impl Serialize for CameraDayKey { + /// Serializes as a string, not as the default bytes. + /// serde_json will only allow string keys for objects. + fn serialize(&self, serializer: &mut S) -> Result<(), S::Error> where S: Serializer { + serializer.serialize_str(str::from_utf8(&self.0[..]).expect("days are always UTF-8")) + } +} + +/// In-memory state about a particular camera on a particular day. +#[derive(Copy, Clone, Debug, PartialEq, Eq, Serialize)] +pub struct CameraDayValue { + /// The number of recordings that overlap with this day. Note that `adjust_day` automatically + /// prunes days with 0 recordings. + pub recordings: i64, + + /// The total duration recorded on this day. This can be 0; because frames' durations are taken + /// from the time of the next frame, a recording that ends unexpectedly after a single frame + /// will have 0 duration of that frame and thus the whole recording. + pub duration: recording::Duration, +} + +/// In-memory state about a camera. +#[derive(Debug, Serialize)] +pub struct Camera { + pub id: i32, + pub uuid: Uuid, + pub short_name: String, + pub description: String, + pub host: String, + pub username: String, + pub password: String, + pub main_rtsp_path: String, + pub sub_rtsp_path: String, + pub retain_bytes: i64, + + /// The time range of recorded data associated with this camera (minimum start time and maximum + /// end time). `None` iff there are no recordings for this camera. + #[serde(skip_serializing)] + pub range: Option>, + pub sample_file_bytes: i64, + + /// The total duration of recorded data. This may not be `range.end - range.start` due to + /// gaps and overlap. + pub duration: recording::Duration, + + /// Mapping of calendar day (in the server's time zone) to a summary of recordings on that day. + pub days: BTreeMap, +} + +/// Adds `delta` to the day represented by `day` in the map `m`. +/// Inserts a map entry if absent; removes the entry if it has 0 entries on exit. +fn adjust_day(day: CameraDayKey, delta: CameraDayValue, + m: &mut BTreeMap) { + enum Do { + Insert, + Remove, + Nothing + }; + let what_to_do = match m.get_mut(&day) { + None => { + Do::Insert + }, + Some(ref mut v) => { + v.recordings += delta.recordings; + v.duration += delta.duration; + if v.recordings == 0 { Do::Remove } else { Do::Nothing } + }, + }; + match what_to_do { + Do::Insert => { m.insert(day, delta); }, + Do::Remove => { m.remove(&day); }, + Do::Nothing => {}, + } +} + +/// Adjusts the day map `m` to reflect the range of the given recording. +/// Note that the specified range may span two days. It will never span more because the maximum +/// length of a recording entry is less than a day (even a 23-hour "spring forward" day). +/// +/// This function swallows/logs date formatting errors because they shouldn't happen and there's +/// not much that can be done about them. (The database operation has already gone through.) +fn adjust_days(r: Range, sign: i64, + m: &mut BTreeMap) { + // Find first day key. + let mut my_tm = time::at(time::Timespec{sec: r.start.unix_seconds(), nsec: 0}); + let day = match CameraDayKey::new(my_tm) { + Ok(d) => d, + Err(ref e) => { + error!("Unable to fill first day key from {:?}: {}; will ignore.", my_tm, e); + return; + } + }; + + // Determine the start of the next day. + // Use mytm to hold a non-normalized representation of the boundary. + my_tm.tm_isdst = -1; + my_tm.tm_hour = 0; + my_tm.tm_min = 0; + my_tm.tm_sec = 0; + my_tm.tm_mday += 1; + let boundary = my_tm.to_timespec(); + let boundary_90k = boundary.sec * TIME_UNITS_PER_SEC; + + // Adjust the first day. + let first_day_delta = CameraDayValue{ + recordings: sign, + duration: recording::Duration(sign * (cmp::min(r.end.0, boundary_90k) - r.start.0)), + }; + adjust_day(day, first_day_delta, m); + + if r.end.0 <= boundary_90k { + return; + } + + // Fill day with the second day. This requires a normalized representation so recalculate. + // (The C mktime(3) already normalized for us once, but .to_timespec() discarded that result.) + let my_tm = time::at(boundary); + let day = match CameraDayKey::new(my_tm) { + Ok(d) => d, + Err(ref e) => { + error!("Unable to fill second day key from {:?}: {}; will ignore.", my_tm, e); + return; + } + }; + let second_day_delta = CameraDayValue{ + recordings: sign, + duration: recording::Duration(sign * (r.end.0 - boundary_90k)), + }; + adjust_day(day, second_day_delta, m); +} + +impl Camera { + /// Adds a single recording with the given properties to the in-memory state. + fn add_recording(&mut self, r: Range, sample_file_bytes: i32) { + self.range = Some(match self.range { + Some(ref e) => cmp::min(e.start, r.start) .. cmp::max(e.end, r.end), + None => r.start .. r.end, + }); + self.duration += r.end - r.start; + self.sample_file_bytes += sample_file_bytes as i64; + adjust_days(r, 1, &mut self.days); + } +} + +/// Gets a uuid from the given SQLite row and column index. +fn get_uuid(row: &rusqlite::Row, i: I) -> Result { + // TODO: avoid this extra allocation+copy into a Vec. + // See . + Ok(Uuid::from_bytes(row.get_checked::<_, Vec>(i)?.as_slice())?) +} + +/// Initializes the recordings associated with the given camera. +fn init_recordings(conn: &mut rusqlite::Connection, camera_id: i32, camera: &mut Camera) + -> Result<(), Error> { + info!("Loading recordings for camera {}", camera.short_name); + let mut stmt = conn.prepare(r#" + select + recording.start_time_90k, + recording.duration_90k, + recording.sample_file_bytes + from + recording + where + camera_id = :camera_id + "#)?; + let mut rows = stmt.query_named(&[(":camera_id", &camera_id)])?; + let mut i = 0; + while let Some(row) = rows.next() { + let row = row?; + let start = recording::Time(row.get_checked(0)?); + let duration = recording::Duration(row.get_checked(1)?); + let bytes = row.get_checked(2)?; + camera.add_recording(start .. start + duration, bytes); + i += 1; + } + info!("Loaded {} recordings for camera {}", i, camera.short_name); + Ok(()) +} + +pub struct LockedDatabase { + conn: rusqlite::Connection, + state: State, +} + +/// In-memory state from the database. +/// This is separated out of `LockedDatabase` so that `Transaction` can mutably borrow `state` +/// while its underlying `rusqlite::Transaction` is borrowing `conn`. +struct State { + cameras_by_id: BTreeMap, + cameras_by_uuid: BTreeMap, + video_sample_entries: BTreeMap>, + list_recordings_sql: String, + recording_cache: RefCell, fnv::FnvBuildHasher>>, +} + +/// A high-level transaction. This manages the SQLite transaction and the matching modification to +/// be applied to the in-memory state on successful commit. +pub struct Transaction<'a> { + state: &'a mut State, + mods_by_camera: fnv::FnvHashMap, + tx: rusqlite::Transaction<'a>, + + /// True if due to an earlier error the transaction must be rolled back rather than committed. + /// Insert and delete are two-part, requiring a delete from the `reserve_sample_files` table + /// and an insert to the `recording` table (or vice versa). If the latter half fails, the + /// former should be aborted as well. We could use savepoints (nested transactions) for this, + /// but for simplicity we just require the entire transaction be rolled back. + must_rollback: bool, + + /// Normally sample file uuids must be reserved prior to a recording being inserted. + /// It's convenient in benchmarks though to allow the same segment to be inserted into the + /// database many times, so this safety check can be disabled. + pub bypass_reservation_for_testing: bool, +} + +/// A modification to be done to a `Camera` after a `Transaction` is committed. +struct CameraModification { + /// Add this to `camera.duration`. Thus, positive values indicate a net addition; + /// negative values indicate a net subtraction. + duration: recording::Duration, + + /// Add this to `camera.sample_file_bytes`. + sample_file_bytes: i64, + + /// Add this to `camera.days`. + days: BTreeMap, + + /// Reset the Camera range to this value. This should be populated immediately prior to the + /// commit. + range: Option>, +} + +impl<'a> Transaction<'a> { + /// Reserves a new, randomly generated UUID to be used as a sample file. + pub fn reserve_sample_file(&mut self) -> Result { + let mut stmt = self.tx.prepare_cached(INSERT_RESERVATION_SQL)?; + let uuid = Uuid::new_v4(); + let uuid_bytes = &uuid.as_bytes()[..]; + stmt.execute_named(&[ + (":uuid", &uuid_bytes), + (":state", &(ReservationState::Writing as i64)) + ])?; + info!("reserved {}", uuid); + Ok(uuid) + } + + /// Deletes the given recordings from the `recording` table. + /// Note they are not fully removed from the database; the uuids are transferred to the + /// `reserved_sample_files` table. The caller should `unlink` the files, then remove the + /// reservation. + pub fn delete_recordings(&mut self, rows: &[ListOldestSampleFilesRow]) -> Result<(), Error> { + let mut del = self.tx.prepare_cached(DELETE_RECORDING_SQL)?; + let mut insert = self.tx.prepare_cached(INSERT_RESERVATION_SQL)?; + + self.check_must_rollback()?; + self.must_rollback = true; + for row in rows { + let changes = del.execute_named(&[(":recording_id", &row.recording_id)])?; + if changes != 1 { + return Err(Error::new(format!("no such recording {} (camera {}, uuid {})", + row.recording_id, row.camera_id, row.uuid))); + } + let uuid = &row.uuid.as_bytes()[..]; + insert.execute_named(&[ + (":uuid", &uuid), + (":state", &(ReservationState::Deleting as i64)) + ])?; + let mut m = Transaction::get_mods_by_camera(&mut self.mods_by_camera, row.camera_id); + m.duration -= row.time.end - row.time.start; + m.sample_file_bytes -= row.sample_file_bytes as i64; + adjust_days(row.time.clone(), -1, &mut m.days); + } + self.must_rollback = false; + Ok(()) + } + + /// Marks the given sample file uuid as deleted. Accepts uuids in either `ReservationState`. + /// This shouldn't be called until the files have been `unlink()`ed and the parent directory + /// `fsync()`ed. + pub fn mark_sample_files_deleted(&mut self, uuids: &[Uuid]) -> Result<(), Error> { + if uuids.is_empty() { return Ok(()); } + let mut stmt = + self.tx.prepare_cached("delete from reserved_sample_files where uuid = :uuid;")?; + for uuid in uuids { + let uuid_bytes = &uuid.as_bytes()[..]; + let changes = stmt.execute_named(&[(":uuid", &uuid_bytes)])?; + if changes != 1 { + return Err(Error::new(format!("no reservation for {}", uuid.hyphenated()))); + } + } + Ok(()) + } + + /// Inserts the specified recording. + /// The sample file uuid must have been previously reserved. (Although this can be bypassed + /// for testing; see the `bypass_reservation_for_testing` field.) + pub fn insert_recording(&mut self, r: &RecordingToInsert) -> Result<(), Error> { + self.check_must_rollback()?; + + // Sanity checking. + if r.time.end < r.time.start { + return Err(Error::new(format!("end time {} must be >= start time {}", + r.time.end, r.time.start))); + } + + // Unreserve the sample file uuid and insert the recording row. + if self.state.cameras_by_id.get_mut(&r.camera_id).is_none() { + return Err(Error::new(format!("no such camera id {}", r.camera_id))); + } + let uuid = &r.sample_file_uuid.as_bytes()[..]; + { + let mut stmt = self.tx.prepare_cached(DELETE_RESERVATION_SQL)?; + let changes = stmt.execute_named(&[(":uuid", &uuid)])?; + if changes != 1 && !self.bypass_reservation_for_testing { + return Err(Error::new(format!("uuid {} is not reserved", r.sample_file_uuid))); + } + } + self.must_rollback = true; + { + let mut stmt = self.tx.prepare_cached(INSERT_RECORDING_SQL)?; + let sha1 = &r.sample_file_sha1[..]; + stmt.execute_named(&[ + (":camera_id", &(r.camera_id as i64)), + (":sample_file_bytes", &r.sample_file_bytes), + (":start_time_90k", &r.time.start.0), + (":duration_90k", &(r.time.end.0 - r.time.start.0)), + (":local_time_delta_90k", &(r.local_time.0 - r.time.start.0)), + (":video_samples", &r.video_samples), + (":video_sync_samples", &r.video_sync_samples), + (":video_sample_entry_id", &r.video_sample_entry_id), + (":sample_file_uuid", &uuid), + (":sample_file_sha1", &sha1), + (":video_index", &r.video_index), + ])?; + } + self.must_rollback = false; + let mut m = Transaction::get_mods_by_camera(&mut self.mods_by_camera, r.camera_id); + m.duration += r.time.end - r.time.start; + m.sample_file_bytes += r.sample_file_bytes as i64; + adjust_days(r.time.clone(), 1, &mut m.days); + Ok(()) + } + + /// Commits these changes, consuming the Transaction. + pub fn commit(mut self) -> Result<(), Error> { + self.check_must_rollback()?; + self.precommit()?; + self.tx.commit()?; + for (&camera_id, m) in &self.mods_by_camera { + let mut camera = self.state.cameras_by_id.get_mut(&camera_id) + .expect("modified camera must exist"); + camera.duration += m.duration; + camera.sample_file_bytes += m.sample_file_bytes; + for (k, v) in &m.days { + adjust_day(*k, *v, &mut camera.days); + } + camera.range = m.range.clone(); + } + Ok(()) + } + + /// Raises an error if `must_rollback` is true. To be used on commit and in modifications. + fn check_must_rollback(&self) -> Result<(), Error> { + if self.must_rollback { + return Err(Error::new("failing due to previous error".to_owned())); + } + Ok(()) + } + + /// Looks up an existing entry in `mods` for a given camera or makes+inserts an identity entry. + fn get_mods_by_camera(mods: &mut fnv::FnvHashMap, camera_id: i32) + -> &mut CameraModification { + mods.entry(camera_id).or_insert_with(|| { + CameraModification{ + duration: recording::Duration(0), + sample_file_bytes: 0, + range: None, + days: BTreeMap::new(), + } + }) + } + + /// Fills the `range` of each `CameraModification`. This is done prior to commit so that if the + /// commit succeeds, there's no possibility that the correct state can't be retrieved. + fn precommit(&mut self) -> Result<(), Error> { + // Recompute start and end times for each camera. + for (&camera_id, m) in &mut self.mods_by_camera { + // The minimum is straightforward, taking advantage of the start_time_90k index. + let mut stmt = self.tx.prepare_cached(CAMERA_MIN_START_SQL)?; + let mut rows = stmt.query_named(&[(":camera_id", &camera_id)])?; + let min_start = match rows.next() { + Some(row) => recording::Time(row?.get_checked(0)?), + None => continue, // no data; leave m.range alone. + }; + + // There was a minimum, so there should be a maximum too. Calculating it is less + // straightforward because recordings could overlap. All recordings starting in the + // last MAX_RECORDING_DURATION must be examined in order to take advantage of the + // start_time_90k index. + let mut stmt = self.tx.prepare_cached(CAMERA_MAX_START_SQL)?; + let mut rows = stmt.query_named(&[(":camera_id", &camera_id)])?; + let mut maxes_opt = None; + while let Some(row) = rows.next() { + let row = row?; + let row_start = recording::Time(row.get_checked(0)?); + let row_duration: i64 = row.get_checked(1)?; + let row_end = recording::Time(row_start.0 + row_duration); + let maxes = match maxes_opt { + None => row_start .. row_end, + Some(Range{start: s, end: e}) => s .. cmp::max(e, row_end), + }; + if row_start.0 <= maxes.start.0 - recording::MAX_RECORDING_DURATION { + break; + } + maxes_opt = Some(maxes); + } + let max_end = match maxes_opt { + Some(Range{end: e, ..}) => e, + None => { + return Err(Error::new(format!("missing max for camera {} which had min {}", + camera_id, min_start))); + } + }; + m.range = Some(min_start .. max_end); + } + Ok(()) + } +} + +impl LockedDatabase { + /// Returns an immutable view of the cameras by id. + pub fn cameras_by_id(&self) -> &BTreeMap { &self.state.cameras_by_id } + + /// Starts a transaction for a write operation. + /// Note transactions are not needed for read operations; this process holds a lock on the + /// database directory, and the connection is locked within the process, so having a + /// `LockedDatabase` is sufficient to ensure a consistent view. + pub fn tx(&mut self) -> Result { + Ok(Transaction{ + state: &mut self.state, + mods_by_camera: fnv::FnvHashMap::default(), + tx: self.conn.transaction()?, + must_rollback: false, + bypass_reservation_for_testing: false, + }) + } + + /// Gets a given camera by uuid. + pub fn get_camera(&self, uuid: Uuid) -> Option<&Camera> { + match self.state.cameras_by_uuid.get(&uuid) { + Some(id) => Some(self.state.cameras_by_id.get(id).expect("uuid->id requires id->cam")), + None => None, + } + } + + /// Lists the specified recordings in ascending order, passing them to a supplied function. + /// Given that the function is called with the database lock held, it should be quick. + pub fn list_recordings(&self, camera_id: i32, desired_time: &Range, + mut f: F) -> Result<(), Error> + where F: FnMut(ListCameraRecordingsRow) -> Result<(), Error> { + let mut stmt = self.conn.prepare_cached(&self.state.list_recordings_sql)?; + let mut rows = stmt.query_named(&[ + (":camera_id", &camera_id), + (":start_time_90k", &desired_time.start.0), + (":end_time_90k", &desired_time.end.0)])?; + while let Some(row) = rows.next() { + let row = row?; + let id = row.get_checked(0)?; + let vse_id = row.get_checked(6)?; + let video_sample_entry = match self.state.video_sample_entries.get(&vse_id) { + Some(v) => v, + None => { + return Err(Error::new(format!( + "recording {} references nonexistent video_sample_entry {}", id, vse_id))); + }, + }; + let out = ListCameraRecordingsRow{ + id: id, + start: recording::Time(row.get_checked(1)?), + duration_90k: row.get_checked(2)?, + sample_file_bytes: row.get_checked(3)?, + video_samples: row.get_checked(4)?, + video_sync_samples: row.get_checked(5)?, + video_sample_entry: video_sample_entry.clone(), + }; + f(out)?; + } + Ok(()) + } + + /// Convenience method which calls `list_recordings` and aggregates consecutive recordings. + pub fn list_aggregated_recordings(&self, camera_id: i32, + desired_time: &Range, + forced_split: recording::Duration, + mut f: F) -> Result<(), Error> + where F: FnMut(ListAggregatedRecordingsRow) -> Result<(), Error> { + let mut agg: Option = None; + self.list_recordings(camera_id, desired_time, |row| { + let needs_flush = if let Some(ref a) = agg { + let new_dur = a.range.end - a.range.start + + recording::Duration(row.duration_90k as i64); + a.range.end != row.start || + row.video_sample_entry.id != a.video_sample_entry.id || new_dur >= forced_split + } else { + false + }; + if needs_flush { + let a = agg.take().expect("needs_flush when agg is none"); + f(a)?; + } + match agg { + None => { + agg = Some(ListAggregatedRecordingsRow{ + range: row.start .. recording::Time(row.start.0 + row.duration_90k as i64), + video_samples: row.video_samples as i64, + video_sync_samples: row.video_sync_samples as i64, + sample_file_bytes: row.sample_file_bytes as i64, + video_sample_entry: row.video_sample_entry, + }); + }, + Some(ref mut a) => { + a.range.end.0 += row.duration_90k as i64; + a.video_samples += row.video_samples as i64; + a.video_sync_samples += row.video_sync_samples as i64; + a.sample_file_bytes += row.sample_file_bytes as i64; + } + }; + Ok(()) + })?; + if let Some(a) = agg { + f(a)?; + } + Ok(()) + } + + /// Gets extra data about a single recording. + /// This uses a LRU cache to reduce the number of retrievals from the database. + pub fn get_recording(&self, recording_id: i64) + -> Result, Error> { + let mut cache = self.state.recording_cache.borrow_mut(); + if let Some(r) = cache.get_mut(&recording_id) { + debug!("cache hit for recording {}", recording_id); + return Ok(r.clone()); + } + debug!("cache miss for recording {}", recording_id); + let mut stmt = self.conn.prepare_cached(GET_RECORDING_SQL)?; + let mut rows = stmt.query_named(&[(":id", &recording_id)])?; + if let Some(row) = rows.next() { + let row = row?; + let r = Arc::new(ExtraRecording{ + sample_file_uuid: get_uuid(&row, 0)?, + video_index: row.get_checked(1)?, + }); + cache.insert(recording_id, r.clone()); + return Ok(r); + } + Err(Error::new(format!("no such recording {}", recording_id))) + } + + /// Lists all reserved sample files. + pub fn list_reserved_sample_files(&self) -> Result, Error> { + let mut reserved = Vec::new(); + let mut stmt = self.conn.prepare_cached("select uuid from reserved_sample_files;")?; + let mut rows = stmt.query_named(&[])?; + while let Some(row) = rows.next() { + let row = row?; + reserved.push(get_uuid(&row, 0)?); + } + Ok(reserved) + } + + /// Lists the oldest sample files (to delete to free room). + /// `f` should return true as long as further rows are desired. + pub fn list_oldest_sample_files(&self, camera_id: i32, mut f: F) -> Result<(), Error> + where F: FnMut(ListOldestSampleFilesRow) -> bool { + let mut stmt = self.conn.prepare_cached(LIST_OLDEST_SAMPLE_FILES_SQL)?; + let mut rows = stmt.query_named(&[(":camera_id", &(camera_id as i64))])?; + while let Some(row) = rows.next() { + let row = row?; + let start = recording::Time(row.get_checked(2)?); + let duration = recording::Duration(row.get_checked(3)?); + let should_continue = f(ListOldestSampleFilesRow{ + recording_id: row.get_checked(0)?, + uuid: get_uuid(&row, 1)?, + camera_id: camera_id, + time: start .. start + duration, + sample_file_bytes: row.get_checked(4)?, + }); + if !should_continue { + break; + } + } + Ok(()) + } + + /// Initializes the video_sample_entries. To be called during construction. + fn init_video_sample_entries(&mut self) -> Result<(), Error> { + info!("Loading video sample entries"); + let mut stmt = self.conn.prepare(r#" + select + id, + sha1, + width, + height, + data + from + video_sample_entry + "#)?; + let mut rows = stmt.query(&[])?; + while let Some(row) = rows.next() { + let row = row?; + let id = row.get_checked(0)?; + let mut sha1 = [0u8; 20]; + let sha1_vec: Vec = row.get_checked(1)?; + if sha1_vec.len() != 20 { + return Err(Error::new(format!( + "video sample entry id {} has sha1 {} of wrong length", + id, sha1_vec.len()))); + } + sha1.copy_from_slice(&sha1_vec); + self.state.video_sample_entries.insert(id, Arc::new(VideoSampleEntry{ + id: id as i32, + width: row.get_checked::<_, i32>(2)? as u16, + height: row.get_checked::<_, i32>(3)? as u16, + sha1: sha1, + data: row.get_checked(4)?, + })); + } + info!("Loaded {} video sample entries", + self.state.video_sample_entries.len()); + Ok(()) + } + + /// Initializes the cameras, but not their matching recordings. + /// To be called during construction. + fn init_cameras(&mut self) -> Result<(), Error> { + info!("Loading cameras"); + let mut stmt = self.conn.prepare(r#" + select + camera.id, + camera.uuid, + camera.short_name, + camera.description, + camera.host, + camera.username, + camera.password, + camera.main_rtsp_path, + camera.sub_rtsp_path, + camera.retain_bytes + from + camera; + "#)?; + let mut rows = stmt.query(&[])?; + while let Some(row) = rows.next() { + let row = row?; + let id = row.get_checked(0)?; + let uuid = get_uuid(&row, 1)?; + self.state.cameras_by_id.insert(id, Camera{ + id: id, + uuid: uuid, + short_name: row.get_checked(2)?, + description: row.get_checked(3)?, + host: row.get_checked(4)?, + username: row.get_checked(5)?, + password: row.get_checked(6)?, + main_rtsp_path: row.get_checked(7)?, + sub_rtsp_path: row.get_checked(8)?, + retain_bytes: row.get_checked(9)?, + range: None, + sample_file_bytes: 0, + duration: recording::Duration(0), + days: BTreeMap::new(), + }); + self.state.cameras_by_uuid.insert(uuid, id); + } + info!("Loaded {} cameras", self.state.cameras_by_id.len()); + Ok(()) + } + + /// Inserts the specified video sample entry if absent. + /// On success, returns the id of a new or existing row. + pub fn insert_video_sample_entry(&mut self, w: u16, h: u16, data: &[u8]) -> Result { + let sha1 = hash::hash(hash::Type::SHA1, data)?; + let mut sha1_bytes = [0u8; 20]; + sha1_bytes.copy_from_slice(&sha1); + + // Check if it already exists. + // There shouldn't be too many entries, so it's fine to enumerate everything. + for (&id, v) in &self.state.video_sample_entries { + if v.sha1 == sha1_bytes { + // The width and height should match given that they're also specified within data + // and thus included in the just-compared hash. + if v.width != w || v.height != h { + return Err(Error::new(format!("database entry for {:?} is {}x{}, not {}x{}", + &sha1[..], v.width, v.height, w, h))); + } + return Ok(id); + } + } + + let mut stmt = self.conn.prepare_cached(INSERT_VIDEO_SAMPLE_ENTRY_SQL)?; + stmt.execute_named(&[ + (":sha1", &sha1), + (":width", &(w as i64)), + (":height", &(h as i64)), + (":data", &data), + ])?; + + let id = self.conn.last_insert_rowid() as i32; + self.state.video_sample_entries.insert(id, Arc::new(VideoSampleEntry{ + id: id, + width: w, + height: h, + sha1: sha1_bytes, + data: data.to_vec(), + })); + + Ok(id) + } +} + +/// The recording database. Abstracts away SQLite queries. Also maintains in-memory state +/// (loaded on startup, and updated on successful commit) to avoid expensive scans over the +/// recording table on common queries. +pub struct Database(Mutex); + +impl Database { + /// Creates the database from a caller-supplied SQLite connection. + pub fn new(conn: rusqlite::Connection) -> Result { + let list_recordings_sql = format!(r#" + select + recording.id, + recording.start_time_90k, + recording.duration_90k, + recording.sample_file_bytes, + recording.video_samples, + recording.video_sync_samples, + recording.video_sample_entry_id + from + recording + where + camera_id = :camera_id and + recording.start_time_90k > :start_time_90k - {} and + recording.start_time_90k < :end_time_90k and + recording.start_time_90k + recording.duration_90k > :start_time_90k + order by + recording.start_time_90k + "#, recording::MAX_RECORDING_DURATION); + let db = Database(Mutex::new(LockedDatabase{ + conn: conn, + state: State{ + cameras_by_id: BTreeMap::new(), + cameras_by_uuid: BTreeMap::new(), + video_sample_entries: BTreeMap::new(), + recording_cache: RefCell::new(LruCache::with_hasher(1024, Default::default())), + list_recordings_sql: list_recordings_sql, + }, + })); + { + let mut l = &mut *db.0.lock().unwrap(); + l.init_video_sample_entries().map_err(Error::annotator("init_video_sample_entries"))?; + l.init_cameras().map_err(Error::annotator("init_cameras"))?; + for (&camera_id, ref mut camera) in &mut l.state.cameras_by_id { + // TODO: we could use one thread per camera if we had multiple db conns. + init_recordings(&mut l.conn, camera_id, camera) + .map_err(Error::annotator("init_recordings"))?; + } + } + Ok(db) + } + + /// Locks the database; the returned reference is the only way to perform (read or write) + /// operations. + pub fn lock(&self) -> MutexGuard { self.0.lock().unwrap() } + + /// For testing. Closes the database and return the connection. This allows verification that + /// a newly opened database is in an acceptable state. + #[cfg(test)] + fn close(self) -> rusqlite::Connection { + self.0.into_inner().unwrap().conn + } +} + +#[cfg(test)] +mod tests { + extern crate test; + + use core::cmp::Ord; + use recording::{self, TIME_UNITS_PER_SEC}; + use rusqlite::Connection; + use std::collections::BTreeMap; + use std::fmt::Debug; + use testutil; + use super::*; + use super::adjust_days; // non-public. + use uuid::Uuid; + + fn setup_conn() -> Connection { + let conn = Connection::open_in_memory().unwrap(); + let schema = include_str!("schema.sql"); + conn.execute_batch(schema).unwrap(); + conn + } + + fn setup_camera(conn: &Connection, uuid: Uuid, short_name: &str) -> i32 { + let uuid_bytes = &uuid.as_bytes()[..]; + conn.execute_named(r#" + insert into camera (uuid, short_name, description, host, username, password, + main_rtsp_path, sub_rtsp_path, retain_bytes) + values (:uuid, :short_name, :description, :host, :username, :password, + :main_rtsp_path, :sub_rtsp_path, :retain_bytes) + "#, &[ + (":uuid", &uuid_bytes), + (":short_name", &short_name), + (":description", &""), + (":host", &"test-camera"), + (":username", &"foo"), + (":password", &"bar"), + (":main_rtsp_path", &"/main"), + (":sub_rtsp_path", &"/sub"), + (":retain_bytes", &42i64), + ]).unwrap(); + conn.last_insert_rowid() as i32 + } + + fn assert_no_recordings(db: &Database, uuid: Uuid) { + let mut rows = 0; + let mut camera_id = -1; + { + let db = db.lock(); + for row in db.cameras_by_id().values() { + rows += 1; + camera_id = row.id; + assert_eq!(uuid, row.uuid); + assert_eq!("test-camera", row.host); + assert_eq!("foo", row.username); + assert_eq!("bar", row.password); + assert_eq!("/main", row.main_rtsp_path); + assert_eq!("/sub", row.sub_rtsp_path); + assert_eq!(42, row.retain_bytes); + assert_eq!(None, row.range); + assert_eq!(recording::Duration(0), row.duration); + assert_eq!(0, row.sample_file_bytes); + } + } + assert_eq!(1, rows); + + rows = 0; + { + let db = db.lock(); + let all_time = recording::Time(i64::min_value()) .. recording::Time(i64::max_value()); + db.list_recordings(camera_id, &all_time, |_row| { + rows += 1; + Ok(()) + }).unwrap(); + } + assert_eq!(0, rows); + } + + fn assert_single_recording(db: &Database, camera_uuid: Uuid, r: &RecordingToInsert) { + let mut rows = 0; + let mut camera_id = -1; + { + let db = db.lock(); + for row in db.cameras_by_id().values() { + rows += 1; + camera_id = row.id; + assert_eq!(camera_uuid, row.uuid); + assert_eq!(Some(r.time.clone()), row.range); + assert_eq!(r.sample_file_bytes as i64, row.sample_file_bytes); + assert_eq!(r.time.end - r.time.start, row.duration); + } + } + assert_eq!(1, rows); + + // TODO(slamb): test that the days logic works correctly. + + rows = 0; + let mut recording_id = -1; + { + let db = db.lock(); + let all_time = recording::Time(i64::min_value()) .. recording::Time(i64::max_value()); + db.list_recordings(camera_id, &all_time, |row| { + rows += 1; + recording_id = row.id; + assert_eq!(r.time, + row.start .. row.start + recording::Duration(row.duration_90k as i64)); + assert_eq!(r.video_samples, row.video_samples); + assert_eq!(r.video_sync_samples, row.video_sync_samples); + assert_eq!(r.sample_file_bytes, row.sample_file_bytes); + Ok(()) + }).unwrap(); + } + assert_eq!(1, rows); + + rows = 0; + db.lock().list_oldest_sample_files(camera_id, |row| { + rows += 1; + assert_eq!(recording_id, row.recording_id); + assert_eq!(r.sample_file_uuid, row.uuid); + assert_eq!(r.time, row.time); + assert_eq!(r.sample_file_bytes, row.sample_file_bytes); + true + }).unwrap(); + assert_eq!(1, rows); + + // TODO: get_recording. + } + + fn assert_unsorted_eq(mut a: Vec, mut b: Vec) + where T: Debug + Ord { + a.sort(); + b.sort(); + assert_eq!(a, b); + } + + #[test] + fn test_adjust_days() { + testutil::init(); + let mut m = BTreeMap::new(); + + // Create a day. + let test_time = recording::Time(130647162600000i64); // 2015-12-31 23:59:00 (Pacific). + let one_min = recording::Duration(60 * TIME_UNITS_PER_SEC); + let two_min = recording::Duration(2 * 60 * TIME_UNITS_PER_SEC); + let three_min = recording::Duration(3 * 60 * TIME_UNITS_PER_SEC); + let four_min = recording::Duration(4 * 60 * TIME_UNITS_PER_SEC); + let test_day1 = &CameraDayKey(*b"2015-12-31"); + let test_day2 = &CameraDayKey(*b"2016-01-01"); + adjust_days(test_time .. test_time + one_min, 1, &mut m); + assert_eq!(1, m.len()); + assert_eq!(Some(&CameraDayValue{recordings: 1, duration: one_min}), m.get(test_day1)); + + // Add to a day. + adjust_days(test_time .. test_time + one_min, 1, &mut m); + assert_eq!(1, m.len()); + assert_eq!(Some(&CameraDayValue{recordings: 2, duration: two_min}), m.get(test_day1)); + + // Subtract from a day. + adjust_days(test_time .. test_time + one_min, -1, &mut m); + assert_eq!(1, m.len()); + assert_eq!(Some(&CameraDayValue{recordings: 1, duration: one_min}), m.get(test_day1)); + + // Remove a day. + adjust_days(test_time .. test_time + one_min, -1, &mut m); + assert_eq!(0, m.len()); + + // Create two days. + adjust_days(test_time .. test_time + three_min, 1, &mut m); + assert_eq!(2, m.len()); + assert_eq!(Some(&CameraDayValue{recordings: 1, duration: one_min}), m.get(test_day1)); + assert_eq!(Some(&CameraDayValue{recordings: 1, duration: two_min}), m.get(test_day2)); + + // Add to two days. + adjust_days(test_time .. test_time + three_min, 1, &mut m); + assert_eq!(2, m.len()); + assert_eq!(Some(&CameraDayValue{recordings: 2, duration: two_min}), m.get(test_day1)); + assert_eq!(Some(&CameraDayValue{recordings: 2, duration: four_min}), m.get(test_day2)); + + // Subtract from two days. + adjust_days(test_time .. test_time + three_min, -1, &mut m); + assert_eq!(2, m.len()); + assert_eq!(Some(&CameraDayValue{recordings: 1, duration: one_min}), m.get(test_day1)); + assert_eq!(Some(&CameraDayValue{recordings: 1, duration: two_min}), m.get(test_day2)); + + // Remove two days. + adjust_days(test_time .. test_time + three_min, -1, &mut m); + assert_eq!(0, m.len()); + } + + /// Basic test of running some queries on an empty database. + #[test] + fn test_empty_db() { + testutil::init(); + let conn = setup_conn(); + let db = Database::new(conn).unwrap(); + let db = db.lock(); + assert_eq!(0, db.cameras_by_id().values().count()); + } + + /// Basic test of the full lifecycle of recording. Does not exercise error cases. + #[test] + fn test_full_lifecycle() { + testutil::init(); + let conn = setup_conn(); + let camera_uuid = Uuid::new_v4(); + let camera_id = setup_camera(&conn, camera_uuid, "testcam"); + let db = Database::new(conn).unwrap(); + assert_no_recordings(&db, camera_uuid); + + assert_eq!(db.lock().list_reserved_sample_files().unwrap(), &[]); + + let (uuid_to_use, uuid_to_keep); + { + let mut db = db.lock(); + let mut tx = db.tx().unwrap(); + uuid_to_use = tx.reserve_sample_file().unwrap(); + uuid_to_keep = tx.reserve_sample_file().unwrap(); + tx.commit().unwrap(); + } + + assert_unsorted_eq(db.lock().list_reserved_sample_files().unwrap(), + vec![uuid_to_use, uuid_to_keep]); + + let vse_id = db.lock().insert_video_sample_entry(768, 512, &[0u8; 100]).unwrap(); + assert!(vse_id > 0, "vse_id = {}", vse_id); + + // Inserting a recording should succeed and remove its uuid from the reserved table. + let start = recording::Time(1430006400 * TIME_UNITS_PER_SEC); + let recording = RecordingToInsert{ + camera_id: camera_id, + sample_file_bytes: 42, + time: start .. start + recording::Duration(TIME_UNITS_PER_SEC), + local_time: start, + video_samples: 1, + video_sync_samples: 1, + video_sample_entry_id: vse_id, + sample_file_uuid: uuid_to_use, + video_index: [0u8; 100].to_vec(), + sample_file_sha1: [0u8; 20], + }; + { + let mut db = db.lock(); + let mut tx = db.tx().unwrap(); + tx.insert_recording(&recording).unwrap(); + tx.commit().unwrap(); + } + assert_unsorted_eq(db.lock().list_reserved_sample_files().unwrap(), + vec![uuid_to_keep]); + + // Queries should return the correct result (with caches update on insert). + assert_single_recording(&db, camera_uuid, &recording); + + // Queries on a fresh database should return the correct result (with caches populated from + // existing database contents rather than built on insert). + let conn = db.close(); + let db = Database::new(conn).unwrap(); + assert_single_recording(&db, camera_uuid, &recording); + + // Deleting a recording should succeed, update the min/max times, and re-reserve the uuid. + { + let mut db = db.lock(); + let mut v = Vec::new(); + db.list_oldest_sample_files(camera_id, |r| { v.push(r); true }).unwrap(); + assert_eq!(1, v.len()); + let mut tx = db.tx().unwrap(); + tx.delete_recordings(&v).unwrap(); + tx.commit().unwrap(); + } + assert_no_recordings(&db, camera_uuid); + assert_unsorted_eq(db.lock().list_reserved_sample_files().unwrap(), + vec![uuid_to_use, uuid_to_keep]); + } + + #[test] + fn test_drop_tx() { + testutil::init(); + let conn = setup_conn(); + let db = Database::new(conn).unwrap(); + let mut db = db.lock(); + { + let mut tx = db.tx().unwrap(); + tx.reserve_sample_file().unwrap(); + // drop tx without committing. + } + + // The dropped tx should have done nothing. + assert_eq!(db.list_reserved_sample_files().unwrap(), &[]); + + // Following transactions should succeed. + let uuid; + { + let mut tx = db.tx().unwrap(); + uuid = tx.reserve_sample_file().unwrap(); + tx.commit().unwrap(); + } + assert_eq!(db.list_reserved_sample_files().unwrap(), &[uuid]); + } +} diff --git a/src/dir.rs b/src/dir.rs new file mode 100644 index 0000000..e27f1bf --- /dev/null +++ b/src/dir.rs @@ -0,0 +1,400 @@ +// This file is part of Moonfire NVR, a security camera digital video recorder. +// Copyright (C) 2016 Scott Lamb +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// In addition, as a special exception, the copyright holders give +// permission to link the code of portions of this program with the +// OpenSSL library under certain conditions as described in each +// individual source file, and distribute linked combinations including +// the two. +// +// You must obey the GNU General Public License in all respects for all +// of the code used other than OpenSSL. If you modify file(s) with this +// exception, you may extend this exception to your version of the +// file(s), but you are not obligated to do so. If you do not wish to do +// so, delete this exception statement from your version. If you delete +// this exception statement from all source files in the program, then +// also delete it here. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program. If not, see . + +//! Sample file directory management. +//! +//! This includes opening files for serving, rotating away old +//! files, and syncing new files to disk. + +use db; +use libc; +use recording; +use error::Error; +use std::ffi; +use std::fs; +use std::io::{self, Write}; +use std::mem; +use std::os::unix::io::FromRawFd; +use std::sync::{Arc, Mutex, MutexGuard}; +use std::sync::mpsc; +use std::thread; +use uuid::Uuid; + +/// A sample file directory. This is currently a singleton in production. (Maybe in the future +/// Moonfire will be extended to support multiple directories on different spindles.) +/// +/// If the directory is used for writing, the `start_syncer` function should be called to start +/// a background thread. This thread manages deleting files and writing new files. It synces the +/// directory and commits these operations to the database in the correct order to maintain the +/// invariants described in `design/schema.md`. +pub struct SampleFileDir { + db: Arc, + + /// The open file descriptor for the directory. The worker uses it to create files and sync the + /// directory. Other threads use it to open sample files for reading during video serving. + fd: Fd, + + // Lock order: don't acquire mutable.lock() while holding db.lock(). + mutable: Mutex, +} + +/// A file descriptor associated with a directory (not necessarily the sample file dir). +pub struct Fd(libc::c_int); + +impl Drop for Fd { + fn drop(&mut self) { + if unsafe { libc::close(self.0) } < 0 { + let e = io::Error::last_os_error(); + warn!("Unable to close sample file dir: {}", e); + } + } +} + +impl Fd { + /// Opens the given path as a directory. + pub fn open(path: &str) -> Result { + let cstring = ffi::CString::new(path) + .map_err(|e| io::Error::new(io::ErrorKind::InvalidInput, e))?; + let fd = unsafe { libc::open(cstring.as_ptr(), libc::O_DIRECTORY | libc::O_RDONLY, 0) }; + if fd < 0 { + return Err(io::Error::last_os_error().into()); + } + Ok(Fd(fd)) + } + + /// Locks the directory with the specified `flock` operation. + pub fn lock(&self, operation: libc::c_int) -> Result<(), io::Error> { + let ret = unsafe { libc::flock(self.0, operation) }; + if ret < 0 { + return Err(io::Error::last_os_error().into()); + } + Ok(()) + } +} + +impl SampleFileDir { + pub fn new(path: &str, db: Arc) -> Result, Error> { + let fd = Fd::open(path)?; + Ok(Arc::new(SampleFileDir{ + db: db, + fd: fd, + mutable: Mutex::new(SharedMutableState{ + next_uuid: None, + }), + })) + } + + /// Opens the given sample file for reading. + pub fn open_sample_file(&self, uuid: Uuid) -> Result { + self.open_int(uuid, libc::O_RDONLY, 0) + } + + /// Creates a new writer. + /// Note this doesn't wait for previous rotation to complete; it's assumed the sample file + /// directory has sufficient space for a couple recordings per camera in addition to the + /// cameras' total `retain_bytes`. + pub fn create_writer(&self, start: recording::Time, local_start: recording::Time, + camera_id: i32, video_sample_entry_id: i32) + -> Result { + // Grab the next uuid. Typically one is cached—a sync has usually completed since the last + // writer was created, and syncs ensure `next_uuid` is filled while performing their + // transaction. But if not, perform an extra database transaction to reserve a new one. + let uuid = match self.mutable.lock().unwrap().next_uuid.take() { + Some(u) => u, + None => { + info!("Committing extra transaction because there's no cached uuid"); + let mut db = self.db.lock(); + let mut tx = db.tx()?; + let u = tx.reserve_sample_file()?; + tx.commit()?; + u + }, + }; + + let f = match self.open_int(uuid, libc::O_WRONLY | libc::O_EXCL | libc::O_CREAT, 0o600) { + Ok(f) => f, + Err(e) => { + self.mutable.lock().unwrap().next_uuid = Some(uuid); + return Err(e.into()); + }, + }; + recording::Writer::open(f, uuid, start, local_start, camera_id, video_sample_entry_id) + } + + /// Opens a sample file within this directory with the given flags and (if creating) mode. + fn open_int(&self, uuid: Uuid, flags: libc::c_int, mode: libc::c_int) + -> Result { + let p = SampleFileDir::get_rel_pathname(uuid); + let fd = unsafe { libc::openat(self.fd.0, p.as_ptr(), flags, mode) }; + if fd < 0 { + return Err(io::Error::last_os_error()) + } + unsafe { Ok(fs::File::from_raw_fd(fd)) } + } + + /// Gets a pathname for a sample file suitable for passing to open or unlink. + fn get_rel_pathname(uuid: Uuid) -> [libc::c_char; 37] { + let mut buf = [0u8; 37]; + write!(&mut buf[..36], "{}", uuid.hyphenated()).expect("can't format uuid to pathname buf"); + + // libc::c_char seems to be i8 on some platforms (Linux/arm) and u8 on others (Linux/amd64). + // Transmute, suppressing the warning that happens on platforms in which it's already u8. + #[allow(useless_transmute)] + unsafe { mem::transmute::<[u8; 37], [libc::c_char; 37]>(buf) } + } + + /// Unlinks the given sample file within this directory. + fn unlink(fd: &Fd, uuid: Uuid) -> Result<(), io::Error> { + let p = SampleFileDir::get_rel_pathname(uuid); + let res = unsafe { libc::unlinkat(fd.0, p.as_ptr(), 0) }; + if res < 0 { + return Err(io::Error::last_os_error()) + } + Ok(()) + } + + /// Syncs the directory itself. + fn sync(&self) -> Result<(), io::Error> { + let res = unsafe { libc::fsync(self.fd.0) }; + if res < 0 { + return Err(io::Error::last_os_error()) + }; + Ok(()) + } +} + +/// State shared between users of the `SampleFileDirectory` struct and the syncer. +struct SharedMutableState { + next_uuid: Option, +} + +/// A command sent to the syncer. These correspond to methods in the `SyncerChannel` struct. +enum SyncerCommand { + AsyncSaveWriter(db::RecordingToInsert, fs::File), + + #[cfg(test)] + Flush(mpsc::SyncSender<()>), +} + +/// A channel which can be used to send commands to the syncer. +/// Can be cloned to allow multiple threads to send commands. +#[derive(Clone)] +pub struct SyncerChannel(mpsc::Sender); + +/// State of the worker thread. +struct SyncerState { + dir: Arc, + to_unlink: Vec, + to_mark_deleted: Vec, + cmds: mpsc::Receiver, +} + +/// Starts a syncer for the given sample file directory. +/// There should be only one syncer per directory, or 0 if operating in read-only mode. +/// This function will perform the initial rotation synchronously, so that it is finished before +/// file writing starts. Afterward the syncing happens in a background thread. +/// +/// Returns a `SyncerChannel` which can be used to send commands (and can be cloned freely) and +/// a `JoinHandle` for the syncer thread. At program shutdown, all `SyncerChannel` clones should be +/// removed and then the handle joined to allow all recordings to be persisted. +pub fn start_syncer(dir: Arc) + -> Result<(SyncerChannel, thread::JoinHandle<()>), Error> { + let to_unlink = dir.db.lock().list_reserved_sample_files()?; + let (snd, rcv) = mpsc::channel(); + let mut state = SyncerState { + dir: dir, + to_unlink: to_unlink, + to_mark_deleted: Vec::new(), + cmds: rcv, + }; + state.initial_rotation()?; + Ok((SyncerChannel(snd), + thread::Builder::new().name("syncer".into()).spawn(move || state.run()).unwrap())) +} + +impl SyncerChannel { + /// Asynchronously syncs the given writer, closes it, records it into the database, and + /// starts rotation. + pub fn async_save_writer(&self, w: recording::Writer) -> Result<(), Error> { + let (recording, f) = w.close()?; + self.0.send(SyncerCommand::AsyncSaveWriter(recording, f)).unwrap(); + Ok(()) + } + + /// For testing: flushes the syncer, waiting for all currently-queued commands to complete. + #[cfg(test)] + pub fn flush(&self) { + let (snd, rcv) = mpsc::sync_channel(0); + self.0.send(SyncerCommand::Flush(snd)).unwrap(); + rcv.recv().unwrap_err(); // syncer should just drop the channel, closing it. + } +} + +impl SyncerState { + fn run(&mut self) { + loop { + match self.cmds.recv() { + Err(_) => return, // all senders have closed the channel; shutdown + Ok(SyncerCommand::AsyncSaveWriter(recording, f)) => self.save_writer(recording, f), + + #[cfg(test)] + Ok(SyncerCommand::Flush(_)) => {}, // just drop the supplied sender, closing it. + }; + } + } + + /// Rotates files for all cameras and deletes stale reserved uuids from previous runs. + fn initial_rotation(&mut self) -> Result<(), Error> { + let mut to_delete = Vec::new(); + { + let mut db = self.dir.db.lock(); + for (camera_id, camera) in db.cameras_by_id() { + self.get_rows_to_delete(&db, *camera_id, camera, 0, &mut to_delete)?; + } + let mut tx = db.tx()?; + tx.delete_recordings(&to_delete)?; + tx.commit()?; + } + for row in to_delete { + self.to_unlink.push(row.uuid); + } + self.try_unlink(); + if !self.to_unlink.is_empty() { + return Err(Error::new(format!("failed to unlink {} sample files", + self.to_unlink.len()))); + } + self.dir.sync()?; + { + let mut db = self.dir.db.lock(); + let mut tx = db.tx()?; + tx.mark_sample_files_deleted(&self.to_mark_deleted)?; + tx.commit()?; + } + self.to_mark_deleted.clear(); + Ok(()) + } + + /// Saves the given writer and causes rotation to happen. + /// Note that part of rotation is deferred for the next cycle (saved writing or program startup) + /// so that there can be only one dir sync and database transaction per save. + fn save_writer(&mut self, recording: db::RecordingToInsert, f: fs::File) { + if let Err(e) = self.save_writer_helper(&recording, f) { + error!("camera {}: will discard recording {} due to error while saving: {}", + recording.camera_id, recording.sample_file_uuid, e); + self.to_unlink.push(recording.sample_file_uuid); + return; + } + } + + /// Internal helper for `save_writer`. This is separated out so that the question-mark operator + /// can be used in the many error paths. + fn save_writer_helper(&mut self, recording: &db::RecordingToInsert, f: fs::File) + -> Result<(), Error> { + self.try_unlink(); + if !self.to_unlink.is_empty() { + return Err(Error::new(format!("failed to unlink {} files.", self.to_unlink.len()))); + } + f.sync_all()?; + self.dir.sync()?; + + let mut to_delete = Vec::new(); + let mut l = self.dir.mutable.lock().unwrap(); + let mut db = self.dir.db.lock(); + let mut new_next_uuid = l.next_uuid; + { + let camera = + db.cameras_by_id().get(&recording.camera_id) + .ok_or_else(|| Error::new(format!("no such camera {}", recording.camera_id)))?; + self.get_rows_to_delete(&db, recording.camera_id, camera, + recording.sample_file_bytes as i64, &mut to_delete)?; + } + let mut tx = db.tx()?; + tx.mark_sample_files_deleted(&self.to_mark_deleted)?; + tx.delete_recordings(&to_delete)?; + if new_next_uuid.is_none() { + new_next_uuid = Some(tx.reserve_sample_file()?); + } + tx.insert_recording(recording)?; + tx.commit()?; + l.next_uuid = new_next_uuid; + + self.to_mark_deleted.clear(); + self.to_unlink.extend(to_delete.iter().map(|row| row.uuid)); + Ok(()) + } + + /// Gets rows to delete to bring a camera's disk usage within bounds. + fn get_rows_to_delete(&self, db: &MutexGuard, camera_id: i32, + camera: &db::Camera, extra_bytes_needed: i64, + to_delete: &mut Vec) -> Result<(), Error> { + let bytes_needed = camera.sample_file_bytes + extra_bytes_needed - camera.retain_bytes; + let mut bytes_to_delete = 0; + if bytes_needed <= 0 { + debug!("{}: have remaining quota of {}", camera.short_name, -bytes_needed); + return Ok(()); + } + let mut n = 0; + db.list_oldest_sample_files(camera_id, |row| { + bytes_to_delete += row.sample_file_bytes as i64; + to_delete.push(row); + n += 1; + bytes_needed > bytes_to_delete // continue as long as more deletions are needed. + })?; + if bytes_needed > bytes_to_delete { + return Err(Error::new(format!("{}: couldn't find enough files to delete: {} left.", + camera.short_name, bytes_needed))); + } + info!("{}: deleting {} bytes in {} recordings ({} bytes needed)", + camera.short_name, bytes_to_delete, n, bytes_needed); + Ok(()) + } + + /// Tries to unlink all the uuids in `self.to_unlink`. Any which can't be unlinked will + /// be retained in the vec. + fn try_unlink(&mut self) { + let to_mark_deleted = &mut self.to_mark_deleted; + let fd = &self.dir.fd; + self.to_unlink.retain(|uuid| { + if let Err(e) = SampleFileDir::unlink(fd, *uuid) { + if e.kind() == io::ErrorKind::NotFound { + warn!("dir: Sample file {} already deleted!", uuid.hyphenated()); + to_mark_deleted.push(*uuid); + false + } else { + warn!("dir: Unable to unlink {}: {}", uuid.hyphenated(), e); + true + } + } else { + to_mark_deleted.push(*uuid); + false + } + }); + } +} diff --git a/src/error.rs b/src/error.rs new file mode 100644 index 0000000..0edb1a0 --- /dev/null +++ b/src/error.rs @@ -0,0 +1,145 @@ +// This file is part of Moonfire NVR, a security camera digital video recorder. +// Copyright (C) 2016 Scott Lamb +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// In addition, as a special exception, the copyright holders give +// permission to link the code of portions of this program with the +// OpenSSL library under certain conditions as described in each +// individual source file, and distribute linked combinations including +// the two. +// +// You must obey the GNU General Public License in all respects for all +// of the code used other than OpenSSL. If you modify file(s) with this +// exception, you may extend this exception to your version of the +// file(s), but you are not obligated to do so. If you do not wish to do +// so, delete this exception statement from your version. If you delete +// this exception statement from all source files in the program, then +// also delete it here. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program. If not, see . + +extern crate rusqlite; +extern crate time; +extern crate uuid; + +use core::ops::Deref; +use core::num; +use ffmpeg; +use openssl::error::ErrorStack; +use serde_json; +use std::boxed::Box; +use std::convert::From; +use std::error; +use std::fmt; +use std::io; +use std::result; +use std::string::String; + +#[derive(Debug)] +pub struct Error { + pub description: String, + pub cause: Option>, +} + +impl Error { + pub fn new(description: String) -> Self { + Error{description: description, cause: None } + } + + // Returns a function to pass to std::result::Result::map_err which annotates the given error + // with a prefix. + pub fn annotator(prefix: &'static str) -> impl Fn(Error) -> Error { + move |e| { Error{description: format!("{}: {}", prefix, e.description), cause: e.cause} } + } +} + +impl error::Error for Error { + fn description(&self) -> &str { &self.description } + fn cause(&self) -> Option<&error::Error> { + match self.cause { + Some(ref b) => Some(b.deref()), + None => None + } + } +} + +impl fmt::Display for Error { + fn fmt(&self, f: &mut fmt::Formatter) -> result::Result<(), fmt::Error> { + write!(f, "Error: {}", self.description) + } +} + +// TODO(slamb): isn't there a "" or some such? + +impl From for Error { + fn from(err: rusqlite::Error) -> Self { + use std::error::{Error as E}; + Error{description: String::from(err.description()), + cause: Some(Box::new(err))} + } +} + +impl From for Error { + fn from(err: io::Error) -> Self { + use std::error::{Error as E}; + Error{description: String::from(err.description()), + cause: Some(Box::new(err))} + } +} + +impl From for Error { + fn from(err: time::ParseError) -> Self { + use std::error::{Error as E}; + Error{description: String::from(err.description()), + cause: Some(Box::new(err))} + } +} + +impl From for Error { + fn from(err: num::ParseIntError) -> Self { + use std::error::{Error as E}; + Error{description: err.description().to_owned(), + cause: Some(Box::new(err))} + } +} + +impl From for Error { + fn from(err: serde_json::Error) -> Self { + use std::error::{Error as E}; + Error{description: format!("{} ({})", err.description(), err), + cause: Some(Box::new(err))} + } +} + +impl From for Error { + fn from(err: ffmpeg::Error) -> Self { + use std::error::{Error as E}; + Error{description: format!("{} ({})", err.description(), err), + cause: Some(Box::new(err))} + } +} + +impl From for Error { + fn from(_: uuid::ParseError) -> Self { + Error{description: String::from("UUID parse error"), + cause: None} + } +} + +impl From for Error { + fn from(_: ErrorStack) -> Self { + Error{description: String::from("openssl error"), cause: None} + } +} + +pub type Result = result::Result; diff --git a/src/ffmpeg.cc b/src/ffmpeg.cc deleted file mode 100644 index 31fc312..0000000 --- a/src/ffmpeg.cc +++ /dev/null @@ -1,316 +0,0 @@ -// This file is part of Moonfire NVR, a security camera digital video recorder. -// Copyright (C) 2016 Scott Lamb -// -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// In addition, as a special exception, the copyright holders give -// permission to link the code of portions of this program with the -// OpenSSL library under certain conditions as described in each -// individual source file, and distribute linked combinations including -// the two. -// -// You must obey the GNU General Public License in all respects for all -// of the code used other than OpenSSL. If you modify file(s) with this -// exception, you may extend this exception to your version of the -// file(s), but you are not obligated to do so. If you do not wish to do -// so, delete this exception statement from your version. If you delete -// this exception statement from all source files in the program, then -// also delete it here. -// -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. -// -// You should have received a copy of the GNU General Public License -// along with this program. If not, see . -// -// ffmpeg.cc: See ffmpeg.h for description. - -#include "ffmpeg.h" - -#include - -extern "C" { -#include -#include -#include -#include -#include -#include -} // extern "C" - -#include - -#include "string.h" - -// libav lacks this ffmpeg constant. -#ifndef AV_ERROR_MAX_STRING_SIZE -#define AV_ERROR_MAX_STRING_SIZE 64 -#endif - -DEFINE_int32(avlevel, AV_LOG_INFO, - "maximum logging level for ffmpeg/libav; " - "higher levels will be ignored."); - -namespace moonfire_nvr { - -namespace { - -std::string AvError2Str(re2::StringPiece function, int err) { - char str[AV_ERROR_MAX_STRING_SIZE]; - if (av_strerror(err, str, sizeof(str)) == 0) { - return StrCat(function, ": ", str); - } - return StrCat(function, ": unknown error ", err); -} - -struct Dictionary { - Dictionary() {} - Dictionary(const Dictionary &) = delete; - Dictionary &operator=(const Dictionary &) = delete; - ~Dictionary() { av_dict_free(&dict); } - - bool Set(const char *key, const char *value, std::string *error_message) { - int ret = av_dict_set(&dict, key, value, 0); - if (ret < 0) { - *error_message = AvError2Str("av_dict_set", ret); - return false; - } - return true; - } - - bool size() const { return av_dict_count(dict); } - - AVDictionary *dict = nullptr; -}; - -google::LogSeverity GlogLevelFromAvLevel(int avlevel) { - if (avlevel >= AV_LOG_INFO) { - return google::GLOG_INFO; - } else if (avlevel >= AV_LOG_WARNING) { - return google::GLOG_WARNING; - } else if (avlevel > AV_LOG_PANIC) { - return google::GLOG_ERROR; - } else { - return google::GLOG_FATAL; - } -} - -void AvLogCallback(void *avcl, int avlevel, const char *fmt, va_list vl) { - if (avlevel > FLAGS_avlevel) { - return; - } - - // google::LogMessage expects a "file" and "line" to be prefixed to the - // log message, like so: - // - // W1210 11:00:32.224936 28739 ffmpeg_rtsp:0] Estimating duration ... - // ^file ^line - // - // Normally this is filled in via the __FILE__ and __LINE__ - // C preprocessor macros. In this case, try to fill in something useful - // based on the information ffmpeg supplies. - std::string file("ffmpeg"); - if (avcl != nullptr) { - auto *avclass = *reinterpret_cast(avcl); - file.push_back('_'); - file.append(avclass->item_name(avcl)); - } - char line[512]; - vsnprintf(line, sizeof(line), fmt, vl); - google::LogSeverity glog_level = GlogLevelFromAvLevel(avlevel); - google::LogMessage(file.c_str(), 0, glog_level).stream() << line; -} - -int AvLockCallback(void **mutex, enum AVLockOp op) { - auto typed_mutex = reinterpret_cast(mutex); - switch (op) { - case AV_LOCK_CREATE: - LOG_IF(DFATAL, *typed_mutex != nullptr) - << "creating mutex over existing value."; - *typed_mutex = new std::mutex; - break; - case AV_LOCK_DESTROY: - delete *typed_mutex; - *typed_mutex = nullptr; - break; - case AV_LOCK_OBTAIN: - (*typed_mutex)->lock(); - break; - case AV_LOCK_RELEASE: - (*typed_mutex)->unlock(); - break; - } - return 0; -} - -std::string StringifyVersion(int version_int) { - return StrCat((version_int >> 16) & 0xFF, ".", (version_int >> 8) & 0xFF, ".", - (version_int)&0xFF); -} - -void LogVersion(const char *library_name, int compiled_version, - int running_version, const char *configuration) { - LOG(INFO) << library_name << ": compiled with version " - << StringifyVersion(compiled_version) << ", running with version " - << StringifyVersion(running_version) - << ", configuration: " << configuration; -} - -class RealInputVideoPacketStream : public InputVideoPacketStream { - public: - RealInputVideoPacketStream() { - ctx_ = CHECK_NOTNULL(avformat_alloc_context()); - } - - RealInputVideoPacketStream(const RealInputVideoPacketStream &) = delete; - RealInputVideoPacketStream &operator=(const RealInputVideoPacketStream &) = - delete; - - ~RealInputVideoPacketStream() final { - avformat_close_input(&ctx_); - avformat_free_context(ctx_); - } - - bool GetNext(VideoPacket *pkt, std::string *error_message) final { - while (true) { - av_packet_unref(pkt->pkt()); - int ret = av_read_frame(ctx_, pkt->pkt()); - if (ret != 0) { - if (ret == AVERROR_EOF) { - error_message->clear(); - } else { - *error_message = AvError2Str("av_read_frame", ret); - } - return false; - } - if (pkt->pkt()->stream_index != stream_index_) { - VLOG(3) << "Ignoring packet for stream " << pkt->pkt()->stream_index - << "; only interested in " << stream_index_; - continue; - } - VLOG(3) << "Read packet with pts=" << pkt->pkt()->pts - << ", dts=" << pkt->pkt()->dts << ", key=" << pkt->is_key(); - return true; - } - } - - const AVStream *stream() const final { return ctx_->streams[stream_index_]; } - - private: - friend class RealVideoSource; - int64_t min_next_pts_ = std::numeric_limits::min(); - int64_t min_next_dts_ = std::numeric_limits::min(); - AVFormatContext *ctx_ = nullptr; // owned. - int stream_index_ = -1; -}; - -class RealVideoSource : public VideoSource { - public: - RealVideoSource() { - CHECK_GE(0, av_lockmgr_register(&AvLockCallback)); - av_log_set_callback(&AvLogCallback); - av_register_all(); - avformat_network_init(); - LogVersion("avutil", LIBAVUTIL_VERSION_INT, avutil_version(), - avutil_configuration()); - LogVersion("avformat", LIBAVFORMAT_VERSION_INT, avformat_version(), - avformat_configuration()); - LogVersion("avcodec", LIBAVCODEC_VERSION_INT, avcodec_version(), - avcodec_configuration()); - } - - std::unique_ptr OpenRtsp( - const std::string &url, std::string *error_message) final { - std::unique_ptr stream; - Dictionary open_options; - if (!open_options.Set("rtsp_transport", "tcp", error_message) || - // https://trac.ffmpeg.org/ticket/5018 workaround attempt. - !open_options.Set("probesize", "262144", error_message) || - !open_options.Set("user-agent", "moonfire-nvr", error_message) || - // 10-second socket timeout, in microseconds. - !open_options.Set("stimeout", "10000000", error_message)) { - return stream; - } - - stream = OpenCommon(url, &open_options.dict, error_message); - if (stream == nullptr) { - return stream; - } - - // Discard the first packet. - LOG(INFO) << "Discarding the first packet to work around " - "https://trac.ffmpeg.org/ticket/5018"; - VideoPacket dummy; - if (!stream->GetNext(&dummy, error_message)) { - stream.reset(); - } - - return stream; - } - - std::unique_ptr OpenFile( - const std::string &filename, std::string *error_message) final { - AVDictionary *open_options = nullptr; - return OpenCommon(filename, &open_options, error_message); - } - - private: - std::unique_ptr OpenCommon( - const std::string &source, AVDictionary **dict, - std::string *error_message) { - std::unique_ptr stream( - new RealInputVideoPacketStream); - - int ret = avformat_open_input(&stream->ctx_, source.c_str(), nullptr, dict); - if (ret != 0) { - *error_message = AvError2Str("avformat_open_input", ret); - return std::unique_ptr(); - } - - if (av_dict_count(*dict) != 0) { - std::vector ignored; - AVDictionaryEntry *ent = nullptr; - while ((ent = av_dict_get(*dict, "", ent, AV_DICT_IGNORE_SUFFIX)) != - nullptr) { - ignored.push_back(StrCat(ent->key, "=", ent->value)); - } - LOG(WARNING) << "avformat_open_input ignored " << ignored.size() - << " options: " << Join(ignored, ", "); - } - - ret = avformat_find_stream_info(stream->ctx_, nullptr); - if (ret < 0) { - *error_message = AvError2Str("avformat_find_stream_info", ret); - return std::unique_ptr(); - } - - // Find the video stream. - for (unsigned int i = 0; i < stream->ctx_->nb_streams; ++i) { - if (stream->ctx_->streams[i]->codec->codec_type == AVMEDIA_TYPE_VIDEO) { - VLOG(1) << "Video stream index is " << i; - stream->stream_index_ = i; - break; - } - } - if (stream->stream() == nullptr) { - *error_message = StrCat("no video stream"); - return std::unique_ptr(); - } - - return std::unique_ptr(stream.release()); - } -}; - -} // namespace - -VideoSource *GetRealVideoSource() { - static auto *real_video_source = new RealVideoSource; // never deleted. - return real_video_source; -} - -} // namespace moonfire_nvr diff --git a/src/ffmpeg.h b/src/ffmpeg.h deleted file mode 100644 index 8528fbf..0000000 --- a/src/ffmpeg.h +++ /dev/null @@ -1,149 +0,0 @@ -// This file is part of Moonfire NVR, a security camera digital video recorder. -// Copyright (C) 2016 Scott Lamb -// -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// In addition, as a special exception, the copyright holders give -// permission to link the code of portions of this program with the -// OpenSSL library under certain conditions as described in each -// individual source file, and distribute linked combinations including -// the two. -// -// You must obey the GNU General Public License in all respects for all -// of the code used other than OpenSSL. If you modify file(s) with this -// exception, you may extend this exception to your version of the -// file(s), but you are not obligated to do so. If you do not wish to do -// so, delete this exception statement from your version. If you delete -// this exception statement from all source files in the program, then -// also delete it here. -// -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. -// -// You should have received a copy of the GNU General Public License -// along with this program. If not, see . -// -// ffmpeg.h: ffmpeg (or libav) wrappers for operations needed by moonfire_nvr. -// This is not a general-purpose wrapper. It makes assumptions about the -// data we will be operated on and the desired operations, such as: -// -// * The input should contain no "B" frames (bi-directionally predicted -// pictures) and thus input frames should be strictly in order of ascending -// PTS as well as DTS. -// -// * Only video frames are of interest. - -#ifndef MOONFIRE_NVR_FFMPEG_H -#define MOONFIRE_NVR_FFMPEG_H - -#include -#include -#include - -#include -#include - -extern "C" { -#include -} // extern "C" - -namespace moonfire_nvr { - -// An encoded video packet. -class VideoPacket { - public: - VideoPacket() { av_init_packet(&pkt_); } - VideoPacket(const VideoPacket &) = delete; - VideoPacket &operator=(const VideoPacket &) = delete; - ~VideoPacket() { av_packet_unref(&pkt_); } - - // Returns iff this packet represents a key frame. - // - // (A key frame is one that can be decoded without previous frames.) - // - // PRE: this packet is valid, as if it has been filled by - // InputVideoPacketStream::Next. - bool is_key() const { return (pkt_.flags & AV_PKT_FLAG_KEY) != 0; } - - int64_t pts() const { return pkt_.pts; } - - AVPacket *pkt() { return &pkt_; } - const AVPacket *pkt() const { return &pkt_; } - - re2::StringPiece data() { - return re2::StringPiece(reinterpret_cast(pkt_.data), - pkt_.size); - } - - private: - AVPacket pkt_; -}; - -// An input stream of (still-encoded) video packets. -class InputVideoPacketStream { - public: - InputVideoPacketStream() {} - InputVideoPacketStream(const InputVideoPacketStream &) = delete; - InputVideoPacketStream &operator=(const InputVideoPacketStream &) = delete; - - // Closes the stream. - virtual ~InputVideoPacketStream() {} - - // Get the next packet. - // - // Returns true iff one is available, false on EOF or failure. - // |error_message| will be filled on failure, empty on EOF. - // - // PRE: the stream is healthy: there was no prior Close() call or GetNext() - // failure. - virtual bool GetNext(VideoPacket *pkt, std::string *error_message) = 0; - - // Returns the video stream. - virtual const AVStream *stream() const = 0; - - re2::StringPiece extradata() const { - return re2::StringPiece( - reinterpret_cast(stream()->codec->extradata), - stream()->codec->extradata_size); - } -}; - -// A class which opens streams. -// There's one of these for proudction use; see GetRealVideoSource(). -// It's an abstract class for testability. -class VideoSource { - public: - virtual ~VideoSource() {} - - // Open the given RTSP URL, accessing the first video stream. - // - // The RTSP URL will be opened with TCP and a hardcoded socket timeout. - // - // The first frame will be automatically discarded as a bug workaround. - // https://trac.ffmpeg.org/ticket/5018 - // - // Returns success, filling |error_message| on failure. - // - // PRE: closed. - virtual std::unique_ptr OpenRtsp( - const std::string &url, std::string *error_message) = 0; - - // Open the given video file, accessing the first video stream. - // - // Returns the stream. On failure, returns nullptr and fills - // |error_message|. - virtual std::unique_ptr OpenFile( - const std::string &filename, std::string *error_message) = 0; -}; - -// Returns a VideoSource for production use, which will never be deleted. -VideoSource *GetRealVideoSource(); - -} // namespace moonfire_nvr - -#endif // MOONFIRE_NVR_FFMPEG_H diff --git a/src/filesystem.cc b/src/filesystem.cc deleted file mode 100644 index f901912..0000000 --- a/src/filesystem.cc +++ /dev/null @@ -1,222 +0,0 @@ -// This file is part of Moonfire NVR, a security camera digital video recorder. -// Copyright (C) 2016 Scott Lamb -// -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// In addition, as a special exception, the copyright holders give -// permission to link the code of portions of this program with the -// OpenSSL library under certain conditions as described in each -// individual source file, and distribute linked combinations including -// the two. -// -// You must obey the GNU General Public License in all respects for all -// of the code used other than OpenSSL. If you modify file(s) with this -// exception, you may extend this exception to your version of the -// file(s), but you are not obligated to do so. If you do not wish to do -// so, delete this exception statement from your version. If you delete -// this exception statement from all source files in the program, then -// also delete it here. -// -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. -// -// You should have received a copy of the GNU General Public License -// along with this program. If not, see . -// -// filesystem.cc: See filesystem.h. - -#include "filesystem.h" - -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include -#include - -#include -#include -#include -#include -#include -#include - -#include "string.h" - -namespace moonfire_nvr { - -namespace { - -class RealFile : public File { - public: - RealFile(re2::StringPiece name, int fd) - : name_(name.data(), name.size()), fd_(fd) {} - RealFile(const RealFile &) = delete; - void operator=(const RealFile &) = delete; - - ~RealFile() final { Close(); } - - const std::string &name() const { return name_; } - - int Access(const char *path, int mode, int flags) final { - return faccessat(fd_, path, mode, flags) < 0 ? errno : 0; - } - - int Close() final { - if (fd_ < 0) { - return 0; - } - int ret; - while ((ret = close(fd_)) != 0 && errno == EINTR) - ; - if (ret != 0) { - return errno; - } - fd_ = -1; - return 0; - } - - int Lock(int operation) final { - return (flock(fd_, operation) < 0) ? errno : 0; - } - - int Open(const char *path, int flags, int *fd) final { - return Open(path, flags, 0, fd); - } - - int Open(const char *path, int flags, std::unique_ptr *f) final { - return Open(path, flags, 0, f); - } - - int Open(const char *path, int flags, mode_t mode, int *fd) final { - int ret = openat(fd_, path, flags, mode); - if (ret < 0) { - return errno; - } - *fd = ret; - return 0; - } - - int Open(const char *path, int flags, mode_t mode, - std::unique_ptr *f) final { - int ret = openat(fd_, path, flags, mode); - if (ret < 0) { - return errno; - } - f->reset(new RealFile(StrCat(name_, "/", path), ret)); - return 0; - } - - int Read(void *buf, size_t size, size_t *bytes_read) final { - ssize_t ret; - while ((ret = read(fd_, buf, size)) == -1 && errno == EINTR) - ; - if (ret < 0) { - return errno; - } - *bytes_read = static_cast(ret); - return 0; - } - - int Stat(struct stat *buf) final { return (fstat(fd_, buf) < 0) ? errno : 0; } - - int Sync() final { return (fsync(fd_) < 0) ? errno : 0; } - - int Truncate(off_t length) final { - return (ftruncate(fd_, length) < 0) ? errno : 0; - } - - int Unlink(const char *pathname) { - return unlinkat(fd_, pathname, 0) != 0 ? errno : 0; - } - - int Write(re2::StringPiece data, size_t *bytes_written) final { - ssize_t ret; - while ((ret = write(fd_, data.data(), data.size())) == -1 && errno == EINTR) - ; - if (ret < 0) { - return errno; - } - *bytes_written = static_cast(ret); - return 0; - } - - private: - std::string name_; - int fd_ = -1; -}; - -class RealFilesystem : public Filesystem { - public: - bool DirForEach(const char *dir_path, - std::function fn, - std::string *error_message) final { - DIR *owned_dir = opendir(dir_path); - if (owned_dir == nullptr) { - int err = errno; - *error_message = - StrCat("Unable to examine ", dir_path, ": ", strerror(err)); - return false; - } - struct dirent *ent; - while (errno = 0, (ent = readdir(owned_dir)) != nullptr) { - if (fn(ent) == IterationControl::kBreak) { - closedir(owned_dir); - return true; - } - } - int err = errno; - closedir(owned_dir); - if (err != 0) { - *error_message = StrCat("readdir failed: ", strerror(err)); - return false; - } - return true; - } - - int Open(const char *path, int flags, std::unique_ptr *f) final { - return Open(path, flags, 0, f); - } - - int Open(const char *path, int flags, mode_t mode, - std::unique_ptr *f) final { - int ret = open(path, flags, mode); - if (ret < 0) { - return errno; - } - f->reset(new RealFile(path, ret)); - return 0; - } - - int Mkdir(const char *path, mode_t mode) final { - return (mkdir(path, mode) < 0) ? errno : 0; - } - - int Rmdir(const char *path) final { return (rmdir(path) < 0) ? errno : 0; } - - int Stat(const char *path, struct stat *buf) final { - return (stat(path, buf) < 0) ? errno : 0; - } - - int Unlink(const char *path) final { return (unlink(path) < 0) ? errno : 0; } -}; - -} // namespace - -Filesystem *GetRealFilesystem() { - static Filesystem *real_filesystem = new RealFilesystem; - return real_filesystem; -} - -} // namespace moonfire_nvr diff --git a/src/filesystem.h b/src/filesystem.h deleted file mode 100644 index 824bd06..0000000 --- a/src/filesystem.h +++ /dev/null @@ -1,140 +0,0 @@ -// This file is part of Moonfire NVR, a security camera digital video recorder. -// Copyright (C) 2016 Scott Lamb -// -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// In addition, as a special exception, the copyright holders give -// permission to link the code of portions of this program with the -// OpenSSL library under certain conditions as described in each -// individual source file, and distribute linked combinations including -// the two. -// -// You must obey the GNU General Public License in all respects for all -// of the code used other than OpenSSL. If you modify file(s) with this -// exception, you may extend this exception to your version of the -// file(s), but you are not obligated to do so. If you do not wish to do -// so, delete this exception statement from your version. If you delete -// this exception statement from all source files in the program, then -// also delete it here. -// -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. -// -// You should have received a copy of the GNU General Public License -// along with this program. If not, see . -// -// filesystem.h: helpers for dealing with the local filesystem. - -#ifndef MOONFIRE_NVR_FILESYSTEM_H -#define MOONFIRE_NVR_FILESYSTEM_H - -#include -#include -#include -#include - -#include -#include -#include - -#include -#include -#include -#include - -#include "common.h" - -namespace moonfire_nvr { - -// Represents an open file descriptor. All methods but Close() are thread-safe. -class File { - public: - // Close the file, ignoring the result. - virtual ~File() {} - - // A name for the file (typically assigned at open time). - virtual const std::string &name() const = 0; - - // faccessat(), returning 0 on success or errno>0 on failure. - virtual int Access(const char *path, int mode, int flags) = 0; - - // Close the file, returning 0 on success or errno>0 on failure. - // Already closed is considered a success. - virtual int Close() = 0; - - // flock(), returning 0 on success or errno>0 on failure. - virtual int Lock(int operation) = 0; - - // openat(), returning 0 on success or errno>0 on failure. - virtual int Open(const char *path, int flags, int *fd) = 0; - virtual int Open(const char *path, int flags, std::unique_ptr *f) = 0; - virtual int Open(const char *path, int flags, mode_t mode, int *fd) = 0; - virtual int Open(const char *path, int flags, mode_t mode, - std::unique_ptr *f) = 0; - - // read(), returning 0 on success or errno>0 on failure. - // On success, |bytes_read| will be updated. - virtual int Read(void *buf, size_t count, size_t *bytes_read) = 0; - - // fstat(), returning 0 on success or errno>0 on failure. - virtual int Stat(struct stat *buf) = 0; - - // fsync(), returning 0 on success or errno>0 on failure. - virtual int Sync() = 0; - - // ftruncate(), returning 0 on success or errno>0 on failure. - virtual int Truncate(off_t length) = 0; - - // unlink() the specified file, returning 0 on success or errno>0 on failure. - virtual int Unlink(const char *path) = 0; - - // Write to the file, returning 0 on success or errno>0 on failure. - // On success, |bytes_written| will be updated. - virtual int Write(re2::StringPiece data, size_t *bytes_written) = 0; -}; - -// Interface to the local filesystem. There's typically one per program, -// but it's an abstract class for testability. Thread-safe. -class Filesystem { - public: - virtual ~Filesystem() {} - - // Execute |fn| for each directory entry in |dir_path|, stopping early - // (successfully) if the callback returns IterationControl::kBreak. - // - // On success, returns true. - // On failure, returns false and updates |error_msg|. - virtual bool DirForEach(const char *dir_path, - std::function fn, - std::string *error_msg) = 0; - - // open() the specified path, returning 0 on success or errno>0 on failure. - // On success, |f| is populated with an open file. - virtual int Open(const char *path, int flags, std::unique_ptr *f) = 0; - virtual int Open(const char *path, int flags, mode_t mode, - std::unique_ptr *f) = 0; - - // mkdir() the specified path, returning 0 on success or errno>0 on failure. - virtual int Mkdir(const char *path, mode_t mode) = 0; - - // rmdir() the specified path, returning 0 on success or errno>0 on failure. - virtual int Rmdir(const char *path) = 0; - - // stat() the specified path, returning 0 on success or errno>0 on failure. - virtual int Stat(const char *path, struct stat *buf) = 0; - - // unlink() the specified file, returning 0 on success or errno>0 on failure. - virtual int Unlink(const char *path) = 0; -}; - -// Get the (singleton) real filesystem, which is never deleted. -Filesystem *GetRealFilesystem(); - -} // namespace moonfire_nvr - -#endif // MOONFIRE_NVR_FILESYSTEM_H diff --git a/src/h264-test.cc b/src/h264-test.cc deleted file mode 100644 index 9fa7a2c..0000000 --- a/src/h264-test.cc +++ /dev/null @@ -1,153 +0,0 @@ -// This file is part of Moonfire NVR, a security camera network video recorder. -// Copyright (C) 2016 Scott Lamb -// -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// In addition, as a special exception, the copyright holders give -// permission to link the code of portions of this program with the -// OpenSSL library under certain conditions as described in each -// individual source file, and distribute linked combinations including -// the two. -// -// You must obey the GNU General Public License in all respects for all -// of the code used other than OpenSSL. If you modify file(s) with this -// exception, you may extend this exception to your version of the -// file(s), but you are not obligated to do so. If you do not wish to do -// so, delete this exception statement from your version. If you delete -// this exception statement from all source files in the program, then -// also delete it here. -// -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. -// -// You should have received a copy of the GNU General Public License -// along with this program. If not, see . -// -// h264-test.cc: tests of the h264.h interface. - -#include -#include -#include -#include - -#include "h264.h" -#include "string.h" - -DECLARE_bool(alsologtostderr); - -namespace moonfire_nvr { -namespace { - -const uint8_t kAnnexBTestInput[] = { - 0x00, 0x00, 0x00, 0x01, 0x67, 0x4d, 0x00, 0x1f, 0x9a, 0x66, 0x02, 0x80, - 0x2d, 0xff, 0x35, 0x01, 0x01, 0x01, 0x40, 0x00, 0x00, 0xfa, 0x00, 0x00, - 0x1d, 0x4c, 0x01, 0x00, 0x00, 0x00, 0x01, 0x68, 0xee, 0x3c, 0x80}; - -const uint8_t kAvcDecoderConfigTestInput[] = { - 0x01, 0x4d, 0x00, 0x1f, 0xff, 0xe1, 0x00, 0x17, 0x67, 0x4d, - 0x00, 0x1f, 0x9a, 0x66, 0x02, 0x80, 0x2d, 0xff, 0x35, 0x01, - 0x01, 0x01, 0x40, 0x00, 0x00, 0xfa, 0x00, 0x00, 0x1d, 0x4c, - 0x01, 0x01, 0x00, 0x04, 0x68, 0xee, 0x3c, 0x80}; - -const char kTestOutput[] = - "00 00 00 84 61 76 63 31 00 00 00 00 00 00 00 01 " - "00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 " - "05 00 02 d0 00 48 00 00 00 48 00 00 00 00 00 00 " - "00 01 00 00 00 00 00 00 00 00 00 00 00 00 00 00 " - "00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 " - "00 00 00 18 ff ff 00 00 00 2e 61 76 63 43 01 4d " - "00 1f ff e1 00 17 67 4d 00 1f 9a 66 02 80 2d ff " - "35 01 01 01 40 00 00 fa 00 00 1d 4c 01 01 00 04 " - "68 ee 3c 80"; - -TEST(H264Test, DecodeOnly) { - std::vector nal_units_hexed; - re2::StringPiece test_input(reinterpret_cast(kAnnexBTestInput), - sizeof(kAnnexBTestInput)); - internal::NalUnitFunction fn = [&nal_units_hexed](re2::StringPiece nal_unit) { - nal_units_hexed.push_back(ToHex(nal_unit, true)); - return IterationControl::kContinue; - }; - std::string error_message; - ASSERT_TRUE(internal::DecodeH264AnnexB(test_input, fn, &error_message)) - << error_message; - EXPECT_THAT(nal_units_hexed, - testing::ElementsAre("67 4d 00 1f 9a 66 02 80 2d ff 35 01 01 01 " - "40 00 00 fa 00 00 1d 4c 01", - "68 ee 3c 80")); -} - -TEST(H264Test, SampleEntryFromAnnexBExtraData) { - re2::StringPiece test_input(reinterpret_cast(kAnnexBTestInput), - sizeof(kAnnexBTestInput)); - std::string sample_entry; - std::string error_message; - bool need_transform; - ASSERT_TRUE(ParseExtraData(test_input, 1280, 720, &sample_entry, - &need_transform, &error_message)) - << error_message; - - EXPECT_EQ(kTestOutput, ToHex(sample_entry, true)); - EXPECT_TRUE(need_transform); -} - -TEST(H264Test, SampleEntryFromAvcDecoderConfigExtraData) { - re2::StringPiece test_input( - reinterpret_cast(kAvcDecoderConfigTestInput), - sizeof(kAvcDecoderConfigTestInput)); - std::string sample_entry; - std::string error_message; - bool need_transform; - ASSERT_TRUE(ParseExtraData(test_input, 1280, 720, &sample_entry, - &need_transform, &error_message)) - << error_message; - - EXPECT_EQ(kTestOutput, ToHex(sample_entry, true)); - EXPECT_FALSE(need_transform); -} - -TEST(H264Test, TransformSampleEntry) { - const uint8_t kInput[] = { - 0x00, 0x00, 0x00, 0x01, 0x67, 0x4d, 0x00, 0x1f, 0x9a, 0x66, - 0x02, 0x80, 0x2d, 0xff, 0x35, 0x01, 0x01, 0x01, 0x40, 0x00, - 0x00, 0xfa, 0x00, 0x00, 0x1d, 0x4c, 0x01, - - 0x00, 0x00, 0x00, 0x01, 0x68, 0xee, 0x3c, 0x80, - - 0x00, 0x00, 0x00, 0x01, 0x06, 0x06, 0x01, 0xc4, 0x80, - - 0x00, 0x00, 0x00, 0x01, 0x65, 0x88, 0x80, 0x10, 0x00, 0x08, - 0x7f, 0x00, 0x5d, 0x27, 0xb5, 0xc1, 0xff, 0x8c, 0xd6, 0x35, - // (truncated) - }; - const char kExpectedOutput[] = - "00 00 00 17 " - "67 4d 00 1f 9a 66 02 80 2d ff 35 01 01 01 40 00 00 fa 00 00 1d 4c 01 " - "00 00 00 04 68 ee 3c 80 " - "00 00 00 05 06 06 01 c4 80 " - "00 00 00 10 " - "65 88 80 10 00 08 7f 00 5d 27 b5 c1 ff 8c d6 35"; - re2::StringPiece input(reinterpret_cast(kInput), - sizeof(kInput)); - std::string out; - std::string error_message; - ASSERT_TRUE(TransformSampleData(input, &out, &error_message)) - << error_message; - EXPECT_EQ(kExpectedOutput, ToHex(out, true)); -} - -} // namespace -} // namespace moonfire_nvr - -int main(int argc, char **argv) { - FLAGS_alsologtostderr = true; - google::ParseCommandLineFlags(&argc, &argv, true); - testing::InitGoogleTest(&argc, argv); - google::InitGoogleLogging(argv[0]); - return RUN_ALL_TESTS(); -} diff --git a/src/h264.cc b/src/h264.cc deleted file mode 100644 index 882c61f..0000000 --- a/src/h264.cc +++ /dev/null @@ -1,251 +0,0 @@ -// This file is part of Moonfire NVR, a security camera network video recorder. -// Copyright (C) 2016 Scott Lamb -// -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// In addition, as a special exception, the copyright holders give -// permission to link the code of portions of this program with the -// OpenSSL library under certain conditions as described in each -// individual source file, and distribute linked combinations including -// the two. -// -// You must obey the GNU General Public License in all respects for all -// of the code used other than OpenSSL. If you modify file(s) with this -// exception, you may extend this exception to your version of the -// file(s), but you are not obligated to do so. If you do not wish to do -// so, delete this exception statement from your version. If you delete -// this exception statement from all source files in the program, then -// also delete it here. -// -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. -// -// You should have received a copy of the GNU General Public License -// along with this program. If not, see . -// -// h264.cc: see h264.h. - -#include "h264.h" - -#include - -#include "coding.h" -#include "string.h" - -namespace moonfire_nvr { - -namespace { - -// See ISO/IEC 14496-10 table 7-1 - NAL unit type codes, syntax element -// categories, and NAL unit type classes. -const int kNalUnitSeqParameterSet = 7; -const int kNalUnitPicParameterSet = 8; - -const uint8_t kNalUnitTypeMask = 0x1F; // bottom 5 bits of first byte of unit. - -// Parse sequence parameter set and picture parameter set from ffmpeg's -// "extra_data". -bool ParseAnnexBExtraData(re2::StringPiece extradata, re2::StringPiece *sps, - re2::StringPiece *pps, std::string *error_message) { - bool ok = true; - internal::NalUnitFunction fn = [&](re2::StringPiece nal_unit) { - // See ISO/IEC 14496-10 section 7.3.1, which defines nal_unit. - uint8_t nal_type = nal_unit[0] & kNalUnitTypeMask; - switch (nal_type) { - case kNalUnitSeqParameterSet: - *sps = nal_unit; - break; - case kNalUnitPicParameterSet: - *pps = nal_unit; - break; - default: - *error_message = - StrCat("Expected only SPS and PPS; got type ", nal_type); - ok = false; - return IterationControl::kBreak; - } - return IterationControl::kContinue; - }; - if (!internal::DecodeH264AnnexB(extradata, fn, error_message) || !ok) { - return false; - } - if (sps->empty() || pps->empty()) { - *error_message = "SPS and PPS must be specified."; - return false; - } - return true; -} - -} // namespace - -namespace internal { - -// See ISO/IEC 14496-10 section B.2: Byte stream NAL unit decoding process. -// This is a relatively simple, unoptimized implementation. -bool DecodeH264AnnexB(re2::StringPiece data, NalUnitFunction process_nal_unit, - std::string *error_message) { - static const RE2 kStartCode("(\\x00{2,}\\x01)"); - - if (!RE2::Consume(&data, kStartCode)) { - *error_message = StrCat("stream does not start with Annex B start code: ", - ToHex(data, true)); - return false; - } - - while (!data.empty()) { - // Now at the start of a NAL unit. Find the end. - re2::StringPiece next_start; - re2::StringPiece this_nal = data; - if (RE2::FindAndConsume(&data, kStartCode, &next_start)) { - // It ends where another start code is found. - this_nal = re2::StringPiece(this_nal.data(), - next_start.data() - this_nal.data()); - } else { - // It ends at the end of |data|. |this_nal| is already correct. - // Set |data| to be empty so the while loop exits after this iteration. - data = re2::StringPiece(); - } - - if (this_nal.empty()) { - *error_message = "NAL unit can't be empty"; - return false; - } - - if (process_nal_unit(this_nal) == IterationControl::kBreak) { - break; - } - } - return true; -} - -} // namespace internal - -bool ParseExtraData(re2::StringPiece extradata, uint16_t width, uint16_t height, - std::string *sample_entry, bool *need_transform, - std::string *error_message) { - uint32_t avcc_len; - re2::StringPiece sps; - re2::StringPiece pps; - if (extradata.starts_with(re2::StringPiece("\x00\x00\x00\x01", 4)) || - extradata.starts_with(re2::StringPiece("\x00\x00\x01", 3))) { - // ffmpeg supplied "extradata" in Annex B format. - if (!ParseAnnexBExtraData(extradata, &sps, &pps, error_message)) { - return false; - } - - // This magic value is checked at the end. - avcc_len = 19 + sps.size() + pps.size(); - *need_transform = true; - } else { - // Assume "extradata" holds an AVCDecoderConfiguration. - avcc_len = 8 + extradata.size(); - *need_transform = false; - } - - // This magic value is also checked at the end. - uint32_t avc1_len = 86 + avcc_len; - - sample_entry->clear(); - sample_entry->reserve(avc1_len); - - // This is a concatenation of the following boxes/classes. - // SampleEntry, ISO/IEC 14496-10 section 8.5.2. - uint32_t avc1_len_pos = sample_entry->size(); - AppendU32(avc1_len, sample_entry); // length - sample_entry->append("avc1"); // type - sample_entry->append(6, '\x00'); // reserved - AppendU16(1, sample_entry); // data_reference_index = 1 - - // VisualSampleEntry, ISO/IEC 14496-12 section 12.1.3. - sample_entry->append(16, '\x00'); // pre_defined + reserved - AppendU16(width, sample_entry); - AppendU16(height, sample_entry); - AppendU32(UINT32_C(0x00480000), sample_entry); // horizresolution - AppendU32(UINT32_C(0x00480000), sample_entry); // vertresolution - AppendU32(0, sample_entry); // reserved - AppendU16(1, sample_entry); // frame count - sample_entry->append(32, '\x00'); // compressorname - AppendU16(0x0018, sample_entry); // depth - Append16(-1, sample_entry); // pre_defined - - // AVCSampleEntry, ISO/IEC 14496-15 section 5.3.4.1. - // AVCConfigurationBox, ISO/IEC 14496-15 section 5.3.4.1. - uint32_t avcc_len_pos = sample_entry->size(); - AppendU32(avcc_len, sample_entry); // length - sample_entry->append("avcC"); // type - - if (!sps.empty() && !pps.empty()) { - // Create the AVCDecoderConfiguration, ISO/IEC 14496-15 section 5.2.4.1. - // The beginning of the AVCDecoderConfiguration takes a few values from - // the SPS (ISO/IEC 14496-10 section 7.3.2.1.1). One caveat: that section - // defines the syntax in terms of RBSP, not NAL. The difference is the - // escaping of 00 00 01 and 00 00 02; see notes about - // "emulation_prevention_three_byte" in ISO/IEC 14496-10 section 7.4. - // It looks like 00 is not a valid value of profile_idc, so this distinction - // shouldn't be relevant here. And ffmpeg seems to ignore it. - sample_entry->push_back(1); // configurationVersion - sample_entry->push_back(sps[1]); // profile_idc -> AVCProfileIndication - sample_entry->push_back( - sps[2]); // ...misc bits... -> profile_compatibility - sample_entry->push_back(sps[3]); // level_idc -> AVCLevelIndication - - // Hardcode lengthSizeMinusOne to 3, matching TransformSampleData's 4-byte - // lengths. - sample_entry->push_back(static_cast(0xff)); - - // Only support one SPS and PPS. - // ffmpeg's ff_isom_write_avcc has the same limitation, so it's probably - // fine. This next byte is a reserved 0b111 + a 5-bit # of SPSs (1). - sample_entry->push_back(static_cast(0xe1)); - AppendU16(sps.size(), sample_entry); - sample_entry->append(sps.data(), sps.size()); - sample_entry->push_back(1); // # of PPSs. - AppendU16(pps.size(), sample_entry); - sample_entry->append(pps.data(), pps.size()); - - if (sample_entry->size() - avcc_len_pos != avcc_len) { - *error_message = StrCat( - "internal error: anticipated AVCConfigurationBox length ", avcc_len, - ", but was actually ", sample_entry->size() - avcc_len_pos, - "; sps length ", sps.size(), ", pps length ", pps.size()); - return false; - } - - } else { - sample_entry->append(extradata.data(), extradata.size()); - } - - if (sample_entry->size() - avc1_len_pos != avc1_len) { - *error_message = - StrCat("internal error: anticipated AVCSampleEntry length ", avc1_len, - ", but was actually ", sample_entry->size() - avc1_len_pos, - "; sps length ", sps.size(), ", pps length ", pps.size()); - return false; - } - - return true; -} - -bool TransformSampleData(re2::StringPiece annexb_sample, - std::string *avc_sample, std::string *error_message) { - // See AVCParameterSamples, ISO/IEC 14496-15 section 5.3.2. - avc_sample->clear(); - auto fn = [&](re2::StringPiece nal_unit) { - // 4-byte length; this must be in sync with ParseExtraData's - // lengthSizeMinusOne == 3. - AppendU32(nal_unit.size(), avc_sample); - avc_sample->append(nal_unit.data(), nal_unit.size()); - return IterationControl::kContinue; - }; - if (!internal::DecodeH264AnnexB(annexb_sample, fn, error_message)) { - return false; - } - return true; -} - -} // namespace moonfire_nvr diff --git a/src/h264.h b/src/h264.h deleted file mode 100644 index d0e888a..0000000 --- a/src/h264.h +++ /dev/null @@ -1,84 +0,0 @@ -// This file is part of Moonfire NVR, a security camera network video recorder. -// Copyright (C) 2016 Scott Lamb -// -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// In addition, as a special exception, the copyright holders give -// permission to link the code of portions of this program with the -// OpenSSL library under certain conditions as described in each -// individual source file, and distribute linked combinations including -// the two. -// -// You must obey the GNU General Public License in all respects for all -// of the code used other than OpenSSL. If you modify file(s) with this -// exception, you may extend this exception to your version of the -// file(s), but you are not obligated to do so. If you do not wish to do -// so, delete this exception statement from your version. If you delete -// this exception statement from all source files in the program, then -// also delete it here. -// -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. -// -// You should have received a copy of the GNU General Public License -// along with this program. If not, see . -// -// h264.h: H.264 decoding. For the most part, Moonfire NVR does not try to -// understand the video codec. However, H.264 has two byte stream encodings: -// ISO/IEC 14496-10 Annex B, and ISO/IEC 14496-15 AVC access units. -// When streaming from RTSP, ffmpeg supplies the former. We need the latter -// to stick into .mp4 files. This file manages the conversion, both for -// the ffmpeg "extra data" (which should become the ISO/IEC 14496-15 -// section 5.2.4.1 AVCDecoderConfigurationRecord) and the actual samples. -// -// ffmpeg of course has logic to do the same thing, but unfortunately it is -// not exposed except through ffmpeg's own generated .mp4 file. Extracting -// just this part of their .mp4 files would be more trouble than it's worth. - -#ifndef MOONFIRE_NVR_H264_H -#define MOONFIRE_NVR_H264_H - -#include -#include - -#include - -#include "common.h" - -namespace moonfire_nvr { - -namespace internal { - -using NalUnitFunction = - std::function; - -// Decode a H.264 Annex B byte stream into NAL units. -// For GetH264SampleEntry; exposed for testing. -// Calls |process_nal_unit| for each NAL unit in the byte stream. -// -// Note: this won't spot all invalid byte streams. For example, several 0x00s -// not followed by a 0x01 will just be considered part of a NAL unit rather -// than proof of an invalid stream. -bool DecodeH264AnnexB(re2::StringPiece data, NalUnitFunction process_nal_unit, - std::string *error_message); - -} // namespace - -// Gets a H.264 sample entry (AVCSampleEntry, which extends -// VisualSampleEntry), given the "extradata", width, and height supplied by -// ffmpeg. -bool ParseExtraData(re2::StringPiece extradata, uint16_t width, uint16_t height, - std::string *sample_entry, bool *need_transform, - std::string *error_message); - -bool TransformSampleData(re2::StringPiece annexb_sample, - std::string *avc_sample, std::string *error_message); - -} // namespace moonfire_nvr - -#endif // MOONFIRE_NVR_H264_H diff --git a/src/h264.rs b/src/h264.rs new file mode 100644 index 0000000..d981084 --- /dev/null +++ b/src/h264.rs @@ -0,0 +1,349 @@ +// This file is part of Moonfire NVR, a security camera digital video recorder. +// Copyright (C) 2016 Scott Lamb +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// In addition, as a special exception, the copyright holders give +// permission to link the code of portions of this program with the +// OpenSSL library under certain conditions as described in each +// individual source file, and distribute linked combinations including +// the two. +// +// You must obey the GNU General Public License in all respects for all +// of the code used other than OpenSSL. If you modify file(s) with this +// exception, you may extend this exception to your version of the +// file(s), but you are not obligated to do so. If you do not wish to do +// so, delete this exception statement from your version. If you delete +// this exception statement from all source files in the program, then +// also delete it here. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program. If not, see . + +//! H.264 decoding +//! +//! For the most part, Moonfire NVR does not try to understand the video codec. However, H.264 has +//! two byte stream encodings: ISO/IEC 14496-10 Annex B, and ISO/IEC 14496-15 AVC access units. +//! When streaming from RTSP, ffmpeg supplies the former. We need the latter to stick into `.mp4` +//! files. This file manages the conversion, both for the ffmpeg "extra data" (which should become +//! the ISO/IEC 14496-15 section 5.2.4.1 `AVCDecoderConfigurationRecord`) and the actual samples. +//! +//! ffmpeg of course has logic to do the same thing, but unfortunately it is not exposed except +//! through ffmpeg's own generated `.mp4` file. Extracting just this part of their `.mp4` files +//! would be more trouble than it's worth. + +use byteorder::{BigEndian, WriteBytesExt}; +use error::{Error, Result}; +use regex::bytes::Regex; + +// See ISO/IEC 14496-10 table 7-1 - NAL unit type codes, syntax element categories, and NAL unit +// type classes. +const NAL_UNIT_SEQ_PARAMETER_SET: u8 = 7; +const NAL_UNIT_PIC_PARAMETER_SET: u8 = 8; + +const NAL_UNIT_TYPE_MASK: u8 = 0x1F; // bottom 5 bits of first byte of unit. + +/// Decodes a H.264 Annex B byte stream into NAL units. Calls `f` for each NAL unit in the byte +/// stream. Aborts if `f` returns error. +/// +/// See ISO/IEC 14496-10 section B.2: Byte stream NAL unit decoding process. +/// This is a relatively simple, unoptimized implementation. +/// +/// TODO: detect invalid byte streams. For example, several 0x00s not followed by a 0x01, a stream +/// stream not starting with 0x00 0x00 0x00 0x01, or an empty NAL unit. +fn decode_h264_annex_b<'a, F>(data: &'a [u8], mut f: F) -> Result<()> +where F: FnMut(&'a [u8]) -> Result<()> { + lazy_static! { + static ref START_CODE: Regex = Regex::new(r"(\x00{2,}\x01)").unwrap(); + } + for unit in START_CODE.split(data) { + if !unit.is_empty() { + f(unit)?; + } + } + Ok(()) +} + +/// Parses Annex B extra data, returning a tuple holding the `sps` and `pps` substrings. +fn parse_annex_b_extra_data(data: &[u8]) -> Result<(&[u8], &[u8])> { + let mut sps = None; + let mut pps = None; + decode_h264_annex_b(data, |unit| { + let nal_type = (unit[0] as u8) & NAL_UNIT_TYPE_MASK; + match nal_type { + NAL_UNIT_SEQ_PARAMETER_SET => { sps = Some(unit); }, + NAL_UNIT_PIC_PARAMETER_SET => { pps = Some(unit); }, + _ => { return Err(Error::new(format!("Expected SPS and PPS; got type {}", nal_type))); } + }; + Ok(()) + })?; + match (sps, pps) { + (Some(s), Some(p)) => Ok((s, p)), + _ => Err(Error::new("SPS and PPS must be specified".to_owned())), + } +} + +/// Parsed representation of ffmpeg's "extradata". +#[derive(Debug, PartialEq, Eq)] +pub struct ExtraData { + pub sample_entry: Vec, + pub width: u16, + pub height: u16, + + /// True iff sample data should be transformed from Annex B format to AVC format via a call to + /// `transform_sample_data`. (The assumption is that if the extra data was in Annex B format, + /// the sample data is also.) + pub need_transform: bool, +} + +impl ExtraData { + /// Parses "extradata" from ffmpeg. This data may be in either Annex B format or AVC format. + pub fn parse(extradata: &[u8], width: u16, height: u16) -> Result { + let mut sps_and_pps = None; + let need_transform; + let avcc_len = if extradata.starts_with(b"\x00\x00\x00\x01") || + extradata.starts_with(b"\x00\x00\x01") { + // ffmpeg supplied "extradata" in Annex B format. + let (s, p) = parse_annex_b_extra_data(extradata)?; + sps_and_pps = Some((s, p)); + need_transform = true; + + // This magic value is checked at the end of the function; + // unit tests confirm its accuracy. + 19 + s.len() + p.len() + } else { + // Assume "extradata" holds an AVCDecoderConfiguration. + need_transform = false; + 8 + extradata.len() + }; + let sps_and_pps = sps_and_pps; + let need_transform = need_transform; + + // This magic value is also checked at the end. + let avc1_len = 86 + avcc_len; + + let mut sample_entry = Vec::with_capacity(avc1_len); + + // This is a concatenation of the following boxes/classes. + + // SampleEntry, ISO/IEC 14496-10 section 8.5.2. + let avc1_len_pos = sample_entry.len(); + sample_entry.write_u32::(avc1_len as u32)?; // length + // type + reserved + data_reference_index = 1 + sample_entry.extend_from_slice(b"avc1\x00\x00\x00\x00\x00\x00\x00\x01"); + + // VisualSampleEntry, ISO/IEC 14496-12 section 12.1.3. + sample_entry.extend_from_slice(&[0; 16]); // pre-defined + reserved + sample_entry.write_u16::(width)?; + sample_entry.write_u16::(height)?; + sample_entry.extend_from_slice(&[ + 0x00, 0x48, 0x00, 0x00, // horizresolution + 0x00, 0x48, 0x00, 0x00, // vertresolution + 0x00, 0x00, 0x00, 0x00, // reserved + 0x00, 0x01, // frame count + 0x00, 0x00, 0x00, 0x00, // compressorname + 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, + 0x00, 0x18, 0xff, 0xff, // depth + pre_defined + ]); + + // AVCSampleEntry, ISO/IEC 14496-15 section 5.3.4.1. + // AVCConfigurationBox, ISO/IEC 14496-15 section 5.3.4.1. + let avcc_len_pos = sample_entry.len(); + sample_entry.write_u32::(avcc_len as u32)?; // length + sample_entry.extend_from_slice(b"avcC"); + + let avc_decoder_config_len = if let Some((sps, pps)) = sps_and_pps { + let before = sample_entry.len(); + + // Create the AVCDecoderConfiguration, ISO/IEC 14496-15 section 5.2.4.1. + // The beginning of the AVCDecoderConfiguration takes a few values from + // the SPS (ISO/IEC 14496-10 section 7.3.2.1.1). One caveat: that section + // defines the syntax in terms of RBSP, not NAL. The difference is the + // escaping of 00 00 01 and 00 00 02; see notes about + // "emulation_prevention_three_byte" in ISO/IEC 14496-10 section 7.4. + // It looks like 00 is not a valid value of profile_idc, so this distinction + // shouldn't be relevant here. And ffmpeg seems to ignore it. + sample_entry.push(1); // configurationVersion + sample_entry.push(sps[1]); // profile_idc . AVCProfileIndication + sample_entry.push(sps[2]); // ...misc bits... . profile_compatibility + sample_entry.push(sps[3]); // level_idc . AVCLevelIndication + + // Hardcode lengthSizeMinusOne to 3, matching TransformSampleData's 4-byte + // lengths. + sample_entry.push(0xff); + + // Only support one SPS and PPS. + // ffmpeg's ff_isom_write_avcc has the same limitation, so it's probably + // fine. This next byte is a reserved 0b111 + a 5-bit # of SPSs (1). + sample_entry.push(0xe1); + sample_entry.write_u16::(sps.len() as u16)?; + sample_entry.extend_from_slice(sps); + sample_entry.push(1); // # of PPSs. + sample_entry.write_u16::(pps.len() as u16)?; + sample_entry.extend_from_slice(pps); + + if sample_entry.len() - avcc_len_pos != avcc_len { + return Err(Error::new(format!("internal error: anticipated AVCConfigurationBox \ + length {}, but was actually {}; sps length \ + {}, pps length {}", + avcc_len, sample_entry.len() - avcc_len_pos, + sps.len(), pps.len()))); + } + sample_entry.len() - before + } else { + sample_entry.extend_from_slice(extradata); + extradata.len() + }; + + if sample_entry.len() - avc1_len_pos != avc1_len { + return Err(Error::new(format!("internal error: anticipated AVCSampleEntry length \ + {}, but was actually {}; AVCDecoderConfiguration \ + length {}", avc1_len, sample_entry.len() - avc1_len_pos, + avc_decoder_config_len))); + } + Ok(ExtraData{ + sample_entry: sample_entry, + width: width, + height: height, + need_transform: need_transform, + }) + } +} + +/// Transforms sample data from Annex B format to AVC format. Should be called on samples iff +/// `ExtraData::need_transform` is true. Uses an out parameter `avc_sample` rather than a return +/// so that memory allocations can be reused from sample to sample. +pub fn transform_sample_data(annexb_sample: &[u8], avc_sample: &mut Vec) -> Result<()> { + // See AVCParameterSamples, ISO/IEC 14496-15 section 5.3.2. + avc_sample.clear(); + + // The output will be about as long as the input. Annex B stop codes require at least three + // bytes; many seem to be four. The output lengths are exactly four. + avc_sample.reserve(annexb_sample.len() + 4); + decode_h264_annex_b(annexb_sample, |unit| { + // 4-byte length; this must match ParseExtraData's lengthSizeMinusOne == 3. + avc_sample.write_u32::(unit.len() as u32)?; // length + avc_sample.extend_from_slice(unit); + Ok(()) + })?; + Ok(()) +} + +#[cfg(test)] +mod tests { + const ANNEX_B_TEST_INPUT: [u8; 35] = [ + 0x00, 0x00, 0x00, 0x01, 0x67, 0x4d, 0x00, 0x1f, + 0x9a, 0x66, 0x02, 0x80, 0x2d, 0xff, 0x35, 0x01, + 0x01, 0x01, 0x40, 0x00, 0x00, 0xfa, 0x00, 0x00, + 0x1d, 0x4c, 0x01, 0x00, 0x00, 0x00, 0x01, 0x68, + 0xee, 0x3c, 0x80, + ]; + + const AVC_DECODER_CONFIG_TEST_INPUT: [u8; 38] = [ + 0x01, 0x4d, 0x00, 0x1f, 0xff, 0xe1, 0x00, 0x17, + 0x67, 0x4d, 0x00, 0x1f, 0x9a, 0x66, 0x02, 0x80, + 0x2d, 0xff, 0x35, 0x01, 0x01, 0x01, 0x40, 0x00, + 0x00, 0xfa, 0x00, 0x00, 0x1d, 0x4c, 0x01, 0x01, + 0x00, 0x04, 0x68, 0xee, 0x3c, 0x80, + ]; + + const TEST_OUTPUT: [u8; 132] = [ + 0x00, 0x00, 0x00, 0x84, 0x61, 0x76, 0x63, 0x31, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x05, 0x00, 0x02, 0xd0, 0x00, 0x48, 0x00, 0x00, + 0x00, 0x48, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x18, 0xff, 0xff, 0x00, 0x00, + 0x00, 0x2e, 0x61, 0x76, 0x63, 0x43, 0x01, 0x4d, + 0x00, 0x1f, 0xff, 0xe1, 0x00, 0x17, 0x67, 0x4d, + 0x00, 0x1f, 0x9a, 0x66, 0x02, 0x80, 0x2d, 0xff, + 0x35, 0x01, 0x01, 0x01, 0x40, 0x00, 0x00, 0xfa, + 0x00, 0x00, 0x1d, 0x4c, 0x01, 0x01, 0x00, 0x04, + 0x68, 0xee, 0x3c, 0x80, + ]; + + #[test] + fn test_decode() { + let data = &ANNEX_B_TEST_INPUT; + let mut pieces = Vec::new(); + super::decode_h264_annex_b(data, |p| { + pieces.push(p); + Ok(()) + }).unwrap(); + assert_eq!(&pieces, &[&data[4 .. 27], &data[31 ..]]); + } + + #[test] + fn test_sample_entry_from_avc_decoder_config() { + let e = super::ExtraData::parse(&AVC_DECODER_CONFIG_TEST_INPUT, 1280, 720).unwrap(); + assert_eq!(&e.sample_entry[..], &TEST_OUTPUT[..]); + assert_eq!(e.width, 1280); + assert_eq!(e.height, 720); + assert_eq!(e.need_transform, false); + } + + #[test] + fn test_sample_entry_from_annex_b() { + let e = super::ExtraData::parse(&ANNEX_B_TEST_INPUT, 1280, 720).unwrap(); + assert_eq!(e.width, 1280); + assert_eq!(e.height, 720); + assert_eq!(e.need_transform, true); + } + + #[test] + fn test_transform_sample_data() { + const INPUT: [u8; 64] = [ + 0x00, 0x00, 0x00, 0x01, 0x67, 0x4d, 0x00, 0x1f, + 0x9a, 0x66, 0x02, 0x80, 0x2d, 0xff, 0x35, 0x01, + 0x01, 0x01, 0x40, 0x00, 0x00, 0xfa, 0x00, 0x00, + 0x1d, 0x4c, 0x01, + + 0x00, 0x00, 0x00, 0x01, 0x68, 0xee, 0x3c, 0x80, + + 0x00, 0x00, 0x00, 0x01, 0x06, 0x06, 0x01, 0xc4, + 0x80, + + 0x00, 0x00, 0x00, 0x01, 0x65, 0x88, 0x80, 0x10, + 0x00, 0x08, 0x7f, 0x00, 0x5d, 0x27, 0xb5, 0xc1, + 0xff, 0x8c, 0xd6, 0x35, + // (truncated) + ]; + const EXPECTED_OUTPUT: [u8; 64] = [ + 0x00, 0x00, 0x00, 0x17, 0x67, 0x4d, 0x00, 0x1f, + 0x9a, 0x66, 0x02, 0x80, 0x2d, 0xff, 0x35, 0x01, + 0x01, 0x01, 0x40, 0x00, 0x00, 0xfa, 0x00, 0x00, + 0x1d, 0x4c, 0x01, + + 0x00, 0x00, 0x00, 0x04, 0x68, 0xee, 0x3c, 0x80, + + 0x00, 0x00, 0x00, 0x05, 0x06, 0x06, 0x01, 0xc4, + 0x80, + + 0x00, 0x00, 0x00, 0x10, 0x65, 0x88, 0x80, 0x10, + 0x00, 0x08, 0x7f, 0x00, 0x5d, 0x27, 0xb5, 0xc1, + 0xff, 0x8c, 0xd6, 0x35, + ]; + let mut out = Vec::new(); + super::transform_sample_data(&INPUT, &mut out).unwrap(); + assert_eq!(&out[..], &EXPECTED_OUTPUT[..]); + } +} diff --git a/src/http-test.cc b/src/http-test.cc deleted file mode 100644 index 41bf0d3..0000000 --- a/src/http-test.cc +++ /dev/null @@ -1,295 +0,0 @@ -// This file is part of Moonfire NVR, a security camera digital video recorder. -// Copyright (C) 2016 Scott Lamb -// -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// In addition, as a special exception, the copyright holders give -// permission to link the code of portions of this program with the -// OpenSSL library under certain conditions as described in each -// individual source file, and distribute linked combinations including -// the two. -// -// You must obey the GNU General Public License in all respects for all -// of the code used other than OpenSSL. If you modify file(s) with this -// exception, you may extend this exception to your version of the -// file(s), but you are not obligated to do so. If you do not wish to do -// so, delete this exception statement from your version. If you delete -// this exception statement from all source files in the program, then -// also delete it here. -// -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. -// -// You should have received a copy of the GNU General Public License -// along with this program. If not, see . -// -// util_test.cc: tests of the util.h interface. - -#include -#include -#include -#include - -#include -#include -#include -#include - -#include "http.h" -#include "string.h" -#include "testutil.h" - -DECLARE_bool(alsologtostderr); - -using moonfire_nvr::internal::ParseRangeHeader; -using moonfire_nvr::internal::RangeHeaderType; - -using testing::_; -using testing::AnyNumber; -using testing::DoAll; -using testing::Return; -using testing::SetArgPointee; - -namespace moonfire_nvr { -namespace { - -class MockFileSlice : public FileSlice { - public: - MOCK_CONST_METHOD0(size, int64_t()); - MOCK_CONST_METHOD3(AddRange, int64_t(ByteRange, EvBuffer *, std::string *)); -}; - -TEST(EvBufferTest, AddFileTest) { - std::string dir = PrepareTempDirOrDie("http"); - std::string foo_filename = StrCat(dir, "/foo"); - WriteFileOrDie(foo_filename, "foo"); - - int in_fd = open(foo_filename.c_str(), O_RDONLY); - PCHECK(in_fd >= 0) << "open: " << foo_filename; - std::string error_message; - - // Ensure adding the whole file succeeds. - EvBuffer buf1; - ASSERT_TRUE(buf1.AddFile(in_fd, 0, 3, &error_message)) << error_message; - in_fd = -1; - EXPECT_EQ(3u, evbuffer_get_length(buf1.get())); - - // Ensure adding an empty region succeeds. - EvBuffer buf2; - ASSERT_TRUE(buf2.AddFile(in_fd, 0, 0, &error_message)) << error_message; - EXPECT_EQ(0u, evbuffer_get_length(buf2.get())); - - // Ensure adding part of a file after another string succeeds. - in_fd = open(foo_filename.c_str(), O_RDONLY); - EvBuffer buf3; - buf3.Add("1234"); - ASSERT_TRUE(buf3.AddFile(in_fd, 1, 2, &error_message)) << error_message; - auto size3 = evbuffer_get_length(buf3.get()); - EXPECT_EQ(6u, size3); - std::string buf3_contents = std::string( - reinterpret_cast(evbuffer_pullup(buf3.get(), size3)), - size3); - EXPECT_EQ("1234oo", buf3_contents); -} - -class FileSlicesTest : public testing::Test { - protected: - void Init(int flags) { - EXPECT_CALL(a_, size()).Times(AnyNumber()).WillRepeatedly(Return(5)); - EXPECT_CALL(b_, size()).Times(AnyNumber()).WillRepeatedly(Return(13)); - EXPECT_CALL(c_, size()).Times(AnyNumber()).WillRepeatedly(Return(7)); - EXPECT_CALL(d_, size()).Times(AnyNumber()).WillRepeatedly(Return(17)); - EXPECT_CALL(e_, size()).Times(AnyNumber()).WillRepeatedly(Return(19)); - - slices_.Append(&a_, flags); - slices_.Append(&b_, flags); - slices_.Append(&c_, flags); - slices_.Append(&d_, flags); - slices_.Append(&e_, flags); - } - - FileSlices slices_; - testing::StrictMock a_; - testing::StrictMock b_; - testing::StrictMock c_; - testing::StrictMock d_; - testing::StrictMock e_; -}; - -TEST_F(FileSlicesTest, Size) { - Init(0); - EXPECT_EQ(5 + 13 + 7 + 17 + 19, slices_.size()); -} - -TEST_F(FileSlicesTest, ExactSlice) { - // Exactly slice b. - Init(0); - std::string error_message; - EXPECT_CALL(b_, AddRange(ByteRange(0, 13), _, _)).WillOnce(Return(13)); - EXPECT_EQ(13, slices_.AddRange(ByteRange(5, 18), nullptr, &error_message)) - << error_message; -} - -TEST_F(FileSlicesTest, Offset) { - // Part of slice b, all of slice c, and part of slice d. - Init(0); - std::string error_message; - EXPECT_CALL(b_, AddRange(ByteRange(12, 13), _, _)).WillOnce(Return(1)); - EXPECT_CALL(c_, AddRange(ByteRange(0, 7), _, _)).WillOnce(Return(7)); - EXPECT_CALL(d_, AddRange(ByteRange(0, 1), _, _)).WillOnce(Return(1)); - EXPECT_EQ(9, slices_.AddRange(ByteRange(17, 26), nullptr, &error_message)) - << error_message; -} - -TEST_F(FileSlicesTest, Everything) { - Init(0); - std::string error_message; - EXPECT_CALL(a_, AddRange(ByteRange(0, 5), _, _)).WillOnce(Return(5)); - EXPECT_CALL(b_, AddRange(ByteRange(0, 13), _, _)).WillOnce(Return(13)); - EXPECT_CALL(c_, AddRange(ByteRange(0, 7), _, _)).WillOnce(Return(7)); - EXPECT_CALL(d_, AddRange(ByteRange(0, 17), _, _)).WillOnce(Return(17)); - EXPECT_CALL(e_, AddRange(ByteRange(0, 19), _, _)).WillOnce(Return(19)); - EXPECT_EQ(61, slices_.AddRange(ByteRange(0, 61), nullptr, &error_message)) - << error_message; -} - -TEST_F(FileSlicesTest, Lazy) { - Init(FileSlices::kLazy); - std::string error_message; - EXPECT_CALL(a_, AddRange(ByteRange(0, 5), _, _)).WillOnce(Return(5)); - EXPECT_EQ(5, slices_.AddRange(ByteRange(0, 61), nullptr, &error_message)) - << error_message; - EXPECT_CALL(b_, AddRange(ByteRange(0, 13), _, _)).WillOnce(Return(13)); - EXPECT_EQ(13, slices_.AddRange(ByteRange(5, 61), nullptr, &error_message)) - << error_message; - EXPECT_CALL(c_, AddRange(ByteRange(0, 7), _, _)).WillOnce(Return(7)); - EXPECT_EQ(7, slices_.AddRange(ByteRange(18, 61), nullptr, &error_message)) - << error_message; - EXPECT_CALL(d_, AddRange(ByteRange(0, 17), _, _)).WillOnce(Return(17)); - EXPECT_EQ(17, slices_.AddRange(ByteRange(25, 61), nullptr, &error_message)) - << error_message; - EXPECT_CALL(e_, AddRange(ByteRange(0, 19), _, _)).WillOnce(Return(19)); - EXPECT_EQ(19, slices_.AddRange(ByteRange(42, 61), nullptr, &error_message)) - << error_message; -} - -TEST_F(FileSlicesTest, SliceWithPartialReturn) { - Init(0); - std::string error_message; - EXPECT_CALL(a_, AddRange(ByteRange(0, 5), _, _)).WillOnce(Return(5)); - EXPECT_CALL(b_, AddRange(ByteRange(0, 13), _, _)).WillOnce(Return(1)); - EXPECT_EQ(6, slices_.AddRange(ByteRange(0, 61), nullptr, &error_message)) - << error_message; -} - -TEST_F(FileSlicesTest, PropagateError) { - Init(0); - std::string error_message; - EXPECT_CALL(a_, AddRange(ByteRange(0, 5), _, _)).WillOnce(Return(5)); - EXPECT_CALL(b_, AddRange(ByteRange(0, 13), _, _)) - .WillRepeatedly(DoAll(SetArgPointee<2>("asdf"), Return(-1))); - EXPECT_EQ(5, slices_.AddRange(ByteRange(0, 61), nullptr, &error_message)); - EXPECT_EQ(-1, slices_.AddRange(ByteRange(5, 61), nullptr, &error_message)); - EXPECT_EQ("asdf", error_message); -} - -// Test the specific examples enumerated in RFC 2616 section 14.35.1. -TEST(RangeHeaderTest, Rfc_2616_Section_14_35_1) { - std::vector ranges; - EXPECT_EQ(RangeHeaderType::kSatisfiable, - ParseRangeHeader("bytes=0-499", 10000, &ranges)); - EXPECT_THAT(ranges, testing::ElementsAre(ByteRange(0, 500))); - - EXPECT_EQ(RangeHeaderType::kSatisfiable, - ParseRangeHeader("bytes=500-999", 10000, &ranges)); - EXPECT_THAT(ranges, testing::ElementsAre(ByteRange(500, 1000))); - - EXPECT_EQ(RangeHeaderType::kSatisfiable, - ParseRangeHeader("bytes=-500", 10000, &ranges)); - EXPECT_THAT(ranges, testing::ElementsAre(ByteRange(9500, 10000))); - - EXPECT_EQ(RangeHeaderType::kSatisfiable, - ParseRangeHeader("bytes=9500-", 10000, &ranges)); - EXPECT_THAT(ranges, testing::ElementsAre(ByteRange(9500, 10000))); - - EXPECT_EQ(RangeHeaderType::kSatisfiable, - ParseRangeHeader("bytes=0-0,-1", 10000, &ranges)); - EXPECT_THAT(ranges, - testing::ElementsAre(ByteRange(0, 1), ByteRange(9999, 10000))); - - // Non-canonical ranges. Possibly the point of these is that the adjacent - // and overlapping ranges are supposed to be coalesced into one? I'm not - // going to do that for now...just trying to get something working... - EXPECT_EQ(RangeHeaderType::kSatisfiable, - ParseRangeHeader("bytes=500-600,601-999", 10000, &ranges)); - EXPECT_THAT(ranges, - testing::ElementsAre(ByteRange(500, 601), ByteRange(601, 1000))); - EXPECT_EQ(RangeHeaderType::kSatisfiable, - ParseRangeHeader("bytes=500-700,601-999", 10000, &ranges)); - EXPECT_THAT(ranges, - testing::ElementsAre(ByteRange(500, 701), ByteRange(601, 1000))); -} - -TEST(RangeHeaderTest, Satisfiability) { - std::vector ranges; - EXPECT_EQ(RangeHeaderType::kNotSatisfiable, - ParseRangeHeader("bytes=10000-", 10000, &ranges)); - EXPECT_EQ(RangeHeaderType::kSatisfiable, - ParseRangeHeader("bytes=0-499,10000-", 10000, &ranges)); - EXPECT_THAT(ranges, testing::ElementsAre(ByteRange(0, 500))); - EXPECT_EQ(RangeHeaderType::kNotSatisfiable, - ParseRangeHeader("bytes=-1", 0, &ranges)); - EXPECT_EQ(RangeHeaderType::kNotSatisfiable, - ParseRangeHeader("bytes=0-0", 0, &ranges)); - EXPECT_EQ(RangeHeaderType::kNotSatisfiable, - ParseRangeHeader("bytes=0-", 0, &ranges)); - EXPECT_EQ(RangeHeaderType::kSatisfiable, - ParseRangeHeader("bytes=0-0", 1, &ranges)); - EXPECT_THAT(ranges, testing::ElementsAre(ByteRange(0, 1))); - EXPECT_EQ(RangeHeaderType::kSatisfiable, - ParseRangeHeader("bytes=0-", 1, &ranges)); - EXPECT_THAT(ranges, testing::ElementsAre(ByteRange(0, 1))); -} - -TEST(RangeHeaderTest, AbsentOrInvalid) { - std::vector ranges; - EXPECT_EQ(RangeHeaderType::kAbsentOrInvalid, - ParseRangeHeader(nullptr, 10000, &ranges)); - EXPECT_EQ(RangeHeaderType::kAbsentOrInvalid, - ParseRangeHeader("", 10000, &ranges)); - EXPECT_EQ(RangeHeaderType::kAbsentOrInvalid, - ParseRangeHeader("foo=0-499", 10000, &ranges)); - EXPECT_EQ(RangeHeaderType::kAbsentOrInvalid, - ParseRangeHeader("foo=0-499", 10000, &ranges)); - EXPECT_EQ(RangeHeaderType::kAbsentOrInvalid, - ParseRangeHeader("bytes=499-0", 10000, &ranges)); - EXPECT_EQ(RangeHeaderType::kAbsentOrInvalid, - ParseRangeHeader("bytes=", 10000, &ranges)); - EXPECT_EQ(RangeHeaderType::kAbsentOrInvalid, - ParseRangeHeader("bytes=,", 10000, &ranges)); - EXPECT_EQ(RangeHeaderType::kAbsentOrInvalid, - ParseRangeHeader("bytes=-", 10000, &ranges)); -} - -// TODO: test HttpServe itself! -// Currently the testing is manual. Three important cases: -// * HTTP request succeeds -// * client aborts (as in hitting ctrl-C in curl during a long request) -// * the VirtualFile returns error (say, by chmod u-r on the file backing -// a RealFileSlice) - -} // namespace -} // namespace moonfire_nvr - -int main(int argc, char **argv) { - FLAGS_alsologtostderr = true; - google::ParseCommandLineFlags(&argc, &argv, true); - testing::InitGoogleTest(&argc, argv); - google::InitGoogleLogging(argv[0]); - return RUN_ALL_TESTS(); -} diff --git a/src/http.cc b/src/http.cc deleted file mode 100644 index e2a3d00..0000000 --- a/src/http.cc +++ /dev/null @@ -1,414 +0,0 @@ -// This file is part of Moonfire NVR, a security camera digital video recorder. -// Copyright (C) 2016 Scott Lamb -// -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// In addition, as a special exception, the copyright holders give -// permission to link the code of portions of this program with the -// OpenSSL library under certain conditions as described in each -// individual source file, and distribute linked combinations including -// the two. -// -// You must obey the GNU General Public License in all respects for all -// of the code used other than OpenSSL. If you modify file(s) with this -// exception, you may extend this exception to your version of the -// file(s), but you are not obligated to do so. If you do not wish to do -// so, delete this exception statement from your version. If you delete -// this exception statement from all source files in the program, then -// also delete it here. -// -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. -// -// You should have received a copy of the GNU General Public License -// along with this program. If not, see . -// -// http.cc: See http.h. - -#include "http.h" - -#include -#include -#include -#include -#include -#include - -#include - -#include -#include -#include -#include -#include - -#include "string.h" - -namespace moonfire_nvr { - -namespace { - -// An HttpServe call still in progress. -struct ServeInProgress { - ByteRange left; - int64_t sent_bytes = 0; - std::shared_ptr file; - evhttp_request *req = nullptr; -}; - -void ServeCloseCallback(evhttp_connection *con, void *arg) { - std::unique_ptr serve( - reinterpret_cast(arg)); - LOG(INFO) << serve->req << ": received client abort after sending " - << serve->sent_bytes << " bytes; there were " << serve->left.size() - << " bytes left."; - - // The call to cancel will guarantee ServeChunkCallback is not called again. - evhttp_cancel_request(serve->req); -} - -void ServeChunkCallback(evhttp_connection *con, void *arg) { - std::unique_ptr serve( - reinterpret_cast(arg)); - - if (serve->left.size() == 0) { - LOG(INFO) << serve->req << ": done; sent " << serve->sent_bytes - << " bytes."; - evhttp_connection_set_closecb(con, nullptr, nullptr); - evhttp_send_reply_end(serve->req); - return; - } - - // Serve more data. - EvBuffer buf; - std::string error_message; - int64_t added = serve->file->AddRange(serve->left, &buf, &error_message); - if (added <= 0) { - // Order is important here: evhttp_cancel_request immediately calls the - // close callback, so remove it first to avoid double-freeing |serve|. - evhttp_connection_set_closecb(con, nullptr, nullptr); - evhttp_cancel_request(serve->req); - LOG(ERROR) << serve->req << ": Failed to serve request after sending " - << serve->sent_bytes << " bytes (" << serve->left.size() - << " bytes left): " << error_message; - return; - } - - serve->sent_bytes += added; - serve->left.begin += added; - VLOG(1) << serve->req << ": sending " << added << " bytes (more) data; still " - << serve->left.size() << " bytes left"; - evhttp_send_reply_chunk_with_cb(serve->req, buf.get(), &ServeChunkCallback, - serve.get()); - evhttp_send_reply_chunk(serve->req, buf.get()); - serve.release(); -} - -} // namespace - -namespace internal { - -RangeHeaderType ParseRangeHeader(const char *inptr, int64_t size, - std::vector *ranges) { - if (inptr == nullptr) { - return RangeHeaderType::kAbsentOrInvalid; // absent. - } - if (strncmp(inptr, "bytes=", strlen("bytes=")) != 0) { - return RangeHeaderType::kAbsentOrInvalid; // invalid syntax. - } - inptr += strlen("bytes="); - ranges->clear(); - int n_ranges = 0; - while (*inptr != 0) { // have more byte-range-sets. - ++n_ranges; - ByteRange r; - - // Parse a number. - const char *endptr; - int64_t value; - if (!strto64(inptr, 10, &endptr, &value)) { - return RangeHeaderType::kAbsentOrInvalid; // invalid syntax. - } - - if (value < 0) { // just parsed suffix-byte-range-spec. - r.begin = std::max(size + value, INT64_C(0)); - r.end = size; - if (r.begin < r.end) { // satisfiable. - ranges->emplace_back(std::move(r)); - } - inptr = endptr; - - } else { // just parsed start of byte-range-spec. - if (*endptr != '-') { - return RangeHeaderType::kAbsentOrInvalid; - } - r.begin = value; - inptr = endptr + 1; // move past the '-'. - if (*inptr == ',' || *inptr == 0) { // no end specified; use EOF. - r.end = size; - } else { // explicit range. - if (!strto64(inptr, 10, &endptr, &value) || value < r.begin) { - return RangeHeaderType::kAbsentOrInvalid; // invalid syntax. - } - inptr = endptr; - r.end = std::min(size, value + 1); // note inclusive->exclusive. - } - if (r.begin < size) { - ranges->emplace_back(std::move(r)); // satisfiable. - } - } - - if (*inptr == ',') { - inptr++; - if (*inptr == 0) { - return RangeHeaderType::kAbsentOrInvalid; // invalid syntax. - } - } - } - - if (n_ranges == 0) { // must be at least one range. - return RangeHeaderType::kAbsentOrInvalid; - } - - return ranges->empty() ? RangeHeaderType::kNotSatisfiable - : RangeHeaderType::kSatisfiable; -} - -} // namespace internal - -bool EvBuffer::AddFile(int fd, ev_off_t offset, ev_off_t length, - std::string *error_message) { - if (length == 0) { - // evbuffer_add_file fails in this trivial case, at least when using mmap. - // Just return true since there's nothing to be done. - return true; - } - - if (evbuffer_get_length(buf_) > 0) { - // Work around https://github.com/libevent/libevent/issues/306 by using a - // fresh buffer for evbuffer_add_file. - EvBuffer fresh_buffer; - if (!fresh_buffer.AddFile(fd, offset, length, error_message)) { - return false; - } - - // Crash if evbuffer_add_buffer fails, because the ownership of |fd| has - // already been transferred, and it's too confusing to support some - // failures in which the caller still owns |fd| and some in which it does - // not. - CHECK_EQ(0, evbuffer_add_buffer(buf_, fresh_buffer.buf_)) - << "evbuffer_add_buffer failed: " << strerror(errno); - return true; - } - - if (evbuffer_add_file(buf_, fd, offset, length) != 0) { - int err = errno; - *error_message = StrCat("evbuffer_add_file failed with offset ", offset, - ", length ", length, ": ", strerror(err)); - return false; - } - return true; -} - -void RealFileSlice::Init(File *dir, re2::StringPiece filename, - ByteRange range) { - dir_ = dir; - filename_ = filename.as_string(); - range_ = range; -} - -int64_t RealFileSlice::AddRange(ByteRange range, EvBuffer *buf, - std::string *error_message) const { - int fd; - int ret = dir_->Open(filename_.c_str(), O_RDONLY, &fd); - if (ret != 0) { - *error_message = StrCat("open ", filename_, ": ", strerror(ret)); - return -1; - } - if (!buf->AddFile(fd, range_.begin + range.begin, range.size(), - error_message)) { - close(fd); - return -1; - } - // |buf| now owns |fd|. - return range.size(); -} - -int64_t FillerFileSlice::AddRange(ByteRange range, EvBuffer *buf, - std::string *error_message) const { - std::unique_ptr s(new std::string); - s->reserve(size_); - if (!fn_(s.get(), error_message)) { - return 0; - } - if (s->size() != size_) { - *error_message = StrCat("Expected filled slice to be ", size_, - " bytes; got ", s->size(), " bytes."); - return 0; - } - std::string *unowned_s = s.release(); - buf->AddReference(unowned_s->data() + range.begin, - range.size(), [](const void *, size_t, void *s) { - delete reinterpret_cast(s); - }, unowned_s); - return range.size(); -} - -int64_t StringPieceSlice::AddRange(ByteRange range, EvBuffer *buf, - std::string *error_message) const { - buf->AddReference(piece_.data() + range.begin, range.size(), nullptr, - nullptr); - return range.size(); -} - -int64_t FileSlices::AddRange(ByteRange range, EvBuffer *buf, - std::string *error_message) const { - if (range.begin < 0 || range.begin > range.end || range.end > size_) { - *error_message = StrCat("Range ", range.DebugString(), - " not valid for file of size ", size_); - return false; - } - int64_t total_bytes_added = 0; - auto it = std::upper_bound(slices_.begin(), slices_.end(), range.begin, - [](int64_t begin, const SliceInfo &info) { - return begin < info.range.end; - }); - for (; it != slices_.end() && range.end > it->range.begin; ++it) { - if (total_bytes_added > 0 && (it->flags & kLazy) != 0) { - VLOG(1) << "early return of " << total_bytes_added << "/" << range.size() - << " bytes from FileSlices " << this << " because slice " - << it->slice << " is lazy."; - break; - } - ByteRange mapped( - std::max(INT64_C(0), range.begin - it->range.begin), - std::min(range.end - it->range.begin, it->range.end - it->range.begin)); - int64_t slice_bytes_added = it->slice->AddRange(mapped, buf, error_message); - total_bytes_added += slice_bytes_added > 0 ? slice_bytes_added : 0; - if (slice_bytes_added < 0 && total_bytes_added == 0) { - LOG(WARNING) << "early return of " << total_bytes_added << "/" - << range.size() << " bytes from FileSlices " << this - << " due to slice " << it->slice - << " returning error: " << *error_message; - return -1; - } else if (slice_bytes_added < mapped.size()) { - LOG(INFO) << "early return of " << total_bytes_added << "/" - << range.size() << " bytes from FileSlices " << this - << " due to slice " << it->slice << " returning " - << slice_bytes_added << "/" << mapped.size() - << " bytes. error_message (maybe populated): " - << *error_message; - break; - } - } - return total_bytes_added; -} - -void HttpSendError(evhttp_request *req, int http_err, const std::string &prefix, - int posix_err) { - evhttp_send_error(req, http_err, - EscapeHtml(prefix + strerror(posix_err)).c_str()); -} - -void HttpServe(const std::shared_ptr &file, evhttp_request *req) { - // We could support HEAD, but there's probably no need. - if (evhttp_request_get_command(req) != EVHTTP_REQ_GET) { - return evhttp_send_error(req, HTTP_BADMETHOD, "only GET allowed"); - } - - const struct evkeyvalq *in_hdrs = evhttp_request_get_input_headers(req); - struct evkeyvalq *out_hdrs = evhttp_request_get_output_headers(req); - - // Construct a Last-Modified: header. - time_t last_modified = file->last_modified(); - struct tm last_modified_tm; - if (gmtime_r(&last_modified, &last_modified_tm) == 0) { - return HttpSendError(req, HTTP_INTERNAL, "gmtime_r failed: ", errno); - } - char last_modified_str[50]; - if (strftime(last_modified_str, sizeof(last_modified_str), - "%a, %d %b %Y %H:%M:%S GMT", &last_modified_tm) == 0) { - return HttpSendError(req, HTTP_INTERNAL, "strftime failed: ", errno); - } - std::string etag = file->etag(); - - // Ignore the "Range:" header if "If-Range:" specifies an incorrect etag. - const char *if_range = evhttp_find_header(in_hdrs, "If-Range"); - const char *range_hdr = evhttp_find_header(in_hdrs, "Range"); - if (if_range != nullptr && etag != if_range) { - LOG(INFO) << req << ": Ignoring Range: because If-Range: is stale."; - range_hdr = nullptr; - } - - EvBuffer buf; - std::vector ranges; - auto range_type = - internal::ParseRangeHeader(range_hdr, file->size(), &ranges); - std::string error_message; - int http_status; - const char *http_status_str; - ByteRange left; - switch (range_type) { - case internal::RangeHeaderType::kNotSatisfiable: { - std::string range_hdr = StrCat("bytes */", file->size()); - evhttp_add_header(out_hdrs, "Content-Range", range_hdr.c_str()); - http_status = 416; - http_status_str = "Range Not Satisfiable"; - LOG(INFO) << req - << ": Replying to non-satisfiable range request: " << range_hdr; - break; - } - - case internal::RangeHeaderType::kSatisfiable: - // We only support the simpler single-range case for now. - // A multi-range request just serves the whole file via the fallthrough. - if (ranges.size() == 1) { - std::string range_hdr = StrCat("bytes ", ranges[0].begin, "-", - ranges[0].end - 1, "/", file->size()); - left = ranges[0]; - evhttp_add_header(out_hdrs, "Content-Range", range_hdr.c_str()); - http_status = 206; - http_status_str = "Partial Content"; - LOG(INFO) << req << ": URI " << evhttp_request_get_uri(req) - << ": client requested byte range " << left - << " (total file size " << file->size() << ")"; - break; - } - // FALLTHROUGH - - case internal::RangeHeaderType::kAbsentOrInvalid: { - left = ByteRange(0, file->size()); - LOG(INFO) << req << ": URI " << evhttp_request_get_uri(req) - << ": Client requested whole file of size " << file->size(); - http_status = HTTP_OK; - http_status_str = "OK"; - break; - } - - default: - LOG(FATAL) << "unexpected range_type: " << static_cast(range_type); - } - - // Successful reply started; add common headers and send. - evhttp_add_header(out_hdrs, "Content-Length", StrCat(left.size()).c_str()); - evhttp_add_header(out_hdrs, "Content-Type", file->mime_type().c_str()); - evhttp_add_header(out_hdrs, "Accept-Ranges", "bytes"); - evhttp_add_header(out_hdrs, "Last-Modified", last_modified_str); - evhttp_add_header(out_hdrs, "ETag", etag.c_str()); - evhttp_send_reply_start(req, http_status, http_status_str); - - ServeInProgress *serve = new ServeInProgress; - serve->file = file; - serve->left = left; - serve->req = req; - evhttp_connection *con = evhttp_request_get_connection(req); - evhttp_connection_set_closecb(con, &ServeCloseCallback, serve); - return ServeChunkCallback(con, serve); -} - -} // namespace moonfire_nvr diff --git a/src/http.h b/src/http.h deleted file mode 100644 index cf2d52f..0000000 --- a/src/http.h +++ /dev/null @@ -1,306 +0,0 @@ -// This file is part of Moonfire NVR, a security camera digital video recorder. -// Copyright (C) 2016 Scott Lamb -// -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// In addition, as a special exception, the copyright holders give -// permission to link the code of portions of this program with the -// OpenSSL library under certain conditions as described in each -// individual source file, and distribute linked combinations including -// the two. -// -// You must obey the GNU General Public License in all respects for all -// of the code used other than OpenSSL. If you modify file(s) with this -// exception, you may extend this exception to your version of the -// file(s), but you are not obligated to do so. If you do not wish to do -// so, delete this exception statement from your version. If you delete -// this exception statement from all source files in the program, then -// also delete it here. -// -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. -// -// You should have received a copy of the GNU General Public License -// along with this program. If not, see . -// -// http.h: classes for HTTP serving. In particular, there are helpers for -// serving HTTP byte range requests with libevent. - -#ifndef MOONFIRE_NVR_HTTP_H -#define MOONFIRE_NVR_HTTP_H - -#include -#include -#include -#include -#include - -#include -#include -#include -#include - -#include -#include -#include -#include -#include - -#include "filesystem.h" -#include "string.h" - -namespace moonfire_nvr { - -// Single-use object to represent a set of HTTP query parameters. -class QueryParameters { - public: - // Parse parameters from the given URI. - // Caller should check ok() afterward. - QueryParameters(const char *uri) { - TAILQ_INIT(&me_); - ok_ = evhttp_parse_query(uri, &me_) == 0; - } - QueryParameters(const QueryParameters &) = delete; - void operator=(const QueryParameters &) = delete; - - ~QueryParameters() { evhttp_clear_headers(&me_); } - - bool ok() const { return ok_; } - const char *Get(const char *param) const { - return evhttp_find_header(&me_, param); - } - - private: - struct evkeyvalq me_; - bool ok_ = false; -}; - -// Wrapped version of libevent's "struct evbuffer" which uses RAII and simply -// aborts the process if allocations fail. (Moonfire NVR is intended to run on -// Linux systems with the default vm.overcommit_memory=0, so there's probably -// no point in trying to gracefully recover from a condition that's unlikely -// to ever happen.) -class EvBuffer { - public: - EvBuffer() { buf_ = CHECK_NOTNULL(evbuffer_new()); } - EvBuffer(const EvBuffer &) = delete; - EvBuffer &operator=(const EvBuffer &) = delete; - ~EvBuffer() { evbuffer_free(buf_); } - - struct evbuffer *get() { - return buf_; - } - - void Add(const re2::StringPiece &s) { - CHECK_EQ(0, evbuffer_add(buf_, s.data(), s.size())); - } - - void AddPrintf(const char *fmt, ...) __attribute__((format(printf, 2, 3))) { - va_list argp; - va_start(argp, fmt); - CHECK_LE(0, evbuffer_add_vprintf(buf_, fmt, argp)); - va_end(argp); - } - - // Delegates to evbuffer_add_file. - // On success, |fd| will be closed by libevent. On failure, it remains open. - bool AddFile(int fd, ev_off_t offset, ev_off_t length, - std::string *error_message); - - void AddReference(const void *data, size_t datlen, - evbuffer_ref_cleanup_cb cleanupfn, void *cleanupfn_arg) { - CHECK_EQ( - 0, evbuffer_add_reference(buf_, data, datlen, cleanupfn, cleanupfn_arg)) - << strerror(errno); - } - - private: - struct evbuffer *buf_; -}; - -struct ByteRange { - ByteRange() {} - ByteRange(int64_t begin, int64_t end) : begin(begin), end(end) {} - int64_t begin = 0; - int64_t end = 0; // exclusive. - int64_t size() const { return end - begin; } - bool operator==(const ByteRange &o) const { - return begin == o.begin && end == o.end; - } - std::string DebugString() const { return StrCat("[", begin, ", ", end, ")"); } -}; - -inline std::ostream &operator<<(std::ostream &out, const ByteRange &range) { - return out << range.DebugString(); -} - -// Helper for sending HTTP errors based on POSIX error returns. -void HttpSendError(evhttp_request *req, int http_err, const std::string &prefix, - int posix_errno); - -class FileSlice { - public: - virtual ~FileSlice() {} - - virtual int64_t size() const = 0; - - // Add some to all of the given non-empty |range| to |buf|. - // Returns the number of bytes added, or < 0 on error. - // On error, |error_message| should be populated. (|error_message| may also be - // populated if 0 <= return value < range.size(), such as if one of a - // FileSlices object's failed. However, it's safe to simply retry such - // partial failures later.) - virtual int64_t AddRange(ByteRange range, EvBuffer *buf, - std::string *error_message) const = 0; -}; - -class VirtualFile : public FileSlice { - public: - virtual ~VirtualFile() {} - - // Return the given property of the file. - virtual time_t last_modified() const = 0; - virtual std::string etag() const = 0; - virtual std::string mime_type() const = 0; -}; - -class RealFileSlice : public FileSlice { - public: - // |dir| must outlive the RealFileSlice. - void Init(File *dir, re2::StringPiece filename, ByteRange range); - - int64_t size() const final { return range_.size(); } - - int64_t AddRange(ByteRange range, EvBuffer *buf, - std::string *error_message) const final; - - private: - File *dir_; - std::string filename_; - ByteRange range_; -}; - -// A FileSlice of a pre-defined length which calls a function which fills the -// slice on demand. The FillerFileSlice is responsible for subsetting. -class FillerFileSlice : public FileSlice { - public: - using FillFunction = - std::function; - - void Init(size_t size, FillFunction fn) { - fn_ = fn; - size_ = size; - } - - int64_t size() const final { return size_; } - - int64_t AddRange(ByteRange range, EvBuffer *buf, - std::string *error_message) const final; - - private: - FillFunction fn_; - size_t size_; -}; - -// A FileSlice backed by in-memory data which outlives this object. -class StringPieceSlice : public FileSlice { - public: - StringPieceSlice() = default; - explicit StringPieceSlice(re2::StringPiece piece) : piece_(piece) {} - void Init(re2::StringPiece piece) { piece_ = piece; } - - int64_t size() const final { return piece_.size(); } - int64_t AddRange(ByteRange range, EvBuffer *buf, - std::string *error_message) const final; - - private: - re2::StringPiece piece_; -}; - -// A slice composed of other slices. -class FileSlices : public FileSlice { - public: - FileSlices() {} - FileSlices(const FileSlices &) = delete; - FileSlices &operator=(const FileSlices &) = delete; - - // |slice| must outlive the FileSlices. - // |slice->size()| should not change after this call. - // |flags| should be a bitmask of Flags values below. - void Append(const FileSlice *slice, int flags = 0) { - int64_t new_size = size_ + slice->size(); - slices_.emplace_back(ByteRange(size_, new_size), slice, flags); - size_ = new_size; - } - - size_t num_slices() const { return slices_.size(); } - - int64_t size() const final { return size_; } - int64_t AddRange(ByteRange range, EvBuffer *buf, - std::string *error_message) const final; - - enum Flags { - // kLazy, as an argument to Append, instructs the FileSlices to append - // this slice in AddRange only if it is the first slice in the requested - // range. Otherwise it returns early, expecting HttpServe to call AddRange - // again after the earlier ranges have been sent. This is useful if it is - // expensive to have the given slice pending. In particular, it is useful - // when serving many file slices on 32-bit machines to avoid exhausting - // the address space with too many memory mappings. - kLazy = 1 - }; - - private: - struct SliceInfo { - SliceInfo(ByteRange range, const FileSlice *slice, int flags) - : range(range), slice(slice), flags(flags) {} - ByteRange range; - const FileSlice *slice = nullptr; - int flags; - }; - int64_t size_ = 0; - - std::vector slices_; -}; - -// Serve an HTTP request |req| from |file|, handling byte range and -// conditional serving. (Similar to golang's http.ServeContent.) -// -// |file| will be retained as long as the request is being served. -void HttpServe(const std::shared_ptr &file, evhttp_request *req); - -namespace internal { - -// Value to represent result of parsing HTTP 1.1 "Range:" header. -enum class RangeHeaderType { - // Ignore the header, serving all bytes in the file. - kAbsentOrInvalid, - - // The server SHOULD return a response with status 416 (Requested range not - // satisfiable). - kNotSatisfiable, - - // The server SHOULD return a response with status 406 (Partial Content). - kSatisfiable -}; - -// Parse an HTTP 1.1 "Range:" header value, following RFC 2616 section 14.35. -// This function is for use by HttpServe; it is exposed for testing only. -// -// |value| on entry should be the header value (after the ": "), or nullptr. -// |size| on entry should be the number of bytes available to serve. -// On kSatisfiable return, |ranges| will be filled with the satisfiable ranges. -// Otherwise, its contents are undefined. -RangeHeaderType ParseRangeHeader(const char *value, int64_t size, - std::vector *ranges); - -} // namespace internal - -} // namespace moonfire_nvr - -#endif // MOONFIRE_NVR_HTTP_H diff --git a/src/main.rs b/src/main.rs new file mode 100644 index 0000000..a5da2ae --- /dev/null +++ b/src/main.rs @@ -0,0 +1,195 @@ +// This file is part of Moonfire NVR, a security camera digital video recorder. +// Copyright (C) 2016 Scott Lamb +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// In addition, as a special exception, the copyright holders give +// permission to link the code of portions of this program with the +// OpenSSL library under certain conditions as described in each +// individual source file, and distribute linked combinations including +// the two. +// +// You must obey the GNU General Public License in all respects for all +// of the code used other than OpenSSL. If you modify file(s) with this +// exception, you may extend this exception to your version of the +// file(s), but you are not obligated to do so. If you do not wish to do +// so, delete this exception statement from your version. If you delete +// this exception statement from all source files in the program, then +// also delete it here. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program. If not, see . + +#![cfg_attr(test, feature(test))] +#![feature(alloc, box_syntax, conservative_impl_trait, plugin, proc_macro)] +#![plugin(clippy)] + +extern crate alloc; +extern crate byteorder; +extern crate core; +#[macro_use] extern crate chan; +extern crate chan_signal; +extern crate docopt; +#[macro_use] extern crate ffmpeg; +extern crate ffmpeg_sys; +extern crate fnv; +extern crate hyper; +#[macro_use] extern crate lazy_static; +extern crate libc; +#[macro_use] extern crate log; +extern crate lru_cache; +extern crate rusqlite; +extern crate memmap; +#[macro_use] extern crate mime; +extern crate openssl; +extern crate regex; +extern crate rustc_serialize; +extern crate serde; +#[macro_use] extern crate serde_derive; +extern crate serde_json; +extern crate slog; +extern crate slog_envlogger; +extern crate slog_stdlog; +extern crate slog_term; +extern crate smallvec; +extern crate time; +extern crate url; +extern crate uuid; + +use hyper::server::Server; +use slog::DrainExt; +use std::path::Path; +use std::sync::Arc; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::thread; + +mod db; +mod dir; +mod error; +mod h264; +mod mmapfile; +mod mp4; +mod pieces; +mod recording; +mod resource; +mod stream; +mod streamer; +#[cfg(test)] mod testutil; +mod web; + +/// Commandline usage string. This is in the particular format expected by the `docopt` crate. +/// Besides being printed on --help or argument parsing error, it's actually parsed to define the +/// allowed commandline arguments and their defaults. +const USAGE: &'static str = " +Usage: moonfire-nvr [options] + moonfire-nvr (--help | --version) + +Options: + -h, --help Show this message. + --version Show the version of moonfire-nvr. + --db-dir DIR Set the directory holding the SQLite3 index database. + This is typically on a flash device. + [default: /var/lib/moonfire-nvr/db] + --sample-file-dir DIR Set the directory holding video data. + This is typically on a hard drive. + [default: /var/lib/moonfire-nvr/sample] + --http-addr ADDR Set the bind address for the unencrypted HTTP server. + [default: 0.0.0.0:8080] + --read-only Forces read-only mode / disables recording. +"; + +/// Commandline arguments corresponding to `USAGE`; automatically filled by the `docopt` crate. +#[derive(RustcDecodable)] +struct Args { + flag_db_dir: String, + flag_sample_file_dir: String, + flag_http_addr: String, + flag_read_only: bool, +} + +fn main() { + // Watch for termination signals. + // This must be started before any threads are spawned (such as the async logger thread) so + // that signals will be blocked in all threads. + let signal = chan_signal::notify(&[chan_signal::Signal::INT, chan_signal::Signal::TERM]); + + // Initialize logging. + let drain = slog_term::StreamerBuilder::new().async().full().build(); + let drain = slog_envlogger::new(drain); + slog_stdlog::set_logger(slog::Logger::root(drain.ignore_err(), None)).unwrap(); + + // Parse commandline arguments. + let version = "Moonfire NVR 0.1.0".to_owned(); + let args: Args = docopt::Docopt::new(USAGE) + .and_then(|d| d.version(Some(version)).decode()) + .unwrap_or_else(|e| e.exit()); + + // Open the database and populate cached state. + let db_dir = dir::Fd::open(&args.flag_db_dir).unwrap(); + db_dir.lock(if args.flag_read_only { libc::LOCK_SH } else { libc::LOCK_EX } | libc::LOCK_NB) + .unwrap(); + let conn = rusqlite::Connection::open_with_flags( + Path::new(&args.flag_db_dir).join("db"), + if args.flag_read_only { + rusqlite::SQLITE_OPEN_READ_ONLY + } else { + rusqlite::SQLITE_OPEN_READ_WRITE + } | + // rusqlite::Connection is not Sync, so there's no reason to tell SQLite3 to use the + // serialized threading mode. + rusqlite::SQLITE_OPEN_NO_MUTEX).unwrap(); + let db = Arc::new(db::Database::new(conn).unwrap()); + let dir = dir::SampleFileDir::new(&args.flag_sample_file_dir, db.clone()).unwrap(); + info!("Database is loaded."); + + // Start a streamer for each camera. + let shutdown = Arc::new(AtomicBool::new(false)); + let mut streamers = Vec::new(); + let syncer = if !args.flag_read_only { + let (syncer_channel, syncer_join) = dir::start_syncer(dir.clone()).unwrap(); + let l = db.lock(); + let cameras = l.cameras_by_id().len(); + for (i, (id, camera)) in l.cameras_by_id().iter().enumerate() { + let rotate_offset_sec = streamer::ROTATE_INTERVAL_SEC * i as i64 / cameras as i64; + let mut streamer = streamer::Streamer::new( + db.clone(), dir.clone(), syncer_channel.clone(), shutdown.clone(), *id, camera, + rotate_offset_sec); + let name = format!("stream-{}", streamer.short_name()); + streamers.push(thread::Builder::new().name(name).spawn(move|| { + streamer.run(); + }).expect("can't create thread")); + } + Some((syncer_channel, syncer_join)) + } else { None }; + + // Start the web interface. + let server = Server::http(args.flag_http_addr.as_str()).unwrap(); + let h = web::Handler::new(db.clone(), dir.clone()); + let _guard = server.handle(h); + info!("Ready to serve HTTP requests"); + + // Wait for a signal and shut down. + chan_select! { + signal.recv() -> signal => info!("Received signal {:?}; shutting down streamers.", signal), + } + shutdown.store(true, Ordering::SeqCst); + for streamer in streamers.drain(..) { + streamer.join().unwrap(); + } + if let Some((syncer_channel, syncer_join)) = syncer { + info!("Shutting down syncer."); + drop(syncer_channel); + syncer_join.join().unwrap(); + } + info!("Exiting."); + // TODO: drain the logger. + std::process::exit(0); +} diff --git a/src/mmapfile.rs b/src/mmapfile.rs new file mode 100644 index 0000000..e544094 --- /dev/null +++ b/src/mmapfile.rs @@ -0,0 +1,71 @@ +// This file is part of Moonfire NVR, a security camera digital video recorder. +// Copyright (C) 2016 Scott Lamb +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// In addition, as a special exception, the copyright holders give +// permission to link the code of portions of this program with the +// OpenSSL library under certain conditions as described in each +// individual source file, and distribute linked combinations including +// the two. +// +// You must obey the GNU General Public License in all respects for all +// of the code used other than OpenSSL. If you modify file(s) with this +// exception, you may extend this exception to your version of the +// file(s), but you are not obligated to do so. If you do not wish to do +// so, delete this exception statement from your version. If you delete +// this exception statement from all source files in the program, then +// also delete it here. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program. If not, see . + +//! Memory-mapped file serving. + +extern crate memmap; + +use error::Result; +use std::fs::File; +use std::io; +use std::ops::Range; + +/// Memory-mapped file slice. +/// This struct is meant to be used in constructing an implementation of the `resource::Resource` +/// or `pieces::ContextWriter` traits. The file in question should be immutable, as files shrinking +/// during `mmap` will cause the process to fail with `SIGBUS`. Moonfire NVR sample files satisfy +/// this requirement: +/// +/// * They should only be modified by Moonfire NVR itself. Installation instructions encourage +/// creating a dedicated user/group for Moonfire NVR and ensuring only this group has +/// permissions to Moonfire NVR's directories. +/// * Moonfire NVR never modifies sample files after inserting their matching recording entries +/// into the database. They are kept as-is until they are deleted. +pub struct MmapFileSlice { + f: File, + range: Range, +} + +impl MmapFileSlice { + pub fn new(f: File, range: Range) -> MmapFileSlice { + MmapFileSlice{f: f, range: range} + } + + pub fn write_to(&self, range: Range, out: &mut io::Write) -> Result<()> { + // TODO: overflow check (in case u64 is larger than usize). + let r = self.range.start + range.start .. self.range.start + range.end; + assert!(r.end <= self.range.end, + "requested={:?} within={:?}", range, self.range); + let mmap = memmap::Mmap::open_with_offset( + &self.f, memmap::Protection::Read, r.start as usize, (r.end - r.start) as usize)?; + unsafe { out.write_all(mmap.as_slice())?; } + Ok(()) + } +} diff --git a/src/moonfire-db-test.cc b/src/moonfire-db-test.cc deleted file mode 100644 index 903aa9f..0000000 --- a/src/moonfire-db-test.cc +++ /dev/null @@ -1,458 +0,0 @@ -// -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// In addition, as a special exception, the copyright holders give -// permission to link the code of portions of this program with the -// OpenSSL library under certain conditions as described in each -// individual source file, and distribute linked combinations including -// the two. -// -// You must obey the GNU General Public License in all respects for all -// of the code used other than OpenSSL. If you modify file(s) with this -// exception, you may extend this exception to your version of the -// file(s), but you are not obligated to do so. If you do not wish to do -// so, delete this exception statement from your version. If you delete -// this exception statement from all source files in the program, then -// also delete it here. -// -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. -// -// You should have received a copy of the GNU General Public License -// along with this program. If not, see . -// -// moonfire-db-test.cc: tests of the moonfire-db.h interface. - -#include - -#include - -#include -#include -#include - -#include "moonfire-db.h" -#include "sqlite.h" -#include "string.h" -#include "testutil.h" - -DECLARE_bool(alsologtostderr); - -using testing::_; -using testing::HasSubstr; -using testing::DoAll; -using testing::Return; -using testing::SetArgPointee; - -namespace moonfire_nvr { -namespace { - -class MoonfireDbTest : public testing::Test { - protected: - MoonfireDbTest() { - tmpdir_ = PrepareTempDirOrDie("moonfire-db-test"); - std::string error_message; - CHECK(db_.Open(StrCat(tmpdir_, "/db").c_str(), - SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE, &error_message)) - << error_message; - std::string create_sql = ReadFileOrDie("../src/schema.sql"); - DatabaseContext ctx(&db_); - CHECK(RunStatements(&ctx, create_sql, &error_message)) << error_message; - } - - int64_t AddCamera(Uuid uuid, re2::StringPiece short_name) { - DatabaseContext ctx(&db_); - auto run = ctx.UseOnce( - R"( - insert into camera (uuid, short_name, host, username, password, - main_rtsp_path, sub_rtsp_path, retain_bytes) - values (:uuid, :short_name, :host, :username, :password, - :main_rtsp_path, :sub_rtsp_path, :retain_bytes); - )"); - run.BindBlob(":uuid", uuid.binary_view()); - run.BindText(":short_name", short_name); - run.BindText(":host", "test-camera"); - run.BindText(":username", "foo"); - run.BindText(":password", "bar"); - run.BindText(":main_rtsp_path", "/main"); - run.BindText(":sub_rtsp_path", "/sub"); - run.BindInt64(":retain_bytes", 42); - CHECK_EQ(SQLITE_DONE, run.Step()) << run.error_message(); - if (run.Step() != SQLITE_DONE) { - ADD_FAILURE() << run.error_message(); - return -1; - } - return ctx.last_insert_rowid(); - } - - void ExpectNoRecordings(Uuid camera_uuid) { - int rows = 0; - mdb_->ListCameras([&](const ListCamerasRow &row) { - ++rows; - EXPECT_EQ(camera_uuid, row.uuid); - EXPECT_EQ("test-camera", row.host); - EXPECT_EQ("foo", row.username); - EXPECT_EQ("bar", row.password); - EXPECT_EQ("/main", row.main_rtsp_path); - EXPECT_EQ("/sub", row.sub_rtsp_path); - EXPECT_EQ(42, row.retain_bytes); - EXPECT_EQ(std::numeric_limits::max(), row.min_start_time_90k); - EXPECT_EQ(std::numeric_limits::min(), row.max_end_time_90k); - EXPECT_EQ(0, row.total_duration_90k); - EXPECT_EQ(0, row.total_sample_file_bytes); - return IterationControl::kContinue; - }); - EXPECT_EQ(1, rows); - - GetCameraRow row; - EXPECT_TRUE(mdb_->GetCamera(camera_uuid, &row)); - EXPECT_THAT(row.days, testing::ElementsAre()); - - std::string error_message; - rows = 0; - EXPECT_TRUE(mdb_->ListCameraRecordings( - camera_uuid, 0, std::numeric_limits::max(), - [&](const ListCameraRecordingsRow &row) { - ++rows; - return IterationControl::kBreak; - }, - &error_message)) - << error_message; - EXPECT_EQ(0, rows); - - rows = 0; - EXPECT_TRUE(mdb_->ListMp4Recordings( - camera_uuid, 0, std::numeric_limits::max(), - [&](Recording &recording, const VideoSampleEntry &entry) { - ++rows; - return IterationControl::kBreak; - }, - &error_message)) - << error_message; - EXPECT_EQ(0, rows); - } - - void ExpectSingleRecording(Uuid camera_uuid, const Recording &recording, - const VideoSampleEntry &entry, - ListOldestSampleFilesRow *save_oldest_row) { - std::string error_message; - int rows = 0; - mdb_->ListCameras([&](const ListCamerasRow &row) { - ++rows; - EXPECT_EQ(camera_uuid, row.uuid); - EXPECT_EQ(recording.start_time_90k, row.min_start_time_90k); - EXPECT_EQ(recording.end_time_90k, row.max_end_time_90k); - EXPECT_EQ(recording.end_time_90k - recording.start_time_90k, - row.total_duration_90k); - EXPECT_EQ(recording.sample_file_bytes, row.total_sample_file_bytes); - return IterationControl::kContinue; - }); - EXPECT_EQ(1, rows); - - std::map expected_days; - internal::AdjustDaysMap(recording.start_time_90k, recording.end_time_90k, 1, - &expected_days); - GetCameraRow row; - EXPECT_TRUE(mdb_->GetCamera(camera_uuid, &row)); - EXPECT_THAT(row.days, testing::Eq(expected_days)); - - GetCameraRow camera_row; - EXPECT_TRUE(mdb_->GetCamera(camera_uuid, &camera_row)); - EXPECT_EQ(recording.start_time_90k, camera_row.min_start_time_90k); - EXPECT_EQ(recording.end_time_90k, camera_row.max_end_time_90k); - EXPECT_EQ(recording.end_time_90k - recording.start_time_90k, - camera_row.total_duration_90k); - EXPECT_EQ(recording.sample_file_bytes, camera_row.total_sample_file_bytes); - - rows = 0; - EXPECT_TRUE(mdb_->ListCameraRecordings( - camera_uuid, 0, std::numeric_limits::max(), - [&](const ListCameraRecordingsRow &row) { - ++rows; - EXPECT_EQ(recording.start_time_90k, row.start_time_90k); - EXPECT_EQ(recording.end_time_90k, row.end_time_90k); - EXPECT_EQ(recording.video_samples, row.video_samples); - EXPECT_EQ(recording.sample_file_bytes, row.sample_file_bytes); - EXPECT_EQ(entry.sha1, row.video_sample_entry_sha1); - EXPECT_EQ(entry.width, row.width); - EXPECT_EQ(entry.height, row.height); - return IterationControl::kContinue; - }, - &error_message)) - << error_message; - EXPECT_EQ(1, rows); - - rows = 0; - EXPECT_TRUE(mdb_->ListOldestSampleFiles( - camera_uuid, - [&](const ListOldestSampleFilesRow &row) { - ++rows; - EXPECT_EQ(recording.id, row.recording_id); - EXPECT_EQ(recording.sample_file_uuid, row.sample_file_uuid); - EXPECT_EQ(recording.end_time_90k - recording.start_time_90k, - row.duration_90k); - EXPECT_EQ(recording.sample_file_bytes, row.sample_file_bytes); - *save_oldest_row = row; - return IterationControl::kContinue; - }, - &error_message)) - << error_message; - EXPECT_EQ(1, rows); - - rows = 0; - EXPECT_TRUE(mdb_->ListMp4Recordings( - camera_uuid, 0, std::numeric_limits::max(), - [&](Recording &some_recording, const VideoSampleEntry &some_entry) { - ++rows; - - EXPECT_EQ(recording.id, some_recording.id); - EXPECT_EQ(recording.camera_id, some_recording.camera_id); - EXPECT_EQ(recording.sample_file_sha1, - some_recording.sample_file_sha1); - EXPECT_EQ(recording.sample_file_uuid, - some_recording.sample_file_uuid); - EXPECT_EQ(recording.video_sample_entry_id, - some_recording.video_sample_entry_id); - EXPECT_EQ(recording.start_time_90k, some_recording.start_time_90k); - EXPECT_EQ(recording.end_time_90k, some_recording.end_time_90k); - EXPECT_EQ(recording.sample_file_bytes, - some_recording.sample_file_bytes); - EXPECT_EQ(recording.video_samples, some_recording.video_samples); - EXPECT_EQ(recording.video_sync_samples, - some_recording.video_sync_samples); - EXPECT_EQ(recording.video_index, some_recording.video_index); - - EXPECT_EQ(entry.id, some_entry.id); - EXPECT_EQ(entry.sha1, some_entry.sha1); - EXPECT_EQ(entry.data, some_entry.data); - EXPECT_EQ(entry.width, some_entry.width); - EXPECT_EQ(entry.height, some_entry.height); - - return IterationControl::kContinue; - }, - &error_message)) - << error_message; - EXPECT_EQ(1, rows); - } - - std::string tmpdir_; - Database db_; - std::unique_ptr mdb_; -}; - -TEST(AdjustDaysMapTest, Basic) { - std::map days; - - // Create a day. - const int64_t kTestTime = INT64_C(130647162600000); // 2015-12-31 23:59:00 - moonfire_nvr::internal::AdjustDaysMap( - kTestTime, kTestTime + 60 * kTimeUnitsPerSecond, 1, &days); - EXPECT_THAT(days, testing::ElementsAre(std::make_pair( - "2015-12-31", 60 * kTimeUnitsPerSecond))); - - // Add to a day. - moonfire_nvr::internal::AdjustDaysMap( - kTestTime, kTestTime + 60 * kTimeUnitsPerSecond, 1, &days); - EXPECT_THAT(days, testing::ElementsAre(std::make_pair( - "2015-12-31", 120 * kTimeUnitsPerSecond))); - - // Subtract from a day. - moonfire_nvr::internal::AdjustDaysMap( - kTestTime, kTestTime + 60 * kTimeUnitsPerSecond, -1, &days); - EXPECT_THAT(days, testing::ElementsAre(std::make_pair( - "2015-12-31", 60 * kTimeUnitsPerSecond))); - - // Remove a day. - moonfire_nvr::internal::AdjustDaysMap( - kTestTime, kTestTime + 60 * kTimeUnitsPerSecond, -1, &days); - EXPECT_THAT(days, testing::ElementsAre()); - - // Create two days. - moonfire_nvr::internal::AdjustDaysMap( - kTestTime, kTestTime + 3 * 60 * kTimeUnitsPerSecond, 1, &days); - EXPECT_THAT(days, - testing::ElementsAre( - std::make_pair("2015-12-31", 1 * 60 * kTimeUnitsPerSecond), - std::make_pair("2016-01-01", 2 * 60 * kTimeUnitsPerSecond))); - - // Add to two days. - moonfire_nvr::internal::AdjustDaysMap( - kTestTime, kTestTime + 3 * 60 * kTimeUnitsPerSecond, 1, &days); - EXPECT_THAT(days, - testing::ElementsAre( - std::make_pair("2015-12-31", 2 * 60 * kTimeUnitsPerSecond), - std::make_pair("2016-01-01", 4 * 60 * kTimeUnitsPerSecond))); - - // Subtract from two days. - moonfire_nvr::internal::AdjustDaysMap( - kTestTime, kTestTime + 3 * 60 * kTimeUnitsPerSecond, -1, &days); - EXPECT_THAT(days, - testing::ElementsAre( - std::make_pair("2015-12-31", 1 * 60 * kTimeUnitsPerSecond), - std::make_pair("2016-01-01", 2 * 60 * kTimeUnitsPerSecond))); - - // Remove two days. - moonfire_nvr::internal::AdjustDaysMap( - kTestTime, kTestTime + 3 * 60 * kTimeUnitsPerSecond, -1, &days); - EXPECT_THAT(days, testing::ElementsAre()); -} - -TEST(GetDayBoundsTest, Basic) { - int64_t start_90k; - int64_t end_90k; - std::string error_msg; - - // Normal day. - EXPECT_TRUE(GetDayBounds("2015-12-31", &start_90k, &end_90k, &error_msg)) - << error_msg; - EXPECT_EQ(INT64_C(130639392000000), start_90k); - EXPECT_EQ(INT64_C(130647168000000), end_90k); - - // Spring forward (23-hour day). - EXPECT_TRUE(GetDayBounds("2016-03-13", &start_90k, &end_90k, &error_msg)); - EXPECT_EQ(INT64_C(131207040000000), start_90k); - EXPECT_EQ(INT64_C(131214492000000), end_90k); - - // Fall back (25-hour day). - EXPECT_TRUE(GetDayBounds("2016-11-06", &start_90k, &end_90k, &error_msg)); - EXPECT_EQ(INT64_C(133057404000000), start_90k); - EXPECT_EQ(INT64_C(133065504000000), end_90k); - - // Unparseable day. - EXPECT_FALSE(GetDayBounds("xxxx-xx-xx", &start_90k, &end_90k, &error_msg)); -} - -// Basic test of running some queries on an empty database. -TEST_F(MoonfireDbTest, EmptyDatabase) { - std::string error_message; - mdb_.reset(new MoonfireDatabase); - ASSERT_TRUE(mdb_->Init(&db_, &error_message)) << error_message; - - mdb_->ListCameras([&](const ListCamerasRow &row) { - ADD_FAILURE() << "row unexpected"; - return IterationControl::kBreak; - }); - - GetCameraRow get_camera_row; - EXPECT_FALSE(mdb_->GetCamera(Uuid(), &get_camera_row)); - - EXPECT_FALSE( - mdb_->ListCameraRecordings(Uuid(), 0, std::numeric_limits::max(), - [&](const ListCameraRecordingsRow &row) { - ADD_FAILURE() << "row unexpected"; - return IterationControl::kBreak; - }, - &error_message)); - - EXPECT_FALSE(mdb_->ListMp4Recordings( - Uuid(), 0, std::numeric_limits::max(), - [&](Recording &recording, const VideoSampleEntry &entry) { - ADD_FAILURE() << "row unexpected"; - return IterationControl::kBreak; - }, - &error_message)); -} - -// Basic test of the full lifecycle of recording. -// Does not exercise many error cases. -TEST_F(MoonfireDbTest, FullLifecycle) { - std::string error_message; - const char kCameraShortName[] = "testcam"; - Uuid camera_uuid = GetRealUuidGenerator()->Generate(); - int64_t camera_id = AddCamera(camera_uuid, kCameraShortName); - ASSERT_GT(camera_id, 0); - mdb_.reset(new MoonfireDatabase); - ASSERT_TRUE(mdb_->Init(&db_, &error_message)) << error_message; - - ExpectNoRecordings(camera_uuid); - - std::vector reserved; - EXPECT_TRUE(mdb_->ListReservedSampleFiles(&reserved, &error_message)) - << error_message; - EXPECT_THAT(reserved, testing::IsEmpty()); - - std::vector uuids = mdb_->ReserveSampleFiles(2, &error_message); - ASSERT_THAT(uuids, testing::SizeIs(2)) << error_message; - - EXPECT_TRUE(mdb_->ListReservedSampleFiles(&reserved, &error_message)) - << error_message; - EXPECT_THAT(reserved, testing::UnorderedElementsAre(uuids[0], uuids[1])); - - VideoSampleEntry entry; - entry.sha1.resize(20); - entry.width = 768; - entry.height = 512; - entry.data.resize(100); - ASSERT_TRUE(mdb_->InsertVideoSampleEntry(&entry, &error_message)) - << error_message; - ASSERT_GT(entry.id, 0); - - Recording recording; - recording.camera_id = camera_id; - recording.sample_file_uuid = GetRealUuidGenerator()->Generate(); - recording.video_sample_entry_id = entry.id; - SampleIndexEncoder encoder; - encoder.Init(&recording, UINT64_C(1430006400) * kTimeUnitsPerSecond); - encoder.AddSample(kTimeUnitsPerSecond, 42, true); - - // Inserting a recording should succeed and remove its uuid from the - // reserved table. - ASSERT_FALSE(mdb_->InsertRecording(&recording, &error_message)); - EXPECT_THAT(error_message, testing::HasSubstr("not reserved")); - recording.sample_file_uuid = uuids.back(); - recording.sample_file_sha1.resize(20); - ASSERT_TRUE(mdb_->InsertRecording(&recording, &error_message)) - << error_message; - ASSERT_GT(recording.id, 0); - EXPECT_TRUE(mdb_->ListReservedSampleFiles(&reserved, &error_message)) - << error_message; - EXPECT_THAT(reserved, testing::ElementsAre(uuids[0])); - - // Queries should return the correct result (with caches updated on insert). - ListOldestSampleFilesRow oldest; - ExpectSingleRecording(camera_uuid, recording, entry, &oldest); - - // Queries on a fresh database should return the correct result (with caches - // populated from existing database contents). - mdb_.reset(new MoonfireDatabase); - ASSERT_TRUE(mdb_->Init(&db_, &error_message)) << error_message; - ExpectSingleRecording(camera_uuid, recording, entry, &oldest); - - // Deleting a recording should succeed, update the min/max times, and mark - // the uuid as reserved. - std::vector to_delete; - to_delete.push_back(oldest); - ASSERT_TRUE(mdb_->DeleteRecordings(to_delete, &error_message)) - << error_message; - EXPECT_TRUE(mdb_->ListReservedSampleFiles(&reserved, &error_message)) - << error_message; - EXPECT_THAT(reserved, testing::UnorderedElementsAre(uuids[0], uuids[1])); - ExpectNoRecordings(camera_uuid); - - EXPECT_TRUE(mdb_->MarkSampleFilesDeleted(uuids, &error_message)) - << error_message; - EXPECT_TRUE(mdb_->ListReservedSampleFiles(&reserved, &error_message)) - << error_message; - EXPECT_THAT(reserved, testing::IsEmpty()); -} - -} // namespace -} // namespace moonfire_nvr - -int main(int argc, char **argv) { - FLAGS_alsologtostderr = true; - google::ParseCommandLineFlags(&argc, &argv, true); - testing::InitGoogleTest(&argc, argv); - google::InitGoogleLogging(argv[0]); - - // The calendar day math assumes this timezone. - CHECK_EQ(0, setenv("TZ", "America/Los_Angeles", 1)) << strerror(errno); - tzset(); - - return RUN_ALL_TESTS(); -} diff --git a/src/moonfire-db.cc b/src/moonfire-db.cc deleted file mode 100644 index a2e0663..0000000 --- a/src/moonfire-db.cc +++ /dev/null @@ -1,954 +0,0 @@ -// This file is part of Moonfire NVR, a security camera network video recorder. -// Copyright (C) 2016 Scott Lamb -// -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// In addition, as a special exception, the copyright holders give -// permission to link the code of portions of this program with the -// OpenSSL library under certain conditions as described in each -// individual source file, and distribute linked combinations including -// the two. -// -// You must obey the GNU General Public License in all respects for all -// of the code used other than OpenSSL. If you modify file(s) with this -// exception, you may extend this exception to your version of the -// file(s), but you are not obligated to do so. If you do not wish to do -// so, delete this exception statement from your version. If you delete -// this exception statement from all source files in the program, then -// also delete it here. -// -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. -// -// You should have received a copy of the GNU General Public License -// along with this program. If not, see . -// -// moonfire-db.cc: implementation of moonfire-db.h interface. -// see top-level comments there on performance & efficiency. - -#include "moonfire-db.h" - -#include - -#include - -#include - -#include "http.h" -#include "mp4.h" -#include "recording.h" - -namespace moonfire_nvr { - -namespace { - -const char kDayFmt[] = "%Y-%m-%d"; -constexpr size_t kDayFmtBufSize = sizeof("YYYY-mm-DD"); - -// Helper for AdjustDaysMap. -void AdjustDay(const std::string &day, int64_t delta, - std::map *days) { - auto it = days->find(day); - if (it != days->end()) { - it->second += delta; - DCHECK_GE(it->second, 0) << day << ", " << delta; - if (it->second == 0) { - days->erase(it); - } - } else { - days->insert(it, std::make_pair(day, delta)); - } -} - -} // namespace - -namespace internal { - -void AdjustDaysMap(int64_t start_time_90k, int64_t end_time_90k, int sign, - std::map *days) { - // There will always be at most two days adjusted, because - // kMaxRecordingDuration is less than a day (even during spring forward). - DCHECK_LE(start_time_90k, end_time_90k); - DCHECK_LE(end_time_90k - start_time_90k, kMaxRecordingDuration); - static_assert(kMaxRecordingDuration <= 23 * 60 * kTimeUnitsPerSecond, - "max duration should be less than a (spring-forward) day"); - if (start_time_90k == end_time_90k) { - return; - } - - // Fill |buf| with the first day. - struct tm mytm; - memset(&mytm, 0, sizeof(mytm)); - time_t start_ts = start_time_90k / kTimeUnitsPerSecond; - localtime_r(&start_ts, &mytm); - char buf[kDayFmtBufSize]; - strftime(buf, sizeof(buf), kDayFmt, &mytm); - - // Determine the start of the next day. - // Note that mktime(3) normalizes tm_mday, so this should work on the last - // day of the month/year. - mytm.tm_isdst = -1; - mytm.tm_hour = 0; - mytm.tm_min = 0; - mytm.tm_sec = 0; - ++mytm.tm_mday; - auto boundary_90k = kTimeUnitsPerSecond * static_cast(mktime(&mytm)); - - // Adjust the first day. - auto first_day_delta = - sign * (std::min(end_time_90k, boundary_90k) - start_time_90k); - DCHECK_NE(first_day_delta, 0) << "start=" << start_time_90k - << ", end=" << end_time_90k; - AdjustDay(buf, first_day_delta, days); - - if (end_time_90k <= boundary_90k) { - return; // no second day. - } - - // Fill |buf| with the second day. - strftime(buf, sizeof(buf), kDayFmt, &mytm); - - // Adjust the second day. - auto second_day_delta = sign * (end_time_90k - boundary_90k); - DCHECK_NE(second_day_delta, 0) << "start=" << start_time_90k - << ", end=" << end_time_90k; - AdjustDay(buf, second_day_delta, days); -} - -} // namespace internal - -bool GetDayBounds(const std::string &day, int64_t *start_time_90k, - int64_t *end_time_90k, std::string *error_message) { - struct tm mytm; - memset(&mytm, 0, sizeof(mytm)); - if (strptime(day.c_str(), kDayFmt, &mytm) != day.c_str() + day.size()) { - *error_message = StrCat("Unparseable day: ", day); - return false; - } - - mytm.tm_isdst = -1; - mytm.tm_hour = 0; - mytm.tm_min = 0; - mytm.tm_sec = 0; - *start_time_90k = kTimeUnitsPerSecond * static_cast(mktime(&mytm)); - - mytm.tm_isdst = -1; - mytm.tm_hour = 0; - mytm.tm_min = 0; - mytm.tm_sec = 0; - ++mytm.tm_mday; - *end_time_90k = kTimeUnitsPerSecond * static_cast(mktime(&mytm)); - return true; -} - -bool MoonfireDatabase::Init(Database *db, std::string *error_message) { - CHECK(db_ == nullptr); - db_ = db; - - { - DatabaseContext ctx(db_); - - auto list_cameras_run = ctx.UseOnce( - R"( - select - camera.id, - camera.uuid, - camera.short_name, - camera.description, - camera.host, - camera.username, - camera.password, - camera.main_rtsp_path, - camera.sub_rtsp_path, - camera.retain_bytes - from - camera; - )"); - while (list_cameras_run.Step() == SQLITE_ROW) { - CameraData data; - data.id = list_cameras_run.ColumnInt64(0); - Uuid uuid; - if (!uuid.ParseBinary(list_cameras_run.ColumnBlob(1))) { - *error_message = - StrCat("bad uuid ", ToHex(list_cameras_run.ColumnBlob(1)), - " for camera id ", data.id); - return false; - } - data.short_name = list_cameras_run.ColumnText(2).as_string(); - data.description = list_cameras_run.ColumnText(3).as_string(); - data.host = list_cameras_run.ColumnText(4).as_string(); - data.username = list_cameras_run.ColumnText(5).as_string(); - data.password = list_cameras_run.ColumnText(6).as_string(); - data.main_rtsp_path = list_cameras_run.ColumnText(7).as_string(); - data.sub_rtsp_path = list_cameras_run.ColumnText(8).as_string(); - data.retain_bytes = list_cameras_run.ColumnInt64(9); - - auto ret = cameras_by_uuid_.insert(std::make_pair(uuid, data)); - if (!ret.second) { - *error_message = StrCat("Duplicate camera uuid ", uuid.UnparseText()); - return false; - } - CameraData *data_p = &ret.first->second; - if (!cameras_by_id_.insert(std::make_pair(data.id, data_p)).second) { - *error_message = StrCat("Duplicate camera id ", data.id); - return false; - } - } - if (list_cameras_run.status() != SQLITE_DONE) { - *error_message = StrCat("Camera list query failed: ", - list_cameras_run.error_message()); - return false; - } - - // This query scans the entirety of the recording table's index. - // It is quite slow, so the results are cached. - auto list_recordings_run = ctx.UseOnce( - R"( - select - recording.start_time_90k, - recording.duration_90k, - recording.sample_file_bytes, - recording.camera_id - from - recording - )"); - while (list_recordings_run.Step() == SQLITE_ROW) { - int64_t start_time_90k = list_recordings_run.ColumnInt64(0); - int64_t duration_90k = list_recordings_run.ColumnInt64(1); - int64_t end_time_90k = start_time_90k + duration_90k; - int64_t sample_file_bytes = list_recordings_run.ColumnInt64(2); - int64_t camera_id = list_recordings_run.ColumnInt64(3); - auto it = cameras_by_id_.find(camera_id); - if (it == cameras_by_id_.end()) { - *error_message = - StrCat("Recording refers to unknown camera ", camera_id); - return false; - } - CameraData *data = it->second; - data->min_start_time_90k = - std::min(data->min_start_time_90k, start_time_90k); - data->max_end_time_90k = std::max(data->max_end_time_90k, end_time_90k); - data->total_sample_file_bytes += sample_file_bytes; - data->total_duration_90k += duration_90k; - internal::AdjustDaysMap(start_time_90k, end_time_90k, 1, &data->days); - } - if (list_cameras_run.status() != SQLITE_DONE) { - *error_message = StrCat("Recording list query failed: ", - list_recordings_run.error_message()); - return false; - } - - // It's simplest to just keep the video sample entries in RAM. - auto video_sample_entries_run = ctx.UseOnce( - R"( - select - id, - sha1, - width, - height, - data - from - video_sample_entry - )"); - while (video_sample_entries_run.Step() == SQLITE_ROW) { - VideoSampleEntry entry; - entry.id = video_sample_entries_run.ColumnInt64(0); - entry.sha1 = video_sample_entries_run.ColumnBlob(1).as_string(); - int64_t width_tmp = video_sample_entries_run.ColumnInt64(2); - int64_t height_tmp = video_sample_entries_run.ColumnInt64(3); - auto max = std::numeric_limits::max(); - if (width_tmp <= 0 || width_tmp > max || height_tmp <= 0 || - height_tmp > max) { - *error_message = - StrCat("video_sample_entry id ", entry.id, " width ", width_tmp, - " / height ", height_tmp, " out of range."); - return false; - } - entry.width = width_tmp; - entry.height = height_tmp; - entry.data = video_sample_entries_run.ColumnBlob(4).as_string(); - CHECK( - video_sample_entries_.insert(std::make_pair(entry.id, entry)).second) - << "duplicate: " << entry.id; - } - } - - std::string list_camera_recordings_sql = StrCat( - R"( - select - recording.start_time_90k, - recording.duration_90k, - recording.video_samples, - recording.sample_file_bytes, - recording.video_sample_entry_id - from - recording - where - camera_id = :camera_id and - recording.start_time_90k > :start_time_90k - )", - kMaxRecordingDuration, " and\n", - R"( - recording.start_time_90k < :end_time_90k and - recording.start_time_90k + recording.duration_90k > :start_time_90k - order by - recording.start_time_90k desc;)"); - list_camera_recordings_stmt_ = - db_->Prepare(list_camera_recordings_sql, nullptr, error_message); - if (!list_camera_recordings_stmt_.valid()) { - return false; - } - - std::string build_mp4_sql = StrCat( - R"( - select - recording.id, - recording.start_time_90k, - recording.duration_90k, - recording.sample_file_bytes, - recording.sample_file_uuid, - recording.sample_file_sha1, - recording.video_index, - recording.video_samples, - recording.video_sync_samples, - recording.video_sample_entry_id - from - recording - where - camera_id = :camera_id and - recording.start_time_90k > :start_time_90k - )", - kMaxRecordingDuration, " and\n", - R"( - recording.start_time_90k < :end_time_90k and - recording.start_time_90k + recording.duration_90k > :start_time_90k - order by - recording.start_time_90k;)"); - build_mp4_stmt_ = db_->Prepare(build_mp4_sql, nullptr, error_message); - if (!build_mp4_stmt_.valid()) { - return false; - } - - insert_reservation_stmt_ = db_->Prepare( - "insert into reserved_sample_files (uuid, state)\n" - " values (:uuid, :state);", - nullptr, error_message); - if (!insert_reservation_stmt_.valid()) { - return false; - } - - delete_reservation_stmt_ = - db_->Prepare("delete from reserved_sample_files where uuid = :uuid;", - nullptr, error_message); - if (!delete_reservation_stmt_.valid()) { - return false; - } - - insert_video_sample_entry_stmt_ = db_->Prepare( - R"( - insert into video_sample_entry (sha1, width, height, data) - values (:sha1, :width, :height, :data); - )", - nullptr, error_message); - if (!insert_video_sample_entry_stmt_.valid()) { - return false; - } - - insert_recording_stmt_ = db_->Prepare( - R"( - insert into recording (camera_id, sample_file_bytes, start_time_90k, - duration_90k, local_time_delta_90k, video_samples, - video_sync_samples, video_sample_entry_id, - sample_file_uuid, sample_file_sha1, video_index) - values (:camera_id, :sample_file_bytes, :start_time_90k, - :duration_90k, :local_time_delta_90k, - :video_samples, :video_sync_samples, - :video_sample_entry_id, :sample_file_uuid, - :sample_file_sha1, :video_index); - )", - nullptr, error_message); - if (!insert_recording_stmt_.valid()) { - return false; - } - - list_oldest_sample_files_stmt_ = db_->Prepare( - R"( - select - id, - sample_file_uuid, - start_time_90k, - duration_90k, - sample_file_bytes - from - recording - where - camera_id = :camera_id - order by - start_time_90k - )", - nullptr, error_message); - if (!list_oldest_sample_files_stmt_.valid()) { - return false; - } - - delete_recording_stmt_ = - db_->Prepare("delete from recording where id = :recording_id;", nullptr, - error_message); - if (!delete_recording_stmt_.valid()) { - return false; - } - - camera_min_start_stmt_ = db_->Prepare( - R"( - select - start_time_90k - from - recording - where - camera_id = :camera_id - order by start_time_90k limit 1; - )", - nullptr, error_message); - if (!camera_min_start_stmt_.valid()) { - return false; - } - - camera_max_start_stmt_ = db_->Prepare( - R"( - select - start_time_90k, - duration_90k - from - recording - where - camera_id = :camera_id - order by start_time_90k desc; - )", - nullptr, error_message); - if (!camera_max_start_stmt_.valid()) { - return false; - } - - return true; -} - -void MoonfireDatabase::ListCameras( - std::function cb) { - DatabaseContext ctx(db_); - ListCamerasRow row; - for (const auto &entry : cameras_by_uuid_) { - row.id = entry.second.id; - row.uuid = entry.first; - row.short_name = entry.second.short_name; - row.description = entry.second.description; - row.host = entry.second.host; - row.username = entry.second.username; - row.password = entry.second.password; - row.main_rtsp_path = entry.second.main_rtsp_path; - row.sub_rtsp_path = entry.second.sub_rtsp_path; - row.retain_bytes = entry.second.retain_bytes; - row.min_start_time_90k = entry.second.min_start_time_90k; - row.max_end_time_90k = entry.second.max_end_time_90k; - row.total_duration_90k = entry.second.total_duration_90k; - row.total_sample_file_bytes = entry.second.total_sample_file_bytes; - if (cb(row) == IterationControl::kBreak) { - return; - } - } - return; -} - -bool MoonfireDatabase::GetCamera(Uuid camera_uuid, GetCameraRow *row) { - DatabaseContext ctx(db_); - const auto it = cameras_by_uuid_.find(camera_uuid); - if (it == cameras_by_uuid_.end()) { - return false; - } - const CameraData &data = it->second; - row->short_name = data.short_name; - row->description = data.description; - row->retain_bytes = data.retain_bytes; - row->min_start_time_90k = data.min_start_time_90k; - row->max_end_time_90k = data.max_end_time_90k; - row->total_duration_90k = data.total_duration_90k; - row->total_sample_file_bytes = data.total_sample_file_bytes; - row->days = data.days; - return true; -} - -bool MoonfireDatabase::ListCameraRecordings( - Uuid camera_uuid, int64_t start_time_90k, int64_t end_time_90k, - std::function cb, - std::string *error_message) { - DatabaseContext ctx(db_); - const auto camera_it = cameras_by_uuid_.find(camera_uuid); - if (camera_it == cameras_by_uuid_.end()) { - *error_message = StrCat("no such camera ", camera_uuid.UnparseText()); - return false; - } - auto run = ctx.Borrow(&list_camera_recordings_stmt_); - run.BindInt64(":camera_id", camera_it->second.id); - run.BindInt64(":start_time_90k", start_time_90k); - run.BindInt64(":end_time_90k", end_time_90k); - ListCameraRecordingsRow row; - while (run.Step() == SQLITE_ROW) { - row.start_time_90k = run.ColumnInt64(0); - row.end_time_90k = row.start_time_90k + run.ColumnInt64(1); - row.video_samples = run.ColumnInt64(2); - row.sample_file_bytes = run.ColumnInt64(3); - int64_t video_sample_entry_id = run.ColumnInt64(4); - const auto it = video_sample_entries_.find(video_sample_entry_id); - if (it == video_sample_entries_.end()) { - *error_message = - StrCat("recording references invalid video sample entry ", - video_sample_entry_id); - return false; - } - const VideoSampleEntry &entry = it->second; - row.video_sample_entry_sha1 = entry.sha1; - row.width = entry.width; - row.height = entry.height; - if (cb(row) == IterationControl::kBreak) { - return true; - } - } - if (run.status() != SQLITE_DONE) { - *error_message = StrCat("sqlite query failed: ", run.error_message()); - return false; - } - return true; -} - -bool MoonfireDatabase::ListMp4Recordings( - Uuid camera_uuid, int64_t start_time_90k, int64_t end_time_90k, - std::function - row_cb, - std::string *error_message) { - VLOG(1) << "...(1/4): Waiting for database lock"; - DatabaseContext ctx(db_); - const auto it = cameras_by_uuid_.find(camera_uuid); - if (it == cameras_by_uuid_.end()) { - *error_message = StrCat("no such camera ", camera_uuid.UnparseText()); - return false; - } - const CameraData &data = it->second; - VLOG(1) << "...(2/4): Querying database"; - auto run = ctx.Borrow(&build_mp4_stmt_); - run.BindInt64(":camera_id", data.id); - run.BindInt64(":end_time_90k", end_time_90k); - run.BindInt64(":start_time_90k", start_time_90k); - Recording recording; - VideoSampleEntry sample_entry; - while (run.Step() == SQLITE_ROW) { - recording.id = run.ColumnInt64(0); - recording.camera_id = data.id; - recording.start_time_90k = run.ColumnInt64(1); - recording.end_time_90k = recording.start_time_90k + run.ColumnInt64(2); - recording.sample_file_bytes = run.ColumnInt64(3); - if (!recording.sample_file_uuid.ParseBinary(run.ColumnBlob(4))) { - *error_message = - StrCat("recording ", recording.id, " has unparseable uuid ", - ToHex(run.ColumnBlob(4))); - return false; - } - recording.sample_file_sha1 = run.ColumnBlob(5).as_string(); - recording.video_index = run.ColumnBlob(6).as_string(); - recording.video_samples = run.ColumnInt64(7); - recording.video_sync_samples = run.ColumnInt64(8); - recording.video_sample_entry_id = run.ColumnInt64(9); - - auto it = video_sample_entries_.find(recording.video_sample_entry_id); - if (it == video_sample_entries_.end()) { - *error_message = StrCat("recording ", recording.id, - " references unknown video sample entry ", - recording.video_sample_entry_id); - return false; - } - const VideoSampleEntry &entry = it->second; - - if (row_cb(recording, entry) == IterationControl::kBreak) { - return true; - } - } - if (run.status() != SQLITE_DONE && run.status() != SQLITE_ROW) { - *error_message = StrCat("sqlite query failed: ", run.error_message()); - return false; - } - return true; -} - -bool MoonfireDatabase::ListReservedSampleFiles(std::vector *reserved, - std::string *error_message) { - reserved->clear(); - DatabaseContext ctx(db_); - auto run = ctx.UseOnce("select uuid from reserved_sample_files;"); - while (run.Step() == SQLITE_ROW) { - Uuid uuid; - if (!uuid.ParseBinary(run.ColumnBlob(0))) { - *error_message = StrCat("unparseable uuid ", ToHex(run.ColumnBlob(0))); - return false; - } - reserved->push_back(uuid); - } - if (run.status() != SQLITE_DONE) { - *error_message = run.error_message(); - return false; - } - return true; -} - -std::vector MoonfireDatabase::ReserveSampleFiles( - int n, std::string *error_message) { - if (n == 0) { - return std::vector(); - } - std::vector uuids; - uuids.reserve(n); - for (int i = 0; i < n; ++i) { - uuids.push_back(uuidgen_->Generate()); - } - DatabaseContext ctx(db_); - if (!ctx.BeginTransaction(error_message)) { - return std::vector(); - } - for (const auto &uuid : uuids) { - auto run = ctx.Borrow(&insert_reservation_stmt_); - run.BindBlob(":uuid", uuid.binary_view()); - run.BindInt64(":state", static_cast(ReservationState::kWriting)); - if (run.Step() != SQLITE_DONE) { - ctx.RollbackTransaction(); - *error_message = run.error_message(); - return std::vector(); - } - } - if (!ctx.CommitTransaction(error_message)) { - return std::vector(); - } - return uuids; -} - -bool MoonfireDatabase::InsertVideoSampleEntry(VideoSampleEntry *entry, - std::string *error_message) { - if (entry->id != -1) { - *error_message = StrCat("video_sample_entry already has id ", entry->id); - return false; - } - DatabaseContext ctx(db_); - for (const auto &some_entry : video_sample_entries_) { - if (some_entry.second.sha1 == entry->sha1) { - if (entry->width != some_entry.second.width || - entry->height != some_entry.second.height) { - *error_message = - StrCat("inconsistent entry for sha1 ", ToHex(entry->sha1), - ": existing entry has ", some_entry.second.width, "x", - some_entry.second.height, ", new entry has ", entry->width, - "x", entry->height); - return false; - } - entry->id = some_entry.first; - return true; - } - } - auto insert_run = ctx.Borrow(&insert_video_sample_entry_stmt_); - insert_run.BindBlob(":sha1", entry->sha1); - insert_run.BindInt64(":width", entry->width); - insert_run.BindInt64(":height", entry->height); - insert_run.BindBlob(":data", entry->data); - if (insert_run.Step() != SQLITE_DONE) { - *error_message = - StrCat("insert video sample entry: ", insert_run.error_message(), - ": sha1=", ToHex(entry->sha1), ", dimensions=", entry->width, - "x", entry->height, ", data=", ToHex(entry->data)); - return false; - } - entry->id = ctx.last_insert_rowid(); - CHECK(video_sample_entries_.insert(std::make_pair(entry->id, *entry)).second) - << "duplicate: " << entry->id; - return true; -} - -bool MoonfireDatabase::InsertRecording(Recording *recording, - std::string *error_message) { - if (recording->id != -1) { - *error_message = StrCat("recording already has id ", recording->id); - return false; - } - if (recording->end_time_90k < recording->start_time_90k) { - *error_message = - StrCat("end time ", recording->end_time_90k, " must be >= start time ", - recording->start_time_90k); - return false; - } - DatabaseContext ctx(db_); - auto it = cameras_by_id_.find(recording->camera_id); - if (it == cameras_by_id_.end()) { - *error_message = StrCat("no camera with id ", recording->camera_id); - return false; - } - CameraData *camera_data = it->second; - if (!ctx.BeginTransaction(error_message)) { - return false; - } - auto delete_run = ctx.Borrow(&delete_reservation_stmt_); - delete_run.BindBlob(":uuid", recording->sample_file_uuid.binary_view()); - if (delete_run.Step() != SQLITE_DONE) { - *error_message = delete_run.error_message(); - ctx.RollbackTransaction(); - return false; - } - if (ctx.changes() != 1) { - *error_message = StrCat("uuid ", recording->sample_file_uuid.UnparseText(), - " is not reserved"); - ctx.RollbackTransaction(); - return false; - } - auto insert_run = ctx.Borrow(&insert_recording_stmt_); - insert_run.BindInt64(":camera_id", recording->camera_id); - insert_run.BindInt64(":sample_file_bytes", recording->sample_file_bytes); - insert_run.BindInt64(":start_time_90k", recording->start_time_90k); - insert_run.BindInt64(":duration_90k", - recording->end_time_90k - recording->start_time_90k); - insert_run.BindInt64(":local_time_delta_90k", - recording->local_time_90k - recording->start_time_90k); - insert_run.BindInt64(":video_samples", recording->video_samples); - insert_run.BindInt64(":video_sync_samples", recording->video_sync_samples); - insert_run.BindInt64(":video_sample_entry_id", - recording->video_sample_entry_id); - insert_run.BindBlob(":sample_file_uuid", - recording->sample_file_uuid.binary_view()); - insert_run.BindBlob(":sample_file_sha1", recording->sample_file_sha1); - insert_run.BindBlob(":video_index", recording->video_index); - if (insert_run.Step() != SQLITE_DONE) { - *error_message = - StrCat("insert failed: ", insert_run.error_message(), ", camera_id=", - recording->camera_id, ", sample_file_bytes=", - recording->sample_file_bytes, ", start_time_90k=", - recording->start_time_90k, ", duration_90k=", - recording->end_time_90k - recording->start_time_90k, - ", local_time_delta_90k=", - recording->local_time_90k - recording->start_time_90k, - ", video_samples=", recording->video_samples, - ", video_sync_samples=", recording->video_sync_samples, - ", video_sample_entry_id=", recording->video_sample_entry_id, - ", sample_file_uuid=", recording->sample_file_uuid.UnparseText(), - ", sample_file_sha1=", ToHex(recording->sample_file_sha1), - ", video_index length ", recording->video_index.size()); - ctx.RollbackTransaction(); - return false; - } - if (!ctx.CommitTransaction(error_message)) { - LOG(ERROR) << "commit failed"; - return false; - } - recording->id = ctx.last_insert_rowid(); - if (camera_data->min_start_time_90k == -1 || - camera_data->min_start_time_90k > recording->start_time_90k) { - camera_data->min_start_time_90k = recording->start_time_90k; - } - if (camera_data->max_end_time_90k == -1 || - camera_data->max_end_time_90k < recording->end_time_90k) { - camera_data->max_end_time_90k = recording->end_time_90k; - } - camera_data->total_duration_90k += - recording->end_time_90k - recording->start_time_90k; - camera_data->total_sample_file_bytes += recording->sample_file_bytes; - internal::AdjustDaysMap(recording->start_time_90k, recording->end_time_90k, 1, - &camera_data->days); - return true; -} - -bool MoonfireDatabase::ListOldestSampleFiles( - Uuid camera_uuid, - std::function row_cb, - std::string *error_message) { - DatabaseContext ctx(db_); - auto it = cameras_by_uuid_.find(camera_uuid); - if (it == cameras_by_uuid_.end()) { - *error_message = StrCat("no such camera ", camera_uuid.UnparseText()); - return false; - } - const CameraData &camera_data = it->second; - auto run = ctx.Borrow(&list_oldest_sample_files_stmt_); - run.BindInt64(":camera_id", camera_data.id); - ListOldestSampleFilesRow row; - while (run.Step() == SQLITE_ROW) { - row.camera_id = camera_data.id; - row.recording_id = run.ColumnInt64(0); - if (!row.sample_file_uuid.ParseBinary(run.ColumnBlob(1))) { - *error_message = - StrCat("recording ", row.recording_id, " has unparseable uuid ", - ToHex(run.ColumnBlob(1))); - return false; - } - row.start_time_90k = run.ColumnInt64(2); - row.duration_90k = run.ColumnInt64(3); - row.sample_file_bytes = run.ColumnInt64(4); - if (row_cb(row) == IterationControl::kBreak) { - return true; - } - } - if (run.status() != SQLITE_DONE) { - *error_message = run.error_message(); - return false; - } - return true; -} - -bool MoonfireDatabase::DeleteRecordings( - const std::vector &recordings, - std::string *error_message) { - if (recordings.empty()) { - return true; - } - - DatabaseContext ctx(db_); - if (!ctx.BeginTransaction(error_message)) { - return false; - } - struct State { - int64_t deleted_duration_90k = 0; - int64_t deleted_sample_file_bytes = 0; - int64_t min_start_time_90k = -1; - int64_t max_end_time_90k = -1; - std::map days; - CameraData *camera_data = nullptr; - }; - std::map state_by_camera_id; - for (const auto &recording : recordings) { - State &state = state_by_camera_id[recording.camera_id]; - state.deleted_duration_90k += recording.duration_90k; - state.deleted_sample_file_bytes += recording.sample_file_bytes; - internal::AdjustDaysMap(recording.start_time_90k, - recording.start_time_90k + recording.duration_90k, - 1, &state.days); - - auto delete_run = ctx.Borrow(&delete_recording_stmt_); - delete_run.BindInt64(":recording_id", recording.recording_id); - if (delete_run.Step() != SQLITE_DONE) { - ctx.RollbackTransaction(); - *error_message = StrCat("delete: ", delete_run.error_message()); - return false; - } - if (ctx.changes() != 1) { - ctx.RollbackTransaction(); - *error_message = StrCat("no such recording ", recording.recording_id); - return false; - } - - auto insert_run = ctx.Borrow(&insert_reservation_stmt_); - insert_run.BindBlob(":uuid", recording.sample_file_uuid.binary_view()); - insert_run.BindInt64(":state", - static_cast(ReservationState::kDeleting)); - if (insert_run.Step() != SQLITE_DONE) { - ctx.RollbackTransaction(); - *error_message = StrCat("insert: ", insert_run.error_message()); - return false; - } - } - - // Recompute start and end times for each camera. - for (auto &state_entry : state_by_camera_id) { - int64_t camera_id = state_entry.first; - State &state = state_entry.second; - auto it = cameras_by_id_.find(camera_id); - if (it == cameras_by_id_.end()) { - *error_message = - StrCat("internal error; can't find camera id ", camera_id); - return false; - } - state.camera_data = it->second; - - // The minimum is straightforward, taking advantage of the start_time_90k - // index for speed. - auto min_run = ctx.Borrow(&camera_min_start_stmt_); - min_run.BindInt64(":camera_id", camera_id); - if (min_run.Step() == SQLITE_ROW) { - state.min_start_time_90k = min_run.ColumnInt64(0); - } else if (min_run.Step() == SQLITE_DONE) { - // There are no recordings left. - state.min_start_time_90k = std::numeric_limits::max(); - state.max_end_time_90k = std::numeric_limits::min(); - continue; // skip additional query below to calculate max. - } else { - ctx.RollbackTransaction(); - *error_message = StrCat("min: ", min_run.error_message()); - return false; - } - - // The maximum is less straightforward in the case of overlap - all - // recordings starting in the last kMaxRecordingDuration must be examined - // to take advantage of the start_time_90k index. - auto max_run = ctx.Borrow(&camera_max_start_stmt_); - max_run.BindInt64(":camera_id", camera_id); - if (max_run.Step() != SQLITE_ROW) { - // If there was a min row, there should be a max row too, so this is an - // error even in the SQLITE_DONE case. - ctx.RollbackTransaction(); - *error_message = StrCat("max[0]: ", max_run.error_message()); - return false; - } - int64_t max_start_90k = max_run.ColumnInt64(0); - do { - auto end_time_90k = max_run.ColumnInt64(0) + max_run.ColumnInt64(1); - state.max_end_time_90k = std::max(state.max_end_time_90k, end_time_90k); - } while (max_run.Step() == SQLITE_ROW && - max_run.ColumnInt64(0) > max_start_90k - kMaxRecordingDuration); - if (max_run.status() != SQLITE_DONE && max_run.status() != SQLITE_ROW) { - *error_message = StrCat("max[1]: ", max_run.error_message()); - ctx.RollbackTransaction(); - return false; - } - } - - if (!ctx.CommitTransaction(error_message)) { - *error_message = StrCat("commit: ", *error_message); - return false; - } - - for (auto &state_entry : state_by_camera_id) { - State &state = state_entry.second; - state.camera_data->total_duration_90k -= state.deleted_duration_90k; - state.camera_data->total_sample_file_bytes -= - state.deleted_sample_file_bytes; - state.camera_data->min_start_time_90k = state.min_start_time_90k; - state.camera_data->max_end_time_90k = state.max_end_time_90k; - for (const auto &day : state.days) { - AdjustDay(day.first, -day.second, &state.camera_data->days); - } - } - return true; -} - -bool MoonfireDatabase::MarkSampleFilesDeleted(const std::vector &uuids, - std::string *error_message) { - if (uuids.empty()) { - return true; - } - DatabaseContext ctx(db_); - if (!ctx.BeginTransaction(error_message)) { - return false; - } - for (const auto &uuid : uuids) { - auto run = ctx.Borrow(&delete_reservation_stmt_); - run.BindBlob(":uuid", uuid.binary_view()); - if (run.Step() != SQLITE_DONE) { - *error_message = run.error_message(); - ctx.RollbackTransaction(); - return false; - } - if (ctx.changes() != 1) { - *error_message = StrCat("no reservation for uuid ", uuid.UnparseText()); - ctx.RollbackTransaction(); - return false; - } - } - if (!ctx.CommitTransaction(error_message)) { - return false; - } - return true; -} - -} // namespace moonfire_nvr diff --git a/src/moonfire-db.h b/src/moonfire-db.h deleted file mode 100644 index 11d575e..0000000 --- a/src/moonfire-db.h +++ /dev/null @@ -1,279 +0,0 @@ -// This file is part of Moonfire NVR, a security camera network video recorder. -// Copyright (C) 2016 Scott Lamb -// -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// In addition, as a special exception, the copyright holders give -// permission to link the code of portions of this program with the -// OpenSSL library under certain conditions as described in each -// individual source file, and distribute linked combinations including -// the two. -// -// You must obey the GNU General Public License in all respects for all -// of the code used other than OpenSSL. If you modify file(s) with this -// exception, you may extend this exception to your version of the -// file(s), but you are not obligated to do so. If you do not wish to do -// so, delete this exception statement from your version. If you delete -// this exception statement from all source files in the program, then -// also delete it here. -// -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. -// -// You should have received a copy of the GNU General Public License -// along with this program. If not, see . -// -// moonfire-db.h: database access logic for the Moonfire NVR SQLite schema. -// Currently focused on stuff needed by WebInterface to build a HTML or JSON -// interface. -// -// This caches data in RAM, making the assumption that only one process is -// accessing the database at a time. (TODO: enforce with flock or some such.) -// Performance and efficiency notes: -// -// * several query operations here feature row callbacks. The callback is -// invoked with the database lock. Thus, the caller mustn't perform database -// operations or other long-running operations. -// -// * startup may be slow, as it scans the entire index for the recording -// table. This seems acceptable. -// -// * the operations used for web file serving should return results with -// acceptable latency. -// -// * however, the database lock may be held for longer than is acceptable for -// the critical path of recording frames. It may be necessary to preallocate -// sample file uuids and such to avoid this. -// -// * the caller may need to perform several different types of write -// operations in a row. It might be worth creating an interface for batching -// these inside a transaction, to reduce latency and SSD write cycles. The -// pre-commit and post-commit logic of each operation would have to be -// pulled apart, with the latter being called by this wrapper class on -// commit of the overall transaction. - -#ifndef MOONFIRE_NVR_MOONFIRE_DB_H -#define MOONFIRE_NVR_MOONFIRE_DB_H - -#include -#include -#include -#include -#include - -#include "common.h" -#include "http.h" -#include "mp4.h" -#include "sqlite.h" -#include "uuid.h" - -namespace moonfire_nvr { - -// For use with MoonfireDatabase::ListCameras. -struct ListCamerasRow { - int64_t id = -1; - Uuid uuid; - std::string short_name; - std::string description; - std::string host; - std::string username; - std::string password; - std::string main_rtsp_path; - std::string sub_rtsp_path; - int64_t retain_bytes = -1; - - // Aggregates summarizing completed recordings. - int64_t min_start_time_90k = -1; - int64_t max_end_time_90k = -1; - int64_t total_duration_90k = -1; - int64_t total_sample_file_bytes = -1; -}; - -// For use with MoonfireDatabase::GetCamera. -// This includes everything in ListCamerasRow. In the future, it will include -// more data. Likely, that will mean a list of calendar days (in the system -// time zone) in which there is any data. -struct GetCameraRow { - std::string short_name; - std::string description; - int64_t retain_bytes = -1; - int64_t min_start_time_90k = -1; - int64_t max_end_time_90k = -1; - int64_t total_duration_90k = -1; - int64_t total_sample_file_bytes = -1; - std::map days; // YYYY-mm-dd -> duration_90k. -}; - -// For use with MoonfireDatabase::ListCameraRecordings. -struct ListCameraRecordingsRow { - // From the recording table. - int64_t start_time_90k = -1; - int64_t end_time_90k = -1; - int64_t video_samples = -1; - int64_t sample_file_bytes = -1; - - // Joined from the video_sample_entry table. - // |video_sample_entry_sha1| is valid as long as the MoonfireDatabase. - re2::StringPiece video_sample_entry_sha1; - uint16_t width = 0; - uint16_t height = 0; -}; - -// For use with MoonfireDatabase::ListOldestSampleFiles. -struct ListOldestSampleFilesRow { - int64_t camera_id = -1; - int64_t recording_id = -1; - Uuid sample_file_uuid; - int64_t start_time_90k = -1; - int64_t duration_90k = -1; - int64_t sample_file_bytes = -1; -}; - -// Thread-safe after Init. -// (Uses a DatabaseContext for locking.) -class MoonfireDatabase { - public: - MoonfireDatabase() {} - MoonfireDatabase(const MoonfireDatabase &) = delete; - void operator=(const MoonfireDatabase &) = delete; - - // |db| must outlive the MoonfireDatabase. - bool Init(Database *db, std::string *error_message); - - // List all cameras in the system, ordered by short name. - void ListCameras(std::function cb); - - // Get a single camera. - // Return true iff the camera exists. - bool GetCamera(Uuid camera_uuid, GetCameraRow *row); - - // List all recordings associated with a camera, descending by end time. - bool ListCameraRecordings( - Uuid camera_uuid, int64_t start_time_90k, int64_t end_time_90k, - std::function, - std::string *error_message); - - bool ListMp4Recordings( - Uuid camera_uuid, int64_t start_time_90k, int64_t end_time_90k, - std::function - row_cb, - std::string *error_message); - - bool ListReservedSampleFiles(std::vector *reserved, - std::string *error_message); - - // Reserve |n| new sample file uuids. - // Returns an empty vector on error. - std::vector ReserveSampleFiles(int n, std::string *error_message); - - // Insert a video sample entry if not already inserted. - // On success, |entry->id| is filled in with the id of a freshly-created or - // existing row. - bool InsertVideoSampleEntry(VideoSampleEntry *entry, - std::string *error_message); - - // Insert a new recording. - // The uuid must have been already reserved with ReserveSampleFileUuid above. - // On success, |recording->id| is filled in. - bool InsertRecording(Recording *recording, std::string *error_message); - - // List sample files, starting from the oldest. - // The caller is expected to supply a |row_cb| that returns kBreak when - // enough have been listed. - bool ListOldestSampleFiles( - Uuid camera_uuid, - std::function row_cb, - std::string *error_message); - - // Delete recording rows, moving their sample file uuids to the deleting - // state. - bool DeleteRecordings(const std::vector &rows, - std::string *error_message); - - // Mark a set of sample files as deleted. - // This shouldn't be called until the files have been unlinke()ed and the - // parent directory fsync()ed. - // Returns error if any sample files are not in the deleting state. - bool MarkSampleFilesDeleted(const std::vector &uuids, - std::string *error_message); - - // Replace the default real UUID generator with the supplied one. - // Exposed only for testing; not thread-safe. - void SetUuidGeneratorForTesting(UuidGenerator *uuidgen) { - uuidgen_ = uuidgen; - } - - private: - struct CameraData { - // Cached values of the matching fields from the camera row. - int64_t id = -1; - std::string short_name; - std::string description; - std::string host; - std::string username; - std::string password; - std::string main_rtsp_path; - std::string sub_rtsp_path; - int64_t retain_bytes = -1; - - // Aggregates of all recordings associated with the camera. - int64_t min_start_time_90k = std::numeric_limits::max(); - int64_t max_end_time_90k = std::numeric_limits::min(); - int64_t total_sample_file_bytes = 0; - int64_t total_duration_90k = 0; - - // A map of calendar days (in the local timezone, "YYYY-mm-DD") to the - // total duration (in 90k units) of recorded data in the day. A day is - // present in the map ff the value is non-zero. - std::map days; - }; - - enum class ReservationState { kWriting = 0, kDeleting = 1 }; - - // Efficiently (re-)compute the bounds of recorded time for a given camera. - bool ComputeCameraRecordingBounds(DatabaseContext *ctx, int64_t camera_id, - int64_t *min_start_time_90k, - int64_t *max_end_time_90k, - std::string *error_message); - - Database *db_ = nullptr; - UuidGenerator *uuidgen_ = GetRealUuidGenerator(); - Statement list_camera_recordings_stmt_; - Statement build_mp4_stmt_; - Statement insert_reservation_stmt_; - Statement delete_reservation_stmt_; - Statement insert_video_sample_entry_stmt_; - Statement insert_recording_stmt_; - Statement list_oldest_sample_files_stmt_; - Statement delete_recording_stmt_; - Statement camera_min_start_stmt_; - Statement camera_max_start_stmt_; - - std::map cameras_by_uuid_; - std::map cameras_by_id_; - std::map video_sample_entries_; -}; - -// Given a key in the day-to-duration map, produce the start and end times of -// the day. (Typically the end time is 24 hours later than the start; but it's -// 23 or 25 hours for the days of spring forward and fall back, respectively.) -bool GetDayBounds(const std::string &day, int64_t *start_time_90k, - int64_t *end_time_90k, std::string *error_message); - -namespace internal { - -// Adjust a day-to-duration map (see MoonfireDatabase::CameraData::days_) -// to reflect a recording. -void AdjustDaysMap(int64_t start_time_90k, int64_t end_time_90k, int sign, - std::map *days); - -} // namespace internal - -} // namespace moonfire_nvr - -#endif // MOONFIRE_NVR_MOONFIRE_DB_H diff --git a/src/moonfire-nvr-main.cc b/src/moonfire-nvr-main.cc deleted file mode 100644 index c21696a..0000000 --- a/src/moonfire-nvr-main.cc +++ /dev/null @@ -1,237 +0,0 @@ -// This file is part of Moonfire NVR, a security camera digital video recorder. -// Copyright (C) 2016 Scott Lamb -// -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// In addition, as a special exception, the copyright holders give -// permission to link the code of portions of this program with the -// OpenSSL library under certain conditions as described in each -// individual source file, and distribute linked combinations including -// the two. -// -// You must obey the GNU General Public License in all respects for all -// of the code used other than OpenSSL. If you modify file(s) with this -// exception, you may extend this exception to your version of the -// file(s), but you are not obligated to do so. If you do not wish to do -// so, delete this exception statement from your version. If you delete -// this exception statement from all source files in the program, then -// also delete it here. -// -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. -// -// You should have received a copy of the GNU General Public License -// along with this program. If not, see . -// -// moonfire-nvr-main.cc: main program. This should be kept as short as -// practical, so that individual parts of the program can be tested with the -// googletest framework. - -#include -#include -#include -#include - -#include - -#include -#include -#include -#include -#include -#include - -#include "ffmpeg.h" -#include "profiler.h" -#include "moonfire-db.h" -#include "moonfire-nvr.h" -#include "sqlite.h" -#include "string.h" -#include "web.h" - -using moonfire_nvr::StrCat; - -DEFINE_int32(http_port, 0, ""); -DEFINE_string(db_dir, "", ""); -DEFINE_string(sample_file_dir, "", ""); -DEFINE_bool(read_only, false, ""); - -namespace { - -const struct timeval kLogFlushInterval = {1, 0}; - -struct event_base* base; - -void EventLogCallback(int severity, const char* msg) { - int vlog_level = 0; - google::LogSeverity glog_level; - if (severity <= EVENT_LOG_DEBUG) { - vlog_level = 1; - glog_level = google::GLOG_INFO; - } else if (severity <= EVENT_LOG_MSG) { - glog_level = google::GLOG_INFO; - } else if (severity <= EVENT_LOG_WARN) { - glog_level = google::GLOG_WARNING; - } else { - glog_level = google::GLOG_ERROR; - } - - if (vlog_level > 0 && !VLOG_IS_ON(vlog_level)) { - return; - } - google::LogMessage("libevent", 0, glog_level).stream() << msg; -} - -// Called on SIGTERM or SIGINT. -void SignalCallback(evutil_socket_t, short, void*) { - event_base_loopexit(base, nullptr); -} - -void FlushLogsCallback(evutil_socket_t, short, void* ev) { - google::FlushLogFiles(google::GLOG_INFO); - CHECK_EQ(0, - event_add(reinterpret_cast(ev), &kLogFlushInterval)); -} - -} // namespace - -// Note that main never returns; it calls exit on either success or failure. -// This avoids the need to design an orderly shutdown for all dependencies, -// instead letting the OS clean up memory allocations en masse. State may be -// allocated in whatever way is most convenient: on the stack, in a unique_ptr -// (that may never go out of scope), or as a bare pointer that is never -// deleted. -int main(int argc, char** argv) { - google::ParseCommandLineFlags(&argc, &argv, true); - google::InitGoogleLogging(argv[0]); - google::InstallFailureSignalHandler(); - signal(SIGPIPE, SIG_IGN); - - if (FLAGS_sample_file_dir.empty()) { - LOG(ERROR) << "--sample_file_dir must be specified; exiting."; - exit(1); - } - - if (FLAGS_db_dir.empty()) { - LOG(ERROR) << "--db_dir must be specified; exiting."; - exit(1); - } - - if (FLAGS_http_port == 0) { - LOG(ERROR) << "--http_port must be specified; exiting."; - exit(1); - } - - moonfire_nvr::Environment env; - env.clock = moonfire_nvr::GetRealClock(); - env.video_source = moonfire_nvr::GetRealVideoSource(); - - std::unique_ptr sample_file_dir; - std::string sample_file_dirname = FLAGS_sample_file_dir; - int ret = moonfire_nvr::GetRealFilesystem()->Open( - sample_file_dirname.c_str(), O_DIRECTORY | O_RDONLY, &sample_file_dir); - if (ret != 0) { - LOG(ERROR) << "Unable to open --sample_file_dir=" << sample_file_dirname - << ": " << strerror(ret) << "; exiting."; - exit(1); - } - - // Separately, ensure the sample file directory is writable. - // (Opening the directory above with O_DIRECTORY|O_RDWR doesn't work even - // when the directory is writable; it fails with EISDIR.) - ret = sample_file_dir->Access(".", W_OK, 0); - if (ret != 0) { - LOG(ERROR) << "--sample_file_dir=" << sample_file_dirname - << " is not writable: " << strerror(ret) << "; exiting."; - exit(1); - } - - env.sample_file_dir = sample_file_dir.release(); - - std::unique_ptr db_dir; - std::string db_dirname = FLAGS_db_dir; - ret = moonfire_nvr::GetRealFilesystem()->Open( - db_dirname.c_str(), O_DIRECTORY | O_RDONLY, &db_dir); - if (ret != 0) { - LOG(ERROR) << "Unable to open --db_dir=" << db_dirname << ": " - << strerror(ret) << "; exiting."; - exit(1); - } - bool read_only = FLAGS_read_only; - ret = db_dir->Lock((read_only ? LOCK_SH : LOCK_EX) | LOCK_NB); - if (ret != 0) { - LOG(ERROR) << "Unable to lock --db_dir=" << db_dirname << ": " - << strerror(ret) << "; exiting."; - exit(1); - } - moonfire_nvr::Database db; - std::string error_msg; - std::string db_path = StrCat(FLAGS_db_dir, "/db"); - if (!db.Open(db_path.c_str(), - read_only ? SQLITE_OPEN_READONLY : SQLITE_OPEN_READWRITE, - &error_msg)) { - LOG(ERROR) << error_msg << "; exiting."; - exit(1); - } - - moonfire_nvr::MoonfireDatabase mdb; - CHECK(mdb.Init(&db, &error_msg)) << error_msg; - env.mdb = &mdb; - - moonfire_nvr::WebInterface web(&env); - - event_set_log_callback(&EventLogCallback); - LOG(INFO) << "libevent: compiled with version " << LIBEVENT_VERSION - << ", running with version " << event_get_version(); - base = CHECK_NOTNULL(event_base_new()); - - std::unique_ptr nvr; - if (!read_only) { - nvr.reset(new moonfire_nvr::Nvr(&env)); - if (!nvr->Init(&error_msg)) { - LOG(ERROR) << "Unable to initialize: " << error_msg << "; exiting."; - exit(1); - } - } - - evhttp* http = CHECK_NOTNULL(evhttp_new(base)); - moonfire_nvr::RegisterProfiler(base, http); - web.Register(http); - if (evhttp_bind_socket(http, "0.0.0.0", FLAGS_http_port) != 0) { - LOG(ERROR) << "Unable to bind to --http_port=" << FLAGS_http_port - << "; exiting."; - exit(1); - } - - // Register for termination signals. - struct event ev_sigterm; - struct event ev_sigint; - CHECK_EQ(0, event_assign(&ev_sigterm, base, SIGTERM, EV_SIGNAL | EV_PERSIST, - &SignalCallback, nullptr)); - CHECK_EQ(0, event_assign(&ev_sigint, base, SIGINT, EV_SIGNAL | EV_PERSIST, - &SignalCallback, nullptr)); - CHECK_EQ(0, event_add(&ev_sigterm, nullptr)); - CHECK_EQ(0, event_add(&ev_sigint, nullptr)); - - // Flush the logfiles regularly for debuggability. - struct event ev_flushlogs; - CHECK_EQ(0, event_assign(&ev_flushlogs, base, 0, 0, &FlushLogsCallback, - &ev_flushlogs)); - CHECK_EQ(0, event_add(&ev_flushlogs, &kLogFlushInterval)); - - // Wait for events. - LOG(INFO) << "Main thread entering event loop."; - CHECK_EQ(0, event_base_loop(base, 0)); - - LOG(INFO) << "Shutting down."; - google::FlushLogFiles(google::GLOG_INFO); - nvr.reset(); - LOG(INFO) << "Done."; - google::ShutdownGoogleLogging(); - exit(0); -} diff --git a/src/moonfire-nvr-test.cc b/src/moonfire-nvr-test.cc deleted file mode 100644 index 0f0a7ed..0000000 --- a/src/moonfire-nvr-test.cc +++ /dev/null @@ -1,423 +0,0 @@ -// This file is part of Moonfire NVR, a security camera digital video recorder. -// Copyright (C) 2016 Scott Lamb -// -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// In addition, as a special exception, the copyright holders give -// permission to link the code of portions of this program with the -// OpenSSL library under certain conditions as described in each -// individual source file, and distribute linked combinations including -// the two. -// -// You must obey the GNU General Public License in all respects for all -// of the code used other than OpenSSL. If you modify file(s) with this -// exception, you may extend this exception to your version of the -// file(s), but you are not obligated to do so. If you do not wish to do -// so, delete this exception statement from your version. If you delete -// this exception statement from all source files in the program, then -// also delete it here. -// -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. -// -// You should have received a copy of the GNU General Public License -// along with this program. If not, see . -// -// moonfire-nvr-test.cc: tests of the moonfire-nvr.cc interface. - -#include -#include -#include - -#include -#include -#include - -#include "moonfire-nvr.h" -#include "string.h" -#include "testutil.h" - -DECLARE_bool(alsologtostderr); - -using testing::_; -using testing::AnyNumber; -using testing::HasSubstr; -using testing::Invoke; -using testing::Return; - -namespace moonfire_nvr { -namespace { - -class MockVideoSource : public VideoSource { - public: - // Proxy, as gmock doesn't support non-copyable return values. - std::unique_ptr OpenRtsp( - const std::string &url, std::string *error_message) final { - return std::unique_ptr( - OpenRtspRaw(url, error_message)); - } - std::unique_ptr OpenFile( - const std::string &file, std::string *error_message) final { - return std::unique_ptr( - OpenFileRaw(file, error_message)); - } - - MOCK_METHOD2(OpenRtspRaw, - InputVideoPacketStream *(const std::string &, std::string *)); - MOCK_METHOD2(OpenFileRaw, - InputVideoPacketStream *(const std::string &, std::string *)); -}; - -class StreamTest : public testing::Test { - public: - StreamTest() { - std::string error_message; - test_dir_ = PrepareTempDirOrDie("moonfire-nvr-stream-copier"); - env_.clock = &clock_; - env_.video_source = &video_source_; - int ret = moonfire_nvr::GetRealFilesystem()->Open( - test_dir_.c_str(), O_DIRECTORY | O_RDONLY, &sample_file_dir_); - CHECK_EQ(0, ret) << "open: " << strerror(ret); - env_.sample_file_dir = sample_file_dir_.get(); - - CHECK(db_.Open(StrCat(test_dir_, "/db").c_str(), - SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE, &error_message)) - << error_message; - std::string create_sql = ReadFileOrDie("../src/schema.sql"); - { - DatabaseContext ctx(&db_); - CHECK(RunStatements(&ctx, create_sql, &error_message)) << error_message; - auto run = ctx.UseOnce( - R"( - insert into camera (uuid, short_name, host, username, password, - main_rtsp_path, sub_rtsp_path, retain_bytes) - values (:uuid, :short_name, :host, :username, :password, - :main_rtsp_path, :sub_rtsp_path, :retain_bytes); - )"); - run.BindBlob(":uuid", GetRealUuidGenerator()->Generate().binary_view()); - run.BindText(":short_name", "test"); - run.BindText(":host", "test-camera"); - run.BindText(":username", "foo"); - run.BindText(":password", "bar"); - run.BindText(":main_rtsp_path", "/main"); - run.BindText(":sub_rtsp_path", "/sub"); - run.BindInt64(":retain_bytes", 1000000); - CHECK_EQ(SQLITE_DONE, run.Step()) << run.error_message(); - } - mdb_.SetUuidGeneratorForTesting(&uuidgen_); - CHECK(mdb_.Init(&db_, &error_message)) << error_message; - env_.mdb = &mdb_; - - ListCamerasRow row; - int n_rows = 0; - mdb_.ListCameras([&row, &n_rows](const ListCamerasRow &some_row) { - ++n_rows; - row = some_row; - return IterationControl::kContinue; - }); - CHECK_EQ(1, n_rows); - - clock_.Sleep({1430006400, 0}); // 2015-04-26 00:00:00 UTC - - stream_.reset(new Stream(&signal_, &env_, row, 0, 5)); - } - - // A function to use in OpenRtspRaw invocations which shuts down the stream - // and indicates that the input video source can't be opened. - InputVideoPacketStream *Shutdown(const std::string &url, - std::string *error_message) { - *error_message = "(shutting down)"; - signal_.Shutdown(); - return nullptr; - } - - struct Frame { - Frame(bool is_key, int64_t pts, int64_t duration) - : is_key(is_key), pts(pts), duration(duration) {} - bool is_key; - int64_t pts; - int64_t duration; - - bool operator==(const Frame &o) const { - return is_key == o.is_key && pts == o.pts && duration == o.duration; - } - - friend std::ostream &operator<<(std::ostream &os, const Frame &f) { - return os << "Frame(" << f.is_key << ", " << f.pts << ", " << f.duration - << ")"; - } - }; - -#if 0 - std::vector GetFrames(const std::string &path) { - std::vector frames; - std::string error_message; - std::string full_path = StrCat(test_dir_, "/test/", path); - auto f = GetRealVideoSource()->OpenFile(full_path, &error_message); - if (f == nullptr) { - ADD_FAILURE() << full_path << ": " << error_message; - return frames; - } - VideoPacket pkt; - while (f->GetNext(&pkt, &error_message)) { - frames.push_back(Frame(pkt.is_key(), pkt.pts(), pkt.pkt()->duration)); - } - EXPECT_EQ("", error_message); - return frames; - } -#else - std::vector GetFrames(const re2::StringPiece uuid_text) { - std::vector frames; - Uuid uuid; - if (!uuid.ParseText(uuid_text)) { - ADD_FAILURE() << "unparseable: " << uuid_text; - return frames; - } - DatabaseContext ctx(&db_); - auto run = ctx.UseOnce( - "select video_index from recording where sample_file_uuid = :uuid;"); - run.BindBlob(":uuid", uuid.binary_view()); - if (run.Step() != SQLITE_ROW) { - ADD_FAILURE() << run.error_message(); - return frames; - } - for (SampleIndexIterator it(run.ColumnBlob(0)); !it.done(); it.Next()) { - frames.push_back(Frame(it.is_key(), it.start_90k(), it.duration_90k())); - } - return frames; - } -#endif - - MockUuidGenerator uuidgen_; - ShutdownSignal signal_; - SimulatedClock clock_; - testing::StrictMock video_source_; - Database db_; - MoonfireDatabase mdb_; - std::unique_ptr sample_file_dir_; - Environment env_; - std::string test_dir_; - std::unique_ptr stream_; -}; - -class ProxyingInputVideoPacketStream : public InputVideoPacketStream { - public: - explicit ProxyingInputVideoPacketStream( - std::unique_ptr base, SimulatedClock *clock) - : base_(std::move(base)), clock_(clock) {} - - bool GetNext(VideoPacket *pkt, std::string *error_message) final { - if (pkts_left_-- == 0) { - *error_message = "(pkt limit reached)"; - return false; - } - - // Advance time to when this packet starts. - clock_->Sleep(SecToTimespec(last_duration_sec_)); - if (!base_->GetNext(pkt, error_message)) { - return false; - } - last_duration_sec_ = - pkt->pkt()->duration * av_q2d(base_->stream()->time_base); - - // Adjust timestamps. - if (ts_offset_pkts_left_ > 0) { - pkt->pkt()->pts += ts_offset_; - pkt->pkt()->dts += ts_offset_; - --ts_offset_pkts_left_; - } - - // Use a fixed duration, as the duration from a real RTSP stream is only - // an estimate. Our test video is 1 fps, 90 kHz time base. - pkt->pkt()->duration = 90000; - - return true; - } - - const AVStream *stream() const final { return base_->stream(); } - - void set_ts_offset(int64_t offset, int pkts) { - ts_offset_ = offset; - ts_offset_pkts_left_ = pkts; - } - - void set_pkts(int num) { pkts_left_ = num; } - - private: - std::unique_ptr base_; - SimulatedClock *clock_ = nullptr; - double last_duration_sec_ = 0.; - int64_t ts_offset_ = 0; - int ts_offset_pkts_left_ = 0; - int pkts_left_ = std::numeric_limits::max(); -}; - -TEST_F(StreamTest, Basic) { - std::string error_message; - - // This is a ~1 fps test video with a timebase of 90 kHz. - auto in_stream = GetRealVideoSource()->OpenFile("../src/testdata/clip.mp4", - &error_message); - ASSERT_TRUE(in_stream != nullptr) << error_message; - auto *proxy_stream = - new ProxyingInputVideoPacketStream(std::move(in_stream), &clock_); - - // The starting pts of the input should be irrelevant. - proxy_stream->set_ts_offset(180000, std::numeric_limits::max()); - - Uuid uuid1; - ASSERT_TRUE(uuid1.ParseText("00000000-0000-0000-0000-000000000001")); - Uuid uuid2; - ASSERT_TRUE(uuid2.ParseText("00000000-0000-0000-0000-000000000002")); - EXPECT_CALL(uuidgen_, Generate()) - .WillOnce(Return(uuid1)) - .WillOnce(Return(uuid2)); - - EXPECT_CALL(video_source_, OpenRtspRaw("rtsp://foo:bar@test-camera/main", _)) - .WillOnce(Return(proxy_stream)) - .WillOnce(Invoke(this, &StreamTest::Shutdown)); - stream_->Run(); - // Compare frame-by-frame. - // Note below that while the rotation is scheduled to happen near 5-second - // boundaries (such as 2016-04-26 00:00:05), it gets deferred until the next - // key frame, which in this case is 00:00:07. - EXPECT_THAT(GetFrames("00000000-0000-0000-0000-000000000001"), - testing::ElementsAre( - Frame(true, 0, 90379), Frame(false, 90379, 89884), - Frame(false, 180263, 89749), Frame(false, 270012, 89981), - Frame(true, 359993, 90055), - Frame(false, 450048, - 89967), // pts_time 5.000533, past rotation time. - Frame(false, 540015, 90021), - Frame(false, 630036, 89958))); - EXPECT_THAT( - GetFrames("00000000-0000-0000-0000-000000000002"), - testing::ElementsAre(Frame(true, 0, 90011), Frame(false, 90011, 0))); -} - -TEST_F(StreamTest, NonIncreasingTimestamp) { - std::string error_message; - auto in_stream = GetRealVideoSource()->OpenFile("../src/testdata/clip.mp4", - &error_message); - ASSERT_TRUE(in_stream != nullptr) << error_message; - auto *proxy_stream = - new ProxyingInputVideoPacketStream(std::move(in_stream), &clock_); - proxy_stream->set_ts_offset(12345678, 1); - EXPECT_CALL(video_source_, OpenRtspRaw("rtsp://foo:bar@test-camera/main", _)) - .WillOnce(Return(proxy_stream)) - .WillOnce(Invoke(this, &StreamTest::Shutdown)); - - Uuid uuid1; - ASSERT_TRUE(uuid1.ParseText("00000000-0000-0000-0000-000000000001")); - EXPECT_CALL(uuidgen_, Generate()).WillOnce(Return(uuid1)); - - { - ScopedMockLog log; - EXPECT_CALL(log, Log(_, _, _)).Times(AnyNumber()); - EXPECT_CALL(log, - Log(_, _, HasSubstr("Rejecting non-increasing pts=90379"))); - log.Start(); - stream_->Run(); - } - - // The output file should still be added to the file manager, with the one - // packet that made it. The final packet on input error will have 0 - // duration. - EXPECT_THAT(GetFrames("00000000-0000-0000-0000-000000000001"), - testing::ElementsAre(Frame(true, 0, 0))); -} - -TEST_F(StreamTest, RetryOnInputError) { - std::string error_message; - auto in_stream_1 = GetRealVideoSource()->OpenFile("../src/testdata/clip.mp4", - &error_message); - ASSERT_TRUE(in_stream_1 != nullptr) << error_message; - auto *proxy_stream_1 = - new ProxyingInputVideoPacketStream(std::move(in_stream_1), &clock_); - proxy_stream_1->set_pkts(1); - - auto in_stream_2 = GetRealVideoSource()->OpenFile("../src/testdata/clip.mp4", - &error_message); - ASSERT_TRUE(in_stream_2 != nullptr) << error_message; - auto *proxy_stream_2 = - new ProxyingInputVideoPacketStream(std::move(in_stream_2), &clock_); - proxy_stream_2->set_pkts(1); - - EXPECT_CALL(video_source_, OpenRtspRaw("rtsp://foo:bar@test-camera/main", _)) - .WillOnce(Return(proxy_stream_1)) - .WillOnce(Return(proxy_stream_2)) - .WillOnce(Invoke(this, &StreamTest::Shutdown)); - - Uuid uuid1; - ASSERT_TRUE(uuid1.ParseText("00000000-0000-0000-0000-000000000001")); - Uuid uuid2; - ASSERT_TRUE(uuid2.ParseText("00000000-0000-0000-0000-000000000002")); - EXPECT_CALL(uuidgen_, Generate()) - .WillOnce(Return(uuid1)) - .WillOnce(Return(uuid2)); - stream_->Run(); - - // Each attempt should have resulted in a file with one packet. - EXPECT_THAT(GetFrames("00000000-0000-0000-0000-000000000001"), - testing::ElementsAre(Frame(true, 0, 0))); - EXPECT_THAT(GetFrames("00000000-0000-0000-0000-000000000002"), - testing::ElementsAre(Frame(true, 0, 0))); -} - -TEST_F(StreamTest, DiscardInitialNonKeyFrames) { - std::string error_message; - auto in_stream = GetRealVideoSource()->OpenFile("../src/testdata/clip.mp4", - &error_message); - ASSERT_TRUE(in_stream != nullptr) << error_message; - - // Discard the initial key frame packet. - VideoPacket dummy; - ASSERT_TRUE(in_stream->GetNext(&dummy, &error_message)) << error_message; - - auto *proxy_stream = - new ProxyingInputVideoPacketStream(std::move(in_stream), &clock_); - EXPECT_CALL(video_source_, OpenRtspRaw("rtsp://foo:bar@test-camera/main", _)) - .WillOnce(Return(proxy_stream)) - .WillOnce(Invoke(this, &StreamTest::Shutdown)); - - Uuid uuid1; - ASSERT_TRUE(uuid1.ParseText("00000000-0000-0000-0000-000000000001")); - Uuid uuid2; - ASSERT_TRUE(uuid2.ParseText("00000000-0000-0000-0000-000000000002")); - EXPECT_CALL(uuidgen_, Generate()) - .WillOnce(Return(uuid1)) - .WillOnce(Return(uuid2)); - stream_->Run(); - - // Skipped: initial key frame packet (duration 90379) - // Ignored: duration 89884, 89749, 89981 (total pts time: 2.99571... sec) - // Thus, the first output file should start at 00:00:02. - EXPECT_THAT( - GetFrames("00000000-0000-0000-0000-000000000001"), - testing::ElementsAre( - Frame(true, 0, 90055), - Frame(false, 90055, 89967), // pts_time 5.000533, past rotation time. - Frame(false, 180022, 90021), Frame(false, 270043, 89958))); - EXPECT_THAT( - GetFrames("00000000-0000-0000-0000-000000000002"), - testing::ElementsAre(Frame(true, 0, 90011), Frame(false, 90011, 0))); -} - -// TODO: test output stream error (on open, writing packet, closing). -// TODO: test rotation! - -} // namespace -} // namespace moonfire_nvr - -int main(int argc, char **argv) { - FLAGS_alsologtostderr = true; - google::ParseCommandLineFlags(&argc, &argv, true); - testing::InitGoogleTest(&argc, argv); - google::InitGoogleLogging(argv[0]); - return RUN_ALL_TESTS(); -} diff --git a/src/moonfire-nvr.cc b/src/moonfire-nvr.cc deleted file mode 100644 index 0ef7cbc..0000000 --- a/src/moonfire-nvr.cc +++ /dev/null @@ -1,462 +0,0 @@ -// This file is part of Moonfire NVR, a security camera network video recorder. -// Copyright (C) 2016 Scott Lamb -// -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// In addition, as a special exception, the copyright holders give -// permission to link the code of portions of this program with the -// OpenSSL library under certain conditions as described in each -// individual source file, and distribute linked combinations including -// the two. -// -// You must obey the GNU General Public License in all respects for all -// of the code used other than OpenSSL. If you modify file(s) with this -// exception, you may extend this exception to your version of the -// file(s), but you are not obligated to do so. If you do not wish to do -// so, delete this exception statement from your version. If you delete -// this exception statement from all source files in the program, then -// also delete it here. -// -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. -// -// You should have received a copy of the GNU General Public License -// along with this program. If not, see . -// -// moonfire-nvr.cc: implementation of moonfire-nvr.h. -// -// Caveats: -// -// Currently the recording thread blocks while a just-finished recording -// is synced to disk and written to the database, which can be 250+ ms. -// Likewise when recordings are being deleted. It would be better to hand -// off to a separate syncer thread, only blocking the recording when there -// would otherwise be insufficient disk space. -// -// This also commits to the SQLite database potentially several times per -// minute per camera: -// -// 1. (rarely) to get a new video sample entry id -// 2. to reserve a new uuid -// 3. to move uuids planned for deletion from "recording" to -// "reserved_sample_Files" -// 4. to mark those uuids as deleted -// 5. to insert the new recording -// -// These could be combined into a single batch per minute per camera or even -// per minute by doing some operations sooner (such as reserving the next -// minute's uuid when inserting the previous minute's recording) and some -// later (such as marking uuids as deleted). - -#define _BSD_SOURCE // for timegm(3). - -#include "moonfire-nvr.h" - -#include -#include -#include -#include -#include -#include -#include - -#include -#include -#include -#include - -#include "filesystem.h" -#include "h264.h" -#include "http.h" -#include "recording.h" -#include "string.h" -#include "time.h" - -using std::string; - -namespace moonfire_nvr { - -namespace { - -const int kRotateIntervalSec = 60; - -} // namespace - -// Call from dedicated thread. Runs until shutdown requested. -void Stream::Run() { - std::string error_message; - - // Do an initial rotation so that if retain_bytes has been reduced, the - // bulk deletion happens now, rather than while an input stream is open. - if (!RotateFiles(&error_message)) { - LOG(WARNING) << row_.short_name - << ": initial rotation failed: " << error_message; - } - - while (!signal_->ShouldShutdown()) { - if (in_ == nullptr && !OpenInput(&error_message)) { - LOG(WARNING) << row_.short_name - << ": Failed to open input; sleeping before retrying: " - << error_message; - env_->clock->Sleep({1, 0}); - continue; - } - - LOG(INFO) << row_.short_name << ": Calling ProcessPackets."; - ProcessPacketsResult res = ProcessPackets(&error_message); - if (res == kInputError) { - CloseOutput(-1); - in_.reset(); - start_localtime_90k_ = -1; - LOG(WARNING) << row_.short_name - << ": Input error; sleeping before retrying: " - << error_message; - env_->clock->Sleep({1, 0}); - continue; - } else if (res == kOutputError) { - CloseOutput(-1); - LOG(WARNING) << row_.short_name - << ": Output error; sleeping before retrying: " - << error_message; - env_->clock->Sleep({1, 0}); - continue; - } - } - CloseOutput(-1); -} - -Stream::ProcessPacketsResult Stream::ProcessPackets( - std::string *error_message) { - moonfire_nvr::VideoPacket pkt; - CHECK(in_ != nullptr); - CHECK(!writer_.is_open()); - while (!signal_->ShouldShutdown()) { - if (!in_->GetNext(&pkt, error_message)) { - if (error_message->empty()) { - *error_message = "unexpected end of stream"; - } - return kInputError; - } - - // With gcc 4.9 (Raspbian Jessie), - // #define AV_NOPTS_VALUE INT64_C(0x8000000000000000) - // produces an unsigned value. Argh. Work around. - static const int64_t kAvNoptsValue = AV_NOPTS_VALUE; - if (pkt.pkt()->pts == kAvNoptsValue || pkt.pkt()->dts == kAvNoptsValue) { - *error_message = "Rejecting packet with missing pts/dts"; - return kInputError; - } - - if (pkt.pkt()->pts != pkt.pkt()->dts) { - *error_message = - StrCat("Rejecting packet with pts=", pkt.pkt()->pts, " != dts=", - pkt.pkt()->dts, "; expecting only I or P frames."); - return kInputError; - } - - if (pkt.pkt()->pts < min_next_pts_) { - *error_message = StrCat("Rejecting non-increasing pts=", pkt.pkt()->pts, - "; expected at least ", min_next_pts_); - return kInputError; - } - min_next_pts_ = pkt.pkt()->pts + 1; - - frame_realtime_ = env_->clock->Now(); - - if (writer_.is_open() && frame_realtime_.tv_sec >= rotate_time_ && - pkt.is_key()) { - LOG(INFO) << row_.short_name << ": Reached rotation time; closing " - << recording_.sample_file_uuid.UnparseText() << "."; - CloseOutput(pkt.pkt()->pts - start_pts_); - } else if (writer_.is_open()) { - VLOG(3) << row_.short_name << ": Rotation time=" << rotate_time_ - << " vs current time=" << frame_realtime_.tv_sec; - } - - // Discard the initial, non-key frames from the input. - if (!seen_key_frame_ && !pkt.is_key()) { - continue; - } else if (!seen_key_frame_) { - seen_key_frame_ = true; - } - - if (!writer_.is_open()) { - start_pts_ = pkt.pts(); - if (!OpenOutput(error_message)) { - return kOutputError; - } - rotate_time_ = frame_realtime_.tv_sec - - (frame_realtime_.tv_sec % rotate_interval_sec_) + - rotate_offset_sec_; - if (rotate_time_ <= frame_realtime_.tv_sec) { - rotate_time_ += rotate_interval_sec_; - } - } - - auto start_time_90k = pkt.pkt()->pts - start_pts_; - if (prev_pkt_start_time_90k_ != -1) { - index_.AddSample(start_time_90k - prev_pkt_start_time_90k_, - prev_pkt_bytes_, prev_pkt_key_); - } - re2::StringPiece data = pkt.data(); - if (need_transform_) { - if (!TransformSampleData(data, &transform_tmp_, error_message)) { - return kInputError; - } - data = transform_tmp_; - } - if (!writer_.Write(data, error_message)) { - return kOutputError; - } - prev_pkt_start_time_90k_ = start_time_90k; - prev_pkt_bytes_ = data.size(); - prev_pkt_key_ = pkt.is_key(); - } - return kStopped; -} - -bool Stream::OpenInput(std::string *error_message) { - CHECK(in_ == nullptr); - string url = StrCat("rtsp://", row_.username, ":", row_.password, "@", - row_.host, row_.main_rtsp_path); - string redacted_url = StrCat("rtsp://", row_.username, ":redacted@", - row_.host, row_.main_rtsp_path); - LOG(INFO) << row_.short_name << ": Opening input: " << redacted_url; - in_ = env_->video_source->OpenRtsp(url, error_message); - min_next_pts_ = std::numeric_limits::min(); - seen_key_frame_ = false; - if (in_ == nullptr) { - return false; - } - - // The time base should match the 90kHz frequency specified in RFC 3551 - // section 5. - if (in_->stream()->time_base.num != 1 || - in_->stream()->time_base.den != kTimeUnitsPerSecond) { - *error_message = - StrCat("unexpected time base ", in_->stream()->time_base.num, "/", - in_->stream()->time_base.den); - return false; - } - - // width and height must fix into 16-bit ints for MP4 encoding. - int max_dimension = std::numeric_limits::max(); - if (in_->stream()->codec->width > max_dimension || - in_->stream()->codec->height > max_dimension) { - *error_message = - StrCat("input dimensions ", in_->stream()->codec->width, "x", - in_->stream()->codec->height, " are too large."); - return false; - } - entry_.id = -1; - entry_.width = in_->stream()->codec->width; - entry_.height = in_->stream()->codec->height; - re2::StringPiece extradata = in_->extradata(); - if (!ParseExtraData(extradata, entry_.width, entry_.height, &entry_.data, - &need_transform_, error_message)) { - in_.reset(); - return false; - } - auto sha1 = Digest::SHA1(); - sha1->Update(entry_.data); - entry_.sha1 = sha1->Finalize(); - if (!env_->mdb->InsertVideoSampleEntry(&entry_, error_message)) { - in_.reset(); - return false; - } - return true; -} - -void Stream::CloseOutput(int64_t pts) { - if (!writer_.is_open()) { - return; - } - std::string error_message; - if (prev_pkt_start_time_90k_ != -1) { - int64_t duration_90k = pts - prev_pkt_start_time_90k_; - index_.AddSample(duration_90k > 0 ? duration_90k : 0, prev_pkt_bytes_, - prev_pkt_key_); - } - if (!writer_.Close(&recording_.sample_file_sha1, &error_message)) { - LOG(ERROR) << row_.short_name << ": Closing output " - << recording_.sample_file_uuid.UnparseText() - << " failed with error: " << error_message; - uuids_to_unlink_.push_back(recording_.sample_file_uuid); - TryUnlink(); - return; - } - int ret = env_->sample_file_dir->Sync(); - if (ret != 0) { - LOG(ERROR) << row_.short_name - << ": Unable to sync sample file dir after writing " - << recording_.sample_file_uuid.UnparseText() << ": " - << strerror(ret); - uuids_to_unlink_.push_back(recording_.sample_file_uuid); - TryUnlink(); - return; - } - if (!env_->mdb->InsertRecording(&recording_, &error_message)) { - LOG(ERROR) << row_.short_name << ": Unable to insert recording " - << recording_.sample_file_uuid.UnparseText() << ": " - << error_message; - uuids_to_unlink_.push_back(recording_.sample_file_uuid); - TryUnlink(); - return; - } - row_.total_sample_file_bytes += recording_.sample_file_bytes; - VLOG(1) << row_.short_name << ": ...wrote " - << recording_.sample_file_uuid.UnparseText() << "; usage now " - << HumanizeWithBinaryPrefix(row_.total_sample_file_bytes, "B"); -} - -void Stream::TryUnlink() { - std::vector still_not_unlinked; - for (const auto &uuid : uuids_to_unlink_) { - std::string text = uuid.UnparseText(); - int ret = env_->sample_file_dir->Unlink(text.c_str()); - if (ret == ENOENT) { - LOG(WARNING) << row_.short_name << ": Sample file " << text - << " already deleted!"; - } else if (ret != 0) { - LOG(WARNING) << row_.short_name << ": Unable to unlink " << text << ": " - << strerror(ret); - still_not_unlinked.push_back(uuid); - continue; - } - uuids_to_mark_deleted_.push_back(uuid); - } - uuids_to_unlink_ = std::move(still_not_unlinked); -} - -bool Stream::OpenOutput(std::string *error_message) { - int64_t frame_localtime_90k = To90k(frame_realtime_); - if (start_localtime_90k_ == -1) { - start_localtime_90k_ = frame_localtime_90k - start_pts_; - } - if (!RotateFiles(error_message)) { - return false; - } - std::vector reserved = env_->mdb->ReserveSampleFiles(1, error_message); - if (reserved.size() != 1) { - return false; - } - CHECK(!writer_.is_open()); - string filename = reserved[0].UnparseText(); - recording_.id = -1; - recording_.camera_id = row_.id; - recording_.sample_file_uuid = reserved[0]; - recording_.video_sample_entry_id = entry_.id; - recording_.local_time_90k = frame_localtime_90k; - index_.Init(&recording_, start_localtime_90k_ + start_pts_); - if (!writer_.Open(filename.c_str(), error_message)) { - return false; - } - prev_pkt_start_time_90k_ = -1; - prev_pkt_bytes_ = -1; - prev_pkt_key_ = false; - LOG(INFO) << row_.short_name << ": Opened output " << filename - << ", using start_pts=" << start_pts_ - << ", input timebase=" << in_->stream()->time_base.num << "/" - << in_->stream()->time_base.den; - return true; -} - -bool Stream::RotateFiles(std::string *error_message) { - int64_t bytes_needed = row_.total_sample_file_bytes - row_.retain_bytes; - int64_t bytes_to_delete = 0; - if (bytes_needed <= 0) { - VLOG(1) << row_.short_name << ": have remaining quota of " - << HumanizeWithBinaryPrefix(-bytes_needed, "B"); - return true; - } - LOG(INFO) << row_.short_name << ": need to delete " - << HumanizeWithBinaryPrefix(bytes_needed, "B"); - std::vector to_delete; - auto row_cb = [&](const ListOldestSampleFilesRow &row) { - bytes_needed -= row.sample_file_bytes; - bytes_to_delete += row.sample_file_bytes; - to_delete.push_back(row); - return bytes_needed < 0 ? IterationControl::kBreak - : IterationControl::kContinue; - }; - if (!env_->mdb->ListOldestSampleFiles(row_.uuid, row_cb, error_message)) { - return false; - } - if (bytes_needed > 0) { - *error_message = - StrCat("couldn't find enough files to delete; ", - HumanizeWithBinaryPrefix(bytes_needed, "B"), " left."); - return false; - } - if (!env_->mdb->DeleteRecordings(to_delete, error_message)) { - return false; - } - for (const auto &to_delete_row : to_delete) { - uuids_to_unlink_.push_back(to_delete_row.sample_file_uuid); - } - row_.total_sample_file_bytes -= bytes_to_delete; - TryUnlink(); - if (!uuids_to_unlink_.empty()) { - *error_message = - StrCat("failed to unlink ", uuids_to_unlink_.size(), " files."); - return false; - } - int ret = env_->sample_file_dir->Sync(); - if (ret != 0) { - *error_message = StrCat("fsync sample directory: ", strerror(ret)); - return false; - } - if (!env_->mdb->MarkSampleFilesDeleted(uuids_to_mark_deleted_, - error_message)) { - *error_message = StrCat("unable to mark ", uuids_to_mark_deleted_.size(), - " sample files as deleted"); - return false; - } - uuids_to_mark_deleted_.clear(); - VLOG(1) << row_.short_name << ": ...deleted successfully; usage now " - << HumanizeWithBinaryPrefix(row_.total_sample_file_bytes, "B"); - return true; -} - -Nvr::~Nvr() { - signal_.Shutdown(); - for (auto &thread : stream_threads_) { - thread.join(); - } - // TODO: cleanup reservations? -} - -bool Nvr::Init(std::string *error_msg) { - std::vector all_reserved; - if (!env_->mdb->ListReservedSampleFiles(&all_reserved, error_msg)) { - return false; - } - for (const auto &reserved : all_reserved) { - int ret = env_->sample_file_dir->Unlink(reserved.UnparseText().c_str()); - if (ret != 0 && ret != ENOENT) { - LOG(WARNING) << "Unable to remove reserved sample file: " - << reserved.UnparseText(); - } - } - - std::vector cameras; - env_->mdb->ListCameras([&](const ListCamerasRow &row) { - cameras.push_back(row); - return IterationControl::kContinue; - }); - for (size_t i = 0; i < cameras.size(); ++i) { - int rotate_offset_sec = kRotateIntervalSec * i / cameras.size(); - auto *stream = new Stream(&signal_, env_, cameras[i], rotate_offset_sec, - kRotateIntervalSec); - streams_.emplace_back(stream); - stream_threads_.emplace_back([stream]() { stream->Run(); }); - }; - return true; -} - -} // namespace moonfire_nvr diff --git a/src/moonfire-nvr.h b/src/moonfire-nvr.h deleted file mode 100644 index 7dbcca4..0000000 --- a/src/moonfire-nvr.h +++ /dev/null @@ -1,187 +0,0 @@ -// This file is part of Moonfire NVR, a security camera digital video recorder. -// Copyright (C) 2016 Scott Lamb -// -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// In addition, as a special exception, the copyright holders give -// permission to link the code of portions of this program with the -// OpenSSL library under certain conditions as described in each -// individual source file, and distribute linked combinations including -// the two. -// -// You must obey the GNU General Public License in all respects for all -// of the code used other than OpenSSL. If you modify file(s) with this -// exception, you may extend this exception to your version of the -// file(s), but you are not obligated to do so. If you do not wish to do -// so, delete this exception statement from your version. If you delete -// this exception statement from all source files in the program, then -// also delete it here. -// -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. -// -// You should have received a copy of the GNU General Public License -// along with this program. If not, see . -// -// moonfire-nvr.h: main digital video recorder components. - -#ifndef MOONFIRE_NVR_NVR_H -#define MOONFIRE_NVR_NVR_H - -#include -#include - -#include -#include -#include -#include -#include -#include - -#include - -#include "filesystem.h" -#include "moonfire-db.h" -#include "ffmpeg.h" -#include "time.h" - -namespace moonfire_nvr { - -// A signal that all streams associated with an Nvr should shut down. -class ShutdownSignal { - public: - ShutdownSignal() {} - ShutdownSignal(const ShutdownSignal &) = delete; - ShutdownSignal &operator=(const ShutdownSignal &) = delete; - - void Shutdown() { shutdown_.store(true, std::memory_order_relaxed); } - - bool ShouldShutdown() const { - return shutdown_.load(std::memory_order_relaxed); - } - - private: - std::atomic_bool shutdown_{false}; -}; - -// The Nvr's environment. This is supplied for testability. -struct Environment { - WallClock *clock = nullptr; - VideoSource *video_source = nullptr; - File *sample_file_dir = nullptr; - MoonfireDatabase *mdb = nullptr; -}; - -// A single video stream, currently always a camera's "main" (as opposed to -// "sub") stream. Methods are thread-compatible rather than thread-safe; the -// Nvr should call Run in a dedicated thread. -class Stream { - public: - Stream(const ShutdownSignal *signal, Environment *const env, - const moonfire_nvr::ListCamerasRow &row, int rotate_offset_sec, - int rotate_interval_sec) - : signal_(signal), - env_(env), - row_(row), - rotate_offset_sec_(rotate_offset_sec), - rotate_interval_sec_(rotate_interval_sec), - writer_(env->sample_file_dir) {} - Stream(const Stream &) = delete; - Stream &operator=(const Stream &) = delete; - - // Call from dedicated thread. Runs until shutdown requested. - void Run(); - - private: - enum ProcessPacketsResult { kInputError, kOutputError, kStopped }; - - ProcessPacketsResult ProcessPackets(std::string *error_message); - bool OpenInput(std::string *error_message); - - // |pts| should be the relative pts within this output segment if closing - // due to normal rotation, or -1 if closing abruptly. - void CloseOutput(int64_t pts); - - bool OpenOutput(std::string *error_message); - bool RotateFiles(std::string *error_message); - void TryUnlink(); - - const ShutdownSignal *signal_; - const Environment *env_; - ListCamerasRow row_; - const int rotate_offset_sec_; - const int rotate_interval_sec_; - - // - // State below is used only by the thread in Run(). - // - - std::unique_ptr in_; - int64_t min_next_pts_ = std::numeric_limits::min(); - bool seen_key_frame_ = false; - - // need_transform_ indicates if TransformSampleData will need to be called - // on each video sample. - bool need_transform_ = false; - - VideoSampleEntry entry_; - std::string transform_tmp_; - std::vector uuids_to_unlink_; - std::vector uuids_to_mark_deleted_; - - // Current output segment. - Recording recording_; - moonfire_nvr::SampleFileWriter writer_; - SampleIndexEncoder index_; - time_t rotate_time_ = 0; // rotate when frame_realtime_ >= rotate_time_. - - // start_pts_ is the pts of the first frame included in the current output. - int64_t start_pts_ = -1; - - // start_localtime_90k_ is the local system's time since epoch (in 90k units) - // to match start_pts_. - int64_t start_localtime_90k_ = -1; - - // These fields describe a packet which has been written to the - // sample file but (because the duration is not yet known) has not been - // added to the index. - int32_t prev_pkt_start_time_90k_ = -1; - int32_t prev_pkt_bytes_ = -1; - bool prev_pkt_key_ = false; - struct timespec frame_realtime_ = {0, 0}; -}; - -// The main network video recorder, which manages a collection of streams. -class Nvr { - public: - explicit Nvr(Environment *env) : env_(env) {} - Nvr(const Nvr &) = delete; - Nvr &operator=(const Nvr &) = delete; - - // Shut down, blocking for outstanding streams. - // Caller only has to guarantee that HttpCallback is not being called / will - // not be called again, likely by having already shut down the event loop. - ~Nvr(); - - // Initialize the NVR. Call before any other operation. - // Verifies configuration and starts background threads to capture/rotate - // streams. - bool Init(std::string *error_msg); - - private: - void HttpCallbackForTopLevel(evhttp_request *req); - - Environment *const env_; - std::vector> streams_; - std::vector stream_threads_; - ShutdownSignal signal_; -}; - -} // namespace moonfire_nvr - -#endif // MOONFIRE_NVR_NVR_H diff --git a/src/mp4-test.cc b/src/mp4-test.cc deleted file mode 100644 index 76e4d5c..0000000 --- a/src/mp4-test.cc +++ /dev/null @@ -1,404 +0,0 @@ -// This file is part of Moonfire DVR, a security camera digital video recorder. -// Copyright (C) 2015 Scott Lamb -// -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// In addition, as a special exception, the copyright holders give -// permission to link the code of portions of this program with the -// OpenSSL library under certain conditions as described in each -// individual source file, and distribute linked combinations including -// the two. -// -// You must obey the GNU General Public License in all respects for all -// of the code used other than OpenSSL. If you modify file(s) with this -// exception, you may extend this exception to your version of the -// file(s), but you are not obligated to do so. If you do not wish to do -// so, delete this exception statement from your version. If you delete -// this exception statement from all source files in the program, then -// also delete it here. -// -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. -// -// You should have received a copy of the GNU General Public License -// along with this program. If not, see . -// -// mp4-test.cc: tests of the mp4.h interface. - -#include -#include -#include - -#include -#include -#include -#include - -#include "ffmpeg.h" -#include "h264.h" -#include "http.h" -#include "mp4.h" -#include "string.h" -#include "testutil.h" - -DECLARE_bool(alsologtostderr); - -using moonfire_nvr::internal::Mp4SampleTablePieces; - -namespace moonfire_nvr { -namespace { - -std::string ToHex(const FileSlice *slice, bool pad) { - EvBuffer buf; - std::string error_message; - size_t size = slice->size(); - CHECK(slice->AddRange(ByteRange(0, size), &buf, &error_message)) - << error_message; - CHECK_EQ(size, evbuffer_get_length(buf.get())); - return ::moonfire_nvr::ToHex( - re2::StringPiece( - reinterpret_cast(evbuffer_pullup(buf.get(), size)), - size), - pad); -} - -std::string Digest(const FileSlice *slice) { - EvBuffer buf; - std::string error_message; - ByteRange left(0, slice->size()); - while (left.size() > 0) { - auto ret = slice->AddRange(left, &buf, &error_message); - CHECK_GT(ret, 0) << error_message; - left.begin += ret; - } - evbuffer_iovec vec; - auto digest = Digest::SHA1(); - while (evbuffer_peek(buf.get(), -1, nullptr, &vec, 1) > 0) { - digest->Update(re2::StringPiece( - reinterpret_cast(vec.iov_base), vec.iov_len)); - evbuffer_drain(buf.get(), vec.iov_len); - } - return ::moonfire_nvr::ToHex(digest->Finalize()); -} - -TEST(Mp4SampleTablePiecesTest, AllSyncFrames) { - Recording recording; - SampleIndexEncoder encoder; - encoder.Init(&recording, 42); - for (int i = 1; i <= 5; ++i) { - int64_t sample_duration_90k = 2 * i; - int64_t sample_bytes = 3 * i; - encoder.AddSample(sample_duration_90k, sample_bytes, true); - } - - Mp4SampleTablePieces pieces; - std::string error_message; - // Time range [2, 2 + 4 + 6 + 8) means the 2nd, 3rd, 4th samples should be - // included. - ASSERT_TRUE(pieces.Init(&recording, 2, 10, 2, 2 + 4 + 6 + 8, &error_message)) - << error_message; - - EXPECT_EQ(3, pieces.stts_entry_count()); - const char kExpectedStts[] = - "00 00 00 01 00 00 00 04 " // run length / timestamps. - "00 00 00 01 00 00 00 06 " - "00 00 00 01 00 00 00 08"; - EXPECT_EQ(kExpectedStts, ToHex(pieces.stts_entries(), true)); - - // Initial index "10" as given above. - EXPECT_EQ(3, pieces.stss_entry_count()); - const char kExpectedStss[] = "00 00 00 0a 00 00 00 0b 00 00 00 0c"; - EXPECT_EQ(kExpectedStss, ToHex(pieces.stss_entries(), true)); - - EXPECT_EQ(3, pieces.stsz_entry_count()); - const char kExpectedStsz[] = "00 00 00 06 00 00 00 09 00 00 00 0c"; - EXPECT_EQ(kExpectedStsz, ToHex(pieces.stsz_entries(), true)); -} - -TEST(Mp4SampleTablePiecesTest, HalfSyncFrames) { - Recording recording; - SampleIndexEncoder encoder; - encoder.Init(&recording, 42); - for (int i = 1; i <= 5; ++i) { - int64_t sample_duration_90k = 2 * i; - int64_t sample_bytes = 3 * i; - encoder.AddSample(sample_duration_90k, sample_bytes, (i % 2) == 1); - } - - Mp4SampleTablePieces pieces; - std::string error_message; - // Time range [2 + 4 + 6, 2 + 4 + 6 + 8) means the 4th samples should be - // included. The 3rd gets pulled in also because it is a sync frame and the - // 4th is not. - ASSERT_TRUE( - pieces.Init(&recording, 2, 10, 2 + 4 + 6, 2 + 4 + 6 + 8, &error_message)) - << error_message; - - EXPECT_EQ(2, pieces.stts_entry_count()); - const char kExpectedStts[] = - "00 00 00 01 00 00 00 06 " - "00 00 00 01 00 00 00 08"; - EXPECT_EQ(kExpectedStts, ToHex(pieces.stts_entries(), true)); - - EXPECT_EQ(1, pieces.stss_entry_count()); - const char kExpectedStss[] = "00 00 00 0a"; - EXPECT_EQ(kExpectedStss, ToHex(pieces.stss_entries(), true)); - - EXPECT_EQ(2, pieces.stsz_entry_count()); - const char kExpectedStsz[] = "00 00 00 09 00 00 00 0c"; - EXPECT_EQ(kExpectedStsz, ToHex(pieces.stsz_entries(), true)); -} - -TEST(Mp4SampleTablePiecesTest, FastPath) { - Recording recording; - SampleIndexEncoder encoder; - encoder.Init(&recording, 42); - for (int i = 1; i <= 5; ++i) { - int64_t sample_duration_90k = 2 * i; - int64_t sample_bytes = 3 * i; - encoder.AddSample(sample_duration_90k, sample_bytes, (i % 2) == 1); - } - auto total_duration_90k = recording.end_time_90k - recording.start_time_90k; - - Mp4SampleTablePieces pieces; - std::string error_message; - // Time range [0, end - start) means to pull in everything. - // This uses a fast path which can determine the size without examining the - // index. - ASSERT_TRUE( - pieces.Init(&recording, 2, 10, 0, total_duration_90k, &error_message)) - << error_message; - - EXPECT_EQ(5, pieces.stts_entry_count()); - const char kExpectedStts[] = - "00 00 00 01 00 00 00 02 " - "00 00 00 01 00 00 00 04 " - "00 00 00 01 00 00 00 06 " - "00 00 00 01 00 00 00 08 " - "00 00 00 01 00 00 00 0a"; - EXPECT_EQ(kExpectedStts, ToHex(pieces.stts_entries(), true)); - - EXPECT_EQ(3, pieces.stss_entry_count()); - const char kExpectedStss[] = "00 00 00 0a 00 00 00 0c 00 00 00 0e"; - EXPECT_EQ(kExpectedStss, ToHex(pieces.stss_entries(), true)); - - EXPECT_EQ(5, pieces.stsz_entry_count()); - const char kExpectedStsz[] = - "00 00 00 03 00 00 00 06 00 00 00 09 00 00 00 0c 00 00 00 0f"; - EXPECT_EQ(kExpectedStsz, ToHex(pieces.stsz_entries(), true)); -} - -class IntegrationTest : public testing::Test { - protected: - IntegrationTest() { - tmpdir_path_ = PrepareTempDirOrDie("mp4-integration-test"); - int ret = GetRealFilesystem()->Open(tmpdir_path_.c_str(), - O_RDONLY | O_DIRECTORY, &tmpdir_); - CHECK_EQ(0, ret) << strerror(ret); - } - - Recording CopyMp4ToSingleRecording() { - std::string error_message; - Recording recording; - SampleIndexEncoder index; - - // Set start time to 2015-04-26 00:00:00 UTC. - index.Init(&recording, UINT64_C(1430006400) * kTimeUnitsPerSecond); - SampleFileWriter writer(tmpdir_.get()); - std::string filename = recording.sample_file_uuid.UnparseText(); - if (!writer.Open(filename.c_str(), &error_message)) { - ADD_FAILURE() << "open " << filename << ": " << error_message; - return recording; - } - auto in = GetRealVideoSource()->OpenFile("../src/testdata/clip.mp4", - &error_message); - if (in == nullptr) { - ADD_FAILURE() << "open clip.mp4" << error_message; - return recording; - } - - video_sample_entry_.width = in->stream()->codec->width; - video_sample_entry_.height = in->stream()->codec->height; - bool need_transform; - if (!ParseExtraData(in->extradata(), in->stream()->codec->width, - in->stream()->codec->height, &video_sample_entry_.data, - &need_transform, &error_message)) { - ADD_FAILURE() << "GetH264SampleEntry: " << error_message; - return recording; - } - EXPECT_FALSE(need_transform); - - while (true) { - VideoPacket pkt; - if (!in->GetNext(&pkt, &error_message)) { - if (!error_message.empty()) { - ADD_FAILURE() << "GetNext: " << error_message; - return recording; - } - break; - } - if (!writer.Write(GetData(pkt), &error_message)) { - ADD_FAILURE() << "Write: " << error_message; - return recording; - } - index.AddSample(pkt.pkt()->duration, pkt.pkt()->size, pkt.is_key()); - } - - if (!writer.Close(&recording.sample_file_sha1, &error_message)) { - ADD_FAILURE() << "Close: " << error_message; - } - return recording; - } - - std::shared_ptr CreateMp4FromSingleRecording( - const Recording &recording, int32_t rel_start_90k, int32_t rel_end_90k, - bool include_ts) { - Mp4FileBuilder builder(tmpdir_.get()); - builder.SetSampleEntry(video_sample_entry_); - builder.Append(Recording(recording), rel_start_90k, rel_end_90k); - builder.include_timestamp_subtitle_track(include_ts); - std::string error_message; - auto mp4 = builder.Build(&error_message); - EXPECT_TRUE(mp4 != nullptr) << error_message; - return mp4; - } - - void WriteMp4(VirtualFile *f) { - EvBuffer buf; - std::string error_message; - ByteRange left(0, f->size()); - while (left.size() > 0) { - auto ret = f->AddRange(left, &buf, &error_message); - ASSERT_GT(ret, 0) << error_message; - left.begin += ret; - } - WriteFileOrDie(StrCat(tmpdir_path_, "/clip.new.mp4"), &buf); - } - - void CompareMp4s(int64_t pts_offset) { - std::string error_message; - auto original = GetRealVideoSource()->OpenFile("../src/testdata/clip.mp4", - &error_message); - ASSERT_TRUE(original != nullptr) << error_message; - auto copied = GetRealVideoSource()->OpenFile( - StrCat(tmpdir_path_, "/clip.new.mp4"), &error_message); - ASSERT_TRUE(copied != nullptr) << error_message; - - EXPECT_EQ(original->extradata(), copied->extradata()); - EXPECT_EQ(original->stream()->codec->width, copied->stream()->codec->width); - EXPECT_EQ(original->stream()->codec->height, - copied->stream()->codec->height); - - int pkt = 0; - while (true) { - VideoPacket original_pkt; - VideoPacket copied_pkt; - - bool original_has_next = original->GetNext(&original_pkt, &error_message); - ASSERT_TRUE(original_has_next || error_message.empty()) - << "pkt " << pkt << ": " << error_message; - bool copied_has_next = copied->GetNext(&copied_pkt, &error_message); - ASSERT_TRUE(copied_has_next || error_message.empty()) - << "pkt " << pkt << ": " << error_message; - if (!original_has_next && !copied_has_next) { - break; - } - ASSERT_TRUE(original_has_next) << "pkt " << pkt; - ASSERT_TRUE(copied_has_next) << "pkt " << pkt; - EXPECT_EQ(original_pkt.pkt()->pts + pts_offset, copied_pkt.pkt()->pts) - << "pkt " << pkt; - - // One would normally expect the duration to be exactly the same, but - // when using an edit list, ffmpeg appears to extend the last packet's - // duration by the amount skipped at the beginning. I think this is a - // bug on their side. - EXPECT_LE(original_pkt.pkt()->duration, copied_pkt.pkt()->duration) - << "pkt " << pkt; - EXPECT_EQ(GetData(original_pkt), GetData(copied_pkt)) << "pkt " << pkt; - ++pkt; - } - } - - re2::StringPiece GetData(const VideoPacket &pkt) { - return re2::StringPiece(reinterpret_cast(pkt.pkt()->data), - pkt.pkt()->size); - } - - std::string tmpdir_path_; - std::unique_ptr tmpdir_; - std::string etag_; - VideoSampleEntry video_sample_entry_; -}; - -TEST_F(IntegrationTest, RoundTrip) { - Recording recording = CopyMp4ToSingleRecording(); - if (HasFailure()) { - return; - } - auto f = CreateMp4FromSingleRecording( - recording, 0, std::numeric_limits::max(), false); - WriteMp4(f.get()); - CompareMp4s(0); - - // This test is brittle, which is the point. Any time the digest comparison - // here fails, it can be updated, but the etag must change as well! - // Otherwise clients may combine ranges from the new format with ranges - // from the old format! - EXPECT_EQ("1e5331e8371bd97ac3158b3a86494abc87cdc70e", Digest(f.get())); - EXPECT_EQ("\"268db2cd6e4814676d38832f1f9340c7555e4e71\"", f->etag()); - - // 10 seconds later than the segment's start time. - EXPECT_EQ(1430006410, f->last_modified()); -} - -TEST_F(IntegrationTest, RoundTripWithSubtitle) { - Recording recording = CopyMp4ToSingleRecording(); - if (HasFailure()) { - return; - } - auto f = CreateMp4FromSingleRecording( - recording, 0, std::numeric_limits::max(), true); - WriteMp4(f.get()); - CompareMp4s(0); - - // This test is brittle, which is the point. Any time the digest comparison - // here fails, it can be updated, but the etag must change as well! - // Otherwise clients may combine ranges from the new format with ranges - // from the old format! - EXPECT_EQ("0081a442ba73092027fc580eeac2ebf25cb1ef50", Digest(f.get())); - EXPECT_EQ("\"8a29042355e1e28c10fbba328d1ddc9d54e450cd\"", f->etag()); -} - -TEST_F(IntegrationTest, RoundTripWithEditList) { - Recording recording = CopyMp4ToSingleRecording(); - if (HasFailure()) { - return; - } - auto f = CreateMp4FromSingleRecording( - recording, 1, std::numeric_limits::max(), false); - WriteMp4(f.get()); - CompareMp4s(-1); - - // This test is brittle, which is the point. Any time the digest comparison - // here fails, it can be updated, but the etag must change as well! - // Otherwise clients may combine ranges from the new format with ranges - // from the old format! - EXPECT_EQ("685e026af44204bc9cc52115c5e17058e9fb7c70", Digest(f.get())); - EXPECT_EQ("\"1373289ddc7c05580deeeb1f1624e2d6cac7ddd3\"", f->etag()); -} - -} // namespace -} // namespace moonfire_nvr - -int main(int argc, char **argv) { - FLAGS_alsologtostderr = true; - google::ParseCommandLineFlags(&argc, &argv, true); - testing::InitGoogleTest(&argc, argv); - google::InitGoogleLogging(argv[0]); - return RUN_ALL_TESTS(); -} diff --git a/src/mp4.cc b/src/mp4.cc deleted file mode 100644 index c95f8c1..0000000 --- a/src/mp4.cc +++ /dev/null @@ -1,1147 +0,0 @@ -// This file is part of Moonfire DVR, a security camera network video recorder. -// Copyright (C) 2016 Scott Lamb -// -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// In addition, as a special exception, the copyright holders give -// permission to link the code of portions of this program with the -// OpenSSL library under certain conditions as described in each -// individual source file, and distribute linked combinations including -// the two. -// -// You must obey the GNU General Public License in all respects for all -// of the code used other than OpenSSL. If you modify file(s) with this -// exception, you may extend this exception to your version of the -// file(s), but you are not obligated to do so. If you do not wish to do -// so, delete this exception statement from your version. If you delete -// this exception statement from all source files in the program, then -// also delete it here. -// -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. -// -// You should have received a copy of the GNU General Public License -// along with this program. If not, see . -// -// mp4.cc: implementation of mp4.h interface. -// -// This implementation will make the most sense when read side-by-side with -// ISO/IEC 14496-12:2015, available at the following URL: -// -// -// mp4.cc generates VirtualFiles via an array of FileSlices. Each FileSlice -// is responsible for some portion of the .mp4 file, generally some subset of -// a single .mp4 "box". Slices fall into these categories: -// -// 1. entirely static data from a const char kConstant[]. This is preferred in -// the interest of simplicity and efficiency when there is only one useful -// value for all the fields in the box, including its length. -// -// These slices are represented using the StringPieceSlice class. -// -// 2. a box's fixed-length fields. In some cases a slice represents the entire -// contents of a FullBox type; in others a slice represents only the -// "length" and "type" fields of a container Box, while other contents -// (such as child boxes) are appended to the box as a separate slice. -// -// These slices are represented using a specific "struct ...Box" type for -// type safety and simplicity. The structs match the actual wire format---in -// particular, they are packed and store fields in network byte order. -// sizeof(...Box) is meaningful and structure data can be simply written -// with memcpy, as opposed to via manually-written or generated serialization -// code. (This approach could be revisited if there's ever a need to run -// on a compiler that doesn't support __attribute__((packed)) or a -// processor that doesn't support unaligned access.) The structs are -// wrapped with the Mp4Box<> template class which manages child slices and -// fills in the box's length field automatically. -// -// 3. variable-length data generated using the Mp4SampleTablePieces class, -// representing part of one box dealing with a single recording. These -// are the largest portion of a typical .mp4's metadata. -// -// These slices are generated using the FillerFileSlice class. They -// determine their sizes eagerly (so that the size of the file is known and -// so that later byte ranges can be served correctly) but only generate -// their contents when the requested byte range overlaps with the slice -// (for memory/CPU efficiency). -// -// 4. file-backed variable-length data, representing actual video samples. -// -// These are represented using the FileSlice class and are mmap()ed via -// libevent, letting the kernel decide how much to page in at once. -// -// The box hierarchy is constructed through append operations on the Mp4Box -// subclasses. Most of the static data is always in RAM when the VirtualFile -// is, but the file-backed and sample table portions are not. This should be -// a reasonable compromise between simplicity of implementation and memory -// efficiency. - -#include "mp4.h" - -#include "coding.h" - -#define NET_UINT64_C(x) ::moonfire_nvr::ToNetworkU64(UINT64_C(x)) -#define NET_INT64_C(x) ::moonfire_nvr::ToNetwork64(UINT64_C(x)) -#define NET_UINT32_C(x) ::moonfire_nvr::ToNetworkU32(UINT32_C(x)) -#define NET_INT32_C(x) ::moonfire_nvr::ToNetwork32(UINT32_C(x)) -#define NET_UINT16_C(x) ::moonfire_nvr::ToNetworkU16(UINT16_C(x)) -#define NET_INT16_C(x) ::moonfire_nvr::ToNetwork16(UINT16_C(x)) - -using ::moonfire_nvr::internal::Mp4FileSegment; - -namespace moonfire_nvr { - -namespace { - -// strftime template for subtitles. Must be a constant length declared below. -const char kSubtitleTemplate[] = "%Y-%m-%d %H:%M:%S %z"; -const size_t kSubtitleLength = strlen("2015-07-02 17:10:00 -0700"); - -// This value should be incremented any time a change is made to this file -// that causes the different bytes to be output for a particular set of -// Mp4Builder options. Incrementing this value will cause the etag to change -// as well. -const char kFormatVersion[] = {0x01}; - -// ISO/IEC 14496-12 section 4.3, ftyp. -const char kFtypBox[] = { - 0x00, 0x00, 0x00, 0x20, // length = 32, sizeof(kFtypBox) - 'f', 't', 'y', 'p', // type - 'i', 's', 'o', 'm', // major_brand - 0x00, 0x00, 0x02, 0x00, // minor_version - 'i', 's', 'o', 'm', // compatible_brands[0] - 'i', 's', 'o', '2', // compatible_brands[1] - 'a', 'v', 'c', '1', // compatible_brands[2] - 'm', 'p', '4', '1', // compatible_brands[3] -}; - -// vmhd and dinf boxes. These are both completely static and adjacent in the -// structure, so they're in a single constant. -const char kVmhdAndDinfBoxes[] = { - // A vmhd box; the "graphicsmode" and "opcolor" values don't have any - // meaningful use. - 0x00, 0x00, 0x00, 0x14, // length == sizeof(kVmhdBox) - 'v', 'm', 'h', 'd', // type = vmhd, ISO/IEC 14496-12 section 12.1.2. - 0x00, 0x00, 0x00, 0x01, // version + flags(1) - 0x00, 0x00, 0x00, 0x00, // graphicsmode (copy), opcolor[0] - 0x00, 0x00, 0x00, 0x00, // opcolor[1], opcolor[2] - - // A dinf box suitable for a "self-contained" .mp4 file (no URL/URN - // references to external data). - 0x00, 0x00, 0x00, 0x24, // length == sizeof(kDinfBox) - 'd', 'i', 'n', 'f', // type = dinf, ISO/IEC 14496-12 section 8.7.1. - 0x00, 0x00, 0x00, 0x1c, // length - 'd', 'r', 'e', 'f', // type = dref, ISO/IEC 14496-12 section 8.7.2. - 0x00, 0x00, 0x00, 0x00, // version and flags - 0x00, 0x00, 0x00, 0x01, // entry_count - 0x00, 0x00, 0x00, 0x0c, // length - 'u', 'r', 'l', ' ', // type = url, ISO/IEC 14496-12 section 8.7.2. - 0x00, 0x00, 0x00, 0x01, // version=0, flags=self-contained -}; - -// Likewise, nmhd + dinf boxes, as used for subtitles. -const char kNmhdAndDinfBoxes[] = { - // A nmhd box; the "graphicsmode" and "opcolor" values don't have any - // meaningful use. - 0x00, 0x00, 0x00, 0x0c, // length == sizeof(kNmhdBox) - 'n', 'm', 'h', 'd', // type = vmhd, ISO/IEC 14496-12 section 12.1.2. - 0x00, 0x00, 0x00, 0x01, // version + flags(1) - - // A dinf box suitable for a "self-contained" .mp4 file (no URL/URN - // references to external data). - 0x00, 0x00, 0x00, 0x24, // length == sizeof(kDinfBox) - 'd', 'i', 'n', 'f', // type = dinf, ISO/IEC 14496-12 section 8.7.1. - 0x00, 0x00, 0x00, 0x1c, // length - 'd', 'r', 'e', 'f', // type = dref, ISO/IEC 14496-12 section 8.7.2. - 0x00, 0x00, 0x00, 0x00, // version and flags - 0x00, 0x00, 0x00, 0x01, // entry_count - 0x00, 0x00, 0x00, 0x0c, // length - 'u', 'r', 'l', ' ', // type = url, ISO/IEC 14496-12 section 8.7.2. - 0x00, 0x00, 0x00, 0x01, // version=0, flags=self-contained -}; - -// A hdlr box suitable for a video track. -const char kVideoHdlrBox[] = { - 0x00, 0x00, 0x00, 0x21, // length == sizeof(kHdlrBox) - 'h', 'd', 'l', 'r', // type == hdlr, ISO/IEC 14496-12 section 8.4.3. - 0x00, 0x00, 0x00, 0x00, // version + flags - 0x00, 0x00, 0x00, 0x00, // pre_defined - 'v', 'i', 'd', 'e', // handler = vide - 0x00, 0x00, 0x00, 0x00, // reserved[0] - 0x00, 0x00, 0x00, 0x00, // reserved[1] - 0x00, 0x00, 0x00, 0x00, // reserved[2] - 0x00, // name, zero-terminated (empty) -}; - -// A hdlr box suitable for a subtitle track. -const char kSubtitleHdlrBox[] = { - 0x00, 0x00, 0x00, 0x21, // length == sizeof(kHdlrBox) - 'h', 'd', 'l', 'r', // type == hdlr, ISO/IEC 14496-12 section 8.4.3. - 0x00, 0x00, 0x00, 0x00, // version + flags - 0x00, 0x00, 0x00, 0x00, // pre_defined - 's', 'b', 't', 'l', // handler = sbtl - 0x00, 0x00, 0x00, 0x00, // reserved[0] - 0x00, 0x00, 0x00, 0x00, // reserved[1] - 0x00, 0x00, 0x00, 0x00, // reserved[2] - 0x00, // name, zero-terminated (empty) -}; - -// A stsd box suitable for timestamp subtitles. -const char kSubtitleStsdBox[] = { - 0x00, 0x00, 0x00, 0x54, // length - 's', 't', 's', 'd', // type == stsd, ISO/IEC 14496-12 section 8.5.2. - 0x00, 0x00, 0x00, 0x00, // version + flags - 0x00, 0x00, 0x00, 0x01, // entry_count == 1 - - // SampleEntry, ISO/IEC 14496-12 section 8.5.2.2. - 0x00, 0x00, 0x00, 0x44, // length - 't', 'x', '3', 'g', // type == tx3g, 3GPP TS 26.245 section 5.16. - 0x00, 0x00, 0x00, 0x00, // reserved - 0x00, 0x00, 0x00, 0x01, // reserved, data_reference_index == 1 - - // TextSampleEntry - 0x00, 0x00, 0x00, 0x00, // displayFlags == none - 0x00, // horizontal-justification == left - 0x00, // vertical-justification == top - 0x00, 0x00, 0x00, 0x00, // background-color-rgba == transparent - - // TextSampleEntry.BoxRecord - 0x00, 0x00, // top - 0x00, 0x00, // left - 0x00, 0x00, // bottom - 0x00, 0x00, // right - - // TextSampleEntry.StyleRecord - 0x00, 0x00, // startChar - 0x00, 0x00, // endChar - 0x00, 0x01, // font-ID - 0x00, // face-style-flags - 0x12, // font-size == 18 px - '\xff', '\xff', '\xff', '\xff', // text-color-rgba == opaque white - - // TextSampleEntry.FontTableBox - 0x00, 0x00, 0x00, 0x16, // length - 'f', 't', 'a', 'b', // type == ftab, section 5.16 - 0x00, 0x01, // entry-count == 1 - 0x00, 0x01, // font-ID == 1 - 0x09, // font-name-length == 9 - 'M', 'o', 'n', 'o', 's', 'p', 'a', 'c', 'e'}; - -// Convert from 90kHz units since 1970-01-01 00:00:00 UTC to -// seconds since 1904-01-01 00:00:00 UTC. -uint32_t ToIso14496Timestamp(uint64_t time_90k) { - return time_90k / kTimeUnitsPerSecond + 24107 * 86400; -} - -struct MovieBox { // ISO/IEC 14496-12 section 8.2.1, moov. - uint32_t size = NET_UINT32_C(0); - const char type[4] = {'m', 'o', 'o', 'v'}; -}; - -struct MovieHeaderBoxVersion0 { // ISO/IEC 14496-12 section 8.2.2, mvhd. - uint32_t size = NET_UINT32_C(0); - const char type[4] = {'m', 'v', 'h', 'd'}; - const uint32_t version_and_flags = NET_UINT32_C(0); - uint32_t creation_time = NET_UINT32_C(0); - uint32_t modification_time = NET_UINT32_C(0); - uint32_t timescale = ToNetworkU32(kTimeUnitsPerSecond); - uint32_t duration = NET_UINT32_C(0); - const int32_t rate = NET_UINT32_C(0x00010000); - const int16_t volume = NET_INT16_C(0x0100); - const int16_t reserved = NET_UINT16_C(0); - const uint32_t more_reserved[2] = {NET_UINT32_C(0), NET_UINT32_C(0)}; - const int32_t matrix[9] = { - NET_INT32_C(0x00010000), NET_INT32_C(0), NET_INT32_C(0), NET_INT32_C(0), - NET_INT32_C(0x00010000), NET_INT32_C(0), NET_INT32_C(0), NET_INT32_C(0), - NET_INT32_C(0x40000000)}; - const uint32_t pre_defined[6] = {NET_UINT32_C(0), NET_UINT32_C(0), - NET_UINT32_C(0), NET_UINT32_C(0), - NET_UINT32_C(0), NET_UINT32_C(0)}; - uint32_t next_track_id = NET_UINT32_C(2); -} __attribute__((packed)); - -struct TrackBox { // ISO/IEC 14496-12 section 8.3.1, trak. - uint32_t size = NET_UINT32_C(0); - const char type[4] = {'t', 'r', 'a', 'k'}; -} __attribute__((packed)); - -struct TrackHeaderBoxVersion0 { // ISO/IEC 14496-12 section 8.3.2, tkhd. - uint32_t size = NET_UINT32_C(0); - const char type[4] = {'t', 'k', 'h', 'd'}; - // flags 7 = track_enabled | track_in_movie | track_in_preview - uint32_t version_and_flags = NET_UINT32_C(7); - uint32_t creation_time = NET_UINT32_C(0); - uint32_t modification_time = NET_UINT32_C(0); - uint32_t track_id = NET_UINT32_C(0); - const uint32_t reserved1 = NET_UINT64_C(0); - uint32_t duration = NET_UINT32_C(0); - const uint32_t reserved2[2] = {NET_UINT32_C(0), NET_UINT32_C(0)}; - const uint16_t layer = NET_UINT16_C(0); - const uint16_t alternate_group = NET_UINT16_C(0); - const uint16_t volume = NET_UINT16_C(0); - const uint16_t reserved3 = NET_UINT16_C(0); - int32_t matrix[9] = { - NET_INT32_C(0x00010000), NET_INT32_C(0), NET_INT32_C(0), NET_INT32_C(0), - NET_INT32_C(0x00010000), NET_INT32_C(0), NET_INT32_C(0), NET_INT32_C(0), - NET_INT32_C(0x40000000)}; - uint32_t width = NET_UINT32_C(0); - uint32_t height = NET_UINT32_C(0); -} __attribute__((packed)); - -struct EditBox { // ISO/IEC 14496-12 section 8.6.5, edts. - uint32_t size = NET_UINT32_C(0); - const char type[4] = {'e', 'd', 't', 's'}; -} __attribute__((packed)); - -struct EditListBoxVersion1 { // ISO/IEC 14496-12 section 8.6.6, elst. - uint32_t size = NET_UINT32_C(0); - const char type[4] = {'e', 'l', 's', 't'}; - const uint32_t version_and_flags = NET_UINT32_C(1 << 24); - uint32_t entry_count = NET_UINT32_C(0); -}; - -struct MediaBox { // ISO/IEC 14496-12 section 8.4.1, mdia. - uint32_t size = NET_UINT32_C(0); - const char type[4] = {'m', 'd', 'i', 'a'}; -} __attribute__((packed)); - -struct MediaHeaderBoxVersion0 { // ISO/IEC 14496-12 section 8.4.2, mdhd. - uint32_t size = NET_UINT32_C(0); - const char type[4] = {'m', 'd', 'h', 'd'}; - const uint32_t version_and_flags = NET_UINT32_C(0); - uint32_t creation_time = NET_UINT32_C(0); - uint32_t modification_time = NET_UINT32_C(0); - uint32_t timescale = ToNetworkU32(kTimeUnitsPerSecond); - uint32_t duration = NET_UINT32_C(0); - uint16_t languages = NET_UINT16_C(0x55c4); // undetermined - const uint16_t pre_defined = NET_UINT32_C(0); -} __attribute__((packed)); - -struct MediaInformationBox { // ISO/IEC 14496-12 section 8.4.4, minf. - uint32_t size = NET_UINT32_C(0); - const char type[4] = {'m', 'i', 'n', 'f'}; -} __attribute__((packed)); - -struct SampleTableBox { // ISO/IEC 14496-12 section 8.5.1, stbl. - uint32_t size = NET_UINT32_C(0); - const char type[4] = {'s', 't', 'b', 'l'}; -} __attribute__((packed)); - -struct SampleDescriptionBoxVersion0 { // ISO/IEC 14496-12 section 8.5.2, stsd. - uint32_t size = NET_UINT32_C(0); - const char type[4] = {'s', 't', 's', 'd'}; - const uint32_t version_and_flags = NET_UINT32_C(0 << 24); - uint32_t entry_count = NET_UINT32_C(0); -} __attribute__((packed)); - -struct TimeToSampleBoxVersion0 { // ISO/IEC 14496-12 section 8.6.1.2, stts. - uint32_t size = NET_UINT32_C(0); - const char type[4] = {'s', 't', 't', 's'}; - const uint32_t version_and_flags = NET_UINT32_C(0); - uint32_t entry_count = NET_UINT32_C(0); -} __attribute__((packed)); - -struct SampleToChunkBoxVersion0 { // ISO/IEC 14496-12 section 8.7.4, stsc. - uint32_t size = NET_UINT32_C(0); - const char type[4] = {'s', 't', 's', 'c'}; - const uint32_t version_and_flags = NET_UINT32_C(0); - uint32_t entry_count = NET_UINT32_C(0); -} __attribute__((packed)); - -struct SampleSizeBoxVersion0 { // ISO/IEC 14496-12 section 8.7.3, stsz. - uint32_t size = NET_UINT32_C(0); - const char type[4] = {'s', 't', 's', 'z'}; - const uint32_t version_and_flags = NET_UINT32_C(0); - uint32_t sample_size = NET_UINT32_C(0); - uint32_t sample_count = NET_UINT32_C(0); -} __attribute__((packed)); - -struct ChunkLargeOffsetBoxVersion0 { // ISO/IEC 14496-12 section 8.7.5, co64. - uint32_t size = NET_UINT32_C(0); - const char type[4] = {'c', 'o', '6', '4'}; - const uint32_t version_and_flags = NET_UINT32_C(0); - uint32_t entry_count = NET_UINT32_C(0); -} __attribute__((packed)); - -struct SyncSampleBoxVersion0 { // ISO/IEC 14496-12 section 8.6.2, stss. - uint32_t size = NET_UINT32_C(0); - const char type[4] = {'s', 't', 's', 's'}; - const uint32_t version_and_flags = NET_UINT32_C(0); - uint32_t entry_count = NET_UINT32_C(0); -} __attribute__((packed)); - -struct LargeMediaDataBox { // ISO/IEC 14496-12 section 8.1.1, mdat. - const uint32_t size = NET_UINT32_C(1); - const char type[4] = {'m', 'd', 'a', 't'}; - uint64_t largesize = NET_UINT64_C(0); -}; - -// Grouping of a box's header and the slice representing the header. -// See also ScopedMp4Box, which calculates the length. -template -class Mp4Box { - public: - Mp4Box() - : header_slice_(re2::StringPiece(reinterpret_cast(&header_), - sizeof(header_))) {} - - Header &header() { return header_; } - const FileSlice *header_slice() const { return &header_slice_; } - - private: - Header header_; - StringPieceSlice header_slice_; -}; - -// Helper for adding a mp4 box which calculates the header's size field. -// Construction appends the box to the FileSlices; destruction automatically -// calculates the length including any other slices added in the meantime. -// See also CONSTRUCT_BOX macro. -template -class ScopedMp4Box { - public: - explicit ScopedMp4Box(FileSlices *slices, Box *box) - : starting_size_(slices->size()), slices_(slices), box_(box) { - slices_->Append(box->header_slice()); - } - - ScopedMp4Box(const ScopedMp4Box &) = delete; - void operator=(const ScopedMp4Box &) = delete; - - ~ScopedMp4Box() { - box_->header().size = ToNetwork32(slices_->size() - starting_size_); - } - - private: - int64_t starting_size_; - FileSlices *slices_; - Box *box_; -}; - -// Macro for less verbose ScopedMp4Box instantiation. -// For use only within Mp4File. -#define CONSTRUCT_BOX(box) \ - ScopedMp4Box _scoped_##box(&slices_, &box); - -// .mp4 file, constructed from boxes arranged in the order suggested by -// ISO/IEC 14496-12 section 6.2.3 (see Table 1): -// -// * ftyp (file type and compatibility) -// * moov (container for all the metadata) -// ** mvhd (movie header, overall declarations) -// -// ** trak (video: container for an individual track or stream) -// *** tkhd (track header, overall information about the track) -// *** (optional) edts (edit list container) -// **** elst (an edit list) -// *** mdia (container for the media information in a track) -// **** mdhd (media header, overall information about the media) -// *** minf (media information container) -// **** vmhd (video media header, overall information (video track only)) -// **** dinf (data information box, container) -// ***** dref (data reference box, declares source(s) of media data in track) -// **** stbl (sample table box, container for the time/space map) -// ***** stsd (sample descriptions (codec types, initilization etc.) -// ***** stts ((decoding) time-to-sample) -// ***** stsc (sample-to-chunk, partial data-offset information) -// ***** stsz (samples sizes (framing)) -// ***** co64 (64-bit chunk offset) -// ***** stss (sync sample table) -// -// ** (optional) trak (subtitle: container for an individual track or stream) -// *** tkhd (track header, overall information about the track) -// *** mdia (container for the media information in a track) -// **** mdhd (media header, overall information about the media) -// *** minf (media information container) -// **** nmhd (null media header, overall information) -// **** dinf (data information box, container) -// ***** dref (data reference box, declares source(s) of media data in track) -// **** stbl (sample table box, container for the time/space map) -// ***** stsd (sample descriptions (codec types, initilization etc.) -// ***** stts ((decoding) time-to-sample) -// ***** stsc (sample-to-chunk, partial data-offset information) -// ***** stsz (samples sizes (framing)) -// ***** co64 (64-bit chunk offset) -// -// * mdat (media data container) -class Mp4File : public VirtualFile { - public: - Mp4File(File *sample_file_dir, - std::vector> segments, - VideoSampleEntry &&video_sample_entry, - bool include_timestamp_subtitle_track) - : sample_file_dir_(sample_file_dir), - segments_(std::move(segments)), - video_sample_entry_(std::move(video_sample_entry)), - ftyp_(re2::StringPiece(kFtypBox, sizeof(kFtypBox))), - moov_video_trak_mdia_hdlr_( - re2::StringPiece(kVideoHdlrBox, sizeof(kVideoHdlrBox))), - moov_video_trak_mdia_minf_vmhddinf_( - re2::StringPiece(kVmhdAndDinfBoxes, sizeof(kVmhdAndDinfBoxes))), - moov_video_trak_mdia_minf_stbl_stsd_entry_(video_sample_entry_.data), - moov_subtitle_trak_mdia_hdlr_( - re2::StringPiece(kSubtitleHdlrBox, sizeof(kSubtitleHdlrBox))), - moov_subtitle_trak_mdia_minf_nmhddinf_( - re2::StringPiece(kNmhdAndDinfBoxes, sizeof(kNmhdAndDinfBoxes))), - moov_subtitle_trak_mdia_minf_stbl_stsd_( - re2::StringPiece(kSubtitleStsdBox, sizeof(kSubtitleStsdBox))), - include_timestamp_subtitle_track_(include_timestamp_subtitle_track) { - uint32_t duration = 0; - int64_t max_time_90k = 0; - for (const auto &segment : segments_) { - duration += segment->pieces.end_90k() - segment->rel_start_90k; - int64_t start_90k = - segment->recording.start_time_90k + segment->rel_start_90k; - int64_t end_90k = - segment->recording.start_time_90k + segment->pieces.end_90k(); - int64_t start_ts = start_90k / kTimeUnitsPerSecond; - int64_t end_ts = - (end_90k + kTimeUnitsPerSecond - 1) / kTimeUnitsPerSecond; - num_subtitle_samples_ += end_ts - start_ts; - max_time_90k = std::max(max_time_90k, end_90k); - } - last_modified_ = max_time_90k / kTimeUnitsPerSecond; - auto creation_ts = ToIso14496Timestamp(max_time_90k); - - slices_.Append(&ftyp_); - AppendMoov(ToNetworkU32(duration), ToNetworkU32(creation_ts)); - - // Add the mdat_ without using CONSTRUCT_BOX. - // mdat_ is special because it uses largesize rather than size. - int64_t size_before_mdat = slices_.size(); - slices_.Append(mdat_.header_slice()); - initial_sample_byte_pos_ = slices_.size(); - for (const auto &segment : segments_) { - segment->sample_file_slice.Init( - sample_file_dir_, segment->recording.sample_file_uuid.UnparseText(), - segment->pieces.sample_pos()); - slices_.Append(&segment->sample_file_slice, FileSlices::kLazy); - } - if (include_timestamp_subtitle_track_) { - subtitle_sample_byte_pos_ = slices_.size(); - mdat_subtitle_.Init( - num_subtitle_samples_ * (sizeof(uint16_t) + kSubtitleLength), - [this](std::string *s, std::string *error_message) { - return FillMdatSubtitle(s, error_message); - }); - slices_.Append(&mdat_subtitle_); - } - mdat_.header().largesize = ToNetworkU64(slices_.size() - size_before_mdat); - - auto etag_digest = Digest::SHA1(); - etag_digest->Update( - re2::StringPiece(kFormatVersion, sizeof(kFormatVersion))); - if (include_timestamp_subtitle_track_) { - etag_digest->Update(":ts:"); - } - std::string segment_times; - for (const auto &segment : segments_) { - segment_times.clear(); - Append64(segment->recording.start_time_90k, &segment_times); - Append32(segment->rel_start_90k, &segment_times); - Append32(segment->pieces.end_90k(), &segment_times); - etag_digest->Update(segment_times); - etag_digest->Update(segment->recording.sample_file_sha1); - } - etag_ = StrCat("\"", ToHex(etag_digest->Finalize()), "\""); - VLOG(1) << "Constructed .mp4 has " << slices_.num_slices() << " slices for " - << segments_.size() << " segments, " << slices_.size() << " bytes."; - } - - time_t last_modified() const final { return last_modified_; } - std::string etag() const final { return etag_; } - std::string mime_type() const final { return "video/mp4"; } - int64_t size() const final { return slices_.size(); } - int64_t AddRange(ByteRange range, EvBuffer *buf, - std::string *error_message) const final { - return slices_.AddRange(range, buf, error_message); - } - - private: - void AppendMoov(uint32_t net_duration, uint32_t net_creation_ts) { - CONSTRUCT_BOX(moov_); - { - CONSTRUCT_BOX(moov_mvhd_); - moov_mvhd_.header().creation_time = net_creation_ts; - moov_mvhd_.header().modification_time = net_creation_ts; - moov_mvhd_.header().duration = net_duration; - moov_mvhd_.header().duration = net_duration; - } - { - CONSTRUCT_BOX(moov_video_trak_); - { - CONSTRUCT_BOX(moov_video_trak_tkhd_); - moov_video_trak_tkhd_.header().creation_time = net_creation_ts; - moov_video_trak_tkhd_.header().modification_time = net_creation_ts; - moov_video_trak_tkhd_.header().track_id = NET_UINT32_C(1); - moov_video_trak_tkhd_.header().duration = net_duration; - moov_video_trak_tkhd_.header().width = - NET_UINT32_C(video_sample_entry_.width << 16); - moov_video_trak_tkhd_.header().height = - NET_UINT32_C(video_sample_entry_.height << 16); - } - MaybeAppendVideoEdts(); - { - CONSTRUCT_BOX(moov_video_trak_mdia_); - { - CONSTRUCT_BOX(moov_video_trak_mdia_mdhd_); - moov_video_trak_mdia_mdhd_.header().creation_time = net_creation_ts; - moov_video_trak_mdia_mdhd_.header().modification_time = - net_creation_ts; - moov_video_trak_mdia_mdhd_.header().duration = net_duration; - } - slices_.Append(&moov_video_trak_mdia_hdlr_); - { - CONSTRUCT_BOX(moov_video_trak_mdia_minf_); - slices_.Append(&moov_video_trak_mdia_minf_vmhddinf_); - AppendVideoStbl(); - } - } - } - if (include_timestamp_subtitle_track_) { - AppendSubtitleTrack(net_duration, net_creation_ts); - } - } - - void MaybeAppendVideoEdts() { - struct Entry { - Entry(int64_t segment_duration, int64_t media_time) - : segment_duration(segment_duration), media_time(media_time) {} - int64_t segment_duration = 0; - int64_t media_time = 0; - int64_t end() const { return segment_duration + media_time; } - }; - std::vector entries; - int64_t cur_media_time = 0; - for (const auto &segment : segments_) { - auto skip = segment->rel_start_90k - segment->pieces.start_90k(); - auto keep = segment->pieces.end_90k() - segment->rel_start_90k; - DCHECK_GE(skip, 0); - DCHECK_GT(keep, 0); - cur_media_time += skip; - if (!entries.empty() && entries.back().end() == cur_media_time) { - entries.back().segment_duration += keep; - } else { - entries.emplace_back(keep, cur_media_time); - } - DCHECK_GT(segment->pieces.duration_90k(), 0); - cur_media_time += keep; - } - if (entries.size() == 1 && entries[0].media_time == 0) { - return; // use implicit one-to-one mapping. - } - - VLOG(1) << "Using edit list with " << entries.size() << " entries."; - std::string *s = &moov_video_trak_edts_elst_entries_str_; - for (const auto &entry : entries) { - VLOG(2) << "...duration=" << entry.segment_duration - << ", time=" << entry.media_time; - AppendU64(entry.segment_duration, s); - AppendU64(entry.media_time, s); - AppendU16(1, s); // media_rate_integer - AppendU16(1, s); // media_rate_fraction - } - CONSTRUCT_BOX(moov_video_trak_edts_); - CONSTRUCT_BOX(moov_video_trak_edts_elst_); - moov_video_trak_edts_elst_.header().entry_count = - ToNetworkU32(entries.size()); - moov_video_trak_edts_elst_entries_.Init( - moov_video_trak_edts_elst_entries_str_); - slices_.Append(&moov_video_trak_edts_elst_entries_); - } - - void AppendVideoStbl() { - CONSTRUCT_BOX(moov_video_trak_mdia_minf_stbl_); - { - CONSTRUCT_BOX(moov_video_trak_mdia_minf_stbl_stsd_); - moov_video_trak_mdia_minf_stbl_stsd_.header().entry_count = - NET_UINT32_C(1); - slices_.Append(&moov_video_trak_mdia_minf_stbl_stsd_entry_); - } - { - CONSTRUCT_BOX(moov_video_trak_mdia_minf_stbl_stts_); - int32_t stts_entry_count = 0; - for (const auto &segment : segments_) { - stts_entry_count += segment->pieces.stts_entry_count(); - slices_.Append(segment->pieces.stts_entries()); - } - moov_video_trak_mdia_minf_stbl_stts_.header().entry_count = - ToNetwork32(stts_entry_count); - } - { - CONSTRUCT_BOX(moov_video_trak_mdia_minf_stbl_stsc_); - moov_video_trak_mdia_minf_stbl_stsc_entries_.Init( - 3 * sizeof(uint32_t) * segments_.size(), - [this](std::string *s, std::string *error_message) { - return FillVideoStscEntries(s, error_message); - }); - moov_video_trak_mdia_minf_stbl_stsc_.header().entry_count = - ToNetwork32(segments_.size()); - slices_.Append(&moov_video_trak_mdia_minf_stbl_stsc_entries_); - } - { - CONSTRUCT_BOX(moov_video_trak_mdia_minf_stbl_stsz_); - uint32_t stsz_entry_count = 0; - for (const auto &segment : segments_) { - stsz_entry_count += segment->pieces.stsz_entry_count(); - slices_.Append(segment->pieces.stsz_entries()); - } - moov_video_trak_mdia_minf_stbl_stsz_.header().sample_count = - ToNetwork32(stsz_entry_count); - } - { - CONSTRUCT_BOX(moov_video_trak_mdia_minf_stbl_co64_); - moov_video_trak_mdia_minf_stbl_co64_entries_.Init( - sizeof(uint64_t) * segments_.size(), - [this](std::string *s, std::string *error_message) { - return FillVideoCo64Entries(s, error_message); - }); - moov_video_trak_mdia_minf_stbl_co64_.header().entry_count = - ToNetwork32(segments_.size()); - slices_.Append(&moov_video_trak_mdia_minf_stbl_co64_entries_); - } - { - CONSTRUCT_BOX(moov_video_trak_mdia_minf_stbl_stss_); - uint32_t stss_entry_count = 0; - for (const auto &segment : segments_) { - stss_entry_count += segment->pieces.stss_entry_count(); - slices_.Append(segment->pieces.stss_entries()); - } - moov_video_trak_mdia_minf_stbl_stss_.header().entry_count = - ToNetwork32(stss_entry_count); - } - } - - bool FillVideoStscEntries(std::string *s, std::string *error_message) { - uint32_t chunk = 0; - for (const auto &segment : segments_) { - AppendU32(++chunk, s); - AppendU32(segment->pieces.samples(), s); - AppendU32(1, s); // TODO: sample_description_index. - } - return true; - } - - bool FillVideoCo64Entries(std::string *s, std::string *error_message) { - int64_t pos = initial_sample_byte_pos_; - for (const auto &segment : segments_) { - AppendU64(pos, s); - pos += segment->sample_file_slice.size(); - } - return true; - } - - void AppendSubtitleTrack(uint32_t net_duration, uint32_t net_creation_ts) { - CONSTRUCT_BOX(moov_subtitle_trak_); - { - CONSTRUCT_BOX(moov_subtitle_trak_tkhd_); - auto &hdr = moov_subtitle_trak_tkhd_.header(); - hdr.creation_time = net_creation_ts; - hdr.modification_time = net_creation_ts; - hdr.track_id = NET_UINT32_C(2); - hdr.duration = net_duration; -#if 0 - hdr.width = NET_UINT32_C(800 /*video_sample_entry_.width*/ << 16); - hdr.height = NET_UINT32_C(60 /*video_sample_entry_.height*/ << 16); - hdr.matrix[0] = NET_INT32_C(1 << 16); // a - hdr.matrix[1] = NET_INT32_C(0 << 16); // b - hdr.matrix[2] = NET_INT32_C(0 << 30); // u - hdr.matrix[3] = NET_INT32_C(0 << 16); // c - hdr.matrix[4] = NET_INT32_C(1 << 16); // d - hdr.matrix[5] = NET_INT32_C(0 << 30); // v - hdr.matrix[6] = NET_INT32_C(240 << 16); // x - hdr.matrix[7] = NET_INT32_C(660 << 16); // y - hdr.matrix[8] = NET_INT32_C(1 << 30); // w -#endif - } - { - CONSTRUCT_BOX(moov_subtitle_trak_mdia_); - { - CONSTRUCT_BOX(moov_subtitle_trak_mdia_mdhd_); - moov_subtitle_trak_mdia_mdhd_.header().creation_time = net_creation_ts; - moov_subtitle_trak_mdia_mdhd_.header().modification_time = - net_creation_ts; - moov_subtitle_trak_mdia_mdhd_.header().duration = net_duration; - } - slices_.Append(&moov_subtitle_trak_mdia_hdlr_); - { - CONSTRUCT_BOX(moov_subtitle_trak_mdia_minf_); - slices_.Append(&moov_subtitle_trak_mdia_minf_nmhddinf_); - AppendSubtitleStbl(); - } - } - } - - void AppendSubtitleStbl() { - CONSTRUCT_BOX(moov_subtitle_trak_mdia_minf_stbl_); - slices_.Append(&moov_subtitle_trak_mdia_minf_stbl_stsd_); - { - CONSTRUCT_BOX(moov_subtitle_trak_mdia_minf_stbl_stts_); - int32_t num_entries = 0; - FillSubtitleSttsEntries(&num_entries); - moov_subtitle_trak_mdia_minf_stbl_stts_.header().entry_count = - ToNetwork32(num_entries); - slices_.Append(&moov_subtitle_trak_mdia_minf_stbl_stts_entries_); - } - { - CONSTRUCT_BOX(moov_subtitle_trak_mdia_minf_stbl_stsc_); - moov_subtitle_trak_mdia_minf_stbl_stsc_entries_.Init( - 3 * sizeof(uint32_t), - [this](std::string *s, std::string *error_message) { - AppendU32(1, s); // first_chunk - AppendU32(num_subtitle_samples_, s); // samples_per_chunk - AppendU32(1, s); // sample_description - return true; - }); - moov_subtitle_trak_mdia_minf_stbl_stsc_.header().entry_count = - ToNetwork32(1); - slices_.Append(&moov_subtitle_trak_mdia_minf_stbl_stsc_entries_); - } - { - CONSTRUCT_BOX(moov_subtitle_trak_mdia_minf_stbl_stsz_); - moov_subtitle_trak_mdia_minf_stbl_stsz_.header().sample_size = - ToNetwork32(sizeof(uint16_t) + kSubtitleLength); - moov_subtitle_trak_mdia_minf_stbl_stsz_.header().sample_count = - ToNetwork32(num_subtitle_samples_); - } - { - CONSTRUCT_BOX(moov_subtitle_trak_mdia_minf_stbl_co64_); - moov_subtitle_trak_mdia_minf_stbl_co64_entries_.Init( - sizeof(uint64_t), [this](std::string *s, std::string *error_message) { - AppendU64(subtitle_sample_byte_pos_, s); - return true; - }); - moov_subtitle_trak_mdia_minf_stbl_co64_.header().entry_count = - ToNetwork32(1); - slices_.Append(&moov_subtitle_trak_mdia_minf_stbl_co64_entries_); - } - } - - // Fills |moov_subtitle_trak_mdia_minf_stbl_stts_entries_| and puts - // the number of STTS entries into |num_entries| (in host byte order). - void FillSubtitleSttsEntries(int32_t *num_entries) { - std::string &s = moov_subtitle_trak_mdia_minf_stbl_stts_entries_str_; - for (const auto &segment : segments_) { - int64_t start_90k = - segment->recording.start_time_90k + segment->rel_start_90k; - int64_t end_90k = - segment->recording.start_time_90k + segment->pieces.end_90k(); - int64_t start_next_90k = - start_90k + kTimeUnitsPerSecond - (start_90k % kTimeUnitsPerSecond); - - if (end_90k <= start_next_90k) { - ++*num_entries; - AppendU32(1, &s); // sample_count - AppendU32(end_90k - start_90k, &s); // sample_duration - } else { - ++*num_entries; - AppendU32(1, &s); // sample_count - AppendU32(start_next_90k - start_90k, &s); // sample_duration - - int64_t end_prev_90k = end_90k - (end_90k % kTimeUnitsPerSecond); - if (start_next_90k < end_prev_90k) { - ++*num_entries; - int64_t interior = - (end_prev_90k - start_next_90k) / kTimeUnitsPerSecond; - AppendU32(interior, &s); // sample_count - AppendU32(kTimeUnitsPerSecond, &s); - } - - ++*num_entries; - AppendU32(1, &s); // sample_count - AppendU32(end_90k - end_prev_90k, &s); // sample_duration - } - } - moov_subtitle_trak_mdia_minf_stbl_stts_entries_.Init( - moov_subtitle_trak_mdia_minf_stbl_stts_entries_str_); - } - - bool FillMdatSubtitle(std::string *s, std::string *error_message) { - char buf[kSubtitleLength + 1 /* null */]; - struct tm mytm; - memset(&mytm, 0, sizeof(mytm)); - for (const auto &segment : segments_) { - int64_t start_90k = - segment->recording.start_time_90k + segment->rel_start_90k; - int64_t end_90k = - segment->recording.start_time_90k + segment->pieces.end_90k(); - int64_t start_ts = start_90k / kTimeUnitsPerSecond; - int64_t end_ts = - (end_90k + kTimeUnitsPerSecond - 1) / kTimeUnitsPerSecond; - for (time_t ts = start_ts; ts < end_ts; ++ts) { - AppendU16(kSubtitleLength, s); - localtime_r(&ts, &mytm); - size_t r = strftime(buf, sizeof(buf), kSubtitleTemplate, &mytm); - if (r != kSubtitleLength) { - *error_message = StrCat("strftime unexpectedly returned ", r); - return false; - } - s->append(buf, r); - } - } - return true; - } - - int64_t initial_sample_byte_pos_ = 0; - int64_t subtitle_sample_byte_pos_ = 0; - int64_t num_subtitle_samples_ = 0; - File *sample_file_dir_ = nullptr; - std::vector> segments_; - VideoSampleEntry video_sample_entry_; - FileSlices slices_; - std::string etag_; - time_t last_modified_ = -1; - - StringPieceSlice ftyp_; - Mp4Box moov_; - Mp4Box moov_mvhd_; - - Mp4Box moov_video_trak_; - Mp4Box moov_video_trak_tkhd_; - Mp4Box moov_video_trak_edts_; - Mp4Box moov_video_trak_edts_elst_; - StringPieceSlice moov_video_trak_edts_elst_entries_; - std::string moov_video_trak_edts_elst_entries_str_; - Mp4Box moov_video_trak_mdia_; - Mp4Box moov_video_trak_mdia_mdhd_; - StringPieceSlice moov_video_trak_mdia_hdlr_; - Mp4Box moov_video_trak_mdia_minf_; - StringPieceSlice moov_video_trak_mdia_minf_vmhddinf_; - Mp4Box moov_video_trak_mdia_minf_stbl_; - Mp4Box moov_video_trak_mdia_minf_stbl_stsd_; - StringPieceSlice moov_video_trak_mdia_minf_stbl_stsd_entry_; - Mp4Box moov_video_trak_mdia_minf_stbl_stts_; - Mp4Box moov_video_trak_mdia_minf_stbl_stsc_; - FillerFileSlice moov_video_trak_mdia_minf_stbl_stsc_entries_; - Mp4Box moov_video_trak_mdia_minf_stbl_stsz_; - Mp4Box moov_video_trak_mdia_minf_stbl_co64_; - FillerFileSlice moov_video_trak_mdia_minf_stbl_co64_entries_; - Mp4Box moov_video_trak_mdia_minf_stbl_stss_; - - Mp4Box moov_subtitle_trak_; - Mp4Box moov_subtitle_trak_tkhd_; - Mp4Box moov_subtitle_trak_mdia_; - Mp4Box moov_subtitle_trak_mdia_mdhd_; - StringPieceSlice moov_subtitle_trak_mdia_hdlr_; - Mp4Box moov_subtitle_trak_mdia_minf_; - StringPieceSlice moov_subtitle_trak_mdia_minf_nmhddinf_; - Mp4Box moov_subtitle_trak_mdia_minf_stbl_; - StringPieceSlice moov_subtitle_trak_mdia_minf_stbl_stsd_; - Mp4Box moov_subtitle_trak_mdia_minf_stbl_stts_; - StringPieceSlice moov_subtitle_trak_mdia_minf_stbl_stts_entries_; - std::string moov_subtitle_trak_mdia_minf_stbl_stts_entries_str_; - Mp4Box moov_subtitle_trak_mdia_minf_stbl_stsc_; - FillerFileSlice moov_subtitle_trak_mdia_minf_stbl_stsc_entries_; - Mp4Box moov_subtitle_trak_mdia_minf_stbl_stsz_; - Mp4Box moov_subtitle_trak_mdia_minf_stbl_co64_; - FillerFileSlice moov_subtitle_trak_mdia_minf_stbl_co64_entries_; - Mp4Box moov_subtitle_trak_mdia_minf_stbl_stss_; - FillerFileSlice mdat_subtitle_; - - Mp4Box mdat_; - - bool include_timestamp_subtitle_track_ = false; -}; - -#undef CONSTRUCT_BOX - -} // namespace - -namespace internal { - -bool Mp4SampleTablePieces::Init(const Recording *recording, - int sample_entry_index, int32_t sample_offset, - int32_t start_90k, int32_t end_90k, - std::string *error_message) { - sample_entry_index_ = sample_entry_index; - sample_offset_ = sample_offset; - desired_end_90k_ = end_90k; - SampleIndexIterator it = SampleIndexIterator(recording->video_index); - auto recording_duration_90k = - recording->end_time_90k - recording->start_time_90k; - bool fast_path = start_90k == 0 && end_90k >= recording_duration_90k; - if (fast_path) { - VLOG(1) << "Fast path, frames=" << recording->video_samples - << ", key=" << recording->video_sync_samples; - sample_pos_.end = recording->sample_file_bytes; - begin_ = it; - frames_ = recording->video_samples; - key_frames_ = recording->video_sync_samples; - actual_end_90k_ = recording_duration_90k; - } else { - VLOG(1) << "Slow path."; - if (!it.done() && !it.is_key()) { - *error_message = "First frame must be a key frame."; - return false; - } - for (; !it.done(); it.Next()) { - VLOG(3) << "Processing frame with start " << it.start_90k() - << (it.is_key() ? " (key)" : " (non-key)"); - // Find boundaries. - if (it.start_90k() <= start_90k && it.is_key()) { - VLOG(3) << "...new start candidate."; - begin_ = it; - sample_pos_.begin = begin_.pos(); - frames_ = 0; - key_frames_ = 0; - } - if (it.start_90k() >= end_90k) { - VLOG(3) << "...past end."; - break; - } - - // Process this frame. - frames_++; - if (it.is_key()) { - key_frames_++; - } - - // This is the current best candidate to end. - actual_end_90k_ = it.end_90k(); - } - sample_pos_.end = it.pos(); - } - if (it.has_error()) { - *error_message = it.error(); - return false; - } - actual_end_90k_ = std::min(actual_end_90k_, desired_end_90k_); - VLOG(1) << "requested ts [" << start_90k << ", " << end_90k << "), got ts [" - << begin_.start_90k() << ", " << actual_end_90k_ << "), " << frames_ - << " frames (" << key_frames_ - << " key), byte positions: " << sample_pos_; - - stts_entries_.Init(2 * sizeof(int32_t) * stts_entry_count(), - [this](std::string *s, std::string *error_message) { - return FillSttsEntries(s, error_message); - }); - stss_entries_.Init(sizeof(int32_t) * stss_entry_count(), - [this](std::string *s, std::string *error_message) { - return FillStssEntries(s, error_message); - }); - stsz_entries_.Init(sizeof(int32_t) * stsz_entry_count(), - [this](std::string *s, std::string *error_message) { - return FillStszEntries(s, error_message); - }); - return true; -} - -bool Mp4SampleTablePieces::FillSttsEntries(std::string *s, - std::string *error_message) const { - SampleIndexIterator it; - for (it = begin_; !it.done() && it.start_90k() < desired_end_90k_; - it.Next()) { - AppendU32(1, s); - - // The final sample may be shortened to the desired end. - if (it.end_90k() > desired_end_90k_) { - auto new_duration = desired_end_90k_ - it.start_90k(); - VLOG(1) << "Shortening final sample duration from " << it.duration_90k() - << " to " << new_duration; - AppendU32(new_duration, s); - break; - } else { - AppendU32(it.duration_90k(), s); - } - } - if (it.has_error()) { - *error_message = it.error(); - return false; - } - return true; -} - -bool Mp4SampleTablePieces::FillStssEntries(std::string *s, - std::string *error_message) const { - SampleIndexIterator it; - uint32_t sample_num = sample_offset_; - for (it = begin_; !it.done() && it.start_90k() < desired_end_90k_; - it.Next()) { - if (it.is_key()) { - Append32(sample_num, s); - } - sample_num++; - } - if (it.has_error()) { - *error_message = it.error(); - return false; - } - return true; -} - -bool Mp4SampleTablePieces::FillStscEntries(std::string *s, - std::string *error_message) const { - Append32(sample_offset_, s); - Append32(frames_, s); - Append32(sample_entry_index_, s); - return true; -} - -bool Mp4SampleTablePieces::FillStszEntries(std::string *s, - std::string *error_message) const { - SampleIndexIterator it; - for (it = begin_; !it.done() && it.start_90k() < desired_end_90k_; - it.Next()) { - Append32(it.bytes(), s); - } - if (it.has_error()) { - *error_message = it.error(); - return false; - } - return true; -} - -} // namespace internal - -Mp4FileBuilder &Mp4FileBuilder::Append(Recording &&recording, - int32_t rel_start_90k, - int32_t rel_end_90k) { - std::unique_ptr s(new Mp4FileSegment); - s->recording = std::move(recording); - s->rel_start_90k = rel_start_90k; - s->rel_end_90k = rel_end_90k; - segments_.push_back(std::move(s)); - return *this; -} - -Mp4FileBuilder &Mp4FileBuilder::SetSampleEntry(const VideoSampleEntry &entry) { - video_sample_entry_ = entry; - return *this; -} - -std::shared_ptr Mp4FileBuilder::Build(std::string *error_message) { - int32_t sample_offset = 1; - for (auto &segment : segments_) { - if (segment->recording.video_sample_entry_id != video_sample_entry_.id) { - *error_message = StrCat( - "inconsistent video sample entries. builder has: ", - video_sample_entry_.id, " (sha1 ", ToHex(video_sample_entry_.sha1), - ", segment has: ", segment->recording.video_sample_entry_id); - return std::shared_ptr(); - } - - if (!segment->pieces.Init(&segment->recording, - 1, // sample entry index - sample_offset, segment->rel_start_90k, - segment->rel_end_90k, error_message)) { - return std::shared_ptr(); - } - sample_offset += segment->pieces.samples(); - } - - if (segments_.empty()) { - *error_message = "Can't construct empty .mp4"; - return std::shared_ptr(); - } - - return std::shared_ptr(new Mp4File( - sample_file_dir_, std::move(segments_), std::move(video_sample_entry_), - include_timestamp_subtitle_track_)); -} - -} // namespace moonfire_nvr diff --git a/src/mp4.h b/src/mp4.h deleted file mode 100644 index 742e104..0000000 --- a/src/mp4.h +++ /dev/null @@ -1,188 +0,0 @@ -// This file is part of Moonfire NVR, a security camera network video recorder. -// Copyright (C) 2016 Scott Lamb -// -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// In addition, as a special exception, the copyright holders give -// permission to link the code of portions of this program with the -// OpenSSL library under certain conditions as described in each -// individual source file, and distribute linked combinations including -// the two. -// -// You must obey the GNU General Public License in all respects for all -// of the code used other than OpenSSL. If you modify file(s) with this -// exception, you may extend this exception to your version of the -// file(s), but you are not obligated to do so. If you do not wish to do -// so, delete this exception statement from your version. If you delete -// this exception statement from all source files in the program, then -// also delete it here. -// -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. -// -// You should have received a copy of the GNU General Public License -// along with this program. If not, see . -// -// mp4.h: interface for building VirtualFiles representing ISO/IEC 14496-12 -// (ISO base media format / MPEG-4 / .mp4) video. These can be constructed -// from one or more recordings and are suitable for HTTP range serving or -// download. - -#ifndef MOONFIRE_NVR_MP4_H -#define MOONFIRE_NVR_MP4_H - -#include -#include -#include - -#include "recording.h" -#include "http.h" - -namespace moonfire_nvr { - -namespace internal { - -// Represents pieces of .mp4 sample tables for one recording. Many recordings, -// and thus many of these objects, may be spliced together into a single -// virtual .mp4 file. For internal use by Mp4FileBuilder. Exposed for testing. -class Mp4SampleTablePieces { - public: - Mp4SampleTablePieces() {} - Mp4SampleTablePieces(const Mp4SampleTablePieces &) = delete; - void operator=(const Mp4SampleTablePieces &) = delete; - - // |recording| must outlive the Mp4SampleTablePieces. - // - // |sample_entry_index| should be the (1-based) index into the "stsd" box - // of an entry matching this recording's video_sample_entry_sha1. It may - // be shared with other recordings. - // - // |sample_offset| should be the (1-based) index of the first sample in - // this file. It should be 1 + the sum of all previous Mp4SampleTablePieces' - // samples() values. - // - // |start_90k| and |end_90k| should be relative to the start of the recording. - // They indicate the *desired* time range. The *actual* time range will - // start at the last sync sample <= |start_90k|. (The caller is responsible - // for creating an edit list to skip the undesired portion.) It will end at - // the desired range, or the end of the recording, whichever is sooner. - bool Init(const Recording *recording, int sample_entry_index, - int32_t sample_offset, int32_t start_90k, int32_t end_90k, - std::string *error_message); - - int32_t stts_entry_count() const { return frames_; } - const FileSlice *stts_entries() const { return &stts_entries_; } - - int32_t stss_entry_count() const { return key_frames_; } - const FileSlice *stss_entries() const { return &stss_entries_; } - - int32_t stsz_entry_count() const { return frames_; } - const FileSlice *stsz_entries() const { return &stsz_entries_; } - - int32_t samples() const { return frames_; } - - // Return the byte range in the sample file of the frames represented here. - ByteRange sample_pos() const { return sample_pos_; } - - // As described above, these may differ from the desired range. - uint64_t duration_90k() const { return actual_end_90k_ - begin_.start_90k(); } - int32_t start_90k() const { return begin_.start_90k(); } - int32_t end_90k() const { return actual_end_90k_; } - - private: - bool FillSttsEntries(std::string *s, std::string *error_message) const; - bool FillStssEntries(std::string *s, std::string *error_message) const; - bool FillStscEntries(std::string *s, std::string *error_message) const; - bool FillStszEntries(std::string *s, std::string *error_message) const; - - // After Init(), |begin_| will be on the first sample after the start of the - // range (or it will be done()). - SampleIndexIterator begin_; - - ByteRange sample_pos_; - - FillerFileSlice stts_entries_; - FillerFileSlice stss_entries_; - FillerFileSlice stsz_entries_; - - int sample_entry_index_ = -1; - int32_t sample_offset_ = -1; - int32_t desired_end_90k_ = -1; - int32_t actual_end_90k_ = -1; - int32_t frames_ = 0; - int32_t key_frames_ = 0; -}; - -struct Mp4FileSegment { - Recording recording; - Mp4SampleTablePieces pieces; - RealFileSlice sample_file_slice; - - // Requested start time, relative to recording.start_90k. - // If there is no key frame at exactly this position, |pieces| will actually - // start sooner, and an edit list should be used to skip the undesired - // prefix. - int32_t rel_start_90k = 0; - - // Requested end time, relative to recording.end_90k. - // This will be clamped to the actual duration of the recording. - int32_t rel_end_90k = std::numeric_limits::max(); -}; - -} // namespace internal - -// Builder for a virtual .mp4 file. -class Mp4FileBuilder { - public: - // |sample_file_dir| must outlive the Mp4FileBuilder and the returned - // VirtualFile. - explicit Mp4FileBuilder(File *sample_file_dir) - : sample_file_dir_(sample_file_dir) {} - Mp4FileBuilder(const Mp4FileBuilder &) = delete; - void operator=(const Mp4FileBuilder &) = delete; - - // Append part or all of a recording. - // Note that |recording.video_sample_entry_sha1| must be added via - // AddSampleEntry. - Mp4FileBuilder &Append(Recording &&recording, int32_t rel_start_300ths, - int32_t rel_end_300ths); - - // TODO: support multiple sample entries? - Mp4FileBuilder &SetSampleEntry(const VideoSampleEntry &entry); - - // Set if a subtitle track should be added with timestamps. - Mp4FileBuilder &include_timestamp_subtitle_track(bool v) { - include_timestamp_subtitle_track_ = v; - return *this; - } - - // TODO: MPEG-DASH / ISO BMFF Byte Stream Format support. - - // Build the .mp4 file, returning it to the caller. - // The Mp4FileBuilder is left in an undefined state; it should not - // be used afterward. On error, nullptr is returned, with |error_message| - // populated. - // - // Errors include: - // * TODO: width/height mismatch? or is this okay? - // * No segments. - // * Non-final segment has zero duration of last sample. - // * Data error in one of the recording sample indexes. - // * Invalid start/end. - std::shared_ptr Build(std::string *error_message); - - private: - File *sample_file_dir_; - std::vector> segments_; - VideoSampleEntry video_sample_entry_; - bool include_timestamp_subtitle_track_ = false; -}; - -} // namespace moonfire_nvr - -#endif // MOONFIRE_NVR_MP4_H diff --git a/src/mp4.rs b/src/mp4.rs new file mode 100644 index 0000000..014e0db --- /dev/null +++ b/src/mp4.rs @@ -0,0 +1,1534 @@ +// This file is part of Moonfire NVR, a security camera digital video recorder. +// Copyright (C) 2016 Scott Lamb +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// In addition, as a special exception, the copyright holders give +// permission to link the code of portions of this program with the +// OpenSSL library under certain conditions as described in each +// individual source file, and distribute linked combinations including +// the two. +// +// You must obey the GNU General Public License in all respects for all +// of the code used other than OpenSSL. If you modify file(s) with this +// exception, you may extend this exception to your version of the +// file(s), but you are not obligated to do so. If you do not wish to do +// so, delete this exception statement from your version. If you delete +// this exception statement from all source files in the program, then +// also delete it here. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program. If not, see . + +//! `.mp4` virtual file serving. +//! +//! The `mp4` module builds virtual files representing ISO/IEC 14496-12 (ISO base media format / +//! MPEG-4 / `.mp4`) video. These can be constructed from one or more recordings and are suitable +//! for HTTP range serving or download. + +extern crate byteorder; +extern crate time; + +use alloc::raw_vec::RawVec; +use byteorder::{BigEndian, ByteOrder, WriteBytesExt}; +use db::{Database, ListCameraRecordingsRow, VideoSampleEntry}; +use dir; +use error::{Error, Result}; +use hyper::header; +use mmapfile; +use mime; +use openssl::crypto::hash; +use pieces; +use pieces::ContextWriter; +use pieces::Slices; +use recording::{self, TIME_UNITS_PER_SEC}; +use resource; +use smallvec::SmallVec; +use std::cell::RefCell; +use std::cmp; +use std::io; +use std::ops::Range; +use std::mem; +use std::sync::Arc; +use time::Timespec; + +/// This value should be incremented any time a change is made to this file that causes different +/// bytes to be output for a particular set of `Mp4Builder` options. Incrementing this value will +/// cause the etag to change as well. +const FORMAT_VERSION: [u8; 1] = [0x02]; + +/// An `ftyp` (ISO/IEC 14496-12 section 4.3 `FileType`) box. +const FTYP_BOX: &'static [u8] = &[ + 0x00, 0x00, 0x00, 0x20, // length = 32, sizeof(FTYP_BOX) + b'f', b't', b'y', b'p', // type + b'i', b's', b'o', b'm', // major_brand + 0x00, 0x00, 0x02, 0x00, // minor_version + b'i', b's', b'o', b'm', // compatible_brands[0] + b'i', b's', b'o', b'2', // compatible_brands[1] + b'a', b'v', b'c', b'1', // compatible_brands[2] + b'm', b'p', b'4', b'1', // compatible_brands[3] +]; + +/// An `hdlr` (ISO/IEC 14496-12 section 8.4.3 `HandlerBox`) box suitable for a video track. +const VIDEO_HDLR_BOX: &'static [u8] = &[ + 0x00, 0x00, 0x00, 0x21, // length == sizeof(kHdlrBox) + b'h', b'd', b'l', b'r', // type == hdlr, ISO/IEC 14496-12 section 8.4.3. + 0x00, 0x00, 0x00, 0x00, // version + flags + 0x00, 0x00, 0x00, 0x00, // pre_defined + b'v', b'i', b'd', b'e', // handler = vide + 0x00, 0x00, 0x00, 0x00, // reserved[0] + 0x00, 0x00, 0x00, 0x00, // reserved[1] + 0x00, 0x00, 0x00, 0x00, // reserved[2] + 0x00, // name, zero-terminated (empty) +]; + +/// An `hdlr` (ISO/IEC 14496-12 section 8.4.3 `HandlerBox`) box suitable for a subtitle track. +const SUBTITLE_HDLR_BOX: &'static [u8] = &[ + 0x00, 0x00, 0x00, 0x21, // length == sizeof(kHdlrBox) + b'h', b'd', b'l', b'r', // type == hdlr, ISO/IEC 14496-12 section 8.4.3. + 0x00, 0x00, 0x00, 0x00, // version + flags + 0x00, 0x00, 0x00, 0x00, // pre_defined + b's', b'b', b't', b'l', // handler = sbtl + 0x00, 0x00, 0x00, 0x00, // reserved[0] + 0x00, 0x00, 0x00, 0x00, // reserved[1] + 0x00, 0x00, 0x00, 0x00, // reserved[2] + 0x00, // name, zero-terminated (empty) +]; + +/// Part of an `mvhd` (`MovieHeaderBox` version 0, ISO/IEC 14496-12 section 8.2.2), used from +/// `append_mvhd`. +const MVHD_JUNK: &'static [u8] = &[ + 0x00, 0x01, 0x00, 0x00, // rate + 0x01, 0x00, // volume + 0x00, 0x00, // reserved + 0x00, 0x00, 0x00, 0x00, // reserved + 0x00, 0x00, 0x00, 0x00, // reserved + 0x00, 0x01, 0x00, 0x00, // matrix[0] + 0x00, 0x00, 0x00, 0x00, // matrix[1] + 0x00, 0x00, 0x00, 0x00, // matrix[2] + 0x00, 0x00, 0x00, 0x00, // matrix[3] + 0x00, 0x01, 0x00, 0x00, // matrix[4] + 0x00, 0x00, 0x00, 0x00, // matrix[5] + 0x00, 0x00, 0x00, 0x00, // matrix[6] + 0x00, 0x00, 0x00, 0x00, // matrix[7] + 0x40, 0x00, 0x00, 0x00, // matrix[8] + 0x00, 0x00, 0x00, 0x00, // pre_defined[0] + 0x00, 0x00, 0x00, 0x00, // pre_defined[1] + 0x00, 0x00, 0x00, 0x00, // pre_defined[2] + 0x00, 0x00, 0x00, 0x00, // pre_defined[3] + 0x00, 0x00, 0x00, 0x00, // pre_defined[4] + 0x00, 0x00, 0x00, 0x00, // pre_defined[5] +]; + +/// Part of a `tkhd` (`TrackHeaderBox` version 0, ISO/IEC 14496-12 section 8.3.2), used from +/// `append_video_tkhd` and `append_subtitle_tkhd`. +const TKHD_JUNK: &'static [u8] = &[ + 0x00, 0x00, 0x00, 0x00, // reserved + 0x00, 0x00, 0x00, 0x00, // reserved + 0x00, 0x00, 0x00, 0x00, // layer + alternate_group + 0x00, 0x00, 0x00, 0x00, // volume + reserved + 0x00, 0x01, 0x00, 0x00, // matrix[0] + 0x00, 0x00, 0x00, 0x00, // matrix[1] + 0x00, 0x00, 0x00, 0x00, // matrix[2] + 0x00, 0x00, 0x00, 0x00, // matrix[3] + 0x00, 0x01, 0x00, 0x00, // matrix[4] + 0x00, 0x00, 0x00, 0x00, // matrix[5] + 0x00, 0x00, 0x00, 0x00, // matrix[6] + 0x00, 0x00, 0x00, 0x00, // matrix[7] + 0x40, 0x00, 0x00, 0x00, // matrix[8] +]; + +/// Part of a `minf` (`MediaInformationBox`, ISO/IEC 14496-12 section 8.4.4), used from +/// `append_video_minf`. +const VIDEO_MINF_JUNK: &'static [u8] = &[ + b'm', b'i', b'n', b'f', // type = minf, ISO/IEC 14496-12 section 8.4.4. + // A vmhd box; the "graphicsmode" and "opcolor" values don't have any + // meaningful use. + 0x00, 0x00, 0x00, 0x14, // length == sizeof(kVmhdBox) + b'v', b'm', b'h', b'd', // type = vmhd, ISO/IEC 14496-12 section 12.1.2. + 0x00, 0x00, 0x00, 0x01, // version + flags(1) + 0x00, 0x00, 0x00, 0x00, // graphicsmode (copy), opcolor[0] + 0x00, 0x00, 0x00, 0x00, // opcolor[1], opcolor[2] + + // A dinf box suitable for a "self-contained" .mp4 file (no URL/URN + // references to external data). + 0x00, 0x00, 0x00, 0x24, // length == sizeof(kDinfBox) + b'd', b'i', b'n', b'f', // type = dinf, ISO/IEC 14496-12 section 8.7.1. + 0x00, 0x00, 0x00, 0x1c, // length + b'd', b'r', b'e', b'f', // type = dref, ISO/IEC 14496-12 section 8.7.2. + 0x00, 0x00, 0x00, 0x00, // version and flags + 0x00, 0x00, 0x00, 0x01, // entry_count + 0x00, 0x00, 0x00, 0x0c, // length + b'u', b'r', b'l', b' ', // type = url, ISO/IEC 14496-12 section 8.7.2. + 0x00, 0x00, 0x00, 0x01, // version=0, flags=self-contained +]; + +/// Part of a `minf` (`MediaInformationBox`, ISO/IEC 14496-12 section 8.4.4), used from +/// `append_subtitle_minf`. +const SUBTITLE_MINF_JUNK: &'static [u8] = &[ + b'm', b'i', b'n', b'f', // type = minf, ISO/IEC 14496-12 section 8.4.4. + // A nmhd box. + 0x00, 0x00, 0x00, 0x0c, // length == sizeof(kNmhdBox) + b'n', b'm', b'h', b'd', // type = nmhd, ISO/IEC 14496-12 section 12.1.2. + 0x00, 0x00, 0x00, 0x01, // version + flags(1) + + // A dinf box suitable for a "self-contained" .mp4 file (no URL/URN + // references to external data). + 0x00, 0x00, 0x00, 0x24, // length == sizeof(kDinfBox) + b'd', b'i', b'n', b'f', // type = dinf, ISO/IEC 14496-12 section 8.7.1. + 0x00, 0x00, 0x00, 0x1c, // length + b'd', b'r', b'e', b'f', // type = dref, ISO/IEC 14496-12 section 8.7.2. + 0x00, 0x00, 0x00, 0x00, // version and flags + 0x00, 0x00, 0x00, 0x01, // entry_count + 0x00, 0x00, 0x00, 0x0c, // length + b'u', b'r', b'l', b' ', // type = url, ISO/IEC 14496-12 section 8.7.2. + 0x00, 0x00, 0x00, 0x01, // version=0, flags=self-contained +]; + +/// Part of a `stbl` (`SampleTableBox`, ISO/IEC 14496 section 8.5.1) used from +/// `append_subtitle_stbl`. +const SUBTITLE_STBL_JUNK: &'static [u8] = &[ + b's', b't', b'b', b'l', // type = stbl, ISO/IEC 14496-12 section 8.5.1. + + // A stsd box. + 0x00, 0x00, 0x00, 0x54, // length + b's', b't', b's', b'd', // type == stsd, ISO/IEC 14496-12 section 8.5.2. + 0x00, 0x00, 0x00, 0x00, // version + flags + 0x00, 0x00, 0x00, 0x01, // entry_count == 1 + + // SampleEntry, ISO/IEC 14496-12 section 8.5.2.2. + 0x00, 0x00, 0x00, 0x44, // length + b't', b'x', b'3', b'g', // type == tx3g, 3GPP TS 26.245 section 5.16. + 0x00, 0x00, 0x00, 0x00, // reserved + 0x00, 0x00, 0x00, 0x01, // reserved, data_reference_index == 1 + + // TextSampleEntry + 0x00, 0x00, 0x00, 0x00, // displayFlags == none + 0x00, // horizontal-justification == left + 0x00, // vertical-justification == top + 0x00, 0x00, 0x00, 0x00, // background-color-rgba == transparent + + // TextSampleEntry.BoxRecord + 0x00, 0x00, // top + 0x00, 0x00, // left + 0x00, 0x00, // bottom + 0x00, 0x00, // right + + // TextSampleEntry.StyleRecord + 0x00, 0x00, // startChar + 0x00, 0x00, // endChar + 0x00, 0x01, // font-ID + 0x00, // face-style-flags + 0x12, // font-size == 18 px + 0xff, 0xff, 0xff, 0xff, // text-color-rgba == opaque white + + // TextSampleEntry.FontTableBox + 0x00, 0x00, 0x00, 0x16, // length + b'f', b't', b'a', b'b', // type == ftab, section 5.16 + 0x00, 0x01, // entry-count == 1 + 0x00, 0x01, // font-ID == 1 + 0x09, // font-name-length == 9 + b'M', b'o', b'n', b'o', b's', b'p', b'a', b'c', b'e', +]; + +/// Pointers to each static bytestrings. +/// The order here must match the `StaticBytestring` enum. +const STATIC_BYTESTRINGS: [&'static [u8]; 8] = [ + FTYP_BOX, + VIDEO_HDLR_BOX, + SUBTITLE_HDLR_BOX, + MVHD_JUNK, + TKHD_JUNK, + VIDEO_MINF_JUNK, + SUBTITLE_MINF_JUNK, + SUBTITLE_STBL_JUNK, +]; + +/// Enumeration of the static bytestrings. The order here must match the `STATIC_BYTESTRINGS` +/// array. The advantage of this enum over direct pointers to the relevant strings is that it +/// fits into a u32 on 64-bit platforms, allowing an `Mp4FileSlice` to fit into 8 bytes. +#[derive(Copy, Clone, Debug)] +enum StaticBytestring { + FtypBox, + VideoHdlrBox, + SubtitleHdlrBox, + MvhdJunk, + TkhdJunk, + VideoMinfJunk, + SubtitleMinfJunk, + SubtitleStblJunk, +} + +/// The template fed into strtime for a timestamp subtitle. This must produce fixed-length output +/// (see `SUBTITLE_LENGTH`) to allow quick calculation of the total size of the subtitles for +/// a given time range. +const SUBTITLE_TEMPLATE: &'static str = "%Y-%m-%d %H:%M:%S %z"; + +/// The length of the output of `SUBTITLE_TEMPLATE`. +const SUBTITLE_LENGTH: usize = 25; // "2015-07-02 17:10:00 -0700".len(); + +/// Holds the sample indexes for a given video segment: `stts`, `stsz`, and `stss`. +struct Mp4SegmentIndex { + /// Holds all three sample indexes: + /// &buf[.. stsz_start] is stts. + /// &buf[stsz_start .. stss_start] is stsz. + /// &buf[stss_start ..] is stss. + buf: Box<[u8]>, + stsz_start: usize, + stss_start: usize, +} + +impl Mp4SegmentIndex { + fn stts(&self) -> &[u8] { &self.buf[.. self.stsz_start] } + fn stsz(&self) -> &[u8] { &self.buf[self.stsz_start .. self.stss_start] } + fn stss(&self) -> &[u8] { &self.buf[self.stss_start ..] } +} + +struct Mp4Segment { + s: recording::Segment, + + /// Holds the `stts`, `stsz`, and `stss` if they've been generated. + /// Access only through `with_index`. + index: RefCell>, + + /// The 1-indexed frame number in the `Mp4File` of the first frame in this segment. + first_frame_num: u32, + num_subtitle_samples: u32, +} + +impl Mp4Segment { + fn with_index(&self, db: &Database, f: F) -> Result + where F: FnOnce(&Mp4SegmentIndex) -> Result { + let mut i = self.index.borrow_mut(); + if let Some(ref i) = *i { + return f(i); + } + let index = self.build_index(db)?; + let r = f(&index); + *i = Some(index); + r + } + + fn build_index(&self, db: &Database) -> Result { + let s = &self.s; + let stts_len = mem::size_of::() * 2 * (s.frames as usize); + let stsz_len = mem::size_of::() * s.frames as usize; + let stss_len = mem::size_of::() * s.key_frames as usize; + let len = stts_len + stsz_len + stss_len; + let mut buf = unsafe { RawVec::with_capacity(len).into_box() }; + { + let (stts, mut rest) = buf.split_at_mut(stts_len); + let (stsz, stss) = rest.split_at_mut(stsz_len); + let mut frame = 0; + let mut key_frame = 0; + let mut last_start_and_dur = None; + s.foreach(db, |it| { + last_start_and_dur = Some((it.start_90k, it.duration_90k)); + BigEndian::write_u32(&mut stts[8*frame .. 8*frame+4], 1); + BigEndian::write_u32(&mut stts[8*frame+4 .. 8*frame+8], it.duration_90k as u32); + BigEndian::write_u32(&mut stsz[4*frame .. 4*frame+4], it.bytes as u32); + if it.is_key { + BigEndian::write_u32(&mut stss[4*key_frame .. 4*key_frame+4], + self.first_frame_num + (frame as u32)); + key_frame += 1; + } + frame += 1; + Ok(()) + })?; + assert_eq!(s.frames, frame as i32); + assert_eq!(s.key_frames, key_frame as i32); + + // Fix up the final frame's duration. + // Doing this after the fact is more efficient than having a condition on every + // iteration. + if let Some((last_start, dur)) = last_start_and_dur { + BigEndian::write_u32(&mut stts[8*frame-4 ..], + cmp::min(s.desired_range_90k.end - last_start, dur) as u32); + } + } + Ok(Mp4SegmentIndex{ + buf: buf, + stsz_start: stts_len, + stss_start: (stts_len + stsz_len), + }) + } +} + +pub struct Mp4FileBuilder { + /// Segments of video: one per "recording" table entry as they should + /// appear in the video. + segments: Vec, + video_sample_entries: SmallVec<[Arc; 1]>, + next_frame_num: u32, + duration_90k: u32, + num_subtitle_samples: u32, + subtitle_co64_pos: Option, + body: BodyState, + include_timestamp_subtitle_track: bool, +} + +/// The portion of `Mp4FileBuilder` which is mutated while building the body of the file. +/// This is separated out from the rest so that it can be borrowed in a loop over +/// `Mp4FileBuilder::segments`; otherwise this would cause a double-self-borrow. +struct BodyState { + slices: Slices, + + /// `self.buf[unflushed_buf_pos .. self.buf.len()]` holds bytes that should be + /// appended to `slices` before any other slice. See `flush_buf()`. + unflushed_buf_pos: usize, + buf: Vec, +} + +#[derive(Debug)] +enum Mp4FileSlice { + Static(StaticBytestring), // index into STATIC_BYTESTRINGS + Buf(u32), // index into m.buf + VideoSampleEntry(u32), // index into m.video_sample_entries + Stts(u32), // index into m.segments + Stsz(u32), // index into m.segments + Co64, + Stss(u32), // index into m.segments + VideoSampleData(u32), // index into m.segments + SubtitleSampleData(u32), // index into m.segments +} + +impl ContextWriter for Mp4FileSlice { + fn write_to(&self, f: &Mp4File, r: Range, l: u64, out: &mut io::Write) + -> Result<()> { + match *self { + Mp4FileSlice::Static(off) => { + let s = STATIC_BYTESTRINGS[off as usize]; + debug!("write static data, range: {:?} slice len: {}", r, s.len()); + let part = &s[r.start as usize .. r.end as usize]; + out.write_all(part)?; + Ok(()) + }, + Mp4FileSlice::Buf(off) => { + let off = off as usize; + debug!("write data from buf starting at offset {}/{}, range: {:?}", + off, f.buf.len(), r); + out.write_all( + &f.buf[off+r.start as usize .. off+r.end as usize])?; + Ok(()) + }, + Mp4FileSlice::VideoSampleEntry(off) => { + let e = &f.video_sample_entries[off as usize]; + debug!("write video sample entry data, range: {:?} data len: {}", r, e.data.len()); + let part = &e.data[r.start as usize .. r.end as usize]; + out.write_all(part)?; + Ok(()) + }, + Mp4FileSlice::Stts(index) => { + debug!("write stts for segment {}/{}, range: {:?}", index, f.segments.len(), r); + f.write_stts(index as usize, r, l, out) + }, + Mp4FileSlice::Stsz(index) => { + debug!("write stsz for segment {}/{}, range: {:?}", index, f.segments.len(), r); + f.write_stsz(index as usize, r, l, out) + }, + Mp4FileSlice::Co64 => { + debug!("write co64, range: {:?}", r); + f.write_co64(r, l, out) + }, + Mp4FileSlice::Stss(index) => { + debug!("write stss for segment {}/{}, range: {:?}", index, f.segments.len(), r); + f.write_stss(index as usize, r, l, out) + }, + Mp4FileSlice::VideoSampleData(index) => { + debug!("write video data for segment {}/{}, range: {:?}", + index, f.segments.len(), r); + f.write_video_sample_data(index as usize, r, out) + }, + Mp4FileSlice::SubtitleSampleData(index) => { + debug!("write subtitle data for segment {}/{}, range: {:?}", + index, f.segments.len(), r); + f.write_subtitle_sample_data(index as usize, r, l, out) + } + } + } +} + +// Convert from 90kHz units since 1970-01-01 00:00:00 UTC to +// seconds since 1904-01-01 00:00:00 UTC. +fn to_iso14496_timestamp(t: recording::Time) -> u32 { (t.unix_seconds() + 24107 * 86400) as u32 } + +// Used only within Mp4FileBuilder::build (and methods it calls internally). +// Writes a box length for everything appended in the supplied scope. +macro_rules! write_length { + ($_self:ident, $b:block) => {{ + let len_pos = $_self.body.buf.len(); + let len_start = $_self.body.slices.len() + $_self.body.buf.len() as u64 - + $_self.body.unflushed_buf_pos as u64; + $_self.body.append_u32(0); // placeholder. + { $b; } + let len_end = $_self.body.slices.len() + $_self.body.buf.len() as u64 - + $_self.body.unflushed_buf_pos as u64; + BigEndian::write_u32(&mut $_self.body.buf[len_pos .. len_pos + 4], + (len_end - len_start) as u32); + }} +} + +fn hex(raw: &[u8]) -> String { + const HEX_CHARS: [u8; 16] = [b'0', b'1', b'2', b'3', b'4', b'5', b'6', b'7', + b'8', b'9', b'a', b'b', b'c', b'd', b'e', b'f']; + let mut hex = Vec::with_capacity(2 * raw.len()); + for b in raw { + hex.push(HEX_CHARS[((b & 0xf0) >> 4) as usize]); + hex.push(HEX_CHARS[( b & 0x0f ) as usize]); + } + unsafe { String::from_utf8_unchecked(hex) } +} + +impl Mp4FileBuilder { + pub fn new() -> Self { + Mp4FileBuilder{ + segments: Vec::new(), + video_sample_entries: SmallVec::new(), + next_frame_num: 1, + duration_90k: 0, + num_subtitle_samples: 0, + subtitle_co64_pos: None, + body: BodyState{ + slices: Slices::new(), + buf: Vec::new(), + unflushed_buf_pos: 0, + }, + include_timestamp_subtitle_track: false, + } + } + + pub fn include_timestamp_subtitle_track(&mut self, b: bool) { + self.include_timestamp_subtitle_track = b; + } + + pub fn reserve(&mut self, additional: usize) { + self.segments.reserve(additional); + } + + pub fn len(&self) -> usize { self.segments.len() } + + pub fn append(&mut self, row: ListCameraRecordingsRow, rel_range_90k: Range) { + self.segments.push(Mp4Segment{ + s: recording::Segment::new(&row, rel_range_90k), + index: RefCell::new(None), + first_frame_num: self.next_frame_num, + num_subtitle_samples: 0, + }); + self.next_frame_num += row.video_samples as u32; + if !self.video_sample_entries.iter().any(|e| e.id == row.video_sample_entry.id) { + self.video_sample_entries.push(row.video_sample_entry); + } + } + + pub fn build(mut self, db: Arc, dir: Arc) -> Result { + let mut max_end = None; + let mut etag = hash::Hasher::new(hash::Type::SHA1)?; + etag.update(&FORMAT_VERSION[..])?; + if self.include_timestamp_subtitle_track { + etag.update(b":ts:")?; + } + for s in &mut self.segments { + s.s.init(&db)?; + let d = &s.s.desired_range_90k; + self.duration_90k += (d.end - d.start) as u32; + let end = s.s.start + recording::Duration(d.end as i64); + max_end = match max_end { + None => Some(end), + Some(v) => Some(cmp::max(v, end)), + }; + + if self.include_timestamp_subtitle_track { + // Calculate the number of subtitle samples: starting to ending time (rounding up). + let start_sec = (s.s.start + recording::Duration(d.start as i64)).unix_seconds(); + let end_sec = (s.s.start + + recording::Duration(d.end as i64 + TIME_UNITS_PER_SEC - 1)) + .unix_seconds(); + s.num_subtitle_samples = (end_sec - start_sec) as u32; + self.num_subtitle_samples += s.num_subtitle_samples; + } + + // Update the etag to reflect this segment. + let mut data = [0_u8; 24]; + let mut cursor = io::Cursor::new(&mut data[..]); + cursor.write_i64::(s.s.id)?; + cursor.write_i64::(s.s.start.0)?; + cursor.write_i32::(d.start)?; + cursor.write_i32::(d.end)?; + etag.update(cursor.into_inner())?; + } + let max_end = match max_end { + None => return Err(Error::new("no segments!".to_owned())), + Some(v) => v, + }; + let creation_ts = to_iso14496_timestamp(max_end); + let mut est_slices = 16 + self.video_sample_entries.len() + 4 * self.segments.len(); + if self.include_timestamp_subtitle_track { + est_slices += 16 + self.segments.len(); + } + self.body.slices.reserve(est_slices); + const EST_BUF_LEN: usize = 2048; + self.body.buf.reserve(EST_BUF_LEN); + self.body.append_static(StaticBytestring::FtypBox); + self.append_moov(creation_ts)?; + + // Write the mdat header. Use the large format to support files over 2^32-1 bytes long. + // Write zeroes for the length as a placeholder; fill it in after it's known. + // It'd be nice to use the until-EOF form, but QuickTime Player doesn't support it. + self.body.buf.extend_from_slice(b"\x00\x00\x00\x01mdat\x00\x00\x00\x00\x00\x00\x00\x00"); + let mdat_len_pos = self.body.buf.len() - 8; + self.body.flush_buf(); + let initial_sample_byte_pos = self.body.slices.len(); + for (i, s) in self.segments.iter().enumerate() { + let r = s.s.sample_file_range(); + self.body.slices.append(r.end - r.start, Mp4FileSlice::VideoSampleData(i as u32)); + } + if let Some(p) = self.subtitle_co64_pos { + BigEndian::write_u64(&mut self.body.buf[p .. p + 8], self.body.slices.len()); + for (i, s) in self.segments.iter().enumerate() { + self.body.slices.append( + s.num_subtitle_samples as u64 * + (mem::size_of::() + SUBTITLE_LENGTH) as u64, + Mp4FileSlice::SubtitleSampleData(i as u32)); + } + } + // Fill in the length left as a placeholder above. Note the 16 here is the length + // of the mdat header. + BigEndian::write_u64(&mut self.body.buf[mdat_len_pos .. mdat_len_pos + 8], + 16 + self.body.slices.len() - initial_sample_byte_pos); + if est_slices < self.body.slices.num() { + warn!("Estimated {} slices; actually were {} slices", est_slices, + self.body.slices.num()); + } else { + debug!("Estimated {} slices; actually were {} slices", est_slices, + self.body.slices.num()); + } + if EST_BUF_LEN < self.body.buf.len() { + warn!("Estimated {} buf bytes; actually were {}", EST_BUF_LEN, self.body.buf.len()); + } else { + debug!("Estimated {} buf bytes; actually were {}", EST_BUF_LEN, self.body.buf.len()); + } + debug!("slices: {:?}", self.body.slices); + Ok(Mp4File{ + db: db, + dir: dir, + segments: self.segments, + slices: self.body.slices, + buf: self.body.buf, + video_sample_entries: self.video_sample_entries, + initial_sample_byte_pos: initial_sample_byte_pos, + last_modified: header::HttpDate(time::at(Timespec::new(max_end.unix_seconds(), 0))), + etag: header::EntityTag::strong(hex(&etag.finish()?)), + }) + } + + // MovieBox, ISO/IEC 14496-12 section 8.2.1. + fn append_moov(&mut self, creation_ts: u32) -> Result<()> { + write_length!(self, { + self.body.buf.extend_from_slice(b"moov"); + self.append_mvhd(creation_ts); + self.append_video_trak(creation_ts)?; + if self.include_timestamp_subtitle_track { + self.append_subtitle_trak(creation_ts); + } + }); + Ok(()) + } + + // MovieHeaderBox version 0, ISO/IEC 14496-12 section 8.2.2. + fn append_mvhd(&mut self, creation_ts: u32) { + write_length!(self, { + self.body.buf.extend_from_slice(b"mvhd\x00\x00\x00\x00"); + self.body.append_u32(creation_ts); + self.body.append_u32(creation_ts); + self.body.append_u32(TIME_UNITS_PER_SEC as u32); + let d = self.duration_90k; + self.body.append_u32(d); + self.body.append_static(StaticBytestring::MvhdJunk); + // TODO: caption track? + self.body.append_u32(2); // next_track_id + }); + } + + // ISO/IEC 14496-12 section 8.3.1, trak. + fn append_video_trak(&mut self, creation_ts: u32) -> Result<()> { + write_length!(self, { + self.body.buf.extend_from_slice(b"trak"); + self.append_video_tkhd(creation_ts); + self.maybe_append_video_edts()?; + self.append_video_mdia(creation_ts); + }); + Ok(()) + } + + // ISO/IEC 14496-12 section 8.3.1, trak. + fn append_subtitle_trak(&mut self, creation_ts: u32) { + write_length!(self, { + self.body.buf.extend_from_slice(b"trak"); + self.append_subtitle_tkhd(creation_ts); + self.append_subtitle_mdia(creation_ts); + }); + } + + // ISO/IEC 14496-12 section 8.3.2. + fn append_video_tkhd(&mut self, creation_ts: u32) { + write_length!(self, { + // flags 7: track_enabled | track_in_movie | track_in_preview + self.body.buf.extend_from_slice(b"tkhd\x00\x00\x00\x07"); + self.body.append_u32(creation_ts); + self.body.append_u32(creation_ts); + self.body.append_u32(1); // track_id + self.body.append_u32(0); // reserved + self.body.append_u32(self.duration_90k); + self.body.append_static(StaticBytestring::TkhdJunk); + let width = self.video_sample_entries.iter().map(|e| e.width).max().unwrap(); + let height = self.video_sample_entries.iter().map(|e| e.height).max().unwrap(); + self.body.append_u32((width as u32) << 16); + self.body.append_u32((height as u32) << 16); + }); + } + + // ISO/IEC 14496-12 section 8.3.2. + fn append_subtitle_tkhd(&mut self, creation_ts: u32) { + write_length!(self, { + // flags 7: track_enabled | track_in_movie | track_in_preview + self.body.buf.extend_from_slice(b"tkhd\x00\x00\x00\x07"); + self.body.append_u32(creation_ts); + self.body.append_u32(creation_ts); + self.body.append_u32(2); // track_id + self.body.append_u32(0); // reserved + self.body.append_u32(self.duration_90k); + self.body.append_static(StaticBytestring::TkhdJunk); + self.body.append_u32(0); // width, unused. + self.body.append_u32(0); // height, unused. + }); + } + + // ISO/IEC 14496-12 section 8.6.5. + fn maybe_append_video_edts(&mut self) -> Result<()> { + #[derive(Debug, Default)] + struct Entry { + segment_duration: u64, + media_time: u64, + }; + let mut flushed: Vec = Vec::new(); + let mut unflushed: Entry = Default::default(); + let mut cur_media_time: u64 = 0; + for s in &self.segments { + // The actual range may start before the desired range because it can only start on a + // key frame. This relationship should hold true: + // actual start <= desired start < desired end + let actual = s.s.actual_time_90k(); + let skip = s.s.desired_range_90k.start - actual.start; + let keep = s.s.desired_range_90k.end - s.s.desired_range_90k.start; + assert!(skip >= 0 && keep > 0, "desired={}..{} actual={}..{}", + s.s.desired_range_90k.start, s.s.desired_range_90k.end, + actual.start, actual.end); + cur_media_time += skip as u64; + if unflushed.segment_duration + unflushed.media_time == cur_media_time { + unflushed.segment_duration += keep as u64; + } else { + if unflushed.segment_duration > 0 { + flushed.push(unflushed); + } + unflushed = Entry{ + segment_duration: keep as u64, + media_time: cur_media_time, + }; + } + cur_media_time += keep as u64; + } + + if flushed.is_empty() && unflushed.media_time == 0 { + return Ok(()); // use implicit one-to-one mapping. + } + + flushed.push(unflushed); + + debug!("Using edit list: {:?}", flushed); + write_length!(self, { + self.body.buf.extend_from_slice(b"edts"); + write_length!(self, { + // Use version 1 for 64-bit times. + self.body.buf.extend_from_slice(b"elst\x01\x00\x00\x00"); + self.body.append_u32(flushed.len() as u32); + for e in &flushed { + self.body.append_u64(e.segment_duration); + self.body.append_u64(e.media_time); + + // media_rate_integer + media_rate_fraction: both fixed at 1 + self.body.buf.extend_from_slice(b"\x00\x01\x00\x01"); + } + }); + }); + Ok(()) + } + + // ISO/IEC 14496-12 section 8.4.1. + fn append_video_mdia(&mut self, creation_ts: u32) { + write_length!(self, { + self.body.buf.extend_from_slice(b"mdia"); + self.append_mdhd(creation_ts); + self.body.append_static(StaticBytestring::VideoHdlrBox); + self.append_video_minf(); + }); + } + + // ISO/IEC 14496-12 section 8.4.1. + fn append_subtitle_mdia(&mut self, creation_ts: u32) { + write_length!(self, { + self.body.buf.extend_from_slice(b"mdia"); + self.append_mdhd(creation_ts); + self.body.append_static(StaticBytestring::SubtitleHdlrBox); + self.append_subtitle_minf(); + // TODO: nmhddinf + }); + } + + /// Appends a mdhd suitable for either the video or subtitle track. + /// See ISO/IEC 14496-12 section 8.4.2. + fn append_mdhd(&mut self, creation_ts: u32) { + write_length!(self, { + self.body.buf.extend_from_slice(b"mdhd\x00\x00\x00\x00"); + self.body.append_u32(creation_ts); + self.body.append_u32(creation_ts); + self.body.append_u32(TIME_UNITS_PER_SEC as u32); + self.body.append_u32(self.duration_90k); + self.body.append_u32(0x55c40000); // language=und + pre_defined + }); + } + + // ISO/IEC 14496-12 section 8.4.4. + fn append_video_minf(&mut self) { + write_length!(self, { + self.body.append_static(StaticBytestring::VideoMinfJunk); + self.append_video_stbl(); + }); + } + + // ISO/IEC 14496-12 section 8.4.4. + fn append_subtitle_minf(&mut self) { + write_length!(self, { + self.body.append_static(StaticBytestring::SubtitleMinfJunk); + self.append_subtitle_stbl(); + }); + } + + // ISO/IEC 14496-12 section 8.5.1. + fn append_video_stbl(&mut self) { + write_length!(self, { + self.body.buf.extend_from_slice(b"stbl"); + self.append_video_stsd(); + self.append_video_stts(); + self.append_video_stsc(); + self.append_video_stsz(); + self.append_video_co64(); + self.append_video_stss(); + }); + } + + // ISO/IEC 14496-12 section 8.5.1. + fn append_subtitle_stbl(&mut self) { + write_length!(self, { + self.body.append_static(StaticBytestring::SubtitleStblJunk); + self.append_subtitle_stts(); + self.append_subtitle_stsc(); + self.append_subtitle_stsz(); + self.append_subtitle_co64(); + }); + } + + // ISO/IEC 14496-12 section 8.5.2. + fn append_video_stsd(&mut self) { + write_length!(self, { + self.body.buf.extend_from_slice(b"stsd\x00\x00\x00\x00"); + let n_entries = self.video_sample_entries.len() as u32; + self.body.append_u32(n_entries); + self.body.flush_buf(); + for (i, e) in self.video_sample_entries.iter().enumerate() { + self.body.slices.append(e.data.len() as u64, + Mp4FileSlice::VideoSampleEntry(i as u32)); + } + }); + } + + // ISO/IEC 14496-12 section 8.6.1. + fn append_video_stts(&mut self) { + write_length!(self, { + self.body.buf.extend_from_slice(b"stts\x00\x00\x00\x00"); + let mut entry_count = 0; + for s in &self.segments { + entry_count += s.s.frames as u32; + } + self.body.append_u32(entry_count); + self.body.flush_buf(); + for (i, s) in self.segments.iter().enumerate() { + self.body.slices.append( + 2 * (mem::size_of::() as u64) * (s.s.frames as u64), + Mp4FileSlice::Stts(i as u32)); + } + }); + } + + // ISO/IEC 14496-12 section 8.6.1. + fn append_subtitle_stts(&mut self) { + write_length!(self, { + self.body.buf.extend_from_slice(b"stts\x00\x00\x00\x00"); + + let entry_count_pos = self.body.buf.len(); + self.body.append_u32(0); // placeholder for entry_count + + let mut entry_count = 0; + for s in &self.segments { + let r = &s.s.desired_range_90k; + let start = s.s.start + recording::Duration(r.start as i64); + let end = s.s.start + recording::Duration(r.end as i64); + let start_next_sec = recording::Time( + start.0 + TIME_UNITS_PER_SEC - (start.0 % TIME_UNITS_PER_SEC)); + if end <= start_next_sec { + // Segment doesn't last past the next second. + entry_count += 1; + self.body.append_u32(1); // count + self.body.append_u32((end - start).0 as u32); // duration + } else { + // The first subtitle just lasts until the next second. + entry_count += 1; + self.body.append_u32(1); // count + self.body.append_u32((start_next_sec - start).0 as u32); // duration + + // Then there are zero or more "interior" subtitles, one second each. + let end_prev_sec = recording::Time(end.0 - (end.0 % TIME_UNITS_PER_SEC)); + if start_next_sec < end_prev_sec { + entry_count += 1; + let interior = (end_prev_sec - start_next_sec).0 / TIME_UNITS_PER_SEC; + self.body.append_u32(interior as u32); // count + self.body.append_u32(TIME_UNITS_PER_SEC as u32); // duration + } + + // Then there's a final subtitle for the remaining fraction of a second. + entry_count += 1; + self.body.append_u32(1); // count + self.body.append_u32((end - end_prev_sec).0 as u32); // duration + } + } + BigEndian::write_u32(&mut self.body.buf[entry_count_pos .. entry_count_pos + 4], + entry_count); + }); + } + + // ISO/IEC 14496-12 section 8.7.4. + fn append_video_stsc(&mut self) { + write_length!(self, { + self.body.buf.extend_from_slice(b"stsc\x00\x00\x00\x00"); + self.body.append_u32(self.segments.len() as u32); + for (i, s) in self.segments.iter().enumerate() { + self.body.append_u32((i + 1) as u32); + self.body.append_u32(s.s.frames as u32); + + // Write sample_description_index. + let i = self.video_sample_entries.iter().position( + |e| e.id == s.s.video_sample_entry_id).unwrap(); + self.body.append_u32((i + 1) as u32); + } + }); + } + + // ISO/IEC 14496-12 section 8.7.4. + fn append_subtitle_stsc(&mut self) { + write_length!(self, { + self.body.buf.extend_from_slice( + b"stsc\x00\x00\x00\x00\x00\x00\x00\x01\x00\x00\x00\x01"); + self.body.append_u32(self.num_subtitle_samples); + self.body.append_u32(1); + }); + } + + // ISO/IEC 14496-12 section 8.7.3. + fn append_video_stsz(&mut self) { + write_length!(self, { + self.body.buf.extend_from_slice(b"stsz\x00\x00\x00\x00\x00\x00\x00\x00"); + let mut entry_count = 0; + for s in &self.segments { + entry_count += s.s.frames as u32; + } + self.body.append_u32(entry_count); + self.body.flush_buf(); + for (i, s) in self.segments.iter().enumerate() { + self.body.slices.append( + (mem::size_of::()) as u64 * (s.s.frames as u64), + Mp4FileSlice::Stsz(i as u32)); + } + }); + } + + // ISO/IEC 14496-12 section 8.7.3. + fn append_subtitle_stsz(&mut self) { + write_length!(self, { + self.body.buf.extend_from_slice(b"stsz\x00\x00\x00\x00"); + self.body.append_u32((mem::size_of::() + SUBTITLE_LENGTH) as u32); + self.body.append_u32(self.num_subtitle_samples); + }); + } + + // ISO/IEC 14496-12 section 8.7.5. + fn append_video_co64(&mut self) { + write_length!(self, { + self.body.buf.extend_from_slice(b"co64\x00\x00\x00\x00"); + self.body.append_u32(self.segments.len() as u32); + self.body.flush_buf(); + self.body.slices.append( + (mem::size_of::()) as u64 * (self.segments.len() as u64), + Mp4FileSlice::Co64); + }); + } + + // ISO/IEC 14496-12 section 8.7.5. + fn append_subtitle_co64(&mut self) { + write_length!(self, { + // Write a placeholder; the actual value will be filled in later. + self.body.buf.extend_from_slice( + b"co64\x00\x00\x00\x00\x00\x00\x00\x01\x00\x00\x00\x00\x00\x00\x00\x00"); + self.subtitle_co64_pos = Some(self.body.buf.len() - 8); + }); + } + + // ISO/IEC 14496-12 section 8.6.2. + fn append_video_stss(&mut self) { + write_length!(self, { + self.body.buf.extend_from_slice(b"stss\x00\x00\x00\x00"); + let mut entry_count = 0; + for s in &self.segments { + entry_count += s.s.key_frames as u32; + } + self.body.append_u32(entry_count); + self.body.flush_buf(); + for (i, s) in self.segments.iter().enumerate() { + self.body.slices.append( + (mem::size_of::() as u64) * (s.s.key_frames as u64), + Mp4FileSlice::Stss(i as u32)); + } + }); + } +} + +impl BodyState { + fn append_u32(&mut self, v: u32) { + self.buf.write_u32::(v).expect("Vec write shouldn't fail"); + } + + fn append_u64(&mut self, v: u64) { + self.buf.write_u64::(v).expect("Vec write shouldn't fail"); + } + + fn flush_buf(&mut self) { + let len = self.buf.len(); + if self.unflushed_buf_pos < len { + self.slices.append((len - self.unflushed_buf_pos) as u64, + Mp4FileSlice::Buf(self.unflushed_buf_pos as u32)); + self.unflushed_buf_pos = len; + } + } + + fn append_static(&mut self, which: StaticBytestring) { + self.flush_buf(); + let s = STATIC_BYTESTRINGS[which as usize]; + self.slices.append(s.len() as u64, Mp4FileSlice::Static(which)); + } +} + +pub struct Mp4File { + db: Arc, + dir: Arc, + segments: Vec, + slices: Slices, + buf: Vec, + video_sample_entries: SmallVec<[Arc; 1]>, + initial_sample_byte_pos: u64, + last_modified: header::HttpDate, + etag: header::EntityTag, +} + +impl Mp4File { + fn write_stts(&self, i: usize, r: Range, _l: u64, out: &mut io::Write) + -> Result<()> { + self.segments[i].with_index(&self.db, |i| { + out.write_all(&i.stts()[r.start as usize .. r.end as usize])?; + Ok(()) + }) + } + + fn write_stsz(&self, i: usize, r: Range, _l: u64, out: &mut io::Write) + -> Result<()> { + self.segments[i].with_index(&self.db, |i| { + out.write_all(&i.stsz()[r.start as usize .. r.end as usize])?; + Ok(()) + }) + } + + fn write_co64(&self, r: Range, l: u64, out: &mut io::Write) -> Result<()> { + pieces::clip_to_range(r, l, out, |w| { + let mut pos = self.initial_sample_byte_pos; + for s in &self.segments { + w.write_u64::(pos)?; + let r = s.s.sample_file_range(); + pos += r.end - r.start; + } + Ok(()) + }) + } + + fn write_stss(&self, i: usize, r: Range, _l: u64, out: &mut io::Write) -> Result<()> { + self.segments[i].with_index(&self.db, |i| { + out.write_all(&i.stss()[r.start as usize .. r.end as usize])?; + Ok(()) + }) + } + + fn write_video_sample_data(&self, i: usize, r: Range, out: &mut io::Write) -> Result<()> { + let s = &self.segments[i]; + let f = self.dir.open_sample_file(self.db.lock().get_recording(s.s.id)?.sample_file_uuid)?; + mmapfile::MmapFileSlice::new(f, s.s.sample_file_range()).write_to(r, out) + } + + fn write_subtitle_sample_data(&self, i: usize, r: Range, l: u64, out: &mut io::Write) + -> Result<()> { + let s = &self.segments[i]; + let d = &s.s.desired_range_90k; + let start_sec = (s.s.start + recording::Duration(d.start as i64)).unix_seconds(); + let end_sec = (s.s.start + recording::Duration(d.end as i64 + TIME_UNITS_PER_SEC - 1)) + .unix_seconds(); + pieces::clip_to_range(r, l, out, |w| { + for ts in start_sec .. end_sec { + w.write_u16::(SUBTITLE_LENGTH as u16)?; + let tm = time::at(time::Timespec{sec: ts, nsec: 0}); + use std::io::Write; + write!(w, "{}", tm.strftime(SUBTITLE_TEMPLATE)?)?; + } + Ok(()) + })?; + Ok(()) + } +} + +impl resource::Resource for Mp4File { + fn content_type(&self) -> mime::Mime { "video/mp4".parse().unwrap() } + fn last_modified(&self) -> &header::HttpDate { &self.last_modified } + fn etag(&self) -> Option<&header::EntityTag> { Some(&self.etag) } + fn len(&self) -> u64 { self.slices.len() } + fn write_to(&self, range: Range, out: &mut io::Write) -> Result<()> { + self.slices.write_to(self, range, out) + } +} + +#[cfg(test)] +mod tests { + extern crate tempdir; + extern crate test; + + use db; + use dir; + use ffmpeg; + use hyper::{self, header}; + use openssl::crypto::hash; + use recording::{self, TIME_UNITS_PER_SEC}; + use resource::{self, Resource}; + use rusqlite; + use self::test::Bencher; + use std::fs; + use std::io; + use std::mem; + use std::path::Path; + use std::sync::Arc; + use std::thread; + use super::*; + use stream::StreamSource; + use self::tempdir::TempDir; + use testutil; + use uuid::Uuid; + + struct Sha1(hash::Hasher); + + impl Sha1 { + fn new() -> Sha1 { Sha1(hash::Hasher::new(hash::Type::SHA1).unwrap()) } + fn finish(mut self) -> Vec { self.0.finish().unwrap() } + } + + impl io::Write for Sha1 { + fn write(&mut self, buf: &[u8]) -> io::Result { + self.0.update(buf).unwrap(); + Ok(buf.len()) + } + fn flush(&mut self) -> io::Result<()> { Ok(()) } + } + + fn digest(r: &Resource) -> Vec { + let mut sha1 = Sha1::new(); + r.write_to(0 .. r.len(), &mut sha1).unwrap(); + sha1.finish() + } + + lazy_static! { + static ref TEST_CAMERA_UUID: Uuid = + Uuid::parse_str("ce2d9bc2-0cd3-4204-9324-7b5ccb07183c").unwrap(); + } + + const TEST_CAMERA_ID: i32 = 1; + + struct TestDb { + db: Arc, + dir: Arc, + syncer_channel: dir::SyncerChannel, + syncer_join: thread::JoinHandle<()>, + tmpdir: TempDir, + } + + // TODO: this seems like an integration test util; move to tests/common/*.rs file? + fn setup_db() -> TestDb { + let tmpdir = TempDir::new("mp4-test").unwrap(); + + let conn = rusqlite::Connection::open_in_memory().unwrap(); + let schema = include_str!("schema.sql"); + conn.execute_batch(schema).unwrap(); + let uuid_bytes = &TEST_CAMERA_UUID.as_bytes()[..]; + conn.execute_named(r#" + insert into camera (uuid, short_name, description, host, username, password, + main_rtsp_path, sub_rtsp_path, retain_bytes) + values (:uuid, :short_name, :description, :host, :username, :password, + :main_rtsp_path, :sub_rtsp_path, :retain_bytes) + "#, &[ + (":uuid", &uuid_bytes), + (":short_name", &"test camera"), + (":description", &""), + (":host", &"test-camera"), + (":username", &"foo"), + (":password", &"bar"), + (":main_rtsp_path", &"/main"), + (":sub_rtsp_path", &"/sub"), + (":retain_bytes", &1048576i64), + ]).unwrap(); + assert_eq!(TEST_CAMERA_ID as i64, conn.last_insert_rowid()); + let db = Arc::new(db::Database::new(conn).unwrap()); + let path = tmpdir.path().to_str().unwrap().to_owned(); + let dir = dir::SampleFileDir::new(&path, db.clone()).unwrap(); + let (syncer_channel, syncer_join) = dir::start_syncer(dir.clone()).unwrap(); + TestDb{ + db: db, + dir: dir, + syncer_channel: syncer_channel, + syncer_join: syncer_join, + tmpdir: tmpdir, + } + } + + fn copy_mp4_to_db(db: &TestDb) { + let mut input = StreamSource::File("src/testdata/clip.mp4").open().unwrap(); + + // 2015-04-26 00:00:00 UTC. + const START_TIME: recording::Time = recording::Time(1430006400i64 * TIME_UNITS_PER_SEC); + let extra_data = input.get_extra_data().unwrap(); + let video_sample_entry_id = db.db.lock().insert_video_sample_entry( + extra_data.width, extra_data.height, &extra_data.sample_entry).unwrap(); + let mut output = db.dir.create_writer(START_TIME, START_TIME, TEST_CAMERA_ID, + video_sample_entry_id).unwrap(); + loop { + let pkt = match input.get_next() { + Ok(p) => p, + Err(ffmpeg::Error::Eof) => { break; }, + Err(e) => { panic!("unexpected input error: {}", e); }, + }; + output.write(pkt.data().expect("packet without data"), pkt.duration() as i32, + pkt.is_key()).unwrap(); + } + db.syncer_channel.async_save_writer(output).unwrap(); + db.syncer_channel.flush(); + } + + fn add_dummy_recordings_to_db(db: &db::Database) { + let mut data = Vec::new(); + data.extend_from_slice(include_bytes!("testdata/video_sample_index.bin")); + let mut db = db.lock(); + let video_sample_entry_id = db.insert_video_sample_entry(1920, 1080, &[0u8; 100]).unwrap(); + const START_TIME: recording::Time = recording::Time(1430006400i64 * TIME_UNITS_PER_SEC); + const DURATION: recording::Duration = recording::Duration(5399985); + let mut recording = db::RecordingToInsert{ + camera_id: TEST_CAMERA_ID, + sample_file_bytes: 30104460, + time: START_TIME .. (START_TIME + DURATION), + local_time: START_TIME, + video_samples: 1800, + video_sync_samples: 60, + video_sample_entry_id: video_sample_entry_id, + sample_file_uuid: Uuid::nil(), + video_index: data, + sample_file_sha1: [0; 20], + }; + let mut tx = db.tx().unwrap(); + tx.bypass_reservation_for_testing = true; + for _ in 0..60 { + tx.insert_recording(&recording).unwrap(); + recording.time.start += DURATION; + recording.local_time += DURATION; + recording.time.end += DURATION; + } + tx.commit().unwrap(); + } + + fn create_mp4_from_db(db: Arc, dir: Arc, skip_90k: i32, + shorten_90k: i32, include_subtitles: bool) -> Mp4File { + let mut builder = Mp4FileBuilder::new(); + builder.include_timestamp_subtitle_track(include_subtitles); + let all_time = recording::Time(i64::min_value()) .. recording::Time(i64::max_value()); + db.lock().list_recordings(TEST_CAMERA_ID, &all_time, |r| { + let d = r.duration_90k; + assert!(skip_90k + shorten_90k < d); + builder.append(r, skip_90k .. d - shorten_90k); + Ok(()) + }).unwrap(); + builder.build(db, dir).unwrap() + } + + fn write_mp4(mp4: &Mp4File, dir: &Path) -> String { + let mut filename = dir.to_path_buf(); + filename.push("clip.new.mp4"); + let mut out = fs::OpenOptions::new().write(true).create_new(true).open(&filename).unwrap(); + mp4.write_to(0 .. mp4.len(), &mut out).unwrap(); + filename.to_str().unwrap().to_string() + } + + fn compare_mp4s(new_filename: &str, pts_offset: i64, shorten: i64) { + let mut orig = StreamSource::File("src/testdata/clip.mp4").open().unwrap(); + let mut new = StreamSource::File(new_filename).open().unwrap(); + assert_eq!(orig.get_extra_data().unwrap(), new.get_extra_data().unwrap()); + let mut final_durations = None; + loop { + let orig_pkt = match orig.get_next() { + Ok(p) => Some(p), + Err(ffmpeg::Error::Eof) => None, + Err(e) => { panic!("unexpected input error: {}", e); }, + }; + let new_pkt = match new.get_next() { + Ok(p) => Some(p), + Err(ffmpeg::Error::Eof) => { break; }, + Err(e) => { panic!("unexpected input error: {}", e); }, + }; + let (orig_pkt, new_pkt) = match (orig_pkt, new_pkt) { + (Some(o), Some(n)) => (o, n), + (None, None) => break, + (o, n) => panic!("orig: {} new: {}", o.is_some(), n.is_some()), + }; + assert_eq!(orig_pkt.pts().unwrap(), new_pkt.pts().unwrap() + pts_offset); + assert_eq!(orig_pkt.dts(), new_pkt.dts() + pts_offset); + assert_eq!(orig_pkt.data(), new_pkt.data()); + assert_eq!(orig_pkt.is_key(), new_pkt.is_key()); + final_durations = Some((orig_pkt.duration(), new_pkt.duration())); + } + + if let Some((orig_dur, new_dur)) = final_durations { + // One would normally expect the duration to be exactly the same, but when using an + // edit list, ffmpeg appears to extend the last packet's duration by the amount skipped + // at the beginning. I think this is a bug on their side. + assert!(orig_dur - shorten + pts_offset == new_dur, + "orig_dur={} new_dur={} shorten={} pts_offset={}", + orig_dur, new_dur, shorten, pts_offset); + } + } + + #[test] + fn test_round_trip() { + testutil::init(); + let db = setup_db(); + copy_mp4_to_db(&db); + let mp4 = create_mp4_from_db(db.db.clone(), db.dir.clone(), 0, 0, false); + let new_filename = write_mp4(&mp4, db.tmpdir.path()); + compare_mp4s(&new_filename, 0, 0); + + // Test the metadata. This is brittle, which is the point. Any time the digest comparison + // here fails, it can be updated, but the etag must change as well! Otherwise clients may + // combine ranges from the new format with ranges from the old format. + let sha1 = digest(&mp4); + assert_eq!("1e5331e8371bd97ac3158b3a86494abc87cdc70e", super::hex(&sha1[..])); + const EXPECTED_ETAG: &'static str = "191730a3c41adc9c5394b4516638ee6fda05649c"; + assert_eq!(Some(&header::EntityTag::strong(EXPECTED_ETAG.to_owned())), mp4.etag()); + drop(db.syncer_channel); + db.syncer_join.join().unwrap(); + } + + #[test] + fn test_round_trip_with_subtitles() { + testutil::init(); + let db = setup_db(); + copy_mp4_to_db(&db); + let mp4 = create_mp4_from_db(db.db.clone(), db.dir.clone(), 0, 0, true); + let new_filename = write_mp4(&mp4, db.tmpdir.path()); + compare_mp4s(&new_filename, 0, 0); + + // Test the metadata. This is brittle, which is the point. Any time the digest comparison + // here fails, it can be updated, but the etag must change as well! Otherwise clients may + // combine ranges from the new format with ranges from the old format. + let sha1 = digest(&mp4); + assert_eq!("0081a442ba73092027fc580eeac2ebf25cb1ef50", super::hex(&sha1[..])); + const EXPECTED_ETAG: &'static str = "d7aedccfae063219974086c43efdc6fda5a7e889"; + assert_eq!(Some(&header::EntityTag::strong(EXPECTED_ETAG.to_owned())), mp4.etag()); + drop(db.syncer_channel); + db.syncer_join.join().unwrap(); + } + + #[test] + fn test_round_trip_with_edit_list() { + testutil::init(); + let db = setup_db(); + copy_mp4_to_db(&db); + let mp4 = create_mp4_from_db(db.db.clone(), db.dir.clone(), 1, 0, false); + let new_filename = write_mp4(&mp4, db.tmpdir.path()); + compare_mp4s(&new_filename, 1, 0); + + // Test the metadata. This is brittle, which is the point. Any time the digest comparison + // here fails, it can be updated, but the etag must change as well! Otherwise clients may + // combine ranges from the new format with ranges from the old format. + let sha1 = digest(&mp4); + assert_eq!("685e026af44204bc9cc52115c5e17058e9fb7c70", super::hex(&sha1[..])); + const EXPECTED_ETAG: &'static str = "4cd904b6746330e63590a60d3bd254df3987caee"; + assert_eq!(Some(&header::EntityTag::strong(EXPECTED_ETAG.to_owned())), mp4.etag()); + drop(db.syncer_channel); + db.syncer_join.join().unwrap(); + } + + #[test] + fn test_round_trip_with_shorten() { + testutil::init(); + let db = setup_db(); + copy_mp4_to_db(&db); + let mp4 = create_mp4_from_db(db.db.clone(), db.dir.clone(), 0, 1, false); + let new_filename = write_mp4(&mp4, db.tmpdir.path()); + compare_mp4s(&new_filename, 0, 1); + + // Test the metadata. This is brittle, which is the point. Any time the digest comparison + // here fails, it can be updated, but the etag must change as well! Otherwise clients may + // combine ranges from the new format with ranges from the old format. + let sha1 = digest(&mp4); + assert_eq!("e0d28ddf08e24575a82657b1ce0b2da73f32fd88", super::hex(&sha1[..])); + const EXPECTED_ETAG: &'static str = "d2e75438be6c3a747bf0d9aa86332604340d2b82"; + assert_eq!(Some(&header::EntityTag::strong(EXPECTED_ETAG.to_owned())), mp4.etag()); + drop(db.syncer_channel); + db.syncer_join.join().unwrap(); + } + + #[test] + fn mp4_file_slice_size() { + assert_eq!(8, mem::size_of::()); + } + + /// An HTTP server for benchmarking. + /// It's used as a singleton via `lazy_static!` for two reasons: + /// + /// * to avoid running out of file descriptors. `#[bench]` functions apparently get called + /// many times as the number of iterations is tuned, and hyper servers + /// [can't be shut down](https://github.com/hyperium/hyper/issues/338), so + /// otherwise the default Ubuntu 16.04.1 ulimit of 1024 files is quickly exhausted. + /// * so that when getting a CPU profile of the benchmark, more of the profile focuses + /// on the HTTP serving rather than the setup. + /// + /// Currently this only serves a single `.mp4` file but we could set up variations to benchmark + /// different scenarios: with/without subtitles and edit lists, different lengths, serving + /// different fractions of the file, etc. + struct BenchServer { + url: hyper::Url, + generated_len: u64, + } + + impl BenchServer { + fn new() -> BenchServer { + let mut listener = hyper::net::HttpListener::new("127.0.0.1:0").unwrap(); + use hyper::net::NetworkListener; + let addr = listener.local_addr().unwrap(); + let server = hyper::Server::new(listener); + let url = hyper::Url::parse( + format!("http://{}:{}/", addr.ip(), addr.port()).as_str()).unwrap(); + let db = setup_db(); + add_dummy_recordings_to_db(&db.db); + let mp4 = create_mp4_from_db(db.db.clone(), db.dir.clone(), 0, 0, false); + let p = mp4.initial_sample_byte_pos; + use std::thread::spawn; + spawn(move || { + use hyper::server::{Request, Response, Fresh}; + let (db, dir) = (db.db.clone(), db.dir.clone()); + let _ = server.handle(move |req: Request, res: Response| { + let mp4 = create_mp4_from_db(db.clone(), dir.clone(), 0, 0, false); + resource::serve(&mp4, &req, res).unwrap(); + }); + }); + BenchServer{ + url: url, + generated_len: p, + } + } + } + + lazy_static! { + static ref SERVER: BenchServer = { BenchServer::new() }; + } + + /// Benchmarks serving the generated part of a `.mp4` file (up to the first byte from disk). + #[bench] + fn serve_generated_bytes_fresh_client(b: &mut Bencher) { + testutil::init(); + let server = &*SERVER; + let p = server.generated_len; + let mut buf = Vec::with_capacity(p as usize); + b.bytes = p; + b.iter(|| { + let client = hyper::Client::new(); + let mut resp = + client.get(server.url.clone()) + .header(header::Range::Bytes(vec![header::ByteRangeSpec::FromTo(0, p - 1)])) + .send() + .unwrap(); + buf.clear(); + use std::io::Read; + let size = resp.read_to_end(&mut buf).unwrap(); + assert_eq!(p, size as u64); + }); + } + + /// Another benchmark of serving generated bytes, but reusing the `hyper::Client`. + /// This should be faster than the `fresh` version, but see + /// [this hyper issue](https://github.com/hyperium/hyper/issues/944) relating to Nagle's + /// algorithm. + #[bench] + fn serve_generated_bytes_reuse_client(b: &mut Bencher) { + testutil::init(); + let server = &*SERVER; + let p = server.generated_len; + let mut buf = Vec::with_capacity(p as usize); + b.bytes = p; + let client = hyper::Client::new(); + b.iter(|| { + let mut resp = + client.get(server.url.clone()) + .header(header::Range::Bytes(vec![header::ByteRangeSpec::FromTo(0, p - 1)])) + .send() + .unwrap(); + buf.clear(); + use std::io::Read; + let size = resp.read_to_end(&mut buf).unwrap(); + assert_eq!(p, size as u64); + }); + } + + #[bench] + fn mp4_construction(b: &mut Bencher) { + testutil::init(); + let db = setup_db(); + add_dummy_recordings_to_db(&db.db); + b.iter(|| { + create_mp4_from_db(db.db.clone(), db.dir.clone(), 0, 0, false); + }); + } +} diff --git a/src/pieces.rs b/src/pieces.rs new file mode 100644 index 0000000..21c5d94 --- /dev/null +++ b/src/pieces.rs @@ -0,0 +1,295 @@ +// This file is part of Moonfire NVR, a security camera digital video recorder. +// Copyright (C) 2016 Scott Lamb +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// In addition, as a special exception, the copyright holders give +// permission to link the code of portions of this program with the +// OpenSSL library under certain conditions as described in each +// individual source file, and distribute linked combinations including +// the two. +// +// You must obey the GNU General Public License in all respects for all +// of the code used other than OpenSSL. If you modify file(s) with this +// exception, you may extend this exception to your version of the +// file(s), but you are not obligated to do so. If you do not wish to do +// so, delete this exception statement from your version. If you delete +// this exception statement from all source files in the program, then +// also delete it here. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program. If not, see . + +use error::{Error, Result}; +use std::fmt; +use std::io; +use std::marker::PhantomData; +use std::ops::Range; + +#[derive(Debug)] +struct SliceInfo { + end: u64, + writer: W, +} + +pub trait ContextWriter { + fn write_to(&self, ctx: &Ctx, r: Range, l: u64, out: &mut io::Write) -> Result<()>; +} + +/// Calls `f` with an `io::Write` which delegates to `inner` only for the section defined by `r`. +/// This is useful for easily implementing the `ContextWriter` interface for pieces that generate +/// data on-the-fly rather than simply copying a buffer. +pub fn clip_to_range(r: Range, l: u64, inner: &mut io::Write, mut f: F) -> Result<()> +where F: FnMut(&mut Vec) -> Result<()> { + // Just create a buffer for the whole slice and copy out the relevant portion. + // One might expect it to be faster to avoid this memory allocation and extra copying, but + // benchmarks show when making many 4-byte writes it's better to be able to inline many + // Vec::write_all calls then make one call through traits to hyper's write logic. + let mut buf = Vec::with_capacity(l as usize); + f(&mut buf)?; + inner.write_all(&buf[r.start as usize .. r.end as usize])?; + Ok(()) +} + +pub struct Slices { + len: u64, + slices: Vec>, + phantom: PhantomData, +} + +impl fmt::Debug for Slices where W: fmt::Debug { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + write!(f, "{} slices with overall length {}:", self.slices.len(), self.len)?; + let mut start = 0; + for (i, s) in self.slices.iter().enumerate() { + write!(f, "\n{:7}: [{:12}, {:12}): {:?}", i, start, s.end, s.writer)?; + start = s.end; + } + Ok(()) + } +} + +impl Slices where W: ContextWriter { + pub fn new() -> Slices { + Slices{len: 0, slices: Vec::new(), phantom: PhantomData} + } + + pub fn reserve(&mut self, additional: usize) { + self.slices.reserve(additional) + } + + pub fn append(&mut self, len: u64, writer: W) { + self.len += len; + self.slices.push(SliceInfo{end: self.len, writer: writer}); + } + + /// Returns the total byte length of all slices. + pub fn len(&self) -> u64 { self.len } + + /// Returns the number of slices. + pub fn num(&self) -> usize { self.slices.len() } + + pub fn write_to(&self, ctx: &C, range: Range, out: &mut io::Write) + -> Result<()> { + if range.start > range.end || range.end > self.len { + return Err(Error{ + description: format!("Bad range {:?} for slice of length {}", range, self.len), + cause: None}); + } + + // Binary search for the first slice of the range to write, determining its index and + // (from the preceding slice) the start of its range. + let (mut i, mut slice_start) = match self.slices.binary_search_by_key(&range.start, + |s| s.end) { + Ok(i) if i == self.slices.len() - 1 => return Ok(()), // at end. + Ok(i) => (i+1, self.slices[i].end), // desired start == slice i's end; first is i+1! + Err(i) if i == 0 => (i, 0), // desired start < slice 0's end; first is 0. + Err(i) => (i, self.slices[i-1].end), // desired start < slice i's end; first is i. + }; + + // Iterate through and write each slice until the end. + let mut start_pos = range.start - slice_start; + loop { + let s = &self.slices[i]; + let l = s.end - slice_start; + if range.end <= s.end { // last slice. + return s.writer.write_to(ctx, start_pos .. range.end - slice_start, l, out); + } + s.writer.write_to(ctx, start_pos .. s.end - slice_start, l, out)?; + + // setup next iteration. + start_pos = 0; + slice_start = s.end; + i += 1; + } + } +} + +#[cfg(test)] +mod tests { + use error::{Error, Result}; + use std::cell::RefCell; + use std::error::Error as E; + use std::io::Write; + use std::ops::Range; + use std::vec::Vec; + use super::{ContextWriter, Slices, clip_to_range}; + + #[derive(Debug, Eq, PartialEq)] + pub struct FakeWrite { + writer: &'static str, + range: Range, + } + + pub struct FakeWriter { + name: &'static str, + } + + impl ContextWriter>> for FakeWriter { + fn write_to(&self, ctx: &RefCell>, r: Range, _l: u64, _out: &mut Write) + -> Result<()> { + ctx.borrow_mut().push(FakeWrite{writer: self.name, range: r}); + Ok(()) + } + } + + pub fn new_slices() -> Slices>> { + let mut s = Slices::new(); + s.append(5, FakeWriter{name: "a"}); + s.append(13, FakeWriter{name: "b"}); + s.append(7, FakeWriter{name: "c"}); + s.append(17, FakeWriter{name: "d"}); + s.append(19, FakeWriter{name: "e"}); + s + } + + #[test] + pub fn size() { + assert_eq!(5 + 13 + 7 + 17 + 19, new_slices().len()); + } + + #[test] + pub fn exact_slice() { + // Test writing exactly slice b. + let s = new_slices(); + let w = RefCell::new(Vec::new()); + let mut dummy = Vec::new(); + s.write_to(&w, 5 .. 18, &mut dummy).unwrap(); + assert_eq!(&[FakeWrite{writer: "b", range: 0 .. 13}], &w.borrow()[..]); + } + + #[test] + pub fn offset_first() { + // Test writing part of slice a. + let s = new_slices(); + let w = RefCell::new(Vec::new()); + let mut dummy = Vec::new(); + s.write_to(&w, 1 .. 3, &mut dummy).unwrap(); + assert_eq!(&[FakeWrite{writer: "a", range: 1 .. 3}], &w.borrow()[..]); + } + + #[test] + pub fn offset_mid() { + // Test writing part of slice b, all of slice c, and part of slice d. + let s = new_slices(); + let w = RefCell::new(Vec::new()); + let mut dummy = Vec::new(); + s.write_to(&w, 17 .. 26, &mut dummy).unwrap(); + assert_eq!(&[ + FakeWrite{writer: "b", range: 12 .. 13}, + FakeWrite{writer: "c", range: 0 .. 7}, + FakeWrite{writer: "d", range: 0 .. 1}, + ], &w.borrow()[..]); + } + + #[test] + pub fn everything() { + // Test writing the whole Slices. + let s = new_slices(); + let w = RefCell::new(Vec::new()); + let mut dummy = Vec::new(); + s.write_to(&w, 0 .. 61, &mut dummy).unwrap(); + assert_eq!(&[ + FakeWrite{writer: "a", range: 0 .. 5}, + FakeWrite{writer: "b", range: 0 .. 13}, + FakeWrite{writer: "c", range: 0 .. 7}, + FakeWrite{writer: "d", range: 0 .. 17}, + FakeWrite{writer: "e", range: 0 .. 19}, + ], &w.borrow()[..]); + } + + #[test] + pub fn at_end() { + let s = new_slices(); + let w = RefCell::new(Vec::new()); + let mut dummy = Vec::new(); + s.write_to(&w, 61 .. 61, &mut dummy).unwrap(); + let empty: &[FakeWrite] = &[]; + assert_eq!(empty, &w.borrow()[..]); + } + + #[test] + pub fn test_clip_to_range() { + let mut out = Vec::new(); + + // Simple case: one write with everything. + clip_to_range(0 .. 5, 5, &mut out, |w| { + w.write_all(b"01234").unwrap(); + Ok(()) + }).unwrap(); + assert_eq!(b"01234", &out[..]); + + // Same in a few writes. + out.clear(); + clip_to_range(0 .. 5, 5, &mut out, |w| { + w.write_all(b"0").unwrap(); + w.write_all(b"123").unwrap(); + w.write_all(b"4").unwrap(); + Ok(()) + }).unwrap(); + assert_eq!(b"01234", &out[..]); + + // Limiting to a prefix. + out.clear(); + clip_to_range(0 .. 2, 5, &mut out, |w| { + w.write_all(b"0").unwrap(); // all of this write + w.write_all(b"123").unwrap(); // some of this write + w.write_all(b"4").unwrap(); // none of this write + Ok(()) + }).unwrap(); + assert_eq!(b"01", &out[..]); + + // Limiting to part in the middle. + out.clear(); + clip_to_range(2 .. 4, 5, &mut out, |w| { + w.write_all(b"0").unwrap(); // none of this write + w.write_all(b"1234").unwrap(); // middle of this write + w.write_all(b"5678").unwrap(); // none of this write + Ok(()) + }).unwrap(); + assert_eq!(b"23", &out[..]); + + // If the callback returns an error, it should be propagated (fast path or not). + out.clear(); + assert_eq!( + clip_to_range(0 .. 4, 4, &mut out, |_| Err(Error::new("some error".to_owned()))) + .unwrap_err().description(), + "some error"); + out.clear(); + assert_eq!( + clip_to_range(0 .. 1, 4, &mut out, |_| Err(Error::new("some error".to_owned()))) + .unwrap_err().description(), + "some error"); + + // TODO: if inner.write does a partial write, the next try should start at the correct + // position. + } +} diff --git a/src/profiler.cc b/src/profiler.cc deleted file mode 100644 index 850f0c2..0000000 --- a/src/profiler.cc +++ /dev/null @@ -1,145 +0,0 @@ -// This file is part of Moonfire NVR, a security camera digital video recorder. -// Copyright (C) 2016 Scott Lamb -// -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// In addition, as a special exception, the copyright holders give -// permission to link the code of portions of this program with the -// OpenSSL library under certain conditions as described in each -// individual source file, and distribute linked combinations including -// the two. -// -// You must obey the GNU General Public License in all respects for all -// of the code used other than OpenSSL. If you modify file(s) with this -// exception, you may extend this exception to your version of the -// file(s), but you are not obligated to do so. If you do not wish to do -// so, delete this exception statement from your version. If you delete -// this exception statement from all source files in the program, then -// also delete it here. -// -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. -// -// You should have received a copy of the GNU General Public License -// along with this program. If not, see . -// -// profiler.cc: See profiler.h. - -#include "profiler.h" - -#include -#include -#include -#include -#include -#include -#include - -#include -#include - -#include -#include -#include -#include -#include - -#include "http.h" -#include "string.h" - -namespace moonfire_nvr { - -namespace { - -const int kDefaultProfileSeconds = 30; - -// Only a single CPU profile may be active at once. Track if it is active now. -bool profiling; - -struct ProfileRequestContext { -#define TEMPLATE "/tmp/moonfire-nvr.profile.XXXXXX" - char filename[sizeof(TEMPLATE)] = TEMPLATE; -#undef TEMPLATE - evhttp_request *req = nullptr; - event *timer = nullptr; - int fd = -1; -}; - -// End a CPU profile. Serve the result from the temporary file and delete it. -void EndProfileCallback(evutil_socket_t, short, void *arg) { - CHECK(profiling); - ProfilerStop(); - profiling = false; - std::unique_ptr ctx( - reinterpret_cast(arg)); - if (unlink(ctx->filename) < 0) { - int err = errno; - LOG(WARNING) << "Unable to unlink temporary profile file: " << ctx->filename - << ": " << strerror(err); - } - event_free(ctx->timer); - struct stat statbuf; - if (fstat(ctx->fd, &statbuf) < 0) { - close(ctx->fd); - return HttpSendError(ctx->req, HTTP_INTERNAL, "fstat: ", errno); - } - EvBuffer buf; - std::string error_message; - if (!buf.AddFile(ctx->fd, 0, statbuf.st_size, &error_message)) { - evhttp_send_error(ctx->req, HTTP_INTERNAL, - EscapeHtml(error_message).c_str()); - close(ctx->fd); - return; - } - evhttp_send_reply(ctx->req, HTTP_OK, "OK", buf.get()); -} - -// Start a CPU profile. Creates a temporary file for the profiler library -// to use and schedules a call to EndProfileCallback. -void StartProfileCallback(struct evhttp_request *req, void *arg) { - auto *base = reinterpret_cast(arg); - if (evhttp_request_get_command(req) != EVHTTP_REQ_GET) { - return evhttp_send_error(req, HTTP_BADMETHOD, "only GET allowed"); - } - if (profiling) { - return evhttp_send_error(req, HTTP_SERVUNAVAIL, - "Profiling already in progress"); - } - struct timeval timeout = {0, 0}; - QueryParameters params(evhttp_request_get_uri(req)); - const char *seconds_value = params.Get("seconds"); - timeout.tv_sec = - seconds_value == nullptr ? kDefaultProfileSeconds : atoi(seconds_value); - if (timeout.tv_sec <= 0) { - return evhttp_send_error(req, HTTP_BADREQUEST, "invalid seconds"); - } - - auto *ctx = new ProfileRequestContext; - ctx->fd = mkstemp(ctx->filename); - if (ctx->fd < 0) { - delete ctx; - return HttpSendError(req, HTTP_INTERNAL, "mkstemp: ", errno); - } - - if (ProfilerStart(ctx->filename) == 0) { - delete ctx; - return evhttp_send_error(req, HTTP_INTERNAL, "ProfilerStart failed"); - } - profiling = true; - ctx->req = req; - ctx->timer = evtimer_new(base, &EndProfileCallback, ctx); - evtimer_add(ctx->timer, &timeout); -} - -} // namespace - -void RegisterProfiler(event_base *base, evhttp *http) { - evhttp_set_cb(http, "/pprof/profile", &StartProfileCallback, base); -} - -} // namespace moonfire_nvr diff --git a/src/recording-bench.cc b/src/recording-bench.cc deleted file mode 100644 index 9abc1db..0000000 --- a/src/recording-bench.cc +++ /dev/null @@ -1,66 +0,0 @@ -// This file is part of Moonfire NVR, a security camera network video recorder. -// Copyright (C) 2016 Scott Lamb -// -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// In addition, as a special exception, the copyright holders give -// permission to link the code of portions of this program with the -// OpenSSL library under certain conditions as described in each -// individual source file, and distribute linked combinations including -// the two. -// -// You must obey the GNU General Public License in all respects for all -// of the code used other than OpenSSL. If you modify file(s) with this -// exception, you may extend this exception to your version of the -// file(s), but you are not obligated to do so. If you do not wish to do -// so, delete this exception statement from your version. If you delete -// this exception statement from all source files in the program, then -// also delete it here. -// -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. -// -// You should have received a copy of the GNU General Public License -// along with this program. If not, see . -// -// recording-bench.cc: benchmarks of the recording.h interface. - -#include -#include - -#include "recording.h" -#include "testutil.h" - -DECLARE_bool(alsologtostderr); - -static void BM_Iterator(benchmark::State &state) { - using moonfire_nvr::ReadFileOrDie; - using moonfire_nvr::SampleIndexIterator; - // state.PauseTiming(); - std::string index = ReadFileOrDie("../src/testdata/video_sample_index.bin"); - // state.ResumeTiming(); - while (state.KeepRunning()) { - SampleIndexIterator it(index); - while (!it.done()) it.Next(); - CHECK(!it.has_error()) << it.error(); - } - state.SetBytesProcessed(int64_t(state.iterations()) * int64_t(index.size())); -} -BENCHMARK(BM_Iterator); - -int main(int argc, char **argv) { - FLAGS_alsologtostderr = true; - - // Sadly, these two flag-parsing libraries don't appear to get along. - // google::ParseCommandLineFlags(&argc, &argv, true); - benchmark::Initialize(&argc, argv); - - google::InitGoogleLogging(argv[0]); - benchmark::RunSpecifiedBenchmarks(); - return 0; -} diff --git a/src/recording-test.cc b/src/recording-test.cc deleted file mode 100644 index c52df0f..0000000 --- a/src/recording-test.cc +++ /dev/null @@ -1,264 +0,0 @@ -// This file is part of Moonfire NVR, a security camera network video recorder. -// Copyright (C) 2016 Scott Lamb -// -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// In addition, as a special exception, the copyright holders give -// permission to link the code of portions of this program with the -// OpenSSL library under certain conditions as described in each -// individual source file, and distribute linked combinations including -// the two. -// -// You must obey the GNU General Public License in all respects for all -// of the code used other than OpenSSL. If you modify file(s) with this -// exception, you may extend this exception to your version of the -// file(s), but you are not obligated to do so. If you do not wish to do -// so, delete this exception statement from your version. If you delete -// this exception statement from all source files in the program, then -// also delete it here. -// -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. -// -// You should have received a copy of the GNU General Public License -// along with this program. If not, see . -// -// recording-test.cc: tests of the recording.h interface. - -#include -#include -#include - -#include -#include -#include - -#include "recording.h" -#include "string.h" -#include "testutil.h" - -DECLARE_bool(alsologtostderr); - -using testing::_; -using testing::HasSubstr; -using testing::DoAll; -using testing::Return; -using testing::SetArgPointee; - -namespace moonfire_nvr { -namespace { - -// Example from design/schema.md. -TEST(SampleIndexTest, EncodeExample) { - Recording recording; - SampleIndexEncoder encoder; - encoder.Init(&recording, 1000); - encoder.AddSample(10, 1000, true); - encoder.AddSample(9, 10, false); - encoder.AddSample(11, 15, false); - encoder.AddSample(10, 12, false); - encoder.AddSample(10, 1050, true); - EXPECT_EQ("29 d0 0f 02 14 08 0a 02 05 01 64", - ToHex(recording.video_index, true)); - EXPECT_EQ(1000, recording.start_time_90k); - EXPECT_EQ(1000 + 10 + 9 + 11 + 10 + 10, recording.end_time_90k); - EXPECT_EQ(1000 + 10 + 15 + 12 + 1050, recording.sample_file_bytes); - EXPECT_EQ(5, recording.video_samples); - EXPECT_EQ(2, recording.video_sync_samples); -} - -TEST(SampleIndexTest, RoundTrip) { - Recording recording; - SampleIndexEncoder encoder; - encoder.Init(&recording, 1000); - encoder.AddSample(10, 30000, true); - encoder.AddSample(9, 1000, false); - encoder.AddSample(11, 1100, false); - encoder.AddSample(18, 31000, true); - - SampleIndexIterator it = SampleIndexIterator(recording.video_index); - std::string error_message; - ASSERT_FALSE(it.done()) << it.error(); - EXPECT_EQ(10, it.duration_90k()); - EXPECT_EQ(30000, it.bytes()); - EXPECT_TRUE(it.is_key()); - - it.Next(); - ASSERT_FALSE(it.done()) << it.error(); - EXPECT_EQ(9, it.duration_90k()); - EXPECT_EQ(1000, it.bytes()); - EXPECT_FALSE(it.is_key()); - - it.Next(); - ASSERT_FALSE(it.done()) << it.error(); - EXPECT_EQ(11, it.duration_90k()); - EXPECT_EQ(1100, it.bytes()); - EXPECT_FALSE(it.is_key()); - - it.Next(); - ASSERT_FALSE(it.done()) << it.error(); - EXPECT_EQ(18, it.duration_90k()); - EXPECT_EQ(31000, it.bytes()); - EXPECT_TRUE(it.is_key()); - - it.Next(); - ASSERT_TRUE(it.done()); - ASSERT_FALSE(it.has_error()) << it.error(); -} - -TEST(SampleIndexTest, IteratorErrors) { - std::string bad_first_varint("\x80"); - SampleIndexIterator it(bad_first_varint); - EXPECT_TRUE(it.has_error()); - EXPECT_EQ("buffer underrun", it.error()); - - std::string bad_second_varint("\x00\x80", 2); - it = SampleIndexIterator(bad_second_varint); - EXPECT_TRUE(it.has_error()); - EXPECT_EQ("buffer underrun", it.error()); - - std::string zero_durations("\x00\x02\x00\x00", 4); - it = SampleIndexIterator(zero_durations); - EXPECT_TRUE(it.has_error()); - EXPECT_THAT(it.error(), HasSubstr("zero duration")); - - std::string negative_duration("\x02\x02", 2); - it = SampleIndexIterator(negative_duration); - EXPECT_TRUE(it.has_error()); - EXPECT_THAT(it.error(), HasSubstr("negative duration")); - - std::string non_positive_bytes("\x04\x00", 2); - it = SampleIndexIterator(non_positive_bytes); - EXPECT_TRUE(it.has_error()); - EXPECT_THAT(it.error(), HasSubstr("non-positive bytes")); -} - -TEST(SampleFileWriterTest, Simple) { - testing::StrictMock parent; - auto *f = new testing::StrictMock; - - re2::StringPiece write_1("write 1"); - re2::StringPiece write_2("write 2"); - - EXPECT_CALL(parent, OpenRaw("foo", O_WRONLY | O_EXCL | O_CREAT, 0600, _)) - .WillOnce(DoAll(SetArgPointee<3>(f), Return(0))); - EXPECT_CALL(*f, Write(write_1, _)) - .WillOnce(DoAll(SetArgPointee<1>(7), Return(0))); - EXPECT_CALL(*f, Write(write_2, _)) - .WillOnce(DoAll(SetArgPointee<1>(7), Return(0))); - EXPECT_CALL(*f, Sync()).WillOnce(Return(0)); - EXPECT_CALL(*f, Close()).WillOnce(Return(0)); - - SampleFileWriter writer(&parent); - std::string error_message; - std::string sha1; - ASSERT_TRUE(writer.Open("foo", &error_message)) << error_message; - EXPECT_TRUE(writer.Write(write_1, &error_message)) << error_message; - EXPECT_TRUE(writer.Write(write_2, &error_message)) << error_message; - EXPECT_TRUE(writer.Close(&sha1, &error_message)) << error_message; - EXPECT_EQ("6bc37325b36fb5fd205e57284429e75764338618", ToHex(sha1)); -} - -TEST(SampleFileWriterTest, PartialWriteIsRetried) { - testing::StrictMock parent; - auto *f = new testing::StrictMock; - - re2::StringPiece write_1("write 1"); - re2::StringPiece write_2("write 2"); - re2::StringPiece write_2b(write_2); - write_2b.remove_prefix(3); - - EXPECT_CALL(parent, OpenRaw("foo", O_WRONLY | O_EXCL | O_CREAT, 0600, _)) - .WillOnce(DoAll(SetArgPointee<3>(f), Return(0))); - EXPECT_CALL(*f, Write(write_1, _)) - .WillOnce(DoAll(SetArgPointee<1>(7), Return(0))); - EXPECT_CALL(*f, Write(write_2, _)) - .WillOnce(DoAll(SetArgPointee<1>(3), Return(0))); - EXPECT_CALL(*f, Write(write_2b, _)) - .WillOnce(DoAll(SetArgPointee<1>(4), Return(0))); - EXPECT_CALL(*f, Sync()).WillOnce(Return(0)); - EXPECT_CALL(*f, Close()).WillOnce(Return(0)); - - SampleFileWriter writer(&parent); - std::string error_message; - std::string sha1; - ASSERT_TRUE(writer.Open("foo", &error_message)) << error_message; - EXPECT_TRUE(writer.Write(write_1, &error_message)) << error_message; - EXPECT_TRUE(writer.Write(write_2, &error_message)) << error_message; - EXPECT_TRUE(writer.Close(&sha1, &error_message)) << error_message; - EXPECT_EQ("6bc37325b36fb5fd205e57284429e75764338618", ToHex(sha1)); -} - -TEST(SampleFileWriterTest, PartialWriteIsTruncated) { - testing::StrictMock parent; - auto *f = new testing::StrictMock; - - re2::StringPiece write_1("write 1"); - re2::StringPiece write_2("write 2"); - re2::StringPiece write_2b(write_2); - write_2b.remove_prefix(3); - - EXPECT_CALL(parent, OpenRaw("foo", O_WRONLY | O_EXCL | O_CREAT, 0600, _)) - .WillOnce(DoAll(SetArgPointee<3>(f), Return(0))); - EXPECT_CALL(*f, Write(write_1, _)) - .WillOnce(DoAll(SetArgPointee<1>(7), Return(0))); - EXPECT_CALL(*f, Write(write_2, _)) - .WillOnce(DoAll(SetArgPointee<1>(3), Return(0))); - EXPECT_CALL(*f, Write(write_2b, _)).WillOnce(Return(ENOSPC)); - EXPECT_CALL(*f, Truncate(7)).WillOnce(Return(0)); - EXPECT_CALL(*f, Sync()).WillOnce(Return(0)); - EXPECT_CALL(*f, Close()).WillOnce(Return(0)); - - SampleFileWriter writer(&parent); - std::string error_message; - std::string sha1; - ASSERT_TRUE(writer.Open("foo", &error_message)) << error_message; - EXPECT_TRUE(writer.Write(write_1, &error_message)) << error_message; - EXPECT_FALSE(writer.Write(write_2, &error_message)) << error_message; - EXPECT_TRUE(writer.Close(&sha1, &error_message)) << error_message; - EXPECT_EQ("b1ccee339b935587c09997a9ec8bb2374e02b5d0", ToHex(sha1)); -} - -TEST(SampleFileWriterTest, PartialWriteTruncateFailureCausesCloseToFail) { - testing::StrictMock parent; - auto *f = new testing::StrictMock; - - re2::StringPiece write_1("write 1"); - re2::StringPiece write_2("write 2"); - re2::StringPiece write_2b(write_2); - write_2b.remove_prefix(3); - - EXPECT_CALL(parent, OpenRaw("foo", O_WRONLY | O_EXCL | O_CREAT, 0600, _)) - .WillOnce(DoAll(SetArgPointee<3>(f), Return(0))); - EXPECT_CALL(*f, Write(write_1, _)) - .WillOnce(DoAll(SetArgPointee<1>(7), Return(0))); - EXPECT_CALL(*f, Write(write_2, _)) - .WillOnce(DoAll(SetArgPointee<1>(3), Return(0))); - EXPECT_CALL(*f, Write(write_2b, _)).WillOnce(Return(EIO)); - EXPECT_CALL(*f, Truncate(7)).WillOnce(Return(EIO)); - EXPECT_CALL(*f, Close()).WillOnce(Return(0)); - - SampleFileWriter writer(&parent); - std::string error_message; - std::string sha1; - ASSERT_TRUE(writer.Open("foo", &error_message)) << error_message; - EXPECT_TRUE(writer.Write(write_1, &error_message)) << error_message; - EXPECT_FALSE(writer.Write(write_2, &error_message)) << error_message; - EXPECT_FALSE(writer.Close(&sha1, &error_message)) << error_message; -} - -} // namespace -} // namespace moonfire_nvr - -int main(int argc, char **argv) { - FLAGS_alsologtostderr = true; - google::ParseCommandLineFlags(&argc, &argv, true); - testing::InitGoogleTest(&argc, argv); - google::InitGoogleLogging(argv[0]); - return RUN_ALL_TESTS(); -} diff --git a/src/recording.cc b/src/recording.cc deleted file mode 100644 index 323511b..0000000 --- a/src/recording.cc +++ /dev/null @@ -1,229 +0,0 @@ -// This file is part of Moonfire NVR, a security camera network video recorder. -// Copyright (C) 2015 Scott Lamb -// -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// In addition, as a special exception, the copyright holders give -// permission to link the code of portions of this program with the -// OpenSSL library under certain conditions as described in each -// individual source file, and distribute linked combinations including -// the two. -// -// You must obey the GNU General Public License in all respects for all -// of the code used other than OpenSSL. If you modify file(s) with this -// exception, you may extend this exception to your version of the -// file(s), but you are not obligated to do so. If you do not wish to do -// so, delete this exception statement from your version. If you delete -// this exception statement from all source files in the program, then -// also delete it here. -// -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. -// -// You should have received a copy of the GNU General Public License -// along with this program. If not, see . -// -// recording.cc: see recording.h. - -#include "recording.h" - -#include -#include -#include - -#include "common.h" -#include "coding.h" -#include "string.h" - -namespace moonfire_nvr { - -void SampleIndexEncoder::Init(Recording *recording, int64_t start_time_90k) { - recording_ = recording; - recording_->start_time_90k = start_time_90k; - recording_->end_time_90k = start_time_90k; - recording_->sample_file_bytes = 0; - recording_->video_samples = 0; - recording_->video_sync_samples = 0; - recording_->video_index.clear(); - prev_duration_90k_ = 0; - prev_bytes_key_ = 0; - prev_bytes_nonkey_ = 0; -} - -void SampleIndexEncoder::AddSample(int32_t duration_90k, int32_t bytes, - bool is_key) { - CHECK_GE(duration_90k, 0); - CHECK_GT(bytes, 0); - int32_t duration_delta = duration_90k - prev_duration_90k_; - prev_duration_90k_ = duration_90k; - int32_t bytes_delta; - recording_->end_time_90k += duration_90k; - recording_->sample_file_bytes += bytes; - ++recording_->video_samples; - if (is_key) { - bytes_delta = bytes - prev_bytes_key_; - prev_bytes_key_ = bytes; - ++recording_->video_sync_samples; - } else { - bytes_delta = bytes - prev_bytes_nonkey_; - prev_bytes_nonkey_ = bytes; - } - uint32_t zigzagged_bytes_delta = Zigzag32(bytes_delta); - AppendVar32((Zigzag32(duration_delta) << 1) | is_key, - &recording_->video_index); - AppendVar32(zigzagged_bytes_delta, &recording_->video_index); -} - -void SampleIndexIterator::Next() { - uint32_t raw1; - uint32_t raw2; - pos_ += bytes_; - if (UNLIKELY(data_.empty()) || - UNLIKELY(!DecodeVar32(&data_, &raw1, &error_)) || - UNLIKELY(!DecodeVar32(&data_, &raw2, &error_))) { - done_ = true; - return; - } - start_90k_ += duration_90k_; - int32_t duration_90k_delta = Unzigzag32(raw1 >> 1); - duration_90k_ += duration_90k_delta; - if (UNLIKELY(duration_90k_ < 0)) { - error_ = StrCat("negative duration ", duration_90k_, - " after applying delta ", duration_90k_delta); - done_ = true; - return; - } - if (UNLIKELY(duration_90k_ == 0 && !data_.empty())) { - error_ = StrCat("zero duration only allowed at end; have ", data_.size(), - "bytes left."); - done_ = true; - return; - } - is_key_ = raw1 & 0x01; - int32_t bytes_delta = Unzigzag32(raw2); - if (UNLIKELY(is_key_)) { - bytes_ = bytes_key_ += bytes_delta; - } else { - bytes_ = bytes_nonkey_ += bytes_delta; - } - if (UNLIKELY(bytes_ <= 0)) { - error_ = StrCat("non-positive bytes ", bytes_, " after applying delta ", - bytes_delta, " to ", (is_key_ ? "key" : "non-key"), - " frame at ts ", start_90k_); - done_ = true; - return; - } - done_ = false; - return; -} - -void SampleIndexIterator::Clear() { - data_.clear(); - error_.clear(); - pos_ = 0; - start_90k_ = 0; - duration_90k_ = 0; - bytes_key_ = 0; - bytes_nonkey_ = 0; - bytes_ = 0; - is_key_ = false; - done_ = true; -} - -SampleFileWriter::SampleFileWriter(File *parent_dir) - : parent_dir_(parent_dir), sha1_(Digest::SHA1()) {} - -bool SampleFileWriter::Open(const char *filename, std::string *error_message) { - if (is_open()) { - *error_message = "already open!"; - return false; - } - int ret = - parent_dir_->Open(filename, O_WRONLY | O_CREAT | O_EXCL, 0600, &file_); - if (ret != 0) { - *error_message = StrCat("open ", filename, " (within dir ", - parent_dir_->name(), "): ", strerror(ret)); - return false; - } - return true; -} - -bool SampleFileWriter::Write(re2::StringPiece pkt, std::string *error_message) { - if (!is_open()) { - *error_message = "not open!"; - return false; - } - auto old_pos = pos_; - re2::StringPiece remaining(pkt); - while (!remaining.empty()) { - size_t written; - int write_ret = file_->Write(remaining, &written); - if (write_ret != 0) { - if (pos_ > old_pos) { - int truncate_ret = file_->Truncate(old_pos); - if (truncate_ret != 0) { - *error_message = - StrCat("write failed with: ", strerror(write_ret), - " and ftruncate failed with: ", strerror(truncate_ret)); - corrupt_ = true; - return false; - } - } - *error_message = StrCat("write: ", strerror(write_ret)); - return false; - } - remaining.remove_prefix(written); - pos_ += written; - } - sha1_->Update(pkt); - return true; -} - -bool SampleFileWriter::Close(std::string *sha1, std::string *error_message) { - if (!is_open()) { - *error_message = "not open!"; - return false; - } - - if (corrupt_) { - *error_message = "File already corrupted."; - } else { - int ret = file_->Sync(); - if (ret != 0) { - *error_message = StrCat("fsync failed with: ", strerror(ret)); - corrupt_ = true; - } - } - - int ret = file_->Close(); - if (ret != 0 && !corrupt_) { - corrupt_ = true; - *error_message = StrCat("close failed with: ", strerror(ret)); - } - - bool ok = !corrupt_; - file_.reset(); - *sha1 = sha1_->Finalize(); - sha1_ = Digest::SHA1(); - pos_ = 0; - corrupt_ = false; - return ok; -} - -std::string PrettyTimestamp(int64_t ts_90k) { - struct tm mytm; - memset(&mytm, 0, sizeof(mytm)); - time_t ts = ts_90k / kTimeUnitsPerSecond; - localtime_r(&ts, &mytm); - const size_t kTimeBufLen = 50; - char tmbuf[kTimeBufLen]; - strftime(tmbuf, kTimeBufLen, "%a, %d %b %Y %H:%M:%S %Z", &mytm); - return tmbuf; -} - -} // namespace moonfire_nvr diff --git a/src/recording.h b/src/recording.h deleted file mode 100644 index a764e4a..0000000 --- a/src/recording.h +++ /dev/null @@ -1,214 +0,0 @@ -// This file is part of Moonfire NVR, a security camera network video recorder. -// Copyright (C) 2016 Scott Lamb -// -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// In addition, as a special exception, the copyright holders give -// permission to link the code of portions of this program with the -// OpenSSL library under certain conditions as described in each -// individual source file, and distribute linked combinations including -// the two. -// -// You must obey the GNU General Public License in all respects for all -// of the code used other than OpenSSL. If you modify file(s) with this -// exception, you may extend this exception to your version of the -// file(s), but you are not obligated to do so. If you do not wish to do -// so, delete this exception statement from your version. If you delete -// this exception statement from all source files in the program, then -// also delete it here. -// -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. -// -// You should have received a copy of the GNU General Public License -// along with this program. If not, see . -// -// recording.h: Write and read recordings. See design/schema.md for a -// description of the storage schema. - -#ifndef MOONFIRE_NVR_RECORDING_H -#define MOONFIRE_NVR_RECORDING_H - -#include - -#include -#include - -#include -#include - -#include "crypto.h" -#include "filesystem.h" -#include "uuid.h" - -namespace moonfire_nvr { - -constexpr int64_t kTimeUnitsPerSecond = 90000; - -// Recordings are never longer than this (5 minutes). -// Having such a limit dramatically speeds up some SQL queries. -// This limit should be more than the normal rotation time, -// as recording doesn't happen until the next key frame. -// 5 minutes is generously more than 1 minute, but still sufficient to -// allow the optimization to be useful. This value must match the CHECK -// constraint on duration_90k in schema.sql. -constexpr int64_t kMaxRecordingDuration = 5 * 60 * kTimeUnitsPerSecond; - -// Various fields from the "recording" table which are useful when viewing -// recordings. -struct Recording { - int64_t id = -1; - int64_t camera_id = -1; - std::string sample_file_sha1; - std::string sample_file_path; - Uuid sample_file_uuid; - int64_t video_sample_entry_id = -1; - int64_t local_time_90k = -1; - - // Fields populated by SampleIndexEncoder. - int64_t start_time_90k = -1; - int64_t end_time_90k = -1; - int64_t sample_file_bytes = -1; - int64_t video_samples = -1; - int64_t video_sync_samples = -1; - std::string video_index; -}; - -// Reusable object to encode sample index data to a Recording object. -class SampleIndexEncoder { - public: - SampleIndexEncoder() {} - SampleIndexEncoder(const SampleIndexEncoder &) = delete; - void operator=(const SampleIndexEncoder &) = delete; - - void Init(Recording *recording, int64_t start_time_90k); - void AddSample(int32_t duration_90k, int32_t bytes, bool is_key); - - private: - Recording *recording_; - int32_t prev_duration_90k_ = 0; - int32_t prev_bytes_key_ = 0; - int32_t prev_bytes_nonkey_ = 0; -}; - -// Iterates through an encoded index, decoding on the fly. Copyable. -// Example usage: -// -// SampleIndexIterator it; -// for (it = index; !it.done(); it.Next()) { -// LOG(INFO) << "sample size: " << it.bytes(); -// } -// if (it.has_error()) { -// LOG(ERROR) << "error: " << it.error(); -// } -class SampleIndexIterator { - public: - SampleIndexIterator() { Clear(); } - - // |index| must outlive the iterator. - explicit SampleIndexIterator(re2::StringPiece index) { - Clear(); - data_ = index; - Next(); - } - - // Iteration control. - void Next(); - bool done() const { return done_; } - bool has_error() const { return !error_.empty(); } - const std::string &error() const { return error_; } - - // Return properties of the current sample. - // Note pos() and start_90k() are valid when done(); the others are not. - int64_t pos() const { return pos_; } - int32_t start_90k() const { return start_90k_; } - int32_t duration_90k() const { - DCHECK(!done_); - return duration_90k_; - } - int32_t end_90k() const { return start_90k_ + duration_90k(); } - int32_t bytes() const { - DCHECK(!done_); - return bytes_; - } - bool is_key() const { - DCHECK(!done_); - return is_key_; - } - - private: - void Clear(); - - re2::StringPiece data_; - std::string error_; - int64_t pos_; - int32_t start_90k_; - int32_t duration_90k_; - int32_t bytes_; // bytes taken by the current sample, or 0 after Clear(). - int32_t bytes_key_; - int32_t bytes_nonkey_; - bool is_key_; - bool done_; -}; - -// Writes a sample file. Can be used repeatedly. Thread-compatible. -class SampleFileWriter { - public: - // |parent_dir| must outlive the writer. - SampleFileWriter(File *parent_dir); - SampleFileWriter(const SampleFileWriter &) = delete; - void operator=(const SampleFileWriter &) = delete; - - // PRE: !is_open(). - bool Open(const char *filename, std::string *error_message); - - // Writes a single packet, returning success. - // On failure, the stream should be closed. If Close() returns true, the - // file contains the results of all packets up to (but not including) this - // one. - // - // PRE: is_open(). - bool Write(re2::StringPiece pkt, std::string *error_message); - - // fsync() and close() the stream. - // Note the caller is still responsible for fsync()ing the parent stream, - // so that operations can be batched. - // On success, |sha1| will be filled with the raw SHA-1 hash of the file. - // On failure, the file should be considered corrupt and discarded. - // - // PRE: is_open(). - bool Close(std::string *sha1, std::string *error_message); - - bool is_open() const { return file_ != nullptr; } - - private: - File *parent_dir_; - std::unique_ptr file_; - std::unique_ptr sha1_; - int64_t pos_ = 0; - bool corrupt_ = false; -}; - -struct VideoSampleEntry { - int64_t id = -1; - std::string sha1; - std::string data; - uint16_t width = 0; - uint16_t height = 0; -}; - -std::string PrettyTimestamp(int64_t ts_90k); - -inline int64_t To90k(const struct timespec &ts) { - return (ts.tv_sec * kTimeUnitsPerSecond) + - (ts.tv_nsec * kTimeUnitsPerSecond / 1000000000); -} - -} // namespace moonfire_nvr - -#endif // MOONFIRE_NVR_RECORDING_H diff --git a/src/recording.rs b/src/recording.rs new file mode 100644 index 0000000..ccb5070 --- /dev/null +++ b/src/recording.rs @@ -0,0 +1,726 @@ +// This file is part of Moonfire NVR, a security camera digital video recorder. +// Copyright (C) 2016 Scott Lamb +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// In addition, as a special exception, the copyright holders give +// permission to link the code of portions of this program with the +// OpenSSL library under certain conditions as described in each +// individual source file, and distribute linked combinations including +// the two. +// +// You must obey the GNU General Public License in all respects for all +// of the code used other than OpenSSL. If you modify file(s) with this +// exception, you may extend this exception to your version of the +// file(s), but you are not obligated to do so. If you do not wish to do +// so, delete this exception statement from your version. If you delete +// this exception statement from all source files in the program, then +// also delete it here. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program. If not, see . + +#![allow(inline_always)] + +extern crate uuid; + +use db; +use std::ops; +use error::Error; +use openssl::crypto::hash; +use std::fmt; +use std::fs; +use std::io::Write; +use std::ops::Range; +use std::string::String; +use time; +use uuid::Uuid; + +pub const TIME_UNITS_PER_SEC: i64 = 90000; +pub const DESIRED_RECORDING_DURATION: i64 = 60 * TIME_UNITS_PER_SEC; +pub const MAX_RECORDING_DURATION: i64 = 5 * 60 * TIME_UNITS_PER_SEC; + +/// A time specified as 90,000ths of a second since 1970-01-01 00:00:00 UTC. +#[derive(Clone, Copy, Debug, Eq, Ord, PartialEq, PartialOrd, Serialize)] +pub struct Time(pub i64); + +impl Time { + pub fn new(tm: time::Timespec) -> Self { + Time(tm.sec * TIME_UNITS_PER_SEC + tm.nsec as i64 * TIME_UNITS_PER_SEC / 1_000_000_000) + } + + pub fn unix_seconds(&self) -> i64 { self.0 / TIME_UNITS_PER_SEC } +} + +impl ops::Sub for Time { + type Output = Duration; + fn sub(self, rhs: Time) -> Duration { Duration(self.0 - rhs.0) } +} + +impl ops::AddAssign for Time { + fn add_assign(&mut self, rhs: Duration) { self.0 += rhs.0 } +} + +impl ops::Add for Time { + type Output = Time; + fn add(self, rhs: Duration) -> Time { Time(self.0 + rhs.0) } +} + +impl ops::Sub for Time { + type Output = Time; + fn sub(self, rhs: Duration) -> Time { Time(self.0 - rhs.0) } +} + +impl fmt::Display for Time { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + let tm = time::at(time::Timespec{sec: self.0 / TIME_UNITS_PER_SEC, nsec: 0}); + write!(f, "{}:{:05}", tm.strftime("%FT%T%Z").or_else(|_| Err(fmt::Error))?, + self.0 % TIME_UNITS_PER_SEC) + } +} + +/// A duration specified in 1/90,000ths of a second. +/// Durations are typically non-negative, but a `db::CameraDayValue::duration` may be negative. +#[derive(Clone, Copy, Debug, Eq, Ord, PartialEq, PartialOrd, Serialize)] +pub struct Duration(pub i64); + +impl fmt::Display for Duration { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + let mut seconds = self.0 / TIME_UNITS_PER_SEC; + const MINUTE_IN_SECONDS: i64 = 60; + const HOUR_IN_SECONDS: i64 = 60 * MINUTE_IN_SECONDS; + const DAY_IN_SECONDS: i64 = 24 * HOUR_IN_SECONDS; + let days = seconds / DAY_IN_SECONDS; + seconds %= DAY_IN_SECONDS; + let hours = seconds / HOUR_IN_SECONDS; + seconds %= HOUR_IN_SECONDS; + let minutes = seconds / MINUTE_IN_SECONDS; + seconds %= MINUTE_IN_SECONDS; + let mut have_written = if days > 0 { + write!(f, "{} day{}", days, if days == 1 { "" } else { "s" })?; + true + } else { + false + }; + if hours > 0 { + write!(f, "{}{} hour{}", if have_written { " " } else { "" }, + hours, if hours == 1 { "" } else { "s" })?; + have_written = true; + } + if minutes > 0 { + write!(f, "{}{} minute{}", if have_written { " " } else { "" }, + minutes, if minutes == 1 { "" } else { "s" })?; + have_written = true; + } + if seconds > 0 || !have_written { + write!(f, "{}{} second{}", if have_written { " " } else { "" }, + seconds, if seconds == 1 { "" } else { "s" })?; + } + Ok(()) + } +} + +impl ops::Add for Duration { + type Output = Duration; + fn add(self, rhs: Duration) -> Duration { Duration(self.0 + rhs.0) } +} + +impl ops::AddAssign for Duration { + fn add_assign(&mut self, rhs: Duration) { self.0 += rhs.0 } +} + +impl ops::SubAssign for Duration { + fn sub_assign(&mut self, rhs: Duration) { self.0 -= rhs.0 } +} + +#[derive(Clone, Copy, Debug)] +pub struct SampleIndexIterator { + i: usize, + pub pos: i32, + pub start_90k: i32, + pub duration_90k: i32, + pub bytes: i32, + bytes_key: i32, + bytes_nonkey: i32, + pub is_key: bool +} + +#[derive(Debug)] +pub struct SampleIndexEncoder { + // Internal state. + prev_duration_90k: i32, + prev_bytes_key: i32, + prev_bytes_nonkey: i32, + + // Eventual output. + // TODO: move to another struct? + pub sample_file_bytes: i32, + pub total_duration_90k: i32, + pub video_samples: i32, + pub video_sync_samples: i32, + pub video_index: Vec, +} + +pub struct Writer { + f: fs::File, + index: SampleIndexEncoder, + uuid: Uuid, + corrupt: bool, + hasher: hash::Hasher, + start_time: Time, + local_time: Time, + camera_id: i32, + video_sample_entry_id: i32, +} + +/// Zigzag-encodes a signed integer, as in [protocol buffer +/// encoding](https://developers.google.com/protocol-buffers/docs/encoding#types). Uses the low bit +/// to indicate signedness (1 = negative, 0 = non-negative). +#[inline(always)] +fn zigzag32(i: i32) -> u32 { ((i << 1) as u32) ^ ((i >> 31) as u32) } + +/// Zigzag-decodes to a signed integer. +/// See `zigzag`. +#[inline(always)] +fn unzigzag32(i: u32) -> i32 { ((i >> 1) as i32) ^ -((i & 1) as i32) } + +#[inline(always)] +fn decode_varint32(data: &[u8], i: usize) -> Result<(u32, usize), ()> { + // Unroll a few likely possibilities before going into the robust out-of-line loop. + // This aids branch prediction. + if data.len() > i && (data[i] & 0x80) == 0 { + return Ok((data[i] as u32, i+1)) + } else if data.len() > i + 1 && (data[i+1] & 0x80) == 0 { + return Ok((( (data[i] & 0x7f) as u32) | + (( data[i+1] as u32) << 7), + i+2)) + } else if data.len() > i + 2 && (data[i+2] & 0x80) == 0 { + return Ok((( (data[i] & 0x7f) as u32) | + (((data[i+1] & 0x7f) as u32) << 7) | + (( data[i+2] as u32) << 14), + i+3)) + } + decode_varint32_slow(data, i) +} + +#[cold] +fn decode_varint32_slow(data: &[u8], mut i: usize) -> Result<(u32, usize), ()> { + let l = data.len(); + let mut out = 0; + let mut shift = 0; + loop { + if i == l { + return Err(()) + } + let b = data[i]; + if shift == 28 && (b & 0xf0) != 0 { + return Err(()) + } + out |= ((b & 0x7f) as u32) << shift; + shift += 7; + i += 1; + if (b & 0x80) == 0 { + break; + } + } + Ok((out, i)) +} + +fn append_varint32(i: u32, data: &mut Vec) { + if i < 1u32 << 7 { + data.push(i as u8); + } else if i < 1u32 << 14 { + data.extend_from_slice(&[(( i & 0x7F) | 0x80) as u8, + (i >> 7) as u8]); + } else if i < 1u32 << 21 { + data.extend_from_slice(&[(( i & 0x7F) | 0x80) as u8, + (((i >> 7) & 0x7F) | 0x80) as u8, + (i >> 14) as u8]); + } else if i < 1u32 << 28 { + data.extend_from_slice(&[(( i & 0x7F) | 0x80) as u8, + (((i >> 7) & 0x7F) | 0x80) as u8, + (((i >> 14) & 0x7F) | 0x80) as u8, + (i >> 21) as u8]); + } else { + data.extend_from_slice(&[(( i & 0x7F) | 0x80) as u8, + (((i >> 7) & 0x7F) | 0x80) as u8, + (((i >> 14) & 0x7F) | 0x80) as u8, + (((i >> 21) & 0x7F) | 0x80) as u8, + (i >> 28) as u8]); + } +} + +impl SampleIndexIterator { + pub fn new() -> SampleIndexIterator { + SampleIndexIterator{i: 0, + pos: 0, + start_90k: 0, + duration_90k: 0, + bytes: 0, + bytes_key: 0, + bytes_nonkey: 0, + is_key: false} + } + + pub fn next(&mut self, data: &[u8]) -> Result { + self.pos += self.bytes; + self.start_90k += self.duration_90k; + if self.i == data.len() { + return Ok(false) + } + let (raw1, i1) = match decode_varint32(data, self.i) { + Ok(tuple) => tuple, + Err(()) => return Err(Error::new(format!("bad varint 1 at offset {}", self.i))), + }; + let (raw2, i2) = match decode_varint32(data, i1) { + Ok(tuple) => tuple, + Err(()) => return Err(Error::new(format!("bad varint 2 at offset {}", i1))), + }; + self.i = i2; + let duration_90k_delta = unzigzag32(raw1 >> 1); + self.duration_90k += duration_90k_delta; + if self.duration_90k < 0 { + return Err(Error{ + description: format!("negative duration {} after applying delta {}", + self.duration_90k, duration_90k_delta), + cause: None}); + } + if self.duration_90k == 0 && data.len() > self.i { + return Err(Error{ + description: format!("zero duration only allowed at end; have {} bytes left", + data.len() - self.i), + cause: None}); + } + self.is_key = (raw1 & 1) == 1; + let bytes_delta = unzigzag32(raw2); + self.bytes = if self.is_key { + self.bytes_key += bytes_delta; + self.bytes_key + } else { + self.bytes_nonkey += bytes_delta; + self.bytes_nonkey + }; + if self.bytes <= 0 { + return Err(Error{ + description: format!("non-positive bytes {} after applying delta {} to key={} frame at ts {}", + self.bytes, bytes_delta, self.is_key, + self.start_90k), + cause: None}); + } + Ok(true) + } +} + +impl SampleIndexEncoder { + pub fn new() -> Self { + SampleIndexEncoder{ + prev_duration_90k: 0, + prev_bytes_key: 0, + prev_bytes_nonkey: 0, + total_duration_90k: 0, + sample_file_bytes: 0, + video_samples: 0, + video_sync_samples: 0, + video_index: Vec::new(), + } + } + + pub fn add_sample(&mut self, duration_90k: i32, bytes: i32, is_key: bool) { + let duration_delta = duration_90k - self.prev_duration_90k; + self.prev_duration_90k = duration_90k; + self.total_duration_90k += duration_90k; + self.sample_file_bytes += bytes; + self.video_samples += 1; + let bytes_delta = bytes - if is_key { + let prev = self.prev_bytes_key; + self.video_sync_samples += 1; + self.prev_bytes_key = bytes; + prev + } else { + let prev = self.prev_bytes_nonkey; + self.prev_bytes_nonkey = bytes; + prev + }; + append_varint32((zigzag32(duration_delta) << 1) | (is_key as u32), &mut self.video_index); + append_varint32(zigzag32(bytes_delta), &mut self.video_index); + } +} + +impl Writer { + pub fn open(f: fs::File, uuid: Uuid, start_time: Time, local_time: Time, + camera_id: i32, video_sample_entry_id: i32) -> Result { + Ok(Writer{ + f: f, + index: SampleIndexEncoder::new(), + uuid: uuid, + corrupt: false, + hasher: hash::Hasher::new(hash::Type::SHA1)?, + start_time: start_time, + local_time: local_time, + camera_id: camera_id, + video_sample_entry_id: video_sample_entry_id, + }) + } + + pub fn write(&mut self, pkt: &[u8], duration_90k: i32, is_key: bool) -> Result<(), Error> { + let mut remaining = pkt; + while !remaining.is_empty() { + let written = match self.f.write(remaining) { + Ok(b) => b, + Err(e) => { + if remaining.len() < pkt.len() { + // Partially written packet. Truncate if possible. + if let Err(e2) = self.f.set_len(self.index.sample_file_bytes as u64) { + error!("After write to {} failed with {}, truncate failed with {}; \ + sample file is corrupt.", self.uuid.hyphenated(), e, e2); + self.corrupt = true; + } + } + return Err(Error::from(e)); + }, + }; + remaining = &remaining[written..]; + } + self.index.add_sample(duration_90k, pkt.len() as i32, is_key); + self.hasher.update(pkt)?; + Ok(()) + } + + pub fn end(&self) -> Time { + self.start_time + Duration(self.index.total_duration_90k as i64) + } + + // TODO: clean up this interface. + pub fn close(mut self) -> Result<(db::RecordingToInsert, fs::File), Error> { + if self.corrupt { + return Err(Error::new(format!("recording {} is corrupt", self.uuid))); + } + let mut sha1_bytes = [0u8; 20]; + sha1_bytes.copy_from_slice(&self.hasher.finish()?[..]); + Ok((db::RecordingToInsert{ + camera_id: self.camera_id, + sample_file_bytes: self.index.sample_file_bytes, + time: self.start_time .. self.end(), + local_time: self.local_time, + video_samples: self.index.video_samples, + video_sync_samples: self.index.video_sync_samples, + video_sample_entry_id: self.video_sample_entry_id, + sample_file_uuid: self.uuid, + video_index: self.index.video_index, + sample_file_sha1: sha1_bytes, + }, self.f)) + } +} + +/// A segment represents a view of some or all of a single recording, starting from a key frame. +/// Used by the `Mp4FileBuilder` class to splice together recordings into a single virtual .mp4. +pub struct Segment { + pub id: i64, + pub start: Time, + begin: SampleIndexIterator, + pub file_end: i32, + pub desired_range_90k: Range, + actual_end_90k: i32, + pub frames: i32, + pub key_frames: i32, + pub video_sample_entry_id: i32, +} + +impl Segment { + /// Creates a segment in a semi-initialized state. This is very light initialization because + /// it is called with the database lock held. `init` must be called before usage, and the + /// Segment should not be used if `init` fails. + /// + /// `desired_range_90k` represents the desired range of the segment relative to the start of + /// the recording. The actual range will start at the first key frame at or before the + /// desired start time. (The caller is responsible for creating an edit list to skip the + /// undesired portion.) It will end at the first frame after the desired range (unless the + /// desired range extends beyond the recording). + pub fn new(recording: &db::ListCameraRecordingsRow, + desired_range_90k: Range) -> Segment { + Segment{ + id: recording.id, + start: recording.start, + begin: SampleIndexIterator::new(), + file_end: recording.sample_file_bytes, + desired_range_90k: desired_range_90k, + actual_end_90k: recording.duration_90k, + frames: recording.video_samples, + key_frames: recording.video_sync_samples, + video_sample_entry_id: recording.video_sample_entry.id, + } + } + + /// Completes initialization of the segment. Must be called without the database lock held; + /// this will use the database to retrieve the video index for partial recordings. + pub fn init(&mut self, db: &db::Database) -> Result<(), Error> { + if self.desired_range_90k.start > self.desired_range_90k.end || + self.desired_range_90k.end > self.actual_end_90k { + return Err(Error::new(format!( + "desired range [{}, {}) invalid for recording of length {}", + self.desired_range_90k.start, self.desired_range_90k.end, self.actual_end_90k))); + } + + if self.desired_range_90k.start == 0 && + self.desired_range_90k.end == self.actual_end_90k { + // Fast path. Existing entry is fine. + return Ok(()) + } + + // Slow path. Need to iterate through the index. + let extra = db.lock().get_recording(self.id)?; + let data = &(&extra).video_index; + let mut it = SampleIndexIterator::new(); + if !it.next(data)? { + return Err(Error{description: String::from("no index"), + cause: None}); + } + if !it.is_key { + return Err(Error{description: String::from("not key frame"), + cause: None}); + } + loop { + if it.start_90k <= self.desired_range_90k.start && it.is_key { + // new start candidate. + self.begin = it; + self.frames = 0; + self.key_frames = 0; + } + if it.start_90k >= self.desired_range_90k.end { + break; + } + self.frames += 1; + self.key_frames += it.is_key as i32; + if !it.next(data)? { + break; + } + } + self.file_end = it.pos; + self.actual_end_90k = it.start_90k; + Ok(()) + } + + /// Returns the byte range within the sample file of data associated with this segment. + pub fn sample_file_range(&self) -> Range { + Range{start: self.begin.pos as u64, end: self.file_end as u64} + } + + /// Returns the actual time range as described in `new`. + pub fn actual_time_90k(&self) -> Range { + Range{start: self.begin.start_90k, end: self.actual_end_90k} + } + + /// Iterates through each frame in the segment. + /// Must be called without the database lock held; retrieves video index from the cache. + pub fn foreach(&self, db: &db::Database, mut f: F) -> Result<(), Error> + where F: FnMut(&SampleIndexIterator) -> Result<(), Error> + { + let extra = db.lock().get_recording(self.id)?; + let data = &(&extra).video_index; + let mut it = self.begin; + if it.i == 0 { + assert!(it.next(data)?); + assert!(it.is_key); + } + loop { + f(&it)?; + if !it.next(data)? { + return Ok(()); + } + } + } +} + +#[cfg(test)] +mod tests { + extern crate test; + + use super::{append_varint32, decode_varint32, unzigzag32, zigzag32}; + use super::*; + use self::test::Bencher; + + #[test] + fn test_zigzag() { + struct Test { + decoded: i32, + encoded: u32, + } + let tests = [ + Test{decoded: 0, encoded: 0}, + Test{decoded: -1, encoded: 1}, + Test{decoded: 1, encoded: 2}, + Test{decoded: -2, encoded: 3}, + Test{decoded: 2147483647, encoded: 4294967294}, + Test{decoded: -2147483648, encoded: 4294967295}, + ]; + for test in &tests { + assert_eq!(test.encoded, zigzag32(test.decoded)); + assert_eq!(test.decoded, unzigzag32(test.encoded)); + } + } + + #[test] + fn test_correct_varints() { + struct Test { + decoded: u32, + encoded: &'static [u8], + } + let tests = [ + Test{decoded: 1, encoded: b"\x01"}, + Test{decoded: 257, encoded: b"\x81\x02"}, + Test{decoded: 49409, encoded: b"\x81\x82\x03"}, + Test{decoded: 8438017, encoded: b"\x81\x82\x83\x04"}, + Test{decoded: 1350615297, encoded: b"\x81\x82\x83\x84\x05"}, + ]; + for test in &tests { + // Test encoding to an empty buffer. + let mut out = Vec::new(); + append_varint32(test.decoded, &mut out); + assert_eq!(&out[..], test.encoded); + + // ...and to a non-empty buffer. + let mut buf = Vec::new(); + out.clear(); + out.push(b'x'); + buf.push(b'x'); + buf.extend_from_slice(test.encoded); + append_varint32(test.decoded, &mut out); + assert_eq!(out, buf); + + // Test decoding from the beginning of the string. + assert_eq!((test.decoded, test.encoded.len()), + decode_varint32(test.encoded, 0).unwrap()); + + // ...and from the middle of a buffer. + buf.push(b'x'); + assert_eq!((test.decoded, test.encoded.len() + 1), + decode_varint32(&buf, 1).unwrap()); + } + } + + #[test] + fn test_display_duration() { + let tests = &[ + // (output, seconds) + ("0 seconds", 0), + ("1 second", 1), + ("1 minute", 60), + ("1 minute 1 second", 61), + ("2 minutes", 120), + ("1 hour", 3600), + ("1 hour 1 minute", 3660), + ("2 hours", 7200), + ("1 day", 86400), + ("1 day 1 hour", 86400 + 3600), + ("2 days", 2 * 86400), + ]; + for test in tests { + assert_eq!(test.0, format!("{}", Duration(test.1 * TIME_UNITS_PER_SEC))); + } + } + + #[test] + fn test_bad_varints() { + let tests: &[&[u8]] = &[ + // buffer underruns + b"", + b"\x80", + b"\x80\x80", + b"\x80\x80\x80", + b"\x80\x80\x80\x80", + + // int32 overflows + b"\x80\x80\x80\x80\x80", + b"\x80\x80\x80\x80\x80\x00", + ]; + for (i, encoded) in tests.iter().enumerate() { + assert!(decode_varint32(encoded, 0).is_err(), "while on test {}", i); + } + } + + /// Tests the example from design/schema.md. + #[test] + fn test_encode_example() { + let mut e = SampleIndexEncoder::new(); + e.add_sample(10, 1000, true); + e.add_sample(9, 10, false); + e.add_sample(11, 15, false); + e.add_sample(10, 12, false); + e.add_sample(10, 1050, true); + assert_eq!(e.video_index, b"\x29\xd0\x0f\x02\x14\x08\x0a\x02\x05\x01\x64"); + assert_eq!(10 + 9 + 11 + 10 + 10, e.total_duration_90k); + assert_eq!(5, e.video_samples); + assert_eq!(2, e.video_sync_samples); + } + + #[test] + fn test_round_trip() { + #[derive(Debug, PartialEq, Eq)] + struct Sample { + duration_90k: i32, + bytes: i32, + is_key: bool, + } + let samples = [ + Sample{duration_90k: 10, bytes: 30000, is_key: true}, + Sample{duration_90k: 9, bytes: 1000, is_key: false}, + Sample{duration_90k: 11, bytes: 1100, is_key: false}, + Sample{duration_90k: 18, bytes: 31000, is_key: true}, + Sample{duration_90k: 0, bytes: 1000, is_key: false}, + ]; + let mut e = SampleIndexEncoder::new(); + for sample in &samples { + e.add_sample(sample.duration_90k, sample.bytes, sample.is_key); + } + let mut it = SampleIndexIterator::new(); + for sample in &samples { + assert!(it.next(&e.video_index).unwrap()); + assert_eq!(sample, + &Sample{duration_90k: it.duration_90k, bytes: it.bytes, is_key: it.is_key}); + } + assert!(!it.next(&e.video_index).unwrap()); + } + + #[test] + fn test_iterator_errors() { + struct Test { + encoded: &'static [u8], + err: &'static str, + } + let tests = [ + Test{encoded: b"\x80", err: "bad varint 1 at offset 0"}, + Test{encoded: b"\x00\x80", err: "bad varint 2 at offset 1"}, + Test{encoded: b"\x00\x02\x00\x00", + err: "zero duration only allowed at end; have 2 bytes left"}, + Test{encoded: b"\x02\x02", + err: "negative duration -1 after applying delta -1"}, + Test{encoded: b"\x04\x00", + err: "non-positive bytes 0 after applying delta 0 to key=false frame at ts 0"}, + ]; + for test in &tests { + let mut it = SampleIndexIterator::new(); + assert_eq!(it.next(test.encoded).unwrap_err().description, test.err); + } + } + + /// Benchmarks the decoder, which is performance-critical for .mp4 serving. + #[bench] + fn bench_decoder(b: &mut Bencher) { + let data = include_bytes!("testdata/video_sample_index.bin"); + b.bytes = data.len() as u64; + b.iter(|| { + let mut it = SampleIndexIterator::new(); + while it.next(data).unwrap() {} + assert_eq!(30104460, it.pos); + assert_eq!(5399985, it.start_90k); + }); + } +} diff --git a/src/resource.rs b/src/resource.rs new file mode 100644 index 0000000..358ac53 --- /dev/null +++ b/src/resource.rs @@ -0,0 +1,682 @@ +// This file is part of Moonfire NVR, a security camera digital video recorder. +// Copyright (C) 2016 Scott Lamb +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// In addition, as a special exception, the copyright holders give +// permission to link the code of portions of this program with the +// OpenSSL library under certain conditions as described in each +// individual source file, and distribute linked combinations including +// the two. +// +// You must obey the GNU General Public License in all respects for all +// of the code used other than OpenSSL. If you modify file(s) with this +// exception, you may extend this exception to your version of the +// file(s), but you are not obligated to do so. If you do not wish to do +// so, delete this exception statement from your version. If you delete +// this exception statement from all source files in the program, then +// also delete it here. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program. If not, see . + +extern crate core; +extern crate hyper; +extern crate time; + +use error::Result; +use hyper::server::{Request, Response}; +use hyper::header; +use hyper::method::Method; +use hyper::net::Fresh; +use mime; +use smallvec::SmallVec; +use std::cmp; +use std::io; +use std::ops::Range; + +/// An HTTP resource for GET and HEAD serving. +pub trait Resource { + /// Returns the length of the slice in bytes. + fn len(&self) -> u64; + + /// Writes bytes within this slice indicated by `range` to `out.` + /// TODO: different result type? + fn write_to(&self, range: Range, out: &mut io::Write) -> Result<()>; + + fn content_type(&self) -> mime::Mime; + fn etag(&self) -> Option<&header::EntityTag>; + fn last_modified(&self) -> &header::HttpDate; +} + +#[derive(Debug, Eq, PartialEq)] +enum ResolvedRanges { + AbsentOrInvalid, + NotSatisfiable, + Satisfiable(SmallVec<[Range; 1]>) +} + +fn parse_range_header(range: Option<&header::Range>, resource_len: u64) -> ResolvedRanges { + if let Some(&header::Range::Bytes(ref byte_ranges)) = range { + let mut ranges: SmallVec<[Range; 1]> = SmallVec::new(); + for range in byte_ranges { + match *range { + header::ByteRangeSpec::FromTo(range_from, range_to) => { + let end = cmp::min(range_to + 1, resource_len); + if range_from >= end { + debug!("Range {:?} not satisfiable with length {:?}", range, resource_len); + continue; + } + ranges.push(Range{start: range_from, end: end}); + }, + header::ByteRangeSpec::AllFrom(range_from) => { + if range_from >= resource_len { + debug!("Range {:?} not satisfiable with length {:?}", range, resource_len); + continue; + } + ranges.push(Range{start: range_from, end: resource_len}); + }, + header::ByteRangeSpec::Last(last) => { + if last >= resource_len { + debug!("Range {:?} not satisfiable with length {:?}", range, resource_len); + continue; + } + ranges.push(Range{start: resource_len - last, + end: resource_len}); + }, + } + } + if !ranges.is_empty() { + debug!("Ranges {:?} all satisfiable with length {:?}", range, resource_len); + return ResolvedRanges::Satisfiable(ranges); + } + return ResolvedRanges::NotSatisfiable; + } + ResolvedRanges::AbsentOrInvalid +} + +/// Returns true if `req` doesn't have an `If-None-Match` header matching `req`. +fn none_match(etag: Option<&header::EntityTag>, req: &Request) -> bool { + match req.headers.get::() { + Some(&header::IfNoneMatch::Any) => false, + Some(&header::IfNoneMatch::Items(ref items)) => { + if let Some(some_etag) = etag { + for item in items { + if item.weak_eq(some_etag) { + return false; + } + } + } + true + }, + None => true, + } +} + +/// Returns true if `req` has no `If-Match` header or one which matches `etag`. +fn any_match(etag: Option<&header::EntityTag>, req: &Request) -> bool { + match req.headers.get::() { + Some(&header::IfMatch::Any) => true, + Some(&header::IfMatch::Items(ref items)) => { + if let Some(some_etag) = etag { + for item in items { + if item.strong_eq(some_etag) { + return true; + } + } + } + false + }, + None => false, + } +} + +/// Serves GET and HEAD requests for a given byte-ranged resource. +/// Handles conditional & subrange requests. +/// The caller is expected to have already determined the correct resource and appended +/// Expires, Cache-Control, and Vary headers. +/// +/// TODO: is it appropriate to include those headers on all response codes used in this function? +/// +/// TODO: check HTTP rules about weak vs strong comparisons with range requests. I don't think I'm +/// doing this correctly. +pub fn serve(rsrc: &Resource, req: &Request, mut res: Response) -> Result<()> { + if req.method != Method::Get && req.method != Method::Head { + *res.status_mut() = hyper::status::StatusCode::MethodNotAllowed; + res.headers_mut().set(header::ContentType(mime!(Text/Plain))); + res.headers_mut().set(header::Allow(vec![Method::Get, Method::Head])); + res.send(b"This resource only supports GET and HEAD.")?; + return Ok(()); + } + + let last_modified = rsrc.last_modified(); + let etag = rsrc.etag(); + res.headers_mut().set(header::AcceptRanges(vec![header::RangeUnit::Bytes])); + res.headers_mut().set(header::LastModified(*last_modified)); + if let Some(some_etag) = etag { + res.headers_mut().set(header::ETag(some_etag.clone())); + } + + if let Some(&header::IfUnmodifiedSince(ref since)) = req.headers.get() { + if last_modified.0.to_timespec() > since.0.to_timespec() { + *res.status_mut() = hyper::status::StatusCode::PreconditionFailed; + res.send(b"Precondition failed")?; + return Ok(()); + } + } + + if any_match(etag, req) { + *res.status_mut() = hyper::status::StatusCode::PreconditionFailed; + res.send(b"Precondition failed")?; + return Ok(()); + } + + if !none_match(etag, req) { + *res.status_mut() = hyper::status::StatusCode::NotModified; + res.send(b"")?; + return Ok(()); + } + + if let Some(&header::IfModifiedSince(ref since)) = req.headers.get() { + if last_modified <= since { + *res.status_mut() = hyper::status::StatusCode::NotModified; + res.send(b"")?; + return Ok(()); + } + } + + let mut range_hdr = req.headers.get::(); + + // See RFC 2616 section 10.2.7: a Partial Content response should include certain + // entity-headers or not based on the If-Range response. + let include_entity_headers_on_range = match req.headers.get::() { + Some(&header::IfRange::EntityTag(ref if_etag)) => { + if let Some(some_etag) = etag { + if if_etag.strong_eq(some_etag) { + false + } else { + range_hdr = None; + true + } + } else { + range_hdr = None; + true + } + }, + Some(&header::IfRange::Date(ref if_date)) => { + // The to_timespec conversion appears necessary because in the If-Range off the wire, + // fields such as tm_yday are absent, causing strict equality to spuriously fail. + if if_date.0.to_timespec() != last_modified.0.to_timespec() { + range_hdr = None; + true + } else { + false + } + }, + None => true, + }; + let len = rsrc.len(); + let (range, include_entity_headers) = match parse_range_header(range_hdr, len) { + ResolvedRanges::AbsentOrInvalid => (0 .. len, true), + ResolvedRanges::Satisfiable(rs) => { + if rs.len() == 1 { + res.headers_mut().set(header::ContentRange( + header::ContentRangeSpec::Bytes{ + range: Some((rs[0].start, rs[0].end-1)), + instance_length: Some(len)})); + *res.status_mut() = hyper::status::StatusCode::PartialContent; + (rs[0].clone(), include_entity_headers_on_range) + } else { + // Ignore multi-part range headers for now. They require additional complexity, and + // I don't see clients sending them in the wild. + (0 .. len, true) + } + }, + ResolvedRanges::NotSatisfiable => { + res.headers_mut().set(header::ContentRange( + header::ContentRangeSpec::Bytes{ + range: None, + instance_length: Some(len)})); + *res.status_mut() = hyper::status::StatusCode::RangeNotSatisfiable; + res.send(b"")?; + return Ok(()); + } + }; + if include_entity_headers { + res.headers_mut().set(header::ContentType(rsrc.content_type())); + } + res.headers_mut().set(header::ContentLength(range.end - range.start)); + let mut stream = res.start()?; + if req.method == Method::Get { + rsrc.write_to(range, &mut stream)?; + } + stream.end()?; + Ok(()) +} + +#[cfg(test)] +mod tests { + use error::Result; + use hyper; + use hyper::header::{self, ByteRangeSpec, ContentRangeSpec, EntityTag}; + use hyper::header::Range::Bytes; + use mime; + use smallvec::SmallVec; + use std::io::{Read, Write}; + use std::ops::Range; + use std::sync::Mutex; + use super::{ResolvedRanges, parse_range_header}; + use super::*; + use testutil; + use time; + + /// Tests the specific examples enumerated in RFC 2616 section 14.35.1. + #[test] + fn test_resolve_ranges_rfc() { + let mut v = SmallVec::new(); + + v.push(0 .. 500); + assert_eq!(ResolvedRanges::Satisfiable(v.clone()), + parse_range_header(Some(&Bytes(vec![ByteRangeSpec::FromTo(0, 499)])), + 10000)); + + v.clear(); + v.push(500 .. 1000); + assert_eq!(ResolvedRanges::Satisfiable(v.clone()), + parse_range_header(Some(&Bytes(vec![ByteRangeSpec::FromTo(500, 999)])), + 10000)); + + v.clear(); + v.push(9500 .. 10000); + assert_eq!(ResolvedRanges::Satisfiable(v.clone()), + parse_range_header(Some(&Bytes(vec![ByteRangeSpec::Last(500)])), + 10000)); + + v.clear(); + v.push(9500 .. 10000); + assert_eq!(ResolvedRanges::Satisfiable(v.clone()), + parse_range_header(Some(&Bytes(vec![ByteRangeSpec::AllFrom(9500)])), + 10000)); + + v.clear(); + v.push(0 .. 1); + v.push(9999 .. 10000); + assert_eq!(ResolvedRanges::Satisfiable(v.clone()), + parse_range_header(Some(&Bytes(vec![ByteRangeSpec::FromTo(0, 0), + ByteRangeSpec::Last(1)])), + 10000)); + + // Non-canonical ranges. Possibly the point of these is that the adjacent and overlapping + // ranges are supposed to be coalesced into one? I'm not going to do that for now. + + v.clear(); + v.push(500 .. 601); + v.push(601 .. 1000); + assert_eq!(ResolvedRanges::Satisfiable(v.clone()), + parse_range_header(Some(&Bytes(vec![ByteRangeSpec::FromTo(500, 600), + ByteRangeSpec::FromTo(601, 999)])), + 10000)); + + v.clear(); + v.push(500 .. 701); + v.push(601 .. 1000); + assert_eq!(ResolvedRanges::Satisfiable(v.clone()), + parse_range_header(Some(&Bytes(vec![ByteRangeSpec::FromTo(500, 700), + ByteRangeSpec::FromTo(601, 999)])), + 10000)); + } + + #[test] + fn test_resolve_ranges_satisfiability() { + assert_eq!(ResolvedRanges::NotSatisfiable, + parse_range_header(Some(&Bytes(vec![ByteRangeSpec::AllFrom(10000)])), + 10000)); + + let mut v = SmallVec::new(); + v.push(0 .. 500); + assert_eq!(ResolvedRanges::Satisfiable(v.clone()), + parse_range_header(Some(&Bytes(vec![ByteRangeSpec::FromTo(0, 499), + ByteRangeSpec::AllFrom(10000)])), + 10000)); + + assert_eq!(ResolvedRanges::NotSatisfiable, + parse_range_header(Some(&Bytes(vec![ByteRangeSpec::Last(1)])), 0)); + assert_eq!(ResolvedRanges::NotSatisfiable, + parse_range_header(Some(&Bytes(vec![ByteRangeSpec::FromTo(0, 0)])), 0)); + assert_eq!(ResolvedRanges::NotSatisfiable, + parse_range_header(Some(&Bytes(vec![ByteRangeSpec::AllFrom(0)])), 0)); + + v.clear(); + v.push(0 .. 1); + assert_eq!(ResolvedRanges::Satisfiable(v.clone()), + parse_range_header(Some(&Bytes(vec![ByteRangeSpec::FromTo(0, 0)])), 1)); + + v.clear(); + v.push(0 .. 500); + assert_eq!(ResolvedRanges::Satisfiable(v.clone()), + parse_range_header(Some(&Bytes(vec![ByteRangeSpec::FromTo(0, 10000)])), + 500)); + } + + #[test] + fn test_resolve_ranges_absent_or_invalid() { + assert_eq!(ResolvedRanges::AbsentOrInvalid, parse_range_header(None, 10000)); + } + + struct FakeResource { + etag: Option, + mime: mime::Mime, + last_modified: header::HttpDate, + body: &'static [u8], + } + + impl Resource for FakeResource { + fn len(&self) -> u64 { self.body.len() as u64 } + fn write_to(&self, range: Range, out: &mut Write) -> Result<()> { + Ok(out.write_all(&self.body[range.start as usize .. range.end as usize])?) + } + fn content_type(&self) -> mime::Mime { self.mime.clone() } + fn etag(&self) -> Option<&EntityTag> { self.etag.as_ref() } + fn last_modified(&self) -> &header::HttpDate { &self.last_modified } + } + + fn new_server() -> String { + let mut listener = hyper::net::HttpListener::new("127.0.0.1:0").unwrap(); + use hyper::net::NetworkListener; + let addr = listener.local_addr().unwrap(); + let server = hyper::Server::new(listener); + use std::thread::spawn; + spawn(move || { + use hyper::server::{Request, Response, Fresh}; + let _ = server.handle(move |req: Request, res: Response| { + let l = RESOURCE.lock().unwrap(); + let resource = l.as_ref().unwrap(); + serve(resource, &req, res).unwrap(); + }); + }); + format!("http://{}:{}/", addr.ip(), addr.port()) + } + + lazy_static! { + static ref RESOURCE: Mutex> = { Mutex::new(None) }; + static ref SERVER: String = { new_server() }; + static ref SOME_DATE: header::HttpDate = { + header::HttpDate(time::at_utc(time::Timespec::new(1430006400i64, 0))) + }; + static ref LATER_DATE: header::HttpDate = { + header::HttpDate(time::at_utc(time::Timespec::new(1430092800i64, 0))) + }; + } + + #[test] + fn serve_without_etag() { + testutil::init(); + *RESOURCE.lock().unwrap() = Some(FakeResource{ + etag: None, + mime: mime!(Application/OctetStream), + last_modified: *SOME_DATE, + body: b"01234", + }); + let client = hyper::Client::new(); + let mut buf = Vec::new(); + + // Full body. + let mut resp = client.get(&*SERVER).send().unwrap(); + assert_eq!(hyper::status::StatusCode::Ok, resp.status); + assert_eq!(Some(&header::ContentType(mime!(Application/OctetStream))), + resp.headers.get::()); + assert_eq!(None, resp.headers.get::()); + buf.clear(); + resp.read_to_end(&mut buf).unwrap(); + assert_eq!(b"01234", &buf[..]); + + // If-None-Match any. + let mut resp = client.get(&*SERVER) + .header(header::IfNoneMatch::Any) + .send() + .unwrap(); + assert_eq!(hyper::status::StatusCode::NotModified, resp.status); + assert_eq!(None, resp.headers.get::()); + buf.clear(); + resp.read_to_end(&mut buf).unwrap(); + assert_eq!(b"", &buf[..]); + + // If-None-Match by etag doesn't match (as this request has no etag). + let mut resp = + client.get(&*SERVER) + .header(header::IfNoneMatch::Items(vec![EntityTag::strong("foo".to_owned())])) + .send() + .unwrap(); + assert_eq!(hyper::status::StatusCode::Ok, resp.status); + assert_eq!(Some(&header::ContentType(mime!(Application/OctetStream))), + resp.headers.get::()); + assert_eq!(None, resp.headers.get::()); + buf.clear(); + resp.read_to_end(&mut buf).unwrap(); + assert_eq!(b"01234", &buf[..]); + + // Unmodified since supplied date. + let mut resp = client.get(&*SERVER) + .header(header::IfModifiedSince(*SOME_DATE)) + .send() + .unwrap(); + assert_eq!(hyper::status::StatusCode::NotModified, resp.status); + assert_eq!(None, resp.headers.get::()); + buf.clear(); + resp.read_to_end(&mut buf).unwrap(); + assert_eq!(b"", &buf[..]); + + // Range serving - basic case. + let mut resp = client.get(&*SERVER) + .header(Bytes(vec![ByteRangeSpec::FromTo(1, 3)])) + .send() + .unwrap(); + assert_eq!(hyper::status::StatusCode::PartialContent, resp.status); + assert_eq!(Some(&header::ContentRange(ContentRangeSpec::Bytes{ + range: Some((1, 3)), + instance_length: Some(5), + })), resp.headers.get()); + buf.clear(); + resp.read_to_end(&mut buf).unwrap(); + assert_eq!(b"123", &buf[..]); + + // Range serving - multiple ranges. Currently falls back to whole range. + let mut resp = client.get(&*SERVER) + .header(Bytes(vec![ByteRangeSpec::FromTo(0, 1), + ByteRangeSpec::FromTo(3, 4)])) + .send() + .unwrap(); + assert_eq!(None, resp.headers.get::()); + assert_eq!(hyper::status::StatusCode::Ok, resp.status); + buf.clear(); + resp.read_to_end(&mut buf).unwrap(); + assert_eq!(b"01234", &buf[..]); + + // Range serving - not satisfiable. + let mut resp = client.get(&*SERVER) + .header(Bytes(vec![ByteRangeSpec::AllFrom(500)])) + .send() + .unwrap(); + assert_eq!(hyper::status::StatusCode::RangeNotSatisfiable, resp.status); + assert_eq!(Some(&header::ContentRange(ContentRangeSpec::Bytes{ + range: None, + instance_length: Some(5), + })), resp.headers.get()); + buf.clear(); + resp.read_to_end(&mut buf).unwrap(); + assert_eq!(b"", &buf[..]); + + // Range serving - matching If-Range by date honors the range. + let mut resp = client.get(&*SERVER) + .header(Bytes(vec![ByteRangeSpec::FromTo(1, 3)])) + .header(header::IfRange::Date(*SOME_DATE)) + .send() + .unwrap(); + assert_eq!(hyper::status::StatusCode::PartialContent, resp.status); + assert_eq!(Some(&header::ContentRange(ContentRangeSpec::Bytes{ + range: Some((1, 3)), + instance_length: Some(5), + })), resp.headers.get()); + buf.clear(); + resp.read_to_end(&mut buf).unwrap(); + assert_eq!(b"123", &buf[..]); + + // Range serving - non-matching If-Range by date ignores the range. + let mut resp = client.get(&*SERVER) + .header(Bytes(vec![ByteRangeSpec::FromTo(1, 3)])) + .header(header::IfRange::Date(*LATER_DATE)) + .send() + .unwrap(); + assert_eq!(hyper::status::StatusCode::Ok, resp.status); + assert_eq!(Some(&header::ContentType(mime!(Application/OctetStream))), + resp.headers.get::()); + assert_eq!(None, resp.headers.get::()); + buf.clear(); + resp.read_to_end(&mut buf).unwrap(); + assert_eq!(b"01234", &buf[..]); + + // Range serving - this resource has no etag, so any If-Range by etag ignores the range. + let mut resp = + client.get(&*SERVER) + .header(Bytes(vec![ByteRangeSpec::FromTo(1, 3)])) + .header(header::IfRange::EntityTag(EntityTag::strong("foo".to_owned()))) + .send() + .unwrap(); + assert_eq!(hyper::status::StatusCode::Ok, resp.status); + assert_eq!(Some(&header::ContentType(mime!(Application/OctetStream))), + resp.headers.get::()); + assert_eq!(None, resp.headers.get::()); + buf.clear(); + resp.read_to_end(&mut buf).unwrap(); + assert_eq!(b"01234", &buf[..]); + } + + #[test] + fn serve_with_strong_etag() { + testutil::init(); + *RESOURCE.lock().unwrap() = Some(FakeResource{ + etag: Some(EntityTag::strong("foo".to_owned())), + mime: mime!(Application/OctetStream), + last_modified: *SOME_DATE, + body: b"01234", + }); + let client = hyper::Client::new(); + let mut buf = Vec::new(); + + // If-None-Match by etag which matches. + let mut resp = + client.get(&*SERVER) + .header(header::IfNoneMatch::Items(vec![EntityTag::strong("foo".to_owned())])) + .send() + .unwrap(); + assert_eq!(hyper::status::StatusCode::NotModified, resp.status); + assert_eq!(None, resp.headers.get::()); + buf.clear(); + resp.read_to_end(&mut buf).unwrap(); + assert_eq!(b"", &buf[..]); + + // If-None-Match by etag which doesn't match. + let mut resp = + client.get(&*SERVER) + .header(header::IfNoneMatch::Items(vec![EntityTag::strong("bar".to_owned())])) + .send() + .unwrap(); + assert_eq!(hyper::status::StatusCode::Ok, resp.status); + assert_eq!(None, resp.headers.get::()); + buf.clear(); + resp.read_to_end(&mut buf).unwrap(); + assert_eq!(b"01234", &buf[..]); + + // Range serving - If-Range matching by etag. + let mut resp = + client.get(&*SERVER) + .header(Bytes(vec![ByteRangeSpec::FromTo(1, 3)])) + .header(header::IfRange::EntityTag(EntityTag::strong("foo".to_owned()))) + .send() + .unwrap(); + assert_eq!(hyper::status::StatusCode::PartialContent, resp.status); + assert_eq!(None, resp.headers.get::()); + assert_eq!(Some(&header::ContentRange(ContentRangeSpec::Bytes{ + range: Some((1, 3)), + instance_length: Some(5), + })), resp.headers.get()); + buf.clear(); + resp.read_to_end(&mut buf).unwrap(); + assert_eq!(b"123", &buf[..]); + + // Range serving - If-Range not matching by etag. + let mut resp = + client.get(&*SERVER) + .header(Bytes(vec![ByteRangeSpec::FromTo(1, 3)])) + .header(header::IfRange::EntityTag(EntityTag::strong("bar".to_owned()))) + .send() + .unwrap(); + assert_eq!(hyper::status::StatusCode::Ok, resp.status); + assert_eq!(Some(&header::ContentType(mime!(Application/OctetStream))), + resp.headers.get::()); + assert_eq!(None, resp.headers.get::()); + buf.clear(); + resp.read_to_end(&mut buf).unwrap(); + assert_eq!(b"01234", &buf[..]); + } + + #[test] + fn serve_with_weak_etag() { + testutil::init(); + *RESOURCE.lock().unwrap() = Some(FakeResource{ + etag: Some(EntityTag::weak("foo".to_owned())), + mime: mime!(Application/OctetStream), + last_modified: *SOME_DATE, + body: b"01234", + }); + let client = hyper::Client::new(); + let mut buf = Vec::new(); + + // If-None-Match by identical weak etag is sufficient. + let mut resp = + client.get(&*SERVER) + .header(header::IfNoneMatch::Items(vec![EntityTag::weak("foo".to_owned())])) + .send() + .unwrap(); + assert_eq!(hyper::status::StatusCode::NotModified, resp.status); + assert_eq!(None, resp.headers.get::()); + buf.clear(); + resp.read_to_end(&mut buf).unwrap(); + assert_eq!(b"", &buf[..]); + + // If-None-Match by etag which doesn't match. + let mut resp = + client.get(&*SERVER) + .header(header::IfNoneMatch::Items(vec![EntityTag::weak("bar".to_owned())])) + .send() + .unwrap(); + assert_eq!(hyper::status::StatusCode::Ok, resp.status); + assert_eq!(None, resp.headers.get::()); + buf.clear(); + resp.read_to_end(&mut buf).unwrap(); + assert_eq!(b"01234", &buf[..]); + + // Range serving - If-Range matching by weak etag isn't sufficient. + let mut resp = + client.get(&*SERVER) + .header(Bytes(vec![ByteRangeSpec::FromTo(1, 3)])) + .header(header::IfRange::EntityTag(EntityTag::weak("foo".to_owned()))) + .send() + .unwrap(); + assert_eq!(hyper::status::StatusCode::Ok, resp.status); + assert_eq!(Some(&header::ContentType(mime!(Application/OctetStream))), + resp.headers.get::()); + assert_eq!(None, resp.headers.get::()); + buf.clear(); + resp.read_to_end(&mut buf).unwrap(); + assert_eq!(b"01234", &buf[..]); + } +} diff --git a/src/schema.sql b/src/schema.sql index 93b660c..13c267f 100644 --- a/src/schema.sql +++ b/src/schema.sql @@ -110,6 +110,7 @@ create index recording_cover on recording ( -- to consult the underlying row. duration_90k, video_samples, + video_sync_samples, video_sample_entry_id, sample_file_bytes ); diff --git a/src/sqlite-test.cc b/src/sqlite-test.cc deleted file mode 100644 index 20834c8..0000000 --- a/src/sqlite-test.cc +++ /dev/null @@ -1,111 +0,0 @@ -// This file is part of Moonfire NVR, a security camera network video recorder. -// Copyright (C) 2016 Scott Lamb -// -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// In addition, as a special exception, the copyright holders give -// permission to link the code of portions of this program with the -// OpenSSL library under certain conditions as described in each -// individual source file, and distribute linked combinations including -// the two. -// -// You must obey the GNU General Public License in all respects for all -// of the code used other than OpenSSL. If you modify file(s) with this -// exception, you may extend this exception to your version of the -// file(s), but you are not obligated to do so. If you do not wish to do -// so, delete this exception statement from your version. If you delete -// this exception statement from all source files in the program, then -// also delete it here. -// -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. -// -// You should have received a copy of the GNU General Public License -// along with this program. If not, see . -// -// sqlite-test.cc: tests of the sqlite.h interface. - -#include - -#include -#include -#include -#include - -#include "sqlite.h" -#include "string.h" -#include "testutil.h" - -DECLARE_bool(alsologtostderr); - -namespace moonfire_nvr { -namespace { - -class SqliteTest : public testing::Test { - protected: - SqliteTest() { - tmpdir_ = PrepareTempDirOrDie("sqlite-test"); - - std::string error_message; - CHECK(db_.Open(StrCat(tmpdir_, "/db").c_str(), - SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE, &error_message)) - << error_message; - - std::string create_sql = ReadFileOrDie("../src/schema.sql"); - DatabaseContext ctx(&db_); - CHECK(RunStatements(&ctx, create_sql, &error_message)) << error_message; - } - - std::string tmpdir_; - Database db_; -}; - -TEST_F(SqliteTest, JustCreate) {} - -TEST_F(SqliteTest, BindAndColumn) { - std::string error_message; - auto insert_stmt = db_.Prepare( - "insert into camera (uuid, short_name, retain_bytes) " - " values (:uuid, :short_name, :retain_bytes)", - nullptr, &error_message); - ASSERT_TRUE(insert_stmt.valid()) << error_message; - const char kBlob[] = {0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, - 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f}; - re2::StringPiece blob_piece = re2::StringPiece(kBlob, sizeof(kBlob)); - const char kText[] = "foo"; - const int64_t kInt64 = INT64_C(0xbeeffeedface); - - DatabaseContext ctx(&db_); - { - auto run = ctx.Borrow(&insert_stmt); - run.BindBlob(1, blob_piece); - run.BindText(2, kText); - run.BindInt64(3, kInt64); - ASSERT_EQ(SQLITE_DONE, run.Step()) << run.error_message(); - } - - { - auto run = ctx.UseOnce("select uuid, short_name, retain_bytes from camera"); - ASSERT_EQ(SQLITE_ROW, run.Step()) << run.error_message(); - EXPECT_EQ(ToHex(blob_piece, true), ToHex(run.ColumnBlob(0), true)); - EXPECT_EQ(kText, run.ColumnText(1).as_string()); - EXPECT_EQ(kInt64, run.ColumnInt64(2)); - ASSERT_EQ(SQLITE_DONE, run.Step()) << run.error_message(); - } -} - -} // namespace -} // namespace moonfire_nvr - -int main(int argc, char **argv) { - FLAGS_alsologtostderr = true; - google::ParseCommandLineFlags(&argc, &argv, true); - testing::InitGoogleTest(&argc, argv); - google::InitGoogleLogging(argv[0]); - return RUN_ALL_TESTS(); -} diff --git a/src/sqlite.cc b/src/sqlite.cc deleted file mode 100644 index 791a2ea..0000000 --- a/src/sqlite.cc +++ /dev/null @@ -1,408 +0,0 @@ -// This file is part of Moonfire NVR, a security camera digital video recorder. -// Copyright (C) 2016 Scott Lamb -// -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// In addition, as a special exception, the copyright holders give -// permission to link the code of portions of this program with the -// OpenSSL library under certain conditions as described in each -// individual source file, and distribute linked combinations including -// the two. -// -// You must obey the GNU General Public License in all respects for all -// of the code used other than OpenSSL. If you modify file(s) with this -// exception, you may extend this exception to your version of the -// file(s), but you are not obligated to do so. If you do not wish to do -// so, delete this exception statement from your version. If you delete -// this exception statement from all source files in the program, then -// also delete it here. -// -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. -// -// You should have received a copy of the GNU General Public License -// along with this program. If not, see . -// -// sqlite.cc: implementation of the sqlite.h interface. - -#include "sqlite.h" - -#include - -#include - -#include "string.h" - -namespace moonfire_nvr { - -namespace { - -void LogCallback(void *, int err_code, const char *msg) { - LOG(ERROR) << "(" << err_code << ") " << msg; -} - -void GlobalSetup() { - VLOG(1) << "Installing sqlite3 log callback"; - sqlite3_config(SQLITE_CONFIG_LOG, &LogCallback, nullptr); -} - -std::once_flag global_setup; - -} // namespace - -Statement::Statement(Statement &&other) { *this = std::move(other); } - -void Statement::operator=(Statement &&other) { - Clear(); - memcpy(this, &other, sizeof(Statement)); - other.me_ = nullptr; - other.borrowed_ = false; -} - -Statement::~Statement() { Clear(); } - -void Statement::Clear() { - CHECK(!borrowed_) << "can't delete statement while still borrowed!"; - sqlite3_finalize(me_); -} - -DatabaseContext::DatabaseContext(Database *db) : db_(db), lock_(db->ctx_mu_) {} - -DatabaseContext::~DatabaseContext() { - if (transaction_open_) { - LOG(WARNING) << this << ": transaction left open! closing in destructor."; - RollbackTransaction(); - } -} - -bool DatabaseContext::BeginTransaction(std::string *error_message) { - if (transaction_open_) { - *error_message = "transaction already open!"; - return false; - } - sqlite3_step(db_->begin_transaction_.me_); - int ret = sqlite3_reset(db_->begin_transaction_.me_); - if (ret != SQLITE_OK) { - *error_message = - StrCat("begin transaction: ", sqlite3_errstr(ret), " (", ret, ")"); - return false; - } - transaction_open_ = true; - return true; -} - -bool DatabaseContext::CommitTransaction(std::string *error_message) { - if (!transaction_open_) { - *error_message = "transaction not open!"; - return false; - } - sqlite3_step(db_->commit_transaction_.me_); - int ret = sqlite3_reset(db_->commit_transaction_.me_); - if (ret != SQLITE_OK) { - *error_message = - StrCat("commit transaction: ", sqlite3_errstr(ret), " (", ret, ")"); - return false; - } - transaction_open_ = false; - return true; -} - -void DatabaseContext::RollbackTransaction() { - if (!transaction_open_) { - LOG(WARNING) << this << ": rollback failed: transaction not open!"; - return; - } - sqlite3_step(db_->rollback_transaction_.me_); - int ret = sqlite3_reset(db_->rollback_transaction_.me_); - if (ret != SQLITE_OK) { - LOG(WARNING) << this << ": rollback failed: " << sqlite3_errstr(ret) << " (" - << ret << ")"; - return; - } - transaction_open_ = false; -} - -RunningStatement DatabaseContext::Borrow(Statement *statement) { - return RunningStatement(statement, std::string(), false); -} - -RunningStatement DatabaseContext::UseOnce(re2::StringPiece sql) { - std::string error_message; - auto *statement = new Statement(db_->Prepare(sql, nullptr, &error_message)); - return RunningStatement(statement, error_message, true); -} - -RunningStatement::RunningStatement(Statement *statement, - const std::string &deferred_error, - bool owns_statement) - : error_message_(deferred_error), owns_statement_(owns_statement) { - if (statement != nullptr && statement->valid()) { - CHECK(!statement->borrowed_) << "Statement already borrowed!"; - statement->borrowed_ = true; - statement_ = statement; - } else if (error_message_.empty()) { - error_message_ = "invalid statement"; - } - - if (!error_message_.empty()) { - status_ = SQLITE_MISUSE; - } -} - -RunningStatement::RunningStatement(RunningStatement &&o) { - statement_ = o.statement_; - status_ = o.status_; - owns_statement_ = o.owns_statement_; - o.statement_ = nullptr; -} - -RunningStatement::~RunningStatement() { - if (statement_ != nullptr) { - CHECK(statement_->borrowed_) << "Statement no longer borrowed!"; - sqlite3_clear_bindings(statement_->me_); - sqlite3_reset(statement_->me_); - statement_->borrowed_ = false; - if (owns_statement_) { - delete statement_; - } - } -} - -void RunningStatement::BindBlob(int param, re2::StringPiece value) { - if (status_ != SQLITE_OK) { - return; - } - status_ = sqlite3_bind_blob64(statement_->me_, param, value.data(), - value.size(), SQLITE_TRANSIENT); - if (status_ != SQLITE_OK) { - error_message_ = StrCat("Unable to bind parameter ", param, ": ", - sqlite3_errstr(status_), " (", status_, ")"); - } -} - -void RunningStatement::BindBlob(const char *name, re2::StringPiece value) { - if (status_ != SQLITE_OK) { - return; - } - int param = sqlite3_bind_parameter_index(statement_->me_, name); - if (param == 0) { - status_ = SQLITE_MISUSE; - error_message_ = StrCat("Unable to bind parameter ", name, ": not found."); - return; - } - status_ = sqlite3_bind_blob64(statement_->me_, param, value.data(), - value.size(), SQLITE_TRANSIENT); - if (status_ != SQLITE_OK) { - error_message_ = StrCat("Unable to bind parameter ", name, ": ", - sqlite3_errstr(status_), " (", status_, ")"); - } -} - -void RunningStatement::BindInt64(int param, int64_t value) { - if (status_ != SQLITE_OK) { - return; - } - status_ = sqlite3_bind_int64(statement_->me_, param, value); - if (status_ != SQLITE_OK) { - error_message_ = StrCat("Unable to bind parameter ", param, ": ", - sqlite3_errstr(status_), " (", status_, ")"); - } -} - -void RunningStatement::BindInt64(const char *name, int64_t value) { - if (status_ != SQLITE_OK) { - return; - } - int param = sqlite3_bind_parameter_index(statement_->me_, name); - if (param == 0) { - status_ = SQLITE_MISUSE; - error_message_ = StrCat("Unable to bind parameter ", name, ": not found."); - return; - } - status_ = sqlite3_bind_int64(statement_->me_, param, value); - if (status_ != SQLITE_OK) { - error_message_ = StrCat("Unable to bind parameter ", name, ": ", - sqlite3_errstr(status_), " (", status_, ")"); - } -} - -void RunningStatement::BindText(int param, re2::StringPiece value) { - if (status_ != SQLITE_OK) { - return; - } - status_ = sqlite3_bind_text64(statement_->me_, param, value.data(), - value.size(), SQLITE_TRANSIENT, SQLITE_UTF8); - if (status_ != SQLITE_OK) { - error_message_ = StrCat("Unable to bind parameter ", param, ": ", - sqlite3_errstr(status_), " (", status_, ")"); - } -} - -void RunningStatement::BindText(const char *name, re2::StringPiece value) { - if (status_ != SQLITE_OK) { - return; - } - int param = sqlite3_bind_parameter_index(statement_->me_, name); - if (param == 0) { - error_message_ = StrCat("Unable to bind parameter ", name, ": not found."); - return; - } - status_ = sqlite3_bind_text64(statement_->me_, param, value.data(), - value.size(), SQLITE_TRANSIENT, SQLITE_UTF8); - if (status_ != SQLITE_OK) { - error_message_ = StrCat("Unable to bind parameter ", name, ": ", - sqlite3_errstr(status_), " (", status_, ")"); - } -} - -int RunningStatement::Step() { - if (status_ != SQLITE_OK && status_ != SQLITE_ROW) { - return status_; - } - status_ = sqlite3_step(statement_->me_); - error_message_ = - StrCat("step: ", sqlite3_errstr(status_), " (", status_, ")"); - return status_; -} - -int RunningStatement::ColumnType(int col) { - return sqlite3_column_type(statement_->me_, col); -} - -re2::StringPiece RunningStatement::ColumnBlob(int col) { - // Order matters: call _blob first, then _bytes. - const void *data = sqlite3_column_blob(statement_->me_, col); - size_t len = sqlite3_column_bytes(statement_->me_, col); - return re2::StringPiece(reinterpret_cast(data), len); -} - -int64_t RunningStatement::ColumnInt64(int col) { - return sqlite3_column_int64(statement_->me_, col); -} - -re2::StringPiece RunningStatement::ColumnText(int col) { - // Order matters: call _text first, then _bytes. - const unsigned char *data = sqlite3_column_text(statement_->me_, col); - size_t len = sqlite3_column_bytes(statement_->me_, col); - return re2::StringPiece(reinterpret_cast(data), len); -} - -Database::~Database() { - begin_transaction_ = Statement(); - commit_transaction_ = Statement(); - rollback_transaction_ = Statement(); - int err = sqlite3_close(me_); - CHECK_EQ(SQLITE_OK, err) << "sqlite3_close: " << sqlite3_errstr(err); -} - -bool Database::Open(const char *filename, int flags, - std::string *error_message) { - std::call_once(global_setup, &GlobalSetup); - int ret = sqlite3_open_v2(filename, &me_, flags, nullptr); - if (ret != SQLITE_OK) { - *error_message = - StrCat("open ", filename, ": ", sqlite3_errstr(ret), " (", ret, ")"); - return false; - } - - ret = sqlite3_extended_result_codes(me_, 1); - if (ret != SQLITE_OK) { - sqlite3_close(me_); - me_ = nullptr; - *error_message = StrCat("while enabling extended result codes: ", - sqlite3_errstr(ret), " (", ret, ")"); - return false; - } - - Statement pragma_foreignkeys; - struct StatementToInitialize { - Statement *p; - re2::StringPiece sql; - }; - StatementToInitialize stmts[] = { - {&begin_transaction_, "begin transaction;"}, - {&commit_transaction_, "commit transaction;"}, - {&rollback_transaction_, "rollback transaction;"}, - {&pragma_foreignkeys, "pragma foreign_keys = true;"}}; - - for (const auto &stmt : stmts) { - *stmt.p = Prepare(stmt.sql, nullptr, error_message); - if (!stmt.p->valid()) { - sqlite3_close(me_); - me_ = nullptr; - *error_message = StrCat("while preparing SQL for \"", stmt.sql, "\": ", - *error_message); - return false; - } - } - - ret = sqlite3_step(pragma_foreignkeys.me_); - sqlite3_reset(pragma_foreignkeys.me_); - if (ret != SQLITE_DONE) { - sqlite3_close(me_); - me_ = nullptr; - *error_message = StrCat("while enabling foreign keys: ", - sqlite3_errstr(ret), " (", ret, ")"); - return false; - } - - return true; -} - -Statement Database::Prepare(re2::StringPiece sql, size_t *used, - std::string *error_message) { - Statement statement; - const char *tail; - int err = - sqlite3_prepare_v2(me_, sql.data(), sql.size(), &statement.me_, &tail); - if (err != SQLITE_OK) { - *error_message = StrCat("prepare: ", sqlite3_errstr(err), " (", err, ")"); - return statement; - } - if (used != nullptr) { - *used = tail - sql.data(); - } - if (statement.me_ == nullptr) { - error_message->clear(); - } - return statement; -} - -bool RunStatements(DatabaseContext *ctx, re2::StringPiece stmts, - std::string *error_message) { - while (true) { - size_t used = 0; - auto stmt = ctx->db()->Prepare(stmts, &used, error_message); - if (!stmt.valid()) { - // Statement didn't parse. If |error_message| is empty, there are just no - // more statements. Otherwise this is due to an error. Either way, return. - return error_message->empty(); - } - VLOG(1) << "Running statement:\n" << stmts.substr(0, used).as_string(); - int64_t rows = 0; - auto run = ctx->Borrow(&stmt); - while (run.Step() == SQLITE_ROW) { - ++rows; - } - if (rows > 0) { - VLOG(1) << "Statement returned " << rows << " row(s)."; - } - if (run.status() != SQLITE_DONE) { - VLOG(1) << "Statement failed with " << run.status() << ": " - << run.error_message(); - *error_message = - StrCat("Unexpected error ", run.error_message(), - " from statement: \"", stmts.substr(0, used), "\""); - return false; - } - VLOG(1) << "Statement succeeded."; - stmts.remove_prefix(used); - } -} - -} // namespace moonfire_nvr diff --git a/src/sqlite.h b/src/sqlite.h deleted file mode 100644 index df82061..0000000 --- a/src/sqlite.h +++ /dev/null @@ -1,276 +0,0 @@ -// This file is part of Moonfire NVR, a security camera digital video recorder. -// Copyright (C) 2016 Scott Lamb -// -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// In addition, as a special exception, the copyright holders give -// permission to link the code of portions of this program with the -// OpenSSL library under certain conditions as described in each -// individual source file, and distribute linked combinations including -// the two. -// -// You must obey the GNU General Public License in all respects for all -// of the code used other than OpenSSL. If you modify file(s) with this -// exception, you may extend this exception to your version of the -// file(s), but you are not obligated to do so. If you do not wish to do -// so, delete this exception statement from your version. If you delete -// this exception statement from all source files in the program, then -// also delete it here. -// -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. -// -// You should have received a copy of the GNU General Public License -// along with this program. If not, see . -// -// sqlite.h: a quick C++ wrapper interface around the SQLite3 C API. -// This provides RAII and takes advantage of some types like re2::StringPiece. -// It makes no attempt to hide how the underlying API works, so read -// alongside: - -#ifndef MOONFIRE_NVR_SQLITE_H -#define MOONFIRE_NVR_SQLITE_H - -#include -#include -#include - -#include -#include -#include - -#include "common.h" - -namespace moonfire_nvr { - -// Prepared statement. Movable, not copyable. -// The caller can obtain a Statement via Database::Prepare -// and use one via DatabaseContext::Borrow. -class Statement { - public: - Statement() {} - Statement(Statement &&); - void operator=(Statement &&); - ~Statement(); - - bool valid() const { return me_ != nullptr; } - - private: - friend class Database; - friend class DatabaseContext; - friend class RunningStatement; - - Statement(const Statement &) = delete; - void operator=(const Statement &) = delete; - - void Clear(); - - sqlite3_stmt *me_ = nullptr; // owned. - bool borrowed_ = false; -}; - -class Database { - public: - Database() {} - Database(const Database &) = delete; - Database &operator=(const Database &) = delete; - - // PRE: all DatabaseContext and Statement objects have been deleted. - ~Database(); - - // Open the database and do initial setup. - // - // Foreign keys will always be enabled via "pragma foreign_keys = true;". - // - // extended result codes will always be enabled via - // sqlite3_extended_result_codes. - bool Open(const char *filename, int flags, std::string *error_message); - - // Prepare a statement. Thread-safe. - // - // |used|, if non-null, will be updated with the number of bytes used from - // |sql| on success. (Only the first statement is parsed.) - // - // Returns a statement, which may or may not be valid(). - // |error_message| will be empty if there is simply no statement to parse. - Statement Prepare(re2::StringPiece sql, size_t *used, - std::string *error_message); - - private: - friend class DatabaseContext; - friend class RunningStatement; - sqlite3 *me_ = nullptr; - Statement begin_transaction_; - Statement commit_transaction_; - Statement rollback_transaction_; - - std::mutex ctx_mu_; // used by DatabaseContext. -}; - -// A running statement; get via DatabaseContext::Borrow or -// DatabaseContext::UseOnce. Example uses: -// -// { -// DatabaseContext ctx(&db); -// auto run = ctx.UseOnce("insert into table (column) values (:column)"); -// run.BindText(":column", "value"); -// if (run.Step() != SQLITE_DONE) { -// LOG(ERROR) << "Operation failed: " << run.error_message(); -// return; -// } -// int64_t rowid = ctx.last_insert_rowid(); -// } -// -// Statement select_stmt = db.Prepare( -// "select rowid from table;", nullptr, &error_message); -// ... -// { -// auto run = ctx.Borrow(&select_stmt); -// while (run.Step() == SQLITE_ROW) { -// int64_t rowid = op.ColumnInt64(0); -// ... -// } -// if (run.status() != SQLITE_DONE) { -// LOG(ERROR) << "Operation failed: " << run.error_message(); -// } -// } -class RunningStatement { - public: - RunningStatement(RunningStatement &&o); - - // Reset/unbind/return the statement for the next use (in the case of - // Borrow) or delete it (in the case of UseOnce). - ~RunningStatement(); - - // Bind a value to a parameter. Call before the first Step. - // |param| is indexed from 1 (unlike columns!). - // - // StringPiece |value|s will be copied; they do not need to outlive the - // Bind{Blob,Text} call. Text values are assumed to be UTF-8. - // - // Errors are deferred until Step() for simplicity of caller code. - void BindBlob(int param, re2::StringPiece value); - void BindBlob(const char *param, re2::StringPiece value); - void BindText(int param, re2::StringPiece value); - void BindText(const char *param, re2::StringPiece value); - void BindInt64(int param, int64_t value); - void BindInt64(const char *param, int64_t value); - - // Advance the statement, returning SQLITE_ROW, SQLITE_DONE, or an error. - // Note that this may return a "deferred error" if UseOnce failed to parse - // the SQL or if a bind failed. - int Step(); - - // Convenience function; re-return the last status from Step(). - int status() { return status_; } - - // Return a stringified version of the last status. - // This may have more information than sqlite3_errstr(status()), - // in the case of "deferred errors". - std::string error_message() { return error_message_; } - - // Column accessors, to be called after Step() returns SQLITE_ROW. - // Columns are indexed from 0 (unlike bind parameters!). - // StringPiece values are valid only until a type conversion, the following - // NextRow() call, or destruction of the RunningStatement, whichever - // happens first. - // - // Note there is no useful way to report error here. In particular, the - // underlying SQLite functions return a default value on error, which can't - // be distinguished from a legitimate value. The error code is set on the - // database, but it's not guaranteed to *not* be set if there's no error. - - // Return the type of a given column; if SQLITE_NULL, the value is null. - // As noted in sqlite3_column_type() documentation, this is only meaningful - // if other Column* calls have not forced a type conversion. - int ColumnType(int col); - - re2::StringPiece ColumnBlob(int col); - int64_t ColumnInt64(int col); - re2::StringPiece ColumnText(int col); - - private: - friend class DatabaseContext; - RunningStatement(Statement *stmt, const std::string &deferred_error, - bool own_statement); - RunningStatement(const RunningStatement &) = delete; - void operator=(const RunningStatement &) = delete; - - Statement *statement_ = nullptr; // maybe owned; see owns_statement_. - std::string error_message_; - int status_ = SQLITE_OK; - bool owns_statement_ = false; -}; - -// A scoped database lock and transaction manager. -// -// Moonfire NVR does all SQLite operations under a lock, to avoid SQLITE_BUSY -// and so that calls such as sqlite3_last_insert_rowid return useful values. -// This class implicitly acquires the lock on entry / releases it on exit. -// In the future, it may have instrumentation to track slow operations. -class DatabaseContext { - public: - // Acquire a lock on |db|, which must already be opened. - explicit DatabaseContext(Database *db); - DatabaseContext(const DatabaseContext &) = delete; - void operator=(const DatabaseContext &) = delete; - - // Release the lock and, if an explicit transaction is active, roll it - // back with a logged warning. - ~DatabaseContext(); - - // Begin a transaction, or return false and fill |error_message|. - // If successful, the caller should explicitly call CommitTransaction or - // RollbackTransaction before the DatabaseContext goes out of scope. - bool BeginTransaction(std::string *error_message); - - // Commit the transaction, or return false and fill |error_message|. - bool CommitTransaction(std::string *error_message); - - // Roll back the transaction, logging error on failure. - // The error code is not returned; there's nothing useful the caller can do. - void RollbackTransaction(); - - // Borrow a prepared statement to run. - // |statement| should outlive the RunningStatement. It can't be borrowed - // twice simultaneously, but two similar statements can be run side-by-side - // (in the same context). - RunningStatement Borrow(Statement *statement); - - // Use the given |sql| once. - // Note that parse errors are "deferred" until RunningStatement::Step(). - RunningStatement UseOnce(re2::StringPiece sql); - - // Return the number of changes for the last DML statement (insert, update, or - // delete), as with sqlite3_changes. - int64_t changes() { return sqlite3_changes(db_->me_); } - - // Return the last rowid inserted into a table that does not specify "WITHOUT - // ROWID", as with sqlite3_last_insert_rowid. - int64_t last_insert_rowid() { return sqlite3_last_insert_rowid(db_->me_); } - - Database *db() { return db_; } - - private: - Database *db_; - std::lock_guard lock_; - bool transaction_open_ = false; -}; - -// Convenience routines below. - -// Run through all the statements in |stmts|. -// Return error if any do not parse or return something other than SQLITE_DONE -// when stepped. (SQLITE_ROW returns are skipped over, though. This is useful -// for "pragma journal_mode = wal;" which returns a row.) -bool RunStatements(DatabaseContext *ctx, re2::StringPiece stmts, - std::string *error_message); - -} // namespace moonfire_nvr - -#endif // MOONFIRE_NVR_SQLITE_H diff --git a/src/stream.rs b/src/stream.rs new file mode 100644 index 0000000..7008059 --- /dev/null +++ b/src/stream.rs @@ -0,0 +1,162 @@ +// This file is part of Moonfire NVR, a security camera digital video recorder. +// Copyright (C) 2016 Scott Lamb +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// In addition, as a special exception, the copyright holders give +// permission to link the code of portions of this program with the +// OpenSSL library under certain conditions as described in each +// individual source file, and distribute linked combinations including +// the two. +// +// You must obey the GNU General Public License in all respects for all +// of the code used other than OpenSSL. If you modify file(s) with this +// exception, you may extend this exception to your version of the +// file(s), but you are not obligated to do so. If you do not wish to do +// so, delete this exception statement from your version. If you delete +// this exception statement from all source files in the program, then +// also delete it here. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program. If not, see . + +use error::Error; +use ffmpeg::{self, format, media}; +use ffmpeg_sys::{self, AVLockOp}; +use h264; +use libc::{self, c_int, c_void}; +use std::mem; +use std::ptr; +use std::result::Result; +use std::slice; +use std::sync; + +static START: sync::Once = sync::ONCE_INIT; + +pub enum StreamSource<'a> { + #[cfg(test)] + File(&'a str), // filename, for testing. + + Rtsp(&'a str), // url, for production use. +} + +// TODO: I think this should be provided by ffmpeg-sys. Otherwise, ffmpeg-sys is thread-hostile, +// which I believe is not allowed at all in Rust. (Also, this method's signature should include +// unsafe.) +extern "C" fn lock_callback(untyped_ptr: *mut *mut c_void, op: AVLockOp) -> c_int { + unsafe { + let ptr = mem::transmute::<*mut *mut c_void, *mut *mut libc::pthread_mutex_t>(untyped_ptr); + match op { + AVLockOp::AV_LOCK_CREATE => { + let m = Box::::new(mem::uninitialized()); + *ptr = Box::into_raw(m); + libc::pthread_mutex_init(*ptr, ptr::null()); + }, + AVLockOp::AV_LOCK_DESTROY => { + libc::pthread_mutex_destroy(*ptr); + Box::from_raw(*ptr); // delete. + *ptr = ptr::null_mut(); + }, + AVLockOp::AV_LOCK_OBTAIN => { + libc::pthread_mutex_lock(*ptr); + }, + AVLockOp::AV_LOCK_RELEASE => { + libc::pthread_mutex_unlock(*ptr); + }, + }; + }; + 0 +} + +impl<'a> StreamSource<'a> { + pub fn open(&self) -> Result { + START.call_once(|| { + unsafe { ffmpeg_sys::av_lockmgr_register(lock_callback); }; + ffmpeg::init().unwrap(); + ffmpeg::format::network::init(); + + }); + + let (input, discard_first) = match *self { + #[cfg(test)] + StreamSource::File(filename) => + (format::input_with(&format!("file:{}", filename), ffmpeg::Dictionary::new())?, + false), + StreamSource::Rtsp(url) => { + let open_options = dict![ + "rtsp_transport" => "tcp", + // https://trac.ffmpeg.org/ticket/5018 workaround attempt. + "probesize" => "262144", + "user-agent" => "moonfire-nvr", + // 10-second socket timeout, in microseconds. + "stimeout" => "10000000" + ]; + (format::input_with(&url, open_options)?, true) + }, + }; + + // Find the video stream. + let mut video_i = None; + for (i, stream) in input.streams().enumerate() { + if stream.codec().medium() == media::Type::Video { + debug!("Video stream index is {}", i); + video_i = Some(i); + break; + } + } + let video_i = match video_i { + Some(i) => i, + None => { return Err(Error::new("no video stream".to_owned())) }, + }; + + let mut stream = Stream{ + input: input, + video_i: video_i, + }; + + if discard_first { + info!("Discarding the first packet to work around https://trac.ffmpeg.org/ticket/5018"); + stream.get_next()?; + } + + Ok(stream) + } +} + +pub struct Stream { + input: format::context::Input, + video_i: usize, +} + +impl Stream { + pub fn get_extra_data(&self) -> Result { + let video = self.input.stream(self.video_i).expect("can't get video stream known to exist"); + let codec = video.codec(); + let (extradata, width, height) = unsafe { + let ptr = codec.as_ptr(); + (slice::from_raw_parts((*ptr).extradata, (*ptr).extradata_size as usize), + (*ptr).width as u16, + (*ptr).height as u16) + }; + // TODO: verify video stream is h264. + h264::ExtraData::parse(extradata, width, height) + } + + pub fn get_next(&mut self) -> Result { + let mut pkt = ffmpeg::Packet::empty(); + loop { + pkt.read(&mut self.input)?; + if pkt.stream() == self.video_i { + return Ok(pkt); + } + } + } +} diff --git a/src/streamer.rs b/src/streamer.rs new file mode 100644 index 0000000..213ff97 --- /dev/null +++ b/src/streamer.rs @@ -0,0 +1,158 @@ +// This file is part of Moonfire NVR, a security camera digital video recorder. +// Copyright (C) 2016 Scott Lamb +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// In addition, as a special exception, the copyright holders give +// permission to link the code of portions of this program with the +// OpenSSL library under certain conditions as described in each +// individual source file, and distribute linked combinations including +// the two. +// +// You must obey the GNU General Public License in all respects for all +// of the code used other than OpenSSL. If you modify file(s) with this +// exception, you may extend this exception to your version of the +// file(s), but you are not obligated to do so. If you do not wish to do +// so, delete this exception statement from your version. If you delete +// this exception statement from all source files in the program, then +// also delete it here. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program. If not, see . + +use db::{Camera, Database}; +use dir; +use error::Error; +use h264; +use recording; +use std::result::Result; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::Arc; +use std::thread; +use std::time::Duration; +use stream::StreamSource; +use time; + +pub static ROTATE_INTERVAL_SEC: i64 = 60; + +pub struct Streamer { + shutdown: Arc, + + // State below is only used by the thread in Run. + rotate_offset_sec: i64, + db: Arc, + dir: Arc, + syncer_channel: dir::SyncerChannel, + camera_id: i32, + short_name: String, + url: String, + redacted_url: String, +} + +impl Streamer { + pub fn new(db: Arc, dir: Arc, syncer_channel: dir::SyncerChannel, + shutdown: Arc, camera_id: i32, c: &Camera, rotate_offset_sec: i64) + -> Self { + Streamer{ + shutdown: shutdown, + rotate_offset_sec: rotate_offset_sec, + db: db, + dir: dir, + syncer_channel: syncer_channel, + camera_id: camera_id, + short_name: c.short_name.to_owned(), + url: format!("rtsp://{}:{}@{}{}", c.username, c.password, c.host, c.main_rtsp_path), + redacted_url: format!("rtsp://{}:redacted@{}{}", c.username, c.host, c.main_rtsp_path), + } + } + + pub fn short_name(&self) -> &str { &self.short_name } + + pub fn run(&mut self) { + while !self.shutdown.load(Ordering::SeqCst) { + if let Err(e) = self.run_once() { + let sleep_time = Duration::from_secs(1); + warn!("{}: sleeping for {:?} after error: {}", self.short_name, sleep_time, e); + thread::sleep(sleep_time); + } + } + info!("{}: shutting down", self.short_name); + } + + fn run_once(&mut self) -> Result<(), Error> { + info!("{}: Opening input: {}", self.short_name, self.redacted_url); + + // TODO: mockability? + let mut stream = StreamSource::Rtsp(&self.url).open()?; + // TODO: verify time base. + // TODO: verify width/height. + let extra_data = stream.get_extra_data()?; + let video_sample_entry_id = + self.db.lock().insert_video_sample_entry(extra_data.width, extra_data.height, + &extra_data.sample_entry)?; + debug!("{}: video_sample_entry_id={}", self.short_name, video_sample_entry_id); + let mut seen_key_frame = false; + let mut rotate = None; + let mut writer: Option = None; + let mut transformed = Vec::new(); + let mut next_start = None; + while !self.shutdown.load(Ordering::SeqCst) { + let pkt = stream.get_next()?; + if !seen_key_frame && !pkt.is_key() { + continue; + } else if !seen_key_frame { + debug!("{}: have first key frame", self.short_name); + seen_key_frame = true; + } + let frame_realtime = time::get_time(); + if let Some(r) = rotate { + if frame_realtime.sec > r && pkt.is_key() { + let w = writer.take().expect("rotate set implies writer is set"); + next_start = Some(w.end()); + // TODO: restore this log message. + // info!("{}: wrote {}: [{}, {})", self.short_name, r.sample_file_uuid, + // r.time.start, r.time.end); + self.syncer_channel.async_save_writer(w)?; + } + }; + let mut w = match writer { + Some(w) => w, + None => { + let r = frame_realtime.sec - + (frame_realtime.sec % ROTATE_INTERVAL_SEC) + + self.rotate_offset_sec; + rotate = Some( + if r <= frame_realtime.sec { r + ROTATE_INTERVAL_SEC } else { r }); + let local_realtime = recording::Time::new(frame_realtime); + + self.dir.create_writer(next_start.unwrap_or(local_realtime), local_realtime, + self.camera_id, video_sample_entry_id)? + }, + }; + let orig_data = match pkt.data() { + Some(d) => d, + None => return Err(Error::new("packet has no data".to_owned())), + }; + let transformed_data = if extra_data.need_transform { + h264::transform_sample_data(orig_data, &mut transformed)?; + transformed.as_slice() + } else { + orig_data + }; + w.write(transformed_data, pkt.duration() as i32, pkt.is_key())?; + writer = Some(w); + } + if let Some(w) = writer { + self.syncer_channel.async_save_writer(w)?; + } + Ok(()) + } +} diff --git a/src/string-test.cc b/src/string-test.cc deleted file mode 100644 index bf62ecb..0000000 --- a/src/string-test.cc +++ /dev/null @@ -1,116 +0,0 @@ -// This file is part of Moonfire NVR, a security camera digital video recorder. -// Copyright (C) 2016 Scott Lamb -// -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// In addition, as a special exception, the copyright holders give -// permission to link the code of portions of this program with the -// OpenSSL library under certain conditions as described in each -// individual source file, and distribute linked combinations including -// the two. -// -// You must obey the GNU General Public License in all respects for all -// of the code used other than OpenSSL. If you modify file(s) with this -// exception, you may extend this exception to your version of the -// file(s), but you are not obligated to do so. If you do not wish to do -// so, delete this exception statement from your version. If you delete -// this exception statement from all source files in the program, then -// also delete it here. -// -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. -// -// You should have received a copy of the GNU General Public License -// along with this program. If not, see . -// -// string-test.cc: tests of the string.h interface. - -#include -#include -#include -#include - -#include "string.h" - -DECLARE_bool(alsologtostderr); - -namespace moonfire_nvr { -namespace { - -TEST(StrCatTest, Simple) { - EXPECT_EQ("foo", StrCat("foo")); - EXPECT_EQ("foobar", StrCat("foo", "bar")); - EXPECT_EQ("foo", StrCat(std::string("foo"))); - - EXPECT_EQ("42", StrCat(uint64_t(42))); - EXPECT_EQ("0", StrCat(uint64_t(0))); - EXPECT_EQ("18446744073709551615", - StrCat(std::numeric_limits::max())); - - EXPECT_EQ("42", StrCat(int64_t(42))); - EXPECT_EQ("0", StrCat(int64_t(0))); - EXPECT_EQ("-9223372036854775808", - StrCat(std::numeric_limits::min())); - EXPECT_EQ("9223372036854775807", StrCat(std::numeric_limits::max())); -} - -TEST(JoinTest, Simple) { - EXPECT_EQ("", Join(std::initializer_list(), ",")); - EXPECT_EQ("a", Join(std::initializer_list({"a"}), ",")); - EXPECT_EQ("a,b", Join(std::initializer_list({"a", "b"}), ",")); - EXPECT_EQ( - "a,b,c", - Join(std::initializer_list({"a", "b", "c"}), ",")); -} - -TEST(EscapeTest, Simple) { - EXPECT_EQ("", moonfire_nvr::EscapeHtml("")); - EXPECT_EQ("no special chars", moonfire_nvr::EscapeHtml("no special chars")); - EXPECT_EQ("<tag> & text", moonfire_nvr::EscapeHtml(" & text")); -} - -TEST(ToHexTest, Simple) { - EXPECT_EQ("", ToHex("", false)); - EXPECT_EQ("", ToHex("", true)); - EXPECT_EQ("1234deadbeef", ToHex("\x12\x34\xde\xad\xbe\xef", false)); - EXPECT_EQ("12 34 de ad be ef", ToHex("\x12\x34\xde\xad\xbe\xef", true)); -} - -TEST(HumanizeTest, Simple) { - EXPECT_EQ("1.0 B", HumanizeWithBinaryPrefix(1.f, "B")); - EXPECT_EQ("1.0 KiB", HumanizeWithBinaryPrefix(UINT64_C(1) << 10, "B")); - EXPECT_EQ("1.0 EiB", HumanizeWithBinaryPrefix(UINT64_C(1) << 60, "B")); - EXPECT_EQ("1.5 EiB", HumanizeWithBinaryPrefix( - (UINT64_C(1) << 60) + (UINT64_C(1) << 59), "B")); - EXPECT_EQ("16.0 EiB", HumanizeWithBinaryPrefix( - std::numeric_limits::max(), "B")); - - EXPECT_EQ("1.0 Mbps", HumanizeWithDecimalPrefix(1e6f, "bps")); - EXPECT_EQ("1000.0 Ebps", HumanizeWithDecimalPrefix(1e21, "bps")); -} - -TEST(AtoiTest, Simple) { - int64_t out; - EXPECT_TRUE(Atoi64("1234", 10, &out)); - EXPECT_EQ(1234, out); - EXPECT_FALSE(Atoi64(nullptr, 10, &out)); - EXPECT_FALSE(Atoi64("", 10, &out)); - EXPECT_FALSE(Atoi64("asdf", 10, &out)); - EXPECT_FALSE(Atoi64("1234asdf", 10, &out)); -} - -} // namespace -} // namespace moonfire_nvr - -int main(int argc, char **argv) { - FLAGS_alsologtostderr = true; - google::ParseCommandLineFlags(&argc, &argv, true); - testing::InitGoogleTest(&argc, argv); - google::InitGoogleLogging(argv[0]); - return RUN_ALL_TESTS(); -} diff --git a/src/string.cc b/src/string.cc deleted file mode 100644 index 506f061..0000000 --- a/src/string.cc +++ /dev/null @@ -1,189 +0,0 @@ -// This file is part of Moonfire NVR, a security camera digital video recorder. -// Copyright (C) 2016 Scott Lamb -// -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// In addition, as a special exception, the copyright holders give -// permission to link the code of portions of this program with the -// OpenSSL library under certain conditions as described in each -// individual source file, and distribute linked combinations including -// the two. -// -// You must obey the GNU General Public License in all respects for all -// of the code used other than OpenSSL. If you modify file(s) with this -// exception, you may extend this exception to your version of the -// file(s), but you are not obligated to do so. If you do not wish to do -// so, delete this exception statement from your version. If you delete -// this exception statement from all source files in the program, then -// also delete it here. -// -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. -// -// You should have received a copy of the GNU General Public License -// along with this program. If not, see . -// -// string.cc: See string.h. - -#include "string.h" - -#include - -#include - -namespace moonfire_nvr { - -namespace { - -char HexDigit(unsigned int i) { - static char kHexadigits[] = "0123456789abcdef"; - return (i < 16) ? kHexadigits[i] : 'x'; -} - -std::string Humanize(std::initializer_list prefixes, - float f, float n, re2::StringPiece suffix) { - size_t i; - for (i = 0; i < prefixes.size() - 1 && n >= f; ++i) n /= f; - char buf[64]; - snprintf(buf, sizeof(buf), "%.1f", n); - return StrCat(buf, *(prefixes.begin() + i), suffix); -} - -} // namespace - -namespace internal { - -StrCatPiece::StrCatPiece(uint64_t p) { - if (p == 0) { - piece_ = "0"; - } else { - size_t i = sizeof(buf_); - while (p != 0) { - buf_[--i] = '0' + (p % 10); - p /= 10; - } - piece_.set(buf_ + i, sizeof(buf_) - i); - } -} - -StrCatPiece::StrCatPiece(int64_t p) { - if (p == 0) { - piece_ = "0"; - } else { - bool negative = p < 0; - size_t i = sizeof(buf_); - while (p != 0) { - buf_[--i] = '0' + std::abs(p % 10); - p /= 10; - } - if (negative) { - buf_[--i] = '-'; - } - piece_.set(buf_ + i, sizeof(buf_) - i); - } -} - -} // namespace internal - -std::string EscapeHtml(const std::string &input) { - std::string output; - output.reserve(input.size()); - for (char c : input) { - switch (c) { - case '&': - output.append("&"); - break; - case '<': - output.append("<"); - break; - case '>': - output.append(">"); - break; - default: - output.push_back(c); - } - } - return output; -} - -std::string ToHex(re2::StringPiece in, bool pad) { - std::string out; - out.reserve(in.size() * (2 + pad) + pad); - for (int i = 0; i < in.size(); ++i) { - if (pad && i > 0) out.push_back(' '); - uint8_t byte = in[i]; - out.push_back(HexDigit(byte >> 4)); - out.push_back(HexDigit(byte & 0x0F)); - } - return out; -} - -std::string HumanizeWithDecimalPrefix(float n, re2::StringPiece suffix) { - static const std::initializer_list kPrefixes = { - " ", " k", " M", " G", " T", " P", " E"}; - return Humanize(kPrefixes, 1000., n, suffix); -} - -std::string HumanizeWithBinaryPrefix(float n, re2::StringPiece suffix) { - static const std::initializer_list kPrefixes = { - " ", " Ki", " Mi", " Gi", " Ti", " Pi", " Ei"}; - return Humanize(kPrefixes, 1024., n, suffix); -} - -std::string HumanizeDuration(int64_t seconds) { - static const int64_t kMinuteInSeconds = 60; - static const int64_t kHourInSeconds = 60 * kMinuteInSeconds; - static const int64_t kDayInSeconds = 24 * kHourInSeconds; - int64_t days = seconds / kDayInSeconds; - seconds %= kDayInSeconds; - int64_t hours = seconds / kHourInSeconds; - seconds %= kHourInSeconds; - int64_t minutes = seconds / kMinuteInSeconds; - seconds %= kMinuteInSeconds; - std::string out; - if (days > 0) { - out.append(StrCat(days, days == 1 ? " day" : " days")); - } - if (hours > 0) { - if (!out.empty()) { - out.append(" "); - } - out.append(StrCat(hours, hours == 1 ? " hour" : " hours")); - } - if (minutes > 0) { - if (!out.empty()) { - out.append(" "); - } - out.append(StrCat(minutes, minutes == 1 ? " minute" : " minutes")); - } - if (seconds > 0 || out.empty()) { - if (!out.empty()) { - out.append(" "); - } - out.append(StrCat(seconds, seconds == 1 ? " second" : " seconds")); - } - return out; -} - -bool strto64(const char *str, int base, const char **endptr, int64_t *value) { - static_assert(sizeof(int64_t) == sizeof(long long int), - "unknown memory model"); - if (str == nullptr) { - return false; - } - errno = 0; - *value = ::strtoll(str, const_cast(endptr), base); - return *endptr != str && errno == 0; -} - -bool Atoi64(const char *str, int base, int64_t *value) { - const char *endptr; - return strto64(str, base, &endptr, value) && *endptr == '\0'; -} - -} // namespace moonfire_nvr diff --git a/src/string.h b/src/string.h deleted file mode 100644 index b5b2b47..0000000 --- a/src/string.h +++ /dev/null @@ -1,138 +0,0 @@ -// This file is part of Moonfire NVR, a security camera digital video recorder. -// Copyright (C) 2016 Scott Lamb -// -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// In addition, as a special exception, the copyright holders give -// permission to link the code of portions of this program with the -// OpenSSL library under certain conditions as described in each -// individual source file, and distribute linked combinations including -// the two. -// -// You must obey the GNU General Public License in all respects for all -// of the code used other than OpenSSL. If you modify file(s) with this -// exception, you may extend this exception to your version of the -// file(s), but you are not obligated to do so. If you do not wish to do -// so, delete this exception statement from your version. If you delete -// this exception statement from all source files in the program, then -// also delete it here. -// -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. -// -// You should have received a copy of the GNU General Public License -// along with this program. If not, see . -// -// string.h: convenience methods for dealing with strings. - -#ifndef MOONFIRE_NVR_STRING_H -#define MOONFIRE_NVR_STRING_H - -#include - -#include - -namespace moonfire_nvr { - -namespace internal { - -// Only used within StrCat() and Join(). -// Note implicit constructor, which is necessary to avoid a copy, -// though it could be wrapped in another type. -// http://stackoverflow.com/questions/34112755/can-i-avoid-a-c11-move-when-initializing-an-array/34113744 -class StrCatPiece { - public: - StrCatPiece(uint64_t p); - StrCatPiece(int64_t p); - StrCatPiece(uint32_t p) : StrCatPiece(static_cast(p)) {} - StrCatPiece(int32_t p) : StrCatPiece(static_cast(p)) {} - -#ifndef __LP64__ // if sizeof(long) == sizeof(int32_t) - // Need to resolve ambiguity. - StrCatPiece(long p) : StrCatPiece(static_cast(p)) {} - StrCatPiece(unsigned long p) : StrCatPiece(static_cast(p)) {} -#endif - - StrCatPiece(re2::StringPiece p) : piece_(p) {} - - StrCatPiece(const StrCatPiece &) = delete; - StrCatPiece &operator=(const StrCatPiece &) = delete; - - const char *data() const { return piece_.data(); } - size_t size() const { return piece_.size(); } - - private: - // Not allowed: ambiguous meaning. - StrCatPiece(char); - - // |piece_| points either to within buf_ (numeric constructors) or to unowned - // string data (StringPiece constructor). - re2::StringPiece piece_; - char buf_[20]; // length of maximum uint64 (no terminator needed). -}; - -} // namespace internal - -// Concatenate any number of strings, StringPieces, and numeric values into a -// single string. -template -std::string StrCat(Types... args) { - internal::StrCatPiece pieces[] = {{args}...}; - size_t size = 0; - for (const auto &p : pieces) { - size += p.size(); - } - std::string out; - out.reserve(size); - for (const auto &p : pieces) { - out.append(p.data(), p.size()); - } - return out; -} - -// Join any number of string fragments (of like type) together into a single -// string, with a separator. -template -std::string Join(const Container &pieces, re2::StringPiece separator) { - std::string out; - bool first = true; - for (const auto &p : pieces) { - if (!first) { - out.append(separator.data(), separator.size()); - } - first = false; - internal::StrCatPiece piece(p); - out.append(piece.data(), piece.size()); - } - return out; -} - -// HTML-escape the given UTF-8-encoded string. -std::string EscapeHtml(const std::string &input); - -// Return a hex-encoded version of |in|, optionally padding between bytes. -// For example, ToHex("\xde\xad\xbe\xef", true) returns "de ad be ef". -std::string ToHex(re2::StringPiece in, bool pad = false); - -// Return a human-friendly approximation of the given non-negative value, using -// SI (base-10) or IEC (base-2) prefixes. -std::string HumanizeWithDecimalPrefix(float n, re2::StringPiece suffix); -std::string HumanizeWithBinaryPrefix(float n, re2::StringPiece suffix); - -std::string HumanizeDuration(int64_t sec); - -// Wrapper around ::strtoll that returns true iff valid and corrects -// constness. Returns false if |str| is null. -bool strto64(const char *str, int base, const char **endptr, int64_t *value); - -// Simpler form that expects the entire string to be a single integer. -bool Atoi64(const char *str, int base, int64_t *value); - -} // namespace moonfire_nvr - -#endif // MOONFIRE_NVR_STRING_H diff --git a/src/testutil.cc b/src/testutil.cc deleted file mode 100644 index f4e7e45..0000000 --- a/src/testutil.cc +++ /dev/null @@ -1,153 +0,0 @@ -// This file is part of Moonfire NVR, a security camera digital video recorder. -// Copyright (C) 2016 Scott Lamb -// -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// In addition, as a special exception, the copyright holders give -// permission to link the code of portions of this program with the -// OpenSSL library under certain conditions as described in each -// individual source file, and distribute linked combinations including -// the two. -// -// You must obey the GNU General Public License in all respects for all -// of the code used other than OpenSSL. If you modify file(s) with this -// exception, you may extend this exception to your version of the -// file(s), but you are not obligated to do so. If you do not wish to do -// so, delete this exception statement from your version. If you delete -// this exception statement from all source files in the program, then -// also delete it here. -// -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. -// -// You should have received a copy of the GNU General Public License -// along with this program. If not, see . -// -// testutil.cc: implementation of testutil.h interface. - -#include "testutil.h" - -#include -#include -#include -#include -#include -#include - -#include -#include - -#include "filesystem.h" -#include "string.h" - -namespace moonfire_nvr { - -namespace { - -bool DeleteChildrenRecursively(const char *dirname, std::string *error_msg) { - bool ok = true; - auto fn = [&dirname, &ok, error_msg](const struct dirent *ent) { - std::string name(ent->d_name); - std::string path = StrCat(dirname, "/", name); - if (name == "." || name == "..") { - return IterationControl::kContinue; - } - bool is_dir = (ent->d_type == DT_DIR); - if (ent->d_type == DT_UNKNOWN) { - struct stat buf; - int ret = GetRealFilesystem()->Stat(path.c_str(), &buf); - CHECK_EQ(ret, 0) << path << ": " << strerror(ret); - is_dir = S_ISDIR(buf.st_mode); - } - if (is_dir) { - ok = ok && DeleteChildrenRecursively(path.c_str(), error_msg); - if (!ok) { - return IterationControl::kBreak; - } - int ret = GetRealFilesystem()->Rmdir(path.c_str()); - if (ret != 0) { - *error_msg = StrCat("rmdir failed on ", path, ": ", strerror(ret)); - ok = false; - return IterationControl::kBreak; - } - } else { - int ret = GetRealFilesystem()->Unlink(path.c_str()); - if (ret != 0) { - *error_msg = StrCat("unlink failed on ", path, ": ", strerror(ret)); - ok = false; - return IterationControl::kBreak; - } - } - return IterationControl::kContinue; - }; - if (!GetRealFilesystem()->DirForEach(dirname, fn, error_msg)) { - return false; - } - return ok; -} - -} // namespace - -std::string PrepareTempDirOrDie(const std::string &test_name) { - std::string dirname = StrCat("/tmp/test.", test_name); - int ret = GetRealFilesystem()->Mkdir(dirname.c_str(), 0700); - if (ret != 0) { - CHECK_EQ(ret, EEXIST) << "mkdir failed: " << strerror(ret); - std::string error_msg; - CHECK(DeleteChildrenRecursively(dirname.c_str(), &error_msg)) << error_msg; - } - return dirname; -} - -void WriteFileOrDie(const std::string &path, re2::StringPiece contents) { - std::unique_ptr f; - int ret = GetRealFilesystem()->Open(path.c_str(), - O_WRONLY | O_CREAT | O_TRUNC, 0600, &f); - CHECK_EQ(ret, 0) << "open " << path << ": " << strerror(ret); - while (!contents.empty()) { - size_t written; - ret = f->Write(contents, &written); - CHECK_EQ(ret, 0) << "write " << path << ": " << strerror(ret); - contents.remove_prefix(written); - } - ret = f->Close(); - CHECK_EQ(ret, 0) << "close " << path << ": " << strerror(ret); -} - -void WriteFileOrDie(const std::string &path, EvBuffer *buf) { - int fd = open(path.c_str(), O_CREAT | O_WRONLY | O_TRUNC, 0600); - PCHECK(fd >= 0) << "open: " << path; - while (evbuffer_get_length(buf->get()) > 0) { - size_t buf_len = evbuffer_get_length(buf->get()); - int written = evbuffer_write(buf->get(), fd); - PCHECK(written >= 0) << "buf_len: " << buf_len << ", written: " << written; - } - PCHECK(close(fd) == 0) << "close"; -} - -std::string ReadFileOrDie(const std::string &path) { - std::unique_ptr f; - int ret = GetRealFilesystem()->Open(path.c_str(), O_RDONLY, &f); - CHECK_EQ(ret, 0) << "open " << path << ": " << strerror(ret); - struct stat statbuf; - ret = f->Stat(&statbuf); - CHECK_EQ(ret, 0) << "fstat " << path << ": " << strerror(ret); - std::string out(statbuf.st_size, '0'); - off_t bytes_read_total = 0; - size_t bytes_read; - while (bytes_read_total < statbuf.st_size) { - ret = f->Read(&out[bytes_read_total], statbuf.st_size - bytes_read_total, - &bytes_read); - CHECK_EQ(ret, 0) << "read " << path << ": " << strerror(ret); - CHECK_GT(bytes_read, 0); - bytes_read_total += bytes_read; - } - return out; -} - -} // namespace moonfire_nvr diff --git a/src/testutil.h b/src/testutil.h deleted file mode 100644 index 874fa14..0000000 --- a/src/testutil.h +++ /dev/null @@ -1,155 +0,0 @@ -// This file is part of Moonfire NVR, a security camera digital video recorder. -// Copyright (C) 2016 Scott Lamb -// -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// In addition, as a special exception, the copyright holders give -// permission to link the code of portions of this program with the -// OpenSSL library under certain conditions as described in each -// individual source file, and distribute linked combinations including -// the two. -// -// You must obey the GNU General Public License in all respects for all -// of the code used other than OpenSSL. If you modify file(s) with this -// exception, you may extend this exception to your version of the -// file(s), but you are not obligated to do so. If you do not wish to do -// so, delete this exception statement from your version. If you delete -// this exception statement from all source files in the program, then -// also delete it here. -// -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. -// -// You should have received a copy of the GNU General Public License -// along with this program. If not, see . -// -// testutil.h: utilities for testing. - -#ifndef MOONFIRE_NVR_TESTUTIL_H -#define MOONFIRE_NVR_TESTUTIL_H - -#include -#include -#include - -#include "filesystem.h" -#include "http.h" -#include "uuid.h" - -namespace moonfire_nvr { - -// Create or empty the given test directory, or die. -// Returns the full path. -std::string PrepareTempDirOrDie(const std::string &test_name); - -// Write the given file contents to the given path, or die. -void WriteFileOrDie(const std::string &path, re2::StringPiece contents); -void WriteFileOrDie(const std::string &path, EvBuffer *buf); - -// Read the contents of the given path, or die. -std::string ReadFileOrDie(const std::string &path); - -// A scoped log sink for testing that the right log messages are sent. -// Modelled after glog's "mock-log.h", which is not exported. -// Use as follows: -// -// { -// ScopedMockLog log; -// EXPECT_CALL(log, Log(ERROR, _, HasSubstr("blah blah"))); -// log.Start(); -// ThingThatLogs(); -// } -class ScopedMockLog : public google::LogSink { - public: - ~ScopedMockLog() final { google::RemoveLogSink(this); } - - // Start logging to this sink. - // This is not done at construction time so that it's possible to set - // expectations first, which is important if some background thread is - // already logging. - void Start() { google::AddLogSink(this); } - - // Set expectations here. - MOCK_METHOD3(Log, void(google::LogSeverity severity, - const std::string &full_filename, - const std::string &message)); - - private: - struct LogEntry { - google::LogSeverity severity = -1; - std::string full_filename; - std::string message; - }; - - // This method is called with locks held and thus shouldn't call Log. - // It just stashes away the log entry for later. - void send(google::LogSeverity severity, const char *full_filename, - const char *base_filename, int line, const tm *tm_time, - const char *message, size_t message_len) final { - pending_.severity = severity; - pending_.full_filename = full_filename; - pending_.message.assign(message, message_len); - } - - // This method is always called after send() without locks. - // It does the actual work of calling Log. It moves data away from - // pending_ in case Log() logs itself (causing a nested call to send() and - // WaitTillSent()). - void WaitTillSent() final { - LogEntry entry = std::move(pending_); - Log(entry.severity, entry.full_filename, entry.message); - } - - LogEntry pending_; -}; - -class MockUuidGenerator : public UuidGenerator { - public: - MOCK_METHOD0(Generate, Uuid()); -}; - -class MockFile : public File { - public: - MOCK_CONST_METHOD0(name, const std::string &()); - MOCK_METHOD3(Access, int(const char *, int, int)); - MOCK_METHOD0(Close, int()); - - // The std::unique_ptr variants of Open are wrapped here because gmock's - // SetArgPointee doesn't work well with std::unique_ptr. - - int Open(const char *path, int flags, std::unique_ptr *f) final { - File *f_tmp = nullptr; - int ret = OpenRaw(path, flags, &f_tmp); - f->reset(f_tmp); - return ret; - } - - int Open(const char *path, int flags, mode_t mode, - std::unique_ptr *f) final { - File *f_tmp = nullptr; - int ret = OpenRaw(path, flags, mode, &f_tmp); - f->reset(f_tmp); - return ret; - } - - MOCK_METHOD1(Lock, int(int)); - MOCK_METHOD3(Open, int(const char *, int, int *)); - MOCK_METHOD4(Open, int(const char *, int, mode_t, int *)); - MOCK_METHOD3(OpenRaw, int(const char *, int, File **)); - MOCK_METHOD4(OpenRaw, int(const char *, int, mode_t, File **)); - MOCK_METHOD3(Read, int(void *, size_t, size_t *)); - MOCK_METHOD1(Stat, int(struct stat *)); - MOCK_METHOD0(Sync, int()); - MOCK_METHOD1(Truncate, int(off_t)); - MOCK_METHOD1(Unlink, int(const char *)); - MOCK_METHOD2(Write, int(re2::StringPiece, size_t *)); -}; - -} // namespace moonfire_nvr - -#endif // MOONFIRE_NVR_TESTUTIL_H diff --git a/src/profiler.h b/src/testutil.rs similarity index 64% rename from src/profiler.h rename to src/testutil.rs index e18df74..07f53cd 100644 --- a/src/profiler.h +++ b/src/testutil.rs @@ -27,25 +27,28 @@ // // You should have received a copy of the GNU General Public License // along with this program. If not, see . -// -// profiler.h: support for on-demand profiling. The interface is described here: -// -// Currently this only supports CPU profiling; heap profiling may be added -// later. -#ifndef MOONFIRE_NVR_PROFILER_H -#define MOONFIRE_NVR_PROFILER_H +use std::env; +use std::sync; +use slog::{self, DrainExt}; +use slog_envlogger; +use slog_stdlog; +use slog_term; +use time; -#include -#include +static INIT: sync::Once = sync::ONCE_INIT; -namespace moonfire_nvr { - -// Register a HTTP URI for a CPU profile with google-perftools. -// Not thread-safe, so |http| must be running on |base|, and |base| must be -// using a single thread. -void RegisterProfiler(event_base *base, evhttp *http); - -} // namespace moonfire_nvr - -#endif // MOONFIRE_NVR_PROFILER_H +/// Performs global initialization for tests. +/// * set up logging. (Note the output can be confusing unless `RUST_TEST_THREADS=1` is set in +/// the program's environment prior to running.) +/// * set `TZ=America/Los_Angeles` so that tests that care about calendar time get the expected +/// results regardless of machine setup.) +pub fn init() { + INIT.call_once(|| { + let drain = slog_term::StreamerBuilder::new().async().full().build(); + let drain = slog_envlogger::new(drain); + slog_stdlog::set_logger(slog::Logger::root(drain.ignore_err(), None)).unwrap(); + env::set_var("TZ", "America/Los_Angeles"); + time::tzset(); + }); +} diff --git a/src/time.cc b/src/time.cc deleted file mode 100644 index 8f5b03b..0000000 --- a/src/time.cc +++ /dev/null @@ -1,90 +0,0 @@ -// This file is part of Moonfire NVR, a security camera digital video recorder. -// Copyright (C) 2016 Scott Lamb -// -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// In addition, as a special exception, the copyright holders give -// permission to link the code of portions of this program with the -// OpenSSL library under certain conditions as described in each -// individual source file, and distribute linked combinations including -// the two. -// -// You must obey the GNU General Public License in all respects for all -// of the code used other than OpenSSL. If you modify file(s) with this -// exception, you may extend this exception to your version of the -// file(s), but you are not obligated to do so. If you do not wish to do -// so, delete this exception statement from your version. If you delete -// this exception statement from all source files in the program, then -// also delete it here. -// -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. -// -// You should have received a copy of the GNU General Public License -// along with this program. If not, see . -// -// time.cc: implementation of time.h interface. - -#include "time.h" - -#include -#include - -#include - -namespace moonfire_nvr { - -namespace { - -class RealClock : public WallClock { - public: - struct timespec Now() const final { - struct timespec now; - CHECK_EQ(0, clock_gettime(CLOCK_REALTIME, &now)) << strerror(errno); - return now; - } - - void Sleep(struct timespec req) final { - struct timespec rem; - while (true) { - int ret = nanosleep(&req, &rem); - if (ret != 0 && errno != EINTR) { - PLOG(FATAL) << "nanosleep"; - } - if (ret == 0) { - return; - } - req = rem; - } - } -}; - -} // namespace - -// Returns the real wall clock, which will never be deleted. -WallClock *GetRealClock() { - static RealClock *real_clock = new RealClock; // never deleted. - return real_clock; -} - -struct timespec SimulatedClock::Now() const { - std::lock_guard l(mu_); - return now_; -} - -void SimulatedClock::Sleep(struct timespec req) { - std::lock_guard l(mu_); - now_.tv_sec += req.tv_sec; - now_.tv_nsec += req.tv_nsec; - if (now_.tv_nsec > kNanos) { - now_.tv_nsec -= kNanos; - now_.tv_sec++; - } -} - -} // namespace moonfire_nvr diff --git a/src/time.h b/src/time.h deleted file mode 100644 index ec04e98..0000000 --- a/src/time.h +++ /dev/null @@ -1,78 +0,0 @@ -// This file is part of Moonfire NVR, a security camera digital video recorder. -// Copyright (C) 2016 Scott Lamb -// -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// In addition, as a special exception, the copyright holders give -// permission to link the code of portions of this program with the -// OpenSSL library under certain conditions as described in each -// individual source file, and distribute linked combinations including -// the two. -// -// You must obey the GNU General Public License in all respects for all -// of the code used other than OpenSSL. If you modify file(s) with this -// exception, you may extend this exception to your version of the -// file(s), but you are not obligated to do so. If you do not wish to do -// so, delete this exception statement from your version. If you delete -// this exception statement from all source files in the program, then -// also delete it here. -// -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. -// -// You should have received a copy of the GNU General Public License -// along with this program. If not, see . -// -// time.h: functions dealing with (wall) time. - -#ifndef MOONFIRE_NVR_TIME_H -#define MOONFIRE_NVR_TIME_H - -#include -#include - -#include - -namespace moonfire_nvr { - -constexpr long kNanos = 1000000000; - -class WallClock { - public: - virtual ~WallClock() {} - virtual struct timespec Now() const = 0; - virtual void Sleep(struct timespec) = 0; -}; - -class SimulatedClock : public WallClock { - public: - SimulatedClock() : now_({0, 0}) {} - struct timespec Now() const final; - void Sleep(struct timespec req) final; - - private: - mutable std::mutex mu_; - struct timespec now_; -}; - -inline struct timespec SecToTimespec(double sec) { - double intpart; - double fractpart = modf(sec, &intpart); - return {static_cast(intpart), static_cast(fractpart * kNanos)}; -} - -inline double TimespecToSec(struct timespec t) { - return t.tv_sec + static_cast(t.tv_nsec) / kNanos; -} - -// Returns the real wall clock, which will never be deleted. -WallClock *GetRealClock(); - -} // namespace moonfire_nvr - -#endif // MOONFIRE_NVR_TIME_H diff --git a/src/uuid.cc b/src/uuid.cc deleted file mode 100644 index 6d07d52..0000000 --- a/src/uuid.cc +++ /dev/null @@ -1,94 +0,0 @@ -// This file is part of Moonfire NVR, a security camera network video recorder. -// Copyright (C) 2016 Lamb -// -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// In addition, as a special exception, the copyright holders give -// permission to link the code of portions of this program with the -// OpenSSL library under certain conditions as described in each -// individual source file, and distribute linked combinations including -// the two. -// -// You must obey the GNU General Public License in all respects for all -// of the code used other than OpenSSL. If you modify file(s) with this -// exception, you may extend this exception to your version of the -// file(s), but you are not obligated to do so. If you do not wish to do -// so, delete this exception statement from your version. If you delete -// this exception statement from all source files in the program, then -// also delete it here. -// -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. -// -// You should have received a copy of the GNU General Public License -// along with this program. If not, see . -// -// uuid.cc: implementation of uuid.h interface. - -#include "uuid.h" - -namespace moonfire_nvr { - -namespace { - -const size_t kTextFormatLength = - sizeof("xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx") - 1; - -} // namespace - -bool Uuid::ParseText(re2::StringPiece input) { - if (input.size() != kTextFormatLength) { - return false; - } - char tmp[kTextFormatLength + 1]; - memcpy(tmp, input.data(), kTextFormatLength); - tmp[kTextFormatLength] = 0; - return uuid_parse(tmp, me_) == 0; -} - -bool Uuid::ParseBinary(re2::StringPiece input) { - if (input.size() != sizeof(uuid_t)) { - return false; - } - memcpy(me_, input.data(), sizeof(uuid_t)); - return true; -} - -std::string Uuid::UnparseText() const { - char tmp[kTextFormatLength + 1]; - uuid_unparse_lower(me_, tmp); - return tmp; -} - -re2::StringPiece Uuid::binary_view() const { - return re2::StringPiece(reinterpret_cast(me_), sizeof(me_)); -} - -bool Uuid::operator==(const Uuid &other) const { - return uuid_compare(me_, other.me_) == 0; -} - -bool Uuid::operator<(const Uuid &other) const { - return uuid_compare(me_, other.me_) < 0; -} - -class RealUuidGenerator : public UuidGenerator { - public: - Uuid Generate() final { - Uuid out; - uuid_generate(out.me_); - return out; - } -}; - -UuidGenerator *GetRealUuidGenerator() { - static RealUuidGenerator *gen = new RealUuidGenerator; // never freed. - return gen; -} - -} // namespace moonfire_nvr diff --git a/src/uuid.h b/src/uuid.h deleted file mode 100644 index 4c3f255..0000000 --- a/src/uuid.h +++ /dev/null @@ -1,81 +0,0 @@ -// This file is part of Moonfire NVR, a security camera network video recorder. -// Copyright (C) 2016 Lamb -// -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// In addition, as a special exception, the copyright holders give -// permission to link the code of portions of this program with the -// OpenSSL library under certain conditions as described in each -// individual source file, and distribute linked combinations including -// the two. -// -// You must obey the GNU General Public License in all respects for all -// of the code used other than OpenSSL. If you modify file(s) with this -// exception, you may extend this exception to your version of the -// file(s), but you are not obligated to do so. If you do not wish to do -// so, delete this exception statement from your version. If you delete -// this exception statement from all source files in the program, then -// also delete it here. -// -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. -// -// You should have received a copy of the GNU General Public License -// along with this program. If not, see . -// -// uuid.h: small wrapper around the C UUID library for generating/parsing -// RFC 4122 UUIDs. - -#ifndef MOONFIRE_NVR_UUID_H -#define MOONFIRE_NVR_UUID_H - -#include -#include - -namespace moonfire_nvr { - -class Uuid { - public: - // Create a null uuid. - Uuid() { uuid_clear(me_); } - - // Parse the text UUID. Returns success. - bool ParseText(re2::StringPiece input); - - // Parse a binary UUID. In practice any 16-byte string is considered valid. - bool ParseBinary(re2::StringPiece input); - - // Return a 36-byte lowercase text representation, such as - // 1b4e28ba-2fa1-11d2-883f-0016d3cca427. - std::string UnparseText() const; - - // Return a reference to the 16-byte binary form. - // Invalidated by any change to the Uuid object. - re2::StringPiece binary_view() const; - - bool operator==(const Uuid &) const; - bool operator<(const Uuid &) const; - - bool is_null() const { return uuid_is_null(me_); } - - private: - friend class RealUuidGenerator; - uuid_t me_; -}; - -class UuidGenerator { - public: - virtual ~UuidGenerator() {} - virtual Uuid Generate() = 0; -}; - -UuidGenerator *GetRealUuidGenerator(); - -} // namespace moonfire_nvr - -#endif // MOONFIRE_NVR_CODING_H diff --git a/src/web.cc b/src/web.cc deleted file mode 100644 index 4d4f843..0000000 --- a/src/web.cc +++ /dev/null @@ -1,505 +0,0 @@ -// This file is part of Moonfire NVR, a security camera network video recorder. -// Copyright (C) 2016 Scott Lamb -// -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// In addition, as a special exception, the copyright holders give -// permission to link the code of portions of this program with the -// OpenSSL library under certain conditions as described in each -// individual source file, and distribute linked combinations including -// the two. -// -// You must obey the GNU General Public License in all respects for all -// of the code used other than OpenSSL. If you modify file(s) with this -// exception, you may extend this exception to your version of the -// file(s), but you are not obligated to do so. If you do not wish to do -// so, delete this exception statement from your version. If you delete -// this exception statement from all source files in the program, then -// also delete it here. -// -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. -// -// You should have received a copy of the GNU General Public License -// along with this program. If not, see . -// -// web.cc: implementation of web.h interface. - -#include "web.h" - -#include -#include -#include -#include - -#include "recording.h" -#include "string.h" - -namespace moonfire_nvr { - -namespace { - -static const char kJsonMimeType[] = "application/json"; - -bool ParseOptionalStartAndEnd(const QueryParameters ¶ms, - int64_t *start_time_90k, int64_t *end_time_90k) { - *start_time_90k = std::numeric_limits::min(); - *end_time_90k = std::numeric_limits::max(); - if (!params.ok() || - (params.Get("start_time_90k") != nullptr && - !Atoi64(params.Get("start_time_90k"), 10, start_time_90k)) || - (params.Get("end_time_90k") != nullptr && - !Atoi64(params.Get("end_time_90k"), 10, end_time_90k))) { - return false; - } - return true; -} - -void ReplyWithJson(evhttp_request *req, const Json::Value &value) { - EvBuffer buf; - buf.Add(Json::FastWriter().write(value)); - evhttp_add_header(evhttp_request_get_output_headers(req), "Content-Type", - kJsonMimeType); - evhttp_send_reply(req, HTTP_OK, "OK", buf.get()); -} - -} // namespace - -void WebInterface::Register(evhttp *http) { - evhttp_set_gencb(http, &WebInterface::DispatchHttpRequest, this); -} - -void WebInterface::DispatchHttpRequest(evhttp_request *req, void *arg) { - static const RE2 kCameraUri("/cameras/([^/]+)/"); - static const RE2 kCameraRecordingsUri("/cameras/([^/]+)/recordings"); - static const RE2 kCameraViewUri("/cameras/([^/]+)/view.mp4"); - - re2::StringPiece accept = - evhttp_find_header(evhttp_request_get_input_headers(req), "Accept"); - bool json = accept == kJsonMimeType; - - auto *this_ = reinterpret_cast(arg); - const evhttp_uri *uri = evhttp_request_get_evhttp_uri(req); - re2::StringPiece path = evhttp_uri_get_path(uri); - re2::StringPiece camera_uuid_str; - Uuid camera_uuid; - if (path == "/" || path == "/cameras/") { - if (json) { - this_->HandleJsonCameraList(req); - } else { - this_->HandleHtmlCameraList(req); - } - } else if (RE2::FullMatch(path, kCameraUri, &camera_uuid_str) && - camera_uuid.ParseText(camera_uuid_str)) { - if (json) { - this_->HandleJsonCameraDetail(req, camera_uuid); - } else { - this_->HandleHtmlCameraDetail(req, camera_uuid); - } - } else if (RE2::FullMatch(path, kCameraRecordingsUri, &camera_uuid_str) && - camera_uuid.ParseText(camera_uuid_str)) { - // The HTML version includes this in the top-level camera view. - // So only support JSON at this URI. - this_->HandleJsonCameraRecordings(req, camera_uuid); - } else if (RE2::FullMatch(path, kCameraViewUri, &camera_uuid_str) && - camera_uuid.ParseText(camera_uuid_str)) { - this_->HandleMp4View(req, camera_uuid); - } else { - evhttp_send_error(req, HTTP_NOTFOUND, "path not understood"); - } -} - -void WebInterface::HandleHtmlCameraList(evhttp_request *req) { - EvBuffer buf; - buf.Add( - "\n" - "\n" - "\n" - "Camera list\n" - "\n" - "\n" - "\n" - "\n" - "\n"); - auto row_cb = [&](const ListCamerasRow &row) { - auto seconds = row.total_duration_90k / kTimeUnitsPerSecond; - std::string min_start_time_90k = - row.min_start_time_90k == -1 ? std::string("n/a") - : PrettyTimestamp(row.min_start_time_90k); - std::string max_end_time_90k = row.max_end_time_90k == -1 - ? std::string("n/a") - : PrettyTimestamp(row.max_end_time_90k); - buf.AddPrintf( - "\n" - "\n" - "\n" - "\n" - "\n" - "\n" - "\n", - row.uuid.UnparseText().c_str(), EscapeHtml(row.short_name).c_str(), - EscapeHtml(row.description).c_str(), - EscapeHtml(HumanizeWithBinaryPrefix(row.total_sample_file_bytes, "B")) - .c_str(), - EscapeHtml(HumanizeWithBinaryPrefix(row.retain_bytes, "B")).c_str(), - 100.f * row.total_sample_file_bytes / row.retain_bytes, - EscapeHtml(row.uuid.UnparseText()).c_str(), - EscapeHtml(min_start_time_90k).c_str(), - EscapeHtml(max_end_time_90k).c_str(), - EscapeHtml(HumanizeDuration(seconds)).c_str()); - return IterationControl::kContinue; - }; - env_->mdb->ListCameras(row_cb); - buf.Add( - "
%s" - "
description%s
space%s / %s (%.1f%%)
uuid%s
oldest recording%s
newest recording%s
total duration%s
\n" - "\n" - "\n"); - evhttp_send_reply(req, HTTP_OK, "OK", buf.get()); -} - -void WebInterface::HandleJsonCameraList(evhttp_request *req) { - Json::Value cameras(Json::arrayValue); - auto row_cb = [&](const ListCamerasRow &row) { - Json::Value camera(Json::objectValue); - camera["uuid"] = row.uuid.UnparseText(); - camera["short_name"] = row.short_name; - camera["description"] = row.description; - camera["retain_bytes"] = static_cast(row.retain_bytes); - camera["total_duration_90k"] = - static_cast(row.total_duration_90k); - camera["total_sample_file_bytes"] = - static_cast(row.total_sample_file_bytes); - if (row.min_start_time_90k != -1) { - camera["min_start_time_90k"] = - static_cast(row.min_start_time_90k); - } - if (row.max_end_time_90k != -1) { - camera["max_end_time_90k"] = - static_cast(row.max_end_time_90k); - } - cameras.append(camera); - return IterationControl::kContinue; - }; - env_->mdb->ListCameras(row_cb); - ReplyWithJson(req, cameras); -} - -bool WebInterface::ListAggregatedCameraRecordings( - Uuid camera_uuid, int64_t start_time_90k, int64_t end_time_90k, - int64_t forced_split_duration_90k, - const std::function &fn, - std::string *error_message) { - ListCameraRecordingsRow aggregated; - auto handle_sql_row = [&](const ListCameraRecordingsRow &row) { - auto new_duration_90k = aggregated.end_time_90k - row.start_time_90k; - if (row.video_sample_entry_sha1 == aggregated.video_sample_entry_sha1 && - row.end_time_90k == aggregated.start_time_90k && - new_duration_90k < forced_split_duration_90k) { - // Append to current .mp4. - aggregated.start_time_90k = row.start_time_90k; - aggregated.video_samples += row.video_samples; - aggregated.sample_file_bytes += row.sample_file_bytes; - } else { - // Start a new .mp4. - if (aggregated.start_time_90k != -1) { fn(aggregated); } - aggregated = row; - } - return IterationControl::kContinue; - }; - if (!env_->mdb->ListCameraRecordings(camera_uuid, start_time_90k, - end_time_90k, handle_sql_row, - error_message)) { - return false; - } - if (aggregated.start_time_90k != -1) { fn(aggregated); } - return true; -} - -void WebInterface::HandleHtmlCameraDetail(evhttp_request *req, - Uuid camera_uuid) { - GetCameraRow camera_row; - if (!env_->mdb->GetCamera(camera_uuid, &camera_row)) { - return evhttp_send_error(req, HTTP_NOTFOUND, "no such camera"); - } - - int64_t start_time_90k; - int64_t end_time_90k; - QueryParameters params(evhttp_request_get_uri(req)); - if (!ParseOptionalStartAndEnd(params, &start_time_90k, &end_time_90k)) { - return evhttp_send_error(req, HTTP_BADREQUEST, "bad query parameters"); - } - - EvBuffer buf; - buf.AddPrintf( - "\n" - "\n" - "\n" - "%s recordings\n" - "\n" - "\n" - "\n" - "\n" - "

%s

\n" - "

%s

\n" - "\n" - "" - "" - "\n", - EscapeHtml(camera_row.short_name).c_str(), - EscapeHtml(camera_row.short_name).c_str(), - EscapeHtml(camera_row.description).c_str()); - - // Rather than listing each 60-second recording, generate a HTML row for - // aggregated .mp4 files of up to kForceSplitDuration90k each, provided - // there is no gap or change in video parameters between recordings. - static const int64_t kForceSplitDuration90k = 60 * 60 * kTimeUnitsPerSecond; - auto finish_html_row = [&](const ListCameraRecordingsRow &aggregated) { - auto seconds = static_cast(aggregated.end_time_90k - - aggregated.start_time_90k) / - kTimeUnitsPerSecond; - buf.AddPrintf( - "" - "\n", - aggregated.start_time_90k, aggregated.end_time_90k, - PrettyTimestamp(aggregated.start_time_90k).c_str(), - PrettyTimestamp(aggregated.end_time_90k).c_str(), - static_cast(aggregated.width), static_cast(aggregated.height), - static_cast(aggregated.video_samples) / seconds, - HumanizeWithBinaryPrefix(aggregated.sample_file_bytes, "B").c_str(), - HumanizeWithDecimalPrefix( - static_cast(aggregated.sample_file_bytes) * 8 / seconds, - "bps") - .c_str()); - }; - std::string error_message; - if (!ListAggregatedCameraRecordings(camera_uuid, start_time_90k, - end_time_90k, kForceSplitDuration90k, - finish_html_row, &error_message)) { - return evhttp_send_error( - req, HTTP_INTERNAL, - StrCat("sqlite query failed: ", EscapeHtml(error_message)).c_str()); - } - buf.Add( - "
startendresolutionfpssizebitrate
%s%s%dx%d%.0f%s%s
\n" - "\n"); - evhttp_send_reply(req, HTTP_OK, "OK", buf.get()); -} - -void WebInterface::HandleJsonCameraDetail(evhttp_request *req, - Uuid camera_uuid) { - GetCameraRow camera_row; - if (!env_->mdb->GetCamera(camera_uuid, &camera_row)) { - return evhttp_send_error(req, HTTP_NOTFOUND, "no such camera"); - } - - Json::Value camera(Json::objectValue); - camera["short_name"] = camera_row.short_name; - camera["description"] = camera_row.description; - camera["retain_bytes"] = static_cast(camera_row.retain_bytes); - camera["total_duration_90k"] = - static_cast(camera_row.total_duration_90k); - camera["total_sample_file_bytes"] = - static_cast(camera_row.total_sample_file_bytes); - if (camera_row.min_start_time_90k != std::numeric_limits::max()) { - camera["min_start_time_90k"] = - static_cast(camera_row.min_start_time_90k); - } - if (camera_row.max_end_time_90k != std::numeric_limits::min()) { - camera["max_end_time_90k"] = - static_cast(camera_row.max_end_time_90k); - } - - Json::Value days(Json::objectValue); - std::string error_message; - for (const auto &day : camera_row.days) { - int64_t start_time_90k; - int64_t end_time_90k; - if (!GetDayBounds(day.first, &start_time_90k, &end_time_90k, - &error_message)) { - return evhttp_send_error( - req, HTTP_INTERNAL, - StrCat("internal error: ", EscapeHtml(error_message)).c_str()); - } - - Json::Value day_val(Json::objectValue); - day_val["start_time_90k"] = static_cast(start_time_90k); - day_val["end_time_90k"] = static_cast(end_time_90k); - day_val["total_duration_90k"] = static_cast(day.second); - days[day.first] = day_val; - } - camera["days"] = days; - ReplyWithJson(req, camera); -} - -void WebInterface::HandleJsonCameraRecordings(evhttp_request *req, - Uuid camera_uuid) { - int64_t start_time_90k; - int64_t end_time_90k; - QueryParameters params(evhttp_request_get_uri(req)); - if (!ParseOptionalStartAndEnd(params, &start_time_90k, &end_time_90k)) { - return evhttp_send_error(req, HTTP_BADREQUEST, "bad query parameters"); - } - - GetCameraRow camera_row; - if (!env_->mdb->GetCamera(camera_uuid, &camera_row)) { - return evhttp_send_error(req, HTTP_NOTFOUND, "no such camera"); - } - - // TODO(slamb): paging support. - - Json::Value recordings(Json::arrayValue); - auto handle_row = [&](const ListCameraRecordingsRow &row) { - Json::Value recording(Json::objectValue); - recording["end_time_90k"] = static_cast(row.end_time_90k); - recording["start_time_90k"] = static_cast(row.start_time_90k); - recording["video_samples"] = static_cast(row.video_samples); - recording["sample_file_bytes"] = - static_cast(row.sample_file_bytes); - recording["video_sample_entry_sha1"] = ToHex(row.video_sample_entry_sha1); - recording["video_sample_entry_width"] = row.width; - recording["video_sample_entry_height"] = row.height; - recordings.append(recording); - return IterationControl::kContinue; - }; - std::string error_message; - const auto kForceSplitDuration90k = std::numeric_limits::max(); - if (!ListAggregatedCameraRecordings(camera_uuid, start_time_90k, - end_time_90k, kForceSplitDuration90k, - handle_row, &error_message)) { - return evhttp_send_error( - req, HTTP_INTERNAL, - StrCat("sqlite query failed: ", EscapeHtml(error_message)).c_str()); - } - - Json::Value response(Json::objectValue); - response["recordings"] = recordings; - ReplyWithJson(req, response); -} - -void WebInterface::HandleMp4View(evhttp_request *req, Uuid camera_uuid) { - int64_t start_time_90k; - int64_t end_time_90k; - QueryParameters params(evhttp_request_get_uri(req)); - if (!params.ok() || - !Atoi64(params.Get("start_time_90k"), 10, &start_time_90k) || - !Atoi64(params.Get("end_time_90k"), 10, &end_time_90k) || - start_time_90k < 0 || start_time_90k >= end_time_90k) { - return evhttp_send_error(req, HTTP_BADREQUEST, "bad query parameters"); - } - bool include_ts = re2::StringPiece(params.Get("ts")) == "true"; - - std::string error_message; - auto file = BuildMp4(camera_uuid, start_time_90k, end_time_90k, include_ts, - &error_message); - if (file == nullptr) { - // TODO: more nuanced HTTP status codes. - LOG(WARNING) << "BuildMp4 failed: " << error_message; - return evhttp_send_error(req, HTTP_INTERNAL, - EscapeHtml(error_message).c_str()); - } - - return HttpServe(file, req); -} - -std::shared_ptr WebInterface::BuildMp4( - Uuid camera_uuid, int64_t start_time_90k, int64_t end_time_90k, - bool include_ts, std::string *error_message) { - LOG(INFO) << "Building mp4 for camera: " << camera_uuid.UnparseText() - << ", start_time_90k: " << start_time_90k - << ", end_time_90k: " << end_time_90k; - - Mp4FileBuilder builder(env_->sample_file_dir); - int64_t next_row_start_time_90k = start_time_90k; - int64_t rows = 0; - bool ok = true; - auto row_cb = [&](Recording &recording, - const VideoSampleEntry &sample_entry) { - if (rows == 0 && recording.start_time_90k > next_row_start_time_90k) { - *error_message = StrCat( - "recording starts late: ", PrettyTimestamp(recording.start_time_90k), - " (", recording.start_time_90k, ") rather than requested: ", - PrettyTimestamp(start_time_90k), " (", start_time_90k, ")"); - ok = false; - return IterationControl::kBreak; - } else if (rows > 0 && - recording.start_time_90k != next_row_start_time_90k) { - *error_message = StrCat("gap/overlap in recording: ", - PrettyTimestamp(next_row_start_time_90k), " (", - next_row_start_time_90k, ") to: ", - PrettyTimestamp(recording.start_time_90k), " (", - recording.start_time_90k, ") before row ", rows); - ok = false; - return IterationControl::kBreak; - } - - next_row_start_time_90k = recording.end_time_90k; - - if (rows > 0 && recording.video_sample_entry_id != sample_entry.id) { - *error_message = - StrCat("inconsistent video sample entries: this recording has id ", - recording.video_sample_entry_id, " previous had ", - sample_entry.id, " (sha1 ", ToHex(sample_entry.sha1), ")"); - ok = false; - return IterationControl::kBreak; - } else if (rows == 0) { - builder.SetSampleEntry(sample_entry); - } - - int32_t rel_start_90k = 0; - int32_t rel_end_90k = std::numeric_limits::max(); - if (recording.start_time_90k < start_time_90k) { - rel_start_90k = start_time_90k - recording.start_time_90k; - } - if (recording.end_time_90k > end_time_90k) { - rel_end_90k = end_time_90k - recording.start_time_90k; - } - builder.Append(std::move(recording), rel_start_90k, rel_end_90k); - ++rows; - return IterationControl::kContinue; - }; - if (!env_->mdb->ListMp4Recordings(camera_uuid, start_time_90k, end_time_90k, - row_cb, error_message) || - !ok) { - return false; - } - if (rows == 0) { - *error_message = StrCat("no recordings in range"); - return false; - } - if (next_row_start_time_90k < end_time_90k) { - *error_message = StrCat("recording ends early: ", - PrettyTimestamp(next_row_start_time_90k), " (", - next_row_start_time_90k, "), not requested: ", - PrettyTimestamp(end_time_90k), " (", end_time_90k, - ") after ", rows, " rows"); - return false; - } - - builder.include_timestamp_subtitle_track(include_ts); - - VLOG(1) << "...(3/4) building VirtualFile from " << rows << " recordings."; - auto file = builder.Build(error_message); - if (file == nullptr) { - return false; - } - - VLOG(1) << "...(4/4) success, " << file->size() << " bytes, etag " - << file->etag(); - return file; -} - -} // namespace moonfire_nvr diff --git a/src/web.h b/src/web.h deleted file mode 100644 index b4a500c..0000000 --- a/src/web.h +++ /dev/null @@ -1,92 +0,0 @@ -// This file is part of Moonfire NVR, a security camera network video recorder. -// Copyright (C) 2016 Scott Lamb -// -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// In addition, as a special exception, the copyright holders give -// permission to link the code of portions of this program with the -// OpenSSL library under certain conditions as described in each -// individual source file, and distribute linked combinations including -// the two. -// -// You must obey the GNU General Public License in all respects for all -// of the code used other than OpenSSL. If you modify file(s) with this -// exception, you may extend this exception to your version of the -// file(s), but you are not obligated to do so. If you do not wish to do -// so, delete this exception statement from your version. If you delete -// this exception statement from all source files in the program, then -// also delete it here. -// -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. -// -// You should have received a copy of the GNU General Public License -// along with this program. If not, see . -// -// web.h: web (HTTP/HTML/JSON) interface to the SQLite-based recording schema. -// See design/api.md for a description of the JSON API. -// -// In the future, the interface will be reworked for tighter integration with -// the recording system to support more features: -// -// * including the recording currently being written in the web interface -// * subscribing to changes -// * reconfiguring the recording system, such as -// adding/removing/starting/stopping/editing cameras -// * showing thumbnails of the latest key frame from each camera -// * ... - -#ifndef MOONFIRE_NVR_WEB_H -#define MOONFIRE_NVR_WEB_H - -#include - -#include - -#include "moonfire-db.h" -#include "moonfire-nvr.h" -#include "http.h" - -namespace moonfire_nvr { - -class WebInterface { - public: - explicit WebInterface(Environment *env) : env_(env) {} - WebInterface(const WebInterface &) = delete; - void operator=(const WebInterface &) = delete; - - void Register(evhttp *http); - - private: - static void DispatchHttpRequest(evhttp_request *req, void *arg); - - void HandleHtmlCameraList(evhttp_request *req); - void HandleJsonCameraList(evhttp_request *req); - void HandleHtmlCameraDetail(evhttp_request *req, Uuid camera_uuid); - void HandleJsonCameraDetail(evhttp_request *req, Uuid camera_uuid); - void HandleJsonCameraRecordings(evhttp_request *req, Uuid camera_uuid); - void HandleMp4View(evhttp_request *req, Uuid camera_uuid); - - bool ListAggregatedCameraRecordings( - Uuid camera_uuid, int64_t start_time_90k, int64_t end_time_90k, - int64_t forced_split_duration_90k, - const std::function &fn, - std::string *error_message); - - // TODO: more nuanced error code for HTTP. - std::shared_ptr BuildMp4(Uuid camera_uuid, - int64_t start_time_90k, - int64_t end_time_90k, bool include_ts, - std::string *error_message); - - Environment *const env_; -}; - -} // namespace moonfire_nvr - -#endif // MOONFIRE_NVR_WEB_H diff --git a/src/web.rs b/src/web.rs new file mode 100644 index 0000000..c06684b --- /dev/null +++ b/src/web.rs @@ -0,0 +1,449 @@ +// This file is part of Moonfire NVR, a security camera digital video recorder. +// Copyright (C) 2016 Scott Lamb +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// In addition, as a special exception, the copyright holders give +// permission to link the code of portions of this program with the +// OpenSSL library under certain conditions as described in each +// individual source file, and distribute linked combinations including +// the two. +// +// You must obey the GNU General Public License in all respects for all +// of the code used other than OpenSSL. If you modify file(s) with this +// exception, you may extend this exception to your version of the +// file(s), but you are not obligated to do so. If you do not wish to do +// so, delete this exception statement from your version. If you delete +// this exception statement from all source files in the program, then +// also delete it here. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program. If not, see . + +extern crate hyper; + +use core::borrow::Borrow; +use core::str::FromStr; +use db; +use dir::SampleFileDir; +use error::{Error, Result}; +use hyper::{header,server,status}; +use hyper::uri::RequestUri; +use mime; +use mp4; +use recording; +use resource; +use serde_json; +use serde::ser::Serializer; +use std::collections::BTreeMap; +use std::fmt; +use std::io::Write; +use std::result; +use std::sync::{Arc,MutexGuard}; +use time; +use url::form_urlencoded; +use uuid::Uuid; + +const BINARY_PREFIXES: &'static [&'static str] = &[" ", " Ki", " Mi", " Gi", " Ti", " Pi", " Ei"]; +const DECIMAL_PREFIXES: &'static [&'static str] =&[" ", " k", " M", " G", " T", " P", " E"]; + +lazy_static! { + static ref JSON: mime::Mime = mime!(Application/Json); + static ref HTML: mime::Mime = mime!(Text/Html); +} + +enum Path { + CamerasList, // "/" or "/cameras/" + Camera(Uuid), // "/cameras//" + CameraRecordings(Uuid), // "/cameras//recordings" + CameraViewMp4(Uuid), // "/cameras//view.mp4" + NotFound, +} + +fn get_path_and_query(uri: &RequestUri) -> (&str, &str) { + match *uri { + RequestUri::AbsolutePath(ref both) => match both.find('?') { + Some(split) => (&both[..split], &both[split+1..]), + None => (both, ""), + }, + RequestUri::AbsoluteUri(ref u) => (u.path(), u.query().unwrap_or("")), + _ => ("", ""), + } +} + +fn decode_path(path: &str) -> Path { + if path == "/" { + return Path::CamerasList; + } + if !path.starts_with("/cameras/") { + return Path::NotFound; + } + let path = &path["/cameras/".len()..]; + if path == "" { + return Path::CamerasList; + } + let slash = match path.find('/') { + None => { return Path::NotFound; }, + Some(s) => s, + }; + let (uuid, path) = path.split_at(slash); + + // TODO(slamb): require uuid to be in canonical format. + let uuid = match Uuid::parse_str(uuid) { + Ok(u) => u, + Err(_) => { return Path::NotFound }, + }; + match path { + "/" => Path::Camera(uuid), + "/recordings" => Path::CameraRecordings(uuid), + "/view.mp4" => Path::CameraViewMp4(uuid), + _ => Path::NotFound, + } +} + +fn is_json(req: &server::Request) -> bool { + if let Some(accept) = req.headers.get::() { + return accept.len() == 1 && accept[0].item == *JSON && + accept[0].quality == header::Quality(1000); + } + false +} + +pub struct HtmlEscaped<'a>(&'a str); + +impl<'a> fmt::Display for HtmlEscaped<'a> { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + let mut last_end = 0; + for (start, part) in self.0.match_indices(|c| c == '<' || c == '&') { + try!(f.write_str(&self.0[last_end..start])); + try!(f.write_str(if part == "<" { "<" } else { "&" })); + last_end = start + 1; + } + f.write_str(&self.0[last_end..]) + } +} + +pub struct Humanized(i64); + +impl Humanized { + fn do_fmt(&self, base: f32, prefixes: &[&str], f: &mut fmt::Formatter) -> fmt::Result { + let mut n = self.0 as f32; + let mut i = 0; + loop { + if n < base || i >= prefixes.len() - 1 { + break; + } + n /= base; + i += 1; + } + write!(f, "{:.1}{}", n, prefixes[i]) + } +} + +impl fmt::Display for Humanized { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + self.do_fmt(1000., DECIMAL_PREFIXES, f) + } +} + +impl fmt::Binary for Humanized { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + self.do_fmt(1024., BINARY_PREFIXES, f) + } +} + +pub struct HumanizedTimestamp(Option); + +impl fmt::Display for HumanizedTimestamp { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + match self.0 { + None => f.write_str("n/a"), + Some(t) => { + let tm = time::at(time::Timespec{sec: t.unix_seconds(), nsec: 0}); + write!(f, "{}", + try!(tm.strftime("%a, %d %b %Y %H:%M:%S %Z").or_else(|_| Err(fmt::Error)))) + } + } + } +} + +pub struct Handler { + db: Arc, + dir: Arc, +} + +#[derive(Serialize)] +struct ListCameras<'a> { + // Use a custom serializer which presents the map's values as a sequence. + #[serde(serialize_with = "ListCameras::serialize_cameras")] + cameras: &'a BTreeMap, +} + +impl<'a> ListCameras<'a> { + fn serialize_cameras(cameras: &BTreeMap, + serializer: &mut S) -> result::Result<(), S::Error> + where S: Serializer { + let mut state = try!(serializer.serialize_seq(Some(cameras.len()))); + for c in cameras.values() { + try!(serializer.serialize_seq_elt(&mut state, c)); + } + serializer.serialize_seq_end(state) + } +} + +impl Handler { + pub fn new(db: Arc, dir: Arc) -> Self { + Handler{db: db, dir: dir} + } + + fn not_found(&self, mut res: server::Response) -> Result<()> { + *res.status_mut() = status::StatusCode::NotFound; + try!(res.send(b"not found")); + Ok(()) + } + + fn list_cameras(&self, req: &server::Request, mut res: server::Response) -> Result<()> { + let json = is_json(req); + let buf = { + let db = self.db.lock(); + if json { + try!(serde_json::to_vec(&ListCameras{cameras: db.cameras_by_id()})) + } else { + try!(self.list_cameras_html(db)) + } + }; + res.headers_mut().set(header::ContentType(if json { JSON.clone() } else { HTML.clone() })); + try!(res.send(&buf)); + Ok(()) + } + + fn list_cameras_html(&self, db: MutexGuard) -> Result> { + let mut buf = Vec::new(); + buf.extend_from_slice(b"\ + \n\ + \n\ + \n\ + Camera list\n\ + \n\ + \n\ + \n\ + \n\ + \n"); + for row in db.cameras_by_id().values() { + try!(write!(&mut buf, "\ + \n\ + \n\ + \n\ + \n\ + \n\ + \n\ + \n", + row.uuid, HtmlEscaped(&row.short_name), HtmlEscaped(&row.description), + Humanized(row.sample_file_bytes), Humanized(row.retain_bytes), + 100. * row.sample_file_bytes as f32 / row.retain_bytes as f32, + row.uuid, HumanizedTimestamp(row.range.as_ref().map(|r| r.start)), + HumanizedTimestamp(row.range.as_ref().map(|r| r.end)), + row.duration)); + } + Ok(buf) + } + + fn camera(&self, uuid: Uuid, req: &server::Request, mut res: server::Response) -> Result<()> { + let json = is_json(req); + let buf = { + let db = self.db.lock(); + if json { + let camera = try!(db.get_camera(uuid) + .ok_or_else(|| Error::new("no such camera".to_owned()))); + try!(serde_json::to_vec(&camera)) + } else { + try!(self.camera_html(db, uuid)) + } + }; + res.headers_mut().set(header::ContentType(if json { JSON.clone() } else { HTML.clone() })); + try!(res.send(&buf)); + Ok(()) + } + + fn camera_html(&self, db: MutexGuard, uuid: Uuid) -> Result> { + let camera = try!(db.get_camera(uuid) + .ok_or_else(|| Error::new("no such camera".to_owned()))); + let mut buf = Vec::new(); + try!(write!(&mut buf, "\ + \n\ + \n\ + \n\ + {0} recordings\n\ + \n\ + \n\ + \n\ + \n\ +

{0}

\n\ +

{1}

\n\ +
{}
description{}
space{:b}B / {:b}B ({:.1}%)
uuid{}
oldest recording{}
newest recording{}
total duration{}
\n\ + \ + \ + \n", + HtmlEscaped(&camera.short_name), HtmlEscaped(&camera.description))); + let r = recording::Time(i64::min_value()) .. recording::Time(i64::max_value()); + + // Rather than listing each 60-second recording, generate a HTML row for aggregated .mp4 + // files of up to FORCE_SPLIT_DURATION each, provided there is no gap or change in video + // parameters between recordings. + static FORCE_SPLIT_DURATION: recording::Duration = + recording::Duration(60 * 60 * recording::TIME_UNITS_PER_SEC); + try!(db.list_aggregated_recordings(camera.id, &r, FORCE_SPLIT_DURATION, |row| { + let seconds = (row.range.end.0 - row.range.start.0) / recording::TIME_UNITS_PER_SEC; + try!(write!(&mut buf, "\ + \ + \n", + row.range.start.0, row.range.end.0, + HumanizedTimestamp(Some(row.range.start)), + HumanizedTimestamp(Some(row.range.end)), row.video_sample_entry.width, + row.video_sample_entry.height, + if seconds == 0 { 0. } else { row.video_samples as f32 / seconds as f32 }, + Humanized(row.sample_file_bytes), + Humanized(if seconds == 0 { 0 } else { row.sample_file_bytes * 8 / seconds }))); + Ok(()) + })); + buf.extend_from_slice(b"
startendresolutionfpssizebitrate
{}{}{}x{}{:.0}{:b}B{}bps
\n\n"); + Ok(buf) + } + + fn camera_recordings(&self, _uuid: Uuid, _req: &server::Request, + mut res: server::Response) -> Result<()> { + *res.status_mut() = status::StatusCode::NotImplemented; + try!(res.send(b"not implemented")); + Ok(()) + } + + fn camera_view_mp4(&self, uuid: Uuid, query: &str, req: &server::Request, + res: server::Response) -> Result<()> { + let camera_id = { + let db = self.db.lock(); + let camera = try!(db.get_camera(uuid) + .ok_or_else(|| Error::new("no such camera".to_owned()))); + camera.id + }; + let mut start = None; + let mut end = None; + let mut include_ts = false; + for (key, value) in form_urlencoded::parse(query.as_bytes()) { + let (key, value) = (key.borrow(), value.borrow()); + match key { + "start_time_90k" => start = Some(recording::Time(try!(i64::from_str(value)))), + "end_time_90k" => end = Some(recording::Time(try!(i64::from_str(value)))), + "ts" => { include_ts = value == "true"; }, + _ => {}, + } + }; + let start = try!(start.ok_or_else(|| Error::new("start_time_90k missing".to_owned()))); + let end = try!(end.ok_or_else(|| Error::new("end_time_90k missing".to_owned()))); + let desired_range = start .. end; + let mut builder = mp4::Mp4FileBuilder::new(); + + // There should be roughly ceil((end - start) / desired_recording_duration) recordings + // in the desired timespan if there are no gaps or overlap. Add a couple more to be safe: + // one for misalignment of the requested timespan with the rotate offset, another because + // rotation only happens at key frames. + let ceil_durations = ((end - start).0 + recording::DESIRED_RECORDING_DURATION - 1) / + recording::DESIRED_RECORDING_DURATION; + let est_records = (ceil_durations + 2) as usize; + let mut next_start = start; + builder.reserve(est_records); + { + let db = self.db.lock(); + try!(db.list_recordings(camera_id, &desired_range, |r| { + if builder.len() == 0 && r.start > next_start { + return Err(Error::new(format!("recording started late ({} vs requested {})", + r.start, start))); + } else if builder.len() != 0 && r.start != next_start { + return Err(Error::new(format!("gap/overlap in recording: {} to {} after row {}", + next_start, r.start, builder.len()))); + } + next_start = r.start + recording::Duration(r.duration_90k as i64); + // TODO: check for inconsistent video sample entries. + + let rel_start = if r.start < start { + (start - r.start).0 as i32 + } else { + 0 + }; + let rel_end = if r.start + recording::Duration(r.duration_90k as i64) > end { + (end - r.start).0 as i32 + } else { + r.duration_90k + }; + builder.append(r, rel_start .. rel_end); + Ok(()) + })); + } + if next_start < end { + return Err(Error::new(format!( + "recording ends early: {}, not requested: {} after {} rows.", + next_start, end, builder.len()))) + } + if builder.len() > est_records { + warn!("Estimated {} records for time [{}, {}); actually were {}", + est_records, start, end, builder.len()); + } else { + debug!("Estimated {} records for time [{}, {}); actually were {}", + est_records, start, end, builder.len()); + } + builder.include_timestamp_subtitle_track(include_ts); + let mp4 = try!(builder.build(self.db.clone(), self.dir.clone())); + try!(resource::serve(&mp4, req, res)); + Ok(()) + } +} + +impl server::Handler for Handler { + fn handle(&self, req: server::Request, res: server::Response) { + let (path, query) = get_path_and_query(&req.uri); + let res = match decode_path(path) { + Path::CamerasList => self.list_cameras(&req, res), + Path::Camera(uuid) => self.camera(uuid, &req, res), + Path::CameraRecordings(uuid) => self.camera_recordings(uuid, &req, res), + Path::CameraViewMp4(uuid) => self.camera_view_mp4(uuid, query, &req, res), + Path::NotFound => self.not_found(res), + }; + if let Err(ref e) = res { + warn!("Error handling request: {}", e); + } + } +} + +#[cfg(test)] +mod tests { + use super::{HtmlEscaped, Humanized}; + + #[test] + fn test_humanize() { + assert_eq!("1.0 B", format!("{:b}B", Humanized(1))); + assert_eq!("1.0 EiB", format!("{:b}B", Humanized(1i64 << 60))); + assert_eq!("1.5 EiB", format!("{:b}B", Humanized((1i64 << 60) + (1i64 << 59)))); + assert_eq!("8.0 EiB", format!("{:b}B", Humanized(i64::max_value()))); + assert_eq!("1.0 Mbps", format!("{}bps", Humanized(1_000_000))); + } + + #[test] + fn test_html_escaped() { + assert_eq!("", format!("{}", HtmlEscaped(""))); + assert_eq!("no special chars", format!("{}", HtmlEscaped("no special chars"))); + assert_eq!("a <tag> & text", format!("{}", HtmlEscaped("a & text"))); + } +}