MyFavMirrors/moonfire-nvr
1
0
Fork 0
mirror of https://github.com/scottlamb/moonfire-nvr.git synced 2025-11-09 05:34:56 -05:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'master' into new-schema

Browse Source
This commit is contained in:
Scott Lamb
2021-10-21 12:26:31 -07:00
parent f7aa71d2af 7b0099fb4e
commit 4a7f22723c
55 changed files with 1345 additions and 701 deletions
Download Patch File Download Diff File Expand all files Collapse all files

281
server/Cargo.lock generated
View File

@@ -30,9 +30,9 @@ dependencies = [
[[package]]
name = "ahash"
version = "0.7.4"
version = "0.7.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "43bb833f0bf979d8475d38fbf09ed3b8a55e1885fe93ad3f93239fc6a4f17b98"
checksum = "fcb51a0695d8f838b1ee009b3fbf66bda078cd64590202a864a8f3e8c4315c47"
dependencies = [
"getrandom 0.2.3",
"once_cell",
@@ -110,9 +110,9 @@ checksum = "cdb031dd78e28731d87d56cc8ffef4a8f36ca26c38fe2de700543e627f8a464a"
[[package]]
name = "backtrace"
version = "0.3.61"
version = "0.3.62"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e7a905d892734eea339e896738c14b9afce22b5318f64b951e70bf3844419b01"
checksum = "091bcdf2da9950f96aa522681ce805e6857f6ca8df73833d35736ab2dc78e152"
dependencies = [
"addr2line",
"cc",
@@ -131,9 +131,9 @@ checksum = "904dfeac50f3cdaba28fc6f57fdcddb75f49ed61346676a78c4ffe55877802fd"
[[package]]
name = "base64ct"
version = "1.0.1"
version = "1.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8a32fd6af2b5827bce66c29053ba0e7c42b9dcab01835835058558c10851a46b"
checksum = "e6b4d9b1225d28d360ec6a231d65af1fd99a2a095154c8040689617290569c5c"
[[package]]
name = "bcrypt"
@@ -148,9 +148,9 @@ dependencies = [
[[package]]
name = "bitflags"
version = "1.2.1"
version = "1.3.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cf1de2fe8c75bc145a2f577add951f8134889b4795d47466a54a5c846d691693"
checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a"
[[package]]
name = "bitreader"
@@ -230,9 +230,9 @@ dependencies = [
[[package]]
name = "bstr"
version = "0.2.16"
version = "0.2.17"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "90682c8d613ad3373e66de8c6411e0ae2ab2571e879d2efbf73558cc66f21279"
checksum = "ba3569f383e8f1598449f1a423e72e99569137b47740b1da11ef19af3d5c3223"
dependencies = [
"lazy_static",
"memchr",
@@ -242,9 +242,9 @@ dependencies = [
[[package]]
name = "bumpalo"
version = "3.7.0"
version = "3.8.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9c59e7af012c713f529e7a3ee57ce9b31ddd858d4b512923602f74608b009631"
checksum = "8f1e260c3a9040a7c19a12468758f4c16f31a81a1fe087482be9570ec864bb6c"
[[package]]
name = "byteorder"
@@ -260,9 +260,9 @@ checksum = "c4872d67bab6358e59559027aa3b9157c53d9358c51423c17554809a8858e0f8"
[[package]]
name = "cc"
version = "1.0.70"
version = "1.0.71"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d26a6ce4b6a484fa3edb70f7efa6fc430fd2b87285fe8b84304fd0936faa0dc0"
checksum = "79c2681d6594606957bbb8631c4b90a7fcaaa72cdb714743a437b156d6a7eedd"
[[package]]
name = "cfg-if"
@@ -377,9 +377,9 @@ dependencies = [
[[package]]
name = "cstr"
version = "0.2.8"
version = "0.2.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c11a39d776a3b35896711da8a04dc1835169dcd36f710878187637314e47941b"
checksum = "f2846d3636dcaff720d311ea8983f5fa7a8288632b2f95145dd4b5819c397fd8"
dependencies = [
"proc-macro2",
"quote",
@@ -501,19 +501,6 @@ dependencies = [
"generic-array",
]
[[package]]
name = "digest_auth"
version = "0.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fa30657988b2ced88f68fe490889e739bf98d342916c33ed3100af1d6f1cbc9c"
dependencies = [
"digest",
"hex",
"md-5",
"rand",
"sha2",
]
[[package]]
name = "dirs"
version = "1.0.5"
@@ -545,9 +532,9 @@ checksum = "a357d28ed41a50f9c765dbfe56cbc04a64e53e5fc58ba79fbc34c10ef3df831f"
[[package]]
name = "encoding_rs"
version = "0.8.28"
version = "0.8.29"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "80df024fbc5ac80f87dfef0d9f5209a252f2a497f7f42944cff24d8253cac065"
checksum = "a74ea89a0a1b98f6332de42c95baff457ada66d1cb4030f9ff151b2041a1c746"
dependencies = [
"cfg-if",
]
@@ -628,9 +615,9 @@ checksum = "7360491ce676a36bf9bb3c56c1aa791658183a54d2744120f27285738d90465a"
[[package]]
name = "flate2"
version = "1.0.21"
version = "1.0.22"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "80edafed416a46fb378521624fab1cfa2eb514784fd8921adbe8a8d8321da811"
checksum = "1e6988e897c1c9c485f43b47a529cef42fde0547f9d8d41a7062518f1d8fc53f"
dependencies = [
"cfg-if",
"crc32fast",
@@ -800,9 +787,9 @@ checksum = "f0a01e0497841a3b2db4f8afa483cce65f7e96a3498bd6c541734792aeac8fe7"
[[package]]
name = "h2"
version = "0.3.4"
version = "0.3.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d7f3675cfef6a30c8031cf9e6493ebdc3bb3272a3fea3923c4210d1830e6a472"
checksum = "6c06815895acec637cd6ed6e9662c935b866d20a106f8361892893a7d9234964"
dependencies = [
"bytes",
"fnv",
@@ -835,7 +822,7 @@ version = "0.11.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ab5ef0d4909ef3724cc8cce6ccc8572c5c817592e9285f5464f8e86f8bd3726e"
dependencies = [
"ahash 0.7.4",
"ahash 0.7.6",
]
[[package]]
@@ -883,15 +870,30 @@ dependencies = [
[[package]]
name = "http"
version = "0.2.4"
version = "0.2.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "527e8c9ac747e28542699a951517aa9a6945af506cd1f2e1b53a576c17b6cc11"
checksum = "1323096b05d41827dadeaee54c9981958c0f94e670bc94ed80037d1a7b8b186b"
dependencies = [
"bytes",
"fnv",
"itoa",
]
[[package]]
name = "http-auth"
version = "0.1.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "805afa6c41edf02ff4643e6672810974a83c2d866be06e0c377e1789084f6a7e"
dependencies = [
"base64",
"digest",
"hex",
"md-5",
"memchr",
"rand",
"sha2",
]
[[package]]
name = "http-body"
version = "0.4.3"
@@ -940,9 +942,9 @@ checksum = "6456b8a6c8f33fee7d958fcd1b60d55b11940a79e63ae87013e6d22e26034440"
[[package]]
name = "hyper"
version = "0.14.12"
version = "0.14.13"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "13f67199e765030fa08fe0bd581af683f0d5bc04ea09c2b1102012c5fb90e7fd"
checksum = "15d1cfb9e4f68655fa04c01f59edb405b6074a0f7118ea881e5026e4a1cd8593"
dependencies = [
"bytes",
"futures-channel",
@@ -991,9 +993,9 @@ dependencies = [
[[package]]
name = "instant"
version = "0.1.10"
version = "0.1.12"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bee0328b1209d157ef001c94dd85b4f8f64139adb0eac2659f4b08382b2f474d"
checksum = "7a5bbe824c507c5da5956355e86a746d82e0e1464f65d862cc5e71da70e94b2c"
dependencies = [
"cfg-if",
]
@@ -1021,9 +1023,9 @@ checksum = "b71991ff56294aa922b450139ee08b3bfc70982c6b2c7562771375cf73542dd4"
[[package]]
name = "js-sys"
version = "0.3.53"
version = "0.3.55"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e4bf49d50e2961077d9c99f4b7997d770a1114f087c3c2e0069b36c13fc2979d"
checksum = "7cc9ffccd38c451a86bf13657df244e9c3f37493cce8e5e21e940963777acc84"
dependencies = [
"wasm-bindgen",
]
@@ -1049,9 +1051,9 @@ dependencies = [
[[package]]
name = "libc"
version = "0.2.101"
version = "0.2.104"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3cb00336871be5ed2c8ed44b60ae9959dc5b9f08539422ed43f09e34ecaeba21"
checksum = "7b2f96d100e1cf1929e7719b7edb3b90ab5298072638fccd77be9ce942ecdfce"
[[package]]
name = "libpasta"
@@ -1076,9 +1078,9 @@ dependencies = [
[[package]]
name = "libsqlite3-sys"
version = "0.22.2"
version = "0.23.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "290b64917f8b0cb885d9de0f9959fe1f775d7fa12f1da2db9001c1c8ab60f89d"
checksum = "abd5850c449b40bacb498b2bbdfaff648b1b055630073ba8db499caf2d0ea9f2"
dependencies = [
"cc",
"pkg-config",
@@ -1155,9 +1157,9 @@ checksum = "2a60c7ce501c71e03a9c9c0d35b861413ae925bd979cc7a4e30d060069aaac8d"
[[package]]
name = "minimal-lexical"
version = "0.1.2"
version = "0.1.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6595bb28ed34f43c3fe088e48f6cfb2e033cab45f25a5384d5fdf564fbc8c4b2"
checksum = "9c64630dcdd71f1a64c435f54885086a0de5d6a12d104d69b165fb7d5286d677"
[[package]]
name = "miniz_oxide"
@@ -1171,9 +1173,9 @@ dependencies = [
[[package]]
name = "mio"
version = "0.7.13"
version = "0.7.14"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8c2bdb6314ec10835cd3293dd268473a835c02b7b352e788be788b3c6ca6bb16"
checksum = "8067b404fe97c70829f082dec8bcf4f71225d7eaea1d8645349cb76fa06205cc"
dependencies = [
"libc",
"log",
@@ -1196,6 +1198,7 @@ name = "moonfire-base"
version = "0.0.1"
dependencies = [
"failure",
"futures",
"lazy_static",
"libc",
"log",
@@ -1203,12 +1206,13 @@ dependencies = [
"parking_lot",
"serde",
"serde_json",
"slab",
"time",
]
[[package]]
name = "moonfire-db"
version = "0.6.5"
version = "0.6.7"
dependencies = [
"base64",
"blake3",
@@ -1261,7 +1265,7 @@ dependencies = [
[[package]]
name = "moonfire-nvr"
version = "0.6.5"
version = "0.6.7"
dependencies = [
"base64",
"blake3",
@@ -1349,9 +1353,9 @@ dependencies = [
[[package]]
name = "nix"
version = "0.22.1"
version = "0.23.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e7555d6c7164cc913be1ce7f95cbecdabda61eb2ccd89008524af306fb7f5031"
checksum = "f305c2c2e4c39a82f7bf0bf65fb557f9070ce06781d4f2454295cc34b1c43188"
dependencies = [
"bitflags",
"cc",
@@ -1485,9 +1489,9 @@ dependencies = [
[[package]]
name = "object"
version = "0.26.2"
version = "0.27.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "39f37e50073ccad23b6d09bcb5b263f4e76d3bb6038e4a3c08e52162ffa8abc2"
checksum = "c821014c18301591b89b843809ef953af9e3df0496c232d5c0611b0a52aac363"
dependencies = [
"memchr",
]
@@ -1609,15 +1613,15 @@ checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184"
[[package]]
name = "pkg-config"
version = "0.3.19"
version = "0.3.20"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3831453b3449ceb48b6d9c7ad7c96d5ea673e9b470a1dc578c2ce6521230884c"
checksum = "7c9b1041b4387893b91ee6746cddfc28516aff326a3519fb2adf820932c5e6cb"
[[package]]
name = "ppv-lite86"
version = "0.2.10"
version = "0.2.14"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ac74c624d6b2d21f425f752262f42188365d7b8ff1aff74c82e45136510a4857"
checksum = "c3ca011bd0129ff4ae15cd04c4eef202cadf6c51c21e47aba319b4e0501db741"
[[package]]
name = "pretty-hex"
@@ -1687,9 +1691,9 @@ checksum = "bc881b2c22681370c6a780e47af9840ef841837bc98118431d4e1868bd0c1086"
[[package]]
name = "proc-macro2"
version = "1.0.29"
version = "1.0.30"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b9f5105d4fdaab20335ca9565e106a5d9b82b6219b5ba735731124ac6711d23d"
checksum = "edc3358ebc67bc8b7fa0c007f945b0b18226f78437d61bec735a9eb96b61ee70"
dependencies = [
"unicode-xid",
]
@@ -1718,9 +1722,9 @@ dependencies = [
[[package]]
name = "quote"
version = "1.0.9"
version = "1.0.10"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c3d0b9745dc2debf507c8422de05d7226cc1f0644216dfdfead988f9b1ab32a7"
checksum = "38bc8cc6a5f2e3655e0899c1b848643b2562f853f114bfec7be120678e3ace05"
dependencies = [
"proc-macro2",
]
@@ -1838,9 +1842,9 @@ dependencies = [
[[package]]
name = "reqwest"
version = "0.11.4"
version = "0.11.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "246e9f61b9bb77df069a947682be06e31ac43ea37862e244a69f177694ea6d22"
checksum = "66d2927ca2f685faf0fc620ac4834690d29e7abb153add10f5812eef20b5e280"
dependencies = [
"base64",
"bytes",
@@ -1870,17 +1874,17 @@ dependencies = [
[[package]]
name = "retina"
version = "0.3.0"
version = "0.3.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "446070f3caf291e982d240c7f921837f6da0cbffbc26f1d785b0a8d214f2cadd"
checksum = "99806071cb433bda0abe688798940f64ea8412482c4049e27b924e4d78675c9a"
dependencies = [
"base64",
"bitreader",
"bytes",
"digest_auth",
"futures",
"h264-reader",
"hex",
"http-auth",
"log",
"once_cell",
"pin-project",
@@ -1888,7 +1892,7 @@ dependencies = [
"rand",
"rtp-rs",
"rtsp-types",
"sdp",
"sdp-types",
"smallvec",
"thiserror",
"time",
@@ -1953,9 +1957,9 @@ dependencies = [
[[package]]
name = "rusqlite"
version = "0.25.3"
version = "0.26.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "57adcf67c8faaf96f3248c2a7b419a0dbc52ebe36ba83dd57fe83827c1ea4eb3"
checksum = "8a82b0b91fad72160c56bf8da7a549b25d7c31109f52cc1437eac4c0ad2550a7"
dependencies = [
"bitflags",
"fallible-iterator",
@@ -2026,14 +2030,13 @@ dependencies = [
]
[[package]]
name = "sdp"
version = "0.1.5"
name = "sdp-types"
version = "0.1.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "db73ce8329b973830407fb1ba0e51bc32716392281f7757a92f372a1420bb8ec"
checksum = "ae499f6886cff026ebd8355c8f67a1881cd15f23ce89de4aab13588cf52142dd"
dependencies = [
"rand",
"thiserror",
"url",
"bstr",
"fallible-iterator",
]
[[package]]
@@ -2067,9 +2070,9 @@ dependencies = [
[[package]]
name = "serde_json"
version = "1.0.67"
version = "1.0.68"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a7f9e390c27c3c0ce8bc5d725f6e4d30a29d26659494aa4b17535f7522c5c950"
checksum = "0f690853975602e1bfe1ccbf50504d67174e3bcf340f23b5ea9992e0587a52d8"
dependencies = [
"indexmap",
"itoa",
@@ -2107,9 +2110,9 @@ dependencies = [
[[package]]
name = "serde_yaml"
version = "0.8.20"
version = "0.8.21"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ad104641f3c958dab30eb3010e834c2622d1f3f4c530fef1dee20ad9485f3c09"
checksum = "d8c608a35705a5d3cdc9fbe403147647ff34b921f8e833e49306df898f9b20af"
dependencies = [
"dtoa",
"indexmap",
@@ -2132,9 +2135,9 @@ dependencies = [
[[package]]
name = "sha2"
version = "0.9.6"
version = "0.9.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9204c41a1597a8c5af23c82d1c921cb01ec0a4c59e07a9c7306062829a3903f3"
checksum = "b69f9a4c9740d74c5baa3fd2e547f9525fa8088a8a958e0ca2409a514e33f5fa"
dependencies = [
"block-buffer",
"cfg-if",
@@ -2164,21 +2167,21 @@ dependencies = [
[[package]]
name = "slab"
version = "0.4.4"
version = "0.4.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c307a32c1c5c437f38c7fd45d753050587732ba8628319fbdf12a7e289ccc590"
checksum = "9def91fd1e018fe007022791f865d0ccc9b3a0d5001e01aabb8b40e46000afb5"
[[package]]
name = "smallvec"
version = "1.6.1"
version = "1.7.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fe0f37c9e8f3c5a4a66ad655a93c74daac4ad00c441533bf5c6e7990bb42604e"
checksum = "1ecab6c735a6bb4139c0caafd0cc3635748bbb3acf4550e8138122099251f309"
[[package]]
name = "socket2"
version = "0.4.1"
version = "0.4.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "765f090f0e423d2b55843402a07915add955e7d60657db13707a159727326cad"
checksum = "5dc90fe6c7be1a323296982db1836d1ea9e47b6839496dde9a541bc496df3516"
dependencies = [
"libc",
"winapi",
@@ -2210,9 +2213,9 @@ checksum = "6446ced80d6c486436db5c078dde11a9f73d42b57fb273121e160b84f63d894c"
[[package]]
name = "structopt"
version = "0.3.23"
version = "0.3.25"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bf9d950ef167e25e0bdb073cf1d68e9ad2795ac826f2f3f59647817cf23c0bfa"
checksum = "40b9788f4202aa75c240ecc9c15c65185e6a39ccdeb0fd5d008b98825464c87c"
dependencies = [
"clap",
"lazy_static",
@@ -2221,9 +2224,9 @@ dependencies = [
[[package]]
name = "structopt-derive"
version = "0.4.16"
version = "0.4.18"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "134d838a2c9943ac3125cf6df165eda53493451b719f3255b2a26b85f772d0ba"
checksum = "dcb5ae327f9cc13b68763b5749770cb9e048a99bd9dfdfa58d0cf05d5f64afe0"
dependencies = [
"heck",
"proc-macro-error",
@@ -2240,9 +2243,9 @@ checksum = "6bdef32e8150c2a081110b42772ffe7d7c9032b606bc226c8260fd97e0976601"
[[package]]
name = "syn"
version = "1.0.75"
version = "1.0.80"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b7f58f7e8eaa0009c5fec437aabf511bd9933e4b2d7407bd05273c01a8906ea7"
checksum = "d010a1623fbd906d51d650a9916aaefc05ffa0e4053ff7fe601167f3e715d194"
dependencies = [
"proc-macro2",
"quote",
@@ -2257,9 +2260,9 @@ checksum = "20518fe4a4c9acf048008599e464deb21beeae3d3578418951a189c235a7a9a8"
[[package]]
name = "synstructure"
version = "0.12.5"
version = "0.12.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "474aaa926faa1603c40b7885a9eaea29b444d1cb2850cb7c0e37bb1a4182f4fa"
checksum = "f36bdaa60a83aca3921b5259d5400cbf5e90fc51931376a9bd4a0eb79aa7210f"
dependencies = [
"proc-macro2",
"quote",
@@ -2320,18 +2323,18 @@ dependencies = [
[[package]]
name = "thiserror"
version = "1.0.28"
version = "1.0.30"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "283d5230e63df9608ac7d9691adc1dfb6e701225436eb64d0b9a7f0a5a04f6ec"
checksum = "854babe52e4df1653706b98fcfc05843010039b406875930a70e4d9644e5c417"
dependencies = [
"thiserror-impl",
]
[[package]]
name = "thiserror-impl"
version = "1.0.28"
version = "1.0.30"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fa3884228611f5cd3608e2d409bf7dce832e4eb3135e3f11addbd7e41bd68e71"
checksum = "aa32fd3f627f367fe16f893e2597ae3c05020f8bba2666a4e6ea73d377e5714b"
dependencies = [
"proc-macro2",
"quote",
@@ -2351,9 +2354,9 @@ dependencies = [
[[package]]
name = "tinyvec"
version = "1.3.1"
version = "1.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "848a1e1181b9f6753b5e96a092749e29b11d19ede67dfbbd6c7dc7e0f49b5338"
checksum = "f83b2a3d4d9091d0abd7eba4dc2710b1718583bd4d8992e2190720ea38f391f7"
dependencies = [
"tinyvec_macros",
]
@@ -2366,9 +2369,9 @@ checksum = "cda74da7e1a664f795bb1f8a87ec406fb89a02522cf6e50620d016add6dbbf5c"
[[package]]
name = "tokio"
version = "1.10.1"
version = "1.12.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "92036be488bb6594459f2e03b60e42df6f937fe6ca5c5ffdcb539c6b84dc40f5"
checksum = "c2c2416fdedca8443ae44b4527de1ea633af61d8f7169ffa6e72c5b53d24efcc"
dependencies = [
"autocfg",
"bytes",
@@ -2386,9 +2389,9 @@ dependencies = [
[[package]]
name = "tokio-macros"
version = "1.3.0"
version = "1.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "54473be61f4ebe4efd09cec9bd5d16fa51d70ea0192213d754d2d500457db110"
checksum = "b2dd85aeaba7b68df939bd357c6afb36c87951be9e80bf9c859f2fc3e9fca0fd"
dependencies = [
"proc-macro2",
"quote",
@@ -2421,9 +2424,9 @@ dependencies = [
[[package]]
name = "tokio-util"
version = "0.6.7"
version = "0.6.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1caa0b0c8d94a049db56b5acf8cba99dc0623aab1b26d5b5f5e2d945846b3592"
checksum = "08d3725d3efa29485e87311c5b699de63cde14b00ed4d256b8318aa30ca452cd"
dependencies = [
"bytes",
"futures-core",
@@ -2450,9 +2453,9 @@ checksum = "360dfd1d6d30e05fda32ace2c8c70e9c0a9da713275777f5a4dbb8a1893930c6"
[[package]]
name = "tracing"
version = "0.1.26"
version = "0.1.29"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "09adeb8c97449311ccd28a427f96fb563e7fd31aabf994189879d9da2394b89d"
checksum = "375a639232caf30edfc78e8d89b2d4c375515393e7af7e16f01cd96917fb2105"
dependencies = [
"cfg-if",
"log",
@@ -2463,9 +2466,9 @@ dependencies = [
[[package]]
name = "tracing-attributes"
version = "0.1.15"
version = "0.1.18"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c42e6fa53307c8a17e4ccd4dc81cf5ec38db9209f59b222210375b54ee40d1e2"
checksum = "f4f480b8f81512e825f337ad51e94c1eb5d3bbdf2b363dcd01e2b19a9ffe3f8e"
dependencies = [
"proc-macro2",
"quote",
@@ -2474,9 +2477,9 @@ dependencies = [
[[package]]
name = "tracing-core"
version = "0.1.19"
version = "0.1.21"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2ca517f43f0fb96e0c3072ed5c275fe5eece87e8cb52f4a77b69226d3b1c9df8"
checksum = "1f4ed65637b8390770814083d20756f87bfa2c21bf2f110babdc5438351746e4"
dependencies = [
"lazy_static",
]
@@ -2508,9 +2511,9 @@ dependencies = [
[[package]]
name = "typenum"
version = "1.13.0"
version = "1.14.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "879f6906492a7cd215bfa4cf595b600146ccfac0c79bcbd1f3000162af5e8b06"
checksum = "b63708a265f51345575b27fe43f9500ad611579e764c79edbc2037b1121959ec"
[[package]]
name = "unchecked-index"
@@ -2520,9 +2523,9 @@ checksum = "eeba86d422ce181a719445e51872fa30f1f7413b62becb52e95ec91aa262d85c"
[[package]]
name = "unicode-bidi"
version = "0.3.6"
version = "0.3.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "246f4c42e67e7a4e3c6106ff716a5d067d4132a642840b242e357e468a2a0085"
checksum = "1a01404663e3db436ed2746d9fefef640d868edae3cceb81c3b8d5732fda678f"
[[package]]
name = "unicode-normalization"
@@ -2541,9 +2544,9 @@ checksum = "8895849a949e7845e06bd6dc1aa51731a103c42707010a5b591c0038fb73385b"
[[package]]
name = "unicode-width"
version = "0.1.8"
version = "0.1.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9337591893a19b88d8d87f2cec1e73fad5cdfd10e5a6f349f498ad6ea2ffb1e3"
checksum = "3ed742d4ea2bd1176e236172c8429aaf54486e7ac098db29ffe6529e0ce50973"
[[package]]
name = "unicode-xid"
@@ -2622,21 +2625,19 @@ checksum = "1a143597ca7c7793eff794def352d41792a93c481eb1042423ff7ff72ba2c31f"
[[package]]
name = "wasm-bindgen"
version = "0.2.76"
version = "0.2.78"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8ce9b1b516211d33767048e5d47fa2a381ed8b76fc48d2ce4aa39877f9f183e0"
checksum = "632f73e236b219150ea279196e54e610f5dbafa5d61786303d4da54f84e47fce"
dependencies = [
"cfg-if",
"serde",
"serde_json",
"wasm-bindgen-macro",
]
[[package]]
name = "wasm-bindgen-backend"
version = "0.2.76"
version = "0.2.78"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cfe8dc78e2326ba5f845f4b5bf548401604fa20b1dd1d365fb73b6c1d6364041"
checksum = "a317bf8f9fba2476b4b2c85ef4c4af8ff39c3c7f0cdfeed4f82c34a880aa837b"
dependencies = [
"bumpalo",
"lazy_static",
@@ -2649,9 +2650,9 @@ dependencies = [
[[package]]
name = "wasm-bindgen-futures"
version = "0.4.26"
version = "0.4.28"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "95fded345a6559c2cfee778d562300c581f7d4ff3edb9b0d230d69800d213972"
checksum = "8e8d7523cb1f2a4c96c1317ca690031b714a51cc14e05f712446691f413f5d39"
dependencies = [
"cfg-if",
"js-sys",
@@ -2661,9 +2662,9 @@ dependencies = [
[[package]]
name = "wasm-bindgen-macro"
version = "0.2.76"
version = "0.2.78"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "44468aa53335841d9d6b6c023eaab07c0cd4bddbcfdee3e2bb1e8d2cb8069fef"
checksum = "d56146e7c495528bf6587663bea13a8eb588d39b36b679d83972e1a2dbbdacf9"
dependencies = [
"quote",
"wasm-bindgen-macro-support",
@@ -2671,9 +2672,9 @@ dependencies = [
[[package]]
name = "wasm-bindgen-macro-support"
version = "0.2.76"
version = "0.2.78"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0195807922713af1e67dc66132c7328206ed9766af3858164fb583eedc25fbad"
checksum = "7803e0eea25835f8abdc585cd3021b3deb11543c6fe226dcd30b228857c5c5ab"
dependencies = [
"proc-macro2",
"quote",
@@ -2684,9 +2685,9 @@ dependencies = [
[[package]]
name = "wasm-bindgen-shared"
version = "0.2.76"
version = "0.2.78"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "acdb075a845574a1fa5f09fd77e43f7747599301ea3417a9fbffdeedfc1f4a29"
checksum = "0237232789cf037d5480773fe568aac745bfe2afbc11a863e97901780a6b47cc"
[[package]]
name = "wasmer_enumset"
@@ -2712,9 +2713,9 @@ dependencies = [
[[package]]
name = "web-sys"
version = "0.3.53"
version = "0.3.55"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "224b2f6b67919060055ef1a67807367c2066ed520c3862cc013d26cf893a783c"
checksum = "38eb105f1c59d9eaa6b5cdc92b859d85b926e82cb2e0945cd0c9259faa6fe9fb"
dependencies = [
"js-sys",
"wasm-bindgen",

10
server/Cargo.toml
View File

@@ -1,6 +1,6 @@
[package]
name = "moonfire-nvr"
version = "0.6.5"
version = "0.6.7"
authors = ["Scott Lamb <slamb@slamb.org>"]
edition = "2018"
license-file = "../LICENSE.txt"
@@ -41,21 +41,21 @@ libc = "0.2"
log = { version = "0.4" }
memchr = "2.0.2"
mylog = { git = "https://github.com/scottlamb/mylog" }
nix = "0.22.0"
nix = "0.23.0"
nom = "7.0.0"
parking_lot = { version = "0.11.1", features = [] }
protobuf = { git = "https://github.com/stepancheg/rust-protobuf" }
reffers = "0.6.0"
retina = "0.3.0"
retina = "0.3.3"
ring = "0.16.2"
rusqlite = "0.25.3"
rusqlite = "0.26.1"
serde = { version = "1.0", features = ["derive"] }
serde_json = "1.0"
smallvec = "1.0"
structopt = { version = "0.3.13", default-features = false }
sync_wrapper = "0.1.0"
time = "0.1"
tokio = { version = "1.0", features = ["macros", "parking_lot", "rt-multi-thread", "signal", "time"] }
tokio = { version = "1.0", features = ["macros", "parking_lot", "rt-multi-thread", "signal", "sync", "time"] }
tokio-stream = "0.1.5"
tokio-tungstenite = "0.15.0"
tracing = { version = "0.1", features = ["log"] }

2
server/base/Cargo.toml
View File

@@ -14,6 +14,7 @@ path = "lib.rs"
[dependencies]
failure = "0.1.1"
futures = "0.3"
lazy_static = "1.0"
libc = "0.2"
log = "0.4"
@@ -21,4 +22,5 @@ parking_lot = { version = "0.11.1", features = [] }
nom = "7.0.0"
serde = { version = "1.0", features = ["derive"] }
serde_json = "1.0"
slab = "0.4"
time = "0.1"

23
server/base/clock.rs
View File

@@ -13,12 +13,17 @@ use std::thread;
use std::time::Duration as StdDuration;
use time::{Duration, Timespec};
use crate::shutdown::ShutdownError;
/// Abstract interface to the system clocks. This is for testability.
pub trait Clocks: Send + Sync + 'static {
/// Gets the current time from `CLOCK_REALTIME`.
fn realtime(&self) -> Timespec;
/// Gets the current time from `CLOCK_MONOTONIC`.
/// Gets the current time from a monotonic clock.
///
/// On Linux, this uses `CLOCK_BOOTTIME`, which includes suspended time.
/// On other systems, it uses `CLOCK_MONOTONIC`.
fn monotonic(&self) -> Timespec;
/// Causes the current thread to sleep for the specified time.
@@ -32,16 +37,21 @@ pub trait Clocks: Send + Sync + 'static {
) -> Result<T, mpsc::RecvTimeoutError>;
}
pub fn retry_forever<C, T, E>(clocks: &C, f: &mut dyn FnMut() -> Result<T, E>) -> T
pub fn retry<C, T, E>(
clocks: &C,
shutdown_rx: &crate::shutdown::Receiver,
f: &mut dyn FnMut() -> Result<T, E>,
) -> Result<T, ShutdownError>
where
C: Clocks,
E: Into<Error>,
{
loop {
let e = match f() {
Ok(t) => return t,
Ok(t) => return Ok(t),
Err(e) => e.into(),
};
shutdown_rx.check()?;
let sleep_time = Duration::seconds(1);
warn!(
"sleeping for {} after error: {}",
@@ -70,6 +80,13 @@ impl Clocks for RealClocks {
fn realtime(&self) -> Timespec {
self.get(libc::CLOCK_REALTIME)
}
#[cfg(target_os = "linux")]
fn monotonic(&self) -> Timespec {
self.get(libc::CLOCK_BOOTTIME)
}
#[cfg(not(target_os = "linux"))]
fn monotonic(&self) -> Timespec {
self.get(libc::CLOCK_MONOTONIC)
}

3
server/base/lib.rs
View File

@@ -1,9 +1,10 @@
// This file is part of Moonfire NVR, a security camera network video recorder.
// Copyright (C) 2018 The Moonfire NVR Authors; see AUTHORS and LICENSE.txt.
// Copyright (C) 2021 The Moonfire NVR Authors; see AUTHORS and LICENSE.txt.
// SPDX-License-Identifier: GPL-v3.0-or-later WITH GPL-3.0-linking-exception.
pub mod clock;
mod error;
pub mod shutdown;
pub mod strutil;
pub mod time;

211
server/base/shutdown.rs Normal file
View File

@@ -0,0 +1,211 @@
// This file is part of Moonfire NVR, a security camera network video recorder.
// Copyright (C) 2021 The Moonfire NVR Authors; see AUTHORS and LICENSE.txt.
// SPDX-License-Identifier: GPL-v3.0-or-later WITH GPL-3.0-linking-exception.
//! Tools for propagating a graceful shutdown signal through the program.
//!
//! The receiver can be cloned, checked and used as a future in async code.
//! Also, for convenience, blocked in synchronous code without going through the
//! runtime.
//!
//! Surprisingly, I couldn't find any simple existing mechanism for anything
//! close to this in `futures::channels` or `tokio::sync`.
use std::pin::Pin;
use std::sync::Arc;
use std::task::{Context, Poll, Waker};
use futures::Future;
use parking_lot::{Condvar, Mutex};
use slab::Slab;
#[derive(Debug)]
pub struct ShutdownError;
impl std::fmt::Display for ShutdownError {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.write_str("shutdown requested")
}
}
impl std::error::Error for ShutdownError {}
struct Inner {
/// `None` iff shutdown has already happened.
wakers: Mutex<Option<Slab<Waker>>>,
condvar: Condvar,
}
pub struct Sender(Arc<Inner>);
impl Drop for Sender {
fn drop(&mut self) {
// Note sequencing: modify the lock state, then notify async/sync waiters.
// The opposite order would create a race in which something might never wake.
let mut wakers = self
.0
.wakers
.lock()
.take()
.expect("only the single Sender takes the slab");
for w in wakers.drain() {
w.wake();
}
self.0.condvar.notify_all();
}
}
#[derive(Clone)]
pub struct Receiver(Arc<Inner>);
pub struct ReceiverRefFuture<'receiver> {
receiver: &'receiver Receiver,
waker_i: usize,
}
pub struct ReceiverFuture {
receiver: Arc<Inner>,
waker_i: usize,
}
/// `waker_i` value to indicate no slot has been assigned.
///
/// There can't be `usize::MAX` items in the slab because there are other things
/// in the address space (and because `Waker` uses more than one byte anyway).
const NO_WAKER: usize = usize::MAX;
impl Receiver {
pub fn check(&self) -> Result<(), ShutdownError> {
if self.0.wakers.lock().is_none() {
Err(ShutdownError)
} else {
Ok(())
}
}
pub fn as_future(&self) -> ReceiverRefFuture {
ReceiverRefFuture {
receiver: self,
waker_i: NO_WAKER,
}
}
pub fn future(&self) -> ReceiverFuture {
ReceiverFuture {
receiver: self.0.clone(),
waker_i: NO_WAKER,
}
}
pub fn into_future(self) -> ReceiverFuture {
ReceiverFuture {
receiver: self.0,
waker_i: NO_WAKER,
}
}
pub fn wait_for(&self, timeout: std::time::Duration) -> Result<(), ShutdownError> {
let mut l = self.0.wakers.lock();
if l.is_none() {
return Err(ShutdownError);
}
if self.0.condvar.wait_for(&mut l, timeout).timed_out() {
Ok(())
} else {
// parking_lot guarantees no spurious wakeups.
debug_assert!(l.is_none());
Err(ShutdownError)
}
}
}
fn poll_impl(inner: &Inner, waker_i: &mut usize, cx: &mut Context<'_>) -> Poll<()> {
let mut l = inner.wakers.lock();
let wakers = match &mut *l {
None => return Poll::Ready(()),
Some(w) => w,
};
let new_waker = cx.waker();
if *waker_i == NO_WAKER {
*waker_i = wakers.insert(new_waker.clone());
} else {
let existing_waker = &mut wakers[*waker_i];
if !new_waker.will_wake(existing_waker) {
*existing_waker = new_waker.clone();
}
}
Poll::Pending
}
impl<'receiver> Future for ReceiverRefFuture<'receiver> {
type Output = ();
fn poll(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Self::Output> {
poll_impl(&self.receiver.0, &mut self.waker_i, cx)
}
}
impl Future for ReceiverFuture {
type Output = ();
fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Self::Output> {
let this = Pin::into_inner(self);
poll_impl(&this.receiver, &mut this.waker_i, cx)
}
}
/// Returns a sender and receiver for graceful shutdown.
///
/// Dropping the sender will request shutdown.
///
/// The receiver can be used as a future or just polled when convenient.
pub fn channel() -> (Sender, Receiver) {
let inner = Arc::new(Inner {
wakers: Mutex::new(Some(Slab::new())),
condvar: Condvar::new(),
});
(Sender(inner.clone()), Receiver(inner))
}
#[cfg(test)]
mod tests {
use futures::Future;
use std::task::{Context, Poll};
#[test]
fn simple_check() {
let (tx, rx) = super::channel();
rx.check().unwrap();
drop(tx);
rx.check().unwrap_err();
}
#[test]
fn blocking() {
let (tx, rx) = super::channel();
rx.wait_for(std::time::Duration::from_secs(0)).unwrap();
let h = std::thread::spawn(move || {
rx.wait_for(std::time::Duration::from_secs(1000))
.unwrap_err()
});
// Make it likely that rx has done its initial check and is waiting on the Condvar.
std::thread::sleep(std::time::Duration::from_millis(10));
drop(tx);
h.join().unwrap();
}
#[test]
fn future() {
let (tx, rx) = super::channel();
let waker = futures::task::noop_waker_ref();
let mut cx = Context::from_waker(waker);
let mut f = rx.as_future();
assert_eq!(std::pin::Pin::new(&mut f).poll(&mut cx), Poll::Pending);
drop(tx);
assert_eq!(std::pin::Pin::new(&mut f).poll(&mut cx), Poll::Ready(()));
// TODO: this doesn't actually check that waker is even used.
}
}

6
server/db/Cargo.toml
View File

@@ -1,6 +1,6 @@
[package]
name = "moonfire-db"
version = "0.6.5"
version = "0.6.7"
authors = ["Scott Lamb <slamb@slamb.org>"]
readme = "../README.md"
edition = "2018"
@@ -28,7 +28,7 @@ libc = "0.2"
libpasta = "0.1.2"
log = "0.4"
mylog = { git = "https://github.com/scottlamb/mylog" }
nix = "0.22.0"
nix = "0.23.0"
num-rational = { version = "0.4.0", default-features = false, features = ["std"] }
odds = { version = "0.4.0", features = ["std-vec"] }
parking_lot = { version = "0.11.1", features = [] }
@@ -36,7 +36,7 @@ pretty-hex = "0.2.1"
prettydiff = { git = "https://github.com/scottlamb/prettydiff", branch = "pr-update-deps" }
protobuf = { git = "https://github.com/stepancheg/rust-protobuf" }
ring = "0.16.2"
rusqlite = "0.25.3"
rusqlite = "0.26.1"
serde = { version = "1.0", features = ["derive"] }
serde_json = "1.0"
smallvec = "1.0"

49
server/db/db.rs
View File

@@ -33,6 +33,7 @@ use crate::raw;
use crate::recording;
use crate::schema;
use crate::signal;
use base::bail_t;
use base::clock::{self, Clocks};
use base::strutil::encode_size;
use failure::{bail, format_err, Error, ResultExt};
@@ -214,6 +215,7 @@ pub struct ListAggregatedRecordingsRow {
pub open_id: u32,
pub first_uncommitted: Option<i32>,
pub growing: bool,
pub has_trailing_zero: bool,
}
impl ListAggregatedRecordingsRow {
@@ -237,6 +239,7 @@ impl ListAggregatedRecordingsRow {
None
},
growing,
has_trailing_zero: (row.flags & RecordingFlags::TrailingZero as i32) != 0,
}
}
}
@@ -341,7 +344,7 @@ impl SampleFileDir {
}
/// Returns expected existing metadata when opening this directory.
fn meta(&self, db_uuid: &Uuid) -> schema::DirMeta {
fn expected_meta(&self, db_uuid: &Uuid) -> schema::DirMeta {
let mut meta = schema::DirMeta::default();
meta.db_uuid.extend_from_slice(&db_uuid.as_bytes()[..]);
meta.dir_uuid.extend_from_slice(&self.uuid.as_bytes()[..]);
@@ -1172,20 +1175,20 @@ impl LockedDatabase {
if dir.dir.is_some() {
continue;
}
let mut meta = dir.meta(&self.uuid);
let mut expected_meta = dir.expected_meta(&self.uuid);
if let Some(o) = self.open.as_ref() {
let open = meta.in_progress_open.set_default();
let open = expected_meta.in_progress_open.set_default();
open.id = o.id;
open.uuid.extend_from_slice(&o.uuid.as_bytes()[..]);
}
let d = dir::SampleFileDir::open(&dir.path, &meta)
let d = dir::SampleFileDir::open(&dir.path, &expected_meta)
.map_err(|e| e.context(format!("Failed to open dir {}", dir.path)))?;
if self.open.is_none() {
// read-only mode; it's already fully opened.
dir.dir = Some(d);
} else {
// read-write mode; there are more steps to do.
e.insert((meta, d));
e.insert((expected_meta, d));
}
}
@@ -1211,8 +1214,7 @@ impl LockedDatabase {
for (id, (mut meta, d)) in in_progress.drain() {
let dir = self.sample_file_dirs_by_id.get_mut(&id).unwrap();
meta.last_complete_open.clear();
mem::swap(&mut meta.last_complete_open, &mut meta.in_progress_open);
meta.last_complete_open = meta.in_progress_open.take().into();
d.write_meta(&meta)?;
dir.dir = Some(d);
}
@@ -1247,10 +1249,10 @@ impl LockedDatabase {
&self,
stream_id: i32,
desired_time: Range<recording::Time>,
f: &mut dyn FnMut(ListRecordingsRow) -> Result<(), Error>,
) -> Result<(), Error> {
f: &mut dyn FnMut(ListRecordingsRow) -> Result<(), base::Error>,
) -> Result<(), base::Error> {
let s = match self.streams_by_id.get(&stream_id) {
None => bail!("no such stream {}", stream_id),
None => bail_t!(NotFound, "no such stream {}", stream_id),
Some(s) => s,
};
raw::list_recordings_by_time(&self.conn, stream_id, desired_time.clone(), f)?;
@@ -1280,10 +1282,10 @@ impl LockedDatabase {
&self,
stream_id: i32,
desired_ids: Range<i32>,
f: &mut dyn FnMut(ListRecordingsRow) -> Result<(), Error>,
) -> Result<(), Error> {
f: &mut dyn FnMut(ListRecordingsRow) -> Result<(), base::Error>,
) -> Result<(), base::Error> {
let s = match self.streams_by_id.get(&stream_id) {
None => bail!("no such stream {}", stream_id),
None => bail_t!(NotFound, "no such stream {}", stream_id),
Some(s) => s,
};
if desired_ids.start < s.cum_recordings {
@@ -1321,8 +1323,8 @@ impl LockedDatabase {
stream_id: i32,
desired_time: Range<recording::Time>,
forced_split: recording::Duration,
f: &mut dyn FnMut(&ListAggregatedRecordingsRow) -> Result<(), Error>,
) -> Result<(), Error> {
f: &mut dyn FnMut(&ListAggregatedRecordingsRow) -> Result<(), base::Error>,
) -> Result<(), base::Error> {
// Iterate, maintaining a map from a recording_id to the aggregated row for the latest
// batch of recordings from the run starting at that id. Runs can be split into multiple
// batches for a few reasons:
@@ -1343,6 +1345,7 @@ impl LockedDatabase {
let run_start_id = recording_id - row.run_offset;
let uncommitted = (row.flags & RecordingFlags::Uncommitted as i32) != 0;
let growing = (row.flags & RecordingFlags::Growing as i32) != 0;
let has_trailing_zero = (row.flags & RecordingFlags::TrailingZero as i32) != 0;
use std::collections::btree_map::Entry;
match aggs.entry(run_start_id) {
Entry::Occupied(mut e) => {
@@ -1359,7 +1362,8 @@ impl LockedDatabase {
} else {
// append.
if a.time.end != row.start {
bail!(
bail_t!(
Internal,
"stream {} recording {} ends at {} but {} starts at {}",
stream_id,
a.ids.end - 1,
@@ -1369,7 +1373,8 @@ impl LockedDatabase {
);
}
if a.open_id != row.open_id {
bail!(
bail_t!(
Internal,
"stream {} recording {} has open id {} but {} has {}",
stream_id,
a.ids.end - 1,
@@ -1387,6 +1392,7 @@ impl LockedDatabase {
a.first_uncommitted = a.first_uncommitted.or(Some(recording_id));
}
a.growing = growing;
a.has_trailing_zero = has_trailing_zero;
}
}
Entry::Vacant(e) => {
@@ -1763,14 +1769,13 @@ impl LockedDatabase {
path,
uuid,
dir: Some(dir),
last_complete_open: None,
last_complete_open: Some(*o),
garbage_needs_unlink: FnvHashSet::default(),
garbage_unlinked: Vec::new(),
}),
Entry::Occupied(_) => bail!("duplicate sample file dir id {}", id),
};
d.last_complete_open = Some(*o);
mem::swap(&mut meta.last_complete_open, &mut meta.in_progress_open);
meta.last_complete_open = meta.in_progress_open.take().into();
d.dir.as_ref().unwrap().write_meta(&meta)?;
Ok(id)
}
@@ -1792,7 +1797,7 @@ impl LockedDatabase {
);
}
let dir = match d.get_mut().dir.take() {
None => dir::SampleFileDir::open(&d.get().path, &d.get().meta(&self.uuid))?,
None => dir::SampleFileDir::open(&d.get().path, &d.get().expected_meta(&self.uuid))?,
Some(arc) => match Arc::strong_count(&arc) {
1 => {
d.get_mut().dir = Some(arc); // put it back.
@@ -1807,7 +1812,7 @@ impl LockedDatabase {
&d.get().path
);
}
let mut meta = d.get().meta(&self.uuid);
let mut meta = d.get().expected_meta(&self.uuid);
meta.in_progress_open = meta.last_complete_open.take().into();
dir.write_meta(&meta)?;
if self

55
server/db/dir/mod.rs
View File

@@ -212,8 +212,8 @@ impl SampleFileDir {
///
/// `db_meta.in_progress_open` should be filled if the directory should be opened in read/write
/// mode; absent in read-only mode.
pub fn open(path: &str, db_meta: &schema::DirMeta) -> Result<Arc<SampleFileDir>, Error> {
let read_write = db_meta.in_progress_open.is_some();
pub fn open(path: &str, expected_meta: &schema::DirMeta) -> Result<Arc<SampleFileDir>, Error> {
let read_write = expected_meta.in_progress_open.is_some();
let s = SampleFileDir::open_self(path, false)?;
s.fd.lock(if read_write {
FlockArg::LockExclusiveNonblock
@@ -222,45 +222,50 @@ impl SampleFileDir {
})
.map_err(|e| e.context(format!("unable to lock dir {}", path)))?;
let dir_meta = read_meta(&s.fd).map_err(|e| e.context("unable to read meta file"))?;
if !SampleFileDir::consistent(db_meta, &dir_meta) {
let serialized = db_meta
.write_length_delimited_to_bytes()
.expect("proto3->vec is infallible");
if let Err(e) = SampleFileDir::check_consistent(expected_meta, &dir_meta) {
bail!(
"metadata mismatch.\ndb: {:#?}\ndir: {:#?}\nserialized db: {:#?}",
db_meta,
&dir_meta,
&serialized
"metadata mismatch: {}.\nexpected:\n{:#?}\n\nactual:\n{:#?}",
e,
expected_meta,
&dir_meta
);
}
if db_meta.in_progress_open.is_some() {
s.write_meta(db_meta)?;
if expected_meta.in_progress_open.is_some() {
s.write_meta(expected_meta)?;
}
Ok(s)
}
/// Returns true if the existing directory and database metadata are consistent; the directory
/// Checks that the existing directory and database metadata are consistent; the directory
/// is then openable.
pub(crate) fn consistent(db_meta: &schema::DirMeta, dir_meta: &schema::DirMeta) -> bool {
if dir_meta.db_uuid != db_meta.db_uuid {
return false;
pub(crate) fn check_consistent(
expected_meta: &schema::DirMeta,
actual_meta: &schema::DirMeta,
) -> Result<(), String> {
if actual_meta.db_uuid != expected_meta.db_uuid {
return Err("db uuid mismatch".into());
}
if dir_meta.dir_uuid != db_meta.dir_uuid {
return false;
if actual_meta.dir_uuid != expected_meta.dir_uuid {
return Err("dir uuid mismatch".into());
}
if db_meta.last_complete_open.is_some()
&& (db_meta.last_complete_open != dir_meta.last_complete_open
&& db_meta.last_complete_open != dir_meta.in_progress_open)
if expected_meta.last_complete_open.is_some()
&& (expected_meta.last_complete_open != actual_meta.last_complete_open
&& expected_meta.last_complete_open != actual_meta.in_progress_open)
{
return false;
return Err(format!(
"expected open {:?}; but got {:?} (complete) or {:?} (in progress)",
&expected_meta.last_complete_open,
&actual_meta.last_complete_open,
&actual_meta.in_progress_open,
));
}
if db_meta.last_complete_open.is_none() && dir_meta.last_complete_open.is_some() {
return false;
if expected_meta.last_complete_open.is_none() && actual_meta.last_complete_open.is_some() {
return Err("expected never opened".into());
}
true
Ok(())
}
pub(crate) fn create(

74
server/db/raw.rs
View File

@@ -6,7 +6,8 @@
use crate::db::{self, CompositeId, FromSqlUuid};
use crate::recording;
use failure::{bail, Error, ResultExt};
use base::{ErrorKind, ResultExt as _};
use failure::{bail, Error, ResultExt as _};
use fnv::FnvHashSet;
use rusqlite::{named_params, params};
use std::ops::Range;
@@ -103,14 +104,18 @@ pub(crate) fn list_recordings_by_time(
conn: &rusqlite::Connection,
stream_id: i32,
desired_time: Range<recording::Time>,
f: &mut dyn FnMut(db::ListRecordingsRow) -> Result<(), Error>,
) -> Result<(), Error> {
let mut stmt = conn.prepare_cached(LIST_RECORDINGS_BY_TIME_SQL)?;
let rows = stmt.query(named_params! {
":stream_id": stream_id,
":start_time_90k": desired_time.start.0,
":end_time_90k": desired_time.end.0,
})?;
f: &mut dyn FnMut(db::ListRecordingsRow) -> Result<(), base::Error>,
) -> Result<(), base::Error> {
let mut stmt = conn
.prepare_cached(LIST_RECORDINGS_BY_TIME_SQL)
.err_kind(ErrorKind::Internal)?;
let rows = stmt
.query(named_params! {
":stream_id": stream_id,
":start_time_90k": desired_time.start.0,
":end_time_90k": desired_time.end.0,
})
.err_kind(ErrorKind::Internal)?;
list_recordings_inner(rows, false, f)
}
@@ -119,39 +124,46 @@ pub(crate) fn list_recordings_by_id(
conn: &rusqlite::Connection,
stream_id: i32,
desired_ids: Range<i32>,
f: &mut dyn FnMut(db::ListRecordingsRow) -> Result<(), Error>,
) -> Result<(), Error> {
let mut stmt = conn.prepare_cached(LIST_RECORDINGS_BY_ID_SQL)?;
let rows = stmt.query(named_params! {
":start": CompositeId::new(stream_id, desired_ids.start).0,
":end": CompositeId::new(stream_id, desired_ids.end).0,
})?;
f: &mut dyn FnMut(db::ListRecordingsRow) -> Result<(), base::Error>,
) -> Result<(), base::Error> {
let mut stmt = conn
.prepare_cached(LIST_RECORDINGS_BY_ID_SQL)
.err_kind(ErrorKind::Internal)?;
let rows = stmt
.query(named_params! {
":start": CompositeId::new(stream_id, desired_ids.start).0,
":end": CompositeId::new(stream_id, desired_ids.end).0,
})
.err_kind(ErrorKind::Internal)?;
list_recordings_inner(rows, true, f)
}
fn list_recordings_inner(
mut rows: rusqlite::Rows,
include_prev: bool,
f: &mut dyn FnMut(db::ListRecordingsRow) -> Result<(), Error>,
) -> Result<(), Error> {
while let Some(row) = rows.next()? {
let wall_duration_90k = row.get(4)?;
let media_duration_delta_90k: i32 = row.get(5)?;
f: &mut dyn FnMut(db::ListRecordingsRow) -> Result<(), base::Error>,
) -> Result<(), base::Error> {
while let Some(row) = rows.next().err_kind(ErrorKind::Internal)? {
let wall_duration_90k = row.get(4).err_kind(ErrorKind::Internal)?;
let media_duration_delta_90k: i32 = row.get(5).err_kind(ErrorKind::Internal)?;
f(db::ListRecordingsRow {
id: CompositeId(row.get(0)?),
run_offset: row.get(1)?,
flags: row.get(2)?,
start: recording::Time(row.get(3)?),
id: CompositeId(row.get(0).err_kind(ErrorKind::Internal)?),
run_offset: row.get(1).err_kind(ErrorKind::Internal)?,
flags: row.get(2).err_kind(ErrorKind::Internal)?,
start: recording::Time(row.get(3).err_kind(ErrorKind::Internal)?),
wall_duration_90k,
media_duration_90k: wall_duration_90k + media_duration_delta_90k,
sample_file_bytes: row.get(6)?,
video_samples: row.get(7)?,
video_sync_samples: row.get(8)?,
video_sample_entry_id: row.get(9)?,
open_id: row.get(10)?,
sample_file_bytes: row.get(6).err_kind(ErrorKind::Internal)?,
video_samples: row.get(7).err_kind(ErrorKind::Internal)?,
video_sync_samples: row.get(8).err_kind(ErrorKind::Internal)?,
video_sample_entry_id: row.get(9).err_kind(ErrorKind::Internal)?,
open_id: row.get(10).err_kind(ErrorKind::Internal)?,
prev_media_duration_and_runs: match include_prev {
false => None,
true => Some((recording::Duration(row.get(11)?), row.get(12)?)),
true => Some((
recording::Duration(row.get(11).err_kind(ErrorKind::Internal)?),
row.get(12).err_kind(ErrorKind::Internal)?,
)),
},
})?;
}

7
server/db/testutil.rs
View File

@@ -51,6 +51,8 @@ pub fn init() {
pub struct TestDb<C: Clocks + Clone> {
pub db: Arc<db::Database<C>>,
pub dirs_by_stream_id: Arc<FnvHashMap<i32, Arc<dir::SampleFileDir>>>,
pub shutdown_tx: base::shutdown::Sender,
pub shutdown_rx: base::shutdown::Receiver,
pub syncer_channel: writer::SyncerChannel<::std::fs::File>,
pub syncer_join: thread::JoinHandle<()>,
pub tmpdir: TempDir,
@@ -119,11 +121,14 @@ impl<C: Clocks + Clone> TestDb<C> {
}
let mut dirs_by_stream_id = FnvHashMap::default();
dirs_by_stream_id.insert(TEST_STREAM_ID, dir);
let (shutdown_tx, shutdown_rx) = base::shutdown::channel();
let (syncer_channel, syncer_join) =
writer::start_syncer(db.clone(), sample_file_dir_id).unwrap();
writer::start_syncer(db.clone(), shutdown_rx.clone(), sample_file_dir_id).unwrap();
TestDb {
db,
dirs_by_stream_id: Arc::new(dirs_by_stream_id),
shutdown_tx,
shutdown_rx,
syncer_channel,
syncer_join,
tmpdir,

7
server/db/upgrade/v4_to_v5.rs
View File

@@ -37,11 +37,12 @@ fn maybe_upgrade_meta(dir: &dir::Fd, db_meta: &schema::DirMeta) -> Result<bool,
dir_meta
.merge_from(&mut s)
.map_err(|e| e.context("Unable to parse metadata proto: {}"))?;
if !dir::SampleFileDir::consistent(&db_meta, &dir_meta) {
if let Err(e) = dir::SampleFileDir::check_consistent(&db_meta, &dir_meta) {
bail!(
"Inconsistent db_meta={:?} dir_meta={:?}",
"Inconsistent db_meta={:?} dir_meta={:?}: {}",
&db_meta,
&dir_meta
&dir_meta,
e
);
}
let mut f = crate::fs::openat(

198
server/db/writer.rs
View File

@@ -8,6 +8,7 @@ use crate::db::{self, CompositeId};
use crate::dir;
use crate::recording::{self, MAX_RECORDING_WALL_DURATION};
use base::clock::{self, Clocks};
use base::shutdown::ShutdownError;
use failure::{bail, format_err, Error};
use fnv::FnvHashMap;
use log::{debug, trace, warn};
@@ -95,6 +96,7 @@ struct Syncer<C: Clocks + Clone, D: DirWriter> {
dir: D,
db: Arc<db::Database<C>>,
planned_flushes: std::collections::BinaryHeap<PlannedFlush>,
shutdown_rx: base::shutdown::Receiver,
}
/// A plan to flush at a given instant due to a recently-saved recording's `flush_if_sec` parameter.
@@ -155,13 +157,14 @@ impl Eq for PlannedFlush {}
/// TODO: add a join wrapper which arranges for the on flush hook to be removed automatically.
pub fn start_syncer<C>(
db: Arc<db::Database<C>>,
shutdown_rx: base::shutdown::Receiver,
dir_id: i32,
) -> Result<(SyncerChannel<::std::fs::File>, thread::JoinHandle<()>), Error>
where
C: Clocks + Clone,
{
let db2 = db.clone();
let (mut syncer, path) = Syncer::new(&db.lock(), db2, dir_id)?;
let (mut syncer, path) = Syncer::new(&db.lock(), shutdown_rx, db2, dir_id)?;
syncer.initial_rotation()?;
let (snd, rcv) = mpsc::channel();
db.lock().on_flush(Box::new({
@@ -199,7 +202,8 @@ pub fn lower_retention(
limits: &[NewLimit],
) -> Result<(), Error> {
let db2 = db.clone();
let (mut syncer, _) = Syncer::new(&db.lock(), db2, dir_id)?;
let (_tx, rx) = base::shutdown::channel();
let (mut syncer, _) = Syncer::new(&db.lock(), rx, db2, dir_id)?;
syncer.do_rotation(|db| {
for l in limits {
let (fs_bytes_before, extra);
@@ -305,6 +309,7 @@ fn list_files_to_abandon(
impl<C: Clocks + Clone> Syncer<C, Arc<dir::SampleFileDir>> {
fn new(
l: &db::LockedDatabase,
shutdown_rx: base::shutdown::Receiver,
db: Arc<db::Database<C>>,
dir_id: i32,
) -> Result<(Self, String), Error> {
@@ -346,6 +351,7 @@ impl<C: Clocks + Clone> Syncer<C, Arc<dir::SampleFileDir>> {
Ok((
Syncer {
dir_id,
shutdown_rx,
dir,
db,
planned_flushes: std::collections::BinaryHeap::new(),
@@ -438,8 +444,16 @@ impl<C: Clocks + Clone, D: DirWriter> Syncer<C, D> {
// Have a command; handle it.
match cmd {
SyncerCommand::AsyncSaveRecording(id, wall_dur, f) => self.save(id, wall_dur, f),
SyncerCommand::DatabaseFlushed => self.collect_garbage(),
SyncerCommand::AsyncSaveRecording(id, wall_dur, f) => {
if self.save(id, wall_dur, f).is_err() {
return false;
}
}
SyncerCommand::DatabaseFlushed => {
if self.collect_garbage().is_err() {
return false;
}
}
SyncerCommand::Flush(flush) => {
// The sender is waiting for the supplied writer to be dropped. If there's no
// timeout, do so immediately; otherwise wait for that timeout then drop it.
@@ -453,7 +467,7 @@ impl<C: Clocks + Clone, D: DirWriter> Syncer<C, D> {
}
/// Collects garbage (without forcing a sync). Called from worker thread.
fn collect_garbage(&mut self) {
fn collect_garbage(&mut self) -> Result<(), ShutdownError> {
trace!("Collecting garbage");
let mut garbage: Vec<_> = {
let l = self.db.lock();
@@ -461,11 +475,11 @@ impl<C: Clocks + Clone, D: DirWriter> Syncer<C, D> {
d.garbage_needs_unlink.iter().copied().collect()
};
if garbage.is_empty() {
return;
return Ok(());
}
let c = &self.db.clocks();
for &id in &garbage {
clock::retry_forever(c, &mut || {
clock::retry(c, &self.shutdown_rx, &mut || {
if let Err(e) = self.dir.unlink_file(id) {
if e == nix::Error::ENOENT {
warn!("dir: recording {} already deleted!", id);
@@ -474,25 +488,33 @@ impl<C: Clocks + Clone, D: DirWriter> Syncer<C, D> {
return Err(e);
}
Ok(())
});
})?;
}
clock::retry_forever(c, &mut || self.dir.sync());
clock::retry_forever(c, &mut || {
clock::retry(c, &self.shutdown_rx, &mut || self.dir.sync())?;
clock::retry(c, &self.shutdown_rx, &mut || {
self.db.lock().delete_garbage(self.dir_id, &mut garbage)
});
})?;
Ok(())
}
/// Saves the given recording and prompts rotation. Called from worker thread.
/// Note that this doesn't flush immediately; SQLite transactions are batched to lower SSD
/// wear. On the next flush, the old recordings will actually be marked as garbage in the
/// database, and shortly afterward actually deleted from disk.
fn save(&mut self, id: CompositeId, wall_duration: recording::Duration, f: D::File) {
fn save(
&mut self,
id: CompositeId,
wall_duration: recording::Duration,
f: D::File,
) -> Result<(), ShutdownError> {
trace!("Processing save for {}", id);
let stream_id = id.stream();
// Free up a like number of bytes.
clock::retry_forever(&self.db.clocks(), &mut || f.sync_all());
clock::retry_forever(&self.db.clocks(), &mut || self.dir.sync());
clock::retry(&self.db.clocks(), &self.shutdown_rx, &mut || f.sync_all())?;
clock::retry(&self.db.clocks(), &self.shutdown_rx, &mut || {
self.dir.sync()
})?;
let mut db = self.db.lock();
db.mark_synced(id).unwrap();
delete_recordings(&mut db, stream_id, 0).unwrap();
@@ -519,6 +541,7 @@ impl<C: Clocks + Clone, D: DirWriter> Syncer<C, D> {
recording: id,
senders: Vec::new(),
});
Ok(())
}
/// Flushes the database if necessary to honor `flush_if_sec` for some recording.
@@ -613,8 +636,8 @@ struct InnerWriter<F: FileWriter> {
hasher: blake3::Hasher,
/// The start time of this segment, based solely on examining the local clock after frames in
/// this segment were received. Frames can suffer from various kinds of delay (initial
/// The start time of this recording, based solely on examining the local clock after frames in
/// this recording were received. Frames can suffer from various kinds of delay (initial
/// buffering, encoding, and network transmission), so this time is set to far in the future on
/// construction, given a real value on the first packet, and decreased as less-delayed packets
/// are discovered. See design/time.md for details.
@@ -626,7 +649,8 @@ struct InnerWriter<F: FileWriter> {
/// the writer is closed cleanly (the caller supplies the next pts), or when the writer is
/// closed uncleanly (with a zero duration, which the `.mp4` format allows only at the end).
///
/// Invariant: this should always be `Some` (briefly violated during `write` call only).
/// `unindexed_sample` should always be `Some`, except when a `write` call has aborted on
/// shutdown. In that case, the close will be unable to write the full segment.
unindexed_sample: Option<UnindexedSample>,
}
@@ -671,7 +695,7 @@ impl<'a, C: Clocks + Clone, D: DirWriter> Writer<'a, C, D> {
/// On successful return, `self.state` will be `WriterState::Open(w)` with `w` violating the
/// invariant that `unindexed_sample` is `Some`. The caller (`write`) is responsible for
/// correcting this.
fn open(&mut self) -> Result<(), Error> {
fn open(&mut self, shutdown_rx: &mut base::shutdown::Receiver) -> Result<(), Error> {
let prev = match self.state {
WriterState::Unopened => None,
WriterState::Open(_) => return Ok(()),
@@ -689,7 +713,9 @@ impl<'a, C: Clocks + Clone, D: DirWriter> Writer<'a, C, D> {
..Default::default()
},
)?;
let f = clock::retry_forever(&self.db.clocks(), &mut || self.dir.create_file(id));
let f = clock::retry(&self.db.clocks(), shutdown_rx, &mut || {
self.dir.create_file(id)
})?;
self.state = WriterState::Open(InnerWriter {
f,
@@ -711,16 +737,17 @@ impl<'a, C: Clocks + Clone, D: DirWriter> Writer<'a, C, D> {
})
}
/// Writes a new frame to this segment.
/// Writes a new frame to this recording.
/// `local_time` should be the local clock's time as of when this packet was received.
pub fn write(
&mut self,
shutdown_rx: &mut base::shutdown::Receiver,
pkt: &[u8],
local_time: recording::Time,
pts_90k: i64,
is_key: bool,
) -> Result<(), Error> {
self.open()?;
self.open(shutdown_rx)?;
let w = match self.state {
WriterState::Open(ref mut w) => w,
_ => unreachable!(),
@@ -764,7 +791,18 @@ impl<'a, C: Clocks + Clone, D: DirWriter> Writer<'a, C, D> {
}
let mut remaining = pkt;
while !remaining.is_empty() {
let written = clock::retry_forever(&self.db.clocks(), &mut || w.f.write(remaining));
let written =
match clock::retry(&self.db.clocks(), shutdown_rx, &mut || w.f.write(remaining)) {
Ok(w) => w,
Err(e) => {
// close() will do nothing because unindexed_sample will be None.
log::warn!(
"Abandoning incompletely written recording {} on shutdown",
w.id
);
return Err(e.into());
}
};
remaining = &remaining[written..];
}
w.unindexed_sample = Some(UnindexedSample {
@@ -857,10 +895,12 @@ impl<F: FileWriter> InnerWriter<F> {
stream_id: i32,
reason: Option<String>,
) -> Result<PreviousWriter, Error> {
let unindexed = self
.unindexed_sample
.take()
.expect("should always be an unindexed sample");
let unindexed = self.unindexed_sample.take().ok_or_else(|| {
format_err!(
"Unable to add recording {} to database due to aborted write",
self.id
)
})?;
let (last_sample_duration, flags) = match next_pts {
None => (0, db::RecordingFlags::TrailingZero as i32),
Some(p) => (i32::try_from(p - unindexed.pts_90k)?, 0),
@@ -1059,8 +1099,10 @@ mod tests {
_tmpdir: ::tempfile::TempDir,
dir: MockDir,
channel: super::SyncerChannel<MockFile>,
_shutdown_tx: base::shutdown::Sender,
shutdown_rx: base::shutdown::Receiver,
syncer: super::Syncer<SimulatedClocks, MockDir>,
syncer_rcv: mpsc::Receiver<super::SyncerCommand<MockFile>>,
syncer_rx: mpsc::Receiver<super::SyncerCommand<MockFile>>,
}
fn new_harness(flush_if_sec: u32) -> Harness {
@@ -1081,6 +1123,7 @@ mod tests {
// Start a mock syncer.
let dir = MockDir::new();
let (shutdown_tx, shutdown_rx) = base::shutdown::channel();
let syncer = super::Syncer {
dir_id: *tdb
.db
@@ -1092,10 +1135,11 @@ mod tests {
dir: dir.clone(),
db: tdb.db.clone(),
planned_flushes: std::collections::BinaryHeap::new(),
shutdown_rx: shutdown_rx.clone(),
};
let (syncer_snd, syncer_rcv) = mpsc::channel();
let (syncer_tx, syncer_rx) = mpsc::channel();
tdb.db.lock().on_flush(Box::new({
let snd = syncer_snd.clone();
let snd = syncer_tx.clone();
move || {
if let Err(e) = snd.send(super::SyncerCommand::DatabaseFlushed) {
warn!("Unable to notify syncer for dir {} of flush: {}", dir_id, e);
@@ -1107,9 +1151,11 @@ mod tests {
dir,
db: tdb.db,
_tmpdir: tdb.tmpdir,
channel: super::SyncerChannel(syncer_snd),
channel: super::SyncerChannel(syncer_tx),
_shutdown_tx: shutdown_tx,
shutdown_rx,
syncer,
syncer_rcv,
syncer_rx,
}
}
@@ -1153,19 +1199,26 @@ mod tests {
));
f.expect(MockFileAction::Write(Box::new(|_| Ok(1))));
f.expect(MockFileAction::SyncAll(Box::new(|| Ok(()))));
w.write(b"1", recording::Time(1), 0, true).unwrap();
w.write(&mut h.shutdown_rx, b"1", recording::Time(1), 0, true)
.unwrap();
let e = w
.write(b"2", recording::Time(2), i32::max_value() as i64 + 1, true)
.write(
&mut h.shutdown_rx,
b"2",
recording::Time(2),
i32::max_value() as i64 + 1,
true,
)
.unwrap_err();
assert!(e.to_string().contains("excessive pts jump"));
h.dir.expect(MockDirAction::Sync(Box::new(|| Ok(()))));
drop(w);
assert!(h.syncer.iter(&h.syncer_rcv)); // AsyncSave
assert!(h.syncer.iter(&h.syncer_rx)); // AsyncSave
assert_eq!(h.syncer.planned_flushes.len(), 1);
assert!(h.syncer.iter(&h.syncer_rcv)); // planned flush
assert!(h.syncer.iter(&h.syncer_rx)); // planned flush
assert_eq!(h.syncer.planned_flushes.len(), 0);
assert!(h.syncer.iter(&h.syncer_rcv)); // DatabaseFlushed
assert!(h.syncer.iter(&h.syncer_rx)); // DatabaseFlushed
f.ensure_done();
h.dir.ensure_done();
}
@@ -1215,14 +1268,15 @@ mod tests {
Ok(3)
})));
f.expect(MockFileAction::SyncAll(Box::new(|| Ok(()))));
w.write(b"123", recording::Time(2), 0, true).unwrap();
w.write(&mut h.shutdown_rx, b"123", recording::Time(2), 0, true)
.unwrap();
h.dir.expect(MockDirAction::Sync(Box::new(|| Ok(()))));
w.close(Some(1), None).unwrap();
assert!(h.syncer.iter(&h.syncer_rcv)); // AsyncSave
assert!(h.syncer.iter(&h.syncer_rx)); // AsyncSave
assert_eq!(h.syncer.planned_flushes.len(), 1);
assert!(h.syncer.iter(&h.syncer_rcv)); // planned flush
assert!(h.syncer.iter(&h.syncer_rx)); // planned flush
assert_eq!(h.syncer.planned_flushes.len(), 0);
assert!(h.syncer.iter(&h.syncer_rcv)); // DatabaseFlushed
assert!(h.syncer.iter(&h.syncer_rx)); // DatabaseFlushed
f.ensure_done();
h.dir.ensure_done();
@@ -1240,7 +1294,8 @@ mod tests {
Ok(1)
})));
f.expect(MockFileAction::SyncAll(Box::new(|| Ok(()))));
w.write(b"4", recording::Time(3), 1, true).unwrap();
w.write(&mut h.shutdown_rx, b"4", recording::Time(3), 1, true)
.unwrap();
h.dir.expect(MockDirAction::Sync(Box::new(|| Ok(()))));
h.dir.expect(MockDirAction::Unlink(
CompositeId::new(1, 0),
@@ -1261,15 +1316,15 @@ mod tests {
drop(w);
trace!("expecting AsyncSave");
assert!(h.syncer.iter(&h.syncer_rcv)); // AsyncSave
assert!(h.syncer.iter(&h.syncer_rx)); // AsyncSave
assert_eq!(h.syncer.planned_flushes.len(), 1);
trace!("expecting planned flush");
assert!(h.syncer.iter(&h.syncer_rcv)); // planned flush
assert!(h.syncer.iter(&h.syncer_rx)); // planned flush
assert_eq!(h.syncer.planned_flushes.len(), 0);
trace!("expecting DatabaseFlushed");
assert!(h.syncer.iter(&h.syncer_rcv)); // DatabaseFlushed
assert!(h.syncer.iter(&h.syncer_rx)); // DatabaseFlushed
trace!("expecting DatabaseFlushed again");
assert!(h.syncer.iter(&h.syncer_rcv)); // DatabaseFlushed again
assert!(h.syncer.iter(&h.syncer_rx)); // DatabaseFlushed again
f.ensure_done();
h.dir.ensure_done();
@@ -1286,13 +1341,13 @@ mod tests {
}
assert_eq!(h.syncer.planned_flushes.len(), 0);
assert!(h.syncer.iter(&h.syncer_rcv)); // DatabaseFlushed
assert!(h.syncer.iter(&h.syncer_rx)); // DatabaseFlushed
// The syncer should shut down cleanly.
drop(h.channel);
h.db.lock().clear_on_flush();
assert_eq!(
h.syncer_rcv.try_recv().err(),
h.syncer_rx.try_recv().err(),
Some(std::sync::mpsc::TryRecvError::Disconnected)
);
assert!(h.syncer.planned_flushes.is_empty());
@@ -1350,16 +1405,17 @@ mod tests {
})));
f.expect(MockFileAction::SyncAll(Box::new(|| Err(eio()))));
f.expect(MockFileAction::SyncAll(Box::new(|| Ok(()))));
w.write(b"1234", recording::Time(1), 0, true).unwrap();
w.write(&mut h.shutdown_rx, b"1234", recording::Time(1), 0, true)
.unwrap();
h.dir
.expect(MockDirAction::Sync(Box::new(|| Err(nix::Error::EIO))));
h.dir.expect(MockDirAction::Sync(Box::new(|| Ok(()))));
drop(w);
assert!(h.syncer.iter(&h.syncer_rcv)); // AsyncSave
assert!(h.syncer.iter(&h.syncer_rx)); // AsyncSave
assert_eq!(h.syncer.planned_flushes.len(), 1);
assert!(h.syncer.iter(&h.syncer_rcv)); // planned flush
assert!(h.syncer.iter(&h.syncer_rx)); // planned flush
assert_eq!(h.syncer.planned_flushes.len(), 0);
assert!(h.syncer.iter(&h.syncer_rcv)); // DatabaseFlushed
assert!(h.syncer.iter(&h.syncer_rx)); // DatabaseFlushed
f.ensure_done();
h.dir.ensure_done();
@@ -1374,7 +1430,7 @@ mod tests {
drop(h.channel);
h.db.lock().clear_on_flush();
assert_eq!(
h.syncer_rcv.try_recv().err(),
h.syncer_rx.try_recv().err(),
Some(std::sync::mpsc::TryRecvError::Disconnected)
);
assert!(h.syncer.planned_flushes.is_empty());
@@ -1424,15 +1480,16 @@ mod tests {
Ok(3)
})));
f.expect(MockFileAction::SyncAll(Box::new(|| Ok(()))));
w.write(b"123", recording::Time(2), 0, true).unwrap();
w.write(&mut h.shutdown_rx, b"123", recording::Time(2), 0, true)
.unwrap();
h.dir.expect(MockDirAction::Sync(Box::new(|| Ok(()))));
w.close(Some(1), None).unwrap();
assert!(h.syncer.iter(&h.syncer_rcv)); // AsyncSave
assert!(h.syncer.iter(&h.syncer_rx)); // AsyncSave
assert_eq!(h.syncer.planned_flushes.len(), 1);
assert!(h.syncer.iter(&h.syncer_rcv)); // planned flush
assert!(h.syncer.iter(&h.syncer_rx)); // planned flush
assert_eq!(h.syncer.planned_flushes.len(), 0);
assert!(h.syncer.iter(&h.syncer_rcv)); // DatabaseFlushed
assert!(h.syncer.iter(&h.syncer_rx)); // DatabaseFlushed
f.ensure_done();
h.dir.ensure_done();
@@ -1450,7 +1507,8 @@ mod tests {
Ok(1)
})));
f.expect(MockFileAction::SyncAll(Box::new(|| Ok(()))));
w.write(b"4", recording::Time(3), 1, true).unwrap();
w.write(&mut h.shutdown_rx, b"4", recording::Time(3), 1, true)
.unwrap();
h.dir.expect(MockDirAction::Sync(Box::new(|| Ok(()))));
h.dir.expect(MockDirAction::Unlink(
CompositeId::new(1, 0),
@@ -1482,11 +1540,11 @@ mod tests {
drop(w);
assert!(h.syncer.iter(&h.syncer_rcv)); // AsyncSave
assert!(h.syncer.iter(&h.syncer_rx)); // AsyncSave
assert_eq!(h.syncer.planned_flushes.len(), 1);
assert!(h.syncer.iter(&h.syncer_rcv)); // planned flush
assert!(h.syncer.iter(&h.syncer_rx)); // planned flush
assert_eq!(h.syncer.planned_flushes.len(), 0);
assert!(h.syncer.iter(&h.syncer_rcv)); // DatabaseFlushed
assert!(h.syncer.iter(&h.syncer_rx)); // DatabaseFlushed
f.ensure_done();
h.dir.ensure_done();
@@ -1502,13 +1560,13 @@ mod tests {
assert!(dir.garbage_unlinked.is_empty());
}
assert!(h.syncer.iter(&h.syncer_rcv)); // DatabaseFlushed
assert!(h.syncer.iter(&h.syncer_rx)); // DatabaseFlushed
// The syncer should shut down cleanly.
drop(h.channel);
h.db.lock().clear_on_flush();
assert_eq!(
h.syncer_rcv.try_recv().err(),
h.syncer_rx.try_recv().err(),
Some(std::sync::mpsc::TryRecvError::Disconnected)
);
assert!(h.syncer.planned_flushes.is_empty());
@@ -1555,6 +1613,7 @@ mod tests {
})));
f1.expect(MockFileAction::SyncAll(Box::new(|| Ok(()))));
w.write(
&mut h.shutdown_rx,
b"123",
recording::Time(recording::TIME_UNITS_PER_SEC),
0,
@@ -1564,12 +1623,12 @@ mod tests {
h.dir.expect(MockDirAction::Sync(Box::new(|| Ok(()))));
drop(w);
assert!(h.syncer.iter(&h.syncer_rcv)); // AsyncSave
assert!(h.syncer.iter(&h.syncer_rx)); // AsyncSave
assert_eq!(h.syncer.planned_flushes.len(), 1);
// Flush and let 30 seconds go by.
h.db.lock().flush("forced").unwrap();
assert!(h.syncer.iter(&h.syncer_rcv)); // DatabaseFlushed
assert!(h.syncer.iter(&h.syncer_rx)); // DatabaseFlushed
assert_eq!(h.syncer.planned_flushes.len(), 1);
h.db.clocks().sleep(time::Duration::seconds(30));
@@ -1595,6 +1654,7 @@ mod tests {
})));
f2.expect(MockFileAction::SyncAll(Box::new(|| Ok(()))));
w.write(
&mut h.shutdown_rx,
b"4",
recording::Time(31 * recording::TIME_UNITS_PER_SEC),
1,
@@ -1605,21 +1665,21 @@ mod tests {
drop(w);
assert!(h.syncer.iter(&h.syncer_rcv)); // AsyncSave
assert!(h.syncer.iter(&h.syncer_rx)); // AsyncSave
assert_eq!(h.syncer.planned_flushes.len(), 2);
assert_eq!(h.syncer.planned_flushes.len(), 2);
let db_flush_count_before = h.db.lock().flushes();
assert_eq!(h.db.clocks().monotonic(), time::Timespec::new(31, 0));
assert!(h.syncer.iter(&h.syncer_rcv)); // planned flush (no-op)
assert!(h.syncer.iter(&h.syncer_rx)); // planned flush (no-op)
assert_eq!(h.db.clocks().monotonic(), time::Timespec::new(61, 0));
assert_eq!(h.db.lock().flushes(), db_flush_count_before);
assert_eq!(h.syncer.planned_flushes.len(), 1);
assert!(h.syncer.iter(&h.syncer_rcv)); // planned flush
assert!(h.syncer.iter(&h.syncer_rx)); // planned flush
assert_eq!(h.db.clocks().monotonic(), time::Timespec::new(91, 0));
assert_eq!(h.db.lock().flushes(), db_flush_count_before + 1);
assert_eq!(h.syncer.planned_flushes.len(), 0);
assert!(h.syncer.iter(&h.syncer_rcv)); // DatabaseFlushed
assert!(h.syncer.iter(&h.syncer_rx)); // DatabaseFlushed
f1.ensure_done();
f2.ensure_done();
@@ -1629,7 +1689,7 @@ mod tests {
drop(h.channel);
h.db.lock().clear_on_flush();
assert_eq!(
h.syncer_rcv.try_recv().err(),
h.syncer_rx.try_recv().err(),
Some(std::sync::mpsc::TryRecvError::Disconnected)
);
assert!(h.syncer.planned_flushes.is_empty());

2
server/src/cmds/check.rs
View File

@@ -46,7 +46,7 @@ pub struct Args {
trash_corrupt_rows: bool,
}
pub fn run(args: &Args) -> Result<i32, Error> {
pub fn run(args: Args) -> Result<i32, Error> {
let (_db_dir, mut conn) = super::open_conn(&args.db_dir, super::OpenMode::ReadWrite)?;
check::run(
&mut conn,

1
server/src/cmds/config/cameras.rs
View File

@@ -197,6 +197,7 @@ fn press_test_inner(url: Url, username: String, password: String) -> Result<Stri
username: if pass_creds { Some(username) } else { None },
password: if pass_creds { Some(password) } else { None },
transport: retina::client::Transport::Tcp,
session_group: Default::default(),
},
)?;
Ok(format!(

2
server/src/cmds/config/mod.rs
View File

@@ -31,7 +31,7 @@ pub struct Args {
db_dir: PathBuf,
}
pub fn run(args: &Args) -> Result<i32, Error> {
pub fn run(args: Args) -> Result<i32, Error> {
let (_db_dir, conn) = super::open_conn(&args.db_dir, super::OpenMode::ReadWrite)?;
let clocks = clock::RealClocks {};
let db = Arc::new(db::Database::new(clocks, conn, true)?);

2
server/src/cmds/init.rs
View File

@@ -19,7 +19,7 @@ pub struct Args {
db_dir: PathBuf,
}
pub fn run(args: &Args) -> Result<i32, Error> {
pub fn run(args: Args) -> Result<i32, Error> {
let (_db_dir, mut conn) = super::open_conn(&args.db_dir, super::OpenMode::Create)?;
// Check if the database has already been initialized.

2
server/src/cmds/login.rs
View File

@@ -53,7 +53,7 @@ pub struct Args {
username: String,
}
pub fn run(args: &Args) -> Result<i32, Error> {
pub fn run(args: Args) -> Result<i32, Error> {
let clocks = clock::RealClocks {};
let (_db_dir, conn) = super::open_conn(&args.db_dir, super::OpenMode::ReadWrite)?;
let db = std::sync::Arc::new(db::Database::new(clocks, conn, true).unwrap());

44
server/src/cmds/mod.rs
View File

@@ -28,7 +28,9 @@ enum OpenMode {
/// The returned `dir::Fd` holds the lock and should be kept open as long as the `Connection` is.
fn open_dir(db_dir: &Path, mode: OpenMode) -> Result<dir::Fd, Error> {
let dir = dir::Fd::open(db_dir, mode == OpenMode::Create).map_err(|e| {
e.context(if e == nix::Error::ENOENT {
e.context(if mode == OpenMode::Create {
format!("unable to create db dir {}", db_dir.display())
} else if e == nix::Error::ENOENT {
format!(
"db dir {} not found; try running moonfire-nvr init",
db_dir.display()
@@ -79,3 +81,43 @@ fn open_conn(db_dir: &Path, mode: OpenMode) -> Result<(dir::Fd, rusqlite::Connec
)?;
Ok((dir, conn))
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn open_dir_error_msg() {
let tmpdir = tempfile::Builder::new()
.prefix("moonfire-nvr-test")
.tempdir()
.unwrap();
let mut nonexistent_dir = tmpdir.path().to_path_buf();
nonexistent_dir.push("nonexistent");
let nonexistent_open = open_dir(&nonexistent_dir, OpenMode::ReadOnly).unwrap_err();
assert!(
nonexistent_open
.to_string()
.contains("try running moonfire-nvr init"),
"unexpected error {}",
&nonexistent_open
);
}
#[test]
fn create_dir_error_msg() {
let tmpdir = tempfile::Builder::new()
.prefix("moonfire-nvr-test")
.tempdir()
.unwrap();
let mut nonexistent_dir = tmpdir.path().to_path_buf();
nonexistent_dir.push("nonexistent");
nonexistent_dir.push("db");
let nonexistent_create = open_dir(&nonexistent_dir, OpenMode::Create).unwrap_err();
assert!(
nonexistent_create.to_string().contains("unable to create"),
"unexpected error {}",
&nonexistent_create
);
}
}

115
server/src/cmds/run.rs
View File

@@ -8,11 +8,10 @@ use base::clock;
use db::{dir, writer};
use failure::{bail, Error, ResultExt};
use fnv::FnvHashMap;
use futures::future::FutureExt;
use hyper::service::{make_service_fn, service_fn};
use log::error;
use log::{info, warn};
use std::path::PathBuf;
use std::sync::atomic::{AtomicBool, Ordering};
use std::sync::Arc;
use std::thread;
use structopt::StructOpt;
@@ -150,9 +149,9 @@ fn resolve_zone() -> Result<String, Error> {
}
};
// If `TIMEZONE_PATH` is a file, use its contents as the zone name.
// If `TIMEZONE_PATH` is a file, use its contents as the zone name, trimming whitespace.
match ::std::fs::read_to_string(TIMEZONE_PATH) {
Ok(z) => Ok(z),
Ok(z) => Ok(z.trim().to_owned()),
Err(e) => {
bail!(
"Unable to resolve timezone from TZ env, {}, or {}. Last error: {}",
@@ -170,16 +169,55 @@ struct Syncer {
join: thread::JoinHandle<()>,
}
pub fn run(args: &Args) -> Result<i32, Error> {
pub fn run(args: Args) -> Result<i32, Error> {
let mut builder = tokio::runtime::Builder::new_multi_thread();
builder.enable_all();
if let Some(worker_threads) = args.worker_threads {
builder.worker_threads(worker_threads);
}
builder.build().unwrap().block_on(async_run(args))
let rt = builder.build()?;
let r = rt.block_on(async_run(args));
// tokio normally waits for all spawned tasks to complete, but:
// * in the graceful shutdown path, we wait for specific tasks with logging.
// * in the immediate shutdown path, we don't want to wait.
rt.shutdown_background();
r
}
async fn async_run(args: &Args) -> Result<i32, Error> {
async fn async_run(args: Args) -> Result<i32, Error> {
let (shutdown_tx, shutdown_rx) = base::shutdown::channel();
let mut shutdown_tx = Some(shutdown_tx);
tokio::pin! {
let int = signal(SignalKind::interrupt())?;
let term = signal(SignalKind::terminate())?;
let inner = inner(args, shutdown_rx);
}
tokio::select! {
_ = int.recv() => {
info!("Received SIGINT; shutting down gracefully. \
Send another SIGINT or SIGTERM to shut down immediately.");
shutdown_tx.take();
},
_ = term.recv() => {
info!("Received SIGTERM; shutting down gracefully. \
Send another SIGINT or SIGTERM to shut down immediately.");
shutdown_tx.take();
},
result = &mut inner => return result,
}
tokio::select! {
_ = int.recv() => bail!("immediate shutdown due to second signal (SIGINT)"),
_ = term.recv() => bail!("immediate shutdown due to second singal (SIGTERM)"),
result = &mut inner => result,
}
}
async fn inner(args: Args, shutdown_rx: base::shutdown::Receiver) -> Result<i32, Error> {
let clocks = clock::RealClocks {};
let (_db_dir, conn) = super::open_conn(
&args.db_dir,
@@ -214,8 +252,9 @@ async fn async_run(args: &Args) -> Result<i32, Error> {
})?);
// Start a streamer for each stream.
let shutdown_streamers = Arc::new(AtomicBool::new(false));
let mut streamers = Vec::new();
let mut session_groups_by_camera: FnvHashMap<i32, Arc<retina::client::SessionGroup>> =
FnvHashMap::default();
let syncers = if !args.read_only {
let l = db.lock();
let mut dirs = FnvHashMap::with_capacity_and_hasher(
@@ -227,7 +266,7 @@ async fn async_run(args: &Args) -> Result<i32, Error> {
db: &db,
opener: args.rtsp_library.opener(),
transport: args.rtsp_transport,
shutdown: &shutdown_streamers,
shutdown_rx: &shutdown_rx,
};
// Get the directories that need syncers.
@@ -253,7 +292,7 @@ async fn async_run(args: &Args) -> Result<i32, Error> {
drop(l);
let mut syncers = FnvHashMap::with_capacity_and_hasher(dirs.len(), Default::default());
for (id, dir) in dirs.drain() {
let (channel, join) = writer::start_syncer(db.clone(), id)?;
let (channel, join) = writer::start_syncer(db.clone(), shutdown_rx.clone(), id)?;
syncers.insert(id, Syncer { dir, channel, join });
}
@@ -279,6 +318,10 @@ async fn async_run(args: &Args) -> Result<i32, Error> {
};
let rotate_offset_sec = streamer::ROTATE_INTERVAL_SEC * i as i64 / streams as i64;
let syncer = syncers.get(&sample_file_dir_id).unwrap();
let session_group = session_groups_by_camera
.entry(camera.id)
.or_default()
.clone();
let mut streamer = streamer::Streamer::new(
&env,
syncer.dir.clone(),
@@ -286,6 +329,7 @@ async fn async_run(args: &Args) -> Result<i32, Error> {
*id,
camera,
stream,
session_group,
rotate_offset_sec,
streamer::ROTATE_INTERVAL_SEC,
)?;
@@ -319,39 +363,44 @@ async fn async_run(args: &Args) -> Result<i32, Error> {
.with_context(|_| format!("unable to bind --http-addr={}", &args.http_addr))?
.tcp_nodelay(true)
.serve(make_svc);
let mut int = signal(SignalKind::interrupt())?;
let mut term = signal(SignalKind::terminate())?;
let shutdown = futures::future::select(Box::pin(int.recv()), Box::pin(term.recv()));
let (shutdown_tx, shutdown_rx) = futures::channel::oneshot::channel();
let server = server.with_graceful_shutdown(shutdown_rx.map(|_| ()));
let server = server.with_graceful_shutdown(shutdown_rx.future());
let server_handle = tokio::spawn(server);
info!("Ready to serve HTTP requests");
shutdown.await;
shutdown_tx.send(()).unwrap();
let _ = shutdown_rx.as_future().await;
info!("Shutting down streamers.");
shutdown_streamers.store(true, Ordering::SeqCst);
for streamer in streamers.drain(..) {
streamer.join().unwrap();
}
if let Some(mut ss) = syncers {
// The syncers shut down when all channels to them have been dropped.
// The database maintains one; and `ss` holds one. Drop both.
db.lock().clear_on_flush();
for (_, s) in ss.drain() {
drop(s.channel);
s.join.join().unwrap();
info!("Shutting down streamers and syncers.");
tokio::task::spawn_blocking({
let db = db.clone();
move || {
for streamer in streamers.drain(..) {
streamer.join().unwrap();
}
if let Some(mut ss) = syncers {
// The syncers shut down when all channels to them have been dropped.
// The database maintains one; and `ss` holds one. Drop both.
db.lock().clear_on_flush();
for (_, s) in ss.drain() {
drop(s.channel);
s.join.join().unwrap();
}
}
}
}
})
.await?;
db.lock().clear_watches();
info!("Waiting for HTTP requests to finish.");
server_handle.await??;
info!("Waiting for TEARDOWN requests to complete.");
for g in session_groups_by_camera.values() {
if let Err(e) = g.await_teardown().await {
error!("{}", e);
}
}
info!("Exiting.");
Ok(0)
}

2
server/src/cmds/sql.rs
View File

@@ -37,7 +37,7 @@ pub struct Args {
arg: Vec<OsString>,
}
pub fn run(args: &Args) -> Result<i32, Error> {
pub fn run(args: Args) -> Result<i32, Error> {
let mode = if args.read_only {
OpenMode::ReadOnly
} else {

2
server/src/cmds/ts.rs
View File

@@ -17,7 +17,7 @@ pub struct Args {
timestamps: Vec<String>,
}
pub fn run(args: &Args) -> Result<i32, Error> {
pub fn run(args: Args) -> Result<i32, Error> {
for timestamp in &args.timestamps {
let t = db::recording::Time::parse(timestamp)?;
println!("{} == {}", t, t.0);

2
server/src/cmds/upgrade/mod.rs
View File

@@ -40,7 +40,7 @@ pub struct Args {
no_vacuum: bool,
}
pub fn run(args: &Args) -> Result<i32, Error> {
pub fn run(args: Args) -> Result<i32, Error> {
let (_db_dir, mut conn) = super::open_conn(&args.db_dir, super::OpenMode::ReadWrite)?;
db::upgrade::run(

3
server/src/json.rs
View File

@@ -477,6 +477,9 @@ pub struct Recording {
#[serde(skip_serializing_if = "Not::not")]
pub growing: bool,
#[serde(skip_serializing_if = "Not::not")]
pub has_trailing_zero: bool,
}
#[derive(Debug, Serialize)]

18
server/src/main.rs
View File

@@ -59,16 +59,16 @@ enum Args {
}
impl Args {
fn run(&self) -> Result<i32, failure::Error> {
fn run(self) -> Result<i32, failure::Error> {
match self {
Args::Check(ref a) => cmds::check::run(a),
Args::Config(ref a) => cmds::config::run(a),
Args::Init(ref a) => cmds::init::run(a),
Args::Login(ref a) => cmds::login::run(a),
Args::Run(ref a) => cmds::run::run(a),
Args::Sql(ref a) => cmds::sql::run(a),
Args::Ts(ref a) => cmds::ts::run(a),
Args::Upgrade(ref a) => cmds::upgrade::run(a),
Args::Check(a) => cmds::check::run(a),
Args::Config(a) => cmds::config::run(a),
Args::Init(a) => cmds::init::run(a),
Args::Login(a) => cmds::login::run(a),
Args::Run(a) => cmds::run::run(a),
Args::Sql(a) => cmds::sql::run(a),
Args::Ts(a) => cmds::ts::run(a),
Args::Upgrade(a) => cmds::upgrade::run(a),
}
}
}

30
server/src/mp4.rs
View File

@@ -2277,7 +2277,7 @@ mod tests {
}
}
fn copy_mp4_to_db(db: &TestDb<RealClocks>) {
fn copy_mp4_to_db(db: &mut TestDb<RealClocks>) {
let (extra_data, mut input) = stream::FFMPEG
.open(
"test".to_owned(),
@@ -2322,7 +2322,13 @@ mod tests {
};
frame_time += recording::Duration(i64::from(pkt.duration));
output
.write(pkt.data, frame_time, pkt.pts, pkt.is_key)
.write(
&mut db.shutdown_rx,
pkt.data,
frame_time,
pkt.pts,
pkt.is_key,
)
.unwrap();
end_pts = Some(pkt.pts + i64::from(pkt.duration));
}
@@ -2811,8 +2817,8 @@ mod tests {
#[tokio::test]
async fn test_round_trip() {
testutil::init();
let db = TestDb::new(RealClocks {});
copy_mp4_to_db(&db);
let mut db = TestDb::new(RealClocks {});
copy_mp4_to_db(&mut db);
let mp4 = create_mp4_from_db(&db, 0, 0, false);
traverse(mp4.clone()).await;
let new_filename = write_mp4(&mp4, db.tmpdir.path()).await;
@@ -2840,8 +2846,8 @@ mod tests {
#[tokio::test]
async fn test_round_trip_with_subtitles() {
testutil::init();
let db = TestDb::new(RealClocks {});
copy_mp4_to_db(&db);
let mut db = TestDb::new(RealClocks {});
copy_mp4_to_db(&mut db);
let mp4 = create_mp4_from_db(&db, 0, 0, true);
traverse(mp4.clone()).await;
let new_filename = write_mp4(&mp4, db.tmpdir.path()).await;
@@ -2869,8 +2875,8 @@ mod tests {
#[tokio::test]
async fn test_round_trip_with_edit_list() {
testutil::init();
let db = TestDb::new(RealClocks {});
copy_mp4_to_db(&db);
let mut db = TestDb::new(RealClocks {});
copy_mp4_to_db(&mut db);
let mp4 = create_mp4_from_db(&db, 1, 0, false);
traverse(mp4.clone()).await;
let new_filename = write_mp4(&mp4, db.tmpdir.path()).await;
@@ -2898,8 +2904,8 @@ mod tests {
#[tokio::test]
async fn test_round_trip_with_edit_list_and_subtitles() {
testutil::init();
let db = TestDb::new(RealClocks {});
copy_mp4_to_db(&db);
let mut db = TestDb::new(RealClocks {});
copy_mp4_to_db(&mut db);
let off = 2 * TIME_UNITS_PER_SEC;
let mp4 = create_mp4_from_db(&db, i32::try_from(off).unwrap(), 0, true);
traverse(mp4.clone()).await;
@@ -2928,8 +2934,8 @@ mod tests {
#[tokio::test]
async fn test_round_trip_with_shorten() {
testutil::init();
let db = TestDb::new(RealClocks {});
copy_mp4_to_db(&db);
let mut db = TestDb::new(RealClocks {});
copy_mp4_to_db(&mut db);
let mp4 = create_mp4_from_db(&db, 0, 1, false);
traverse(mp4.clone()).await;
let new_filename = write_mp4(&mp4, db.tmpdir.path()).await;

6
server/src/stream.rs
View File

@@ -15,6 +15,7 @@ use std::convert::TryFrom;
use std::ffi::CString;
use std::pin::Pin;
use std::result::Result;
use std::sync::Arc;
use url::Url;
static START_FFMPEG: parking_lot::Once = parking_lot::Once::new();
@@ -62,6 +63,7 @@ pub enum Source<'a> {
username: Option<String>,
password: Option<String>,
transport: Transport,
session_group: Arc<retina::client::SessionGroup>,
},
}
@@ -73,6 +75,7 @@ pub enum Source {
username: Option<String>,
password: Option<String>,
transport: Transport,
session_group: Arc<retina::client::SessionGroup>,
},
}
@@ -141,6 +144,7 @@ impl Opener for Ffmpeg {
username,
password,
transport,
..
} => {
let mut open_options = ffmpeg::avutil::Dictionary::new();
open_options
@@ -301,6 +305,7 @@ impl Opener for RetinaOpener {
username,
password,
transport,
session_group,
} => (
url,
retina::client::SessionOptions::default()
@@ -313,6 +318,7 @@ impl Opener for RetinaOpener {
_ => bail!("must supply username when supplying password"),
})
.transport(transport)
.session_group(session_group)
.user_agent(format!("Moonfire NVR {}", env!("CARGO_PKG_VERSION"))),
),
};

59
server/src/streamer.rs
View File

@@ -8,7 +8,6 @@ use db::{dir, recording, writer, Camera, Database, Stream};
use failure::{bail, format_err, Error};
use log::{debug, info, trace, warn};
use std::result::Result;
use std::sync::atomic::{AtomicBool, Ordering};
use std::sync::Arc;
use url::Url;
@@ -22,7 +21,7 @@ where
pub opener: &'a dyn stream::Opener,
pub transport: retina::client::Transport,
pub db: &'tmp Arc<Database<C>>,
pub shutdown: &'tmp Arc<AtomicBool>,
pub shutdown_rx: &'tmp base::shutdown::Receiver,
}
/// Connects to a given RTSP stream and writes recordings to the database via [`writer::Writer`].
@@ -31,7 +30,7 @@ pub struct Streamer<'a, C>
where
C: Clocks + Clone,
{
shutdown: Arc<AtomicBool>,
shutdown_rx: base::shutdown::Receiver,
// State below is only used by the thread in Run.
rotate_offset_sec: i64,
@@ -42,6 +41,7 @@ where
opener: &'a dyn stream::Opener,
transport: retina::client::Transport,
stream_id: i32,
session_group: Arc<retina::client::SessionGroup>,
short_name: String,
url: Url,
username: String,
@@ -59,6 +59,7 @@ where
stream_id: i32,
c: &Camera,
s: &Stream,
session_group: Arc<retina::client::SessionGroup>,
rotate_offset_sec: i64,
rotate_interval_sec: i64,
) -> Result<Self, Error> {
@@ -71,7 +72,7 @@ where
bail!("RTSP URL shouldn't include credentials");
}
Ok(Streamer {
shutdown: env.shutdown.clone(),
shutdown_rx: env.shutdown_rx.clone(),
rotate_offset_sec,
rotate_interval_sec,
db: env.db.clone(),
@@ -80,6 +81,7 @@ where
opener: env.opener,
transport: env.transport,
stream_id,
session_group,
short_name: format!("{}-{}", c.short_name, s.type_.as_str()),
url: url.clone(),
username: c.config.username.clone(),
@@ -95,7 +97,7 @@ where
/// Note that when using Retina as the RTSP library, this must be called
/// within a tokio runtime context; see [tokio::runtime::Handle].
pub fn run(&mut self) {
while !self.shutdown.load(Ordering::SeqCst) {
while self.shutdown_rx.check().is_ok() {
if let Err(e) = self.run_once() {
let sleep_time = time::Duration::seconds(1);
warn!(
@@ -114,6 +116,31 @@ where
info!("{}: Opening input: {}", self.short_name, self.url.as_str());
let clocks = self.db.clocks();
let mut waited = false;
loop {
let status = self.session_group.stale_sessions();
if let Some(max_expires) = status.max_expires {
log::info!(
"{}: waiting up to {:?} for TEARDOWN or expiration of {} stale sessions",
&self.short_name,
max_expires.saturating_duration_since(tokio::time::Instant::now()),
status.num_sessions
);
tokio::runtime::Handle::current().block_on(async {
tokio::select! {
_ = self.session_group.await_stale_sessions(&status) => Ok(()),
_ = self.shutdown_rx.as_future() => Err(base::shutdown::ShutdownError),
}
})?;
waited = true;
} else {
if waited {
log::info!("{}: done waiting; no more stale sessions", &self.short_name);
}
break;
}
}
let (extra_data, mut stream) = {
let _t = TimerGuard::new(&clocks, || format!("opening {}", self.url.as_str()));
self.opener.open(
@@ -131,6 +158,7 @@ where
Some(self.password.clone())
},
transport: self.transport,
session_group: self.session_group.clone(),
},
)?
};
@@ -150,7 +178,7 @@ where
self.stream_id,
video_sample_entry_id,
);
while !self.shutdown.load(Ordering::SeqCst) {
while self.shutdown_rx.check().is_ok() {
let pkt = {
let _t = TimerGuard::new(&clocks, || "getting next packet");
stream.next()
@@ -207,7 +235,13 @@ where
}
};
let _t = TimerGuard::new(&clocks, || format!("writing {} bytes", pkt.data.len()));
w.write(pkt.data, local_time, pkt.pts, pkt.is_key)?;
w.write(
&mut self.shutdown_rx,
pkt.data,
local_time,
pkt.pts,
pkt.is_key,
)?;
rotate = Some(r);
}
if rotate.is_some() {
@@ -229,7 +263,6 @@ mod tests {
use parking_lot::Mutex;
use std::cmp;
use std::convert::TryFrom;
use std::sync::atomic::{AtomicBool, Ordering};
use std::sync::Arc;
use time;
@@ -305,7 +338,7 @@ mod tests {
struct MockOpener {
expected_url: url::Url,
streams: Mutex<Vec<(h264::ExtraData, Box<dyn stream::Stream>)>>,
shutdown: Arc<AtomicBool>,
shutdown_tx: Mutex<Option<base::shutdown::Sender>>,
}
impl stream::Opener for MockOpener {
@@ -326,7 +359,7 @@ mod tests {
}
None => {
trace!("MockOpener shutting down");
self.shutdown.store(true, Ordering::SeqCst);
self.shutdown_tx.lock().take();
bail!("done")
}
}
@@ -373,16 +406,17 @@ mod tests {
stream.ts_offset = 123456; // starting pts of the input should be irrelevant
stream.ts_offset_pkts_left = u32::max_value();
stream.pkts_left = u32::max_value();
let (shutdown_tx, shutdown_rx) = base::shutdown::channel();
let opener = MockOpener {
expected_url: url::Url::parse("rtsp://test-camera/main").unwrap(),
streams: Mutex::new(vec![(extra_data, Box::new(stream))]),
shutdown: Arc::new(AtomicBool::new(false)),
shutdown_tx: Mutex::new(Some(shutdown_tx)),
};
let db = testutil::TestDb::new(clocks.clone());
let env = super::Environment {
opener: &opener,
db: &db.db,
shutdown: &opener.shutdown,
shutdown_rx: &shutdown_rx,
transport: retina::client::Transport::Tcp,
};
let mut stream;
@@ -402,6 +436,7 @@ mod tests {
testutil::TEST_STREAM_ID,
camera,
s,
Arc::new(retina::client::SessionGroup::default()),
0,
3,
)

23
server/src/web.rs
View File

@@ -11,7 +11,7 @@ use core::borrow::Borrow;
use core::str::FromStr;
use db::dir::SampleFileDir;
use db::{auth, recording};
use failure::{bail, format_err, Error};
use failure::{format_err, Error};
use fnv::FnvHashMap;
use futures::stream::StreamExt;
use futures::{future::Either, sink::SinkExt};
@@ -762,6 +762,7 @@ impl Service {
video_samples: row.video_samples,
video_sample_entry_id: row.video_sample_entry_id,
growing: row.growing,
has_trailing_zero: row.has_trailing_zero,
});
if !out
.video_sample_entries
@@ -856,7 +857,8 @@ impl Service {
if let Some(o) = s.open_id {
if r.open_id != o {
bail!(
bail_t!(
NotFound,
"recording {} has open id {}, requested {}",
r.id,
r.open_id,
@@ -868,9 +870,14 @@ impl Service {
// Check for missing recordings.
match prev {
None if recording_id == s.ids.start => {}
None => bail!("no such recording {}/{}", stream_id, s.ids.start),
None => bail_t!(
NotFound,
"no such recording {}/{}",
stream_id,
s.ids.start
),
Some(id) if r.id.recording() != id + 1 => {
bail!("no such recording {}/{}", stream_id, id + 1);
bail_t!(NotFound, "no such recording {}/{}", stream_id, id + 1);
}
_ => {}
};
@@ -909,8 +916,7 @@ impl Service {
}
cur_off += wd;
Ok(())
})
.map_err(internal_server_err)?;
})?;
// Check for missing recordings.
match prev {
@@ -1368,15 +1374,16 @@ impl<'a> StaticFileRequest<'a> {
};
let ext = &path[last_dot + 1..];
let mime = match ext {
"css" => "text/css",
"html" => "text/html",
"ico" => "image/x-icon",
"js" | "map" => "text/javascript",
"json" => "application/json",
"png" => "image/png",
"webmanifest" => "application/manifest+json",
"svg" => "image/svg+xml",
"txt" => "text/plain",
"webmanifest" => "application/manifest+json",
"woff2" => "font/woff2",
"css" => "text/css",
_ => return None,
};