mirror of
https://github.com/scottlamb/moonfire-nvr.git
synced 2024-12-29 08:33:22 -05:00
b41a6c43da
After a frustrating search for a suitable channel to use for shutdown (tokio::sync:⌚:Receiver and futures::future::Shared<tokio::sync::oneshot::Receiver> didn't look quite right) in which I rethought my life decisions, I finally just made my own (server/base/shutdown.rs). We can easily poll it or wait for it in async or sync contexts. Most importantly, it's convenient; not that it really matters here, but it's also efficient. We now do a slightly better job of propagating a "graceful" shutdown signal, and this channel will give us tools to improve it over time. * Shut down even when writer or syncer operations are stuck. Fixes #117 * Not done yet: streamers should instantly shut down without waiting for a connection attempt or frame or something. I'll probably implement that when removing --rtsp-library=ffmpeg. The code should be cleaner then. * Not done yet: fix a couple places that sleep for up to a second when they could shut down immediately. I just need to do the plumbing for mock clocks to work. I also implemented an immediate shutdown mode, activated by a second signal. I think this will mitigate the streamer wait situation.
188 lines
5.0 KiB
Rust
188 lines
5.0 KiB
Rust
// This file is part of Moonfire NVR, a security camera network video recorder.
|
|
// Copyright (C) 2018 The Moonfire NVR Authors; see AUTHORS and LICENSE.txt.
|
|
// SPDX-License-Identifier: GPL-v3.0-or-later WITH GPL-3.0-linking-exception.
|
|
|
|
//! Clock interface and implementations for testability.
|
|
|
|
use failure::Error;
|
|
use log::warn;
|
|
use parking_lot::Mutex;
|
|
use std::mem;
|
|
use std::sync::{mpsc, Arc};
|
|
use std::thread;
|
|
use std::time::Duration as StdDuration;
|
|
use time::{Duration, Timespec};
|
|
|
|
use crate::shutdown::ShutdownError;
|
|
|
|
/// Abstract interface to the system clocks. This is for testability.
|
|
pub trait Clocks: Send + Sync + 'static {
|
|
/// Gets the current time from `CLOCK_REALTIME`.
|
|
fn realtime(&self) -> Timespec;
|
|
|
|
/// Gets the current time from a monotonic clock.
|
|
///
|
|
/// On Linux, this uses `CLOCK_BOOTTIME`, which includes suspended time.
|
|
/// On other systems, it uses `CLOCK_MONOTONIC`.
|
|
fn monotonic(&self) -> Timespec;
|
|
|
|
/// Causes the current thread to sleep for the specified time.
|
|
fn sleep(&self, how_long: Duration);
|
|
|
|
/// Calls `rcv.recv_timeout` or substitutes a test implementation.
|
|
fn recv_timeout<T>(
|
|
&self,
|
|
rcv: &mpsc::Receiver<T>,
|
|
timeout: StdDuration,
|
|
) -> Result<T, mpsc::RecvTimeoutError>;
|
|
}
|
|
|
|
pub fn retry<C, T, E>(
|
|
clocks: &C,
|
|
shutdown_rx: &crate::shutdown::Receiver,
|
|
f: &mut dyn FnMut() -> Result<T, E>,
|
|
) -> Result<T, ShutdownError>
|
|
where
|
|
C: Clocks,
|
|
E: Into<Error>,
|
|
{
|
|
loop {
|
|
let e = match f() {
|
|
Ok(t) => return Ok(t),
|
|
Err(e) => e.into(),
|
|
};
|
|
shutdown_rx.check()?;
|
|
let sleep_time = Duration::seconds(1);
|
|
warn!(
|
|
"sleeping for {} after error: {}",
|
|
sleep_time,
|
|
crate::error::prettify_failure(&e)
|
|
);
|
|
clocks.sleep(sleep_time);
|
|
}
|
|
}
|
|
|
|
#[derive(Copy, Clone)]
|
|
pub struct RealClocks {}
|
|
|
|
impl RealClocks {
|
|
fn get(&self, clock: libc::clockid_t) -> Timespec {
|
|
unsafe {
|
|
let mut ts = mem::MaybeUninit::uninit();
|
|
assert_eq!(0, libc::clock_gettime(clock, ts.as_mut_ptr()));
|
|
let ts = ts.assume_init();
|
|
Timespec::new(ts.tv_sec as i64, ts.tv_nsec as i32)
|
|
}
|
|
}
|
|
}
|
|
|
|
impl Clocks for RealClocks {
|
|
fn realtime(&self) -> Timespec {
|
|
self.get(libc::CLOCK_REALTIME)
|
|
}
|
|
|
|
#[cfg(target_os = "linux")]
|
|
fn monotonic(&self) -> Timespec {
|
|
self.get(libc::CLOCK_BOOTTIME)
|
|
}
|
|
|
|
#[cfg(not(target_os = "linux"))]
|
|
fn monotonic(&self) -> Timespec {
|
|
self.get(libc::CLOCK_MONOTONIC)
|
|
}
|
|
|
|
fn sleep(&self, how_long: Duration) {
|
|
match how_long.to_std() {
|
|
Ok(d) => thread::sleep(d),
|
|
Err(e) => warn!("Invalid duration {:?}: {}", how_long, e),
|
|
};
|
|
}
|
|
|
|
fn recv_timeout<T>(
|
|
&self,
|
|
rcv: &mpsc::Receiver<T>,
|
|
timeout: StdDuration,
|
|
) -> Result<T, mpsc::RecvTimeoutError> {
|
|
rcv.recv_timeout(timeout)
|
|
}
|
|
}
|
|
|
|
/// Logs a warning if the TimerGuard lives "too long", using the label created by a supplied
|
|
/// function.
|
|
pub struct TimerGuard<'a, C: Clocks + ?Sized, S: AsRef<str>, F: FnOnce() -> S + 'a> {
|
|
clocks: &'a C,
|
|
label_f: Option<F>,
|
|
start: Timespec,
|
|
}
|
|
|
|
impl<'a, C: Clocks + ?Sized, S: AsRef<str>, F: FnOnce() -> S + 'a> TimerGuard<'a, C, S, F> {
|
|
pub fn new(clocks: &'a C, label_f: F) -> Self {
|
|
TimerGuard {
|
|
clocks,
|
|
label_f: Some(label_f),
|
|
start: clocks.monotonic(),
|
|
}
|
|
}
|
|
}
|
|
|
|
impl<'a, C, S, F> Drop for TimerGuard<'a, C, S, F>
|
|
where
|
|
C: Clocks + ?Sized,
|
|
S: AsRef<str>,
|
|
F: FnOnce() -> S + 'a,
|
|
{
|
|
fn drop(&mut self) {
|
|
let elapsed = self.clocks.monotonic() - self.start;
|
|
if elapsed.num_seconds() >= 1 {
|
|
let label_f = self.label_f.take().unwrap();
|
|
warn!("{} took {}!", label_f().as_ref(), elapsed);
|
|
}
|
|
}
|
|
}
|
|
|
|
/// Simulated clock for testing.
|
|
#[derive(Clone)]
|
|
pub struct SimulatedClocks(Arc<SimulatedClocksInner>);
|
|
|
|
struct SimulatedClocksInner {
|
|
boot: Timespec,
|
|
uptime: Mutex<Duration>,
|
|
}
|
|
|
|
impl SimulatedClocks {
|
|
pub fn new(boot: Timespec) -> Self {
|
|
SimulatedClocks(Arc::new(SimulatedClocksInner {
|
|
boot,
|
|
uptime: Mutex::new(Duration::seconds(0)),
|
|
}))
|
|
}
|
|
}
|
|
|
|
impl Clocks for SimulatedClocks {
|
|
fn realtime(&self) -> Timespec {
|
|
self.0.boot + *self.0.uptime.lock()
|
|
}
|
|
fn monotonic(&self) -> Timespec {
|
|
Timespec::new(0, 0) + *self.0.uptime.lock()
|
|
}
|
|
|
|
/// Advances the clock by the specified amount without actually sleeping.
|
|
fn sleep(&self, how_long: Duration) {
|
|
let mut l = self.0.uptime.lock();
|
|
*l = *l + how_long;
|
|
}
|
|
|
|
/// Advances the clock by the specified amount if data is not immediately available.
|
|
fn recv_timeout<T>(
|
|
&self,
|
|
rcv: &mpsc::Receiver<T>,
|
|
timeout: StdDuration,
|
|
) -> Result<T, mpsc::RecvTimeoutError> {
|
|
let r = rcv.recv_timeout(StdDuration::new(0, 0));
|
|
if r.is_err() {
|
|
self.sleep(Duration::from_std(timeout).unwrap());
|
|
}
|
|
r
|
|
}
|
|
}
|