moonfire-nvr/server/base/clock.rs
Scott Lamb b41a6c43da shutdown better
After a frustrating search for a suitable channel to use for shutdown
(tokio::sync::Receiver and
futures::future::Shared<tokio::sync::oneshot::Receiver> didn't look
quite right) in which I rethought my life decisions, I finally just made
my own (server/base/shutdown.rs). We can easily poll it or wait for it
in async or sync contexts. Most importantly, it's convenient; not that
it really matters here, but it's also efficient.

We now do a slightly better job of propagating a "graceful" shutdown
signal, and this channel will give us tools to improve it over time.

* Shut down even when writer or syncer operations are stuck. Fixes #117
* Not done yet: streamers should instantly shut down without waiting for
  a connection attempt or frame or something. I'll probably
  implement that when removing --rtsp-library=ffmpeg. The code should be
  cleaner then.
* Not done yet: fix a couple places that sleep for up to a second when
  they could shut down immediately. I just need to do the plumbing for
  mock clocks to work.

I also implemented an immediate shutdown mode, activated by a second
signal. I think this will mitigate the streamer wait situation.
2021-09-23 16:33:29 -07:00

188 lines
5.0 KiB
Rust

// This file is part of Moonfire NVR, a security camera network video recorder.
// Copyright (C) 2018 The Moonfire NVR Authors; see AUTHORS and LICENSE.txt.
// SPDX-License-Identifier: GPL-v3.0-or-later WITH GPL-3.0-linking-exception.
//! Clock interface and implementations for testability.
use failure::Error;
use log::warn;
use parking_lot::Mutex;
use std::mem;
use std::sync::{mpsc, Arc};
use std::thread;
use std::time::Duration as StdDuration;
use time::{Duration, Timespec};
use crate::shutdown::ShutdownError;
/// Abstract interface to the system clocks. This is for testability.
pub trait Clocks: Send + Sync + 'static {
/// Gets the current time from `CLOCK_REALTIME`.
fn realtime(&self) -> Timespec;
/// Gets the current time from a monotonic clock.
///
/// On Linux, this uses `CLOCK_BOOTTIME`, which includes suspended time.
/// On other systems, it uses `CLOCK_MONOTONIC`.
fn monotonic(&self) -> Timespec;
/// Causes the current thread to sleep for the specified time.
fn sleep(&self, how_long: Duration);
/// Calls `rcv.recv_timeout` or substitutes a test implementation.
fn recv_timeout<T>(
&self,
rcv: &mpsc::Receiver<T>,
timeout: StdDuration,
) -> Result<T, mpsc::RecvTimeoutError>;
}
pub fn retry<C, T, E>(
clocks: &C,
shutdown_rx: &crate::shutdown::Receiver,
f: &mut dyn FnMut() -> Result<T, E>,
) -> Result<T, ShutdownError>
where
C: Clocks,
E: Into<Error>,
{
loop {
let e = match f() {
Ok(t) => return Ok(t),
Err(e) => e.into(),
};
shutdown_rx.check()?;
let sleep_time = Duration::seconds(1);
warn!(
"sleeping for {} after error: {}",
sleep_time,
crate::error::prettify_failure(&e)
);
clocks.sleep(sleep_time);
}
}
#[derive(Copy, Clone)]
pub struct RealClocks {}
impl RealClocks {
fn get(&self, clock: libc::clockid_t) -> Timespec {
unsafe {
let mut ts = mem::MaybeUninit::uninit();
assert_eq!(0, libc::clock_gettime(clock, ts.as_mut_ptr()));
let ts = ts.assume_init();
Timespec::new(ts.tv_sec as i64, ts.tv_nsec as i32)
}
}
}
impl Clocks for RealClocks {
fn realtime(&self) -> Timespec {
self.get(libc::CLOCK_REALTIME)
}
#[cfg(target_os = "linux")]
fn monotonic(&self) -> Timespec {
self.get(libc::CLOCK_BOOTTIME)
}
#[cfg(not(target_os = "linux"))]
fn monotonic(&self) -> Timespec {
self.get(libc::CLOCK_MONOTONIC)
}
fn sleep(&self, how_long: Duration) {
match how_long.to_std() {
Ok(d) => thread::sleep(d),
Err(e) => warn!("Invalid duration {:?}: {}", how_long, e),
};
}
fn recv_timeout<T>(
&self,
rcv: &mpsc::Receiver<T>,
timeout: StdDuration,
) -> Result<T, mpsc::RecvTimeoutError> {
rcv.recv_timeout(timeout)
}
}
/// Logs a warning if the TimerGuard lives "too long", using the label created by a supplied
/// function.
pub struct TimerGuard<'a, C: Clocks + ?Sized, S: AsRef<str>, F: FnOnce() -> S + 'a> {
clocks: &'a C,
label_f: Option<F>,
start: Timespec,
}
impl<'a, C: Clocks + ?Sized, S: AsRef<str>, F: FnOnce() -> S + 'a> TimerGuard<'a, C, S, F> {
pub fn new(clocks: &'a C, label_f: F) -> Self {
TimerGuard {
clocks,
label_f: Some(label_f),
start: clocks.monotonic(),
}
}
}
impl<'a, C, S, F> Drop for TimerGuard<'a, C, S, F>
where
C: Clocks + ?Sized,
S: AsRef<str>,
F: FnOnce() -> S + 'a,
{
fn drop(&mut self) {
let elapsed = self.clocks.monotonic() - self.start;
if elapsed.num_seconds() >= 1 {
let label_f = self.label_f.take().unwrap();
warn!("{} took {}!", label_f().as_ref(), elapsed);
}
}
}
/// Simulated clock for testing.
#[derive(Clone)]
pub struct SimulatedClocks(Arc<SimulatedClocksInner>);
struct SimulatedClocksInner {
boot: Timespec,
uptime: Mutex<Duration>,
}
impl SimulatedClocks {
pub fn new(boot: Timespec) -> Self {
SimulatedClocks(Arc::new(SimulatedClocksInner {
boot,
uptime: Mutex::new(Duration::seconds(0)),
}))
}
}
impl Clocks for SimulatedClocks {
fn realtime(&self) -> Timespec {
self.0.boot + *self.0.uptime.lock()
}
fn monotonic(&self) -> Timespec {
Timespec::new(0, 0) + *self.0.uptime.lock()
}
/// Advances the clock by the specified amount without actually sleeping.
fn sleep(&self, how_long: Duration) {
let mut l = self.0.uptime.lock();
*l = *l + how_long;
}
/// Advances the clock by the specified amount if data is not immediately available.
fn recv_timeout<T>(
&self,
rcv: &mpsc::Receiver<T>,
timeout: StdDuration,
) -> Result<T, mpsc::RecvTimeoutError> {
let r = rcv.recv_timeout(StdDuration::new(0, 0));
if r.is_err() {
self.sleep(Duration::from_std(timeout).unwrap());
}
r
}
}