remove half-baked analytics module

This is (slightly) complicating the switch from ffmpeg to retina
as the RTSP client. And it's not really that close to what I want
to end up with for analytics:

*   I'd prefer the analytics happen in a separate process for
    several reasons
*   Feeding the entire frame to the object detector doesn't produce
    good results.
*   It doesn't do anything with the results yet anyway.

server/Cargo.lock

@@ -1252,7 +1252,6 @@ dependencies = [
  "moonfire-base",
  "moonfire-db",
  "moonfire-ffmpeg",
- "moonfire-tflite",
  "mylog",
  "nix",
  "nom",
@@ -1278,16 +1277,6 @@ dependencies = [
  "uuid",
 ]
 
-[[package]]
-name = "moonfire-tflite"
-version = "0.0.1"
-source = "git+https://github.com/scottlamb/moonfire-tflite#b1d30c09045c02966249676fd716e917761a7de5"
-dependencies = [
- "cc",
- "libc",
- "log",
-]
-
 [[package]]
 name = "mp4ra-rust"
 version = "0.1.0"

server/Cargo.toml

@@ -15,8 +15,6 @@ nightly = ["db/nightly", "parking_lot/nightly", "smallvec/union"]
 # native libraries where possible.
 bundled = ["rusqlite/bundled"]
 
-analytics = ["moonfire-tflite", "ffmpeg/swscale"]
-
 [workspace]
 members = ["base", "db"]
 
@@ -41,7 +39,6 @@ lazy_static = "1.0"
 libc = "0.2"
 log = { version = "0.4" }
 memchr = "2.0.2"
-moonfire-tflite = { git = "https://github.com/scottlamb/moonfire-tflite", features = ["edgetpu"], optional = true }
 mylog = { git = "https://github.com/scottlamb/mylog" }
 nix = "0.20.0"
 nom = "6.0.0"

server/src/analytics.rs

@@ -1,246 +0,0 @@
-// This file is part of Moonfire NVR, a security camera network video recorder.
-// Copyright (C) 2020 The Moonfire NVR Authors; see AUTHORS and LICENSE.txt.
-// SPDX-License-Identifier: GPL-v3.0-or-later WITH GPL-3.0-linking-exception.
-
-//! Video analytics via TensorFlow Lite and an Edge TPU.
-//!
-//! Note this module is only compiled with `--features=analytics`. There's a stub implementation in
-//! `src/main.rs` which is used otherwise.
-//!
-//! Currently results are only logged (rather spammily, on each frame), not persisted to the
-//! database. This will change soon.
-//!
-//! Currently does object detection on every frame with a single hardcoded model: the 300x300
-//! MobileNet SSD v2 (COCO) from https://coral.ai/models/. Eventually analytics might include:
-//!
-//! *   an object detection model retrained on surveillance images and/or larger input sizes
-//!     for increased accuracy.
-//! *   multiple invocations per image to improve resolution with current model sizes
-//!     (either fixed, overlapping subsets of the image or zooming in on full-frame detections to
-//!     increase confidence).
-//! *   support for other hardware setups (GPUs, other brands of NPUs).
-//! *   a motion detection model.
-//! *   H.264/H.265 decoding on every frame but performing object detection at a minimum pts
-//!     interval to cut down on expense.
-
-use cstr::cstr;
-use failure::{format_err, Error};
-use ffmpeg;
-use log::info;
-use std::sync::Arc;
-
-static MODEL: &[u8] = include_bytes!("edgetpu.tflite");
-
-//static MODEL_UUID: Uuid = Uuid::from_u128(0x02054a38_62cf_42ff_9ffa_04876a2970d0_u128);
-
-pub static MODEL_LABELS: [Option<&str>; 90] = [
-    Some("person"),
-    Some("bicycle"),
-    Some("car"),
-    Some("motorcycle"),
-    Some("airplane"),
-    Some("bus"),
-    Some("train"),
-    Some("truck"),
-    Some("boat"),
-    Some("traffic light"),
-    Some("fire hydrant"),
-    None,
-    Some("stop sign"),
-    Some("parking meter"),
-    Some("bench"),
-    Some("bird"),
-    Some("cat"),
-    Some("dog"),
-    Some("horse"),
-    Some("sheep"),
-    Some("cow"),
-    Some("elephant"),
-    Some("bear"),
-    Some("zebra"),
-    Some("giraffe"),
-    None,
-    Some("backpack"),
-    Some("umbrella"),
-    None,
-    None,
-    Some("handbag"),
-    Some("tie"),
-    Some("suitcase"),
-    Some("frisbee"),
-    Some("skis"),
-    Some("snowboard"),
-    Some("sports ball"),
-    Some("kite"),
-    Some("baseball bat"),
-    Some("baseball glove"),
-    Some("skateboard"),
-    Some("surfboard"),
-    Some("tennis racket"),
-    Some("bottle"),
-    None,
-    Some("wine glass"),
-    Some("cup"),
-    Some("fork"),
-    Some("knife"),
-    Some("spoon"),
-    Some("bowl"),
-    Some("banana"),
-    Some("apple"),
-    Some("sandwich"),
-    Some("orange"),
-    Some("broccoli"),
-    Some("carrot"),
-    Some("hot dog"),
-    Some("pizza"),
-    Some("donut"),
-    Some("cake"),
-    Some("chair"),
-    Some("couch"),
-    Some("potted plant"),
-    Some("bed"),
-    None,
-    Some("dining table"),
-    None,
-    None,
-    Some("toilet"),
-    None,
-    Some("tv"),
-    Some("laptop"),
-    Some("mouse"),
-    Some("remote"),
-    Some("keyboard"),
-    Some("cell phone"),
-    Some("microwave"),
-    Some("oven"),
-    Some("toaster"),
-    Some("sink"),
-    Some("refrigerator"),
-    None,
-    Some("book"),
-    Some("clock"),
-    Some("vase"),
-    Some("scissors"),
-    Some("teddy bear"),
-    Some("hair drier"),
-    Some("toothbrush"),
-];
-
-pub struct ObjectDetector {
-    interpreter: parking_lot::Mutex<moonfire_tflite::Interpreter<'static>>,
-    width: i32,
-    height: i32,
-}
-
-impl ObjectDetector {
-    pub fn new(/*db: &db::LockedDatabase*/) -> Result<Arc<Self>, Error> {
-        let model = moonfire_tflite::Model::from_static(MODEL)
-            .map_err(|()| format_err!("TensorFlow Lite model initialization failed"))?;
-        let devices = moonfire_tflite::edgetpu::Devices::list();
-        let device = devices
-            .first()
-            .ok_or_else(|| format_err!("No Edge TPU device available"))?;
-        info!(
-            "Using device {:?}/{:?} for object detection",
-            device.type_(),
-            device.path()
-        );
-        let mut builder = moonfire_tflite::Interpreter::builder();
-        builder.add_owned_delegate(device.create_delegate().map_err(|()| {
-            format_err!(
-                "Unable to create delegate for {:?}/{:?}",
-                device.type_(),
-                device.path()
-            )
-        })?);
-        let interpreter = builder
-            .build(&model)
-            .map_err(|()| format_err!("TensorFlow Lite initialization failed"))?;
-        Ok(Arc::new(Self {
-            interpreter: parking_lot::Mutex::new(interpreter),
-            width: 300, // TODO
-            height: 300,
-        }))
-    }
-}
-
-pub struct ObjectDetectorStream {
-    decoder: ffmpeg::avcodec::DecodeContext,
-    frame: ffmpeg::avutil::VideoFrame,
-    scaler: ffmpeg::swscale::Scaler,
-    scaled: ffmpeg::avutil::VideoFrame,
-}
-
-/// Copies from a RGB24 VideoFrame to a 1xHxWx3 Tensor.
-fn copy(from: &ffmpeg::avutil::VideoFrame, to: &mut moonfire_tflite::Tensor) {
-    let from = from.plane(0);
-    let to = to.bytes_mut();
-    let (w, h) = (from.width, from.height);
-    let mut from_i = 0;
-    let mut to_i = 0;
-    for _y in 0..h {
-        to[to_i..to_i + 3 * w].copy_from_slice(&from.data[from_i..from_i + 3 * w]);
-        from_i += from.linesize;
-        to_i += 3 * w;
-    }
-}
-
-const SCORE_THRESHOLD: f32 = 0.5;
-
-impl ObjectDetectorStream {
-    pub fn new(
-        par: ffmpeg::avcodec::InputCodecParameters<'_>,
-        detector: &ObjectDetector,
-    ) -> Result<Self, Error> {
-        let mut dopt = ffmpeg::avutil::Dictionary::new();
-        dopt.set(cstr!("refcounted_frames"), cstr!("0"))?;
-        let decoder = par.new_decoder(&mut dopt)?;
-        let scaled = ffmpeg::avutil::VideoFrame::owned(ffmpeg::avutil::ImageDimensions {
-            width: detector.width,
-            height: detector.height,
-            pix_fmt: ffmpeg::avutil::PixelFormat::rgb24(),
-        })?;
-        let frame = ffmpeg::avutil::VideoFrame::empty()?;
-        let scaler = ffmpeg::swscale::Scaler::new(par.dims(), scaled.dims())?;
-        Ok(Self {
-            decoder,
-            frame,
-            scaler,
-            scaled,
-        })
-    }
-
-    pub fn process_frame(
-        &mut self,
-        pkt: &ffmpeg::avcodec::Packet<'_>,
-        detector: &ObjectDetector,
-    ) -> Result<(), Error> {
-        if !self.decoder.decode_video(pkt, &mut self.frame)? {
-            return Ok(());
-        }
-        self.scaler.scale(&self.frame, &mut self.scaled);
-        let mut interpreter = detector.interpreter.lock();
-        copy(&self.scaled, &mut interpreter.inputs()[0]);
-        interpreter
-            .invoke()
-            .map_err(|()| format_err!("TFLite interpreter invocation failed"))?;
-        let outputs = interpreter.outputs();
-        let classes = outputs[1].f32s();
-        let scores = outputs[2].f32s();
-        for (i, &score) in scores.iter().enumerate() {
-            if score < SCORE_THRESHOLD {
-                continue;
-            }
-            let class = classes[i] as usize;
-            if class >= MODEL_LABELS.len() {
-                continue;
-            }
-            let label = match MODEL_LABELS[class] {
-                None => continue,
-                Some(l) => l,
-            };
-            info!("{}, score {}", label, score);
-        }
-        Ok(())
-    }
-}

server/src/cmds/run.rs

@@ -65,12 +65,6 @@ pub struct Args {
     /// --http-addr=127.0.0.1:8080.
     #[structopt(long)]
     trust_forward_hdrs: bool,
-
-    /// Perform object detection on SUB streams.
-    ///
-    /// Note: requires compilation with --feature=analytics.
-    #[structopt(long)]
-    object_detection: bool,
 }
 
 // These are used in a hack to get the name of the current time zone (e.g. America/Los_Angeles).
@@ -176,11 +170,6 @@ pub async fn run(args: &Args) -> Result<i32, Error> {
     let db = Arc::new(db::Database::new(clocks, conn, !args.read_only).unwrap());
     info!("Database is loaded.");
 
-    let object_detector = match args.object_detection {
-        false => None,
-        true => Some(crate::analytics::ObjectDetector::new()?),
-    };
-
     {
         let mut l = db.lock();
         let dirs_to_open: Vec<_> = l
@@ -258,10 +247,6 @@ pub async fn run(args: &Args) -> Result<i32, Error> {
             };
             let rotate_offset_sec = streamer::ROTATE_INTERVAL_SEC * i as i64 / streams as i64;
             let syncer = syncers.get(&sample_file_dir_id).unwrap();
-            let object_detector = match stream.type_ {
-                db::StreamType::Sub => object_detector.clone(),
-                _ => None,
-            };
             let mut streamer = streamer::Streamer::new(
                 &env,
                 syncer.dir.clone(),
@@ -271,7 +256,6 @@ pub async fn run(args: &Args) -> Result<i32, Error> {
                 stream,
                 rotate_offset_sec,
                 streamer::ROTATE_INTERVAL_SEC,
-                object_detector,
             )?;
             info!("Starting streamer for {}", streamer.short_name());
             let name = format!("s-{}", streamer.short_name());

server/src/main.rs

@@ -9,42 +9,6 @@ use std::fmt::Write;
 use std::str::FromStr;
 use structopt::StructOpt;
 
-#[cfg(feature = "analytics")]
-mod analytics;
-
-/// Stub implementation of analytics module when not compiled with TensorFlow Lite.
-#[cfg(not(feature = "analytics"))]
-mod analytics {
-    use failure::{bail, Error};
-
-    pub struct ObjectDetector;
-
-    impl ObjectDetector {
-        pub fn new() -> Result<std::sync::Arc<ObjectDetector>, Error> {
-            bail!("Recompile with --features=analytics for object detection.");
-        }
-    }
-
-    pub struct ObjectDetectorStream;
-
-    impl ObjectDetectorStream {
-        pub fn new(
-            _par: ffmpeg::avcodec::InputCodecParameters<'_>,
-            _detector: &ObjectDetector,
-        ) -> Result<Self, Error> {
-            unimplemented!();
-        }
-
-        pub fn process_frame(
-            &mut self,
-            _pkt: &ffmpeg::avcodec::Packet<'_>,
-            _detector: &ObjectDetector,
-        ) -> Result<(), Error> {
-            unimplemented!();
-        }
-    }
-}
-
 mod body;
 mod cmds;
 mod h264;

server/src/streamer.rs

@@ -46,7 +46,6 @@ where
     short_name: String,
     url: Url,
     redacted_url: Url,
-    detector: Option<Arc<crate::analytics::ObjectDetector>>,
 }
 
 impl<'a, C, S> Streamer<'a, C, S>
@@ -63,7 +62,6 @@ where
         s: &Stream,
         rotate_offset_sec: i64,
         rotate_interval_sec: i64,
-        detector: Option<Arc<crate::analytics::ObjectDetector>>,
     ) -> Result<Self, Error> {
         let mut url = Url::parse(&s.rtsp_url)?;
         let mut redacted_url = url.clone();
@@ -86,7 +84,6 @@ where
             short_name: format!("{}-{}", c.short_name, s.type_.as_str()),
             url,
             redacted_url,
-            detector,
         })
     }
 
@@ -122,14 +119,6 @@ where
             })?
         };
         let realtime_offset = self.db.clocks().realtime() - clocks.monotonic();
-        // TODO: verify width/height.
-        let mut detector_stream = match self.detector.as_ref() {
-            None => None,
-            Some(od) => Some(crate::analytics::ObjectDetectorStream::new(
-                stream.get_video_codecpar(),
-                &od,
-            )?),
-        };
         let extra_data = stream.get_extra_data()?;
         let video_sample_entry_id = {
             let _t = TimerGuard::new(&clocks, || "inserting video sample entry");
@@ -159,9 +148,6 @@ where
                 debug!("{}: have first key frame", self.short_name);
                 seen_key_frame = true;
             }
-            if let (Some(a_s), Some(a)) = (detector_stream.as_mut(), self.detector.as_ref()) {
-                a_s.process_frame(&pkt, &a)?;
-            }
             let frame_realtime = clocks.monotonic() + realtime_offset;
             let local_time = recording::Time::new(frame_realtime);
             rotate = if let Some(r) = rotate {
@@ -412,7 +398,6 @@ mod tests {
                 s,
                 0,
                 3,
-                None,
             )
             .unwrap();
         }