seamlessly merge minor VSE changes

Improves #302.

CHANGELOG.md

@@ -8,6 +8,12 @@ upgrades, e.g. `v0.6.x` -> `v0.7.x`. The config file format and
 [API](ref/api.md) currently have no stability guarantees, so they may change
 even on minor releases, e.g. `v0.7.5` -> `v0.7.6`.
 
+## unreleased
+
+*   seamlessly merge together recordings which have imperceptible changes in
+    their `VideoSampleEntry`. Improves
+    [#302](https://github.com/scottlamb/moonfire-nvr/issues/302).
+
 ## v0.7.12 (2024-01-08)
 
 *   update to Retina 0.4.7, supporting RTSP servers that do not set

ref/api.md

@@ -342,6 +342,7 @@ arbitrary order. Each recording object has the following properties:
     together are as described. Adjacent recordings from the same RTSP session
     may be coalesced in this fashion to reduce the amount of redundant data
     transferred.
+*   `runStartId`. The id of the first recording in this run.
 *   `firstUncommitted` (optional). If this range is not fully committed to the
     database, the first id that is uncommitted. This is significant because
     it's possible that after a crash and restart, this id will refer to a

server/src/json.rs

@@ -470,6 +470,7 @@ pub struct Recording {
     pub video_sample_entry_id: i32,
     pub start_id: i32,
     pub open_id: u32,
+    pub run_start_id: i32,
 
     #[serde(skip_serializing_if = "Option::is_none")]
     pub first_uncommitted: Option<i32>,

server/src/web/mod.rs

@@ -496,6 +496,7 @@ impl Service {
                 } else {
                     Some(end)
                 },
+                run_start_id: row.run_start_id,
                 start_time_90k: row.time.start.0,
                 end_time_90k: row.time.end.0,
                 sample_file_bytes: row.sample_file_bytes,

ui/src/List/VideoList.test.tsx

@@ -10,7 +10,7 @@ import { setupServer } from "msw/node";
 import { Recording, VideoSampleEntry } from "../api";
 import { renderWithCtx } from "../testutil";
 import { Camera, Stream } from "../types";
-import VideoList from "./VideoList";
+import VideoList, { combine } from "./VideoList";
 import { beforeAll, afterAll, afterEach, expect, test } from "vitest";
 
 const TEST_CAMERA: Camera = {
@@ -45,9 +45,30 @@ const TEST_RANGE2: [number, number] = [
 ];
 
 const TEST_RECORDINGS1: Recording[] = [
+  {
+    startId: 44,
+    openId: 1,
+    runStartId: 44,
+    startTime90k: 145750553550000, // 2021-04-26T08:23:15:00000-07:00
+    endTime90k: 145750558950000, // 2021-04-26T08:24:15:00000-07:00
+    videoSampleEntryId: 4,
+    videoSamples: 1860,
+    sampleFileBytes: 248000,
+  },
+  {
+    startId: 43,
+    openId: 1,
+    runStartId: 40,
+    startTime90k: 145750548150000, // 2021-04-26T08:22:15:00000-07:00
+    endTime90k: 145750553550000, // 2021-04-26T08:23:15:00000-07:00
+    videoSampleEntryId: 4,
+    videoSamples: 1860,
+    sampleFileBytes: 248000,
+  },
   {
     startId: 42,
     openId: 1,
+    runStartId: 40,
     startTime90k: 145750542570000, // 2021-04-26T08:21:13:00000-07:00
     endTime90k: 145750548150000, // 2021-04-26T08:22:15:00000-07:00
     videoSampleEntryId: 4,
@@ -60,6 +81,7 @@ const TEST_RECORDINGS2: Recording[] = [
   {
     startId: 42,
     openId: 1,
+    runStartId: 40,
     startTime90k: 145757651670000, // 2021-04-27T06:17:43:00000-07:00
     endTime90k: 145757656980000, // 2021-04-27T06:18:42:00000-07:00
     videoSampleEntryId: 4,
@@ -116,6 +138,59 @@ beforeAll(() => server.listen({ onUnhandledRequest: "error" }));
 afterEach(() => server.resetHandlers());
 afterAll(() => server.close());
 
+test("combine", () => {
+  const actual = combine(undefined, {
+    videoSampleEntries: TEST_VIDEO_SAMPLE_ENTRIES,
+    recordings: TEST_RECORDINGS1,
+  });
+  const expected = [
+    // 44 shouldn't be combined; it's not from the same run as the others.
+    {
+      startId: 44,
+      endId: 44,
+      openId: 1,
+      runStartId: 44,
+      startTime90k: 145750553550000, // 2021-04-26T08:23:15:00000-07:00
+      endTime90k: 145750558950000, // 2021-04-26T08:24:15:00000-07:00
+      videoSamples: 1860,
+      sampleFileBytes: 248000,
+      aspectWidth: TEST_VIDEO_SAMPLE_ENTRIES[4].aspectWidth,
+      aspectHeight: TEST_VIDEO_SAMPLE_ENTRIES[4].aspectHeight,
+      width: TEST_VIDEO_SAMPLE_ENTRIES[4].width,
+      height: TEST_VIDEO_SAMPLE_ENTRIES[4].height,
+      firstUncommitted: undefined,
+      growing: undefined,
+    },
+    // 42 and 43 are combinable.
+    {
+      startId: 42,
+      endId: 43,
+      openId: 1,
+      runStartId: 40,
+      startTime90k: 145750542570000, // 2021-04-26T08:21:13:00000-07:00
+      endTime90k: 145750553550000, // 2021-04-26T08:23:15:00000-07:00
+      videoSamples: 3720,
+      sampleFileBytes: 496000,
+      aspectWidth: TEST_VIDEO_SAMPLE_ENTRIES[4].aspectWidth,
+      aspectHeight: TEST_VIDEO_SAMPLE_ENTRIES[4].aspectHeight,
+      width: TEST_VIDEO_SAMPLE_ENTRIES[4].width,
+      height: TEST_VIDEO_SAMPLE_ENTRIES[4].height,
+      firstUncommitted: undefined,
+      growing: undefined,
+    },
+  ];
+
+  // XXX: unsure why this doesn't work:
+  //
+  // expect(actual).toContainEqual(expected)
+  //
+  // ...but this does:
+  expect(actual).toHaveLength(expected.length);
+  for (let i = 0; i < expected.length; i++) {
+    expect(actual[i]).toEqual(expected[i]);
+  }
+});
+
 test("load", async () => {
   renderWithCtx(
     <table>

ui/src/List/VideoList.tsx

@@ -17,12 +17,97 @@ interface Props {
   range90k: [number, number] | null;
   split90k?: number;
   trimStartAndEnd: boolean;
-  setActiveRecording: (
+  setActiveRecording: (recording: [Stream, CombinedRecording] | null) => void;
-    recording: [Stream, api.Recording, api.VideoSampleEntry] | null
-  ) => void;
   formatTime: (time90k: number) => string;
 }
 
+/**
+ * Matches `api.Recording`, except that two entries with differing
+ * `videoSampleEntryId` but the same resolution may be combined.
+ */
+export interface CombinedRecording {
+  startId: number;
+  endId?: number;
+  runStartId: number;
+  firstUncommitted?: number;
+  growing?: boolean;
+  openId: number;
+  startTime90k: number;
+  endTime90k: number;
+  videoSamples: number;
+  sampleFileBytes: number;
+  width: number;
+  height: number;
+  aspectWidth: number;
+  aspectHeight: number;
+}
+
+/**
+ * Combines recordings, which are assumed to already be sorted in descending
+ * chronological order.
+ *
+ * This is exported only for testing.
+ */
+export function combine(
+  split90k: number | undefined,
+  response: api.RecordingsResponse
+): CombinedRecording[] {
+  let out = [];
+  let cur = null;
+
+  for (const r of response.recordings) {
+    const vse = response.videoSampleEntries[r.videoSampleEntryId];
+
+    // Combine `r` into `cur` if `r` precedes r, shouldn't be split, and
+    // has similar resolution. It doesn't have to have exactly the same
+    // video sample entry; minor changes to encoding can be seamlessly
+    // combined into one `.mp4` file.
+    if (
+      cur !== null &&
+      r.openId === cur.openId &&
+      r.runStartId === cur.runStartId &&
+      (r.endId ?? r.startId) + 1 === cur.startId &&
+      cur.width === vse.width &&
+      cur.height === vse.height &&
+      cur.aspectWidth === vse.aspectWidth &&
+      cur.aspectHeight === vse.aspectHeight &&
+      (split90k === undefined || r.endTime90k - cur.startTime90k <= split90k)
+    ) {
+      cur.startId = r.startId;
+      cur.firstUncommitted == r.firstUncommitted ?? cur.firstUncommitted;
+      cur.startTime90k = r.startTime90k;
+      cur.videoSamples += r.videoSamples;
+      cur.sampleFileBytes += r.sampleFileBytes;
+      continue;
+    }
+
+    // Otherwise, start a new `cur`, flushing any existing one.
+    if (cur !== null) {
+      out.push(cur);
+    }
+    cur = {
+      startId: r.startId,
+      endId: r.endId ?? r.startId,
+      runStartId: r.runStartId,
+      firstUncommitted: r.firstUncommitted,
+      growing: r.growing,
+      openId: r.openId,
+      startTime90k: r.startTime90k,
+      endTime90k: r.endTime90k,
+      videoSamples: r.videoSamples,
+      sampleFileBytes: r.sampleFileBytes,
+      width: vse.width,
+      height: vse.height,
+      aspectWidth: vse.aspectWidth,
+      aspectHeight: vse.aspectHeight,
+    };
+  }
+  if (cur !== null) {
+    out.push(cur);
+  }
+  return out;
+}
+
 const frameRateFmt = new Intl.NumberFormat([], {
   maximumFractionDigits: 0,
 });
@@ -37,7 +122,8 @@ interface State {
    * During loading, this can differ from the requested range.
    */
   range90k: [number, number];
-  response: { status: "skeleton" } | api.FetchResult<api.RecordingsResponse>;
+  split90k?: number;
+  response: { status: "skeleton" } | api.FetchResult<CombinedRecording[]>;
 }
 
 interface RowProps extends TableRowProps {
@@ -111,12 +197,21 @@ const VideoList = ({
         split90k,
       };
       let response = await api.recordings(req, { signal });
-      if (response.status === "success") {
-        // Sort recordings in descending order by start time.
-        response.response.recordings.sort((a, b) => b.startId - a.startId);
-      }
       clearTimeout(timerId);
-      setState({ range90k, response });
+      if (response.status === "success") {
+        // Sort recordings in descending order.
+        response.response.recordings.sort((a, b) => b.startId - a.startId);
+        setState({
+          range90k,
+          split90k,
+          response: {
+            status: "success",
+            response: combine(split90k, response.response),
+          },
+        });
+      } else {
+        setState({ range90k, split90k, response });
+      }
     };
     if (range90k !== null) {
       const timerId = setTimeout(
@@ -157,8 +252,7 @@ const VideoList = ({
     );
   } else if (state.response.status === "success") {
     const resp = state.response.response;
-    body = resp.recordings.map((r: api.Recording) => {
+    body = resp.map((r: CombinedRecording) => {
-      const vse = resp.videoSampleEntries[r.videoSampleEntryId];
       const durationSec = (r.endTime90k - r.startTime90k) / 90000;
       const rate = (r.sampleFileBytes / durationSec) * 0.000008;
       const start = trimStartAndEnd
@@ -171,10 +265,10 @@ const VideoList = ({
         <Row
           key={r.startId}
           className="recording"
-          onClick={() => setActiveRecording([stream, r, vse])}
+          onClick={() => setActiveRecording([stream, r])}
           start={formatTime(start)}
           end={formatTime(end)}
-          resolution={`${vse.width}x${vse.height}`}
+          resolution={`${r.width}x${r.height}`}
           fps={frameRateFmt.format(r.videoSamples / durationSec)}
           storage={`${sizeFmt.format(r.sampleFileBytes / 1048576)} MiB`}
           bitrate={`${sizeFmt.format(rate)} Mbps`}

ui/src/List/index.tsx

@@ -16,7 +16,7 @@ import { Stream } from "../types";
 import DisplaySelector, { DEFAULT_DURATION } from "./DisplaySelector";
 import StreamMultiSelector from "./StreamMultiSelector";
 import TimerangeSelector from "./TimerangeSelector";
-import VideoList from "./VideoList";
+import VideoList, { CombinedRecording } from "./VideoList";
 import { useLayoutEffect } from "react";
 import { fillAspect } from "../aspect";
 import useResizeObserver from "@react-hook/resize-observer";
@@ -208,7 +208,7 @@ const Main = ({ toplevel, timeZoneName, Frame }: Props) => {
   );
 
   const [activeRecording, setActiveRecording] = useState<
-    [Stream, api.Recording, api.VideoSampleEntry] | null
+    [Stream, CombinedRecording] | null
   >(null);
   const formatTime = useMemo(() => {
     return (time90k: number) => {
@@ -341,8 +341,8 @@ const Main = ({ toplevel, timeZoneName, Frame }: Props) => {
                 trimStartAndEnd ? range90k! : undefined
               )}
               aspect={[
-                activeRecording[2].aspectWidth,
+                activeRecording[1].aspectWidth,
-                activeRecording[2].aspectHeight,
+                activeRecording[1].aspectHeight,
               ]}
             />
           </Modal>

ui/src/api.ts

@@ -325,6 +325,16 @@ export async function deleteUser(
   });
 }
 
+export interface RecordingSpecifier {
+  startId: number;
+  endId?: number;
+  firstUncommitted?: number;
+  growing?: boolean;
+  openId: number;
+  startTime90k: number;
+  endTime90k: number;
+}
+
 /**
  * Represents a range of one or more recordings as in a single array entry of
  * <tt>GET /api/cameras/&lt;uuid>/&lt;stream>/&lt;recordings></tt>.
@@ -339,6 +349,9 @@ export interface Recording {
    */
   endId?: number;
 
+  /** id of the first recording in this run. */
+  runStartId: number;
+
   /**
    * If this range is not fully committed to the database, the first id that is
    * uncommitted. This is significant because it's possible that after a crash
@@ -459,7 +472,7 @@ export async function recordings(req: RecordingsRequest, init: RequestInit) {
 export function recordingUrl(
   cameraUuid: string,
   stream: StreamType,
-  r: Recording,
+  r: RecordingSpecifier,
   timestampTrack: boolean,
   trimToRange90k?: [number, number]
 ): string {