mirror of
https://github.com/scottlamb/moonfire-nvr.git
synced 2025-01-11 23:13:23 -05:00
add live stream viewing to React prototype
It's a start. It can display several streams at once, which is nice.
There are lots of opportunities for improvement:
* it doesn't keep the videos approximately in sync.
* it accumulates extra buffering, drifting behind live. This is
particularly noticeable when it's paused and played again; it can
be several seconds before it jumps to after the break.
* it always uses the sub stream rather main. I'd prefer it support
"auto" (use main if the viewport is larger than the sub stream and
there's sufficient bandwidth), "main", or "sub".
* it has a kludgy heuristic where it throws away everything buffered 5
seconds before the current timestamp. It should throw away
everything before the current GOP instead, but I need to alter the
API so it can easily know when that is.
* it can't tell you when a camera connection is down. This needs an
API change also.
* it'd be nice to quickly double-click on a stream to view only it,
then double-click again to go back to the multi-pane view.
* it doesn't allow you to zoom in on part of the video. This would be
nice particularly when viewing 4k video streams on small screens.
* it has only four preconfigured layouts that subdivide a 16x9
viewport. You have to choose every camera every time. It'd be nice
to both allow more flexibility and have more memory.
React prototype: #111
live stream: #59
This commit is contained in:
Scott Lamb
parent
2215032a78
commit
0236ab8d64
37
ui/package-lock.json
generated
37
ui/package-lock.json
generated
@ -2123,6 +2123,28 @@
|
||||
"resolved": "https://registry.npmjs.org/@popperjs/core/-/core-2.9.0.tgz",
|
||||
"integrity": "sha512-wjtKehFAIARq2OxK8j3JrggNlEslJfNuSm2ArteIbKyRMts2g0a7KzTxfRVNUM+O0gnBJ2hNV8nWPOYBgI1sew=="
|
||||
},
|
||||
"@react-hook/latest": {
|
||||
"version": "1.0.3",
|
||||
"resolved": "https://registry.npmjs.org/@react-hook/latest/-/latest-1.0.3.tgz",
|
||||
"integrity": "sha512-dy6duzl+JnAZcDbNTfmaP3xHiKtbXYOaz3G51MGVljh548Y8MWzTr+PHLOfvpypEVW9zwvl+VyKjbWKEVbV1Rg=="
|
||||
},
|
||||
"@react-hook/passive-layout-effect": {
|
||||
"version": "1.2.1",
|
||||
"resolved": "https://registry.npmjs.org/@react-hook/passive-layout-effect/-/passive-layout-effect-1.2.1.tgz",
|
||||
"integrity": "sha512-IwEphTD75liO8g+6taS+4oqz+nnroocNfWVHWz7j+N+ZO2vYrc6PV1q7GQhuahL0IOR7JccFTsFKQ/mb6iZWAg=="
|
||||
},
|
||||
"@react-hook/resize-observer": {
|
||||
"version": "1.2.0",
|
||||
"resolved": "https://registry.npmjs.org/@react-hook/resize-observer/-/resize-observer-1.2.0.tgz",
|
||||
"integrity": "sha512-7Cpy0aaZ3xXlSabQ43aZcfgzwYSidrshEKGDqpfvdx7pHnGDGskr8m1Ajb6yamJUSdTRgqCemYQcwouCqMmZoQ==",
|
||||
"requires": {
|
||||
"@react-hook/latest": "^1.0.2",
|
||||
"@react-hook/passive-layout-effect": "^1.2.0",
|
||||
"@types/raf-schd": "^4.0.0",
|
||||
"raf-schd": "^4.0.2",
|
||||
"resize-observer-polyfill": "^1.5.1"
|
||||
}
|
||||
},
|
||||
"@rollup/plugin-node-resolve": {
|
||||
"version": "7.1.3",
|
||||
"resolved": "https://registry.npmjs.org/@rollup/plugin-node-resolve/-/plugin-node-resolve-7.1.3.tgz",
|
||||
@ -2683,6 +2705,11 @@
|
||||
"resolved": "https://registry.npmjs.org/@types/q/-/q-1.5.4.tgz",
|
||||
"integrity": "sha512-1HcDas8SEj4z1Wc696tH56G8OlRaH/sqZOynNNB+HF0WOeXPaxTtbYzJY2oEfiUxjSKjhCKr+MvR7dCHcEelug=="
|
||||
},
|
||||
"@types/raf-schd": {
|
||||
"version": "4.0.0",
|
||||
"resolved": "https://registry.npmjs.org/@types/raf-schd/-/raf-schd-4.0.0.tgz",
|
||||
"integrity": "sha512-EsXE+pu4MjOhU+H2Ut/8zOGUtictm87anwxOcNY1HjxkH8ipLR+SzYnCzT4lQvyKJdcMwIGxhb8w/KZhOy6vaQ=="
|
||||
},
|
||||
"@types/react": {
|
||||
"version": "17.0.2",
|
||||
"resolved": "https://registry.npmjs.org/@types/react/-/react-17.0.2.tgz",
|
||||
@ -12978,6 +13005,11 @@
|
||||
"performance-now": "^2.1.0"
|
||||
}
|
||||
},
|
||||
"raf-schd": {
|
||||
"version": "4.0.2",
|
||||
"resolved": "https://registry.npmjs.org/raf-schd/-/raf-schd-4.0.2.tgz",
|
||||
"integrity": "sha512-VhlMZmGy6A6hrkJWHLNTGl5gtgMUm+xfGza6wbwnE914yeQ5Ybm18vgM734RZhMgfw4tacUrWseGZlpUrrakEQ=="
|
||||
},
|
||||
"randombytes": {
|
||||
"version": "2.1.0",
|
||||
"resolved": "https://registry.npmjs.org/randombytes/-/randombytes-2.1.0.tgz",
|
||||
@ -13600,6 +13632,11 @@
|
||||
"resolved": "https://registry.npmjs.org/requires-port/-/requires-port-1.0.0.tgz",
|
||||
"integrity": "sha1-kl0mAdOaxIXgkc8NpcbmlNw9yv8="
|
||||
},
|
||||
"resize-observer-polyfill": {
|
||||
"version": "1.5.1",
|
||||
"resolved": "https://registry.npmjs.org/resize-observer-polyfill/-/resize-observer-polyfill-1.5.1.tgz",
|
||||
"integrity": "sha512-LwZrotdHOo12nQuZlHEmtuXdqGoOD0OhaxopaNFxWzInpEgaLWoVuAMbTzixuosCx2nEG58ngzW3vxdWoxIgdg=="
|
||||
},
|
||||
"resolve": {
|
||||
"version": "1.20.0",
|
||||
"resolved": "https://registry.npmjs.org/resolve/-/resolve-1.20.0.tgz",
|
||||
|
||||
@ -9,6 +9,7 @@
|
||||
"@material-ui/core": "^5.0.0-alpha.26",
|
||||
"@material-ui/icons": "^5.0.0-alpha.26",
|
||||
"@material-ui/lab": "^5.0.0-alpha.26",
|
||||
"@react-hook/resize-observer": "^1.2.0",
|
||||
"@types/jest": "^26.0.20",
|
||||
"@types/node": "^14.14.22",
|
||||
"@types/react": "^17.0.0",
|
||||
|
||||
@ -41,6 +41,6 @@
|
||||
</head>
|
||||
<body>
|
||||
<noscript>You need to enable JavaScript to run this app.</noscript>
|
||||
<div id="root"></div>
|
||||
<div id="root" style="display: flex; flex-direction: column"></div>
|
||||
</body>
|
||||
</html>
|
||||
|
||||
104
ui/src/App.tsx
104
ui/src/App.tsx
@ -9,26 +9,50 @@ import MoonfireMenu from "./AppMenu";
|
||||
import Login from "./Login";
|
||||
import { useSnackbars } from "./snackbars";
|
||||
import { Camera, Session } from "./types";
|
||||
import List from "./List";
|
||||
import ListActivity from "./List";
|
||||
import AppBar from "@material-ui/core/AppBar";
|
||||
import LiveActivity, { MultiviewChooser } from "./Live";
|
||||
import Drawer from "@material-ui/core/Drawer";
|
||||
import List from "@material-ui/core/List";
|
||||
import ListItem from "@material-ui/core/ListItem";
|
||||
import ListItemText from "@material-ui/core/ListItemText";
|
||||
import ListIcon from "@material-ui/icons/List";
|
||||
import Videocam from "@material-ui/icons/Videocam";
|
||||
import ListItemIcon from "@material-ui/core/ListItemIcon";
|
||||
import FilterList from "@material-ui/icons/FilterList";
|
||||
import IconButton from "@material-ui/core/IconButton";
|
||||
|
||||
type LoginState =
|
||||
export type LoginState =
|
||||
| "unknown"
|
||||
| "logged-in"
|
||||
| "not-logged-in"
|
||||
| "server-requires-login"
|
||||
| "user-requested-login";
|
||||
|
||||
type Activity = "list" | "live";
|
||||
|
||||
function App() {
|
||||
const [showMenu, toggleShowMenu] = useReducer((m: boolean) => !m, true);
|
||||
const [showMenu, toggleShowMenu] = useReducer((m: boolean) => !m, false);
|
||||
const [showListSelectors, toggleShowListSelectors] = useReducer(
|
||||
(m: boolean) => !m,
|
||||
true
|
||||
);
|
||||
const [activity, setActivity] = useState<Activity>("list");
|
||||
const [multiviewLayoutIndex, setMultiviewLayoutIndex] = useState(0);
|
||||
const [session, setSession] = useState<Session | null>(null);
|
||||
const [cameras, setCameras] = useState<Camera[] | null>(null);
|
||||
const [timeZoneName, setTimeZoneName] = useState<string | null>(null);
|
||||
const [fetchSeq, setFetchSeq] = useState(0);
|
||||
const [loginState, setLoginState] = useState<LoginState>("not-logged-in");
|
||||
const [loginState, setLoginState] = useState<LoginState>("unknown");
|
||||
const [error, setError] = useState<api.FetchError | null>(null);
|
||||
const needNewFetch = () => setFetchSeq((seq) => seq + 1);
|
||||
const snackbars = useSnackbars();
|
||||
|
||||
const clickActivity = (activity: Activity) => {
|
||||
toggleShowMenu();
|
||||
setActivity(activity);
|
||||
};
|
||||
|
||||
const onLoginSuccess = () => {
|
||||
setLoginState("logged-in");
|
||||
needNewFetch();
|
||||
@ -85,19 +109,79 @@ function App() {
|
||||
abort.abort();
|
||||
};
|
||||
}, [fetchSeq]);
|
||||
let activityMenu = null;
|
||||
let activityMain = null;
|
||||
if (error === null && cameras !== null && cameras.length > 0) {
|
||||
switch (activity) {
|
||||
case "list":
|
||||
activityMenu = (
|
||||
<IconButton
|
||||
aria-label="selectors"
|
||||
onClick={toggleShowListSelectors}
|
||||
color="inherit"
|
||||
size="small"
|
||||
>
|
||||
<FilterList />
|
||||
</IconButton>
|
||||
);
|
||||
activityMain = (
|
||||
<ListActivity
|
||||
cameras={cameras}
|
||||
showSelectors={showListSelectors}
|
||||
timeZoneName={timeZoneName!}
|
||||
/>
|
||||
);
|
||||
break;
|
||||
case "live":
|
||||
activityMenu = (
|
||||
<MultiviewChooser
|
||||
layoutIndex={multiviewLayoutIndex}
|
||||
onChoice={setMultiviewLayoutIndex}
|
||||
/>
|
||||
);
|
||||
activityMain = (
|
||||
<LiveActivity cameras={cameras} layoutIndex={multiviewLayoutIndex} />
|
||||
);
|
||||
break;
|
||||
}
|
||||
}
|
||||
return (
|
||||
<>
|
||||
<AppBar position="static">
|
||||
<MoonfireMenu
|
||||
session={session}
|
||||
loginState={loginState}
|
||||
setSession={setSession}
|
||||
requestLogin={() => {
|
||||
setLoginState("user-requested-login");
|
||||
}}
|
||||
logout={logout}
|
||||
menuClick={toggleShowMenu}
|
||||
activityMenuPart={activityMenu}
|
||||
/>
|
||||
</AppBar>
|
||||
<Drawer
|
||||
variant="temporary"
|
||||
open={showMenu}
|
||||
onClose={toggleShowMenu}
|
||||
ModalProps={{
|
||||
keepMounted: true,
|
||||
}}
|
||||
>
|
||||
<List>
|
||||
<ListItem button key="list" onClick={() => clickActivity("list")}>
|
||||
<ListItemIcon>
|
||||
<ListIcon />
|
||||
</ListItemIcon>
|
||||
<ListItemText primary="List view" />
|
||||
</ListItem>
|
||||
<ListItem button key="live" onClick={() => clickActivity("live")}>
|
||||
<ListItemIcon>
|
||||
<Videocam />
|
||||
</ListItemIcon>
|
||||
<ListItemText primary="Live view (experimental)" />
|
||||
</ListItem>
|
||||
</List>
|
||||
</Drawer>
|
||||
<Login
|
||||
onSuccess={onLoginSuccess}
|
||||
open={
|
||||
@ -110,7 +194,7 @@ function App() {
|
||||
);
|
||||
}}
|
||||
/>
|
||||
{error != null && (
|
||||
{error !== null && (
|
||||
<Container>
|
||||
<h2>Error querying server</h2>
|
||||
<pre>{error.message}</pre>
|
||||
@ -120,13 +204,7 @@ function App() {
|
||||
</p>
|
||||
</Container>
|
||||
)}
|
||||
{cameras != null && cameras.length > 0 && (
|
||||
<List
|
||||
cameras={cameras}
|
||||
showMenu={showMenu}
|
||||
timeZoneName={timeZoneName!}
|
||||
/>
|
||||
)}
|
||||
{activityMain}
|
||||
</>
|
||||
);
|
||||
}
|
||||
|
||||
@ -12,6 +12,7 @@ import Typography from "@material-ui/core/Typography";
|
||||
import AccountCircle from "@material-ui/icons/AccountCircle";
|
||||
import MenuIcon from "@material-ui/icons/Menu";
|
||||
import React from "react";
|
||||
import { LoginState } from "./App";
|
||||
import { Session } from "./types";
|
||||
|
||||
const useStyles = makeStyles((theme: Theme) =>
|
||||
@ -19,21 +20,24 @@ const useStyles = makeStyles((theme: Theme) =>
|
||||
title: {
|
||||
flexGrow: 1,
|
||||
},
|
||||
activity: {
|
||||
marginRight: theme.spacing(2),
|
||||
},
|
||||
})
|
||||
);
|
||||
|
||||
interface Props {
|
||||
session: Session | null;
|
||||
loginState: LoginState;
|
||||
setSession: (session: Session | null) => void;
|
||||
requestLogin: () => void;
|
||||
logout: () => void;
|
||||
menuClick?: () => void;
|
||||
activityMenuPart: JSX.Element | null;
|
||||
}
|
||||
|
||||
// https://material-ui.com/components/app-bar/
|
||||
function MoonfireMenu(props: Props) {
|
||||
const classes = useStyles();
|
||||
const auth = props.session !== null;
|
||||
const [
|
||||
accountMenuAnchor,
|
||||
setAccountMenuAnchor,
|
||||
@ -68,12 +72,15 @@ function MoonfireMenu(props: Props) {
|
||||
<Typography variant="h6" className={classes.title}>
|
||||
Moonfire NVR
|
||||
</Typography>
|
||||
{auth || (
|
||||
{props.activityMenuPart !== null && (
|
||||
<div className={classes.activity}>{props.activityMenuPart}</div>
|
||||
)}
|
||||
{props.loginState !== "unknown" && props.loginState !== "logged-in" && (
|
||||
<Button color="inherit" onClick={props.requestLogin}>
|
||||
Log in
|
||||
</Button>
|
||||
)}
|
||||
{auth && (
|
||||
{props.loginState === "logged-in" && (
|
||||
<div>
|
||||
<IconButton
|
||||
aria-label="account of current user"
|
||||
|
||||
@ -73,10 +73,10 @@ const useStyles = makeStyles((theme: Theme) => ({
|
||||
interface Props {
|
||||
timeZoneName: string;
|
||||
cameras: Camera[];
|
||||
showMenu: boolean;
|
||||
showSelectors: boolean;
|
||||
}
|
||||
|
||||
const Main = ({ cameras, timeZoneName, showMenu }: Props) => {
|
||||
const Main = ({ cameras, timeZoneName, showSelectors }: Props) => {
|
||||
const classes = useStyles();
|
||||
|
||||
/**
|
||||
@ -134,7 +134,7 @@ const Main = ({ cameras, timeZoneName, showMenu }: Props) => {
|
||||
<div className={classes.root}>
|
||||
<Box
|
||||
className={classes.selectors}
|
||||
sx={{ display: showMenu ? "block" : "none" }}
|
||||
sx={{ display: showSelectors ? "block" : "none" }}
|
||||
>
|
||||
<StreamMultiSelector
|
||||
cameras={cameras}
|
||||
|
||||
363
ui/src/Live/LiveCamera.tsx
Normal file
363
ui/src/Live/LiveCamera.tsx
Normal file
@ -0,0 +1,363 @@
|
||||
// This file is part of Moonfire NVR, a security camera network video recorder.
|
||||
// Copyright (C) 2021 The Moonfire NVR Authors; see AUTHORS and LICENSE.txt.
|
||||
// SPDX-License-Identifier: GPL-v3.0-or-later WITH GPL-3.0-linking-exception
|
||||
|
||||
import React, { SyntheticEvent } from "react";
|
||||
import { Camera } from "../types";
|
||||
import { Part, parsePart } from "./parser";
|
||||
import * as api from "../api";
|
||||
import Box from "@material-ui/core/Box";
|
||||
import CircularProgress from "@material-ui/core/CircularProgress";
|
||||
import Alert from "@material-ui/core/Alert";
|
||||
|
||||
interface LiveCameraProps {
|
||||
camera: Camera;
|
||||
}
|
||||
|
||||
interface BufferStateClosed {
|
||||
state: "closed";
|
||||
}
|
||||
|
||||
interface BufferStateOpen {
|
||||
state: "open";
|
||||
srcBuf: SourceBuffer;
|
||||
busy: boolean;
|
||||
mimeType: string;
|
||||
videoSampleEntryId: number;
|
||||
}
|
||||
|
||||
interface BufferStateError {
|
||||
state: "error";
|
||||
}
|
||||
|
||||
type BufferState = BufferStateClosed | BufferStateOpen | BufferStateError;
|
||||
|
||||
interface PlaybackStateNormal {
|
||||
state: "normal";
|
||||
}
|
||||
|
||||
interface PlaybackStateWaiting {
|
||||
state: "waiting";
|
||||
}
|
||||
|
||||
interface PlaybackStateError {
|
||||
state: "error";
|
||||
message: string;
|
||||
}
|
||||
|
||||
type PlaybackState =
|
||||
| PlaybackStateNormal
|
||||
| PlaybackStateWaiting
|
||||
| PlaybackStateError;
|
||||
|
||||
/**
|
||||
* Drives a live camera.
|
||||
* Implementation detail of LiveCamera which listens to various DOM events and
|
||||
* drives the WebSocket feed and the MediaSource and SourceBuffers.
|
||||
*/
|
||||
class LiveCameraDriver {
|
||||
constructor(
|
||||
camera: Camera,
|
||||
setPlaybackState: (state: PlaybackState) => void,
|
||||
videoRef: React.RefObject<HTMLVideoElement>
|
||||
) {
|
||||
this.camera = camera;
|
||||
this.setPlaybackState = setPlaybackState;
|
||||
this.videoRef = videoRef;
|
||||
this.src.addEventListener("sourceopen", this.onMediaSourceOpen);
|
||||
}
|
||||
|
||||
onMediaSourceOpen = () => {
|
||||
this.startStream("sourceopen");
|
||||
};
|
||||
|
||||
startStream = (reason: string) => {
|
||||
if (this.ws !== undefined) {
|
||||
return;
|
||||
}
|
||||
console.log(`${this.camera.shortName}: starting stream: ${reason}`);
|
||||
const loc = window.location;
|
||||
const proto = loc.protocol === "https:" ? "wss" : "ws";
|
||||
|
||||
// TODO: switch between sub and main based on window size/bandwidth.
|
||||
const url = `${proto}://${loc.host}/api/cameras/${this.camera.uuid}/sub/live.m4s`;
|
||||
this.ws = new WebSocket(url);
|
||||
this.ws.addEventListener("close", this.onWsClose);
|
||||
this.ws.addEventListener("error", this.onWsError);
|
||||
this.ws.addEventListener("message", this.onWsMessage);
|
||||
};
|
||||
|
||||
error = (reason: string) => {
|
||||
console.error(`${this.camera.shortName}: aborting due to ${reason}`);
|
||||
this.stopStream(reason);
|
||||
this.buf = { state: "error" };
|
||||
this.src.endOfStream("network");
|
||||
this.setPlaybackState({ state: "error", message: reason });
|
||||
};
|
||||
|
||||
onWsClose = (e: CloseEvent) => {
|
||||
this.error(`ws close: ${e.code} ${e.reason}`);
|
||||
};
|
||||
|
||||
onWsError = (_e: Event) => {
|
||||
this.error("ws error");
|
||||
};
|
||||
|
||||
onWsMessage = async (e: MessageEvent) => {
|
||||
let raw;
|
||||
try {
|
||||
raw = new Uint8Array(await e.data.arrayBuffer());
|
||||
} catch (e) {
|
||||
this.error(`error reading part: ${e.message}`);
|
||||
return;
|
||||
}
|
||||
if (this.buf.state === "error") {
|
||||
console.log("onWsMessage while in state error");
|
||||
return;
|
||||
}
|
||||
let result = parsePart(raw);
|
||||
if (result.status === "error") {
|
||||
this.error("unparseable part");
|
||||
return;
|
||||
}
|
||||
const part = result.part;
|
||||
if (!MediaSource.isTypeSupported(part.mimeType)) {
|
||||
this.error(`unsupported mime type ${part.mimeType}`);
|
||||
return;
|
||||
}
|
||||
|
||||
this.queue.push(part);
|
||||
this.queuedBytes += part.body.byteLength;
|
||||
if (this.buf.state === "closed") {
|
||||
const srcBuf = this.src.addSourceBuffer(part.mimeType);
|
||||
srcBuf.mode = "segments";
|
||||
srcBuf.addEventListener("updateend", this.bufUpdateEnd);
|
||||
srcBuf.addEventListener("error", this.bufEvent);
|
||||
srcBuf.addEventListener("abort", this.bufEvent);
|
||||
this.buf = {
|
||||
state: "open",
|
||||
srcBuf,
|
||||
busy: true,
|
||||
mimeType: part.mimeType,
|
||||
videoSampleEntryId: part.videoSampleEntryId,
|
||||
};
|
||||
let initSegmentResult = await api.init(part.videoSampleEntryId, {});
|
||||
if (initSegmentResult.status !== "success") {
|
||||
this.error(`init segment fetch status ${initSegmentResult.status}`);
|
||||
return;
|
||||
}
|
||||
srcBuf.appendBuffer(initSegmentResult.response);
|
||||
return;
|
||||
} else if (this.buf.state === "open") {
|
||||
this.tryAppendPart(this.buf);
|
||||
}
|
||||
};
|
||||
|
||||
bufUpdateEnd = () => {
|
||||
if (this.buf.state !== "open") {
|
||||
console.error("bufUpdateEnd in state", this.buf.state);
|
||||
return;
|
||||
}
|
||||
if (!this.buf.busy) {
|
||||
this.error("bufUpdateEnd when not busy");
|
||||
return;
|
||||
}
|
||||
this.buf.busy = false;
|
||||
this.tryTrimBuffer();
|
||||
this.tryAppendPart(this.buf);
|
||||
};
|
||||
|
||||
tryAppendPart = (buf: BufferStateOpen) => {
|
||||
if (buf.busy) {
|
||||
return;
|
||||
}
|
||||
|
||||
const part = this.queue.shift();
|
||||
if (part === undefined) {
|
||||
return;
|
||||
}
|
||||
this.queuedBytes -= part.body.byteLength;
|
||||
|
||||
if (
|
||||
part.mimeType !== buf.mimeType ||
|
||||
part.videoSampleEntryId !== buf.videoSampleEntryId
|
||||
) {
|
||||
this.error("Switching MIME type or videoSampleEntryId unimplemented");
|
||||
return;
|
||||
}
|
||||
|
||||
// Always put the new part at the end. SourceBuffer.mode "sequence" is
|
||||
// supposed to generate timestamps automatically, but on Chrome 89.0.4389.90
|
||||
// it doesn't appear to work as expected. So use SourceBuffer.mode
|
||||
// "segments" and use the existing end as the timestampOffset.
|
||||
const b = buf.srcBuf.buffered;
|
||||
buf.srcBuf.timestampOffset = b.length > 0 ? b.end(b.length - 1) : 0;
|
||||
|
||||
try {
|
||||
buf.srcBuf.appendBuffer(part.body);
|
||||
} catch (e) {
|
||||
// In particular, appendBuffer can throw QuotaExceededError.
|
||||
// <https://developers.google.com/web/updates/2017/10/quotaexceedederror>
|
||||
// tryTrimBuffer removes already-played stuff from the buffer to avoid
|
||||
// this, but in theory even one GOP could be more than the total buffer
|
||||
// size. At least report error properly.
|
||||
this.error(`${e.name} while appending buffer`);
|
||||
return;
|
||||
}
|
||||
buf.busy = true;
|
||||
};
|
||||
|
||||
tryTrimBuffer = () => {
|
||||
if (
|
||||
this.buf.state !== "open" ||
|
||||
this.buf.busy ||
|
||||
this.buf.srcBuf.buffered.length === 0 ||
|
||||
this.videoRef.current === null
|
||||
) {
|
||||
return;
|
||||
}
|
||||
const curTs = this.videoRef.current.currentTime;
|
||||
|
||||
// TODO: call out key frames in the part headers. The "- 5" here is a guess
|
||||
// to avoid removing anything from the current GOP.
|
||||
const firstTs = this.buf.srcBuf.buffered.start(0);
|
||||
if (firstTs < curTs - 5) {
|
||||
console.log(`${this.camera.shortName}: trimming ${firstTs}-${curTs}`);
|
||||
this.buf.srcBuf.remove(firstTs, curTs - 5);
|
||||
this.buf.busy = true;
|
||||
}
|
||||
};
|
||||
|
||||
bufEvent = (e: Event) => {
|
||||
this.error(`bufEvent: ${e}`);
|
||||
};
|
||||
|
||||
videoPlaying = (e: SyntheticEvent<HTMLVideoElement, Event>) => {
|
||||
if (this.buf.state !== "error") {
|
||||
this.setPlaybackState({ state: "normal" });
|
||||
}
|
||||
};
|
||||
|
||||
videoWaiting = (e: SyntheticEvent<HTMLVideoElement, Event>) => {
|
||||
if (this.buf.state !== "error") {
|
||||
this.setPlaybackState({ state: "waiting" });
|
||||
}
|
||||
};
|
||||
|
||||
stopStream = (reason: string) => {
|
||||
if (this.ws === undefined) {
|
||||
return;
|
||||
}
|
||||
console.log(`${this.camera.shortName}: stopping stream: ${reason}`);
|
||||
const NORMAL_CLOSURE = 1000; // https://developer.mozilla.org/en-US/docs/Web/API/CloseEvent
|
||||
this.ws.close(NORMAL_CLOSURE);
|
||||
this.ws.removeEventListener("close", this.onWsClose);
|
||||
this.ws.removeEventListener("error", this.onWsError);
|
||||
this.ws.removeEventListener("message", this.onWsMessage);
|
||||
this.ws = undefined;
|
||||
};
|
||||
|
||||
camera: Camera;
|
||||
setPlaybackState: (state: PlaybackState) => void;
|
||||
videoRef: React.RefObject<HTMLVideoElement>;
|
||||
|
||||
src = new MediaSource();
|
||||
buf: BufferState = { state: "closed" };
|
||||
queue: Part[] = [];
|
||||
queuedBytes: number = 0;
|
||||
|
||||
/// The object URL for the HTML video element, not the WebSocket URL.
|
||||
url = URL.createObjectURL(this.src);
|
||||
|
||||
ws?: WebSocket;
|
||||
}
|
||||
|
||||
/**
|
||||
* A live view of a camera.
|
||||
* Note there's a significant setup cost to creating a LiveCamera, so the parent
|
||||
* should use React's <tt>key</tt> attribute to avoid unnecessarily mounting
|
||||
* and unmounting a camera.
|
||||
*/
|
||||
const LiveCamera = ({ camera }: LiveCameraProps) => {
|
||||
const videoRef = React.useRef<HTMLVideoElement>(null);
|
||||
const [playbackState, setPlaybackState] = React.useState<PlaybackState>({
|
||||
state: "normal",
|
||||
});
|
||||
|
||||
// Load the camera driver.
|
||||
const [driver, setDriver] = React.useState<LiveCameraDriver | null>(null);
|
||||
React.useEffect(() => {
|
||||
const d = new LiveCameraDriver(camera, setPlaybackState, videoRef);
|
||||
setDriver(d);
|
||||
return () => {
|
||||
// Explictly stop the stream on unmount. There don't seem to be any DOM
|
||||
// event handlers that run in this case. (In particular, the MediaSource's
|
||||
// sourceclose doesn't run.)
|
||||
d.stopStream("unmount or camera change");
|
||||
};
|
||||
}, [camera]);
|
||||
|
||||
// Display circular progress after 100 ms of waiting.
|
||||
const [showProgress, setShowProgress] = React.useState(false);
|
||||
React.useEffect(() => {
|
||||
setShowProgress(false);
|
||||
if (playbackState.state !== "waiting") {
|
||||
return;
|
||||
}
|
||||
const timerId = setTimeout(() => setShowProgress(true), 100);
|
||||
return () => clearTimeout(timerId);
|
||||
}, [playbackState]);
|
||||
|
||||
if (driver === null) {
|
||||
return <Box />;
|
||||
}
|
||||
return (
|
||||
<Box
|
||||
sx={{
|
||||
"& video": { width: "100%", height: "100%", objectFit: "contain" },
|
||||
"& .progress-overlay": {
|
||||
position: "absolute",
|
||||
display: "flex",
|
||||
alignItems: "center",
|
||||
justifyContent: "center",
|
||||
height: "100%",
|
||||
width: "100%",
|
||||
zIndex: 1,
|
||||
},
|
||||
"& .alert-overlay": {
|
||||
position: "absolute",
|
||||
display: "flex",
|
||||
height: "100%",
|
||||
width: "100%",
|
||||
alignItems: "flex-end",
|
||||
zIndex: 1,
|
||||
p: 1,
|
||||
},
|
||||
}}
|
||||
>
|
||||
{showProgress && (
|
||||
<div className="progress-overlay">
|
||||
<CircularProgress />
|
||||
</div>
|
||||
)}
|
||||
{playbackState.state === "error" && (
|
||||
<div className="alert-overlay">
|
||||
<Alert severity="error">{playbackState.message}</Alert>
|
||||
</div>
|
||||
)}
|
||||
<video
|
||||
ref={videoRef}
|
||||
muted
|
||||
autoPlay
|
||||
src={driver.url}
|
||||
onPause={() => driver.stopStream("pause")}
|
||||
onPlay={() => driver.startStream("play")}
|
||||
onPlaying={driver.videoPlaying}
|
||||
onTimeUpdate={driver.tryTrimBuffer}
|
||||
onWaiting={driver.videoWaiting}
|
||||
/>
|
||||
</Box>
|
||||
);
|
||||
};
|
||||
|
||||
export default LiveCamera;
|
||||
258
ui/src/Live/Multiview.tsx
Normal file
258
ui/src/Live/Multiview.tsx
Normal file
@ -0,0 +1,258 @@
|
||||
// This file is part of Moonfire NVR, a security camera network video recorder.
|
||||
// Copyright (C) 2021 The Moonfire NVR Authors; see AUTHORS and LICENSE.txt.
|
||||
// SPDX-License-Identifier: GPL-v3.0-or-later WITH GPL-3.0-linking-exception
|
||||
|
||||
import Select from "@material-ui/core/Select";
|
||||
import MenuItem from "@material-ui/core/MenuItem";
|
||||
import React, { useReducer, useState } from "react";
|
||||
import { Camera } from "../types";
|
||||
import { makeStyles } from "@material-ui/core/styles";
|
||||
import useResizeObserver from "@react-hook/resize-observer";
|
||||
import Box from "@material-ui/core/Box";
|
||||
|
||||
export interface Layout {
|
||||
className: string;
|
||||
cameras: number;
|
||||
}
|
||||
|
||||
// These class names must match useStyles rules (below).
|
||||
const LAYOUTS: Layout[] = [
|
||||
{ className: "solo", cameras: 1 },
|
||||
{ className: "main-plus-five", cameras: 6 },
|
||||
{ className: "two-by-two", cameras: 4 },
|
||||
{ className: "three-by-three", cameras: 9 },
|
||||
];
|
||||
const MAX_CAMERAS = 9;
|
||||
|
||||
const useStyles = makeStyles((theme) => ({
|
||||
root: {
|
||||
flex: "1 0 0",
|
||||
overflow: "hidden",
|
||||
color: "white",
|
||||
marginTop: theme.spacing(2),
|
||||
},
|
||||
mid: {
|
||||
display: "none",
|
||||
position: "relative",
|
||||
padding: 0,
|
||||
margin: 0,
|
||||
"&.wider, &.wider img": {
|
||||
height: "100%",
|
||||
display: "inline-block",
|
||||
},
|
||||
"&.taller, &.taller img": {
|
||||
width: "100%",
|
||||
display: "inline-block",
|
||||
},
|
||||
"& img": {
|
||||
objectFit: "contain",
|
||||
},
|
||||
},
|
||||
inner: {
|
||||
// match parent's size without influencing it.
|
||||
overflow: "hidden",
|
||||
position: "absolute",
|
||||
top: 0,
|
||||
bottom: 0,
|
||||
left: 0,
|
||||
right: 0,
|
||||
|
||||
display: "grid",
|
||||
gridGap: "0px",
|
||||
|
||||
// These class names must match LAYOUTS (above).
|
||||
"&.solo": {
|
||||
gridTemplateColumns: "100%",
|
||||
gridTemplateRows: "100%",
|
||||
},
|
||||
"&.two-by-two": {
|
||||
gridTemplateColumns: "repeat(2, calc(100% / 2))",
|
||||
gridTemplateRows: "repeat(2, calc(100% / 2))",
|
||||
},
|
||||
"&.main-plus-five, &.three-by-three": {
|
||||
gridTemplateColumns: "repeat(3, calc(100% / 3))",
|
||||
gridTemplateRows: "repeat(3, calc(100% / 3))",
|
||||
},
|
||||
"&.main-plus-five > div:nth-child(1)": {
|
||||
gridColumn: "span 2",
|
||||
gridRow: "span 2",
|
||||
},
|
||||
},
|
||||
}));
|
||||
|
||||
export interface MultiviewProps {
|
||||
cameras: Camera[];
|
||||
layoutIndex: number;
|
||||
renderCamera: (camera: Camera) => JSX.Element;
|
||||
}
|
||||
|
||||
export interface MultiviewChooserProps {
|
||||
/// An index into <tt>LAYOUTS</tt>.
|
||||
layoutIndex: number;
|
||||
onChoice: (selectedIndex: number) => void;
|
||||
}
|
||||
|
||||
/**
|
||||
* Chooses the layout for a Multiview.
|
||||
* Styled for placement in the app menu bar.
|
||||
*/
|
||||
export const MultiviewChooser = (props: MultiviewChooserProps) => {
|
||||
return (
|
||||
<Select
|
||||
id="layout"
|
||||
value={props.layoutIndex}
|
||||
onChange={(e) => props.onChoice(e.target.value)}
|
||||
size="small"
|
||||
sx={{
|
||||
// Hacky attempt to style for the app menu.
|
||||
color: "inherit",
|
||||
"& svg": {
|
||||
color: "inherit",
|
||||
},
|
||||
}}
|
||||
>
|
||||
{LAYOUTS.map((e, i) => (
|
||||
<MenuItem key={e.className} value={i}>
|
||||
{e.className}
|
||||
</MenuItem>
|
||||
))}
|
||||
</Select>
|
||||
);
|
||||
};
|
||||
|
||||
/**
|
||||
* The cameras selected for the multiview.
|
||||
* This is always an array of length <tt>MAX_CAMERAS</tt>; only the first
|
||||
* LAYOUTS[layoutIndex].cameras are currently visible. There are no duplicates;
|
||||
* setting one element to a given camera unsets any others pointing to the same
|
||||
* camera.
|
||||
*/
|
||||
type SelectedCameras = Array<number | null>;
|
||||
|
||||
interface SelectOp {
|
||||
selectedIndex: number;
|
||||
cameraIndex: number | null;
|
||||
}
|
||||
|
||||
function selectedReducer(old: SelectedCameras, op: SelectOp): SelectedCameras {
|
||||
let selected = [...old]; // shallow clone.
|
||||
if (op.cameraIndex !== null) {
|
||||
// de-dupe.
|
||||
for (let i = 0; i < selected.length; i++) {
|
||||
if (selected[i] === op.cameraIndex) {
|
||||
selected[i] = null;
|
||||
}
|
||||
}
|
||||
}
|
||||
selected[op.selectedIndex] = op.cameraIndex ?? null;
|
||||
return selected;
|
||||
}
|
||||
|
||||
/**
|
||||
* Presents one or more camera views in one of several layouts.
|
||||
*
|
||||
* The parent should arrange for the multiview's outer div to be as large
|
||||
* as possible. Internally, multiview uses the largest possible aspect
|
||||
* ratio-constrained section of it. It uses a ResizeObserver to determine if
|
||||
* the outer div is wider or taller than 16x9, and then sets an appropriate CSS
|
||||
* class to constrain the width or height respectively using a technique like
|
||||
* <https://stackoverflow.com/a/14911949/23584>. The goal is to have the
|
||||
* smoothest resizing by changing the DOM/CSS as little as possible.
|
||||
*/
|
||||
const Multiview = (props: MultiviewProps) => {
|
||||
const [selected, updateSelected] = useReducer(
|
||||
selectedReducer,
|
||||
Array(MAX_CAMERAS).fill(null)
|
||||
);
|
||||
const [widerOrTaller, setWiderOrTaller] = useState("wider");
|
||||
const outerRef = React.useRef<HTMLDivElement>(null);
|
||||
useResizeObserver(outerRef, (entry: ResizeObserverEntry) => {
|
||||
const w = entry.contentRect.width;
|
||||
const h = entry.contentRect.height;
|
||||
setWiderOrTaller((w * 9) / 16 > h ? "wider" : "taller");
|
||||
});
|
||||
const classes = useStyles();
|
||||
const layout = LAYOUTS[props.layoutIndex];
|
||||
const monoviews = selected.slice(0, layout.cameras).map((e, i) => {
|
||||
// When a camera is selected, use the camera's index as the key.
|
||||
// This allows swapping cameras' positions without tearing down their
|
||||
// WebSocket connections and buffers.
|
||||
//
|
||||
// When no camera is selected, use the index within selected. (Actually,
|
||||
// its negation, to disambiguate between the two cases.)
|
||||
const key = e ?? -i;
|
||||
return (
|
||||
<Monoview
|
||||
key={key}
|
||||
cameras={props.cameras}
|
||||
cameraIndex={e}
|
||||
renderCamera={props.renderCamera}
|
||||
onSelect={(cameraIndex) =>
|
||||
updateSelected({ selectedIndex: i, cameraIndex })
|
||||
}
|
||||
/>
|
||||
);
|
||||
});
|
||||
return (
|
||||
<div className={classes.root} ref={outerRef}>
|
||||
<div className={`${classes.mid} ${widerOrTaller}`}>
|
||||
{/* a 16x9 black png from png-pixel.com */}
|
||||
<img
|
||||
src="data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAABAAAAAJCAQAAACRI2S5AAAAEklEQVR42mNk+M+AFzCOKgADALyGCQGyq8YeAAAAAElFTkSuQmCC"
|
||||
alt=""
|
||||
/>
|
||||
<div className={`${classes.inner} ${layout.className}`}>
|
||||
{monoviews}
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
};
|
||||
|
||||
interface MonoviewProps {
|
||||
cameras: Camera[];
|
||||
cameraIndex: number | null;
|
||||
onSelect: (cameraIndex: number | null) => void;
|
||||
renderCamera: (camera: Camera) => JSX.Element;
|
||||
}
|
||||
|
||||
/** A single pane of a Multiview, including its camera chooser. */
|
||||
const Monoview = (props: MonoviewProps) => {
|
||||
return (
|
||||
<Box>
|
||||
<Box
|
||||
sx={{
|
||||
zIndex: 1,
|
||||
position: "absolute",
|
||||
height: "100%",
|
||||
width: "100%",
|
||||
}}
|
||||
>
|
||||
<Select
|
||||
value={props.cameraIndex == null ? undefined : props.cameraIndex}
|
||||
onChange={(e) => props.onSelect(e.target.value ?? null)}
|
||||
displayEmpty
|
||||
size="small"
|
||||
sx={{
|
||||
// Restyle to fit over the video (or black).
|
||||
backgroundColor: "rgba(255, 255, 255, 0.5)",
|
||||
"& svg": {
|
||||
color: "inherit",
|
||||
},
|
||||
}}
|
||||
>
|
||||
<MenuItem value={undefined}>(none)</MenuItem>
|
||||
{props.cameras.map((e, i) => (
|
||||
<MenuItem key={i} value={i}>
|
||||
{e.shortName}
|
||||
</MenuItem>
|
||||
))}
|
||||
</Select>
|
||||
</Box>
|
||||
{props.cameraIndex !== null &&
|
||||
props.renderCamera(props.cameras[props.cameraIndex])}
|
||||
</Box>
|
||||
);
|
||||
};
|
||||
|
||||
export default Multiview;
|
||||
25
ui/src/Live/index.tsx
Normal file
25
ui/src/Live/index.tsx
Normal file
@ -0,0 +1,25 @@
|
||||
// This file is part of Moonfire NVR, a security camera network video recorder.
|
||||
// Copyright (C) 2021 The Moonfire NVR Authors; see AUTHORS and LICENSE.txt.
|
||||
// SPDX-License-Identifier: GPL-v3.0-or-later WITH GPL-3.0-linking-exception
|
||||
|
||||
import { Camera } from "../types";
|
||||
import LiveCamera from "./LiveCamera";
|
||||
import Multiview from "./Multiview";
|
||||
|
||||
export interface LiveProps {
|
||||
cameras: Camera[];
|
||||
layoutIndex: number;
|
||||
}
|
||||
|
||||
const Live = ({ cameras, layoutIndex }: LiveProps) => {
|
||||
return (
|
||||
<Multiview
|
||||
layoutIndex={layoutIndex}
|
||||
cameras={cameras}
|
||||
renderCamera={(camera: Camera) => <LiveCamera camera={camera} />}
|
||||
/>
|
||||
);
|
||||
};
|
||||
|
||||
export { MultiviewChooser } from "./Multiview";
|
||||
export default Live;
|
||||
74
ui/src/Live/parser.ts
Normal file
74
ui/src/Live/parser.ts
Normal file
@ -0,0 +1,74 @@
|
||||
// This file is part of Moonfire NVR, a security camera network video recorder.
|
||||
// Copyright (C) 2021 The Moonfire NVR Authors; see AUTHORS and LICENSE.txt.
|
||||
// SPDX-License-Identifier: GPL-v3.0-or-later WITH GPL-3.0-linking-exception
|
||||
|
||||
export interface Part {
|
||||
mimeType: string;
|
||||
videoSampleEntryId: number;
|
||||
body: Uint8Array;
|
||||
}
|
||||
|
||||
interface ParseSuccess {
|
||||
status: "success";
|
||||
part: Part;
|
||||
}
|
||||
|
||||
interface ParseError {
|
||||
status: "error";
|
||||
errorMessage: string;
|
||||
}
|
||||
|
||||
const ASCII_DECODER = new TextDecoder("ascii");
|
||||
const CR = "\r".charCodeAt(0);
|
||||
const NL = "\n".charCodeAt(0);
|
||||
|
||||
type ParseResult = ParseSuccess | ParseError;
|
||||
|
||||
/// Parses a live stream message.
|
||||
export function parsePart(raw: Uint8Array): ParseResult {
|
||||
// Parse into headers and body.
|
||||
const headers = new Headers();
|
||||
let pos = 0;
|
||||
while (true) {
|
||||
const cr = raw.indexOf(CR, pos);
|
||||
if (cr === -1 || raw.length === cr + 1 || raw[cr + 1] !== NL) {
|
||||
return {
|
||||
status: "error",
|
||||
errorMessage: "header that never ends (no '\\r\\n')!",
|
||||
};
|
||||
}
|
||||
const line = ASCII_DECODER.decode(raw.slice(pos, cr));
|
||||
pos = cr + 2;
|
||||
if (line.length === 0) {
|
||||
break;
|
||||
}
|
||||
const colon = line.indexOf(":");
|
||||
if (colon === -1 || line.length === colon + 1 || line[colon + 1] !== " ") {
|
||||
return {
|
||||
status: "error",
|
||||
errorMessage: "invalid name/value separator (no ': ')!",
|
||||
};
|
||||
}
|
||||
const name = line.substring(0, colon);
|
||||
const value = line.substring(colon + 2);
|
||||
headers.append(name, value);
|
||||
}
|
||||
const body = raw.slice(pos);
|
||||
|
||||
const mimeType = headers.get("Content-Type");
|
||||
if (mimeType === null) {
|
||||
return { status: "error", errorMessage: "no Content-Type" };
|
||||
}
|
||||
const videoSampleEntryIdStr = headers.get("X-Video-Sample-Entry-Id");
|
||||
if (videoSampleEntryIdStr === null) {
|
||||
return { status: "error", errorMessage: "no X-Video-Sample-Entry-Id" };
|
||||
}
|
||||
const videoSampleEntryId = parseInt(videoSampleEntryIdStr, 10);
|
||||
if (isNaN(videoSampleEntryId)) {
|
||||
return { status: "error", errorMessage: "invalid X-Video-Sample-Entry-Id" };
|
||||
}
|
||||
return {
|
||||
status: "success",
|
||||
part: { mimeType, videoSampleEntryId, body },
|
||||
};
|
||||
}
|
||||
@ -80,10 +80,10 @@ async function myfetch(
|
||||
|
||||
/** Fetches an initialization segment. */
|
||||
export async function init(
|
||||
hash: string,
|
||||
videoSampleEntryId: number,
|
||||
init: RequestInit
|
||||
): Promise<FetchResult<ArrayBuffer>> {
|
||||
const url = `/api/init/${hash}.mp4`;
|
||||
const url = `/api/init/${videoSampleEntryId}.mp4`;
|
||||
const fetchRes = await myfetch(url, init);
|
||||
if (fetchRes.status !== "success") {
|
||||
return fetchRes;
|
||||
|
||||
@ -4,6 +4,13 @@
|
||||
* SPDX-License-Identifier: GPL-v3.0-or-later WITH GPL-3.0-linking-exception
|
||||
*/
|
||||
|
||||
html,
|
||||
body,
|
||||
#root {
|
||||
width: 100%;
|
||||
height: 100%;
|
||||
}
|
||||
|
||||
@media (pointer: fine) {
|
||||
/*
|
||||
* The spacing defined at https://material.io/components/date-pickers#specs
|
||||
|
||||
@ -7,3 +7,18 @@
|
||||
// expect(element).toHaveTextContent(/react/i)
|
||||
// learn more: https://github.com/testing-library/jest-dom
|
||||
import "@testing-library/jest-dom";
|
||||
import { TextDecoder } from "util";
|
||||
|
||||
// LiveCamera/parser.ts uses TextDecoder, which works fine from the browser
|
||||
// but isn't available from node.js without a little help.
|
||||
// https://create-react-app.dev/docs/running-tests/#initializing-test-environment
|
||||
// https://stackoverflow.com/questions/51090515/global-functions-in-typescript-for-jest-testing#comment89270564_51091150
|
||||
declare let global: any;
|
||||
|
||||
// TODO: There's likely an elegant way to add TextDecoder to global's type.
|
||||
// Some promising links:
|
||||
// https://www.typescriptlang.org/docs/handbook/declaration-merging.html#global-augmentation
|
||||
// https://stackoverflow.com/a/62011156/23584
|
||||
// https://github.com/facebook/create-react-app/issues/6553#issuecomment-475491096
|
||||
|
||||
global.TextDecoder = TextDecoder;
|
||||
|
||||
Loading…
Reference in New Issue
Block a user