From f92a23fd74e1342de2650558d57a1e1bc2ece57b Mon Sep 17 00:00:00 2001
From: Scott Lamb <slamb@slamb.org>
Date: Mon, 29 Mar 2021 11:49:33 -0700
Subject: [PATCH] fix Multiview to work in Safari and Firefox

LiveCamera itself still doesn't work in Safari, but small steps.
---
 ui/src/Live/LiveCamera.tsx | 110 ++++++++++++++++++----------
 ui/src/Live/Multiview.tsx  | 143 ++++++++++++++++++++-----------------
 ui/src/Live/index.tsx      |   4 +-
 3 files changed, 151 insertions(+), 106 deletions(-)

diff --git a/ui/src/Live/LiveCamera.tsx b/ui/src/Live/LiveCamera.tsx
index d59c562..5d56c80 100644
--- a/ui/src/Live/LiveCamera.tsx
+++ b/ui/src/Live/LiveCamera.tsx
@@ -11,7 +11,8 @@ import CircularProgress from "@material-ui/core/CircularProgress";
 import Alert from "@material-ui/core/Alert";
 
 interface LiveCameraProps {
-  camera: Camera;
+  camera: Camera | null;
+  chooser: JSX.Element;
 }
 
 interface BufferStateClosed {
@@ -274,11 +275,15 @@ class LiveCameraDriver {
 
 /**
  * A live view of a camera.
+ *
+ * The caller is currently expected to put this into a 16x9 block.
+ *
  * Note there's a significant setup cost to creating a LiveCamera, so the parent
  * should use React's <tt>key</tt> attribute to avoid unnecessarily mounting
  * and unmounting a camera.
+ *
  */
-const LiveCamera = ({ camera }: LiveCameraProps) => {
+const LiveCamera = ({ camera, chooser }: LiveCameraProps) => {
   const videoRef = React.useRef<HTMLVideoElement>(null);
   const [playbackState, setPlaybackState] = React.useState<PlaybackState>({
     state: "normal",
@@ -287,6 +292,10 @@ const LiveCamera = ({ camera }: LiveCameraProps) => {
   // Load the camera driver.
   const [driver, setDriver] = React.useState<LiveCameraDriver | null>(null);
   React.useEffect(() => {
+    if (camera === null) {
+      setDriver(null);
+      return;
+    }
     const d = new LiveCameraDriver(camera, setPlaybackState, videoRef);
     setDriver(d);
     return () => {
@@ -308,43 +317,10 @@ const LiveCamera = ({ camera }: LiveCameraProps) => {
     return () => clearTimeout(timerId);
   }, [playbackState]);
 
-  if (driver === null) {
-    return <Box />;
-  }
-  return (
-    <Box
-      sx={{
-        "& video": { width: "100%", height: "100%", objectFit: "contain" },
-        "& .progress-overlay": {
-          position: "absolute",
-          display: "flex",
-          alignItems: "center",
-          justifyContent: "center",
-          height: "100%",
-          width: "100%",
-          zIndex: 1,
-        },
-        "& .alert-overlay": {
-          position: "absolute",
-          display: "flex",
-          height: "100%",
-          width: "100%",
-          alignItems: "flex-end",
-          zIndex: 1,
-          p: 1,
-        },
-      }}
-    >
-      {showProgress && (
-        <div className="progress-overlay">
-          <CircularProgress />
-        </div>
-      )}
-      {playbackState.state === "error" && (
-        <div className="alert-overlay">
-          <Alert severity="error">{playbackState.message}</Alert>
-        </div>
-      )}
+  const videoElement =
+    driver === null ? (
+      <video />
+    ) : (
       <video
         ref={videoRef}
         muted
@@ -356,6 +332,62 @@ const LiveCamera = ({ camera }: LiveCameraProps) => {
         onTimeUpdate={driver.tryTrimBuffer}
         onWaiting={driver.videoWaiting}
       />
+    );
+  return (
+    <Box
+      sx={{
+        width: "100%",
+        height: "100%",
+        position: "relative",
+        "& video": {
+          width: "100%",
+          height: "100%",
+
+          // It'd be nice to use "contain" here so non-16x9 videos display
+          // with letterboxing rather than by being stretched. Unfortunately
+          // Firefox 87.0 doesn't honor the PixelAspectRatioBox of anamorphic
+          // sub streams. For now, make anamorphic 16x9 sub streams display
+          // correctly (at the expense of non-16x9 streams).
+          // TODO: adjust width/height dynamically to handle the letterboxing
+          // on non-16x9 streams.
+          objectFit: "fill",
+        },
+        "& .controls": {
+          position: "absolute",
+          width: "100%",
+          height: "100%",
+          zIndex: 1,
+        },
+        "& .progress-overlay": {
+          position: "absolute",
+          display: "flex",
+          alignItems: "center",
+          justifyContent: "center",
+          width: "100%",
+          height: "100%",
+        },
+        "& .alert-overlay": {
+          position: "absolute",
+          display: "flex",
+          width: "100%",
+          height: "100%",
+          alignItems: "flex-end",
+          p: 1,
+        },
+      }}
+    >
+      <div className="controls">{chooser}</div>
+      {showProgress && (
+        <div className="progress-overlay">
+          <CircularProgress />
+        </div>
+      )}
+      {playbackState.state === "error" && (
+        <div className="alert-overlay">
+          <Alert severity="error">{playbackState.message}</Alert>
+        </div>
+      )}
+      {videoElement}
     </Box>
   );
 };
diff --git a/ui/src/Live/Multiview.tsx b/ui/src/Live/Multiview.tsx
index a51d803..71a1f9d 100644
--- a/ui/src/Live/Multiview.tsx
+++ b/ui/src/Live/Multiview.tsx
@@ -8,7 +8,6 @@ import React, { useReducer, useState } from "react";
 import { Camera } from "../types";
 import { makeStyles } from "@material-ui/core/styles";
 import useResizeObserver from "@react-hook/resize-observer";
-import Box from "@material-ui/core/Box";
 
 export interface Layout {
   className: string;
@@ -27,36 +26,34 @@ const MAX_CAMERAS = 9;
 const useStyles = makeStyles((theme) => ({
   root: {
     flex: "1 0 0",
-    overflow: "hidden",
     color: "white",
     marginTop: theme.spacing(2),
-  },
-  mid: {
-    display: "none",
-    position: "relative",
-    padding: 0,
-    margin: 0,
-    "&.wider, &.wider img": {
+    overflow: "hidden",
+
+    "& .mid": {
+      position: "relative",
+      aspectRatio: "16 / 9",
+      display: "inline-block",
+    },
+
+    // Set the width based on the height.
+    "& .mid.wider": {
       height: "100%",
-      display: "inline-block",
     },
-    "&.taller, &.taller img": {
+
+    // Set the height based on the width.
+    "& .mid.taller": {
       width: "100%",
-      display: "inline-block",
-    },
-    "& img": {
-      objectFit: "contain",
     },
   },
   inner: {
     // match parent's size without influencing it.
-    overflow: "hidden",
     position: "absolute",
-    top: 0,
-    bottom: 0,
-    left: 0,
-    right: 0,
+    width: "100%",
+    height: "100%",
 
+    backgroundColor: "#000",
+    overflow: "hidden",
     display: "grid",
     gridGap: "0px",
 
@@ -83,7 +80,7 @@ const useStyles = makeStyles((theme) => ({
 export interface MultiviewProps {
   cameras: Camera[];
   layoutIndex: number;
-  renderCamera: (camera: Camera) => JSX.Element;
+  renderCamera: (camera: Camera | null, chooser: JSX.Element) => JSX.Element;
 }
 
 export interface MultiviewChooserProps {
@@ -155,8 +152,7 @@ function selectedReducer(old: SelectedCameras, op: SelectOp): SelectedCameras {
  * as possible. Internally, multiview uses the largest possible aspect
  * ratio-constrained section of it. It uses a ResizeObserver to determine if
  * the outer div is wider or taller than 16x9, and then sets an appropriate CSS
- * class to constrain the width or height respectively using a technique like
- * <https://stackoverflow.com/a/14911949/23584>. The goal is to have the
+ * class to constrain the width or height respectively. The goal is to have the
  * smoothest resizing by changing the DOM/CSS as little as possible.
  */
 const Multiview = (props: MultiviewProps) => {
@@ -164,12 +160,41 @@ const Multiview = (props: MultiviewProps) => {
     selectedReducer,
     Array(MAX_CAMERAS).fill(null)
   );
-  const [widerOrTaller, setWiderOrTaller] = useState("wider");
+  const [widerOrTaller, setWiderOrTaller] = useState("");
   const outerRef = React.useRef<HTMLDivElement>(null);
+  const midRef = React.useRef<HTMLDivElement>(null);
+
+  // Keep a constant 16x9 aspect ratio. Chrome 89.0.4389.90 supports the
+  // "aspect-ratio" CSS property and seems to behave in a predictable way.
+  // Intuition suggests using that is more performant than extra DOM
+  // manipulations. Firefox 87.0 doesn't support aspect-ratio. Emulating it
+  // with an <img> child doesn't work well either for using a (flex item)
+  // ancestor's (calculated) height to compute
+  // the <img>'s width and then the parent's width. There are some open bugs
+  // that look related, eg:
+  // https://bugzilla.mozilla.org/show_bug.cgi?id=1349738
+  // https://bugzilla.mozilla.org/show_bug.cgi?id=1690423
+  // so when there's no "aspect-ratio", just calculate everything here.
+  const aspectRatioSupported = CSS.supports("aspect-ratio: 16 / 9");
   useResizeObserver(outerRef, (entry: ResizeObserverEntry) => {
     const w = entry.contentRect.width;
     const h = entry.contentRect.height;
-    setWiderOrTaller((w * 9) / 16 > h ? "wider" : "taller");
+    const hFromW = (w * 9) / 16;
+    if (aspectRatioSupported) {
+      setWiderOrTaller(hFromW > h ? "wider" : "taller");
+      return;
+    }
+    const mid = midRef.current;
+    if (mid === null) {
+      return;
+    }
+    if (hFromW > h) {
+      mid.style.width = `${(h * 16) / 9}px`;
+      mid.style.height = `${h}px`;
+    } else {
+      mid.style.width = `${w}px`;
+      mid.style.height = `${hFromW}px`;
+    }
   });
   const classes = useStyles();
   const layout = LAYOUTS[props.layoutIndex];
@@ -195,12 +220,7 @@ const Multiview = (props: MultiviewProps) => {
   });
   return (
     <div className={classes.root} ref={outerRef}>
-      <div className={`${classes.mid} ${widerOrTaller}`}>
-        {/* a 16x9 black png from png-pixel.com */}
-        <img
-          src="data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAABAAAAAJCAQAAACRI2S5AAAAEklEQVR42mNk+M+AFzCOKgADALyGCQGyq8YeAAAAAElFTkSuQmCC"
-          alt=""
-        />
+      <div className={`mid ${widerOrTaller}`} ref={midRef}>
         <div className={`${classes.inner} ${layout.className}`}>
           {monoviews}
         </div>
@@ -213,45 +233,36 @@ interface MonoviewProps {
   cameras: Camera[];
   cameraIndex: number | null;
   onSelect: (cameraIndex: number | null) => void;
-  renderCamera: (camera: Camera) => JSX.Element;
+  renderCamera: (camera: Camera | null, chooser: JSX.Element) => JSX.Element;
 }
 
 /** A single pane of a Multiview, including its camera chooser. */
 const Monoview = (props: MonoviewProps) => {
-  return (
-    <Box>
-      <Box
-        sx={{
-          zIndex: 1,
-          position: "absolute",
-          height: "100%",
-          width: "100%",
-        }}
-      >
-        <Select
-          value={props.cameraIndex == null ? undefined : props.cameraIndex}
-          onChange={(e) => props.onSelect(e.target.value ?? null)}
-          displayEmpty
-          size="small"
-          sx={{
-            // Restyle to fit over the video (or black).
-            backgroundColor: "rgba(255, 255, 255, 0.5)",
-            "& svg": {
-              color: "inherit",
-            },
-          }}
-        >
-          <MenuItem value={undefined}>(none)</MenuItem>
-          {props.cameras.map((e, i) => (
-            <MenuItem key={i} value={i}>
-              {e.shortName}
-            </MenuItem>
-          ))}
-        </Select>
-      </Box>
-      {props.cameraIndex !== null &&
-        props.renderCamera(props.cameras[props.cameraIndex])}
-    </Box>
+  const chooser = (
+    <Select
+      value={props.cameraIndex == null ? undefined : props.cameraIndex}
+      onChange={(e) => props.onSelect(e.target.value ?? null)}
+      displayEmpty
+      size="small"
+      sx={{
+        // Restyle to fit over the video (or black).
+        backgroundColor: "rgba(255, 255, 255, 0.5)",
+        "& svg": {
+          color: "inherit",
+        },
+      }}
+    >
+      <MenuItem value={undefined}>(none)</MenuItem>
+      {props.cameras.map((e, i) => (
+        <MenuItem key={i} value={i}>
+          {e.shortName}
+        </MenuItem>
+      ))}
+    </Select>
+  );
+  return props.renderCamera(
+    props.cameraIndex === null ? null : props.cameras[props.cameraIndex],
+    chooser
   );
 };
 
diff --git a/ui/src/Live/index.tsx b/ui/src/Live/index.tsx
index 9bcb4e8..03f04b4 100644
--- a/ui/src/Live/index.tsx
+++ b/ui/src/Live/index.tsx
@@ -16,7 +16,9 @@ const Live = ({ cameras, layoutIndex }: LiveProps) => {
     <Multiview
       layoutIndex={layoutIndex}
       cameras={cameras}
-      renderCamera={(camera: Camera) => <LiveCamera camera={camera} />}
+      renderCamera={(camera: Camera | null, chooser: JSX.Element) => (
+        <LiveCamera camera={camera} chooser={chooser} />
+      )}
     />
   );
 };