tweak the docker release flows

docker/Dockerfile

@@ -12,6 +12,7 @@ FROM --platform=$BUILDPLATFORM ubuntu:20.04 AS dev-common
 LABEL maintainer="slamb@slamb.org"
 ARG BUILD_UID=1000
 ARG BUILD_GID=1000
+ARG INVALIDATE_CACHE_DEV_COMMON=
 ENV LC_ALL=C.UTF-8
 COPY docker/dev-common.bash /
 RUN --mount=type=cache,id=var-cache-apt,target=/var/cache/apt,sharing=locked \
@@ -21,9 +22,10 @@ CMD [ "/bin/bash", "--login" ]
 # "dev" is a full development environment, suitable for shelling into or
 # using with the VS Code container plugin.
 FROM --platform=$BUILDPLATFORM dev-common AS dev
+LABEL maintainer="slamb@slamb.org"
 ARG BUILDARCH
 ARG TARGETARCH
-LABEL maintainer="slamb@slamb.org"
+ARG INVALIDATE_CACHE_DEV=
 COPY docker/dev.bash /
 RUN --mount=type=cache,id=var-cache-apt,target=/var/cache/apt,sharing=locked \
     /dev.bash
@@ -32,50 +34,38 @@ WORKDIR /var/lib/moonfire-nvr
 
 # Build the UI with node_modules and ui-dist outside the src dir.
 FROM --platform=$BUILDPLATFORM dev-common AS build-ui
+ARG INVALIDATE_CACHE_BUILD_UI=
 LABEL maintainer="slamb@slamb.org"
 WORKDIR /var/lib/moonfire-nvr/src/ui
+COPY docker/build-ui.bash /
 COPY ui /var/lib/moonfire-nvr/src/ui
 RUN --mount=type=tmpfs,target=/var/lib/moonfire-nvr/src/ui/node_modules \
-    npm ci && npm run build
+    /build-ui.bash
 
 # Build the Rust components. Note that dev.sh set up an environment variable
 # in .buildrc that similarly changes the target dir path.
-#
-# The "mode" argument to cache mounts doesn't seem to work reliably (as of
-# Docker version 20.10.5, build 55c4c88, using a docker-container builder),
-# thus the chmod command.
 FROM --platform=$BUILDPLATFORM dev AS build-server
 LABEL maintainer="slamb@slamb.org"
+ARG INVALIDATE_CACHE_BUILD_SERVER=
+COPY docker/build-server.bash /
 RUN --mount=type=cache,id=target,target=/var/lib/moonfire-nvr/target,sharing=locked,mode=777 \
-    --mount=type=cache,id=cargo-registry,target=/var/lib/moonfire-nvr/.cargo/registry,sharing=locked,mode=777 \
+    --mount=type=cache,id=cargo,target=/cargo-cache,sharing=locked,mode=777 \
     --mount=type=bind,source=server,target=/var/lib/moonfire-nvr/src/server,readonly \
-    bash -c 'set -o xtrace && \
+    /build-server.bash
-             source ~/.buildrc && \
-             sudo chmod 777 /var/lib/moonfire-nvr/{.cargo/registry,target} && \
-             cd src/server && \
-             cargo test && \
-             cargo build --release && \
-             sudo install -m 755 ~/moonfire-nvr /usr/local/bin/moonfire-nvr'
 
 # Deployment environment, now in the target platform.
 FROM --platform=$TARGETPLATFORM ubuntu:20.04 AS deploy
 LABEL maintainer="slamb@slamb.org"
+ARG INVALIDATE_CACHE_BUILD_DEPLOY=
 ENV LC_ALL=C.UTF-8
+COPY docker/deploy.bash /
 RUN --mount=type=cache,id=var-cache-apt,target=/var/cache/apt,sharing=locked \
-    export DEBIAN_FRONTEND=noninteractive && \
+    /deploy.bash
-    apt-get update && \
+COPY --from=dev-common /docker-build-debug/dev-common/ /docker-build-debug/dev-common/
-    apt-get install --assume-yes --no-install-recommends \
+COPY --from=dev /docker-build-debug/dev/ /docker-build-debug/dev/
-        ffmpeg \
+COPY --from=build-server /docker-build-debug/build-server/ /docker-build-debug/build-server/
-        libncurses6 \
-        libncursesw6 \
-        locales \
-        sudo \
-        sqlite3 \
-        tzdata \
-        vim-nox && \
-    rm -rf /var/lib/apt/lists/* && \
-    ln -s moonfire-nvr /usr/local/bin/nvr
 COPY --from=build-server /usr/local/bin/moonfire-nvr /usr/local/bin/moonfire-nvr
+COPY --from=build-ui /docker-build-debug/build-ui /docker-build-debug/build-ui
 COPY --from=build-ui /var/lib/moonfire-nvr/src/ui/dist /usr/local/lib/moonfire-nvr/ui
 
 # The install instructions say to use --user in the docker run commandline.

docker/build-server.bash

@@ -0,0 +1,34 @@
+#!/bin/bash
+# This file is part of Moonfire NVR, a security camera network video recorder.
+# Copyright (C) 2021 The Moonfire NVR Authors; see AUTHORS and LICENSE.txt.
+# SPDX-License-Identifier: GPL-v3.0-or-later WITH GPL-3.0-linking-exception.
+
+# Build the "build-server" target. See Dockerfile.
+
+set -o errexit
+set -o pipefail
+set -o xtrace
+
+mkdir /docker-build-debug/build-server
+exec > >(tee -i /docker-build-debug/build-server/output) 2>&1
+ls -laFR /cargo-cache > /docker-build-debug/build-server/cargo-cache-before
+ls -laFR /var/lib/moonfire-nvr/target \
+    > /docker-build-debug/build-server/target-before
+
+source ~/.buildrc
+
+# The "mode" argument to cache mounts doesn't seem to work reliably
+# (as of Docker version 20.10.5, build 55c4c88, using a docker-container
+# builder), thus the chmod command.
+sudo chmod 777 /cargo-cache /var/lib/moonfire-nvr/target
+mkdir -p /cargo-cache/{git,registry}
+ln -s /cargo-cache/{git,registry} ~/.cargo
+
+cd src/server
+time cargo test
+time cargo build --release
+sudo install -m 755 ~/moonfire-nvr /usr/local/bin/moonfire-nvr
+
+ls -laFR /cargo-cache > /docker-build-debug/build-server/cargo-cache-after
+ls -laFR /var/lib/moonfire-nvr/target \
+    > /docker-build-debug/build-server/target-after

docker/build-ui.bash

@@ -0,0 +1,18 @@
+#!/bin/bash
+# This file is part of Moonfire NVR, a security camera network video recorder.
+# Copyright (C) 2021 The Moonfire NVR Authors; see AUTHORS and LICENSE.txt.
+# SPDX-License-Identifier: GPL-v3.0-or-later WITH GPL-3.0-linking-exception.
+
+# Build the "build-ui" target. See Dockerfile.
+
+set -o errexit
+set -o pipefail
+set -o xtrace
+
+mkdir /docker-build-debug/build-ui
+exec > >(tee -i /docker-build-debug/build-ui/output) 2>&1
+
+time npm ci
+time npm run build
+ls -laFR /var/lib/moonfire-nvr/src/ui/node_modules \
+    > /docker-build-debug/build-ui/node_modules-after

docker/deploy.bash

@@ -0,0 +1,32 @@
+#!/bin/bash
+# This file is part of Moonfire NVR, a security camera network video recorder.
+# Copyright (C) 2021 The Moonfire NVR Authors; see AUTHORS and LICENSE.txt.
+# SPDX-License-Identifier: GPL-v3.0-or-later WITH GPL-3.0-linking-exception.
+
+# Build the "deploy" target. See Dockerfile.
+
+set -o errexit
+set -o pipefail
+set -o xtrace
+
+mkdir -p /docker-build-debug/deploy
+exec > >(tee -i /docker-build-debug/deploy/output) 2>&1
+ls -laFR /var/cache/apt \
+    > /docker-build-debug/deploy/var-cache-apt-before
+
+export DEBIAN_FRONTEND=noninteractive
+time apt-get update
+time apt-get install --assume-yes --no-install-recommends \
+    ffmpeg \
+    libncurses6 \
+    libncursesw6 \
+    locales \
+    sudo \
+    sqlite3 \
+    tzdata \
+    vim-nox && \
+rm -rf /var/lib/apt/lists/*
+ln -s moonfire-nvr /usr/local/bin/nvr
+
+ls -laFR /var/cache/apt \
+    > /docker-build-debug/deploy/var-cache-apt-after

docker/dev-common.bash

@@ -11,6 +11,11 @@ set -o xtrace
 
 export DEBIAN_FRONTEND=noninteractive
 
+mkdir --mode=1777 /docker-build-debug
+mkdir /docker-build-debug/dev-common
+exec > >(tee -i /docker-build-debug/dev-common/output) 2>&1
+ls -laFR /var/cache/apt > /docker-build-debug/dev-common/var-cache-apt-before
+
 # This file cleans apt caches after every invocation. Instead, we use a
 # buildkit cachemount to avoid putting them in the image while still allowing
 # some reuse.
@@ -30,8 +35,8 @@ packages+=(
     tzdata
     vim-nox
 )
-apt-get update
+time apt-get update
-apt-get install --assume-yes --no-install-recommends "${packages[@]}"
+time apt-get install --assume-yes --no-install-recommends "${packages[@]}"
 
 # Create the user. On the dev environment, allow sudo.
 groupadd \
@@ -47,9 +52,11 @@ useradd \
     moonfire-nvr
 echo 'moonfire-nvr ALL=(ALL) NOPASSWD: ALL' >>/etc/sudoers
 
+
 # Install Rust. Note curl was already installed for yarn above.
-su moonfire-nvr -lc "curl --proto =https --tlsv1.2 -sSf https://sh.rustup.rs |
+time su moonfire-nvr -lc "
-                     sh -s - -y"
+    curl --proto =https --tlsv1.2 -sSf https://sh.rustup.rs |
+    sh -s - -y"
 
 # Put configuration for the Rust build into a new ".buildrc" which is used
 # both (1) interactively from ~/.bashrc when logging into the dev container
@@ -65,3 +72,5 @@ source \$HOME/.cargo/env
 export CARGO_BUILD_TARGET_DIR=/var/lib/moonfire-nvr/target
 EOF
 chown moonfire-nvr:moonfire-nvr /var/lib/moonfire-nvr/.buildrc
+
+ls -laFR /var/cache/apt > /docker-build-debug/dev-common/var-cache-apt-after

docker/dev.bash

@@ -13,6 +13,9 @@ export DEBIAN_FRONTEND=noninteractive
 
 packages=()
 
+mkdir -p /docker-build-debug/dev
+ls -laFR /var/cache/apt > /docker-build-debug/dev/var-cache-apt-before
+
 if [[ "${BUILDARCH}" != "${TARGETARCH}" ]]; then
     # Set up cross compilation.
     case "${TARGETARCH}" in
@@ -44,7 +47,7 @@ if [[ "${BUILDARCH}" != "${TARGETARCH}" ]]; then
     *) host_is_port=1 ;;
     esac
 
-    dpkg --add-architecture "${dpkg_arch}"
+    time dpkg --add-architecture "${dpkg_arch}"
 
     if [[ $target_is_port -ne $host_is_port ]]; then
         # Ubuntu stores non-x86 architectures at a different URL, so futz the
@@ -67,7 +70,7 @@ if [[ "${BUILDARCH}" != "${TARGETARCH}" ]]; then
     )
 fi
 
-apt-get update
+time apt-get update
 
 # Install the packages for the target architecture.
 packages+=(
@@ -78,10 +81,8 @@ packages+=(
     libncurses-dev"${apt_target_suffix}"
     libsqlite3-dev"${apt_target_suffix}"
 )
-apt-get update
+time apt-get update
-apt-get install --assume-yes --no-install-recommends "${packages[@]}"
+time apt-get install --assume-yes --no-install-recommends "${packages[@]}"
-apt-get clean
-rm -rf /var/lib/apt/lists/*
 
 # Set environment variables for cross-compiling.
 # Also set up a symlink that points to the release binary, because the
@@ -113,3 +114,5 @@ EOF
 else
     su moonfire-nvr -lc "ln -s target/release/moonfire-nvr ."
 fi
+
+ls -laFR /var/cache/apt > /docker-build-debug/dev/var-cache-apt-after

guide/build.md

@@ -10,6 +10,14 @@ tracker](https://github.com/scottlamb/moonfire-nvr/issues) or
 [mailing list](https://groups.google.com/d/forum/moonfire-nvr-users) when
 stuck. Please also send pull requests to improve this doc.
 
+* [Building Moonfire NVR](#building-moonfire-nvr)
+    * [Downloading](#downloading)
+    * [Docker builds](#docker-builds)
+        * [Release procedure](#release-procedure)
+    * [Non-Docker setup](#non-docker-setup)
+        * [Running interactively straight from the working copy](#running-interactively-straight-from-the-working-copy)
+        * [Running as a `systemd` service](#running-as-a-systemd-service)
+
 ## Downloading
 
 See the [github page](https://github.com/scottlamb/moonfire-nvr) (in case
@@ -94,6 +102,41 @@ Error while loading /usr/sbin/dpkg-split: No such file or directory
 Error while loading /usr/sbin/dpkg-deb: No such file or directory
 ```
 
+Moonfire NVR's `Dockerfile` has some built-in debugging tools:
+
+*   Each stage saves some debug info to `/docker-build-debug/<stage>`, and
+    the `deploy` stage preserves the output from previous stages. The debug
+    info includes:
+    *    output (stdout + stderr) from the build script, running long operations
+         through the `time` command.
+    *    `ls -laFR` of cache mounts before and after.
+*   Each stage accepts a `INVALIDATE_CACHE_<stage>` argument. You can use eg
+    `--build-arg=INVALIDATE_CACHE_BUILD_SERVER=$(date +%s)` to force the
+    `build-server` stage to be rebuilt rather than use cached Docker layers.
+
+### Release procedure
+
+Releases are currently a bit manual. From a completely clean git work tree,
+
+1.  manually verify the current commit is pushed to github's master branch and
+    has a green checkmark indicating CI passed.
+2.  update versions:
+    * update `server/Cargo.toml` version by hand; run `cargo test --workspace`
+      to update `Cargo.lock`.
+    * ensure `README.md` and `CHANGELOG.md` refer to the new version.
+3.  run commands:
+    ```bash
+    VERSION=x.y.z
+    git commit -am "prepare version ${VERSION}"
+    git tag -a "v${VERSION}" -m "version ${VERSION}"
+    ./release.bash
+    git push
+    git push "v${VERSION}"
+    ```
+
+The `release.bash` script needs [`jq`](https://stedolan.github.io/jq/)
+installed to work.
+
 ## Non-Docker setup
 
 You may prefer building without Docker on the host. Moonfire NVR should run
@@ -196,7 +239,7 @@ terminal, with no extra arguments, until you abort with Ctrl-C. Likewise,
 some of the shell script's subcommands that wrap Docker (`start`, `stop`, and
 `logs`) have no parallel with this `nvr`.
 
-## Running as a `systemd` service
+### Running as a `systemd` service
 
 If you want to deploy a non-Docker build on Linux, you may want to use
 `systemd`. Create `/etc/systemd/system/moonfire-nvr.service`:

release.bash

@@ -0,0 +1,44 @@
+#!/bin/bash
+# This file is part of Moonfire NVR, a security camera network video recorder.
+# Copyright (C) 2021 The Moonfire NVR Authors; see AUTHORS and LICENSE.txt.
+# SPDX-License-Identifier: GPL-v3.0-or-later WITH GPL-3.0-linking-exception
+
+# Pushes a release to Docker. See guides/build.md#release-procedure.
+
+set -o errexit
+set -o pipefail
+set -o xtrace
+
+set_latest() {
+    # Our images are manifest lists (for multiple architectures).
+    # "docker tag" won't copy those. The technique below is adopted from here:
+    # https://github.com/docker/buildx/issues/459#issuecomment-750011371
+    local image="$1"
+    local hashes="$(docker manifest inspect "${image}:${version}" |
+                    jq --raw-output '.manifests[].digest')"
+    time docker manifest create "${image}:latest" "${hashes[@]/#/${image}:}"
+}
+
+version="$(git describe --dirty)"
+if [[ ! "${version}" =~ v[0-9]+\.[0-9]+\.[0-9]+-dirty ]]; then
+    echo "Expected a vX.Y.Z version tag, got ${version}." >&2
+    exit 1
+fi
+
+if [[ -n "$(git status --porcelain)" ]]; then
+    echo "git status says there's extra stuff in this directory." >&2
+    exit 1
+fi
+
+time docker buildx build \
+    --push \
+    --tag="scottlamb/moonfire-nvr:${version}" \
+    -f docker/Dockerfile .
+time docker buildx build \
+    --push \
+    --tag="scottlamb/moonfire-dev:${version}" \
+    --target="dev" \
+    -f docker/Dockerfile .
+
+set_latest scottlamb/moonfire-nvr
+set_latest scottlamb/moonfire-dev