new docker-based build

This replaces the previous Dockerfile, which was a single stage for building and deployment. The new one is a multi-stage build. Its "dev" target has the full development environment; its "deploy" target is more slim. It supports cross-compiled builds via BuildKit, eg to prepare a build suitable for a Raspberry Pi: docker buildx build --load --platform=linux/arm64/v8 --tag=moonfire-nvr --progress=plain --target=deploy -f docker/Dockerfile . Coming next: updating the installation docs.
2025-01-11 23:13:23 -05:00 · 2021-01-15 09:54:10 -08:00 · 2021-01-15 09:54:10 -08:00 · 9349a2a164
commit 9349a2a164
parent b2bf7bdfa4
8 changed files with 244 additions and 184 deletions
--- a/Cargo.lock
+++ b/Cargo.lock
@ -339,22 +339,6 @@ version = "0.1.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "245097e9a4535ee1e3e3931fcfcd55a796a44c643e8596ff6566d68f09b87bbc"

-[[package]]
-name = "core-foundation"
-version = "0.9.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0a89e2ae426ea83155dccf10c0fa6b1463ef6d5fcb44cee0b224a408fa640a62"
-dependencies = [
- "core-foundation-sys",
- "libc",
-]
-
-[[package]]
-name = "core-foundation-sys"
-version = "0.8.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ea221b5284a47e40033bf9b66f35f984ec0ea2931eb03505246cd27a963f981b"
-
 [[package]]
 name = "cpuid-bool"
 version = "0.1.2"
@ -695,21 +679,6 @@ version = "1.0.7"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1"

-[[package]]
-name = "foreign-types"
-version = "0.3.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1"
-dependencies = [
- "foreign-types-shared",
-]
-
-[[package]]
-name = "foreign-types-shared"
-version = "0.1.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b"
-
 [[package]]
 name = "form_urlencoded"
 version = "1.0.0"
@ -1039,19 +1008,6 @@ dependencies = [
 "want",
 ]

-[[package]]
-name = "hyper-tls"
-version = "0.4.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d979acc56dcb5b8dddba3917601745e877576475aa046df3226eabdecef78eed"
-dependencies = [
- "bytes",
- "hyper",
- "native-tls",
- "tokio",
- "tokio-tls",
-]
-
 [[package]]
 name = "ident_case"
 version = "1.0.1"
@ -1453,24 +1409,6 @@ dependencies = [
 "parking_lot",
 ]

-[[package]]
-name = "native-tls"
-version = "0.2.6"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6fcc7939b5edc4e4f86b1b4a04bb1498afaaf871b1a6691838ed06fcb48d3a3f"
-dependencies = [
- "lazy_static",
- "libc",
- "log",
- "openssl",
- "openssl-probe",
- "openssl-sys",
- "schannel",
- "security-framework",
- "security-framework-sys",
- "tempfile",
-]
-
 [[package]]
 name = "ncurses"
 version = "5.99.0"
@ -1625,39 +1563,6 @@ version = "0.3.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "624a8340c38c1b80fd549087862da4ba43e08858af025b236e509b6649fc13d5"

-[[package]]
-name = "openssl"
-version = "0.10.30"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8d575eff3665419f9b83678ff2815858ad9d11567e082f5ac1814baba4e2bcb4"
-dependencies = [
- "bitflags",
- "cfg-if 0.1.10",
- "foreign-types",
- "lazy_static",
- "libc",
- "openssl-sys",
-]
-
-[[package]]
-name = "openssl-probe"
-version = "0.1.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "77af24da69f9d9341038eba93a073b1fdaaa1b788221b00a69bce9e762cb32de"
-
-[[package]]
-name = "openssl-sys"
-version = "0.9.58"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a842db4709b604f0fe5d1170ae3565899be2ad3d9cbc72dedc789ac0511f78de"
-dependencies = [
- "autocfg",
- "cc",
- "libc",
- "pkg-config",
- "vcpkg",
-]
-
 [[package]]
 name = "owning_ref"
 version = "0.4.1"
@ -2064,21 +1969,18 @@ dependencies = [
 "http",
 "http-body",
 "hyper",
- "hyper-tls",
 "ipnet",
 "js-sys",
 "lazy_static",
 "log",
 "mime",
 "mime_guess",
- "native-tls",
 "percent-encoding",
 "pin-project-lite 0.2.0",
 "serde",
 "serde_json",
 "serde_urlencoded",
 "tokio",
- "tokio-tls",
 "url",
 "wasm-bindgen",
 "wasm-bindgen-futures",
@ -2179,16 +2081,6 @@ dependencies = [
 "cipher",
 ]

-[[package]]
-name = "schannel"
-version = "0.1.19"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8f05ba609c234e60bee0d547fe94a4c7e9da733d1c962cf6e59efa4cd9c8bc75"
-dependencies = [
- "lazy_static",
- "winapi 0.3.9",
-]
-
 [[package]]
 name = "scoped-tls"
 version = "1.0.0"
@ -2223,29 +2115,6 @@ dependencies = [
 "subtle",
 ]

-[[package]]
-name = "security-framework"
-version = "2.0.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c1759c2e3c8580017a484a7ac56d3abc5a6c1feadf88db2f3633f12ae4268c69"
-dependencies = [
- "bitflags",
- "core-foundation",
- "core-foundation-sys",
- "libc",
- "security-framework-sys",
-]
-
-[[package]]
-name = "security-framework-sys"
-version = "2.0.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f99b9d5e26d2a71633cc4f2ebae7cc9f874044e0c351a27e17892d76dce5678b"
-dependencies = [
- "core-foundation-sys",
- "libc",
-]
-
 [[package]]
 name = "serde"
 version = "1.0.117"
@ -2537,20 +2406,6 @@ dependencies = [
 "remove_dir_all",
 ]

-[[package]]
-name = "tempfile"
-version = "3.1.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7a6e24d9338a0a5be79593e2fa15a648add6138caa803e2d5bc782c371732ca9"
-dependencies = [
- "cfg-if 0.1.10",
- "libc",
- "rand 0.7.3",
- "redox_syscall",
- "remove_dir_all",
- "winapi 0.3.9",
-]
-
 [[package]]
 name = "term"
 version = "0.5.2"
@ -2687,16 +2542,6 @@ dependencies = [
 "syn 1.0.51",
 ]

-[[package]]
-name = "tokio-tls"
-version = "0.3.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9a70f4fcd7b3b24fb194f837560168208f669ca8cb70d0c4b862944452396343"
-dependencies = [
- "native-tls",
- "tokio",
-]
-
 [[package]]
 name = "tokio-tungstenite"
 version = "0.11.0"
--- a/Cargo.toml
+++ b/Cargo.toml
@ -61,7 +61,7 @@ url = "2.1.1"
 uuid = { version = "0.8", features = ["serde", "std", "v4"] }

 [dev-dependencies]
-reqwest = { version = "0.10.1", features = ["json"] }
+reqwest = { version = "0.10.1", default-features = false, features = ["json"] }
 tempdir = "0.3"

 [profile.dev.package.scrypt]
--- a/20
+++ b/20
@ -1,20 +0,0 @@
-FROM ubuntu:latest
-MAINTAINER Dolf Starreveld "dolf@starreveld.com"
-
-ENV	DEBIAN_FRONTEND noninteractive
-RUN	apt-get update && \
-	apt-get install -y apt-utils && \
-	apt-get install -y apt-transport-https tzdata git curl sudo vim  && \
-	rm -rf /var/lib/apt/lists/*
-RUN	useradd --no-log-init --home-dir /var/lib/moonfire-nvr --system --user-group moonfire-nvr && \
-	echo 'moonfire-nvr ALL=(ALL) NOPASSWD: ALL' >>/etc/sudoers
-ENV	HOME /var/lib/moonfire-nvr
-COPY	--chown=moonfire-nvr:moonfire-nvr . /home/moonfire-nvr/moonfire-nvr
-USER	moonfire-nvr
-WORKDIR /var/lib/moonfire-nvr/moonfire-nvr
-RUN 	whoami && ls -l && \
-	./scripts/setup-ubuntu.sh && \
-	./scripts/setup-ubuntu.sh && \
-	./scripts/build.sh -B
-
-CMD	[ "/bin/bash" ]
--- a/db/build.rs
+++ b/db/build.rs
@ -28,11 +28,14 @@
 // You should have received a copy of the GNU General Public License
 // along with this program.  If not, see <http://www.gnu.org/licenses/>.

-fn main() {
-    protobuf_codegen_pure::Codegen::new()
-        .out_dir(".")
+fn main() -> Result<(), Box<dyn std::error::Error>> {
+    Ok(protobuf_codegen_pure::Codegen::new()
+        .out_dir(std::env::var("OUT_DIR")?)
        .inputs(&["proto/schema.proto"])
        .include("proto")
-        .run()
-        .expect("protoc");
+        .customize(protobuf_codegen_pure::Customize {
+            gen_mod_rs: Some(true),
+            ..Default::default()
+        })
+        .run()?)
 }
--- a/db/lib.rs
+++ b/db/lib.rs
@ -37,9 +37,12 @@ mod compare;
 pub mod db;
 pub mod dir;
 mod fs;
+mod proto {
+    include!(concat!(env!("OUT_DIR"), "/mod.rs"));
+}
 mod raw;
 pub mod recording;
-mod schema;
+use proto::schema;
 pub mod signal;
 pub mod upgrade;
 pub mod writer;
--- a/docker/Dockerfile
+++ b/docker/Dockerfile
@ -0,0 +1,71 @@
+# syntax=docker/dockerfile:1.2.1
+
+# See documentation here:
+# https://github.com/moby/buildkit/blob/master/frontend/dockerfile/docs/syntax.md
+
+# Moonfire NVR development environment, using the build platform.
+FROM --platform=$BUILDPLATFORM ubuntu:20.04 AS dev
+LABEL maintainer="slamb@slamb.org"
+ARG TARGETARCH
+ARG BUILDARCH
+ARG BUILD_UID=1000
+ARG BUILD_GID=1000
+LABEL maintainer="slamb@slamb.org"
+ENV LC_ALL=C.UTF-8
+COPY docker/dev.bash /
+RUN /dev.bash
+USER moonfire-nvr:moonfire-nvr
+WORKDIR /var/lib/moonfire-nvr
+CMD	[ "/bin/bash", "--login" ]
+
+# Build the webpack with node_modules and ui-dist outside the src dir.
+FROM dev AS build-webpack
+LABEL maintainer="slamb@slamb.org"
+RUN --mount=type=bind,target=/var/lib/moonfire-nvr/src,readonly \
+    ln -s src/package.json src/yarn.lock src/ui-src src/webpack . && \
+    yarn install && yarn build && rm -rf node_modules
+
+# Build the Rust components. Note that dev.sh set up an environment variable
+# in .buildrc that similarly changes the target dir path.
+FROM dev AS build-server
+LABEL maintainer="slamb@slamb.org"
+RUN --mount=type=bind,target=/var/lib/moonfire-nvr/src,readonly \
+    bash -c 'set -o xtrace && source ~/.buildrc && cd src && cargo test && cargo build --release'
+
+# Deployment environment, now in the target platform.
+FROM --platform=$TARGETPLATFORM ubuntu:20.04 AS deploy
+ARG DEPLOY_UID=10000
+ARG DEPLOY_GID=10000
+LABEL maintainer="slamb@slamb.org"
+ENV LC_ALL=C.UTF-8
+RUN export DEBIAN_FRONTEND=noninteractive && \
+    apt-get update && \
+    apt-get install --assume-yes --no-install-recommends \
+        ffmpeg \
+        libncurses6 \
+        libncursesw6 \
+        locales \
+        sudo \
+        sqlite3 \
+        tzdata \
+        vim-nox && \
+    apt-get clean && \
+    rm -rf /var/lib/apt/lists/* && \
+    groupadd \
+        --gid="${DEPLOY_GID}" \
+        moonfire-nvr && \
+    useradd \
+        --no-log-init \
+        --home-dir=/var/lib/moonfire-nvr \
+        --uid="${DEPLOY_UID}" \
+        --gid=moonfire-nvr \
+        --shell=/bin/bash \
+        --create-home \
+        moonfire-nvr && \
+    ln -s moonfire-nvr /usr/local/bin/nvr
+COPY --from=build-server /var/lib/moonfire-nvr/moonfire-nvr /usr/local/bin/moonfire-nvr
+COPY --from=build-webpack /var/lib/moonfire-nvr/ui-dist/* /usr/local/moonfire-nvr/ui
+
+USER moonfire-nvr:moonfire-nvr
+WORKDIR /var/lib/moonfire-nvr
+ENTRYPOINT [ "/usr/local/bin/moonfire-nvr" ]
--- a/docker/dev.bash
+++ b/docker/dev.bash
@ -0,0 +1,158 @@
+#!/bin/bash
+
+set -o errexit
+set -o pipefail
+set -o xtrace
+
+export DEBIAN_FRONTEND=noninteractive
+
+packages=()
+
+if [[ "${BUILDARCH}" != "${TARGETARCH}" ]]; then
+    # Set up cross compilation.
+    case "${TARGETARCH}" in
+    arm64)
+      dpkg_arch=arm64
+      gcc_target=aarch64-linux-gnu
+      rust_target=aarch64-unknown-linux-gnu
+      target_is_port=1
+      ;;
+    arm)
+      dpkg_arch=armhf
+      gcc_target=arm-linux-gnueabihf
+      rust_target=arm-unknown-linux-gnueabihf
+      target_is_port=1
+      ;;
+    amd64)
+      dpkg_arch=amd64
+      gcc_target=x86_64-linux-gnu
+      rust_target=x86_64-unknown-linux-gnu
+      target_is_port=0
+      ;;
+    *)
+      echo "Unsupported cross-compile target ${TARGETARCH}." >&2
+      exit 1
+    esac
+    apt_target_suffix=":${dpkg_arch}"
+    case "${BUILDARCH}" in
+    amd64|386) host_is_port=0 ;;
+    *) host_is_port=1 ;;
+    esac
+
+    dpkg --add-architecture "${dpkg_arch}"
+
+    if [[ $target_is_port -ne $host_is_port ]]; then
+        # Ubuntu stores non-x86 architectures at a different URL, so futz the
+        # sources file to allow installing both host and target.
+        # See https://github.com/rust-embedded/cross/blob/master/docker/common.sh
+        perl -pi.bak -e '
+            s{^deb (http://.*.ubuntu.com/ubuntu/) (.*)}
+             {deb [arch=amd64,i386] \1 \2\ndeb [arch-=amd64,i386] http://ports.ubuntu.com/ubuntu-ports \2};
+            s{^deb (http://ports.ubuntu.com/ubuntu-ports/) (.*)}
+             {deb [arch=amd64,i386] http://archive.ubuntu.com/ubuntu \2\ndeb [arch-=amd64,i386] \1 \2}' \
+            /etc/apt/sources.list
+        cat /etc/apt/sources.list
+    fi
+
+    packages+=(
+        g++-${gcc_target}
+        libc6-dev-${dpkg_arch}-cross
+        pkg-config-${gcc_target}
+        qemu-user
+    )
+fi
+
+apt-get update
+
+# Add yarn repository.
+apt-get --assume-yes --no-install-recommends install curl gnupg ca-certificates
+curl -sS https://dl.yarnpkg.com/debian/pubkey.gpg | apt-key add -
+echo "deb https://dl.yarnpkg.com/debian/ stable main" \
+    >> /etc/apt/sources.list.d/yarn.list
+
+# Install all packages necessary for building (and some for testing/debugging).
+packages+=(
+    build-essential
+    pkgconf
+    ffmpeg"${apt_target_suffix}"
+    libavcodec-dev"${apt_target_suffix}"
+    libavformat-dev"${apt_target_suffix}"
+    libavutil-dev"${apt_target_suffix}"
+    libncurses-dev"${apt_target_suffix}"
+    libsqlite3-dev"${apt_target_suffix}"
+    locales
+    nodejs
+    sudo
+    sqlite3
+    tzdata
+    vim-nox
+    yarn
+)
+apt-get update
+apt-get install --assume-yes --no-install-recommends "${packages[@]}"
+apt-get clean
+rm -rf /var/lib/apt/lists/*
+
+# Create the user. On the dev environment, allow sudo.
+groupadd \
+    --gid="${BUILD_GID}" \
+    moonfire-nvr
+useradd \
+    --no-log-init \
+    --home-dir=/var/lib/moonfire-nvr \
+    --uid="${BUILD_UID}" \
+    --gid=moonfire-nvr \
+    --shell=/bin/bash \
+    --create-home \
+    moonfire-nvr
+echo 'moonfire-nvr ALL=(ALL) NOPASSWD: ALL' >>/etc/sudoers
+
+# Install Rust. Note curl was already installed for yarn above.
+su moonfire-nvr -lc "curl --proto =https --tlsv1.2 -sSf https://sh.rustup.rs |
+                     sh -s - -y"
+
+# Put configuration for the Rust build into a new ".buildrc" which is used
+# both (1) interactively from ~/.bashrc when logging into the dev container
+# and (2) from a build-server RUN command. In particular, the latter can't
+# use ~/.bashrc because that script immediately exits when run from a
+# non-interactive shell.
+echo 'source $HOME/.buildrc' >> /var/lib/moonfire-nvr/.bashrc
+cat >> /var/lib/moonfire-nvr/.buildrc <<EOF
+source \$HOME/.cargo/env
+
+# Set the target directory to be outside the src bind mount.
+# https://doc.rust-lang.org/cargo/reference/config.html#buildtarget-dir
+export CARGO_BUILD_TARGET_DIR=/var/lib/moonfire-nvr/target
+EOF
+chown moonfire-nvr:moonfire-nvr /var/lib/moonfire-nvr/.buildrc
+
+# Set environment variables for cross-compiling.
+# Also set up a symlink that points to the release binary, because the
+# release binary's location varies when cross-compiling, as described here:
+# https://doc.rust-lang.org/cargo/guide/build-cache.html
+if [[ -n "${rust_target}" ]]; then
+    su moonfire-nvr -lc "rustup target install ${rust_target} &&
+                         ln -s target/${rust_target}/release/moonfire-nvr ."
+    underscore_rust_target="${rust_target//-/_}"
+    uppercase_underscore_rust_target="${underscore_rust_target^^}"
+    cat >> /var/lib/moonfire-nvr/.buildrc <<EOF
+
+# https://doc.rust-lang.org/cargo/reference/config.html
+export CARGO_BUILD_TARGET=${rust_target}
+export CARGO_TARGET_${uppercase_underscore_rust_target}_LINKER=${gcc_target}-gcc
+
+# https://github.com/rust-lang/pkg-config-rs uses the "PKG_CONFIG"
+# variable to to select the pkg-config binary to use. As of pkg-config 0.3.19,
+# it unfortunately doesn't understand the <target>_ prefix that the README.md
+# describes for other vars. Fortunately Moonfire NVR doesn't have any host tools
+# that need pkg-config.
+export PKG_CONFIG=${gcc_target}-pkg-config
+
+# https://github.com/alexcrichton/cc-rs uses these variables to decide what
+# compiler to invoke.
+export CC_${underscore_rust_target}=${gcc_target}-gcc
+export CXX_${underscore_rust_target}=${gcc_target}-g++
+EOF
+else
+    su moonfire-nvr -lc "ln -s target/release/moonfire-nvr ."
+fi
--- a/scripts/setup-ubuntu.sh
+++ b/scripts/setup-ubuntu.sh
@ -47,8 +47,8 @@ PKGS="build-essential \
      libavcodec-dev \
      libavformat-dev \
      libavutil-dev \
-      libncurses5-dev \
-      libncursesw5-dev \
+      libncurses-dev \
+      libncursesw-dev \
      libsqlite3-dev \
      pkgconf \
      sqlite3 \