From dd66c7b0dd7426a85070a1872ff49174577bc2bf Mon Sep 17 00:00:00 2001 From: Scott Lamb Date: Fri, 22 Jan 2021 14:43:41 -0800 Subject: [PATCH] restructure into "server" and "ui" subdirs Besides being more clear about what belongs to which, this helps with docker caching. The server and ui parts are only rebuilt when their respective subdirectories change. Extend this a bit further by making the webpack build not depend on the target architecture. And adding cache dirs so parts of the server and ui build process can be reused when layer-wide caching fails. --- .dockerignore | 7 +- .github/workflows/ci.yml | 10 +-- .gitignore | 11 ++- docker/Dockerfile | 63 ++++++++-------- docker/dev-common.bash | 68 ++++++++++++++++++ docker/dev.bash | 51 +------------ guide/build.md | 8 +-- guide/developing-ui.md | 1 + Cargo.lock => server/Cargo.lock | 0 Cargo.toml => server/Cargo.toml | 0 {base => server/base}/Cargo.toml | 0 {base => server/base}/clock.rs | 0 {base => server/base}/error.rs | 0 {base => server/base}/lib.rs | 0 {base => server/base}/strutil.rs | 0 {base => server/base}/time.rs | 0 {db => server/db}/Cargo.toml | 0 {db => server/db}/auth.rs | 0 {db => server/db}/build.rs | 0 {db => server/db}/check.rs | 0 {db => server/db}/coding.rs | 0 {db => server/db}/compare.rs | 0 {db => server/db}/db.rs | 0 {db => server/db}/dir.rs | 0 {db => server/db}/fs.rs | 0 {db => server/db}/lib.rs | 0 {db => server/db}/proto/schema.proto | 0 {db => server/db}/raw.rs | 0 {db => server/db}/recording.rs | 0 {db => server/db}/schema.sql | 0 {db => server/db}/signal.rs | 0 {db => server/db}/testdata/avc1 | Bin .../db}/testdata/video_sample_index.bin | Bin {db => server/db}/testutil.rs | 0 {db => server/db}/upgrade/mod.rs | 0 {db => server/db}/upgrade/v0.sql | 0 {db => server/db}/upgrade/v0_to_v1.rs | 0 {db => server/db}/upgrade/v1.sql | 0 {db => server/db}/upgrade/v1_to_v2.rs | 0 {db => server/db}/upgrade/v2_to_v3.rs | 0 {db => server/db}/upgrade/v3.sql | 0 {db => server/db}/upgrade/v3_to_v4.rs | 0 {db => server/db}/upgrade/v4_to_v5.rs | 0 {db => server/db}/upgrade/v5.sql | 0 {db => server/db}/upgrade/v5_to_v6.rs | 0 {db => server/db}/writer.rs | 0 {src => server/src}/analytics.rs | 0 {src => server/src}/body.rs | 0 {src => server/src}/cmds/check.rs | 0 {src => server/src}/cmds/config/cameras.rs | 0 {src => server/src}/cmds/config/dirs.rs | 0 {src => server/src}/cmds/config/mod.rs | 0 {src => server/src}/cmds/config/users.rs | 0 {src => server/src}/cmds/init.rs | 0 {src => server/src}/cmds/login.rs | 0 {src => server/src}/cmds/mod.rs | 0 {src => server/src}/cmds/run.rs | 0 {src => server/src}/cmds/sql.rs | 0 {src => server/src}/cmds/ts.rs | 0 {src => server/src}/cmds/upgrade/mod.rs | 0 {src => server/src}/edgetpu.tflite | Bin {src => server/src}/h264.rs | 0 {src => server/src}/json.rs | 0 {src => server/src}/main.rs | 0 {src => server/src}/mp4.rs | 0 {src => server/src}/slices.rs | 0 {src => server/src}/stream.rs | 0 {src => server/src}/streamer.rs | 0 {src => server/src}/testdata/clip.mp4 | Bin {src => server/src}/web.rs | 0 .eslintrc.json => ui/.eslintrc.json | 0 package.json => ui/package.json | 3 +- {ui-src => ui/src}/NVRApplication.js | 0 {ui-src => ui/src}/favicon.svg | 0 {ui-src => ui/src}/index.css | 0 {ui-src => ui/src}/index.html | 0 {ui-src => ui/src}/index.js | 0 {ui-src => ui/src}/lib/MoonfireAPI.js | 0 .../src}/lib/models/CalendarTSRange.js | 0 {ui-src => ui/src}/lib/models/Camera.js | 0 {ui-src => ui/src}/lib/models/Range.js | 0 {ui-src => ui/src}/lib/models/Range90k.js | 0 {ui-src => ui/src}/lib/models/Recording.js | 0 {ui-src => ui/src}/lib/models/Stream.js | 0 .../src}/lib/support/RecordingFormatter.js | 0 .../src}/lib/support/Time90kParser.js | 0 .../src}/lib/support/TimeFormatter.js | 0 .../src}/lib/support/TimeStamp90kFormatter.js | 0 {ui-src => ui/src}/lib/support/URLBuilder.js | 0 {ui-src => ui/src}/lib/views/CalendarView.js | 0 .../src}/lib/views/DatePickerView.js | 0 .../src}/lib/views/NVRSettingsView.js | 0 .../src}/lib/views/RecordingsView.js | 0 .../src}/lib/views/StreamSelectorView.js | 0 {ui-src => ui/src}/lib/views/StreamView.js | 0 .../src}/lib/views/VideoDialogView.js | 0 {webpack => ui/webpack}/base.config.js | 10 +-- {webpack => ui/webpack}/dev.config.js | 0 {webpack => ui/webpack}/prod.config.js | 0 yarn.lock => ui/yarn.lock | 0 100 files changed, 127 insertions(+), 105 deletions(-) create mode 100755 docker/dev-common.bash rename Cargo.lock => server/Cargo.lock (100%) rename Cargo.toml => server/Cargo.toml (100%) rename {base => server/base}/Cargo.toml (100%) rename {base => server/base}/clock.rs (100%) rename {base => server/base}/error.rs (100%) rename {base => server/base}/lib.rs (100%) rename {base => server/base}/strutil.rs (100%) rename {base => server/base}/time.rs (100%) rename {db => server/db}/Cargo.toml (100%) rename {db => server/db}/auth.rs (100%) rename {db => server/db}/build.rs (100%) rename {db => server/db}/check.rs (100%) rename {db => server/db}/coding.rs (100%) rename {db => server/db}/compare.rs (100%) rename {db => server/db}/db.rs (100%) rename {db => server/db}/dir.rs (100%) rename {db => server/db}/fs.rs (100%) rename {db => server/db}/lib.rs (100%) rename {db => server/db}/proto/schema.proto (100%) rename {db => server/db}/raw.rs (100%) rename {db => server/db}/recording.rs (100%) rename {db => server/db}/schema.sql (100%) rename {db => server/db}/signal.rs (100%) rename {db => server/db}/testdata/avc1 (100%) rename {db => server/db}/testdata/video_sample_index.bin (100%) rename {db => server/db}/testutil.rs (100%) rename {db => server/db}/upgrade/mod.rs (100%) rename {db => server/db}/upgrade/v0.sql (100%) rename {db => server/db}/upgrade/v0_to_v1.rs (100%) rename {db => server/db}/upgrade/v1.sql (100%) rename {db => server/db}/upgrade/v1_to_v2.rs (100%) rename {db => server/db}/upgrade/v2_to_v3.rs (100%) rename {db => server/db}/upgrade/v3.sql (100%) rename {db => server/db}/upgrade/v3_to_v4.rs (100%) rename {db => server/db}/upgrade/v4_to_v5.rs (100%) rename {db => server/db}/upgrade/v5.sql (100%) rename {db => server/db}/upgrade/v5_to_v6.rs (100%) rename {db => server/db}/writer.rs (100%) rename {src => server/src}/analytics.rs (100%) rename {src => server/src}/body.rs (100%) rename {src => server/src}/cmds/check.rs (100%) rename {src => server/src}/cmds/config/cameras.rs (100%) rename {src => server/src}/cmds/config/dirs.rs (100%) rename {src => server/src}/cmds/config/mod.rs (100%) rename {src => server/src}/cmds/config/users.rs (100%) rename {src => server/src}/cmds/init.rs (100%) rename {src => server/src}/cmds/login.rs (100%) rename {src => server/src}/cmds/mod.rs (100%) rename {src => server/src}/cmds/run.rs (100%) rename {src => server/src}/cmds/sql.rs (100%) rename {src => server/src}/cmds/ts.rs (100%) rename {src => server/src}/cmds/upgrade/mod.rs (100%) rename {src => server/src}/edgetpu.tflite (100%) rename {src => server/src}/h264.rs (100%) rename {src => server/src}/json.rs (100%) rename {src => server/src}/main.rs (100%) rename {src => server/src}/mp4.rs (100%) rename {src => server/src}/slices.rs (100%) rename {src => server/src}/stream.rs (100%) rename {src => server/src}/streamer.rs (100%) rename {src => server/src}/testdata/clip.mp4 (100%) rename {src => server/src}/web.rs (100%) rename .eslintrc.json => ui/.eslintrc.json (100%) rename package.json => ui/package.json (98%) rename {ui-src => ui/src}/NVRApplication.js (100%) rename {ui-src => ui/src}/favicon.svg (100%) rename {ui-src => ui/src}/index.css (100%) rename {ui-src => ui/src}/index.html (100%) rename {ui-src => ui/src}/index.js (100%) rename {ui-src => ui/src}/lib/MoonfireAPI.js (100%) rename {ui-src => ui/src}/lib/models/CalendarTSRange.js (100%) rename {ui-src => ui/src}/lib/models/Camera.js (100%) rename {ui-src => ui/src}/lib/models/Range.js (100%) rename {ui-src => ui/src}/lib/models/Range90k.js (100%) rename {ui-src => ui/src}/lib/models/Recording.js (100%) rename {ui-src => ui/src}/lib/models/Stream.js (100%) rename {ui-src => ui/src}/lib/support/RecordingFormatter.js (100%) rename {ui-src => ui/src}/lib/support/Time90kParser.js (100%) rename {ui-src => ui/src}/lib/support/TimeFormatter.js (100%) rename {ui-src => ui/src}/lib/support/TimeStamp90kFormatter.js (100%) rename {ui-src => ui/src}/lib/support/URLBuilder.js (100%) rename {ui-src => ui/src}/lib/views/CalendarView.js (100%) rename {ui-src => ui/src}/lib/views/DatePickerView.js (100%) rename {ui-src => ui/src}/lib/views/NVRSettingsView.js (100%) rename {ui-src => ui/src}/lib/views/RecordingsView.js (100%) rename {ui-src => ui/src}/lib/views/StreamSelectorView.js (100%) rename {ui-src => ui/src}/lib/views/StreamView.js (100%) rename {ui-src => ui/src}/lib/views/VideoDialogView.js (100%) rename {webpack => ui/webpack}/base.config.js (94%) rename {webpack => ui/webpack}/dev.config.js (100%) rename {webpack => ui/webpack}/prod.config.js (100%) rename yarn.lock => ui/yarn.lock (100%) diff --git a/.dockerignore b/.dockerignore index 0ddbf08..14d189a 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,3 +1,4 @@ -node_modules -target -ui-dist +/server/target +/ui/dist +/ui/node_modules +/ui/yarn-error.log diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d1e24ad..dc240f4 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -27,7 +27,7 @@ jobs: path: | ~/.cargo/registry ~/.cargo/git - target + server/target key: ${{ matrix.rust }}-cargo-${{ hashFiles('**/Cargo.lock') }} - name: Install dependencies run: sudo apt-get update && sudo apt-get install libavcodec-dev libavformat-dev libavutil-dev libncurses-dev libsqlite3-dev pkgconf @@ -38,7 +38,7 @@ jobs: toolchain: ${{ matrix.rust }} override: true - name: Test - run: cargo test ${{ matrix.extra_args }} --all + run: cd server && cargo test ${{ matrix.extra_args }} --all js: name: Build and lint Javascript frontend runs-on: ubuntu-20.04 @@ -57,6 +57,6 @@ jobs: key: ${{ runner.os }}-yarn-${{ hashFiles('**/yarn.lock') }} restore-keys: | ${{ runner.os }}-yarn- - - run: yarn install - - run: yarn build - - run: yarn lint + - run: cd ui && yarn install + - run: cd ui && yarn build + - run: cd ui && yarn lint diff --git a/.gitignore b/.gitignore index cbc053a..a965172 100644 --- a/.gitignore +++ b/.gitignore @@ -1,9 +1,6 @@ .DS_Store *.swp -*.sublime-workspace -node_modules -prep.config -db/schema.rs -target -ui-dist -yarn-error.log +/server/target +/ui/dist +/ui/node_modules +/ui/yarn-error.log diff --git a/docker/Dockerfile b/docker/Dockerfile index f085924..0e4154b 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -3,39 +3,51 @@ # See documentation here: # https://github.com/moby/buildkit/blob/master/frontend/dockerfile/docs/syntax.md -# Moonfire NVR development environment, using the build platform. -FROM --platform=$BUILDPLATFORM ubuntu:20.04 AS dev +# "dev-common" is the portion of "dev" (see below) which isn't specific to the +# target arch. It's sufficient for building the non-arch-specific webpack. +FROM --platform=$BUILDPLATFORM ubuntu:20.04 AS dev-common LABEL maintainer="slamb@slamb.org" -ARG TARGETARCH -ARG BUILDARCH ARG BUILD_UID=1000 ARG BUILD_GID=1000 -LABEL maintainer="slamb@slamb.org" ENV LC_ALL=C.UTF-8 +COPY docker/dev-common.bash / +RUN /dev-common.bash +CMD [ "/bin/bash", "--login" ] + +# "dev" is a full development environment, suitable for shelling into or +# using with the VS Code container plugin. +FROM --platform=$BUILDPLATFORM dev-common AS dev +ARG BUILDARCH +ARG TARGETARCH +LABEL maintainer="slamb@slamb.org" COPY docker/dev.bash / RUN /dev.bash USER moonfire-nvr:moonfire-nvr WORKDIR /var/lib/moonfire-nvr -CMD [ "/bin/bash", "--login" ] -# Build the webpack with node_modules and ui-dist outside the src dir. -FROM dev AS build-webpack +# Build the UI with node_modules and ui-dist outside the src dir. +FROM --platform=$BUILDPLATFORM dev-common AS build-ui LABEL maintainer="slamb@slamb.org" -RUN --mount=type=bind,target=/var/lib/moonfire-nvr/src,readonly \ - ln -s src/package.json src/yarn.lock src/ui-src src/webpack . && \ - yarn install && yarn build && rm -rf node_modules +WORKDIR /var/lib/moonfire-nvr/src/ui +COPY ui /var/lib/moonfire-nvr/src/ui +RUN --mount=type=cache,target=/var/lib/moonfire-nvr/src/ui/node_modules,sharing=locked,mode=1777 \ + yarn install && yarn build # Build the Rust components. Note that dev.sh set up an environment variable # in .buildrc that similarly changes the target dir path. -FROM dev AS build-server +FROM --platform=$BUILDPLATFORM dev AS build-server LABEL maintainer="slamb@slamb.org" -RUN --mount=type=bind,target=/var/lib/moonfire-nvr/src,readonly \ - bash -c 'set -o xtrace && source ~/.buildrc && cd src && cargo test && cargo build --release' +RUN --mount=type=cache,id=target,target=/var/lib/moonfire-nvr/target,sharing=locked,mode=1777 \ + --mount=type=bind,source=server,target=/var/lib/moonfire-nvr/src/server,readonly \ + bash -c 'set -o xtrace && \ + source ~/.buildrc && \ + cd src/server && \ + cargo test && \ + cargo build --release && \ + sudo install -m 755 ~/moonfire-nvr /usr/local/bin/moonfire-nvr' # Deployment environment, now in the target platform. FROM --platform=$TARGETPLATFORM ubuntu:20.04 AS deploy -ARG DEPLOY_UID=10000 -ARG DEPLOY_GID=10000 LABEL maintainer="slamb@slamb.org" ENV LC_ALL=C.UTF-8 RUN export DEBIAN_FRONTEND=noninteractive && \ @@ -51,21 +63,12 @@ RUN export DEBIAN_FRONTEND=noninteractive && \ vim-nox && \ apt-get clean && \ rm -rf /var/lib/apt/lists/* && \ - groupadd \ - --gid="${DEPLOY_GID}" \ - moonfire-nvr && \ - useradd \ - --no-log-init \ - --home-dir=/var/lib/moonfire-nvr \ - --uid="${DEPLOY_UID}" \ - --gid=moonfire-nvr \ - --shell=/bin/bash \ - --create-home \ - moonfire-nvr && \ ln -s moonfire-nvr /usr/local/bin/nvr -COPY --from=build-server /var/lib/moonfire-nvr/moonfire-nvr /usr/local/bin/moonfire-nvr -COPY --from=build-webpack /var/lib/moonfire-nvr/ui-dist /usr/local/lib/moonfire-nvr/ui +COPY --from=build-server /usr/local/bin/moonfire-nvr /usr/local/bin/moonfire-nvr +COPY --from=build-ui /var/lib/moonfire-nvr/ui/dist /usr/local/lib/moonfire-nvr/ui -USER moonfire-nvr:moonfire-nvr +# The install instructions say to use --user in the docker run commandline. +# Specify a non-root user just in case someone forgets. +USER 10000:10000 WORKDIR /var/lib/moonfire-nvr ENTRYPOINT [ "/usr/local/bin/moonfire-nvr" ] diff --git a/docker/dev-common.bash b/docker/dev-common.bash new file mode 100755 index 0000000..0bd1ff0 --- /dev/null +++ b/docker/dev-common.bash @@ -0,0 +1,68 @@ +#!/bin/bash +# Build the "dev" target. See Dockerfile. + +set -o errexit +set -o pipefail +set -o xtrace + +export DEBIAN_FRONTEND=noninteractive + +packages=() + +apt-get update + +# Add yarn repository. +apt-get --assume-yes --no-install-recommends install curl gnupg ca-certificates +curl -sS https://dl.yarnpkg.com/debian/pubkey.gpg | apt-key add - +echo "deb https://dl.yarnpkg.com/debian/ stable main" \ + >> /etc/apt/sources.list.d/yarn.list + +# Install all packages necessary for building (and some for testing/debugging). +packages+=( + build-essential + pkgconf + locales + nodejs + sudo + sqlite3 + tzdata + vim-nox + yarn +) +apt-get update +apt-get install --assume-yes --no-install-recommends "${packages[@]}" +apt-get clean +rm -rf /var/lib/apt/lists/* + +# Create the user. On the dev environment, allow sudo. +groupadd \ + --gid="${BUILD_GID}" \ + moonfire-nvr +useradd \ + --no-log-init \ + --home-dir=/var/lib/moonfire-nvr \ + --uid="${BUILD_UID}" \ + --gid=moonfire-nvr \ + --shell=/bin/bash \ + --create-home \ + moonfire-nvr +echo 'moonfire-nvr ALL=(ALL) NOPASSWD: ALL' >>/etc/sudoers + +# Install Rust. Note curl was already installed for yarn above. +su moonfire-nvr -lc "curl --proto =https --tlsv1.2 -sSf https://sh.rustup.rs | + sh -s - -y" + +# Put configuration for the Rust build into a new ".buildrc" which is used +# both (1) interactively from ~/.bashrc when logging into the dev container +# and (2) from a build-server RUN command. In particular, the latter can't +# use ~/.bashrc because that script immediately exits when run from a +# non-interactive shell. +echo 'source $HOME/.buildrc' >> /var/lib/moonfire-nvr/.bashrc +cat >> /var/lib/moonfire-nvr/.buildrc <> /etc/apt/sources.list.d/yarn.list - -# Install all packages necessary for building (and some for testing/debugging). +# Install the packages for the target architecture. packages+=( - build-essential - pkgconf ffmpeg"${apt_target_suffix}" libavcodec-dev"${apt_target_suffix}" libavformat-dev"${apt_target_suffix}" libavutil-dev"${apt_target_suffix}" libncurses-dev"${apt_target_suffix}" libsqlite3-dev"${apt_target_suffix}" - locales - nodejs - sudo - sqlite3 - tzdata - vim-nox - yarn ) apt-get update apt-get install --assume-yes --no-install-recommends "${packages[@]}" apt-get clean rm -rf /var/lib/apt/lists/* -# Create the user. On the dev environment, allow sudo. -groupadd \ - --gid="${BUILD_GID}" \ - moonfire-nvr -useradd \ - --no-log-init \ - --home-dir=/var/lib/moonfire-nvr \ - --uid="${BUILD_UID}" \ - --gid=moonfire-nvr \ - --shell=/bin/bash \ - --create-home \ - moonfire-nvr -echo 'moonfire-nvr ALL=(ALL) NOPASSWD: ALL' >>/etc/sudoers - -# Install Rust. Note curl was already installed for yarn above. -su moonfire-nvr -lc "curl --proto =https --tlsv1.2 -sSf https://sh.rustup.rs | - sh -s - -y" - -# Put configuration for the Rust build into a new ".buildrc" which is used -# both (1) interactively from ~/.bashrc when logging into the dev container -# and (2) from a build-server RUN command. In particular, the latter can't -# use ~/.bashrc because that script immediately exits when run from a -# non-interactive shell. -echo 'source $HOME/.buildrc' >> /var/lib/moonfire-nvr/.bashrc -cat >> /var/lib/moonfire-nvr/.buildrc <