Merge remote-tracking branch 'origin/next' into command-refactor

Fixed conflict with commit 78502aa6b1
2 years ago · 4bbff69a24
parent 87225e70c3 b4755ba15b
commit 4bbff69a24
23 changed files with 338 additions and 184 deletions
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@ -8,6 +8,10 @@ variables:
  GIT_SUBMODULE_STRATEGY: recursive
  FF_USE_FASTZIP: 1
  CACHE_COMPRESSION_LEVEL: fastest
+  # Docker in Docker
+  DOCKER_HOST: tcp://docker:2375/
+  DOCKER_TLS_CERTDIR: ""
+  DOCKER_DRIVER: overlay2

 # --------------------------------------------------------------------- #
 #  Cargo: Compiling for different architectures                         #
@ -20,28 +24,45 @@ variables:
    - if: '$CI_COMMIT_BRANCH == "master"'
    - if: '$CI_COMMIT_BRANCH == "next"'
    - if: "$CI_COMMIT_TAG"
+    - if: '($CI_MERGE_REQUEST_APPROVED == "true") || $BUILD_EVERYTHING' # Once MR is approved, test all builds. Or if BUILD_EVERYTHING is set.
  interruptible: true
-  image: "rust:1.56"
+  image: "registry.gitlab.com/jfowl/conduit-containers/rust-with-tools:latest"
  tags: ["docker"]
+  services: ["docker:dind"]
  variables:
+    SHARED_PATH: $CI_PROJECT_DIR/shared
    CARGO_PROFILE_RELEASE_LTO: "true"
    CARGO_PROFILE_RELEASE_CODEGEN_UNITS: "1"
    CARGO_INCREMENTAL: "false" # https://matklad.github.io/2021/09/04/fast-rust-builds.html#ci-workflow
  before_script:
    - 'echo "Building for target $TARGET"'
-    - "rustc --version && cargo --version && rustup show" # Print version info for debugging
-    - "rustup target add $TARGET"
-    # If provided, bring in caching through sccache, which uses an external S3 endpoint to store compilation results:
-    - if [ -n "${SCCACHE_BIN_URL}" ]; then curl $SCCACHE_BIN_URL --output /sccache && chmod +x /sccache && export RUSTC_WRAPPER=/sccache; fi
+    - "rustup show && rustc --version && cargo --version" # Print version info for debugging
+    # fix cargo and rustup mounts from this container (https://gitlab.com/gitlab-org/gitlab-foss/-/issues/41227)
+    - "mkdir -p $SHARED_PATH/cargo"
+    - "cp -r $CARGO_HOME/bin $SHARED_PATH/cargo"
+    - "cp -r $RUSTUP_HOME $SHARED_PATH"
+    - "export CARGO_HOME=$SHARED_PATH/cargo RUSTUP_HOME=$SHARED_PATH/rustup"
+    # If provided, bring in caching through sccache, which uses an external S3 endpoint to store compilation results.
+    - if [ -n "${SCCACHE_ENDPOINT}" ]; then export RUSTC_WRAPPER=/sccache; fi
  script:
-    - time cargo build --target $TARGET --release
-    - 'cp "target/$TARGET/release/conduit" "conduit-$TARGET"'
+    # cross-compile conduit for target
+    - 'time cross build --target="$TARGET" --locked --release'
+    - 'mv "target/$TARGET/release/conduit" "conduit-$TARGET"'
+    # print information about linking for debugging
+    - "file conduit-$TARGET" # print file information
+    - 'readelf --dynamic conduit-$TARGET | sed -e "/NEEDED/q1"' # ensure statically linked
+  cache:
+    # https://doc.rust-lang.org/cargo/guide/cargo-home.html#caching-the-cargo-home-in-ci
+    key: "cargo-cache-$TARGET"
+    paths:
+      - $SHARED_PATH/cargo/registry/index
+      - $SHARED_PATH/cargo/registry/cache
+      - $SHARED_PATH/cargo/git/db
  artifacts:
    expire_in: never

 build:release:cargo:x86_64-unknown-linux-musl-with-debug:
  extends: .build-cargo-shared-settings
-  image: messense/rust-musl-cross:x86_64-musl
  variables:
    CARGO_PROFILE_RELEASE_DEBUG: 2 # Enable debug info for flamegraph profiling
    TARGET: "x86_64-unknown-linux-musl"
@ -55,7 +76,6 @@ build:release:cargo:x86_64-unknown-linux-musl-with-debug:

 build:release:cargo:x86_64-unknown-linux-musl:
  extends: .build-cargo-shared-settings
-  image: messense/rust-musl-cross:x86_64-musl
  variables:
    TARGET: "x86_64-unknown-linux-musl"
  artifacts:
@ -66,7 +86,6 @@ build:release:cargo:x86_64-unknown-linux-musl:

 build:release:cargo:arm-unknown-linux-musleabihf:
  extends: .build-cargo-shared-settings
-  image: messense/rust-musl-cross:arm-musleabihf
  variables:
    TARGET: "arm-unknown-linux-musleabihf"
  artifacts:
@ -77,7 +96,6 @@ build:release:cargo:arm-unknown-linux-musleabihf:

 build:release:cargo:armv7-unknown-linux-musleabihf:
  extends: .build-cargo-shared-settings
-  image: messense/rust-musl-cross:armv7-musleabihf
  variables:
    TARGET: "armv7-unknown-linux-musleabihf"
  artifacts:
@ -88,7 +106,6 @@ build:release:cargo:armv7-unknown-linux-musleabihf:

 build:release:cargo:aarch64-unknown-linux-musl:
  extends: .build-cargo-shared-settings
-  image: messense/rust-musl-cross:aarch64-musl
  variables:
    TARGET: "aarch64-unknown-linux-musl"
  artifacts:
@ -104,14 +121,17 @@ build:release:cargo:aarch64-unknown-linux-musl:
  cache:
    key: "build_cache--$TARGET--$CI_COMMIT_BRANCH--debug"
  script:
-    - "time cargo build --target $TARGET"
+    # cross-compile conduit for target
+    - 'time time cross build --target="$TARGET" --locked'
    - 'mv "target/$TARGET/debug/conduit" "conduit-debug-$TARGET"'
+    # print information about linking for debugging
+    - "file conduit-debug-$TARGET" # print file information
+    - 'readelf --dynamic conduit-debug-$TARGET | sed -e "/NEEDED/q1"' # ensure statically linked
  artifacts:
    expire_in: 4 weeks

 build:debug:cargo:x86_64-unknown-linux-musl:
  extends: ".cargo-debug-shared-settings"
-  image: messense/rust-musl-cross:x86_64-musl
  variables:
    TARGET: "x86_64-unknown-linux-musl"
  artifacts:
@ -136,9 +156,6 @@ build:debug:cargo:x86_64-unknown-linux-musl:
    - "build:release:cargo:armv7-unknown-linux-musleabihf"
    - "build:release:cargo:aarch64-unknown-linux-musl"
  variables:
-    DOCKER_HOST: tcp://docker:2375/
-    DOCKER_TLS_CERTDIR: ""
-    DOCKER_DRIVER: overlay2
    PLATFORMS: "linux/arm/v6,linux/arm/v7,linux/arm64,linux/amd64"
    DOCKER_FILE: "docker/ci-binaries-packaging.Dockerfile"
  cache:
@ -210,24 +227,20 @@ docker:master:dockerhub:
 test:cargo:
  stage: "test"
  needs: []
-  image: "rust:latest"
+  image: "registry.gitlab.com/jfowl/conduit-containers/rust-with-tools:latest"
  tags: ["docker"]
  variables:
    CARGO_INCREMENTAL: "false" # https://matklad.github.io/2021/09/04/fast-rust-builds.html#ci-workflow
  interruptible: true
  before_script:
-    #    - mkdir -p $CARGO_HOME
-    - apt-get update -yqq
-    - apt-get install -yqq --no-install-recommends build-essential libssl-dev pkg-config libclang-dev
    - rustup component add clippy rustfmt
-    - curl "https://faulty-storage.de/gitlab-report" --output ./gitlab-report && chmod +x ./gitlab-report
    # If provided, bring in caching through sccache, which uses an external S3 endpoint to store compilation results:
-    - if [ -n "${SCCACHE_BIN_URL}" ]; then curl $SCCACHE_BIN_URL --output /sccache && chmod +x /sccache && export RUSTC_WRAPPER=/sccache; fi
+    - if [ -n "${SCCACHE_ENDPOINT}" ]; then export RUSTC_WRAPPER=/usr/local/cargo/bin/sccache; fi
  script:
    - rustc --version && cargo --version # Print version info for debugging
    - cargo fmt --all -- --check
-    - "cargo test --color always --workspace --verbose --locked --no-fail-fast -- -Z unstable-options --format json | ./gitlab-report -p test > $CI_PROJECT_DIR/report.xml"
-    - "cargo clippy --color always --verbose --message-format=json | ./gitlab-report -p clippy > $CI_PROJECT_DIR/gl-code-quality-report.json"
+    - "cargo test --color always --workspace --verbose --locked --no-fail-fast -- -Z unstable-options --format json | gitlab-report -p test > $CI_PROJECT_DIR/report.xml"
+    - "cargo clippy --color always --verbose --message-format=json | gitlab-report -p clippy > $CI_PROJECT_DIR/gl-code-quality-report.json"
  artifacts:
    when: always
    reports:
--- a/CROSS_COMPILE.md
+++ b/CROSS_COMPILE.md
@ -1,11 +0,0 @@
-Install docker:
-
-```
-$ sudo apt install docker
-$ sudo usermod -aG docker $USER
-$ exec sudo su -l $USER
-$ sudo systemctl start docker
-$ cargo install cross
-$ cross build --release --target armv7-unknown-linux-musleabihf
-```
-The cross-compiled binary is at target/armv7-unknown-linux-musleabihf/release/conduit
--- a/Cargo.lock
+++ b/Cargo.lock
@ -2174,7 +2174,7 @@ dependencies = [
 [[package]]
 name = "ruma"
 version = "0.4.0"
-source = "git+https://github.com/ruma/ruma?rev=08d60b3d376b63462f769d4b9bd3bbfb560d501a#08d60b3d376b63462f769d4b9bd3bbfb560d501a"
+source = "git+https://github.com/ruma/ruma?rev=82becb86c837570224964425929d1b5305784435#82becb86c837570224964425929d1b5305784435"
 dependencies = [
 "assign",
 "js_int",
@ -2195,7 +2195,7 @@ dependencies = [
 [[package]]
 name = "ruma-api"
 version = "0.18.5"
-source = "git+https://github.com/ruma/ruma?rev=08d60b3d376b63462f769d4b9bd3bbfb560d501a#08d60b3d376b63462f769d4b9bd3bbfb560d501a"
+source = "git+https://github.com/ruma/ruma?rev=82becb86c837570224964425929d1b5305784435#82becb86c837570224964425929d1b5305784435"
 dependencies = [
 "bytes",
 "http",
@ -2211,7 +2211,7 @@ dependencies = [
 [[package]]
 name = "ruma-api-macros"
 version = "0.18.5"
-source = "git+https://github.com/ruma/ruma?rev=08d60b3d376b63462f769d4b9bd3bbfb560d501a#08d60b3d376b63462f769d4b9bd3bbfb560d501a"
+source = "git+https://github.com/ruma/ruma?rev=82becb86c837570224964425929d1b5305784435#82becb86c837570224964425929d1b5305784435"
 dependencies = [
 "proc-macro-crate",
 "proc-macro2",
@ -2222,7 +2222,7 @@ dependencies = [
 [[package]]
 name = "ruma-appservice-api"
 version = "0.4.0"
-source = "git+https://github.com/ruma/ruma?rev=08d60b3d376b63462f769d4b9bd3bbfb560d501a#08d60b3d376b63462f769d4b9bd3bbfb560d501a"
+source = "git+https://github.com/ruma/ruma?rev=82becb86c837570224964425929d1b5305784435#82becb86c837570224964425929d1b5305784435"
 dependencies = [
 "ruma-api",
 "ruma-common",
@ -2236,7 +2236,7 @@ dependencies = [
 [[package]]
 name = "ruma-client-api"
 version = "0.12.3"
-source = "git+https://github.com/ruma/ruma?rev=08d60b3d376b63462f769d4b9bd3bbfb560d501a#08d60b3d376b63462f769d4b9bd3bbfb560d501a"
+source = "git+https://github.com/ruma/ruma?rev=82becb86c837570224964425929d1b5305784435#82becb86c837570224964425929d1b5305784435"
 dependencies = [
 "assign",
 "bytes",
@ -2256,7 +2256,7 @@ dependencies = [
 [[package]]
 name = "ruma-common"
 version = "0.6.0"
-source = "git+https://github.com/ruma/ruma?rev=08d60b3d376b63462f769d4b9bd3bbfb560d501a#08d60b3d376b63462f769d4b9bd3bbfb560d501a"
+source = "git+https://github.com/ruma/ruma?rev=82becb86c837570224964425929d1b5305784435#82becb86c837570224964425929d1b5305784435"
 dependencies = [
 "indexmap",
 "js_int",
@ -2271,7 +2271,7 @@ dependencies = [
 [[package]]
 name = "ruma-events"
 version = "0.24.6"
-source = "git+https://github.com/ruma/ruma?rev=08d60b3d376b63462f769d4b9bd3bbfb560d501a#08d60b3d376b63462f769d4b9bd3bbfb560d501a"
+source = "git+https://github.com/ruma/ruma?rev=82becb86c837570224964425929d1b5305784435#82becb86c837570224964425929d1b5305784435"
 dependencies = [
 "indoc",
 "js_int",
@ -2288,7 +2288,7 @@ dependencies = [
 [[package]]
 name = "ruma-events-macros"
 version = "0.24.6"
-source = "git+https://github.com/ruma/ruma?rev=08d60b3d376b63462f769d4b9bd3bbfb560d501a#08d60b3d376b63462f769d4b9bd3bbfb560d501a"
+source = "git+https://github.com/ruma/ruma?rev=82becb86c837570224964425929d1b5305784435#82becb86c837570224964425929d1b5305784435"
 dependencies = [
 "proc-macro-crate",
 "proc-macro2",
@ -2299,7 +2299,7 @@ dependencies = [
 [[package]]
 name = "ruma-federation-api"
 version = "0.3.1"
-source = "git+https://github.com/ruma/ruma?rev=08d60b3d376b63462f769d4b9bd3bbfb560d501a#08d60b3d376b63462f769d4b9bd3bbfb560d501a"
+source = "git+https://github.com/ruma/ruma?rev=82becb86c837570224964425929d1b5305784435#82becb86c837570224964425929d1b5305784435"
 dependencies = [
 "js_int",
 "ruma-api",
@ -2314,7 +2314,7 @@ dependencies = [
 [[package]]
 name = "ruma-identifiers"
 version = "0.20.0"
-source = "git+https://github.com/ruma/ruma?rev=08d60b3d376b63462f769d4b9bd3bbfb560d501a#08d60b3d376b63462f769d4b9bd3bbfb560d501a"
+source = "git+https://github.com/ruma/ruma?rev=82becb86c837570224964425929d1b5305784435#82becb86c837570224964425929d1b5305784435"
 dependencies = [
 "percent-encoding",
 "rand 0.8.4",
@ -2329,7 +2329,7 @@ dependencies = [
 [[package]]
 name = "ruma-identifiers-macros"
 version = "0.20.0"
-source = "git+https://github.com/ruma/ruma?rev=08d60b3d376b63462f769d4b9bd3bbfb560d501a#08d60b3d376b63462f769d4b9bd3bbfb560d501a"
+source = "git+https://github.com/ruma/ruma?rev=82becb86c837570224964425929d1b5305784435#82becb86c837570224964425929d1b5305784435"
 dependencies = [
 "quote",
 "ruma-identifiers-validation",
@ -2339,7 +2339,7 @@ dependencies = [
 [[package]]
 name = "ruma-identifiers-validation"
 version = "0.5.0"
-source = "git+https://github.com/ruma/ruma?rev=08d60b3d376b63462f769d4b9bd3bbfb560d501a#08d60b3d376b63462f769d4b9bd3bbfb560d501a"
+source = "git+https://github.com/ruma/ruma?rev=82becb86c837570224964425929d1b5305784435#82becb86c837570224964425929d1b5305784435"
 dependencies = [
 "thiserror",
 ]
@ -2347,7 +2347,7 @@ dependencies = [
 [[package]]
 name = "ruma-identity-service-api"
 version = "0.3.0"
-source = "git+https://github.com/ruma/ruma?rev=08d60b3d376b63462f769d4b9bd3bbfb560d501a#08d60b3d376b63462f769d4b9bd3bbfb560d501a"
+source = "git+https://github.com/ruma/ruma?rev=82becb86c837570224964425929d1b5305784435#82becb86c837570224964425929d1b5305784435"
 dependencies = [
 "js_int",
 "ruma-api",
@ -2360,7 +2360,7 @@ dependencies = [
 [[package]]
 name = "ruma-push-gateway-api"
 version = "0.3.0"
-source = "git+https://github.com/ruma/ruma?rev=08d60b3d376b63462f769d4b9bd3bbfb560d501a#08d60b3d376b63462f769d4b9bd3bbfb560d501a"
+source = "git+https://github.com/ruma/ruma?rev=82becb86c837570224964425929d1b5305784435#82becb86c837570224964425929d1b5305784435"
 dependencies = [
 "js_int",
 "ruma-api",
@ -2375,7 +2375,7 @@ dependencies = [
 [[package]]
 name = "ruma-serde"
 version = "0.5.0"
-source = "git+https://github.com/ruma/ruma?rev=08d60b3d376b63462f769d4b9bd3bbfb560d501a#08d60b3d376b63462f769d4b9bd3bbfb560d501a"
+source = "git+https://github.com/ruma/ruma?rev=82becb86c837570224964425929d1b5305784435#82becb86c837570224964425929d1b5305784435"
 dependencies = [
 "base64 0.13.0",
 "bytes",
@ -2390,7 +2390,7 @@ dependencies = [
 [[package]]
 name = "ruma-serde-macros"
 version = "0.5.0"
-source = "git+https://github.com/ruma/ruma?rev=08d60b3d376b63462f769d4b9bd3bbfb560d501a#08d60b3d376b63462f769d4b9bd3bbfb560d501a"
+source = "git+https://github.com/ruma/ruma?rev=82becb86c837570224964425929d1b5305784435#82becb86c837570224964425929d1b5305784435"
 dependencies = [
 "proc-macro-crate",
 "proc-macro2",
@ -2401,7 +2401,7 @@ dependencies = [
 [[package]]
 name = "ruma-signatures"
 version = "0.9.0"
-source = "git+https://github.com/ruma/ruma?rev=08d60b3d376b63462f769d4b9bd3bbfb560d501a#08d60b3d376b63462f769d4b9bd3bbfb560d501a"
+source = "git+https://github.com/ruma/ruma?rev=82becb86c837570224964425929d1b5305784435#82becb86c837570224964425929d1b5305784435"
 dependencies = [
 "base64 0.13.0",
 "ed25519-dalek",
@ -2418,7 +2418,7 @@ dependencies = [
 [[package]]
 name = "ruma-state-res"
 version = "0.4.1"
-source = "git+https://github.com/ruma/ruma?rev=08d60b3d376b63462f769d4b9bd3bbfb560d501a#08d60b3d376b63462f769d4b9bd3bbfb560d501a"
+source = "git+https://github.com/ruma/ruma?rev=82becb86c837570224964425929d1b5305784435#82becb86c837570224964425929d1b5305784435"
 dependencies = [
 "itertools",
 "js_int",
--- a/Cargo.toml
+++ b/Cargo.toml
@ -20,7 +20,7 @@ rocket = { version = "0.5.0-rc.1", features = ["tls"] } # Used to handle request

 # Used for matrix spec type definitions and helpers
 #ruma = { version = "0.4.0", features = ["compat", "rand", "appservice-api-c", "client-api", "federation-api", "push-gateway-api-c", "state-res", "unstable-pre-spec", "unstable-exhaustive-types"] }
-ruma = { git = "https://github.com/ruma/ruma", rev = "08d60b3d376b63462f769d4b9bd3bbfb560d501a", features = ["compat", "rand", "appservice-api-c", "client-api", "federation-api", "push-gateway-api-c", "state-res", "unstable-pre-spec", "unstable-exhaustive-types"] }
+ruma = { git = "https://github.com/ruma/ruma", rev = "82becb86c837570224964425929d1b5305784435", features = ["compat", "rand", "appservice-api-c", "client-api", "federation-api", "push-gateway-api-c", "state-res", "unstable-pre-spec", "unstable-exhaustive-types"] }
 #ruma = { git = "https://github.com/timokoesters/ruma", rev = "50c1db7e0a3a21fc794b0cce3b64285a4c750c71", features = ["compat", "rand", "appservice-api-c", "client-api", "federation-api", "push-gateway-api-c", "state-res", "unstable-pre-spec", "unstable-exhaustive-types"] }
 #ruma = { path = "../ruma/crates/ruma", features = ["compat", "rand", "appservice-api-c", "client-api", "federation-api", "push-gateway-api-c", "state-res", "unstable-pre-spec", "unstable-exhaustive-types"] }

--- a/Cross.toml
+++ b/Cross.toml
@ -0,0 +1,23 @@
+[build.env]
+# CI uses an S3 endpoint to store sccache artifacts, so their config needs to
+# be available in the cross container as well
+passthrough = [
+    "RUSTC_WRAPPER",
+    "AWS_ACCESS_KEY_ID",
+    "AWS_SECRET_ACCESS_KEY",
+    "SCCACHE_BUCKET",
+    "SCCACHE_ENDPOINT",
+    "SCCACHE_S3_USE_SSL",
+]
+
+[target.aarch64-unknown-linux-musl]
+image = "registry.gitlab.com/jfowl/conduit-containers/rust-cross-aarch64-unknown-linux-musl:latest"
+
+[target.arm-unknown-linux-musleabihf]
+image = "registry.gitlab.com/jfowl/conduit-containers/rust-cross-arm-unknown-linux-musleabihf:latest"
+
+[target.armv7-unknown-linux-musleabihf]
+image = "registry.gitlab.com/jfowl/conduit-containers/rust-cross-armv7-unknown-linux-musleabihf:latest"
+
+[target.x86_64-unknown-linux-musl]
+image = "registry.gitlab.com/jfowl/conduit-containers/rust-cross-x86_64-unknown-linux-musl:latest"
--- a/DEPLOY.md
+++ b/DEPLOY.md
@ -29,7 +29,11 @@ $ sudo wget -O /usr/local/bin/matrix-conduit <url>
 $ sudo chmod +x /usr/local/bin/matrix-conduit
 ```

-Alternatively, you may compile the binary yourself using
+Alternatively, you may compile the binary yourself
+
+```bash
+$ sudo apt install libclang-dev build-essential
+```

 ```bash
 $ cargo build --release
@ -37,7 +41,7 @@ $ cargo build --release

 Note that this currently requires Rust 1.50.

-If you want to cross compile Conduit to another architecture, read the [Cross-Compile Guide](CROSS_COMPILE.md).
+If you want to cross compile Conduit to another architecture, read the [Cross-Compile Guide](cross/README.md).

 ## Adding a Conduit user

--- a/45
+++ b/45
@ -1,9 +1,9 @@
 # syntax=docker/dockerfile:1
-FROM docker.io/rust:1.58-alpine AS builder
+FROM docker.io/rust:1.58-bullseye AS builder
 WORKDIR /usr/src/conduit

 # Install required packages to build Conduit and it's dependencies
-RUN apk add musl-dev
+RUN apt update && apt -y install libclang-dev

 # == Build dependencies without our own code separately for caching ==
 #
@ -26,28 +26,27 @@ COPY src src
 # Builds conduit and places the binary at /usr/src/conduit/target/release/conduit
 RUN touch src/main.rs && touch src/lib.rs && cargo build --release

-
-
-
 # ---------------------------------------------------------------------------------------------------------------
 # Stuff below this line actually ends up in the resulting docker image
 # ---------------------------------------------------------------------------------------------------------------
-FROM docker.io/alpine:3.15.0 AS runner
+FROM docker.io/debian:bullseye-slim AS runner

 # Standard port on which Conduit launches.
 # You still need to map the port when using the docker command or docker-compose.
 EXPOSE 6167

-# Note from @jfowl: I would like to remove this in the future and just have the Docker version be configured with envs.
-ENV CONDUIT_CONFIG="/srv/conduit/conduit.toml"
+# Note from @jfowl: I would like to remove the config file in the future and just have the Docker version be configured with envs.
+ENV CONDUIT_CONFIG="/srv/conduit/conduit.toml" \
+    CONDUIT_PORT=6167

 # Conduit needs:
 #   ca-certificates: for https
-#   libgcc: Apparently this is needed, even if I (@jfowl) don't know exactly why. But whatever, it's not that big.
-RUN apk add --no-cache \
+#   iproute2 & wget: for the healthcheck script
+RUN apt update && apt -y install \
    ca-certificates \
-    libgcc
-
+    iproute2 \
+    wget \
+    && rm -rf /var/lib/apt/lists/*

 # Created directory for the database and media files
 RUN mkdir -p /srv/conduit/.local/share/conduit
@ -59,20 +58,20 @@ HEALTHCHECK --start-period=5s --interval=5s CMD ./healthcheck.sh
 # Copy over the actual Conduit binary from the builder stage
 COPY --from=builder /usr/src/conduit/target/release/conduit /srv/conduit/conduit

-# Improve security: Don't run stuff as root, that does not need to run as root:
-# Add www-data user and group with UID 82, as used by alpine
-# https://git.alpinelinux.org/aports/tree/main/nginx/nginx.pre-install
+# Improve security: Don't run stuff as root, that does not need to run as root
+# Add 'conduit' user and group (100:82). The UID:GID choice is to be compatible
+# with previous, Alpine-based containers, where the user and group were both
+# named 'www-data'.
 RUN set -x ; \
-    addgroup -Sg 82 www-data 2>/dev/null ; \
-    adduser -S -D -H -h /srv/conduit -G www-data -g www-data www-data 2>/dev/null ; \
-    addgroup www-data www-data 2>/dev/null && exit 0 ; exit 1
+    groupadd -r -g 82 conduit ; \
+    useradd -r -M -d /srv/conduit -o -u 100 -g conduit conduit && exit 0 ; exit 1

-# Change ownership of Conduit files to www-data user and group
-RUN chown -cR www-data:www-data /srv/conduit
-RUN chmod +x /srv/conduit/healthcheck.sh
+# Change ownership of Conduit files to conduit user and group and make the healthcheck executable:
+RUN chown -cR conduit:conduit /srv/conduit && \
+    chmod +x /srv/conduit/healthcheck.sh

-# Change user to www-data
-USER www-data
+# Change user to conduit, no root permissions afterwards:
+USER conduit
 # Set container home directory
 WORKDIR /srv/conduit

--- a/conduit-example.toml
+++ b/conduit-example.toml
@ -22,6 +22,7 @@ database_backend = "rocksdb"
 # The port Conduit will be running on. You need to set up a reverse proxy in
 # your web server (e.g. apache or nginx), so all requests to /_matrix on port
 # 443 and 8448 will be forwarded to the Conduit instance running on this port
+# Docker users: Don't change this, you'll need to map an external port to this.
 port = 6167

 # Max size for uploads
--- a/cross/README.md
+++ b/cross/README.md
@ -0,0 +1,37 @@
+## Cross compilation
+
+The `cross` folder contains a set of convenience scripts (`build.sh` and `test.sh`) for cross-compiling Conduit.
+
+Currently supported targets are
+
+- aarch64-unknown-linux-musl
+- arm-unknown-linux-musleabihf
+- armv7-unknown-linux-musleabihf
+- x86\_64-unknown-linux-musl
+
+### Install prerequisites
+#### Docker
+[Installation guide](https://docs.docker.com/get-docker/).
+```sh
+$ sudo apt install docker
+$ sudo systemctl start docker
+$ sudo usermod -aG docker $USER
+$ newgrp docker
+```
+
+#### Cross
+[Installation guide](https://github.com/rust-embedded/cross/#installation).
+```sh
+$ cargo install cross
+```
+
+### Buiding Conduit
+```sh
+$ TARGET=armv7-unknown-linux-musleabihf ./cross/build.sh --release
+```
+The cross-compiled binary is at `target/armv7-unknown-linux-musleabihf/release/conduit`
+
+### Testing Conduit
+```sh
+$ TARGET=armv7-unknown-linux-musleabihf ./cross/test.sh --release
+```
--- a/docker/ci-binaries-packaging.Dockerfile
+++ b/docker/ci-binaries-packaging.Dockerfile
@ -9,19 +9,21 @@

 FROM docker.io/alpine:3.15.0 AS runner

+
 # Standard port on which Conduit launches.
 # You still need to map the port when using the docker command or docker-compose.
 EXPOSE 6167

-# Note from @jfowl: I would like to remove this in the future and just have the Docker version be configured with envs. 
-ENV CONDUIT_CONFIG="/srv/conduit/conduit.toml"
+# Note from @jfowl: I would like to remove the config file in the future and just have the Docker version be configured with envs. 
+ENV CONDUIT_CONFIG="/srv/conduit/conduit.toml" \
+    CONDUIT_PORT=6167

 # Conduit needs:
 #   ca-certificates: for https
-#   libgcc: Apparently this is needed, even if I (@jfowl) don't know exactly why. But whatever, it's not that big.
+#   iproute2: for `ss` for the healthcheck script
 RUN apk add --no-cache \
    ca-certificates \
-    libgcc
+    iproute2


 ARG CREATED
--- a/docker/healthcheck.sh
+++ b/docker/healthcheck.sh
@ -1,8 +1,9 @@
 #!/bin/sh

-# If the port is not specified as env var, take it from the config file
-if [ -z ${CONDUIT_PORT} ]; then
-  CONDUIT_PORT=$(grep -m1 -o 'port\s=\s[0-9]*' conduit.toml | grep -m1 -o '[0-9]*')
+# If the config file does not contain a default port and the CONDUIT_PORT env is not set, create
+# try to get port from process list
+if [ -z "${CONDUIT_PORT}" ]; then
+  CONDUIT_PORT=$(ss -tlpn | grep conduit | grep -m1 -o ':[0-9]*' | grep -m1 -o '[0-9]*')
 fi

 # The actual health check.
--- a/src/client_server/account.rs
+++ b/src/client_server/account.rs
@ -60,15 +60,16 @@ pub async fn get_register_available_route(
    body: Ruma<get_username_availability::Request<'_>>,
 ) -> ConduitResult<get_username_availability::Response> {
    // Validate user id
-    let user_id = UserId::parse_with_server_name(body.username.clone(), db.globals.server_name())
-        .ok()
-        .filter(|user_id| {
-            !user_id.is_historical() && user_id.server_name() == db.globals.server_name()
-        })
-        .ok_or(Error::BadRequest(
-            ErrorKind::InvalidUsername,
-            "Username is invalid.",
-        ))?;
+    let user_id =
+        UserId::parse_with_server_name(body.username.to_lowercase(), db.globals.server_name())
+            .ok()
+            .filter(|user_id| {
+                !user_id.is_historical() && user_id.server_name() == db.globals.server_name()
+            })
+            .ok_or(Error::BadRequest(
+                ErrorKind::InvalidUsername,
+                "Username is invalid.",
+            ))?;

    // Check if username is creative enough
    if db.users.exists(&user_id)? {
--- a/src/client_server/media.rs
+++ b/src/client_server/media.rs
@ -4,7 +4,10 @@ use crate::{
 };
 use ruma::api::client::{
    error::ErrorKind,
-    r0::media::{create_content, get_content, get_content_thumbnail, get_media_config},
+    r0::media::{
+        create_content, get_content, get_content_as_filename, get_content_thumbnail,
+        get_media_config,
+    },
 };
 use std::convert::TryInto;

@ -71,7 +74,39 @@ pub async fn create_content_route(
    .into())
 }

-/// # `POST /_matrix/media/r0/download/{serverName}/{mediaId}`
+pub async fn get_remote_content(
+    db: &DatabaseGuard,
+    mxc: &str,
+    server_name: &ruma::ServerName,
+    media_id: &str,
+) -> Result<get_content::Response, Error> {
+    let content_response = db
+        .sending
+        .send_federation_request(
+            &db.globals,
+            server_name,
+            get_content::Request {
+                allow_remote: false,
+                server_name,
+                media_id,
+            },
+        )
+        .await?;
+
+    db.media
+        .create(
+            mxc.to_string(),
+            &db.globals,
+            &content_response.content_disposition.as_deref(),
+            &content_response.content_type.as_deref(),
+            &content_response.file,
+        )
+        .await?;
+
+    Ok(content_response)
+}
+
+/// # `GET /_matrix/media/r0/download/{serverName}/{mediaId}`
 ///
 /// Load media from our server or over federation.
 ///
@ -100,36 +135,58 @@ pub async fn get_content_route(
        }
        .into())
    } else if &*body.server_name != db.globals.server_name() && body.allow_remote {
-        let get_content_response = db
-            .sending
-            .send_federation_request(
-                &db.globals,
-                &body.server_name,
-                get_content::Request {
-                    allow_remote: false,
-                    server_name: &body.server_name,
-                    media_id: &body.media_id,
-                },
-            )
-            .await?;
+        let remote_content_response =
+            get_remote_content(&db, &mxc, &body.server_name, &body.media_id).await?;
+        Ok(remote_content_response.into())
+    } else {
+        Err(Error::BadRequest(ErrorKind::NotFound, "Media not found."))
+    }
+}

-        db.media
-            .create(
-                mxc,
-                &db.globals,
-                &get_content_response.content_disposition.as_deref(),
-                &get_content_response.content_type.as_deref(),
-                &get_content_response.file,
-            )
-            .await?;
+/// # `GET /_matrix/media/r0/download/{serverName}/{mediaId}/{fileName}`
+///
+/// Load media from our server or over federation, permitting desired filename.
+///
+/// - Only allows federation if `allow_remote` is true
+#[cfg_attr(
+    feature = "conduit_bin",
+    get("/_matrix/media/r0/download/<_>/<_>/<_>", data = "<body>")
+)]
+#[tracing::instrument(skip(db, body))]
+pub async fn get_content_as_filename_route(
+    db: DatabaseGuard,
+    body: Ruma<get_content_as_filename::Request<'_>>,
+) -> ConduitResult<get_content_as_filename::Response> {
+    let mxc = format!("mxc://{}/{}", body.server_name, body.media_id);

-        Ok(get_content_response.into())
+    if let Some(FileMeta {
+        content_disposition: _,
+        content_type,
+        file,
+    }) = db.media.get(&db.globals, &mxc).await?
+    {
+        Ok(get_content_as_filename::Response {
+            file,
+            content_type,
+            content_disposition: Some(format!("inline; filename={}", body.filename)),
+        }
+        .into())
+    } else if &*body.server_name != db.globals.server_name() && body.allow_remote {
+        let remote_content_response =
+            get_remote_content(&db, &mxc, &body.server_name, &body.media_id).await?;
+
+        Ok(get_content_as_filename::Response {
+            content_disposition: Some(format!("inline: filename={}", body.filename)),
+            content_type: remote_content_response.content_type,
+            file: remote_content_response.file,
+        }
+        .into())
    } else {
        Err(Error::BadRequest(ErrorKind::NotFound, "Media not found."))
    }
 }

-/// # `POST /_matrix/media/r0/thumbnail/{serverName}/{mediaId}`
+/// # `GET /_matrix/media/r0/thumbnail/{serverName}/{mediaId}`
 ///
 /// Load media thumbnail from our server or over federation.
 ///
@ -150,7 +207,7 @@ pub async fn get_content_thumbnail_route(
    }) = db
        .media
        .get_thumbnail(
-            mxc.clone(),
+            &mxc,
            &db.globals,
            body.width
                .try_into()
--- a/src/client_server/membership.rs
+++ b/src/client_server/membership.rs
@ -655,7 +655,7 @@ async fn join_room_by_id_helper(

        db.rooms.get_or_create_shortroomid(room_id, &db.globals)?;

-        let pdu = PduEvent::from_id_val(event_id, join_event.clone())
+        let parsed_pdu = PduEvent::from_id_val(event_id, join_event.clone())
            .map_err(|_| Error::BadServerResponse("Invalid join event PDU."))?;

        let mut state = HashMap::new();
@ -695,14 +695,15 @@ async fn join_room_by_id_helper(
        }

        let incoming_shortstatekey = db.rooms.get_or_create_shortstatekey(
-            &pdu.kind,
-            pdu.state_key
+            &parsed_pdu.kind,
+            parsed_pdu
+                .state_key
                .as_ref()
                .expect("Pdu is a membership state event"),
            &db.globals,
        )?;

-        state.insert(incoming_shortstatekey, pdu.event_id.clone());
+        state.insert(incoming_shortstatekey, parsed_pdu.event_id.clone());

        let create_shortstatekey = db
            .rooms
@ -738,12 +739,12 @@ async fn join_room_by_id_helper(

        // We append to state before appending the pdu, so we don't have a moment in time with the
        // pdu without it's state. This is okay because append_pdu can't fail.
-        let statehashid = db.rooms.append_to_state(&pdu, &db.globals)?;
+        let statehashid = db.rooms.append_to_state(&parsed_pdu, &db.globals)?;

        db.rooms.append_pdu(
-            &pdu,
-            utils::to_canonical_object(&pdu).expect("Pdu is valid canonical object"),
-            iter::once(&*pdu.event_id),
+            &parsed_pdu,
+            join_event,
+            iter::once(&*parsed_pdu.event_id),
            db,
        )?;

--- a/src/client_server/room.rs
+++ b/src/client_server/room.rs
@ -344,10 +344,13 @@ pub async fn create_room_route(

    // 6. Events listed in initial_state
    for event in &body.initial_state {
-        let pdu_builder = PduBuilder::from(event.deserialize().map_err(|e| {
+        let mut pdu_builder = event.deserialize_as::<PduBuilder>().map_err(|e| {
            warn!("Invalid initial state event: {:?}", e);
            Error::BadRequest(ErrorKind::InvalidParam, "Invalid initial state event.")
-        })?);
+        })?;
+
+        // Implicit state key defaults to ""
+        pdu_builder.state_key.get_or_insert_with(|| "".to_owned());

        // Silently skip encryption events if they are not allowed
        if pdu_builder.event_type == EventType::RoomEncryption && !db.globals.allow_encryption() {
--- a/src/database.rs
+++ b/src/database.rs
@ -49,6 +49,8 @@ pub struct Config {
    database_path: String,
    #[serde(default = "default_db_cache_capacity_mb")]
    db_cache_capacity_mb: f64,
+    #[serde(default = "default_conduit_cache_capacity_modifier")]
+    conduit_cache_capacity_modifier: f64,
    #[serde(default = "default_rocksdb_max_open_files")]
    rocksdb_max_open_files: i32,
    #[serde(default = "default_pdu_cache_capacity")]
@ -129,8 +131,12 @@ fn default_db_cache_capacity_mb() -> f64 {
    10.0
 }

+fn default_conduit_cache_capacity_modifier() -> f64 {
+    1.0
+}
+
 fn default_rocksdb_max_open_files() -> i32 {
-    512
+    20
 }

 fn default_pdu_cache_capacity() -> u32 {
@ -361,15 +367,27 @@ impl Database {
                        .try_into()
                        .expect("pdu cache capacity fits into usize"),
                )),
-                auth_chain_cache: Mutex::new(LruCache::new(1_000_000)),
-                shorteventid_cache: Mutex::new(LruCache::new(1_000_000)),
-                eventidshort_cache: Mutex::new(LruCache::new(1_000_000)),
-                shortstatekey_cache: Mutex::new(LruCache::new(1_000_000)),
-                statekeyshort_cache: Mutex::new(LruCache::new(1_000_000)),
+                auth_chain_cache: Mutex::new(LruCache::new(
+                    (100_000.0 * config.conduit_cache_capacity_modifier) as usize,
+                )),
+                shorteventid_cache: Mutex::new(LruCache::new(
+                    (100_000.0 * config.conduit_cache_capacity_modifier) as usize,
+                )),
+                eventidshort_cache: Mutex::new(LruCache::new(
+                    (100_000.0 * config.conduit_cache_capacity_modifier) as usize,
+                )),
+                shortstatekey_cache: Mutex::new(LruCache::new(
+                    (100_000.0 * config.conduit_cache_capacity_modifier) as usize,
+                )),
+                statekeyshort_cache: Mutex::new(LruCache::new(
+                    (100_000.0 * config.conduit_cache_capacity_modifier) as usize,
+                )),
                our_real_users_cache: RwLock::new(HashMap::new()),
                appservice_in_room_cache: RwLock::new(HashMap::new()),
                lazy_load_waiting: Mutex::new(HashMap::new()),
-                stateinfo_cache: Mutex::new(LruCache::new(1000)),
+                stateinfo_cache: Mutex::new(LruCache::new(
+                    (100.0 * config.conduit_cache_capacity_modifier) as usize,
+                )),
            },
            account_data: account_data::AccountData {
                roomuserdataid_accountdata: builder.open_tree("roomuserdataid_accountdata")?,
--- a/src/database/admin.rs
+++ b/src/database/admin.rs
@ -262,7 +262,10 @@ fn process_admin_command(
                let parsed_config = serde_yaml::from_str::<serde_yaml::Value>(&appservice_config);
                match parsed_config {
                    Ok(yaml) => match db.appservice.register_appservice(yaml) {
-                        Ok(()) => RoomMessageEventContent::text_plain("Appservice registered."),
+                        Ok(id) => RoomMessageEventContent::text_plain(format!(
+                            "Appservice registered with ID: {}.",
+                            id
+                        )),
                        Err(e) => RoomMessageEventContent::text_plain(format!(
                            "Failed to register appservice: {}",
                            e
--- a/src/database/appservice.rs
+++ b/src/database/appservice.rs
@ -12,7 +12,9 @@ pub struct Appservice {
 }

 impl Appservice {
-    pub fn register_appservice(&self, yaml: serde_yaml::Value) -> Result<()> {
+    /// Registers an appservice and returns the ID to the caller
+    ///
+    pub fn register_appservice(&self, yaml: serde_yaml::Value) -> Result<String> {
        // TODO: Rumaify
        let id = yaml.get("id").unwrap().as_str().unwrap();
        self.id_appserviceregistrations.insert(
@ -22,9 +24,9 @@ impl Appservice {
        self.cached_registrations
            .write()
            .unwrap()
-            .insert(id.to_owned(), yaml);
+            .insert(id.to_owned(), yaml.to_owned());

-        Ok(())
+        Ok(id.to_owned())
    }

    /// Remove an appservice registration
--- a/src/database/key_backups.rs
+++ b/src/database/key_backups.rs
@ -7,7 +7,6 @@ use ruma::{
    serde::Raw,
    RoomId, UserId,
 };
-use serde_json::json;
 use std::{collections::BTreeMap, sync::Arc};

 use super::abstraction::Tree;
@ -212,13 +211,13 @@ impl KeyBackups {
        &self,
        user_id: &UserId,
        version: &str,
-    ) -> Result<BTreeMap<Box<RoomId>, Raw<RoomKeyBackup>>> {
+    ) -> Result<BTreeMap<Box<RoomId>, RoomKeyBackup>> {
        let mut prefix = user_id.as_bytes().to_vec();
        prefix.push(0xff);
        prefix.extend_from_slice(version.as_bytes());
        prefix.push(0xff);

-        let mut rooms = BTreeMap::<Box<RoomId>, Raw<RoomKeyBackup>>::new();
+        let mut rooms = BTreeMap::<Box<RoomId>, RoomKeyBackup>::new();

        for result in self
            .backupkeyid_backup
@ -244,7 +243,7 @@ impl KeyBackups {
                    Error::bad_database("backupkeyid_backup room_id is invalid room id.")
                })?;

-                let key_data: serde_json::Value = serde_json::from_slice(&value).map_err(|_| {
+                let key_data = serde_json::from_slice(&value).map_err(|_| {
                    Error::bad_database("KeyBackupData in backupkeyid_backup is invalid.")
                })?;

@ -252,25 +251,13 @@ impl KeyBackups {
            })
        {
            let (room_id, session_id, key_data) = result?;
-            let room_key_backup = rooms.entry(room_id).or_insert_with(|| {
-                Raw::new(&RoomKeyBackup {
+            rooms
+                .entry(room_id)
+                .or_insert_with(|| RoomKeyBackup {
                    sessions: BTreeMap::new(),
                })
-                .expect("RoomKeyBackup serialization")
-            });
-
-            let mut object = room_key_backup
-                .deserialize_as::<serde_json::Map<String, serde_json::Value>>()
-                .map_err(|_| Error::bad_database("RoomKeyBackup is not an object"))?;
-
-            let sessions = object.entry("session").or_insert_with(|| json!({}));
-            if let serde_json::Value::Object(unsigned_object) = sessions {
-                unsigned_object.insert(session_id, key_data);
-            }
-
-            *room_key_backup = Raw::from_json(
-                serde_json::value::to_raw_value(&object).expect("Value => RawValue serialization"),
-            );
+                .sessions
+                .insert(session_id, key_data);
        }

        Ok(rooms)
--- a/src/database/media.rs
+++ b/src/database/media.rs
@ -171,7 +171,7 @@ impl Media {
    /// For width,height <= 96 the server uses another thumbnailing algorithm which crops the image afterwards.
    pub async fn get_thumbnail(
        &self,
-        mxc: String,
+        mxc: &str,
        globals: &Globals,
        width: u32,
        height: u32,
--- a/src/database/sending.rs
+++ b/src/database/sending.rs
@ -480,6 +480,26 @@ impl Sending {
        hash.as_ref().to_owned()
    }

+    /// Cleanup event data
+    /// Used for instance after we remove an appservice registration
+    ///
+    #[tracing::instrument(skip(self))]
+    pub fn cleanup_events(&self, key_id: &str) -> Result<()> {
+        let mut prefix = b"+".to_vec();
+        prefix.extend_from_slice(key_id.as_bytes());
+        prefix.push(0xff);
+
+        for (key, _) in self.servercurrentevent_data.scan_prefix(prefix.clone()) {
+            self.servercurrentevent_data.remove(&key).unwrap();
+        }
+
+        for (key, _) in self.servernameevent_data.scan_prefix(prefix.clone()) {
+            self.servernameevent_data.remove(&key).unwrap();
+        }
+
+        Ok(())
+    }
+
    #[tracing::instrument(skip(db, events, kind))]
    async fn handle_events(
        kind: OutgoingKind,
@ -520,8 +540,15 @@ impl Sending {
                    &db.globals,
                    db.appservice
                        .get_registration(server.as_str())
-                        .unwrap()
-                        .unwrap(), // TODO: handle error
+                        .map_err(|e| (kind.clone(), e))?
+                        .ok_or_else(|| {
+                            (
+                                kind.clone(),
+                                Error::bad_database(
+                                    "[Appservice] Could not load registration from db.",
+                                ),
+                            )
+                        })?,
                    appservice::event::push_events::v1::Request {
                        events: &pdu_jsons,
                        txn_id: (&*base64::encode_config(
--- a/src/main.rs
+++ b/src/main.rs
@ -129,6 +129,7 @@ fn setup_rocket(config: Figment, data: Arc<RwLock<Database>>) -> rocket::Rocket<
                client_server::send_event_to_device_route,
                client_server::get_media_config_route,
                client_server::create_content_route,
+                client_server::get_content_as_filename_route,
                client_server::get_content_route,
                client_server::get_content_thumbnail_route,
                client_server::get_devices_route,
--- a/src/pdu.rs
+++ b/src/pdu.rs
@ -1,9 +1,8 @@
 use crate::Error;
 use ruma::{
    events::{
-        room::member::RoomMemberEventContent, AnyEphemeralRoomEvent, AnyInitialStateEvent,
-        AnyRoomEvent, AnyStateEvent, AnyStrippedStateEvent, AnySyncRoomEvent, AnySyncStateEvent,
-        EventType, StateEvent,
+        room::member::RoomMemberEventContent, AnyEphemeralRoomEvent, AnyRoomEvent, AnyStateEvent,
+        AnyStrippedStateEvent, AnySyncRoomEvent, AnySyncStateEvent, EventType, StateEvent,
    },
    serde::{CanonicalJsonObject, CanonicalJsonValue, Raw},
    state_res, EventId, MilliSecondsSinceUnixEpoch, RoomId, RoomVersionId, UInt, UserId,
@ -361,17 +360,3 @@ pub struct PduBuilder {
    pub state_key: Option<String>,
    pub redacts: Option<Arc<EventId>>,
 }
-
-/// Direct conversion prevents loss of the empty `state_key` that ruma requires.
-impl From<AnyInitialStateEvent> for PduBuilder {
-    fn from(event: AnyInitialStateEvent) -> Self {
-        Self {
-            event_type: EventType::from(event.event_type()),
-            content: to_raw_value(&event.content())
-                .expect("AnyStateEventContent came from JSON and can thus turn back into JSON."),
-            unsigned: None,
-            state_key: Some(event.state_key().to_owned()),
-            redacts: None,
-        }
-    }
-}