From d94f3c1e9aad363aff552933b104944094f7ddc2 Mon Sep 17 00:00:00 2001 From: Maxim De Clercq Date: Fri, 21 Jan 2022 17:06:15 +0100 Subject: [PATCH 01/40] fix: make sure cc-rs and bindgen use the correct paths when cross-compiling --- .gitlab-ci.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index cdc1d4cb..d0d4f3e3 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -33,6 +33,12 @@ variables: - "rustup target add $TARGET" # If provided, bring in caching through sccache, which uses an external S3 endpoint to store compilation results: - if [ -n "${SCCACHE_BIN_URL}" ]; then curl $SCCACHE_BIN_URL --output /sccache && chmod +x /sccache && export RUSTC_WRAPPER=/sccache; fi + # Make sure that cc-rs links the correct libraries when cross-compiling (required for compiling librocksdb-sys) + # See https://github.com/alexcrichton/cc-rs#external-configuration-via-environment-variables for more information + - 'export TARGET_CFLAGS="-L$TARGET_HOME/lib -latomic"' + # Make sure that rust-bindgen uses the correct include path when cross-compiling (required for compiling librocksdb-sys) + # See https://github.com/rust-lang/rust-bindgen#environment-variables for more information + - 'export BINDGEN_EXTRA_CLANG_ARGS="-I$TARGET_C_INCLUDE_PATH"' script: - time cargo build --target $TARGET --release - 'cp "target/$TARGET/release/conduit" "conduit-$TARGET"' From bfef94f5f4f465156317e5a6d60fffc8e1fd9240 Mon Sep 17 00:00:00 2001 From: Maxim De Clercq Date: Fri, 21 Jan 2022 17:26:25 +0100 Subject: [PATCH 02/40] fix: linking against libatomic is no longer required since the library path is fixed --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index d0d4f3e3..236ce0ae 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -35,7 +35,7 @@ variables: - if [ -n "${SCCACHE_BIN_URL}" ]; then curl $SCCACHE_BIN_URL --output /sccache && chmod +x /sccache && export RUSTC_WRAPPER=/sccache; fi # Make sure that cc-rs links the correct libraries when cross-compiling (required for compiling librocksdb-sys) # See https://github.com/alexcrichton/cc-rs#external-configuration-via-environment-variables for more information - - 'export TARGET_CFLAGS="-L$TARGET_HOME/lib -latomic"' + - 'export TARGET_CFLAGS="-L$TARGET_HOME/lib"' # Make sure that rust-bindgen uses the correct include path when cross-compiling (required for compiling librocksdb-sys) # See https://github.com/rust-lang/rust-bindgen#environment-variables for more information - 'export BINDGEN_EXTRA_CLANG_ARGS="-I$TARGET_C_INCLUDE_PATH"' From 3e9abfedb43e6f52ebb3f24adaf8bf0871712181 Mon Sep 17 00:00:00 2001 From: Maxim De Clercq Date: Sat, 22 Jan 2022 00:14:19 +0100 Subject: [PATCH 03/40] fix: make sure libstdc++ is linked statically when cross-compiling --- .gitlab-ci.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 236ce0ae..5cae7439 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -33,10 +33,10 @@ variables: - "rustup target add $TARGET" # If provided, bring in caching through sccache, which uses an external S3 endpoint to store compilation results: - if [ -n "${SCCACHE_BIN_URL}" ]; then curl $SCCACHE_BIN_URL --output /sccache && chmod +x /sccache && export RUSTC_WRAPPER=/sccache; fi - # Make sure that cc-rs links the correct libraries when cross-compiling (required for compiling librocksdb-sys) + # Make sure that cc-rs links the correct libraries statically when cross-compiling # See https://github.com/alexcrichton/cc-rs#external-configuration-via-environment-variables for more information - - 'export TARGET_CFLAGS="-L$TARGET_HOME/lib"' - # Make sure that rust-bindgen uses the correct include path when cross-compiling (required for compiling librocksdb-sys) + - 'export CARGO_BUILD_RUSTFLAGS="-L$TARGET_HOME/lib" CXXSTDLIB="static=stdc++"' + # Make sure that rust-bindgen uses the correct include path when cross-compiling # See https://github.com/rust-lang/rust-bindgen#environment-variables for more information - 'export BINDGEN_EXTRA_CLANG_ARGS="-I$TARGET_C_INCLUDE_PATH"' script: From a021680591cf581fccd05a9dbb0914163f69e8ba Mon Sep 17 00:00:00 2001 From: Maxim De Clercq Date: Sat, 22 Jan 2022 01:14:36 +0100 Subject: [PATCH 04/40] fix: make sure libatomic is always linked because it's skipped on arm targets --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 5cae7439..b863de98 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -35,7 +35,7 @@ variables: - if [ -n "${SCCACHE_BIN_URL}" ]; then curl $SCCACHE_BIN_URL --output /sccache && chmod +x /sccache && export RUSTC_WRAPPER=/sccache; fi # Make sure that cc-rs links the correct libraries statically when cross-compiling # See https://github.com/alexcrichton/cc-rs#external-configuration-via-environment-variables for more information - - 'export CARGO_BUILD_RUSTFLAGS="-L$TARGET_HOME/lib" CXXSTDLIB="static=stdc++"' + - 'export CARGO_BUILD_RUSTFLAGS="-L$TARGET_HOME/lib -latomic" CXXSTDLIB="static=stdc++"' # Make sure that rust-bindgen uses the correct include path when cross-compiling # See https://github.com/rust-lang/rust-bindgen#environment-variables for more information - 'export BINDGEN_EXTRA_CLANG_ARGS="-I$TARGET_C_INCLUDE_PATH"' From cd9902637ddf3f8e7711f01a5cf044725704e28a Mon Sep 17 00:00:00 2001 From: Maxim De Clercq Date: Sat, 22 Jan 2022 17:34:30 +0100 Subject: [PATCH 05/40] feat: use rustembedded/cross images and use static relocation model to fix cross-compile --- .gitlab-ci.yml | 39 ++++++++++++++++++++++++--------------- CROSS_COMPILE.md | 11 ----------- Cross.toml | 11 +++++++++++ DEPLOY.md | 2 +- cross/README.md | 37 +++++++++++++++++++++++++++++++++++++ cross/build.sh | 31 +++++++++++++++++++++++++++++++ cross/test.sh | 8 ++++++++ 7 files changed, 112 insertions(+), 27 deletions(-) delete mode 100644 CROSS_COMPILE.md create mode 100644 Cross.toml create mode 100644 cross/README.md create mode 100755 cross/build.sh create mode 100755 cross/test.sh diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index b863de98..993145a1 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -21,33 +21,46 @@ variables: - if: '$CI_COMMIT_BRANCH == "next"' - if: "$CI_COMMIT_TAG" interruptible: true - image: "rust:1.56" + image: "rust:1.58" tags: ["docker"] + services: ["docker:dind"] variables: + DOCKER_HOST: tcp://docker:2375/ + DOCKER_TLS_CERTDIR: "" + DOCKER_DRIVER: overlay2 + SHARED_PATH: $CI_PROJECT_DIR/shared/ CARGO_PROFILE_RELEASE_LTO: "true" CARGO_PROFILE_RELEASE_CODEGEN_UNITS: "1" CARGO_INCREMENTAL: "false" # https://matklad.github.io/2021/09/04/fast-rust-builds.html#ci-workflow before_script: - 'echo "Building for target $TARGET"' - - "rustc --version && cargo --version && rustup show" # Print version info for debugging - - "rustup target add $TARGET" + - "rustup show && rustc --version && cargo --version" # Print version info for debugging # If provided, bring in caching through sccache, which uses an external S3 endpoint to store compilation results: - if [ -n "${SCCACHE_BIN_URL}" ]; then curl $SCCACHE_BIN_URL --output /sccache && chmod +x /sccache && export RUSTC_WRAPPER=/sccache; fi - # Make sure that cc-rs links the correct libraries statically when cross-compiling - # See https://github.com/alexcrichton/cc-rs#external-configuration-via-environment-variables for more information - - 'export CARGO_BUILD_RUSTFLAGS="-L$TARGET_HOME/lib -latomic" CXXSTDLIB="static=stdc++"' - # Make sure that rust-bindgen uses the correct include path when cross-compiling - # See https://github.com/rust-lang/rust-bindgen#environment-variables for more information - - 'export BINDGEN_EXTRA_CLANG_ARGS="-I$TARGET_C_INCLUDE_PATH"' script: - - time cargo build --target $TARGET --release + # install cross-compiling prerequisites + - 'apt-get update && apt-get install -y docker.io && docker version' # install docker + - 'cargo install cross && cross --version' # install cross + # fix cargo and rustup mounts from this container (https://gitlab.com/gitlab-org/gitlab-foss/-/issues/41227) + - 'mkdir -p $SHARED_PATH/cargo' + - 'cp -r $CARGO_HOME/bin $SHARED_PATH/cargo' + - 'cp -r $RUSTUP_HOME $SHARED_PATH' + - 'export CARGO_HOME=$SHARED_PATH/cargo RUSTUP_HOME=$SHARED_PATH/rustup' + # cross-compile conduit for target + - 'time ./cross/build.sh --locked --release' - 'cp "target/$TARGET/release/conduit" "conduit-$TARGET"' + cache: + # https://doc.rust-lang.org/cargo/guide/cargo-home.html#caching-the-cargo-home-in-ci + - key: 'cargo-cache-$TARGET' + paths: + - $SHARED_PATH/cargo/registry/index + - $SHARED_PATH/cargo/registry/cache + - $SHARED_PATH/cargo/git/db artifacts: expire_in: never build:release:cargo:x86_64-unknown-linux-musl-with-debug: extends: .build-cargo-shared-settings - image: messense/rust-musl-cross:x86_64-musl variables: CARGO_PROFILE_RELEASE_DEBUG: 2 # Enable debug info for flamegraph profiling TARGET: "x86_64-unknown-linux-musl" @@ -61,7 +74,6 @@ build:release:cargo:x86_64-unknown-linux-musl-with-debug: build:release:cargo:x86_64-unknown-linux-musl: extends: .build-cargo-shared-settings - image: messense/rust-musl-cross:x86_64-musl variables: TARGET: "x86_64-unknown-linux-musl" artifacts: @@ -72,7 +84,6 @@ build:release:cargo:x86_64-unknown-linux-musl: build:release:cargo:arm-unknown-linux-musleabihf: extends: .build-cargo-shared-settings - image: messense/rust-musl-cross:arm-musleabihf variables: TARGET: "arm-unknown-linux-musleabihf" artifacts: @@ -83,7 +94,6 @@ build:release:cargo:arm-unknown-linux-musleabihf: build:release:cargo:armv7-unknown-linux-musleabihf: extends: .build-cargo-shared-settings - image: messense/rust-musl-cross:armv7-musleabihf variables: TARGET: "armv7-unknown-linux-musleabihf" artifacts: @@ -94,7 +104,6 @@ build:release:cargo:armv7-unknown-linux-musleabihf: build:release:cargo:aarch64-unknown-linux-musl: extends: .build-cargo-shared-settings - image: messense/rust-musl-cross:aarch64-musl variables: TARGET: "aarch64-unknown-linux-musl" artifacts: diff --git a/CROSS_COMPILE.md b/CROSS_COMPILE.md deleted file mode 100644 index e38a6ad7..00000000 --- a/CROSS_COMPILE.md +++ /dev/null @@ -1,11 +0,0 @@ -Install docker: - -``` -$ sudo apt install docker -$ sudo usermod -aG docker $USER -$ exec sudo su -l $USER -$ sudo systemctl start docker -$ cargo install cross -$ cross build --release --target armv7-unknown-linux-musleabihf -``` -The cross-compiled binary is at target/armv7-unknown-linux-musleabihf/release/conduit diff --git a/Cross.toml b/Cross.toml new file mode 100644 index 00000000..491efcb7 --- /dev/null +++ b/Cross.toml @@ -0,0 +1,11 @@ +[target.aarch64-unknown-linux-musl] +image = "rust-cross:aarch64-unknown-linux-musl" + +[target.arm-unknown-linux-musleabihf] +image = "rust-cross:arm-unknown-linux-musleabihf" + +[target.armv7-unknown-linux-musleabihf] +image = "rust-cross:armv7-unknown-linux-musleabihf" + +[target.x86_64-unknown-linux-musl] +image = "rust-cross:x86_64-unknown-linux-musl" diff --git a/DEPLOY.md b/DEPLOY.md index 0058b93d..38e1e286 100644 --- a/DEPLOY.md +++ b/DEPLOY.md @@ -37,7 +37,7 @@ $ cargo build --release Note that this currently requires Rust 1.50. -If you want to cross compile Conduit to another architecture, read the [Cross-Compile Guide](CROSS_COMPILE.md). +If you want to cross compile Conduit to another architecture, read the [Cross-Compile Guide](cross/README.md). ## Adding a Conduit user diff --git a/cross/README.md b/cross/README.md new file mode 100644 index 00000000..2829d239 --- /dev/null +++ b/cross/README.md @@ -0,0 +1,37 @@ +## Cross compilation + +The `cross` folder contains a set of convenience scripts (`build.sh` and `test.sh`) for cross-compiling Conduit. + +Currently supported targets are + +- aarch64-unknown-linux-musl +- arm-unknown-linux-musleabihf +- armv7-unknown-linux-musleabihf +- x86\_64-unknown-linux-musl + +### Install prerequisites +#### Docker +[Installation guide](https://docs.docker.com/get-docker/). +```sh +$ sudo apt install docker +$ sudo systemctl start docker +$ sudo usermod -aG docker $USER +$ newgrp docker +``` + +#### Cross +[Installation guide](https://github.com/rust-embedded/cross/#installation). +```sh +$ cargo install cross +``` + +### Buiding Conduit +```sh +$ TARGET=armv7-unknown-linux-musleabihf ./cross/build.sh --release +``` +The cross-compiled binary is at `target/armv7-unknown-linux-musleabihf/release/conduit` + +### Testing Conduit +```sh +$ TARGET=armv7-unknown-linux-musleabihf ./cross/test.sh --release +``` diff --git a/cross/build.sh b/cross/build.sh new file mode 100755 index 00000000..4a6d4493 --- /dev/null +++ b/cross/build.sh @@ -0,0 +1,31 @@ +#!/bin/bash +set -ex + +# build custom container with libclang and static compilation +tag="rust-cross:${TARGET:?}" +docker build --tag="$tag" - << EOF +FROM rustembedded/cross:$TARGET + +# Install libclang for generating bindings with rust-bindgen +# The architecture is not relevant here since it's not used for compilation +RUN apt-get update && \ + apt-get install --assume-yes libclang-dev + +# Set the target prefix +ENV TARGET_PREFIX="/usr/local/$(echo "${TARGET:?}" | sed -e 's/armv7/arm/' -e 's/-unknown//')" + +# Make sure that cc-rs links libc/libstdc++ statically when cross-compiling +# See https://github.com/alexcrichton/cc-rs#external-configuration-via-environment-variables for more information +ENV RUSTFLAGS="-L\$TARGET_PREFIX/lib" CXXSTDLIB="static=stdc++" +# Forcefully linking against libatomic, libc and libgcc is required for arm32, otherwise symbols are missing +$([[ $TARGET =~ arm ]] && echo 'ENV RUSTFLAGS="$RUSTFLAGS -Clink-arg=-lgcc -Clink-arg=-latomic -lstatic=c"') +# Strip symbols while compiling in release mode +$([[ $@ =~ -r ]] && echo 'ENV RUSTFLAGS="$RUSTFLAGS -Clink-arg=-s"') + +# Make sure that rust-bindgen uses the correct include path when cross-compiling +# See https://github.com/rust-lang/rust-bindgen#environment-variables for more information +ENV BINDGEN_EXTRA_CLANG_ARGS="-I\$TARGET_PREFIX/include" +EOF + +# build conduit for a specific target +cross build --target="$TARGET" $@ diff --git a/cross/test.sh b/cross/test.sh new file mode 100755 index 00000000..0aa0909c --- /dev/null +++ b/cross/test.sh @@ -0,0 +1,8 @@ +#!/usr/bin/env sh +set -ex + +# Build conduit for a specific target +cross/build.sh $@ + +# Test conduit for a specific target +cross test --target="$TARGET" $@ From fd67cd7450e33b97050372bdd13832828fa75458 Mon Sep 17 00:00:00 2001 From: Maxim De Clercq Date: Sun, 23 Jan 2022 15:10:42 +0100 Subject: [PATCH 06/40] feat: support targetting i686 --- .gitlab-ci.yml | 28 +++++++++++++++++++--------- Cross.toml | 3 +++ cross/build.sh | 6 ++++-- 3 files changed, 26 insertions(+), 11 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 993145a1..b5a12f3d 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -37,7 +37,6 @@ variables: - "rustup show && rustc --version && cargo --version" # Print version info for debugging # If provided, bring in caching through sccache, which uses an external S3 endpoint to store compilation results: - if [ -n "${SCCACHE_BIN_URL}" ]; then curl $SCCACHE_BIN_URL --output /sccache && chmod +x /sccache && export RUSTC_WRAPPER=/sccache; fi - script: # install cross-compiling prerequisites - 'apt-get update && apt-get install -y docker.io && docker version' # install docker - 'cargo install cross && cross --version' # install cross @@ -46,16 +45,17 @@ variables: - 'cp -r $CARGO_HOME/bin $SHARED_PATH/cargo' - 'cp -r $RUSTUP_HOME $SHARED_PATH' - 'export CARGO_HOME=$SHARED_PATH/cargo RUSTUP_HOME=$SHARED_PATH/rustup' + script: # cross-compile conduit for target - 'time ./cross/build.sh --locked --release' - - 'cp "target/$TARGET/release/conduit" "conduit-$TARGET"' + - 'mv "target/$TARGET/release/conduit" "conduit-$TARGET"' cache: # https://doc.rust-lang.org/cargo/guide/cargo-home.html#caching-the-cargo-home-in-ci - - key: 'cargo-cache-$TARGET' - paths: - - $SHARED_PATH/cargo/registry/index - - $SHARED_PATH/cargo/registry/cache - - $SHARED_PATH/cargo/git/db + key: 'cargo-cache-$TARGET' + paths: + - $SHARED_PATH/cargo/registry/index + - $SHARED_PATH/cargo/registry/cache + - $SHARED_PATH/cargo/git/db artifacts: expire_in: never @@ -82,6 +82,16 @@ build:release:cargo:x86_64-unknown-linux-musl: - "conduit-x86_64-unknown-linux-musl" expose_as: "Conduit for x86_64-unknown-linux-musl" +build:release:cargo:i686-unknown-linux-musl: + extends: .build-cargo-shared-settings + variables: + TARGET: "i686-unknown-linux-musl" + artifacts: + name: "conduit-i686-unknown-linux-musl" + paths: + - "conduit-i686-unknown-linux-musl" + expose_as: "Conduit for i686-unknown-linux-musl" + build:release:cargo:arm-unknown-linux-musleabihf: extends: .build-cargo-shared-settings variables: @@ -119,14 +129,14 @@ build:release:cargo:aarch64-unknown-linux-musl: cache: key: "build_cache--$TARGET--$CI_COMMIT_BRANCH--debug" script: - - "time cargo build --target $TARGET" + # cross-compile conduit for target + - 'time ./cross/build.sh --locked' - 'mv "target/$TARGET/debug/conduit" "conduit-debug-$TARGET"' artifacts: expire_in: 4 weeks build:debug:cargo:x86_64-unknown-linux-musl: extends: ".cargo-debug-shared-settings" - image: messense/rust-musl-cross:x86_64-musl variables: TARGET: "x86_64-unknown-linux-musl" artifacts: diff --git a/Cross.toml b/Cross.toml index 491efcb7..22c84b97 100644 --- a/Cross.toml +++ b/Cross.toml @@ -7,5 +7,8 @@ image = "rust-cross:arm-unknown-linux-musleabihf" [target.armv7-unknown-linux-musleabihf] image = "rust-cross:armv7-unknown-linux-musleabihf" +[target.i686-unknown-linux-musl] +image = "rust-cross:i686-unknown-linux-musl" + [target.x86_64-unknown-linux-musl] image = "rust-cross:x86_64-unknown-linux-musl" diff --git a/cross/build.sh b/cross/build.sh index 4a6d4493..24a2224b 100755 --- a/cross/build.sh +++ b/cross/build.sh @@ -17,8 +17,10 @@ ENV TARGET_PREFIX="/usr/local/$(echo "${TARGET:?}" | sed -e 's/armv7/arm/' -e 's # Make sure that cc-rs links libc/libstdc++ statically when cross-compiling # See https://github.com/alexcrichton/cc-rs#external-configuration-via-environment-variables for more information ENV RUSTFLAGS="-L\$TARGET_PREFIX/lib" CXXSTDLIB="static=stdc++" -# Forcefully linking against libatomic, libc and libgcc is required for arm32, otherwise symbols are missing -$([[ $TARGET =~ arm ]] && echo 'ENV RUSTFLAGS="$RUSTFLAGS -Clink-arg=-lgcc -Clink-arg=-latomic -lstatic=c"') +# Forcefully linking against libatomic and libgcc is required for arm32, otherwise symbols are missing +$([[ $TARGET =~ arm ]] && echo 'ENV RUSTFLAGS="$RUSTFLAGS -Clink-arg=-lgcc -Clink-arg=-latomic"') +# Forcefully linking against libc is required for 32-bit, otherwise symbols are missing +$([[ $TARGET =~ arm|i686 ]] && echo 'ENV RUSTFLAGS="$RUSTFLAGS -lstatic=c"') # Strip symbols while compiling in release mode $([[ $@ =~ -r ]] && echo 'ENV RUSTFLAGS="$RUSTFLAGS -Clink-arg=-s"') From 219dfbabd58ff6008f8a85c291f8cf4f6da1318a Mon Sep 17 00:00:00 2001 From: Maxim De Clercq Date: Sun, 23 Jan 2022 17:31:12 +0100 Subject: [PATCH 07/40] fix: pass RUSTC_WRAPPER to the cross container and enforce static builds --- .gitlab-ci.yml | 18 ++++++------------ Cross.toml | 3 --- cross/build.sh | 9 +++++---- 3 files changed, 11 insertions(+), 19 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index b5a12f3d..fac678cf 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -35,8 +35,6 @@ variables: before_script: - 'echo "Building for target $TARGET"' - "rustup show && rustc --version && cargo --version" # Print version info for debugging - # If provided, bring in caching through sccache, which uses an external S3 endpoint to store compilation results: - - if [ -n "${SCCACHE_BIN_URL}" ]; then curl $SCCACHE_BIN_URL --output /sccache && chmod +x /sccache && export RUSTC_WRAPPER=/sccache; fi # install cross-compiling prerequisites - 'apt-get update && apt-get install -y docker.io && docker version' # install docker - 'cargo install cross && cross --version' # install cross @@ -45,10 +43,14 @@ variables: - 'cp -r $CARGO_HOME/bin $SHARED_PATH/cargo' - 'cp -r $RUSTUP_HOME $SHARED_PATH' - 'export CARGO_HOME=$SHARED_PATH/cargo RUSTUP_HOME=$SHARED_PATH/rustup' + # If provided, bring in caching through sccache, which uses an external S3 endpoint to store compilation results: + - if [ -n "${SCCACHE_BIN_URL}" ]; then export RUSTC_WRAPPER=$SHARED_PATH/cargo/bin/sccache && curl $SCCACHE_BIN_URL --output $RUSTC_WRAPPER && chmod +x $RUSTC_WRAPPER; fi script: # cross-compile conduit for target - 'time ./cross/build.sh --locked --release' - 'mv "target/$TARGET/release/conduit" "conduit-$TARGET"' + # assert that the binary is statically linked + - 'file conduit-$TARGET | grep "static\(-pie\|ally\) linked"' cache: # https://doc.rust-lang.org/cargo/guide/cargo-home.html#caching-the-cargo-home-in-ci key: 'cargo-cache-$TARGET' @@ -82,16 +84,6 @@ build:release:cargo:x86_64-unknown-linux-musl: - "conduit-x86_64-unknown-linux-musl" expose_as: "Conduit for x86_64-unknown-linux-musl" -build:release:cargo:i686-unknown-linux-musl: - extends: .build-cargo-shared-settings - variables: - TARGET: "i686-unknown-linux-musl" - artifacts: - name: "conduit-i686-unknown-linux-musl" - paths: - - "conduit-i686-unknown-linux-musl" - expose_as: "Conduit for i686-unknown-linux-musl" - build:release:cargo:arm-unknown-linux-musleabihf: extends: .build-cargo-shared-settings variables: @@ -132,6 +124,8 @@ build:release:cargo:aarch64-unknown-linux-musl: # cross-compile conduit for target - 'time ./cross/build.sh --locked' - 'mv "target/$TARGET/debug/conduit" "conduit-debug-$TARGET"' + # assert that the binary is statically linked + - 'file conduit-debug-$TARGET | grep "static\(-pie\|ally\) linked"' artifacts: expire_in: 4 weeks diff --git a/Cross.toml b/Cross.toml index 22c84b97..491efcb7 100644 --- a/Cross.toml +++ b/Cross.toml @@ -7,8 +7,5 @@ image = "rust-cross:arm-unknown-linux-musleabihf" [target.armv7-unknown-linux-musleabihf] image = "rust-cross:armv7-unknown-linux-musleabihf" -[target.i686-unknown-linux-musl] -image = "rust-cross:i686-unknown-linux-musl" - [target.x86_64-unknown-linux-musl] image = "rust-cross:x86_64-unknown-linux-musl" diff --git a/cross/build.sh b/cross/build.sh index 24a2224b..34082606 100755 --- a/cross/build.sh +++ b/cross/build.sh @@ -17,13 +17,14 @@ ENV TARGET_PREFIX="/usr/local/$(echo "${TARGET:?}" | sed -e 's/armv7/arm/' -e 's # Make sure that cc-rs links libc/libstdc++ statically when cross-compiling # See https://github.com/alexcrichton/cc-rs#external-configuration-via-environment-variables for more information ENV RUSTFLAGS="-L\$TARGET_PREFIX/lib" CXXSTDLIB="static=stdc++" -# Forcefully linking against libatomic and libgcc is required for arm32, otherwise symbols are missing -$([[ $TARGET =~ arm ]] && echo 'ENV RUSTFLAGS="$RUSTFLAGS -Clink-arg=-lgcc -Clink-arg=-latomic"') -# Forcefully linking against libc is required for 32-bit, otherwise symbols are missing -$([[ $TARGET =~ arm|i686 ]] && echo 'ENV RUSTFLAGS="$RUSTFLAGS -lstatic=c"') +# Forcefully linking against libatomic, libc and libgcc is required for arm32, otherwise symbols are missing +$([[ $TARGET =~ arm ]] && echo 'ENV RUSTFLAGS="$RUSTFLAGS -Clink-arg=-lgcc -Clink-arg=-latomic -lstatic=c"') # Strip symbols while compiling in release mode $([[ $@ =~ -r ]] && echo 'ENV RUSTFLAGS="$RUSTFLAGS -Clink-arg=-s"') +# Support a rustc wrapper like sccache when cross-compiling +ENV RUSTC_WRAPPER="$RUSTC_WRAPPER" + # Make sure that rust-bindgen uses the correct include path when cross-compiling # See https://github.com/rust-lang/rust-bindgen#environment-variables for more information ENV BINDGEN_EXTRA_CLANG_ARGS="-I\$TARGET_PREFIX/include" From c2ad2b3dd747e7a8baa5ff2f9ade8edb92204aa6 Mon Sep 17 00:00:00 2001 From: Maxim De Clercq Date: Sun, 23 Jan 2022 17:38:13 +0100 Subject: [PATCH 08/40] fix: pass sccache variables to cross container with build.env.passthrough --- Cross.toml | 12 ++++++++++++ cross/build.sh | 3 --- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/Cross.toml b/Cross.toml index 491efcb7..a989a98f 100644 --- a/Cross.toml +++ b/Cross.toml @@ -1,3 +1,15 @@ +[build.env] +# CI uses an S3 endpoint to store sccache artifacts, so their config needs to +# be available in the cross container as well +passthrough = [ + "RUSTC_WRAPPER", + "AWS_ACCESS_KEY_ID", + "AWS_SECRET_ACCESS_KEY", + "SCCACHE_BUCKET", + "SCCACHE_ENDPOINT", + "SCCACHE_S3_USE_SSL", +] + [target.aarch64-unknown-linux-musl] image = "rust-cross:aarch64-unknown-linux-musl" diff --git a/cross/build.sh b/cross/build.sh index 34082606..4a6d4493 100755 --- a/cross/build.sh +++ b/cross/build.sh @@ -22,9 +22,6 @@ $([[ $TARGET =~ arm ]] && echo 'ENV RUSTFLAGS="$RUSTFLAGS -Clink-arg=-lgcc -Clin # Strip symbols while compiling in release mode $([[ $@ =~ -r ]] && echo 'ENV RUSTFLAGS="$RUSTFLAGS -Clink-arg=-s"') -# Support a rustc wrapper like sccache when cross-compiling -ENV RUSTC_WRAPPER="$RUSTC_WRAPPER" - # Make sure that rust-bindgen uses the correct include path when cross-compiling # See https://github.com/rust-lang/rust-bindgen#environment-variables for more information ENV BINDGEN_EXTRA_CLANG_ARGS="-I\$TARGET_PREFIX/include" From c7560b3502d27a49f935c347458df6421459c485 Mon Sep 17 00:00:00 2001 From: Maxim De Clercq Date: Sun, 23 Jan 2022 18:09:14 +0100 Subject: [PATCH 09/40] fix: remove libgcc dependency in ci builds since the binary is ensured to be statically compiled --- docker/ci-binaries-packaging.Dockerfile | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/docker/ci-binaries-packaging.Dockerfile b/docker/ci-binaries-packaging.Dockerfile index f4603105..a6339be3 100644 --- a/docker/ci-binaries-packaging.Dockerfile +++ b/docker/ci-binaries-packaging.Dockerfile @@ -9,6 +9,7 @@ FROM docker.io/alpine:3.15.0 AS runner + # Standard port on which Conduit launches. # You still need to map the port when using the docker command or docker-compose. EXPOSE 6167 @@ -18,10 +19,8 @@ ENV CONDUIT_CONFIG="/srv/conduit/conduit.toml" # Conduit needs: # ca-certificates: for https -# libgcc: Apparently this is needed, even if I (@jfowl) don't know exactly why. But whatever, it's not that big. RUN apk add --no-cache \ - ca-certificates \ - libgcc + ca-certificates ARG CREATED From 64c25ea4a15739f75ebe2811e84dc00280ba5fb0 Mon Sep 17 00:00:00 2001 From: Maxim De Clercq Date: Sun, 23 Jan 2022 18:31:40 +0100 Subject: [PATCH 10/40] fix: always print ELF information --- .gitlab-ci.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index fac678cf..defd66ee 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -50,7 +50,8 @@ variables: - 'time ./cross/build.sh --locked --release' - 'mv "target/$TARGET/release/conduit" "conduit-$TARGET"' # assert that the binary is statically linked - - 'file conduit-$TARGET | grep "static\(-pie\|ally\) linked"' + - 'ldd conduit-$TARGET' # print linking information + - 'file conduit-$TARGET | sed -e "/static\(-pie\|ally\) linked/!q1"' # print elf information cache: # https://doc.rust-lang.org/cargo/guide/cargo-home.html#caching-the-cargo-home-in-ci key: 'cargo-cache-$TARGET' @@ -125,7 +126,8 @@ build:release:cargo:aarch64-unknown-linux-musl: - 'time ./cross/build.sh --locked' - 'mv "target/$TARGET/debug/conduit" "conduit-debug-$TARGET"' # assert that the binary is statically linked - - 'file conduit-debug-$TARGET | grep "static\(-pie\|ally\) linked"' + - 'ldd conduit-debug-$TARGET' # print linking information + - 'file conduit-debug-$TARGET | sed -e "/static\(-pie\|ally\) linked/!q1"' # print elf information artifacts: expire_in: 4 weeks From 77ad4cb8f8c69b563c890494b5d203d96195253d Mon Sep 17 00:00:00 2001 From: Maxim De Clercq Date: Sun, 23 Jan 2022 19:24:36 +0100 Subject: [PATCH 11/40] fix: use readelf for checking static compilation --- .gitlab-ci.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index defd66ee..9e584a2a 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -49,9 +49,9 @@ variables: # cross-compile conduit for target - 'time ./cross/build.sh --locked --release' - 'mv "target/$TARGET/release/conduit" "conduit-$TARGET"' - # assert that the binary is statically linked - - 'ldd conduit-$TARGET' # print linking information - - 'file conduit-$TARGET | sed -e "/static\(-pie\|ally\) linked/!q1"' # print elf information + # print information about linking for debugging + - 'file conduit-$TARGET' # print file information + - 'readelf --dynamic conduit-$TARGET | sed -e "/NEEDED/q1"' # ensure statically linked cache: # https://doc.rust-lang.org/cargo/guide/cargo-home.html#caching-the-cargo-home-in-ci key: 'cargo-cache-$TARGET' @@ -125,9 +125,9 @@ build:release:cargo:aarch64-unknown-linux-musl: # cross-compile conduit for target - 'time ./cross/build.sh --locked' - 'mv "target/$TARGET/debug/conduit" "conduit-debug-$TARGET"' - # assert that the binary is statically linked - - 'ldd conduit-debug-$TARGET' # print linking information - - 'file conduit-debug-$TARGET | sed -e "/static\(-pie\|ally\) linked/!q1"' # print elf information + # print information about linking for debugging + - 'file conduit-debug-$TARGET' # print file information + - 'readelf --dynamic conduit-debug-$TARGET | sed -e "/NEEDED/q1"' # ensure statically linked artifacts: expire_in: 4 weeks From 067fcfc0e40ced1c2ed28f32b04940c6e74d6f5a Mon Sep 17 00:00:00 2001 From: Maxim De Clercq Date: Sun, 23 Jan 2022 21:19:19 +0100 Subject: [PATCH 12/40] fix: remove trailing slash from shared path --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 9e584a2a..aac773c2 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -28,7 +28,7 @@ variables: DOCKER_HOST: tcp://docker:2375/ DOCKER_TLS_CERTDIR: "" DOCKER_DRIVER: overlay2 - SHARED_PATH: $CI_PROJECT_DIR/shared/ + SHARED_PATH: $CI_PROJECT_DIR/shared CARGO_PROFILE_RELEASE_LTO: "true" CARGO_PROFILE_RELEASE_CODEGEN_UNITS: "1" CARGO_INCREMENTAL: "false" # https://matklad.github.io/2021/09/04/fast-rust-builds.html#ci-workflow From acf1585fc35e7851df9c30208f654d5c085267d6 Mon Sep 17 00:00:00 2001 From: Maxim De Clercq Date: Mon, 24 Jan 2022 11:45:07 +0100 Subject: [PATCH 13/40] fix: make sure that libatomic is linked statically --- cross/build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cross/build.sh b/cross/build.sh index 4a6d4493..8f64ff87 100755 --- a/cross/build.sh +++ b/cross/build.sh @@ -18,7 +18,7 @@ ENV TARGET_PREFIX="/usr/local/$(echo "${TARGET:?}" | sed -e 's/armv7/arm/' -e 's # See https://github.com/alexcrichton/cc-rs#external-configuration-via-environment-variables for more information ENV RUSTFLAGS="-L\$TARGET_PREFIX/lib" CXXSTDLIB="static=stdc++" # Forcefully linking against libatomic, libc and libgcc is required for arm32, otherwise symbols are missing -$([[ $TARGET =~ arm ]] && echo 'ENV RUSTFLAGS="$RUSTFLAGS -Clink-arg=-lgcc -Clink-arg=-latomic -lstatic=c"') +$([[ $TARGET =~ arm ]] && echo 'ENV RUSTFLAGS="$RUSTFLAGS -Clink-arg=-static-libgcc -Clink-arg=-lgcc -lstatic=atomic -lstatic=c"') # Strip symbols while compiling in release mode $([[ $@ =~ -r ]] && echo 'ENV RUSTFLAGS="$RUSTFLAGS -Clink-arg=-s"') From ff167299766d41c361398243db6bf97e9d45fa65 Mon Sep 17 00:00:00 2001 From: Maxim De Clercq Date: Tue, 25 Jan 2022 22:36:51 +0100 Subject: [PATCH 14/40] fix: correct RUSTC_WRAPPER path in cross container --- .gitlab-ci.yml | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index aac773c2..741b5327 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -8,6 +8,10 @@ variables: GIT_SUBMODULE_STRATEGY: recursive FF_USE_FASTZIP: 1 CACHE_COMPRESSION_LEVEL: fastest + # Docker in Docker + DOCKER_HOST: tcp://docker:2375/ + DOCKER_TLS_CERTDIR: "" + DOCKER_DRIVER: overlay2 # --------------------------------------------------------------------- # # Cargo: Compiling for different architectures # @@ -25,9 +29,6 @@ variables: tags: ["docker"] services: ["docker:dind"] variables: - DOCKER_HOST: tcp://docker:2375/ - DOCKER_TLS_CERTDIR: "" - DOCKER_DRIVER: overlay2 SHARED_PATH: $CI_PROJECT_DIR/shared CARGO_PROFILE_RELEASE_LTO: "true" CARGO_PROFILE_RELEASE_CODEGEN_UNITS: "1" @@ -43,8 +44,9 @@ variables: - 'cp -r $CARGO_HOME/bin $SHARED_PATH/cargo' - 'cp -r $RUSTUP_HOME $SHARED_PATH' - 'export CARGO_HOME=$SHARED_PATH/cargo RUSTUP_HOME=$SHARED_PATH/rustup' - # If provided, bring in caching through sccache, which uses an external S3 endpoint to store compilation results: - - if [ -n "${SCCACHE_BIN_URL}" ]; then export RUSTC_WRAPPER=$SHARED_PATH/cargo/bin/sccache && curl $SCCACHE_BIN_URL --output $RUSTC_WRAPPER && chmod +x $RUSTC_WRAPPER; fi + # If provided, bring in caching through sccache, which uses an external S3 endpoint to store compilation results. + # The sccache binary is stored in the sysroot of the rustc installation since that directory is added to the path of the cross container. + - if [ -n "${SCCACHE_BIN_URL}" ]; then RUSTC_SYSROOT=$(rustc --print sysroot) && curl $SCCACHE_BIN_URL --output $RUSTC_SYSROOT/bin/sccache && chmod +x $RUSTC_SYSROOT/bin/sccache && export RUSTC_WRAPPER=sccache; fi script: # cross-compile conduit for target - 'time ./cross/build.sh --locked --release' @@ -157,9 +159,6 @@ build:debug:cargo:x86_64-unknown-linux-musl: - "build:release:cargo:armv7-unknown-linux-musleabihf" - "build:release:cargo:aarch64-unknown-linux-musl" variables: - DOCKER_HOST: tcp://docker:2375/ - DOCKER_TLS_CERTDIR: "" - DOCKER_DRIVER: overlay2 PLATFORMS: "linux/arm/v6,linux/arm/v7,linux/arm64,linux/amd64" DOCKER_FILE: "docker/ci-binaries-packaging.Dockerfile" cache: From 8472eff277faf55808dc794a8eb9023fafb75763 Mon Sep 17 00:00:00 2001 From: Andrej Kacian Date: Thu, 27 Jan 2022 00:25:20 +0100 Subject: [PATCH 15/40] Implement media download with custom filename --- src/client_server/media.rs | 64 +++++++++++++++++++++++++++++++++++++- src/main.rs | 1 + 2 files changed, 64 insertions(+), 1 deletion(-) diff --git a/src/client_server/media.rs b/src/client_server/media.rs index 0a7f4bb5..2e3cf056 100644 --- a/src/client_server/media.rs +++ b/src/client_server/media.rs @@ -4,7 +4,10 @@ use crate::{ }; use ruma::api::client::{ error::ErrorKind, - r0::media::{create_content, get_content, get_content_thumbnail, get_media_config}, + r0::media::{ + create_content, get_content, get_content_as_filename, get_content_thumbnail, + get_media_config, + }, }; use std::convert::TryInto; @@ -129,6 +132,65 @@ pub async fn get_content_route( } } +/// # `GET /_matrix/media/r0/download/{serverName}/{mediaId}/{fileName}` +/// +/// Load media from our server or over federation, permitting desired filename. +/// +/// - Only allows federation if `allow_remote` is true +#[cfg_attr( + feature = "conduit_bin", + get("/_matrix/media/r0/download/<_>/<_>/<_>", data = "") +)] +#[tracing::instrument(skip(db, body))] +pub async fn get_content_as_filename_route( + db: DatabaseGuard, + body: Ruma>, +) -> ConduitResult { + let mxc = format!("mxc://{}/{}", body.server_name, body.media_id); + + if let Some(FileMeta { + content_disposition: _, + content_type, + file, + }) = db.media.get(&db.globals, &mxc).await? + { + Ok(get_content_as_filename::Response { + file, + content_type, + content_disposition: Some(format!("inline; filename={}", body.filename)), + } + .into()) + } else if &*body.server_name != db.globals.server_name() && body.allow_remote { + let get_content_response = db + .sending + .send_federation_request( + &db.globals, + &body.server_name, + get_content_as_filename::Request { + allow_remote: false, + server_name: &body.server_name, + media_id: &body.media_id, + filename: &body.filename, + }, + ) + .await?; + + db.media + .create( + mxc, + &db.globals, + &get_content_response.content_disposition.as_deref(), + &get_content_response.content_type.as_deref(), + &get_content_response.file, + ) + .await?; + + Ok(get_content_response.into()) + } else { + Err(Error::BadRequest(ErrorKind::NotFound, "Media not found.")) + } +} + /// # `POST /_matrix/media/r0/thumbnail/{serverName}/{mediaId}` /// /// Load media thumbnail from our server or over federation. diff --git a/src/main.rs b/src/main.rs index 56faa3e7..514c2448 100644 --- a/src/main.rs +++ b/src/main.rs @@ -136,6 +136,7 @@ fn setup_rocket(config: Figment, data: Arc>) -> rocket::Rocket< client_server::send_event_to_device_route, client_server::get_media_config_route, client_server::create_content_route, + client_server::get_content_as_filename_route, client_server::get_content_route, client_server::get_content_thumbnail_route, client_server::get_devices_route, From 52873c88b7e204f5b3c9295448c15780fff8084c Mon Sep 17 00:00:00 2001 From: Andrej Kacian Date: Thu, 27 Jan 2022 00:27:13 +0100 Subject: [PATCH 16/40] Fix incorrect HTTP method in doc comments of two media routes --- src/client_server/media.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/client_server/media.rs b/src/client_server/media.rs index 2e3cf056..5b196dff 100644 --- a/src/client_server/media.rs +++ b/src/client_server/media.rs @@ -74,7 +74,7 @@ pub async fn create_content_route( .into()) } -/// # `POST /_matrix/media/r0/download/{serverName}/{mediaId}` +/// # `GET /_matrix/media/r0/download/{serverName}/{mediaId}` /// /// Load media from our server or over federation. /// @@ -191,7 +191,7 @@ pub async fn get_content_as_filename_route( } } -/// # `POST /_matrix/media/r0/thumbnail/{serverName}/{mediaId}` +/// # `GET /_matrix/media/r0/thumbnail/{serverName}/{mediaId}` /// /// Load media thumbnail from our server or over federation. /// From 9c2000cb8973894512940b96996a2b5937f5cc8f Mon Sep 17 00:00:00 2001 From: Jonas Platte Date: Thu, 27 Jan 2022 16:17:55 +0100 Subject: [PATCH 17/40] Upgrade Ruma --- Cargo.lock | 36 ++++++++++++++++++------------------ Cargo.toml | 2 +- src/database/key_backups.rs | 29 ++++++++--------------------- 3 files changed, 27 insertions(+), 40 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 493ac082..794a0257 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2107,7 +2107,7 @@ dependencies = [ [[package]] name = "ruma" version = "0.4.0" -source = "git+https://github.com/ruma/ruma?rev=08d60b3d376b63462f769d4b9bd3bbfb560d501a#08d60b3d376b63462f769d4b9bd3bbfb560d501a" +source = "git+https://github.com/ruma/ruma?rev=82becb86c837570224964425929d1b5305784435#82becb86c837570224964425929d1b5305784435" dependencies = [ "assign", "js_int", @@ -2128,7 +2128,7 @@ dependencies = [ [[package]] name = "ruma-api" version = "0.18.5" -source = "git+https://github.com/ruma/ruma?rev=08d60b3d376b63462f769d4b9bd3bbfb560d501a#08d60b3d376b63462f769d4b9bd3bbfb560d501a" +source = "git+https://github.com/ruma/ruma?rev=82becb86c837570224964425929d1b5305784435#82becb86c837570224964425929d1b5305784435" dependencies = [ "bytes", "http", @@ -2144,7 +2144,7 @@ dependencies = [ [[package]] name = "ruma-api-macros" version = "0.18.5" -source = "git+https://github.com/ruma/ruma?rev=08d60b3d376b63462f769d4b9bd3bbfb560d501a#08d60b3d376b63462f769d4b9bd3bbfb560d501a" +source = "git+https://github.com/ruma/ruma?rev=82becb86c837570224964425929d1b5305784435#82becb86c837570224964425929d1b5305784435" dependencies = [ "proc-macro-crate", "proc-macro2", @@ -2155,7 +2155,7 @@ dependencies = [ [[package]] name = "ruma-appservice-api" version = "0.4.0" -source = "git+https://github.com/ruma/ruma?rev=08d60b3d376b63462f769d4b9bd3bbfb560d501a#08d60b3d376b63462f769d4b9bd3bbfb560d501a" +source = "git+https://github.com/ruma/ruma?rev=82becb86c837570224964425929d1b5305784435#82becb86c837570224964425929d1b5305784435" dependencies = [ "ruma-api", "ruma-common", @@ -2169,7 +2169,7 @@ dependencies = [ [[package]] name = "ruma-client-api" version = "0.12.3" -source = "git+https://github.com/ruma/ruma?rev=08d60b3d376b63462f769d4b9bd3bbfb560d501a#08d60b3d376b63462f769d4b9bd3bbfb560d501a" +source = "git+https://github.com/ruma/ruma?rev=82becb86c837570224964425929d1b5305784435#82becb86c837570224964425929d1b5305784435" dependencies = [ "assign", "bytes", @@ -2189,7 +2189,7 @@ dependencies = [ [[package]] name = "ruma-common" version = "0.6.0" -source = "git+https://github.com/ruma/ruma?rev=08d60b3d376b63462f769d4b9bd3bbfb560d501a#08d60b3d376b63462f769d4b9bd3bbfb560d501a" +source = "git+https://github.com/ruma/ruma?rev=82becb86c837570224964425929d1b5305784435#82becb86c837570224964425929d1b5305784435" dependencies = [ "indexmap", "js_int", @@ -2204,7 +2204,7 @@ dependencies = [ [[package]] name = "ruma-events" version = "0.24.6" -source = "git+https://github.com/ruma/ruma?rev=08d60b3d376b63462f769d4b9bd3bbfb560d501a#08d60b3d376b63462f769d4b9bd3bbfb560d501a" +source = "git+https://github.com/ruma/ruma?rev=82becb86c837570224964425929d1b5305784435#82becb86c837570224964425929d1b5305784435" dependencies = [ "indoc", "js_int", @@ -2221,7 +2221,7 @@ dependencies = [ [[package]] name = "ruma-events-macros" version = "0.24.6" -source = "git+https://github.com/ruma/ruma?rev=08d60b3d376b63462f769d4b9bd3bbfb560d501a#08d60b3d376b63462f769d4b9bd3bbfb560d501a" +source = "git+https://github.com/ruma/ruma?rev=82becb86c837570224964425929d1b5305784435#82becb86c837570224964425929d1b5305784435" dependencies = [ "proc-macro-crate", "proc-macro2", @@ -2232,7 +2232,7 @@ dependencies = [ [[package]] name = "ruma-federation-api" version = "0.3.1" -source = "git+https://github.com/ruma/ruma?rev=08d60b3d376b63462f769d4b9bd3bbfb560d501a#08d60b3d376b63462f769d4b9bd3bbfb560d501a" +source = "git+https://github.com/ruma/ruma?rev=82becb86c837570224964425929d1b5305784435#82becb86c837570224964425929d1b5305784435" dependencies = [ "js_int", "ruma-api", @@ -2247,7 +2247,7 @@ dependencies = [ [[package]] name = "ruma-identifiers" version = "0.20.0" -source = "git+https://github.com/ruma/ruma?rev=08d60b3d376b63462f769d4b9bd3bbfb560d501a#08d60b3d376b63462f769d4b9bd3bbfb560d501a" +source = "git+https://github.com/ruma/ruma?rev=82becb86c837570224964425929d1b5305784435#82becb86c837570224964425929d1b5305784435" dependencies = [ "percent-encoding", "rand 0.8.4", @@ -2262,7 +2262,7 @@ dependencies = [ [[package]] name = "ruma-identifiers-macros" version = "0.20.0" -source = "git+https://github.com/ruma/ruma?rev=08d60b3d376b63462f769d4b9bd3bbfb560d501a#08d60b3d376b63462f769d4b9bd3bbfb560d501a" +source = "git+https://github.com/ruma/ruma?rev=82becb86c837570224964425929d1b5305784435#82becb86c837570224964425929d1b5305784435" dependencies = [ "quote", "ruma-identifiers-validation", @@ -2272,7 +2272,7 @@ dependencies = [ [[package]] name = "ruma-identifiers-validation" version = "0.5.0" -source = "git+https://github.com/ruma/ruma?rev=08d60b3d376b63462f769d4b9bd3bbfb560d501a#08d60b3d376b63462f769d4b9bd3bbfb560d501a" +source = "git+https://github.com/ruma/ruma?rev=82becb86c837570224964425929d1b5305784435#82becb86c837570224964425929d1b5305784435" dependencies = [ "thiserror", ] @@ -2280,7 +2280,7 @@ dependencies = [ [[package]] name = "ruma-identity-service-api" version = "0.3.0" -source = "git+https://github.com/ruma/ruma?rev=08d60b3d376b63462f769d4b9bd3bbfb560d501a#08d60b3d376b63462f769d4b9bd3bbfb560d501a" +source = "git+https://github.com/ruma/ruma?rev=82becb86c837570224964425929d1b5305784435#82becb86c837570224964425929d1b5305784435" dependencies = [ "js_int", "ruma-api", @@ -2293,7 +2293,7 @@ dependencies = [ [[package]] name = "ruma-push-gateway-api" version = "0.3.0" -source = "git+https://github.com/ruma/ruma?rev=08d60b3d376b63462f769d4b9bd3bbfb560d501a#08d60b3d376b63462f769d4b9bd3bbfb560d501a" +source = "git+https://github.com/ruma/ruma?rev=82becb86c837570224964425929d1b5305784435#82becb86c837570224964425929d1b5305784435" dependencies = [ "js_int", "ruma-api", @@ -2308,7 +2308,7 @@ dependencies = [ [[package]] name = "ruma-serde" version = "0.5.0" -source = "git+https://github.com/ruma/ruma?rev=08d60b3d376b63462f769d4b9bd3bbfb560d501a#08d60b3d376b63462f769d4b9bd3bbfb560d501a" +source = "git+https://github.com/ruma/ruma?rev=82becb86c837570224964425929d1b5305784435#82becb86c837570224964425929d1b5305784435" dependencies = [ "base64 0.13.0", "bytes", @@ -2323,7 +2323,7 @@ dependencies = [ [[package]] name = "ruma-serde-macros" version = "0.5.0" -source = "git+https://github.com/ruma/ruma?rev=08d60b3d376b63462f769d4b9bd3bbfb560d501a#08d60b3d376b63462f769d4b9bd3bbfb560d501a" +source = "git+https://github.com/ruma/ruma?rev=82becb86c837570224964425929d1b5305784435#82becb86c837570224964425929d1b5305784435" dependencies = [ "proc-macro-crate", "proc-macro2", @@ -2334,7 +2334,7 @@ dependencies = [ [[package]] name = "ruma-signatures" version = "0.9.0" -source = "git+https://github.com/ruma/ruma?rev=08d60b3d376b63462f769d4b9bd3bbfb560d501a#08d60b3d376b63462f769d4b9bd3bbfb560d501a" +source = "git+https://github.com/ruma/ruma?rev=82becb86c837570224964425929d1b5305784435#82becb86c837570224964425929d1b5305784435" dependencies = [ "base64 0.13.0", "ed25519-dalek", @@ -2351,7 +2351,7 @@ dependencies = [ [[package]] name = "ruma-state-res" version = "0.4.1" -source = "git+https://github.com/ruma/ruma?rev=08d60b3d376b63462f769d4b9bd3bbfb560d501a#08d60b3d376b63462f769d4b9bd3bbfb560d501a" +source = "git+https://github.com/ruma/ruma?rev=82becb86c837570224964425929d1b5305784435#82becb86c837570224964425929d1b5305784435" dependencies = [ "itertools", "js_int", diff --git a/Cargo.toml b/Cargo.toml index 78a4c8ff..9ba1ac05 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -20,7 +20,7 @@ rocket = { version = "0.5.0-rc.1", features = ["tls"] } # Used to handle request # Used for matrix spec type definitions and helpers #ruma = { version = "0.4.0", features = ["compat", "rand", "appservice-api-c", "client-api", "federation-api", "push-gateway-api-c", "state-res", "unstable-pre-spec", "unstable-exhaustive-types"] } -ruma = { git = "https://github.com/ruma/ruma", rev = "08d60b3d376b63462f769d4b9bd3bbfb560d501a", features = ["compat", "rand", "appservice-api-c", "client-api", "federation-api", "push-gateway-api-c", "state-res", "unstable-pre-spec", "unstable-exhaustive-types"] } +ruma = { git = "https://github.com/ruma/ruma", rev = "82becb86c837570224964425929d1b5305784435", features = ["compat", "rand", "appservice-api-c", "client-api", "federation-api", "push-gateway-api-c", "state-res", "unstable-pre-spec", "unstable-exhaustive-types"] } #ruma = { git = "https://github.com/timokoesters/ruma", rev = "50c1db7e0a3a21fc794b0cce3b64285a4c750c71", features = ["compat", "rand", "appservice-api-c", "client-api", "federation-api", "push-gateway-api-c", "state-res", "unstable-pre-spec", "unstable-exhaustive-types"] } #ruma = { path = "../ruma/crates/ruma", features = ["compat", "rand", "appservice-api-c", "client-api", "federation-api", "push-gateway-api-c", "state-res", "unstable-pre-spec", "unstable-exhaustive-types"] } diff --git a/src/database/key_backups.rs b/src/database/key_backups.rs index b74bc408..2eefe481 100644 --- a/src/database/key_backups.rs +++ b/src/database/key_backups.rs @@ -7,7 +7,6 @@ use ruma::{ serde::Raw, RoomId, UserId, }; -use serde_json::json; use std::{collections::BTreeMap, sync::Arc}; use super::abstraction::Tree; @@ -212,13 +211,13 @@ impl KeyBackups { &self, user_id: &UserId, version: &str, - ) -> Result, Raw>> { + ) -> Result, RoomKeyBackup>> { let mut prefix = user_id.as_bytes().to_vec(); prefix.push(0xff); prefix.extend_from_slice(version.as_bytes()); prefix.push(0xff); - let mut rooms = BTreeMap::, Raw>::new(); + let mut rooms = BTreeMap::, RoomKeyBackup>::new(); for result in self .backupkeyid_backup @@ -244,7 +243,7 @@ impl KeyBackups { Error::bad_database("backupkeyid_backup room_id is invalid room id.") })?; - let key_data: serde_json::Value = serde_json::from_slice(&value).map_err(|_| { + let key_data = serde_json::from_slice(&value).map_err(|_| { Error::bad_database("KeyBackupData in backupkeyid_backup is invalid.") })?; @@ -252,25 +251,13 @@ impl KeyBackups { }) { let (room_id, session_id, key_data) = result?; - let room_key_backup = rooms.entry(room_id).or_insert_with(|| { - Raw::new(&RoomKeyBackup { + rooms + .entry(room_id) + .or_insert_with(|| RoomKeyBackup { sessions: BTreeMap::new(), }) - .expect("RoomKeyBackup serialization") - }); - - let mut object = room_key_backup - .deserialize_as::>() - .map_err(|_| Error::bad_database("RoomKeyBackup is not an object"))?; - - let sessions = object.entry("session").or_insert_with(|| json!({})); - if let serde_json::Value::Object(unsigned_object) = sessions { - unsigned_object.insert(session_id, key_data); - } - - *room_key_backup = Raw::from_json( - serde_json::value::to_raw_value(&object).expect("Value => RawValue serialization"), - ); + .sessions + .insert(session_id, key_data); } Ok(rooms) From c4317a7a962fcc4b41b1abfc273034d9827a6563 Mon Sep 17 00:00:00 2001 From: Andrej Kacian Date: Thu, 27 Jan 2022 16:12:39 +0100 Subject: [PATCH 18/40] Reduce code duplication in media download route handlers --- src/client_server/media.rs | 99 ++++++++++++++++++++------------------ 1 file changed, 51 insertions(+), 48 deletions(-) diff --git a/src/client_server/media.rs b/src/client_server/media.rs index 5b196dff..bd73cff5 100644 --- a/src/client_server/media.rs +++ b/src/client_server/media.rs @@ -74,6 +74,38 @@ pub async fn create_content_route( .into()) } +pub async fn get_remote_content( + db: &DatabaseGuard, + mxc: &str, + server_name: &ruma::ServerName, + media_id: &str +) -> ConduitResult { + let content_response = db + .sending + .send_federation_request( + &db.globals, + server_name, + get_content::Request { + allow_remote: false, + server_name, + media_id + }, + ) + .await?; + + db.media + .create( + mxc.to_string(), + &db.globals, + &content_response.content_disposition.as_deref(), + &content_response.content_type.as_deref(), + &content_response.file, + ) + .await?; + + Ok(content_response.into()) +} + /// # `GET /_matrix/media/r0/download/{serverName}/{mediaId}` /// /// Load media from our server or over federation. @@ -103,30 +135,13 @@ pub async fn get_content_route( } .into()) } else if &*body.server_name != db.globals.server_name() && body.allow_remote { - let get_content_response = db - .sending - .send_federation_request( - &db.globals, - &body.server_name, - get_content::Request { - allow_remote: false, - server_name: &body.server_name, - media_id: &body.media_id, - }, - ) - .await?; - - db.media - .create( - mxc, - &db.globals, - &get_content_response.content_disposition.as_deref(), - &get_content_response.content_type.as_deref(), - &get_content_response.file, - ) - .await?; - - Ok(get_content_response.into()) + let remote_content_response = get_remote_content( + &db, + &mxc, + &body.server_name, + &body.media_id + ).await?; + Ok(remote_content_response) } else { Err(Error::BadRequest(ErrorKind::NotFound, "Media not found.")) } @@ -161,31 +176,19 @@ pub async fn get_content_as_filename_route( } .into()) } else if &*body.server_name != db.globals.server_name() && body.allow_remote { - let get_content_response = db - .sending - .send_federation_request( - &db.globals, - &body.server_name, - get_content_as_filename::Request { - allow_remote: false, - server_name: &body.server_name, - media_id: &body.media_id, - filename: &body.filename, - }, - ) - .await?; - - db.media - .create( - mxc, - &db.globals, - &get_content_response.content_disposition.as_deref(), - &get_content_response.content_type.as_deref(), - &get_content_response.file, - ) - .await?; + let remote_content_response = get_remote_content( + &db, + &mxc, + &body.server_name, + &body.media_id + ).await?; - Ok(get_content_response.into()) + Ok(get_content_as_filename::Response { + content_disposition: Some(format!("inline: filename={}", body.filename)), + content_type: remote_content_response.0.content_type, + file: remote_content_response.0.file + } + .into()) } else { Err(Error::BadRequest(ErrorKind::NotFound, "Media not found.")) } From ccfc243c2c1fec9b859ab6accec1246fa63aef94 Mon Sep 17 00:00:00 2001 From: Andrej Kacian Date: Thu, 27 Jan 2022 17:00:08 +0100 Subject: [PATCH 19/40] Make get_remote_content() return Result instead of ConduitResult --- src/client_server/media.rs | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/client_server/media.rs b/src/client_server/media.rs index bd73cff5..dd8e7b0f 100644 --- a/src/client_server/media.rs +++ b/src/client_server/media.rs @@ -79,7 +79,7 @@ pub async fn get_remote_content( mxc: &str, server_name: &ruma::ServerName, media_id: &str -) -> ConduitResult { +) -> Result { let content_response = db .sending .send_federation_request( @@ -103,7 +103,7 @@ pub async fn get_remote_content( ) .await?; - Ok(content_response.into()) + Ok(content_response) } /// # `GET /_matrix/media/r0/download/{serverName}/{mediaId}` @@ -141,7 +141,7 @@ pub async fn get_content_route( &body.server_name, &body.media_id ).await?; - Ok(remote_content_response) + Ok(remote_content_response.into()) } else { Err(Error::BadRequest(ErrorKind::NotFound, "Media not found.")) } @@ -185,8 +185,8 @@ pub async fn get_content_as_filename_route( Ok(get_content_as_filename::Response { content_disposition: Some(format!("inline: filename={}", body.filename)), - content_type: remote_content_response.0.content_type, - file: remote_content_response.0.file + content_type: remote_content_response.content_type, + file: remote_content_response.file } .into()) } else { From 0f6d232cb1117d959a1984551ee2872558172767 Mon Sep 17 00:00:00 2001 From: Andrej Kacian Date: Thu, 27 Jan 2022 17:08:04 +0100 Subject: [PATCH 20/40] Style fixes from 'cargo fmt' --- src/client_server/media.rs | 22 +++++++--------------- 1 file changed, 7 insertions(+), 15 deletions(-) diff --git a/src/client_server/media.rs b/src/client_server/media.rs index dd8e7b0f..a827d64f 100644 --- a/src/client_server/media.rs +++ b/src/client_server/media.rs @@ -78,7 +78,7 @@ pub async fn get_remote_content( db: &DatabaseGuard, mxc: &str, server_name: &ruma::ServerName, - media_id: &str + media_id: &str, ) -> Result { let content_response = db .sending @@ -88,7 +88,7 @@ pub async fn get_remote_content( get_content::Request { allow_remote: false, server_name, - media_id + media_id, }, ) .await?; @@ -135,12 +135,8 @@ pub async fn get_content_route( } .into()) } else if &*body.server_name != db.globals.server_name() && body.allow_remote { - let remote_content_response = get_remote_content( - &db, - &mxc, - &body.server_name, - &body.media_id - ).await?; + let remote_content_response = + get_remote_content(&db, &mxc, &body.server_name, &body.media_id).await?; Ok(remote_content_response.into()) } else { Err(Error::BadRequest(ErrorKind::NotFound, "Media not found.")) @@ -176,17 +172,13 @@ pub async fn get_content_as_filename_route( } .into()) } else if &*body.server_name != db.globals.server_name() && body.allow_remote { - let remote_content_response = get_remote_content( - &db, - &mxc, - &body.server_name, - &body.media_id - ).await?; + let remote_content_response = + get_remote_content(&db, &mxc, &body.server_name, &body.media_id).await?; Ok(get_content_as_filename::Response { content_disposition: Some(format!("inline: filename={}", body.filename)), content_type: remote_content_response.content_type, - file: remote_content_response.file + file: remote_content_response.file, } .into()) } else { From 529e88c7f9d8997a4fbbc84f98120d1b31d2e39e Mon Sep 17 00:00:00 2001 From: Andrej Kacian Date: Thu, 27 Jan 2022 17:47:09 +0100 Subject: [PATCH 21/40] Do not copy mxc string unnecessarily in db.get_thumbnail() --- src/client_server/media.rs | 2 +- src/database/media.rs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/client_server/media.rs b/src/client_server/media.rs index a827d64f..8524c57e 100644 --- a/src/client_server/media.rs +++ b/src/client_server/media.rs @@ -207,7 +207,7 @@ pub async fn get_content_thumbnail_route( }) = db .media .get_thumbnail( - mxc.clone(), + &mxc, &db.globals, body.width .try_into() diff --git a/src/database/media.rs b/src/database/media.rs index 46630131..a4bb4025 100644 --- a/src/database/media.rs +++ b/src/database/media.rs @@ -171,7 +171,7 @@ impl Media { /// For width,height <= 96 the server uses another thumbnailing algorithm which crops the image afterwards. pub async fn get_thumbnail( &self, - mxc: String, + mxc: &str, globals: &Globals, width: u32, height: u32, From 44f7a85077e5c249ec618004b0386f3d66f01911 Mon Sep 17 00:00:00 2001 From: Jonas Zohren Date: Fri, 28 Jan 2022 22:19:19 +0100 Subject: [PATCH 22/40] fix: Use default port for healthcheck as fallback Conduit can start without a specific port being configured. This adjusts the healthcheck script to tolerate that state. Closes https://gitlab.com/famedly/conduit/-/issues/222 --- docker/healthcheck.sh | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/docker/healthcheck.sh b/docker/healthcheck.sh index 7ca04602..efc94917 100644 --- a/docker/healthcheck.sh +++ b/docker/healthcheck.sh @@ -1,10 +1,15 @@ #!/bin/sh # If the port is not specified as env var, take it from the config file -if [ -z ${CONDUIT_PORT} ]; then +if [ -z "${CONDUIT_PORT}" ]; then CONDUIT_PORT=$(grep -m1 -o 'port\s=\s[0-9]*' conduit.toml | grep -m1 -o '[0-9]*') fi +# If the config file also does not contain a default port, just use the default one: 6167. +if [ -z "${CONDUIT_PORT}" ]; then + CONDUIT_PORT=6167 +fi + # The actual health check. # We try to first get a response on HTTP and when that fails on HTTPS and when that fails, we exit with code 1. # TODO: Change this to a single wget call. Do we have a config value that we can check for that? From 401b88d16d43f0c58e5a4ccf777815fd8d538ff8 Mon Sep 17 00:00:00 2001 From: Jonas Zohren Date: Fri, 28 Jan 2022 23:23:58 +0100 Subject: [PATCH 23/40] fix: Healtcheck use netstat for port as fallback --- docker/healthcheck.sh | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/docker/healthcheck.sh b/docker/healthcheck.sh index efc94917..df7f18a5 100644 --- a/docker/healthcheck.sh +++ b/docker/healthcheck.sh @@ -1,13 +1,9 @@ #!/bin/sh -# If the port is not specified as env var, take it from the config file +# If the config file does not contain a default port and the CONDUIT_PORT env is not set, create +# try to get port from process list if [ -z "${CONDUIT_PORT}" ]; then - CONDUIT_PORT=$(grep -m1 -o 'port\s=\s[0-9]*' conduit.toml | grep -m1 -o '[0-9]*') -fi - -# If the config file also does not contain a default port, just use the default one: 6167. -if [ -z "${CONDUIT_PORT}" ]; then - CONDUIT_PORT=6167 + CONDUIT_PORT=$(netstat -tlp | grep conduit | grep -m1 -o ':[0-9]*' | grep -m1 -o '[0-9]*') fi # The actual health check. From 8ff95a5a48c055549b3652a33faddc3d89351d91 Mon Sep 17 00:00:00 2001 From: user Date: Fri, 28 Jan 2022 22:26:56 -0800 Subject: [PATCH 24/40] fix: mention dependencies to build from source --- DEPLOY.md | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/DEPLOY.md b/DEPLOY.md index 38e1e286..d9f91e03 100644 --- a/DEPLOY.md +++ b/DEPLOY.md @@ -29,7 +29,11 @@ $ sudo wget -O /usr/local/bin/matrix-conduit $ sudo chmod +x /usr/local/bin/matrix-conduit ``` -Alternatively, you may compile the binary yourself using +Alternatively, you may compile the binary yourself + +```bash +$ sudo apt install libclang-dev build-essential +``` ```bash $ cargo build --release From cc13112592b1666e75b4e5d0d340d6124afe4071 Mon Sep 17 00:00:00 2001 From: Torsten Flammiger Date: Mon, 31 Jan 2022 09:27:31 +0100 Subject: [PATCH 25/40] Cleanup appservice events after removing the appservice --- src/database/admin.rs | 13 ++++++++++++- src/database/sending.rs | 31 +++++++++++++++++++++++++++++-- 2 files changed, 41 insertions(+), 3 deletions(-) diff --git a/src/database/admin.rs b/src/database/admin.rs index 81e98393..9895a83b 100644 --- a/src/database/admin.rs +++ b/src/database/admin.rs @@ -112,7 +112,18 @@ impl Admin { guard.appservice.register_appservice(yaml).unwrap(); // TODO handle error } AdminCommand::UnregisterAppservice(service_name) => { - guard.appservice.unregister_appservice(&service_name).unwrap(); // TODO: see above + if let Ok(_) = guard.appservice.unregister_appservice(&service_name) { + if let Ok(_) = guard.sending.cleanup_events(&service_name) { + let msg: String = format!("OK. Appservice {} removed", service_name); + send_message(RoomMessageEventContent::text_plain(msg), guard, &state_lock); + } else { + let msg: String = format!("WARN: Appservice {} removed, but failed to cleanup events", service_name); + send_message(RoomMessageEventContent::text_plain(msg), guard, &state_lock); + } + } else { + let msg: String = format!("ERR. Appservice {} not removed", service_name); + send_message(RoomMessageEventContent::text_plain(msg), guard, &state_lock); + } } AdminCommand::ListAppservices => { if let Ok(appservices) = guard.appservice.iter_ids().map(|ids| ids.collect::>()) { diff --git a/src/database/sending.rs b/src/database/sending.rs index 69f7c444..af4ac676 100644 --- a/src/database/sending.rs +++ b/src/database/sending.rs @@ -480,6 +480,26 @@ impl Sending { hash.as_ref().to_owned() } + /// Cleanup event data + /// Used for instance after we remove an appservice registration + /// + #[tracing::instrument(skip(self))] + pub fn cleanup_events(&self, key_id: &str) -> Result<()> { + let mut prefix = b"+".to_vec(); + prefix.extend_from_slice(key_id.as_bytes()); + prefix.push(0xff); + + for (key, _) in self.servercurrentevent_data.scan_prefix(prefix.clone()) { + self.servercurrentevent_data.remove(&key).unwrap(); + } + + for (key, _) in self.servernameevent_data.scan_prefix(prefix.clone()) { + self.servernameevent_data.remove(&key).unwrap(); + } + + Ok(()) + } + #[tracing::instrument(skip(db, events, kind))] async fn handle_events( kind: OutgoingKind, @@ -520,8 +540,15 @@ impl Sending { &db.globals, db.appservice .get_registration(server.as_str()) - .unwrap() - .unwrap(), // TODO: handle error + .map_err(|e| (kind.clone(), e))? + .ok_or_else(|| { + ( + kind.clone(), + Error::bad_database( + "[Appservice] Could not load registration from db.", + ), + ) + })?, appservice::event::push_events::v1::Request { events: &pdu_jsons, txn_id: (&*base64::encode_config( From 78502aa6b10d97f3af3fe006fcdbd19b585d3b58 Mon Sep 17 00:00:00 2001 From: Torsten Flammiger Date: Mon, 31 Jan 2022 10:07:49 +0100 Subject: [PATCH 26/40] add error handling for register_appservice too --- src/database/admin.rs | 13 ++++++++++++- src/database/appservice.rs | 4 ++-- 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/src/database/admin.rs b/src/database/admin.rs index 9895a83b..eef6ce10 100644 --- a/src/database/admin.rs +++ b/src/database/admin.rs @@ -109,7 +109,18 @@ impl Admin { } } AdminCommand::RegisterAppservice(yaml) => { - guard.appservice.register_appservice(yaml).unwrap(); // TODO handle error + match guard.appservice.register_appservice(yaml) { + Ok(Some(id)) => { + let msg: String = format!("OK. Appservice {} created", id); + send_message(RoomMessageEventContent::text_plain(msg), guard, &state_lock); + } + Ok(None) => { + send_message(RoomMessageEventContent::text_plain("WARN. Appservice created, but its ID was not returned!"), guard, &state_lock); + } + Err(_) => { + send_message(RoomMessageEventContent::text_plain("ERR: Failed register appservice. Check server log"), guard, &state_lock); + } + } } AdminCommand::UnregisterAppservice(service_name) => { if let Ok(_) = guard.appservice.unregister_appservice(&service_name) { diff --git a/src/database/appservice.rs b/src/database/appservice.rs index 88de1f33..8b29aca9 100644 --- a/src/database/appservice.rs +++ b/src/database/appservice.rs @@ -12,7 +12,7 @@ pub struct Appservice { } impl Appservice { - pub fn register_appservice(&self, yaml: serde_yaml::Value) -> Result<()> { + pub fn register_appservice(&self, yaml: serde_yaml::Value) -> Result> { // TODO: Rumaify let id = yaml.get("id").unwrap().as_str().unwrap(); self.id_appserviceregistrations.insert( @@ -24,7 +24,7 @@ impl Appservice { .unwrap() .insert(id.to_owned(), yaml); - Ok(()) + Ok(Some(id.to_owned())) } /// Remove an appservice registration From 28d3b348d2fc23e6b2b78c468f682018ab472652 Mon Sep 17 00:00:00 2001 From: Torsten Flammiger Date: Mon, 31 Jan 2022 11:52:33 +0100 Subject: [PATCH 27/40] Return the ID of the appservice that was created by register_appservice --- src/database/admin.rs | 5 +---- src/database/appservice.rs | 8 +++++--- 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/src/database/admin.rs b/src/database/admin.rs index eef6ce10..a214796b 100644 --- a/src/database/admin.rs +++ b/src/database/admin.rs @@ -110,13 +110,10 @@ impl Admin { } AdminCommand::RegisterAppservice(yaml) => { match guard.appservice.register_appservice(yaml) { - Ok(Some(id)) => { + Ok(id) => { let msg: String = format!("OK. Appservice {} created", id); send_message(RoomMessageEventContent::text_plain(msg), guard, &state_lock); } - Ok(None) => { - send_message(RoomMessageEventContent::text_plain("WARN. Appservice created, but its ID was not returned!"), guard, &state_lock); - } Err(_) => { send_message(RoomMessageEventContent::text_plain("ERR: Failed register appservice. Check server log"), guard, &state_lock); } diff --git a/src/database/appservice.rs b/src/database/appservice.rs index 8b29aca9..edd5009b 100644 --- a/src/database/appservice.rs +++ b/src/database/appservice.rs @@ -12,7 +12,9 @@ pub struct Appservice { } impl Appservice { - pub fn register_appservice(&self, yaml: serde_yaml::Value) -> Result> { + /// Registers an appservice and returns the ID to the caller + /// + pub fn register_appservice(&self, yaml: serde_yaml::Value) -> Result { // TODO: Rumaify let id = yaml.get("id").unwrap().as_str().unwrap(); self.id_appserviceregistrations.insert( @@ -22,9 +24,9 @@ impl Appservice { self.cached_registrations .write() .unwrap() - .insert(id.to_owned(), yaml); + .insert(id.to_owned(), yaml.to_owned()); - Ok(Some(id.to_owned())) + Ok(id.to_owned()) } /// Remove an appservice registration From e17bbdd42d3245f2fb3730753f1feb51cd452207 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timo=20K=C3=B6sters?= Date: Fri, 28 Jan 2022 17:26:43 +0100 Subject: [PATCH 28/40] tests --- Cargo.toml | 2 +- src/database.rs | 14 +++++++------- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 78a4c8ff..0089e7f8 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -76,7 +76,7 @@ crossbeam = { version = "0.8.1", optional = true } num_cpus = "1.13.0" threadpool = "1.8.1" heed = { git = "https://github.com/timokoesters/heed.git", rev = "f6f825da7fb2c758867e05ad973ef800a6fe1d5d", optional = true } -rocksdb = { version = "0.17.0", default-features = false, features = ["multi-threaded-cf", "zstd"], optional = true } +rocksdb = { version = "0.17.0", default-features = true, features = ["multi-threaded-cf", "zstd"], optional = true } thread_local = "1.1.3" # used for TURN server authentication diff --git a/src/database.rs b/src/database.rs index 4f230f32..79b82088 100644 --- a/src/database.rs +++ b/src/database.rs @@ -130,7 +130,7 @@ fn default_db_cache_capacity_mb() -> f64 { } fn default_rocksdb_max_open_files() -> i32 { - 512 + 20 } fn default_pdu_cache_capacity() -> u32 { @@ -361,15 +361,15 @@ impl Database { .try_into() .expect("pdu cache capacity fits into usize"), )), - auth_chain_cache: Mutex::new(LruCache::new(1_000_000)), - shorteventid_cache: Mutex::new(LruCache::new(1_000_000)), - eventidshort_cache: Mutex::new(LruCache::new(1_000_000)), - shortstatekey_cache: Mutex::new(LruCache::new(1_000_000)), - statekeyshort_cache: Mutex::new(LruCache::new(1_000_000)), + auth_chain_cache: Mutex::new(LruCache::new(100_000)), + shorteventid_cache: Mutex::new(LruCache::new(100_000)), + eventidshort_cache: Mutex::new(LruCache::new(100_000)), + shortstatekey_cache: Mutex::new(LruCache::new(100_000)), + statekeyshort_cache: Mutex::new(LruCache::new(100_000)), our_real_users_cache: RwLock::new(HashMap::new()), appservice_in_room_cache: RwLock::new(HashMap::new()), lazy_load_waiting: Mutex::new(HashMap::new()), - stateinfo_cache: Mutex::new(LruCache::new(1000)), + stateinfo_cache: Mutex::new(LruCache::new(100)), }, account_data: account_data::AccountData { roomuserdataid_accountdata: builder.open_tree("roomuserdataid_accountdata")?, From 23aecb78c7c5ba5872a058f806cd722787eefc10 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timo=20K=C3=B6sters?= Date: Mon, 31 Jan 2022 15:39:46 +0100 Subject: [PATCH 29/40] fix: use to_lowercase on /register/available username --- src/client_server/account.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/client_server/account.rs b/src/client_server/account.rs index c4e118c9..80c6f702 100644 --- a/src/client_server/account.rs +++ b/src/client_server/account.rs @@ -60,7 +60,7 @@ pub async fn get_register_available_route( body: Ruma>, ) -> ConduitResult { // Validate user id - let user_id = UserId::parse_with_server_name(body.username.clone(), db.globals.server_name()) + let user_id = UserId::parse_with_server_name(body.username.to_lowercase(), db.globals.server_name()) .ok() .filter(|user_id| { !user_id.is_historical() && user_id.server_name() == db.globals.server_name() From caf9834e50b540fc48bf8cf50dc439c57c503de9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timo=20K=C3=B6sters?= Date: Tue, 1 Feb 2022 14:42:13 +0100 Subject: [PATCH 30/40] feat: cache capacity modifier --- src/database.rs | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/src/database.rs b/src/database.rs index 79b82088..449d71bd 100644 --- a/src/database.rs +++ b/src/database.rs @@ -49,6 +49,8 @@ pub struct Config { database_path: String, #[serde(default = "default_db_cache_capacity_mb")] db_cache_capacity_mb: f64, + #[serde(default = "default_conduit_cache_capacity_modifier")] + conduit_cache_capacity_modifier: f64, #[serde(default = "default_rocksdb_max_open_files")] rocksdb_max_open_files: i32, #[serde(default = "default_pdu_cache_capacity")] @@ -129,6 +131,10 @@ fn default_db_cache_capacity_mb() -> f64 { 10.0 } +fn default_conduit_cache_capacity_modifier() -> f64 { + 1.0 +} + fn default_rocksdb_max_open_files() -> i32 { 20 } @@ -361,15 +367,15 @@ impl Database { .try_into() .expect("pdu cache capacity fits into usize"), )), - auth_chain_cache: Mutex::new(LruCache::new(100_000)), - shorteventid_cache: Mutex::new(LruCache::new(100_000)), - eventidshort_cache: Mutex::new(LruCache::new(100_000)), - shortstatekey_cache: Mutex::new(LruCache::new(100_000)), - statekeyshort_cache: Mutex::new(LruCache::new(100_000)), + auth_chain_cache: Mutex::new(LruCache::new((100_000.0 * config.conduit_cache_capacity_modifier) as usize)), + shorteventid_cache: Mutex::new(LruCache::new((100_000.0 * config.conduit_cache_capacity_modifier) as usize)), + eventidshort_cache: Mutex::new(LruCache::new((100_000.0 * config.conduit_cache_capacity_modifier) as usize)), + shortstatekey_cache: Mutex::new(LruCache::new((100_000.0 * config.conduit_cache_capacity_modifier) as usize)), + statekeyshort_cache: Mutex::new(LruCache::new((100_000.0 * config.conduit_cache_capacity_modifier) as usize)), our_real_users_cache: RwLock::new(HashMap::new()), appservice_in_room_cache: RwLock::new(HashMap::new()), lazy_load_waiting: Mutex::new(HashMap::new()), - stateinfo_cache: Mutex::new(LruCache::new(100)), + stateinfo_cache: Mutex::new(LruCache::new((100.0 * config.conduit_cache_capacity_modifier) as usize)), }, account_data: account_data::AccountData { roomuserdataid_accountdata: builder.open_tree("roomuserdataid_accountdata")?, From fa4099b138b3a4cdf6727acb14c47f20ace5f38e Mon Sep 17 00:00:00 2001 From: Jonas Zohren Date: Tue, 1 Feb 2022 23:51:38 +0000 Subject: [PATCH 31/40] Use prebuilt CI-containers from https://gitlab.com/jfowl/conduit-containers Also run all builds on approved MRs --- .gitlab-ci.yml | 39 ++++++++++++++++----------------------- Cross.toml | 8 ++++---- cross/build.sh | 31 ------------------------------- cross/test.sh | 8 -------- 4 files changed, 20 insertions(+), 66 deletions(-) delete mode 100755 cross/build.sh delete mode 100755 cross/test.sh diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 741b5327..6f1a19f0 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -24,8 +24,9 @@ variables: - if: '$CI_COMMIT_BRANCH == "master"' - if: '$CI_COMMIT_BRANCH == "next"' - if: "$CI_COMMIT_TAG" + - if: '($CI_MERGE_REQUEST_APPROVED == "true") || $BUILD_EVERYTHING' # Once MR is approved, test all builds. Or if BUILD_EVERYTHING is set. interruptible: true - image: "rust:1.58" + image: "registry.gitlab.com/jfowl/conduit-containers/rust-with-tools:latest" tags: ["docker"] services: ["docker:dind"] variables: @@ -36,27 +37,23 @@ variables: before_script: - 'echo "Building for target $TARGET"' - "rustup show && rustc --version && cargo --version" # Print version info for debugging - # install cross-compiling prerequisites - - 'apt-get update && apt-get install -y docker.io && docker version' # install docker - - 'cargo install cross && cross --version' # install cross # fix cargo and rustup mounts from this container (https://gitlab.com/gitlab-org/gitlab-foss/-/issues/41227) - - 'mkdir -p $SHARED_PATH/cargo' - - 'cp -r $CARGO_HOME/bin $SHARED_PATH/cargo' - - 'cp -r $RUSTUP_HOME $SHARED_PATH' - - 'export CARGO_HOME=$SHARED_PATH/cargo RUSTUP_HOME=$SHARED_PATH/rustup' + - "mkdir -p $SHARED_PATH/cargo" + - "cp -r $CARGO_HOME/bin $SHARED_PATH/cargo" + - "cp -r $RUSTUP_HOME $SHARED_PATH" + - "export CARGO_HOME=$SHARED_PATH/cargo RUSTUP_HOME=$SHARED_PATH/rustup" # If provided, bring in caching through sccache, which uses an external S3 endpoint to store compilation results. - # The sccache binary is stored in the sysroot of the rustc installation since that directory is added to the path of the cross container. - - if [ -n "${SCCACHE_BIN_URL}" ]; then RUSTC_SYSROOT=$(rustc --print sysroot) && curl $SCCACHE_BIN_URL --output $RUSTC_SYSROOT/bin/sccache && chmod +x $RUSTC_SYSROOT/bin/sccache && export RUSTC_WRAPPER=sccache; fi + - if [ -n "${SCCACHE_ENDPOINT}" ]; then export RUSTC_WRAPPER=/sccache; fi script: # cross-compile conduit for target - - 'time ./cross/build.sh --locked --release' + - 'time cross build --target="$TARGET" --locked --release' - 'mv "target/$TARGET/release/conduit" "conduit-$TARGET"' # print information about linking for debugging - - 'file conduit-$TARGET' # print file information + - "file conduit-$TARGET" # print file information - 'readelf --dynamic conduit-$TARGET | sed -e "/NEEDED/q1"' # ensure statically linked cache: # https://doc.rust-lang.org/cargo/guide/cargo-home.html#caching-the-cargo-home-in-ci - key: 'cargo-cache-$TARGET' + key: "cargo-cache-$TARGET" paths: - $SHARED_PATH/cargo/registry/index - $SHARED_PATH/cargo/registry/cache @@ -125,10 +122,10 @@ build:release:cargo:aarch64-unknown-linux-musl: key: "build_cache--$TARGET--$CI_COMMIT_BRANCH--debug" script: # cross-compile conduit for target - - 'time ./cross/build.sh --locked' + - 'time time cross build --target="$TARGET" --locked' - 'mv "target/$TARGET/debug/conduit" "conduit-debug-$TARGET"' # print information about linking for debugging - - 'file conduit-debug-$TARGET' # print file information + - "file conduit-debug-$TARGET" # print file information - 'readelf --dynamic conduit-debug-$TARGET | sed -e "/NEEDED/q1"' # ensure statically linked artifacts: expire_in: 4 weeks @@ -230,24 +227,20 @@ docker:master:dockerhub: test:cargo: stage: "test" needs: [] - image: "rust:latest" + image: "registry.gitlab.com/jfowl/conduit-containers/rust-with-tools:latest" tags: ["docker"] variables: CARGO_INCREMENTAL: "false" # https://matklad.github.io/2021/09/04/fast-rust-builds.html#ci-workflow interruptible: true before_script: - # - mkdir -p $CARGO_HOME - - apt-get update -yqq - - apt-get install -yqq --no-install-recommends build-essential libssl-dev pkg-config libclang-dev - rustup component add clippy rustfmt - - curl "https://faulty-storage.de/gitlab-report" --output ./gitlab-report && chmod +x ./gitlab-report # If provided, bring in caching through sccache, which uses an external S3 endpoint to store compilation results: - - if [ -n "${SCCACHE_BIN_URL}" ]; then curl $SCCACHE_BIN_URL --output /sccache && chmod +x /sccache && export RUSTC_WRAPPER=/sccache; fi + - if [ -n "${SCCACHE_ENDPOINT}" ]; then export RUSTC_WRAPPER=/usr/local/cargo/bin/sccache; fi script: - rustc --version && cargo --version # Print version info for debugging - cargo fmt --all -- --check - - "cargo test --color always --workspace --verbose --locked --no-fail-fast -- -Z unstable-options --format json | ./gitlab-report -p test > $CI_PROJECT_DIR/report.xml" - - "cargo clippy --color always --verbose --message-format=json | ./gitlab-report -p clippy > $CI_PROJECT_DIR/gl-code-quality-report.json" + - "cargo test --color always --workspace --verbose --locked --no-fail-fast -- -Z unstable-options --format json | gitlab-report -p test > $CI_PROJECT_DIR/report.xml" + - "cargo clippy --color always --verbose --message-format=json | gitlab-report -p clippy > $CI_PROJECT_DIR/gl-code-quality-report.json" artifacts: when: always reports: diff --git a/Cross.toml b/Cross.toml index a989a98f..a1387b43 100644 --- a/Cross.toml +++ b/Cross.toml @@ -11,13 +11,13 @@ passthrough = [ ] [target.aarch64-unknown-linux-musl] -image = "rust-cross:aarch64-unknown-linux-musl" +image = "registry.gitlab.com/jfowl/conduit-containers/rust-cross-aarch64-unknown-linux-musl:latest" [target.arm-unknown-linux-musleabihf] -image = "rust-cross:arm-unknown-linux-musleabihf" +image = "registry.gitlab.com/jfowl/conduit-containers/rust-cross-arm-unknown-linux-musleabihf:latest" [target.armv7-unknown-linux-musleabihf] -image = "rust-cross:armv7-unknown-linux-musleabihf" +image = "registry.gitlab.com/jfowl/conduit-containers/rust-cross-armv7-unknown-linux-musleabihf:latest" [target.x86_64-unknown-linux-musl] -image = "rust-cross:x86_64-unknown-linux-musl" +image = "registry.gitlab.com/jfowl/conduit-containers/rust-cross-x86_64-unknown-linux-musl:latest" diff --git a/cross/build.sh b/cross/build.sh deleted file mode 100755 index 8f64ff87..00000000 --- a/cross/build.sh +++ /dev/null @@ -1,31 +0,0 @@ -#!/bin/bash -set -ex - -# build custom container with libclang and static compilation -tag="rust-cross:${TARGET:?}" -docker build --tag="$tag" - << EOF -FROM rustembedded/cross:$TARGET - -# Install libclang for generating bindings with rust-bindgen -# The architecture is not relevant here since it's not used for compilation -RUN apt-get update && \ - apt-get install --assume-yes libclang-dev - -# Set the target prefix -ENV TARGET_PREFIX="/usr/local/$(echo "${TARGET:?}" | sed -e 's/armv7/arm/' -e 's/-unknown//')" - -# Make sure that cc-rs links libc/libstdc++ statically when cross-compiling -# See https://github.com/alexcrichton/cc-rs#external-configuration-via-environment-variables for more information -ENV RUSTFLAGS="-L\$TARGET_PREFIX/lib" CXXSTDLIB="static=stdc++" -# Forcefully linking against libatomic, libc and libgcc is required for arm32, otherwise symbols are missing -$([[ $TARGET =~ arm ]] && echo 'ENV RUSTFLAGS="$RUSTFLAGS -Clink-arg=-static-libgcc -Clink-arg=-lgcc -lstatic=atomic -lstatic=c"') -# Strip symbols while compiling in release mode -$([[ $@ =~ -r ]] && echo 'ENV RUSTFLAGS="$RUSTFLAGS -Clink-arg=-s"') - -# Make sure that rust-bindgen uses the correct include path when cross-compiling -# See https://github.com/rust-lang/rust-bindgen#environment-variables for more information -ENV BINDGEN_EXTRA_CLANG_ARGS="-I\$TARGET_PREFIX/include" -EOF - -# build conduit for a specific target -cross build --target="$TARGET" $@ diff --git a/cross/test.sh b/cross/test.sh deleted file mode 100755 index 0aa0909c..00000000 --- a/cross/test.sh +++ /dev/null @@ -1,8 +0,0 @@ -#!/usr/bin/env sh -set -ex - -# Build conduit for a specific target -cross/build.sh $@ - -# Test conduit for a specific target -cross test --target="$TARGET" $@ From a5f004d7e9c783caf280884d7fd332c7bafa67ce Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timo=20K=C3=B6sters?= Date: Wed, 2 Feb 2022 12:36:55 +0100 Subject: [PATCH 32/40] fix: signature mismatch on odd send_join servers --- Cargo.toml | 2 +- src/client_server/account.rs | 19 ++++++++++--------- src/client_server/membership.rs | 17 +++++++++-------- src/database.rs | 24 ++++++++++++++++++------ 4 files changed, 38 insertions(+), 24 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 0089e7f8..78a4c8ff 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -76,7 +76,7 @@ crossbeam = { version = "0.8.1", optional = true } num_cpus = "1.13.0" threadpool = "1.8.1" heed = { git = "https://github.com/timokoesters/heed.git", rev = "f6f825da7fb2c758867e05ad973ef800a6fe1d5d", optional = true } -rocksdb = { version = "0.17.0", default-features = true, features = ["multi-threaded-cf", "zstd"], optional = true } +rocksdb = { version = "0.17.0", default-features = false, features = ["multi-threaded-cf", "zstd"], optional = true } thread_local = "1.1.3" # used for TURN server authentication diff --git a/src/client_server/account.rs b/src/client_server/account.rs index 80c6f702..ff348545 100644 --- a/src/client_server/account.rs +++ b/src/client_server/account.rs @@ -60,15 +60,16 @@ pub async fn get_register_available_route( body: Ruma>, ) -> ConduitResult { // Validate user id - let user_id = UserId::parse_with_server_name(body.username.to_lowercase(), db.globals.server_name()) - .ok() - .filter(|user_id| { - !user_id.is_historical() && user_id.server_name() == db.globals.server_name() - }) - .ok_or(Error::BadRequest( - ErrorKind::InvalidUsername, - "Username is invalid.", - ))?; + let user_id = + UserId::parse_with_server_name(body.username.to_lowercase(), db.globals.server_name()) + .ok() + .filter(|user_id| { + !user_id.is_historical() && user_id.server_name() == db.globals.server_name() + }) + .ok_or(Error::BadRequest( + ErrorKind::InvalidUsername, + "Username is invalid.", + ))?; // Check if username is creative enough if db.users.exists(&user_id)? { diff --git a/src/client_server/membership.rs b/src/client_server/membership.rs index 70352784..216c4c0a 100644 --- a/src/client_server/membership.rs +++ b/src/client_server/membership.rs @@ -655,7 +655,7 @@ async fn join_room_by_id_helper( db.rooms.get_or_create_shortroomid(room_id, &db.globals)?; - let pdu = PduEvent::from_id_val(event_id, join_event.clone()) + let parsed_pdu = PduEvent::from_id_val(event_id, join_event.clone()) .map_err(|_| Error::BadServerResponse("Invalid join event PDU."))?; let mut state = HashMap::new(); @@ -695,14 +695,15 @@ async fn join_room_by_id_helper( } let incoming_shortstatekey = db.rooms.get_or_create_shortstatekey( - &pdu.kind, - pdu.state_key + &parsed_pdu.kind, + parsed_pdu + .state_key .as_ref() .expect("Pdu is a membership state event"), &db.globals, )?; - state.insert(incoming_shortstatekey, pdu.event_id.clone()); + state.insert(incoming_shortstatekey, parsed_pdu.event_id.clone()); let create_shortstatekey = db .rooms @@ -738,12 +739,12 @@ async fn join_room_by_id_helper( // We append to state before appending the pdu, so we don't have a moment in time with the // pdu without it's state. This is okay because append_pdu can't fail. - let statehashid = db.rooms.append_to_state(&pdu, &db.globals)?; + let statehashid = db.rooms.append_to_state(&parsed_pdu, &db.globals)?; db.rooms.append_pdu( - &pdu, - utils::to_canonical_object(&pdu).expect("Pdu is valid canonical object"), - iter::once(&*pdu.event_id), + &parsed_pdu, + join_event, + iter::once(&*parsed_pdu.event_id), db, )?; diff --git a/src/database.rs b/src/database.rs index 449d71bd..8d245b7f 100644 --- a/src/database.rs +++ b/src/database.rs @@ -367,15 +367,27 @@ impl Database { .try_into() .expect("pdu cache capacity fits into usize"), )), - auth_chain_cache: Mutex::new(LruCache::new((100_000.0 * config.conduit_cache_capacity_modifier) as usize)), - shorteventid_cache: Mutex::new(LruCache::new((100_000.0 * config.conduit_cache_capacity_modifier) as usize)), - eventidshort_cache: Mutex::new(LruCache::new((100_000.0 * config.conduit_cache_capacity_modifier) as usize)), - shortstatekey_cache: Mutex::new(LruCache::new((100_000.0 * config.conduit_cache_capacity_modifier) as usize)), - statekeyshort_cache: Mutex::new(LruCache::new((100_000.0 * config.conduit_cache_capacity_modifier) as usize)), + auth_chain_cache: Mutex::new(LruCache::new( + (100_000.0 * config.conduit_cache_capacity_modifier) as usize, + )), + shorteventid_cache: Mutex::new(LruCache::new( + (100_000.0 * config.conduit_cache_capacity_modifier) as usize, + )), + eventidshort_cache: Mutex::new(LruCache::new( + (100_000.0 * config.conduit_cache_capacity_modifier) as usize, + )), + shortstatekey_cache: Mutex::new(LruCache::new( + (100_000.0 * config.conduit_cache_capacity_modifier) as usize, + )), + statekeyshort_cache: Mutex::new(LruCache::new( + (100_000.0 * config.conduit_cache_capacity_modifier) as usize, + )), our_real_users_cache: RwLock::new(HashMap::new()), appservice_in_room_cache: RwLock::new(HashMap::new()), lazy_load_waiting: Mutex::new(HashMap::new()), - stateinfo_cache: Mutex::new(LruCache::new((100.0 * config.conduit_cache_capacity_modifier) as usize)), + stateinfo_cache: Mutex::new(LruCache::new( + (100.0 * config.conduit_cache_capacity_modifier) as usize, + )), }, account_data: account_data::AccountData { roomuserdataid_accountdata: builder.open_tree("roomuserdataid_accountdata")?, From bfcf2db497ffab518b946922205fb9a5661d8c27 Mon Sep 17 00:00:00 2001 From: user Date: Fri, 28 Jan 2022 22:26:56 -0800 Subject: [PATCH 33/40] fix: mention dependencies to build from source --- DEPLOY.md | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/DEPLOY.md b/DEPLOY.md index 38e1e286..d9f91e03 100644 --- a/DEPLOY.md +++ b/DEPLOY.md @@ -29,7 +29,11 @@ $ sudo wget -O /usr/local/bin/matrix-conduit $ sudo chmod +x /usr/local/bin/matrix-conduit ``` -Alternatively, you may compile the binary yourself using +Alternatively, you may compile the binary yourself + +```bash +$ sudo apt install libclang-dev build-essential +``` ```bash $ cargo build --release From da7b55b39c1ea592c0d5ec86a1988465bedaad0e Mon Sep 17 00:00:00 2001 From: Torsten Flammiger Date: Mon, 31 Jan 2022 09:27:31 +0100 Subject: [PATCH 34/40] Cleanup appservice events after removing the appservice --- src/database/admin.rs | 13 ++++++++++++- src/database/sending.rs | 31 +++++++++++++++++++++++++++++-- 2 files changed, 41 insertions(+), 3 deletions(-) diff --git a/src/database/admin.rs b/src/database/admin.rs index 81e98393..9895a83b 100644 --- a/src/database/admin.rs +++ b/src/database/admin.rs @@ -112,7 +112,18 @@ impl Admin { guard.appservice.register_appservice(yaml).unwrap(); // TODO handle error } AdminCommand::UnregisterAppservice(service_name) => { - guard.appservice.unregister_appservice(&service_name).unwrap(); // TODO: see above + if let Ok(_) = guard.appservice.unregister_appservice(&service_name) { + if let Ok(_) = guard.sending.cleanup_events(&service_name) { + let msg: String = format!("OK. Appservice {} removed", service_name); + send_message(RoomMessageEventContent::text_plain(msg), guard, &state_lock); + } else { + let msg: String = format!("WARN: Appservice {} removed, but failed to cleanup events", service_name); + send_message(RoomMessageEventContent::text_plain(msg), guard, &state_lock); + } + } else { + let msg: String = format!("ERR. Appservice {} not removed", service_name); + send_message(RoomMessageEventContent::text_plain(msg), guard, &state_lock); + } } AdminCommand::ListAppservices => { if let Ok(appservices) = guard.appservice.iter_ids().map(|ids| ids.collect::>()) { diff --git a/src/database/sending.rs b/src/database/sending.rs index 69f7c444..af4ac676 100644 --- a/src/database/sending.rs +++ b/src/database/sending.rs @@ -480,6 +480,26 @@ impl Sending { hash.as_ref().to_owned() } + /// Cleanup event data + /// Used for instance after we remove an appservice registration + /// + #[tracing::instrument(skip(self))] + pub fn cleanup_events(&self, key_id: &str) -> Result<()> { + let mut prefix = b"+".to_vec(); + prefix.extend_from_slice(key_id.as_bytes()); + prefix.push(0xff); + + for (key, _) in self.servercurrentevent_data.scan_prefix(prefix.clone()) { + self.servercurrentevent_data.remove(&key).unwrap(); + } + + for (key, _) in self.servernameevent_data.scan_prefix(prefix.clone()) { + self.servernameevent_data.remove(&key).unwrap(); + } + + Ok(()) + } + #[tracing::instrument(skip(db, events, kind))] async fn handle_events( kind: OutgoingKind, @@ -520,8 +540,15 @@ impl Sending { &db.globals, db.appservice .get_registration(server.as_str()) - .unwrap() - .unwrap(), // TODO: handle error + .map_err(|e| (kind.clone(), e))? + .ok_or_else(|| { + ( + kind.clone(), + Error::bad_database( + "[Appservice] Could not load registration from db.", + ), + ) + })?, appservice::event::push_events::v1::Request { events: &pdu_jsons, txn_id: (&*base64::encode_config( From 8f69f02e592299dbe3713e238b94b19bfc445ec8 Mon Sep 17 00:00:00 2001 From: Torsten Flammiger Date: Mon, 31 Jan 2022 10:07:49 +0100 Subject: [PATCH 35/40] add error handling for register_appservice too --- src/database/admin.rs | 13 ++++++++++++- src/database/appservice.rs | 4 ++-- 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/src/database/admin.rs b/src/database/admin.rs index 9895a83b..eef6ce10 100644 --- a/src/database/admin.rs +++ b/src/database/admin.rs @@ -109,7 +109,18 @@ impl Admin { } } AdminCommand::RegisterAppservice(yaml) => { - guard.appservice.register_appservice(yaml).unwrap(); // TODO handle error + match guard.appservice.register_appservice(yaml) { + Ok(Some(id)) => { + let msg: String = format!("OK. Appservice {} created", id); + send_message(RoomMessageEventContent::text_plain(msg), guard, &state_lock); + } + Ok(None) => { + send_message(RoomMessageEventContent::text_plain("WARN. Appservice created, but its ID was not returned!"), guard, &state_lock); + } + Err(_) => { + send_message(RoomMessageEventContent::text_plain("ERR: Failed register appservice. Check server log"), guard, &state_lock); + } + } } AdminCommand::UnregisterAppservice(service_name) => { if let Ok(_) = guard.appservice.unregister_appservice(&service_name) { diff --git a/src/database/appservice.rs b/src/database/appservice.rs index 88de1f33..8b29aca9 100644 --- a/src/database/appservice.rs +++ b/src/database/appservice.rs @@ -12,7 +12,7 @@ pub struct Appservice { } impl Appservice { - pub fn register_appservice(&self, yaml: serde_yaml::Value) -> Result<()> { + pub fn register_appservice(&self, yaml: serde_yaml::Value) -> Result> { // TODO: Rumaify let id = yaml.get("id").unwrap().as_str().unwrap(); self.id_appserviceregistrations.insert( @@ -24,7 +24,7 @@ impl Appservice { .unwrap() .insert(id.to_owned(), yaml); - Ok(()) + Ok(Some(id.to_owned())) } /// Remove an appservice registration From e24d75cffc8f00d526848a93a4e2cfce54bf69a2 Mon Sep 17 00:00:00 2001 From: Torsten Flammiger Date: Mon, 31 Jan 2022 11:52:33 +0100 Subject: [PATCH 36/40] Return the ID of the appservice that was created by register_appservice --- src/database/admin.rs | 5 +---- src/database/appservice.rs | 8 +++++--- 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/src/database/admin.rs b/src/database/admin.rs index eef6ce10..a214796b 100644 --- a/src/database/admin.rs +++ b/src/database/admin.rs @@ -110,13 +110,10 @@ impl Admin { } AdminCommand::RegisterAppservice(yaml) => { match guard.appservice.register_appservice(yaml) { - Ok(Some(id)) => { + Ok(id) => { let msg: String = format!("OK. Appservice {} created", id); send_message(RoomMessageEventContent::text_plain(msg), guard, &state_lock); } - Ok(None) => { - send_message(RoomMessageEventContent::text_plain("WARN. Appservice created, but its ID was not returned!"), guard, &state_lock); - } Err(_) => { send_message(RoomMessageEventContent::text_plain("ERR: Failed register appservice. Check server log"), guard, &state_lock); } diff --git a/src/database/appservice.rs b/src/database/appservice.rs index 8b29aca9..edd5009b 100644 --- a/src/database/appservice.rs +++ b/src/database/appservice.rs @@ -12,7 +12,9 @@ pub struct Appservice { } impl Appservice { - pub fn register_appservice(&self, yaml: serde_yaml::Value) -> Result> { + /// Registers an appservice and returns the ID to the caller + /// + pub fn register_appservice(&self, yaml: serde_yaml::Value) -> Result { // TODO: Rumaify let id = yaml.get("id").unwrap().as_str().unwrap(); self.id_appserviceregistrations.insert( @@ -22,9 +24,9 @@ impl Appservice { self.cached_registrations .write() .unwrap() - .insert(id.to_owned(), yaml); + .insert(id.to_owned(), yaml.to_owned()); - Ok(Some(id.to_owned())) + Ok(id.to_owned()) } /// Remove an appservice registration From 9478c75f9dcd040cb9f03deb5ea809f117985de2 Mon Sep 17 00:00:00 2001 From: Jonas Zohren Date: Tue, 1 Feb 2022 23:51:38 +0000 Subject: [PATCH 37/40] Use prebuilt CI-containers from https://gitlab.com/jfowl/conduit-containers Also run all builds on approved MRs --- .gitlab-ci.yml | 39 ++++++++++--------------- Cross.toml | 8 ++--- cross/build.sh | 31 -------------------- cross/test.sh | 8 ----- docker/ci-binaries-packaging.Dockerfile | 4 ++- docker/healthcheck.sh | 2 +- 6 files changed, 24 insertions(+), 68 deletions(-) delete mode 100755 cross/build.sh delete mode 100755 cross/test.sh diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 741b5327..6f1a19f0 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -24,8 +24,9 @@ variables: - if: '$CI_COMMIT_BRANCH == "master"' - if: '$CI_COMMIT_BRANCH == "next"' - if: "$CI_COMMIT_TAG" + - if: '($CI_MERGE_REQUEST_APPROVED == "true") || $BUILD_EVERYTHING' # Once MR is approved, test all builds. Or if BUILD_EVERYTHING is set. interruptible: true - image: "rust:1.58" + image: "registry.gitlab.com/jfowl/conduit-containers/rust-with-tools:latest" tags: ["docker"] services: ["docker:dind"] variables: @@ -36,27 +37,23 @@ variables: before_script: - 'echo "Building for target $TARGET"' - "rustup show && rustc --version && cargo --version" # Print version info for debugging - # install cross-compiling prerequisites - - 'apt-get update && apt-get install -y docker.io && docker version' # install docker - - 'cargo install cross && cross --version' # install cross # fix cargo and rustup mounts from this container (https://gitlab.com/gitlab-org/gitlab-foss/-/issues/41227) - - 'mkdir -p $SHARED_PATH/cargo' - - 'cp -r $CARGO_HOME/bin $SHARED_PATH/cargo' - - 'cp -r $RUSTUP_HOME $SHARED_PATH' - - 'export CARGO_HOME=$SHARED_PATH/cargo RUSTUP_HOME=$SHARED_PATH/rustup' + - "mkdir -p $SHARED_PATH/cargo" + - "cp -r $CARGO_HOME/bin $SHARED_PATH/cargo" + - "cp -r $RUSTUP_HOME $SHARED_PATH" + - "export CARGO_HOME=$SHARED_PATH/cargo RUSTUP_HOME=$SHARED_PATH/rustup" # If provided, bring in caching through sccache, which uses an external S3 endpoint to store compilation results. - # The sccache binary is stored in the sysroot of the rustc installation since that directory is added to the path of the cross container. - - if [ -n "${SCCACHE_BIN_URL}" ]; then RUSTC_SYSROOT=$(rustc --print sysroot) && curl $SCCACHE_BIN_URL --output $RUSTC_SYSROOT/bin/sccache && chmod +x $RUSTC_SYSROOT/bin/sccache && export RUSTC_WRAPPER=sccache; fi + - if [ -n "${SCCACHE_ENDPOINT}" ]; then export RUSTC_WRAPPER=/sccache; fi script: # cross-compile conduit for target - - 'time ./cross/build.sh --locked --release' + - 'time cross build --target="$TARGET" --locked --release' - 'mv "target/$TARGET/release/conduit" "conduit-$TARGET"' # print information about linking for debugging - - 'file conduit-$TARGET' # print file information + - "file conduit-$TARGET" # print file information - 'readelf --dynamic conduit-$TARGET | sed -e "/NEEDED/q1"' # ensure statically linked cache: # https://doc.rust-lang.org/cargo/guide/cargo-home.html#caching-the-cargo-home-in-ci - key: 'cargo-cache-$TARGET' + key: "cargo-cache-$TARGET" paths: - $SHARED_PATH/cargo/registry/index - $SHARED_PATH/cargo/registry/cache @@ -125,10 +122,10 @@ build:release:cargo:aarch64-unknown-linux-musl: key: "build_cache--$TARGET--$CI_COMMIT_BRANCH--debug" script: # cross-compile conduit for target - - 'time ./cross/build.sh --locked' + - 'time time cross build --target="$TARGET" --locked' - 'mv "target/$TARGET/debug/conduit" "conduit-debug-$TARGET"' # print information about linking for debugging - - 'file conduit-debug-$TARGET' # print file information + - "file conduit-debug-$TARGET" # print file information - 'readelf --dynamic conduit-debug-$TARGET | sed -e "/NEEDED/q1"' # ensure statically linked artifacts: expire_in: 4 weeks @@ -230,24 +227,20 @@ docker:master:dockerhub: test:cargo: stage: "test" needs: [] - image: "rust:latest" + image: "registry.gitlab.com/jfowl/conduit-containers/rust-with-tools:latest" tags: ["docker"] variables: CARGO_INCREMENTAL: "false" # https://matklad.github.io/2021/09/04/fast-rust-builds.html#ci-workflow interruptible: true before_script: - # - mkdir -p $CARGO_HOME - - apt-get update -yqq - - apt-get install -yqq --no-install-recommends build-essential libssl-dev pkg-config libclang-dev - rustup component add clippy rustfmt - - curl "https://faulty-storage.de/gitlab-report" --output ./gitlab-report && chmod +x ./gitlab-report # If provided, bring in caching through sccache, which uses an external S3 endpoint to store compilation results: - - if [ -n "${SCCACHE_BIN_URL}" ]; then curl $SCCACHE_BIN_URL --output /sccache && chmod +x /sccache && export RUSTC_WRAPPER=/sccache; fi + - if [ -n "${SCCACHE_ENDPOINT}" ]; then export RUSTC_WRAPPER=/usr/local/cargo/bin/sccache; fi script: - rustc --version && cargo --version # Print version info for debugging - cargo fmt --all -- --check - - "cargo test --color always --workspace --verbose --locked --no-fail-fast -- -Z unstable-options --format json | ./gitlab-report -p test > $CI_PROJECT_DIR/report.xml" - - "cargo clippy --color always --verbose --message-format=json | ./gitlab-report -p clippy > $CI_PROJECT_DIR/gl-code-quality-report.json" + - "cargo test --color always --workspace --verbose --locked --no-fail-fast -- -Z unstable-options --format json | gitlab-report -p test > $CI_PROJECT_DIR/report.xml" + - "cargo clippy --color always --verbose --message-format=json | gitlab-report -p clippy > $CI_PROJECT_DIR/gl-code-quality-report.json" artifacts: when: always reports: diff --git a/Cross.toml b/Cross.toml index a989a98f..a1387b43 100644 --- a/Cross.toml +++ b/Cross.toml @@ -11,13 +11,13 @@ passthrough = [ ] [target.aarch64-unknown-linux-musl] -image = "rust-cross:aarch64-unknown-linux-musl" +image = "registry.gitlab.com/jfowl/conduit-containers/rust-cross-aarch64-unknown-linux-musl:latest" [target.arm-unknown-linux-musleabihf] -image = "rust-cross:arm-unknown-linux-musleabihf" +image = "registry.gitlab.com/jfowl/conduit-containers/rust-cross-arm-unknown-linux-musleabihf:latest" [target.armv7-unknown-linux-musleabihf] -image = "rust-cross:armv7-unknown-linux-musleabihf" +image = "registry.gitlab.com/jfowl/conduit-containers/rust-cross-armv7-unknown-linux-musleabihf:latest" [target.x86_64-unknown-linux-musl] -image = "rust-cross:x86_64-unknown-linux-musl" +image = "registry.gitlab.com/jfowl/conduit-containers/rust-cross-x86_64-unknown-linux-musl:latest" diff --git a/cross/build.sh b/cross/build.sh deleted file mode 100755 index 8f64ff87..00000000 --- a/cross/build.sh +++ /dev/null @@ -1,31 +0,0 @@ -#!/bin/bash -set -ex - -# build custom container with libclang and static compilation -tag="rust-cross:${TARGET:?}" -docker build --tag="$tag" - << EOF -FROM rustembedded/cross:$TARGET - -# Install libclang for generating bindings with rust-bindgen -# The architecture is not relevant here since it's not used for compilation -RUN apt-get update && \ - apt-get install --assume-yes libclang-dev - -# Set the target prefix -ENV TARGET_PREFIX="/usr/local/$(echo "${TARGET:?}" | sed -e 's/armv7/arm/' -e 's/-unknown//')" - -# Make sure that cc-rs links libc/libstdc++ statically when cross-compiling -# See https://github.com/alexcrichton/cc-rs#external-configuration-via-environment-variables for more information -ENV RUSTFLAGS="-L\$TARGET_PREFIX/lib" CXXSTDLIB="static=stdc++" -# Forcefully linking against libatomic, libc and libgcc is required for arm32, otherwise symbols are missing -$([[ $TARGET =~ arm ]] && echo 'ENV RUSTFLAGS="$RUSTFLAGS -Clink-arg=-static-libgcc -Clink-arg=-lgcc -lstatic=atomic -lstatic=c"') -# Strip symbols while compiling in release mode -$([[ $@ =~ -r ]] && echo 'ENV RUSTFLAGS="$RUSTFLAGS -Clink-arg=-s"') - -# Make sure that rust-bindgen uses the correct include path when cross-compiling -# See https://github.com/rust-lang/rust-bindgen#environment-variables for more information -ENV BINDGEN_EXTRA_CLANG_ARGS="-I\$TARGET_PREFIX/include" -EOF - -# build conduit for a specific target -cross build --target="$TARGET" $@ diff --git a/cross/test.sh b/cross/test.sh deleted file mode 100755 index 0aa0909c..00000000 --- a/cross/test.sh +++ /dev/null @@ -1,8 +0,0 @@ -#!/usr/bin/env sh -set -ex - -# Build conduit for a specific target -cross/build.sh $@ - -# Test conduit for a specific target -cross test --target="$TARGET" $@ diff --git a/docker/ci-binaries-packaging.Dockerfile b/docker/ci-binaries-packaging.Dockerfile index a6339be3..bb67bb22 100644 --- a/docker/ci-binaries-packaging.Dockerfile +++ b/docker/ci-binaries-packaging.Dockerfile @@ -19,8 +19,10 @@ ENV CONDUIT_CONFIG="/srv/conduit/conduit.toml" # Conduit needs: # ca-certificates: for https +# iproute2: for `ss` for the healthcheck script RUN apk add --no-cache \ - ca-certificates + ca-certificates \ + iproute2 ARG CREATED diff --git a/docker/healthcheck.sh b/docker/healthcheck.sh index df7f18a5..42b2e103 100644 --- a/docker/healthcheck.sh +++ b/docker/healthcheck.sh @@ -3,7 +3,7 @@ # If the config file does not contain a default port and the CONDUIT_PORT env is not set, create # try to get port from process list if [ -z "${CONDUIT_PORT}" ]; then - CONDUIT_PORT=$(netstat -tlp | grep conduit | grep -m1 -o ':[0-9]*' | grep -m1 -o '[0-9]*') + CONDUIT_PORT=$(ss -tlpn | grep conduit | grep -m1 -o ':[0-9]*' | grep -m1 -o '[0-9]*') fi # The actual health check. From e5bac5e4f53fa3e6565cca96b687dc8ff976f7f0 Mon Sep 17 00:00:00 2001 From: Jonas Zohren Date: Wed, 2 Feb 2022 14:07:35 +0100 Subject: [PATCH 38/40] fix: Running in Docker --- Dockerfile | 44 ++++++++++++------------- conduit-example.toml | 1 + docker/ci-binaries-packaging.Dockerfile | 5 +-- 3 files changed, 26 insertions(+), 24 deletions(-) diff --git a/Dockerfile b/Dockerfile index b629690d..0da4aace 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,9 +1,9 @@ # syntax=docker/dockerfile:1 -FROM docker.io/rust:1.58-alpine AS builder +FROM docker.io/rust:1.58-bullseye AS builder WORKDIR /usr/src/conduit # Install required packages to build Conduit and it's dependencies -RUN apk add musl-dev +RUN apt update && apt -y install libclang-11-dev # == Build dependencies without our own code separately for caching == # @@ -26,28 +26,28 @@ COPY src src # Builds conduit and places the binary at /usr/src/conduit/target/release/conduit RUN touch src/main.rs && touch src/lib.rs && cargo build --release - - - # --------------------------------------------------------------------------------------------------------------- # Stuff below this line actually ends up in the resulting docker image # --------------------------------------------------------------------------------------------------------------- -FROM docker.io/alpine:3.15.0 AS runner +FROM docker.io/debian:bullseye-slim AS runner # Standard port on which Conduit launches. # You still need to map the port when using the docker command or docker-compose. EXPOSE 6167 -# Note from @jfowl: I would like to remove this in the future and just have the Docker version be configured with envs. -ENV CONDUIT_CONFIG="/srv/conduit/conduit.toml" +# Note from @jfowl: I would like to remove the config file in the future and just have the Docker version be configured with envs. +ENV CONDUIT_CONFIG="/srv/conduit/conduit.toml" \ + CONDUIT_PORT=6167 # Conduit needs: # ca-certificates: for https -# libgcc: Apparently this is needed, even if I (@jfowl) don't know exactly why. But whatever, it's not that big. -RUN apk add --no-cache \ +# iproute2 & wget: for the healthcheck script +RUN apt update && apt -y install \ ca-certificates \ - libgcc + iproute2 \ + wget +RUN rm -rf /var/lib/apt/lists/* # Created directory for the database and media files RUN mkdir -p /srv/conduit/.local/share/conduit @@ -59,20 +59,20 @@ HEALTHCHECK --start-period=5s --interval=5s CMD ./healthcheck.sh # Copy over the actual Conduit binary from the builder stage COPY --from=builder /usr/src/conduit/target/release/conduit /srv/conduit/conduit -# Improve security: Don't run stuff as root, that does not need to run as root: -# Add www-data user and group with UID 82, as used by alpine -# https://git.alpinelinux.org/aports/tree/main/nginx/nginx.pre-install +# Improve security: Don't run stuff as root, that does not need to run as root +# Add 'conduit' user and group (100:82). The UID:GID choice is to be compatible +# with previous, Alpine-based containers, where the user and group were both +# named 'www-data'. RUN set -x ; \ - addgroup -Sg 82 www-data 2>/dev/null ; \ - adduser -S -D -H -h /srv/conduit -G www-data -g www-data www-data 2>/dev/null ; \ - addgroup www-data www-data 2>/dev/null && exit 0 ; exit 1 + groupadd -r -g 82 conduit ; \ + useradd -r -M -d /srv/conduit -o -u 100 -g conduit conduit && exit 0 ; exit 1 -# Change ownership of Conduit files to www-data user and group -RUN chown -cR www-data:www-data /srv/conduit -RUN chmod +x /srv/conduit/healthcheck.sh +# Change ownership of Conduit files to conduit user and group and make the healthcheck executable: +RUN chown -cR conduit:conduit /srv/conduit && \ + chmod +x /srv/conduit/healthcheck.sh -# Change user to www-data -USER www-data +# Change user to conduit, no root permissions afterwards: +USER conduit # Set container home directory WORKDIR /srv/conduit diff --git a/conduit-example.toml b/conduit-example.toml index c0274a4d..f1578078 100644 --- a/conduit-example.toml +++ b/conduit-example.toml @@ -22,6 +22,7 @@ database_backend = "rocksdb" # The port Conduit will be running on. You need to set up a reverse proxy in # your web server (e.g. apache or nginx), so all requests to /_matrix on port # 443 and 8448 will be forwarded to the Conduit instance running on this port +# Docker users: Don't change this, you'll need to map an external port to this. port = 6167 # Max size for uploads diff --git a/docker/ci-binaries-packaging.Dockerfile b/docker/ci-binaries-packaging.Dockerfile index bb67bb22..3731bac1 100644 --- a/docker/ci-binaries-packaging.Dockerfile +++ b/docker/ci-binaries-packaging.Dockerfile @@ -14,8 +14,9 @@ FROM docker.io/alpine:3.15.0 AS runner # You still need to map the port when using the docker command or docker-compose. EXPOSE 6167 -# Note from @jfowl: I would like to remove this in the future and just have the Docker version be configured with envs. -ENV CONDUIT_CONFIG="/srv/conduit/conduit.toml" +# Note from @jfowl: I would like to remove the config file in the future and just have the Docker version be configured with envs. +ENV CONDUIT_CONFIG="/srv/conduit/conduit.toml" \ + CONDUIT_PORT=6167 # Conduit needs: # ca-certificates: for https From c4733676cf16267ffbb0b348848e87a7d103cf37 Mon Sep 17 00:00:00 2001 From: Jonas Zohren Date: Wed, 2 Feb 2022 13:35:15 +0000 Subject: [PATCH 39/40] Apply feedback from Ticho --- Dockerfile | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/Dockerfile b/Dockerfile index 0da4aace..b631f297 100644 --- a/Dockerfile +++ b/Dockerfile @@ -3,7 +3,7 @@ FROM docker.io/rust:1.58-bullseye AS builder WORKDIR /usr/src/conduit # Install required packages to build Conduit and it's dependencies -RUN apt update && apt -y install libclang-11-dev +RUN apt update && apt -y install libclang-dev # == Build dependencies without our own code separately for caching == # @@ -45,9 +45,8 @@ ENV CONDUIT_CONFIG="/srv/conduit/conduit.toml" \ RUN apt update && apt -y install \ ca-certificates \ iproute2 \ - wget - -RUN rm -rf /var/lib/apt/lists/* + wget \ + && rm -rf /var/lib/apt/lists/* # Created directory for the database and media files RUN mkdir -p /srv/conduit/.local/share/conduit From 9ef3abacd43571300a7fbd7d35ba05d040816d8d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timo=20K=C3=B6sters?= Date: Wed, 2 Feb 2022 18:03:50 +0100 Subject: [PATCH 40/40] fix: initial state deserialize->serialize error --- src/client_server/room.rs | 7 +++++-- src/pdu.rs | 19 ++----------------- 2 files changed, 7 insertions(+), 19 deletions(-) diff --git a/src/client_server/room.rs b/src/client_server/room.rs index 52d25425..a2339639 100644 --- a/src/client_server/room.rs +++ b/src/client_server/room.rs @@ -344,10 +344,13 @@ pub async fn create_room_route( // 6. Events listed in initial_state for event in &body.initial_state { - let pdu_builder = PduBuilder::from(event.deserialize().map_err(|e| { + let mut pdu_builder = event.deserialize_as::().map_err(|e| { warn!("Invalid initial state event: {:?}", e); Error::BadRequest(ErrorKind::InvalidParam, "Invalid initial state event.") - })?); + })?; + + // Implicit state key defaults to "" + pdu_builder.state_key.get_or_insert_with(|| "".to_owned()); // Silently skip encryption events if they are not allowed if pdu_builder.event_type == EventType::RoomEncryption && !db.globals.allow_encryption() { diff --git a/src/pdu.rs b/src/pdu.rs index db9375e4..fe004609 100644 --- a/src/pdu.rs +++ b/src/pdu.rs @@ -1,9 +1,8 @@ use crate::Error; use ruma::{ events::{ - room::member::RoomMemberEventContent, AnyEphemeralRoomEvent, AnyInitialStateEvent, - AnyRoomEvent, AnyStateEvent, AnyStrippedStateEvent, AnySyncRoomEvent, AnySyncStateEvent, - EventType, StateEvent, + room::member::RoomMemberEventContent, AnyEphemeralRoomEvent, AnyRoomEvent, AnyStateEvent, + AnyStrippedStateEvent, AnySyncRoomEvent, AnySyncStateEvent, EventType, StateEvent, }, serde::{CanonicalJsonObject, CanonicalJsonValue, Raw}, state_res, EventId, MilliSecondsSinceUnixEpoch, RoomId, RoomVersionId, UInt, UserId, @@ -361,17 +360,3 @@ pub struct PduBuilder { pub state_key: Option, pub redacts: Option>, } - -/// Direct conversion prevents loss of the empty `state_key` that ruma requires. -impl From for PduBuilder { - fn from(event: AnyInitialStateEvent) -> Self { - Self { - event_type: EventType::from(event.event_type()), - content: to_raw_value(&event.content()) - .expect("AnyStateEventContent came from JSON and can thus turn back into JSON."), - unsigned: None, - state_key: Some(event.state_key().to_owned()), - redacts: None, - } - } -}