mirror of https://gitlab.com/famedly/conduit
Merge branch 'lints' into 'next'
fix CI See merge request famedly/conduit!564merge-requests/594/head
Timo Kösters
5 months ago
commit
5cf9f3df48
@ -1,244 +1,66 @@
|
||||
stages:
|
||||
- build
|
||||
- build docker image
|
||||
- test
|
||||
- upload artifacts
|
||||
- ci
|
||||
- artifacts
|
||||
|
||||
variables:
|
||||
# Make GitLab CI go fast:
|
||||
GIT_SUBMODULE_STRATEGY: recursive
|
||||
FF_USE_FASTZIP: 1
|
||||
CACHE_COMPRESSION_LEVEL: fastest
|
||||
# Makes some things print in color
|
||||
TERM: ansi
|
||||
|
||||
# --------------------------------------------------------------------- #
|
||||
# Create and publish docker image #
|
||||
# --------------------------------------------------------------------- #
|
||||
before_script:
|
||||
# Enable nix-command and flakes
|
||||
- if command -v nix > /dev/null; then echo "experimental-features = nix-command flakes" >> /etc/nix/nix.conf; fi
|
||||
|
||||
.docker-shared-settings:
|
||||
stage: "build docker image"
|
||||
needs: []
|
||||
tags: [ "docker" ]
|
||||
variables:
|
||||
# Docker in Docker:
|
||||
DOCKER_BUILDKIT: 1
|
||||
image:
|
||||
name: docker.io/docker
|
||||
services:
|
||||
- name: docker.io/docker:dind
|
||||
alias: docker
|
||||
script:
|
||||
- apk add openssh-client
|
||||
- eval $(ssh-agent -s)
|
||||
- mkdir -p ~/.ssh && chmod 700 ~/.ssh
|
||||
- printf "Host *\n\tStrictHostKeyChecking no\n\n" >> ~/.ssh/config
|
||||
- sh .gitlab/setup-buildx-remote-builders.sh
|
||||
# Authorize against this project's own image registry:
|
||||
- docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY
|
||||
# Build multiplatform image and push to temporary tag:
|
||||
- >
|
||||
docker buildx build
|
||||
--platform "linux/arm/v7,linux/arm64,linux/amd64"
|
||||
--pull
|
||||
--tag "$CI_REGISTRY_IMAGE/temporary-ci-images:$CI_JOB_ID"
|
||||
--push
|
||||
--provenance=false
|
||||
--file "Dockerfile" .
|
||||
# Build multiplatform image to deb stage and extract their .deb files:
|
||||
- >
|
||||
docker buildx build
|
||||
--platform "linux/arm/v7,linux/arm64,linux/amd64"
|
||||
--target "packager-result"
|
||||
--output="type=local,dest=/tmp/build-output"
|
||||
--provenance=false
|
||||
--file "Dockerfile" .
|
||||
# Build multiplatform image to binary stage and extract their binaries:
|
||||
- >
|
||||
docker buildx build
|
||||
--platform "linux/arm/v7,linux/arm64,linux/amd64"
|
||||
--target "builder-result"
|
||||
--output="type=local,dest=/tmp/build-output"
|
||||
--provenance=false
|
||||
--file "Dockerfile" .
|
||||
# Copy to GitLab container registry:
|
||||
- >
|
||||
docker buildx imagetools create
|
||||
--tag "$CI_REGISTRY_IMAGE/$TAG"
|
||||
--tag "$CI_REGISTRY_IMAGE/$TAG-bullseye"
|
||||
--tag "$CI_REGISTRY_IMAGE/$TAG-commit-$CI_COMMIT_SHORT_SHA"
|
||||
"$CI_REGISTRY_IMAGE/temporary-ci-images:$CI_JOB_ID"
|
||||
# if DockerHub credentials exist, also copy to dockerhub:
|
||||
- if [ -n "${DOCKER_HUB}" ]; then docker login -u "$DOCKER_HUB_USER" -p "$DOCKER_HUB_PASSWORD" "$DOCKER_HUB"; fi
|
||||
- >
|
||||
if [ -n "${DOCKER_HUB}" ]; then
|
||||
docker buildx imagetools create
|
||||
--tag "$DOCKER_HUB_IMAGE/$TAG"
|
||||
--tag "$DOCKER_HUB_IMAGE/$TAG-bullseye"
|
||||
--tag "$DOCKER_HUB_IMAGE/$TAG-commit-$CI_COMMIT_SHORT_SHA"
|
||||
"$CI_REGISTRY_IMAGE/temporary-ci-images:$CI_JOB_ID"
|
||||
; fi
|
||||
- mv /tmp/build-output ./
|
||||
artifacts:
|
||||
paths:
|
||||
- "./build-output/"
|
||||
|
||||
docker:next:
|
||||
extends: .docker-shared-settings
|
||||
rules:
|
||||
- if: '$BUILD_SERVER_SSH_PRIVATE_KEY && $CI_COMMIT_BRANCH == "next"'
|
||||
variables:
|
||||
TAG: "matrix-conduit:next"
|
||||
|
||||
docker:master:
|
||||
extends: .docker-shared-settings
|
||||
rules:
|
||||
- if: '$BUILD_SERVER_SSH_PRIVATE_KEY && $CI_COMMIT_BRANCH == "master"'
|
||||
variables:
|
||||
TAG: "matrix-conduit:latest"
|
||||
# Add nix-community binary cache
|
||||
- if command -v nix > /dev/null; then echo "extra-substituters = https://nix-community.cachix.org" >> /etc/nix/nix.conf; fi
|
||||
- if command -v nix > /dev/null; then echo "extra-trusted-public-keys = nix-community.cachix.org-1:mB9FSh9qf2dCimDSUo8Zy7bkq5CX+/rkCWyvRCYg3Fs=" >> /etc/nix/nix.conf; fi
|
||||
|
||||
docker:tags:
|
||||
extends: .docker-shared-settings
|
||||
rules:
|
||||
- if: "$BUILD_SERVER_SSH_PRIVATE_KEY && $CI_COMMIT_TAG"
|
||||
variables:
|
||||
TAG: "matrix-conduit:$CI_COMMIT_TAG"
|
||||
# Install direnv and nix-direnv
|
||||
- if command -v nix > /dev/null; then nix-env -iA nixpkgs.direnv nixpkgs.nix-direnv; fi
|
||||
|
||||
# Allow .envrc
|
||||
- if command -v nix > /dev/null; then direnv allow; fi
|
||||
|
||||
docker build debugging:
|
||||
extends: .docker-shared-settings
|
||||
rules:
|
||||
- if: "$CI_MERGE_REQUEST_TITLE =~ /.*[Dd]ocker.*/"
|
||||
variables:
|
||||
TAG: "matrix-conduit-docker-tests:latest"
|
||||
# Set CARGO_HOME to a cacheable path
|
||||
- export CARGO_HOME="$(git rev-parse --show-toplevel)/.gitlab-ci.d/cargo"
|
||||
|
||||
# --------------------------------------------------------------------- #
|
||||
# Run tests #
|
||||
# --------------------------------------------------------------------- #
|
||||
|
||||
cargo check:
|
||||
stage: test
|
||||
image: docker.io/rust:1.70.0-bullseye
|
||||
needs: []
|
||||
interruptible: true
|
||||
before_script:
|
||||
- "rustup show && rustc --version && cargo --version" # Print version info for debugging
|
||||
- apt-get update && apt-get -y --no-install-recommends install libclang-dev # dependency for rocksdb
|
||||
ci:
|
||||
stage: ci
|
||||
image: nixos/nix:2.19.2
|
||||
script:
|
||||
- cargo check
|
||||
|
||||
|
||||
.test-shared-settings:
|
||||
stage: "test"
|
||||
needs: []
|
||||
image: "registry.gitlab.com/jfowl/conduit-containers/rust-with-tools:latest"
|
||||
tags: ["docker"]
|
||||
variables:
|
||||
CARGO_INCREMENTAL: "false" # https://matklad.github.io/2021/09/04/fast-rust-builds.html#ci-workflow
|
||||
interruptible: true
|
||||
- direnv exec . engage
|
||||
cache:
|
||||
key: nix
|
||||
paths:
|
||||
- target
|
||||
- .gitlab-ci.d
|
||||
|
||||
test:cargo:
|
||||
extends: .test-shared-settings
|
||||
before_script:
|
||||
- apt-get update && apt-get -y --no-install-recommends install libclang-dev # dependency for rocksdb
|
||||
docker:
|
||||
stage: artifacts
|
||||
image: nixos/nix:2.19.2
|
||||
script:
|
||||
- rustc --version && cargo --version # Print version info for debugging
|
||||
- "cargo test --color always --workspace --verbose --locked --no-fail-fast"
|
||||
- nix build .#oci-image
|
||||
|
||||
test:clippy:
|
||||
extends: .test-shared-settings
|
||||
allow_failure: true
|
||||
before_script:
|
||||
- rustup component add clippy
|
||||
- apt-get update && apt-get -y --no-install-recommends install libclang-dev # dependency for rocksdb
|
||||
script:
|
||||
- rustc --version && cargo --version # Print version info for debugging
|
||||
- "cargo clippy --color always --verbose --message-format=json | gitlab-report -p clippy > $CI_PROJECT_DIR/gl-code-quality-report.json"
|
||||
# Make the output less difficult to find
|
||||
- cp result docker-image.tar.gz
|
||||
artifacts:
|
||||
when: always
|
||||
reports:
|
||||
codequality: gl-code-quality-report.json
|
||||
paths:
|
||||
- docker-image.tar.gz
|
||||
|
||||
test:format:
|
||||
extends: .test-shared-settings
|
||||
before_script:
|
||||
- rustup component add rustfmt
|
||||
debian:
|
||||
stage: artifacts
|
||||
image: rust:1.70.0
|
||||
script:
|
||||
- cargo fmt --all -- --check
|
||||
- apt-get update && apt-get install -y --no-install-recommends libclang-dev
|
||||
- cargo install cargo-deb
|
||||
- cargo deb
|
||||
|
||||
test:audit:
|
||||
extends: .test-shared-settings
|
||||
allow_failure: true
|
||||
script:
|
||||
- cargo audit --color always || true
|
||||
- cargo audit --stale --json | gitlab-report -p audit > gl-sast-report.json
|
||||
artifacts:
|
||||
when: always
|
||||
reports:
|
||||
sast: gl-sast-report.json
|
||||
|
||||
test:dockerlint:
|
||||
stage: "test"
|
||||
needs: []
|
||||
image: "ghcr.io/hadolint/hadolint@sha256:6c4b7c23f96339489dd35f21a711996d7ce63047467a9a562287748a03ad5242" # 2.8.0-alpine
|
||||
interruptible: true
|
||||
script:
|
||||
- hadolint --version
|
||||
# First pass: Print for CI log:
|
||||
- >
|
||||
hadolint
|
||||
--no-fail --verbose
|
||||
./Dockerfile
|
||||
# Then output the results into a json for GitLab to pretty-print this in the MR:
|
||||
- >
|
||||
hadolint
|
||||
--format gitlab_codeclimate
|
||||
--failure-threshold error
|
||||
./Dockerfile > dockerlint.json
|
||||
# Make the output less difficult to find
|
||||
- mv target/debian/*.deb .
|
||||
artifacts:
|
||||
when: always
|
||||
reports:
|
||||
codequality: dockerlint.json
|
||||
paths:
|
||||
- dockerlint.json
|
||||
rules:
|
||||
- if: '$CI_COMMIT_REF_NAME != "master"'
|
||||
changes:
|
||||
- docker/*Dockerfile
|
||||
- Dockerfile
|
||||
- .gitlab-ci.yml
|
||||
- if: '$CI_COMMIT_REF_NAME == "master"'
|
||||
- if: '$CI_COMMIT_REF_NAME == "next"'
|
||||
|
||||
# --------------------------------------------------------------------- #
|
||||
# Store binaries as package so they have download urls #
|
||||
# --------------------------------------------------------------------- #
|
||||
|
||||
# DISABLED FOR NOW, NEEDS TO BE FIXED AT A LATER TIME:
|
||||
|
||||
#publish:package:
|
||||
# stage: "upload artifacts"
|
||||
# needs:
|
||||
# - "docker:tags"
|
||||
# rules:
|
||||
# - if: "$CI_COMMIT_TAG"
|
||||
# image: curlimages/curl:latest
|
||||
# tags: ["docker"]
|
||||
# variables:
|
||||
# GIT_STRATEGY: "none" # Don't need a clean copy of the code, we just operate on artifacts
|
||||
# script:
|
||||
# - 'BASE_URL="${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/packages/generic/conduit-${CI_COMMIT_REF_SLUG}/build-${CI_PIPELINE_ID}"'
|
||||
# - 'curl --header "JOB-TOKEN: $CI_JOB_TOKEN" --upload-file build-output/linux_amd64/conduit "${BASE_URL}/conduit-x86_64-unknown-linux-gnu"'
|
||||
# - 'curl --header "JOB-TOKEN: $CI_JOB_TOKEN" --upload-file build-output/linux_arm_v7/conduit "${BASE_URL}/conduit-armv7-unknown-linux-gnu"'
|
||||
# - 'curl --header "JOB-TOKEN: $CI_JOB_TOKEN" --upload-file build-output/linux_arm64/conduit "${BASE_URL}/conduit-aarch64-unknown-linux-gnu"'
|
||||
# - 'curl --header "JOB-TOKEN: $CI_JOB_TOKEN" --upload-file build-output/linux_amd64/conduit.deb "${BASE_URL}/conduit-x86_64-unknown-linux-gnu.deb"'
|
||||
# - 'curl --header "JOB-TOKEN: $CI_JOB_TOKEN" --upload-file build-output/linux_arm_v7/conduit.deb "${BASE_URL}/conduit-armv7-unknown-linux-gnu.deb"'
|
||||
# - 'curl --header "JOB-TOKEN: $CI_JOB_TOKEN" --upload-file build-output/linux_arm64/conduit.deb "${BASE_URL}/conduit-aarch64-unknown-linux-gnu.deb"'
|
||||
|
||||
# Avoid duplicate pipelines
|
||||
# See: https://docs.gitlab.com/ee/ci/yaml/workflow.html#switch-between-branch-pipelines-and-merge-request-pipelines
|
||||
workflow:
|
||||
rules:
|
||||
- if: '$CI_PIPELINE_SOURCE == "merge_request_event"'
|
||||
- if: "$CI_COMMIT_BRANCH && $CI_OPEN_MERGE_REQUESTS"
|
||||
when: never
|
||||
- if: "$CI_COMMIT_BRANCH"
|
||||
- if: "$CI_COMMIT_TAG"
|
||||
- "*.deb"
|
||||
cache:
|
||||
key: debian
|
||||
paths:
|
||||
- target
|
||||
- .gitlab-ci.d
|
||||
|
||||
@ -1,132 +0,0 @@
|
||||
# syntax=docker/dockerfile:1
|
||||
FROM docker.io/rust:1.70-bullseye AS base
|
||||
|
||||
FROM base AS builder
|
||||
WORKDIR /usr/src/conduit
|
||||
|
||||
# Install required packages to build Conduit and it's dependencies
|
||||
RUN apt-get update && \
|
||||
apt-get -y --no-install-recommends install libclang-dev=1:11.0-51+nmu5
|
||||
|
||||
# == Build dependencies without our own code separately for caching ==
|
||||
#
|
||||
# Need a fake main.rs since Cargo refuses to build anything otherwise.
|
||||
#
|
||||
# See https://github.com/rust-lang/cargo/issues/2644 for a Cargo feature
|
||||
# request that would allow just dependencies to be compiled, presumably
|
||||
# regardless of whether source files are available.
|
||||
RUN mkdir src && touch src/lib.rs && echo 'fn main() {}' > src/main.rs
|
||||
COPY Cargo.toml Cargo.lock ./
|
||||
RUN cargo build --release && rm -r src
|
||||
|
||||
# Copy over actual Conduit sources
|
||||
COPY src src
|
||||
|
||||
# main.rs and lib.rs need their timestamp updated for this to work correctly since
|
||||
# otherwise the build with the fake main.rs from above is newer than the
|
||||
# source files (COPY preserves timestamps).
|
||||
#
|
||||
# Builds conduit and places the binary at /usr/src/conduit/target/release/conduit
|
||||
RUN touch src/main.rs && touch src/lib.rs && cargo build --release
|
||||
|
||||
|
||||
# ONLY USEFUL FOR CI: target stage to extract build artifacts
|
||||
FROM scratch AS builder-result
|
||||
COPY --from=builder /usr/src/conduit/target/release/conduit /conduit
|
||||
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------------------------------------------
|
||||
# Build cargo-deb, a tool to package up rust binaries into .deb packages for Debian/Ubuntu based systems:
|
||||
# ---------------------------------------------------------------------------------------------------------------
|
||||
FROM base AS build-cargo-deb
|
||||
|
||||
RUN apt-get update && \
|
||||
apt-get install -y --no-install-recommends \
|
||||
dpkg \
|
||||
dpkg-dev \
|
||||
liblzma-dev
|
||||
|
||||
RUN cargo install cargo-deb
|
||||
# => binary is in /usr/local/cargo/bin/cargo-deb
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------------------------------------------
|
||||
# Package conduit build-result into a .deb package:
|
||||
# ---------------------------------------------------------------------------------------------------------------
|
||||
FROM builder AS packager
|
||||
WORKDIR /usr/src/conduit
|
||||
|
||||
COPY ./LICENSE ./LICENSE
|
||||
COPY ./README.md ./README.md
|
||||
COPY debian ./debian
|
||||
COPY --from=build-cargo-deb /usr/local/cargo/bin/cargo-deb /usr/local/cargo/bin/cargo-deb
|
||||
|
||||
# --no-build makes cargo-deb reuse already compiled project
|
||||
RUN cargo deb --no-build
|
||||
# => Package is in /usr/src/conduit/target/debian/<project_name>_<version>_<arch>.deb
|
||||
|
||||
|
||||
# ONLY USEFUL FOR CI: target stage to extract build artifacts
|
||||
FROM scratch AS packager-result
|
||||
COPY --from=packager /usr/src/conduit/target/debian/*.deb /conduit.deb
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------------------------------------------
|
||||
# Stuff below this line actually ends up in the resulting docker image
|
||||
# ---------------------------------------------------------------------------------------------------------------
|
||||
FROM docker.io/debian:bullseye-slim AS runner
|
||||
|
||||
# Standard port on which Conduit launches.
|
||||
# You still need to map the port when using the docker command or docker-compose.
|
||||
EXPOSE 6167
|
||||
|
||||
ARG DEFAULT_DB_PATH=/var/lib/matrix-conduit
|
||||
|
||||
ENV CONDUIT_PORT=6167 \
|
||||
CONDUIT_ADDRESS="0.0.0.0" \
|
||||
CONDUIT_DATABASE_PATH=${DEFAULT_DB_PATH} \
|
||||
CONDUIT_CONFIG=''
|
||||
# └─> Set no config file to do all configuration with env vars
|
||||
|
||||
# Conduit needs:
|
||||
# dpkg: to install conduit.deb
|
||||
# ca-certificates: for https
|
||||
# iproute2 & wget: for the healthcheck script
|
||||
RUN apt-get update && apt-get -y --no-install-recommends install \
|
||||
dpkg \
|
||||
ca-certificates \
|
||||
iproute2 \
|
||||
wget \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
# Test if Conduit is still alive, uses the same endpoint as Element
|
||||
COPY ./docker/healthcheck.sh /srv/conduit/healthcheck.sh
|
||||
HEALTHCHECK --start-period=5s --interval=5s CMD ./healthcheck.sh
|
||||
|
||||
# Install conduit.deb:
|
||||
COPY --from=packager /usr/src/conduit/target/debian/*.deb /srv/conduit/
|
||||
RUN dpkg -i /srv/conduit/*.deb
|
||||
|
||||
# Improve security: Don't run stuff as root, that does not need to run as root
|
||||
# Most distros also use 1000:1000 for the first real user, so this should resolve volume mounting problems.
|
||||
ARG USER_ID=1000
|
||||
ARG GROUP_ID=1000
|
||||
RUN set -x ; \
|
||||
groupadd -r -g ${GROUP_ID} conduit ; \
|
||||
useradd -l -r -M -d /srv/conduit -o -u ${USER_ID} -g conduit conduit && exit 0 ; exit 1
|
||||
|
||||
# Create database directory, change ownership of Conduit files to conduit user and group and make the healthcheck executable:
|
||||
RUN chown -cR conduit:conduit /srv/conduit && \
|
||||
chmod +x /srv/conduit/healthcheck.sh && \
|
||||
mkdir -p ${DEFAULT_DB_PATH} && \
|
||||
chown -cR conduit:conduit ${DEFAULT_DB_PATH}
|
||||
|
||||
# Change user to conduit, no root permissions afterwards:
|
||||
USER conduit
|
||||
# Set container home directory
|
||||
WORKDIR /srv/conduit
|
||||
|
||||
# Run Conduit and print backtraces on panics
|
||||
ENV RUST_BACKTRACE=1
|
||||
ENTRYPOINT [ "/usr/sbin/matrix-conduit" ]
|
||||
@ -0,0 +1,64 @@
|
||||
interpreter = ["bash", "-euo", "pipefail", "-c"]
|
||||
|
||||
[[task]]
|
||||
name = "engage"
|
||||
group = "versions"
|
||||
script = "engage --version"
|
||||
|
||||
[[task]]
|
||||
name = "rustc"
|
||||
group = "versions"
|
||||
script = "rustc --version"
|
||||
|
||||
[[task]]
|
||||
name = "cargo"
|
||||
group = "versions"
|
||||
script = "cargo --version"
|
||||
|
||||
[[task]]
|
||||
name = "cargo-fmt"
|
||||
group = "versions"
|
||||
script = "cargo fmt --version"
|
||||
|
||||
[[task]]
|
||||
name = "rustdoc"
|
||||
group = "versions"
|
||||
script = "rustdoc --version"
|
||||
|
||||
[[task]]
|
||||
name = "cargo-clippy"
|
||||
group = "versions"
|
||||
script = "cargo clippy -- --version"
|
||||
|
||||
[[task]]
|
||||
name = "cargo-fmt"
|
||||
group = "lints"
|
||||
script = "cargo fmt --check -- --color=always"
|
||||
|
||||
[[task]]
|
||||
name = "cargo-doc"
|
||||
group = "lints"
|
||||
script = """
|
||||
RUSTDOCFLAGS="-D warnings" cargo doc \
|
||||
--workspace \
|
||||
--no-deps \
|
||||
--document-private-items \
|
||||
--color always
|
||||
"""
|
||||
|
||||
[[task]]
|
||||
name = "cargo-clippy"
|
||||
group = "lints"
|
||||
script = "cargo clippy --workspace --all-targets --color=always -- -D warnings"
|
||||
|
||||
[[task]]
|
||||
name = "cargo"
|
||||
group = "tests"
|
||||
script = """
|
||||
cargo test \
|
||||
--workspace \
|
||||
--all-targets \
|
||||
--color=always \
|
||||
-- \
|
||||
--color=always
|
||||
"""
|
||||
Loading…
Reference in New Issue