stages: - build - build docker image - test - upload artifacts variables: GIT_SUBMODULE_STRATEGY: recursive FF_USE_FASTZIP: 1 CACHE_COMPRESSION_LEVEL: fastest # Docker in Docker DOCKER_HOST: tcp://docker:2375/ DOCKER_TLS_CERTDIR: "" DOCKER_DRIVER: overlay2 # --------------------------------------------------------------------- # # Cargo: Compiling for different architectures # # --------------------------------------------------------------------- # .build-cargo-shared-settings: stage: "build" needs: [] rules: - if: '$CI_COMMIT_BRANCH == "master"' - if: '$CI_COMMIT_BRANCH == "next"' - if: "$CI_COMMIT_TAG" - if: '($CI_MERGE_REQUEST_APPROVED == "true") || $BUILD_EVERYTHING' # Once MR is approved, test all builds. Or if BUILD_EVERYTHING is set. interruptible: true image: "registry.gitlab.com/jfowl/conduit-containers/rust-with-tools@sha256:69ab327974aef4cc0daf4273579253bf7ae5e379a6c52729b83137e4caa9d093" tags: ["docker"] services: ["docker:dind"] variables: SHARED_PATH: $CI_PROJECT_DIR/shared CARGO_PROFILE_RELEASE_LTO: "true" CARGO_PROFILE_RELEASE_CODEGEN_UNITS: "1" CARGO_INCREMENTAL: "false" # https://matklad.github.io/2021/09/04/fast-rust-builds.html#ci-workflow before_script: - 'echo "Building for target $TARGET"' - "rustup show && rustc --version && cargo --version" # Print version info for debugging # fix cargo and rustup mounts from this container (https://gitlab.com/gitlab-org/gitlab-foss/-/issues/41227) - "mkdir -p $SHARED_PATH/cargo" - "cp -r $CARGO_HOME/bin $SHARED_PATH/cargo" - "cp -r $RUSTUP_HOME $SHARED_PATH" - "export CARGO_HOME=$SHARED_PATH/cargo RUSTUP_HOME=$SHARED_PATH/rustup" # If provided, bring in caching through sccache, which uses an external S3 endpoint to store compilation results. - if [ -n "${SCCACHE_ENDPOINT}" ]; then export RUSTC_WRAPPER=/sccache; fi script: # cross-compile conduit for target - 'time cross build --target="$TARGET" --locked --release' - 'mv "target/$TARGET/release/conduit" "conduit-$TARGET"' # print information about linking for debugging - "file conduit-$TARGET" # print file information - 'readelf --dynamic conduit-$TARGET | sed -e "/NEEDED/q1"' # ensure statically linked cache: # https://doc.rust-lang.org/cargo/guide/cargo-home.html#caching-the-cargo-home-in-ci key: "cargo-cache-$TARGET" paths: - $SHARED_PATH/cargo/registry/index - $SHARED_PATH/cargo/registry/cache - $SHARED_PATH/cargo/git/db artifacts: expire_in: never build:release:cargo:x86_64-unknown-linux-musl-with-debug: extends: .build-cargo-shared-settings variables: CARGO_PROFILE_RELEASE_DEBUG: 2 # Enable debug info for flamegraph profiling TARGET: "x86_64-unknown-linux-musl" after_script: - "mv ./conduit-x86_64-unknown-linux-musl ./conduit-x86_64-unknown-linux-musl-with-debug" artifacts: name: "conduit-x86_64-unknown-linux-musl-with-debug" paths: - "conduit-x86_64-unknown-linux-musl-with-debug" expose_as: "Conduit for x86_64-unknown-linux-musl-with-debug" build:release:cargo:x86_64-unknown-linux-musl: extends: .build-cargo-shared-settings variables: TARGET: "x86_64-unknown-linux-musl" artifacts: name: "conduit-x86_64-unknown-linux-musl" paths: - "conduit-x86_64-unknown-linux-musl" expose_as: "Conduit for x86_64-unknown-linux-musl" build:release:cargo:arm-unknown-linux-musleabihf: extends: .build-cargo-shared-settings variables: TARGET: "arm-unknown-linux-musleabihf" artifacts: name: "conduit-arm-unknown-linux-musleabihf" paths: - "conduit-arm-unknown-linux-musleabihf" expose_as: "Conduit for arm-unknown-linux-musleabihf" build:release:cargo:armv7-unknown-linux-musleabihf: extends: .build-cargo-shared-settings variables: TARGET: "armv7-unknown-linux-musleabihf" artifacts: name: "conduit-armv7-unknown-linux-musleabihf" paths: - "conduit-armv7-unknown-linux-musleabihf" expose_as: "Conduit for armv7-unknown-linux-musleabihf" build:release:cargo:aarch64-unknown-linux-musl: extends: .build-cargo-shared-settings variables: TARGET: "aarch64-unknown-linux-musl" artifacts: name: "conduit-aarch64-unknown-linux-musl" paths: - "conduit-aarch64-unknown-linux-musl" expose_as: "Conduit for aarch64-unknown-linux-musl" .cargo-debug-shared-settings: extends: ".build-cargo-shared-settings" rules: - when: "always" cache: key: "build_cache--$TARGET--$CI_COMMIT_BRANCH--debug" script: # cross-compile conduit for target - 'time time cross build --target="$TARGET" --locked' - 'mv "target/$TARGET/debug/conduit" "conduit-debug-$TARGET"' # print information about linking for debugging - "file conduit-debug-$TARGET" # print file information - 'readelf --dynamic conduit-debug-$TARGET | sed -e "/NEEDED/q1"' # ensure statically linked artifacts: expire_in: 4 weeks build:debug:cargo:x86_64-unknown-linux-musl: extends: ".cargo-debug-shared-settings" variables: TARGET: "x86_64-unknown-linux-musl" artifacts: name: "conduit-debug-x86_64-unknown-linux-musl" paths: - "conduit-debug-x86_64-unknown-linux-musl" expose_as: "Conduit DEBUG for x86_64-unknown-linux-musl" # --------------------------------------------------------------------- # # Create and publish docker image # # --------------------------------------------------------------------- # .docker-shared-settings: stage: "build docker image" image: jdrouet/docker-with-buildx:stable tags: ["docker"] services: - docker:dind needs: - "build:release:cargo:x86_64-unknown-linux-musl" - "build:release:cargo:arm-unknown-linux-musleabihf" - "build:release:cargo:armv7-unknown-linux-musleabihf" - "build:release:cargo:aarch64-unknown-linux-musl" variables: PLATFORMS: "linux/arm/v6,linux/arm/v7,linux/arm64,linux/amd64" DOCKER_FILE: "docker/ci-binaries-packaging.Dockerfile" cache: paths: - docker_cache key: "$CI_JOB_NAME" before_script: - docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY # Only log in to Dockerhub if the credentials are given: - if [ -n "${DOCKER_HUB}" ]; then docker login -u "$DOCKER_HUB_USER" -p "$DOCKER_HUB_PASSWORD" "$DOCKER_HUB"; fi script: # Prepare buildx to build multiarch stuff: - docker context create 'ci-context' - docker buildx create --name 'multiarch-builder' --use 'ci-context' # Copy binaries to their docker arch path - mkdir -p linux/ && mv ./conduit-x86_64-unknown-linux-musl linux/amd64 - mkdir -p linux/arm/ && mv ./conduit-arm-unknown-linux-musleabihf linux/arm/v6 - mkdir -p linux/arm/ && mv ./conduit-armv7-unknown-linux-musleabihf linux/arm/v7 - mv ./conduit-aarch64-unknown-linux-musl linux/arm64 - 'export CREATED=$(date -u +''%Y-%m-%dT%H:%M:%SZ'') && echo "Docker image creation date: $CREATED"' # Build and push image: - > docker buildx build --pull --push --cache-from=type=local,src=$CI_PROJECT_DIR/docker_cache --cache-to=type=local,dest=$CI_PROJECT_DIR/docker_cache --build-arg CREATED=$CREATED --build-arg VERSION=$(grep -m1 -o '[0-9].[0-9].[0-9]' Cargo.toml) --build-arg "GIT_REF=$CI_COMMIT_SHORT_SHA" --platform "$PLATFORMS" --tag "$TAG" --tag "$TAG-alpine" --tag "$TAG-commit-$CI_COMMIT_SHORT_SHA" --file "$DOCKER_FILE" . docker:next:gitlab: extends: .docker-shared-settings rules: - if: '$CI_COMMIT_BRANCH == "next"' variables: TAG: "$CI_REGISTRY_IMAGE/matrix-conduit:next" docker:next:dockerhub: extends: .docker-shared-settings rules: - if: '$CI_COMMIT_BRANCH == "next" && $DOCKER_HUB' variables: TAG: "$DOCKER_HUB_IMAGE/matrixconduit/matrix-conduit:next" docker:master:gitlab: extends: .docker-shared-settings rules: - if: '$CI_COMMIT_BRANCH == "master"' variables: TAG: "$CI_REGISTRY_IMAGE/matrix-conduit:latest" docker:master:dockerhub: extends: .docker-shared-settings rules: - if: '$CI_COMMIT_BRANCH == "master" && $DOCKER_HUB' variables: TAG: "$DOCKER_HUB_IMAGE/matrixconduit/matrix-conduit:latest" docker:tags:gitlab: extends: .docker-shared-settings rules: - if: "$CI_COMMIT_TAG" variables: TAG: "$CI_REGISTRY_IMAGE/matrix-conduit:$CI_COMMIT_TAG" docker:tags:dockerhub: extends: .docker-shared-settings rules: - if: "$CI_COMMIT_TAG && $DOCKER_HUB" variables: TAG: "$DOCKER_HUB_IMAGE/matrixconduit/matrix-conduit:$CI_COMMIT_TAG" # --------------------------------------------------------------------- # # Run tests # # --------------------------------------------------------------------- # .test-shared-settings: stage: "test" needs: [] image: "registry.gitlab.com/jfowl/conduit-containers/rust-with-tools:latest" tags: ["docker"] variables: CARGO_INCREMENTAL: "false" # https://matklad.github.io/2021/09/04/fast-rust-builds.html#ci-workflow interruptible: true test:cargo: extends: .test-shared-settings before_script: - rustup component add clippy # If provided, bring in caching through sccache, which uses an external S3 endpoint to store compilation results: - if [ -n "${SCCACHE_ENDPOINT}" ]; then export RUSTC_WRAPPER=/usr/local/cargo/bin/sccache; fi script: - rustc --version && cargo --version # Print version info for debugging - "cargo test --color always --workspace --verbose --locked --no-fail-fast -- -Z unstable-options --format json | gitlab-report -p test > $CI_PROJECT_DIR/report.xml" - "cargo clippy --color always --verbose --message-format=json | gitlab-report -p clippy > $CI_PROJECT_DIR/gl-code-quality-report.json" artifacts: when: always reports: junit: report.xml codequality: gl-code-quality-report.json test:format: extends: .test-shared-settings before_script: - rustup component add rustfmt script: - cargo fmt --all -- --check test:audit: extends: .test-shared-settings allow_failure: true script: - cargo audit --color always || true - cargo audit --stale --json | gitlab-report -p audit > gl-sast-report.json artifacts: when: always reports: sast: gl-sast-report.json test:sytest: stage: "test" allow_failure: true needs: - "build:debug:cargo:x86_64-unknown-linux-musl" image: name: "valkum/sytest-conduit:latest" entrypoint: [""] tags: ["docker"] variables: PLUGINS: "https://github.com/valkum/sytest_conduit/archive/master.tar.gz" interruptible: true before_script: - "mkdir -p /app" - "cp ./conduit-debug-x86_64-unknown-linux-musl /app/conduit" - "chmod +x /app/conduit" - "rm -rf /src && ln -s $CI_PROJECT_DIR/ /src" - "mkdir -p /work/server-0/database/ && mkdir -p /work/server-1/database/ && mkdir -p /work/server-2/database/" - "cd /" script: - "SYTEST_EXIT_CODE=0" - "/bootstrap.sh conduit || SYTEST_EXIT_CODE=1" - 'perl /sytest/tap-to-junit-xml.pl --puretap --input /logs/results.tap --output $CI_PROJECT_DIR/sytest.xml "Sytest" && cp /logs/results.tap $CI_PROJECT_DIR/results.tap' - "exit $SYTEST_EXIT_CODE" artifacts: when: always paths: - "$CI_PROJECT_DIR/sytest.xml" - "$CI_PROJECT_DIR/results.tap" reports: junit: "$CI_PROJECT_DIR/sytest.xml" test:dockerlint: stage: "test" needs: [] image: "ghcr.io/hadolint/hadolint@sha256:6c4b7c23f96339489dd35f21a711996d7ce63047467a9a562287748a03ad5242" # 2.8.0-alpine interruptible: true script: - hadolint --version # First pass: Print for CI log: - > hadolint --no-fail --verbose ./Dockerfile ./docker/ci-binaries-packaging.Dockerfile # Then output the results into a json for GitLab to pretty-print this in the MR: - > hadolint --format gitlab_codeclimate --failure-threshold error ./Dockerfile ./docker/ci-binaries-packaging.Dockerfile > dockerlint.json artifacts: when: always reports: codequality: dockerlint.json paths: - dockerlint.json rules: - if: '$CI_COMMIT_REF_NAME != "master"' changes: - docker/*Dockerfile - Dockerfile - .gitlab-ci.yml - if: '$CI_COMMIT_REF_NAME == "master"' - if: '$CI_COMMIT_REF_NAME == "next"' # --------------------------------------------------------------------- # # Store binaries as package so they have download urls # # --------------------------------------------------------------------- # publish:package: stage: "upload artifacts" needs: - "build:release:cargo:x86_64-unknown-linux-musl" - "build:release:cargo:arm-unknown-linux-musleabihf" - "build:release:cargo:armv7-unknown-linux-musleabihf" - "build:release:cargo:aarch64-unknown-linux-musl" # - "build:cargo-deb:x86_64-unknown-linux-gnu" rules: - if: '$CI_COMMIT_BRANCH == "master"' - if: '$CI_COMMIT_BRANCH == "next"' - if: "$CI_COMMIT_TAG" image: curlimages/curl:latest tags: ["docker"] variables: GIT_STRATEGY: "none" # Don't need a clean copy of the code, we just operate on artifacts script: - 'BASE_URL="${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/packages/generic/conduit-${CI_COMMIT_REF_SLUG}/build-${CI_PIPELINE_ID}"' - 'curl --header "JOB-TOKEN: $CI_JOB_TOKEN" --upload-file conduit-x86_64-unknown-linux-musl "${BASE_URL}/conduit-x86_64-unknown-linux-musl"' - 'curl --header "JOB-TOKEN: $CI_JOB_TOKEN" --upload-file conduit-arm-unknown-linux-musleabihf "${BASE_URL}/conduit-arm-unknown-linux-musleabihf"' - 'curl --header "JOB-TOKEN: $CI_JOB_TOKEN" --upload-file conduit-armv7-unknown-linux-musleabihf "${BASE_URL}/conduit-armv7-unknown-linux-musleabihf"' - 'curl --header "JOB-TOKEN: $CI_JOB_TOKEN" --upload-file conduit-aarch64-unknown-linux-musl "${BASE_URL}/conduit-aarch64-unknown-linux-musl"' # Avoid duplicate pipelines # See: https://docs.gitlab.com/ee/ci/yaml/workflow.html#switch-between-branch-pipelines-and-merge-request-pipelines workflow: rules: - if: '$CI_PIPELINE_SOURCE == "merge_request_event"' - if: "$CI_COMMIT_BRANCH && $CI_OPEN_MERGE_REQUESTS" when: never - if: "$CI_COMMIT_BRANCH" - if: "$CI_COMMIT_TAG"