From 3336d3f8125539f20b5dd4b39eefad6ff7790c0c Mon Sep 17 00:00:00 2001 From: Charles Hall Date: Tue, 30 Apr 2024 16:00:19 -0700 Subject: [PATCH] factor out nix code into new files via `makeScope` This makes the Nix code a lot easier to reason about. --- flake.nix | 341 +++++---------------- nix/pkgs/book/default.nix | 34 ++ nix/pkgs/default/cross-compilation-env.nix | 100 ++++++ nix/pkgs/default/default.nix | 64 ++++ nix/pkgs/oci-image/default.nix | 25 ++ nix/shell.nix | 55 ++++ 6 files changed, 354 insertions(+), 265 deletions(-) create mode 100644 nix/pkgs/book/default.nix create mode 100644 nix/pkgs/default/cross-compilation-env.nix create mode 100644 nix/pkgs/default/default.nix create mode 100644 nix/pkgs/oci-image/default.nix create mode 100644 nix/shell.nix diff --git a/flake.nix b/flake.nix index 984398db..09aa9e09 100644 --- a/flake.nix +++ b/flake.nix @@ -19,286 +19,97 @@ attic.url = "github:zhaofengli/attic?ref=main"; }; - outputs = inputs: inputs.flake-utils.lib.eachDefaultSystem (system: + outputs = inputs: let - pkgsHost = inputs.nixpkgs.legacyPackages.${system}; + # Keep sorted + mkScope = pkgs: pkgs.lib.makeScope pkgs.newScope (self: { + craneLib = + (inputs.crane.mkLib pkgs).overrideToolchain self.toolchain; - # Nix-accessible `Cargo.toml` - cargoToml = builtins.fromTOML (builtins.readFile ./Cargo.toml); + default = self.callPackage ./nix/pkgs/default {}; - # The Rust toolchain to use - toolchain = inputs.fenix.packages.${system}.fromToolchainFile { - file = ./rust-toolchain.toml; + inherit inputs; - # See also `rust-toolchain.toml` - sha256 = "sha256-SXRtAuO4IqNOQq+nLbrsDFbVk+3aVA8NNpSZsKlVH/8="; - }; + oci-image = self.callPackage ./nix/pkgs/oci-image {}; - builder = pkgs: - ((inputs.crane.mkLib pkgs).overrideToolchain toolchain).buildPackage; + book = self.callPackage ./nix/pkgs/book {}; - nativeBuildInputs = pkgs: [ - # bindgen needs the build platform's libclang. Apparently due to - # "splicing weirdness", pkgs.rustPlatform.bindgenHook on its own doesn't - # quite do the right thing here. - pkgs.pkgsBuildHost.rustPlatform.bindgenHook - ]; - - rocksdb' = pkgs: - let - version = "9.1.1"; - in - pkgs.rocksdb.overrideAttrs (old: { - inherit version; - src = pkgs.fetchFromGitHub { - owner = "facebook"; - repo = "rocksdb"; - rev = "v${version}"; - hash = "sha256-/Xf0bzNJPclH9IP80QNaABfhj4IAR5LycYET18VFCXc="; - }; - }); - - env = pkgs: { - CONDUIT_VERSION_EXTRA = - inputs.self.shortRev or inputs.self.dirtyShortRev; - ROCKSDB_INCLUDE_DIR = "${rocksdb' pkgs}/include"; - ROCKSDB_LIB_DIR = "${rocksdb' pkgs}/lib"; - } - // pkgs.lib.optionalAttrs pkgs.stdenv.hostPlatform.isStatic { - ROCKSDB_STATIC = ""; - } - // { - CARGO_BUILD_RUSTFLAGS = let inherit (pkgs) lib stdenv; in - lib.concatStringsSep " " ([] - ++ lib.optionals - # This disables PIE for static builds, which isn't great in terms - # of security. Unfortunately, my hand is forced because nixpkgs' - # `libstdc++.a` is built without `-fPIE`, which precludes us from - # leaving PIE enabled. - stdenv.hostPlatform.isStatic - ["-C" "relocation-model=static"] - ++ lib.optionals - (stdenv.buildPlatform.config != stdenv.hostPlatform.config) - ["-l" "c"] - ++ lib.optionals - # This check has to match the one [here][0]. We only need to set - # these flags when using a different linker. Don't ask me why, - # though, because I don't know. All I know is it breaks otherwise. - # - # [0]: https://github.com/NixOS/nixpkgs/blob/5cdb38bb16c6d0a38779db14fcc766bc1b2394d6/pkgs/build-support/rust/lib/default.nix#L37-L40 - ( - # Nixpkgs doesn't check for x86_64 here but we do, because I - # observed a failure building statically for x86_64 without - # including it here. Linkers are weird. - (stdenv.hostPlatform.isAarch64 || stdenv.hostPlatform.isx86_64) - && stdenv.hostPlatform.isStatic - && !stdenv.isDarwin - && !stdenv.cc.bintools.isLLVM - ) - [ - "-l" - "stdc++" - "-L" - "${stdenv.cc.cc.lib}/${stdenv.hostPlatform.config}/lib" - ] - ); - } - - # What follows is stolen from [here][0]. Its purpose is to properly - # configure compilers and linkers for various stages of the build, and - # even covers the case of build scripts that need native code compiled and - # run on the build platform (I think). - # - # [0]: https://github.com/NixOS/nixpkgs/blob/5cdb38bb16c6d0a38779db14fcc766bc1b2394d6/pkgs/build-support/rust/lib/default.nix#L57-L80 - // ( - let - inherit (pkgs.rust.lib) envVars; - in - pkgs.lib.optionalAttrs - (pkgs.stdenv.targetPlatform.rust.rustcTarget - != pkgs.stdenv.hostPlatform.rust.rustcTarget) - ( - let - inherit (pkgs.stdenv.targetPlatform.rust) cargoEnvVarTarget; - in - { - "CC_${cargoEnvVarTarget}" = envVars.ccForTarget; - "CXX_${cargoEnvVarTarget}" = envVars.cxxForTarget; - "CARGO_TARGET_${cargoEnvVarTarget}_LINKER" = - envVars.linkerForTarget; - } - ) - // ( - let - inherit (pkgs.stdenv.hostPlatform.rust) cargoEnvVarTarget rustcTarget; - in - { - "CC_${cargoEnvVarTarget}" = envVars.ccForHost; - "CXX_${cargoEnvVarTarget}" = envVars.cxxForHost; - "CARGO_TARGET_${cargoEnvVarTarget}_LINKER" = envVars.linkerForHost; - CARGO_BUILD_TARGET = rustcTarget; - } - ) - // ( + rocksdb = let - inherit (pkgs.stdenv.buildPlatform.rust) cargoEnvVarTarget; + version = "9.1.1"; in - { - "CC_${cargoEnvVarTarget}" = envVars.ccForBuild; - "CXX_${cargoEnvVarTarget}" = envVars.cxxForBuild; - "CARGO_TARGET_${cargoEnvVarTarget}_LINKER" = envVars.linkerForBuild; - HOST_CC = "${pkgs.pkgsBuildHost.stdenv.cc}/bin/cc"; - HOST_CXX = "${pkgs.pkgsBuildHost.stdenv.cc}/bin/c++"; - } - )); - - package = pkgs: builder pkgs { - src = inputs.nix-filter { - root = ./.; - include = [ - "src" - "Cargo.toml" - "Cargo.lock" - ]; - }; - - # This is redundant with CI - doCheck = false; + pkgs.rocksdb.overrideAttrs (old: { + inherit version; + src = pkgs.fetchFromGitHub { + owner = "facebook"; + repo = "rocksdb"; + rev = "v${version}"; + hash = "sha256-/Xf0bzNJPclH9IP80QNaABfhj4IAR5LycYET18VFCXc="; + }; + }); - env = env pkgs; - nativeBuildInputs = nativeBuildInputs pkgs; + shell = self.callPackage ./nix/shell.nix {}; - meta.mainProgram = cargoToml.package.name; - }; + # The Rust toolchain to use + toolchain = inputs + .fenix + .packages + .${pkgs.pkgsBuildHost.system} + .fromToolchainFile { + file = ./rust-toolchain.toml; - mkOciImage = pkgs: package: - pkgs.dockerTools.buildImage { - name = package.pname; - tag = "next"; - copyToRoot = [ - pkgs.dockerTools.caCertificates - ]; - config = { - # Use the `tini` init system so that signals (e.g. ctrl+c/SIGINT) - # are handled as expected - Entrypoint = [ - "${pkgs.lib.getExe' pkgs.tini "tini"}" - "--" - ]; - Cmd = [ - "${pkgs.lib.getExe package}" - ]; + # See also `rust-toolchain.toml` + sha256 = "sha256-SXRtAuO4IqNOQq+nLbrsDFbVk+3aVA8NNpSZsKlVH/8="; }; - }; + }); in - { - packages = { - default = package pkgsHost; - oci-image = mkOciImage pkgsHost inputs.self.packages.${system}.default; - - book = - let - package = inputs.self.packages.${system}.default; - in - pkgsHost.stdenv.mkDerivation { - pname = "${package.pname}-book"; - version = package.version; - - src = inputs.nix-filter { - root = ./.; - include = [ - "book.toml" - "conduit-example.toml" - "README.md" - "debian/README.md" - "docs" - ]; - }; - - nativeBuildInputs = (with pkgsHost; [ - mdbook - ]); - - buildPhase = '' - mdbook build - mv public $out - ''; - }; - } - // - builtins.listToAttrs - (builtins.concatLists - (builtins.map - (crossSystem: - let - binaryName = "static-${crossSystem}"; - pkgsCrossStatic = - (import inputs.nixpkgs { - inherit system; - crossSystem = { - config = crossSystem; - }; - }).pkgsStatic; - in + inputs.flake-utils.lib.eachDefaultSystem (system: + let + pkgs = inputs.nixpkgs.legacyPackages.${system}; + in + { + packages = { + default = (mkScope pkgs).default; + oci-image = (mkScope pkgs).oci-image; + book = (mkScope pkgs).book; + } + // + builtins.listToAttrs + (builtins.concatLists + (builtins.map + (crossSystem: + let + binaryName = "static-${crossSystem}"; + pkgsCrossStatic = + (import inputs.nixpkgs { + inherit system; + crossSystem = { + config = crossSystem; + }; + }).pkgsStatic; + in + [ + # An output for a statically-linked binary + { + name = binaryName; + value = (mkScope pkgsCrossStatic).default; + } + + # An output for an OCI image based on that binary + { + name = "oci-image-${crossSystem}"; + value = (mkScope pkgsCrossStatic).oci-image; + } + ] + ) [ - # An output for a statically-linked binary - { - name = binaryName; - value = package pkgsCrossStatic; - } - - # An output for an OCI image based on that binary - { - name = "oci-image-${crossSystem}"; - value = mkOciImage - pkgsCrossStatic - inputs.self.packages.${system}.${binaryName}; - } + "x86_64-unknown-linux-musl" + "aarch64-unknown-linux-musl" ] ) - [ - "x86_64-unknown-linux-musl" - "aarch64-unknown-linux-musl" - ] - ) - ); - - devShells.default = pkgsHost.mkShell { - env = env pkgsHost // { - # Rust Analyzer needs to be able to find the path to default crate - # sources, and it can read this environment variable to do so. The - # `rust-src` component is required in order for this to work. - RUST_SRC_PATH = "${toolchain}/lib/rustlib/src/rust/library"; - }; - - # Development tools - nativeBuildInputs = nativeBuildInputs pkgsHost ++ [ - # Always use nightly rustfmt because most of its options are unstable - # - # This needs to come before `toolchain` in this list, otherwise - # `$PATH` will have stable rustfmt instead. - inputs.fenix.packages.${system}.latest.rustfmt - - toolchain - ] ++ (with pkgsHost; [ - engage - - # Needed for producing Debian packages - cargo-deb - - # Needed for Complement - go - olm - - # Needed for our script for Complement - jq - - # Needed for finding broken markdown links - lychee + ); - # Useful for editing the book locally - mdbook - ]); - }; - }); + devShells.default = (mkScope pkgs).shell; + } + ); } diff --git a/nix/pkgs/book/default.nix b/nix/pkgs/book/default.nix new file mode 100644 index 00000000..cc0464d6 --- /dev/null +++ b/nix/pkgs/book/default.nix @@ -0,0 +1,34 @@ +# Keep sorted +{ default +, inputs +, mdbook +, stdenv +}: + +stdenv.mkDerivation { + pname = "${default.pname}-book"; + version = default.version; + + + src = let filter = inputs.nix-filter.lib; in filter { + root = inputs.self; + + # Keep sorted + include = [ + "book.toml" + "conduit-example.toml" + "debian/README.md" + "docs" + "README.md" + ]; + }; + + nativeBuildInputs = [ + mdbook + ]; + + buildPhase = '' + mdbook build + mv public $out + ''; +} diff --git a/nix/pkgs/default/cross-compilation-env.nix b/nix/pkgs/default/cross-compilation-env.nix new file mode 100644 index 00000000..fac85e02 --- /dev/null +++ b/nix/pkgs/default/cross-compilation-env.nix @@ -0,0 +1,100 @@ +{ lib +, pkgsBuildHost +, rust +, stdenv +}: + +lib.optionalAttrs stdenv.hostPlatform.isStatic { + ROCKSDB_STATIC = ""; +} +// +{ + CARGO_BUILD_RUSTFLAGS = + lib.concatStringsSep + " " + ([] + # This disables PIE for static builds, which isn't great in terms of + # security. Unfortunately, my hand is forced because nixpkgs' + # `libstdc++.a` is built without `-fPIE`, which precludes us from + # leaving PIE enabled. + ++ lib.optionals + stdenv.hostPlatform.isStatic + [ "-C" "relocation-model=static" ] + ++ lib.optionals + (stdenv.buildPlatform.config != stdenv.hostPlatform.config) + [ "-l" "c" ] + ++ lib.optionals + # This check has to match the one [here][0]. We only need to set + # these flags when using a different linker. Don't ask me why, though, + # because I don't know. All I know is it breaks otherwise. + # + # [0]: https://github.com/NixOS/nixpkgs/blob/5cdb38bb16c6d0a38779db14fcc766bc1b2394d6/pkgs/build-support/rust/lib/default.nix#L37-L40 + ( + # Nixpkgs doesn't check for x86_64 here but we do, because I + # observed a failure building statically for x86_64 without + # including it here. Linkers are weird. + (stdenv.hostPlatform.isAarch64 || stdenv.hostPlatform.isx86_64) + && stdenv.hostPlatform.isStatic + && !stdenv.isDarwin + && !stdenv.cc.bintools.isLLVM + ) + [ + "-l" + "stdc++" + "-L" + "${stdenv.cc.cc.lib}/${stdenv.hostPlatform.config}/lib" + ] + ); +} + +# What follows is stolen from [here][0]. Its purpose is to properly configure +# compilers and linkers for various stages of the build, and even covers the +# case of build scripts that need native code compiled and run on the build +# platform (I think). +# +# [0]: https://github.com/NixOS/nixpkgs/blob/5cdb38bb16c6d0a38779db14fcc766bc1b2394d6/pkgs/build-support/rust/lib/default.nix#L57-L80 +// +( + let + inherit (rust.lib) envVars; + in + lib.optionalAttrs + (stdenv.targetPlatform.rust.rustcTarget + != stdenv.hostPlatform.rust.rustcTarget) + ( + let + inherit (stdenv.targetPlatform.rust) cargoEnvVarTarget; + in + { + "CC_${cargoEnvVarTarget}" = envVars.ccForTarget; + "CXX_${cargoEnvVarTarget}" = envVars.cxxForTarget; + "CARGO_TARGET_${cargoEnvVarTarget}_LINKER" = + envVars.linkerForTarget; + } + ) + // + ( + let + inherit (stdenv.hostPlatform.rust) cargoEnvVarTarget rustcTarget; + in + { + "CC_${cargoEnvVarTarget}" = envVars.ccForHost; + "CXX_${cargoEnvVarTarget}" = envVars.cxxForHost; + "CARGO_TARGET_${cargoEnvVarTarget}_LINKER" = envVars.linkerForHost; + CARGO_BUILD_TARGET = rustcTarget; + } + ) + // + ( + let + inherit (stdenv.buildPlatform.rust) cargoEnvVarTarget; + in + { + "CC_${cargoEnvVarTarget}" = envVars.ccForBuild; + "CXX_${cargoEnvVarTarget}" = envVars.cxxForBuild; + "CARGO_TARGET_${cargoEnvVarTarget}_LINKER" = envVars.linkerForBuild; + HOST_CC = "${pkgsBuildHost.stdenv.cc}/bin/cc"; + HOST_CXX = "${pkgsBuildHost.stdenv.cc}/bin/c++"; + } + ) +) diff --git a/nix/pkgs/default/default.nix b/nix/pkgs/default/default.nix new file mode 100644 index 00000000..75c87b42 --- /dev/null +++ b/nix/pkgs/default/default.nix @@ -0,0 +1,64 @@ +# Keep sorted +{ craneLib +, inputs +, lib +, pkgsBuildHost +, rocksdb +, rust +, stdenv +}: + +let + env = { + CONDUIT_VERSION_EXTRA = inputs.self.shortRev or inputs.self.dirtyShortRev; + ROCKSDB_INCLUDE_DIR = "${rocksdb}/include"; + ROCKSDB_LIB_DIR = "${rocksdb}/lib"; + } + // + (import ./cross-compilation-env.nix { + # Keep sorted + inherit + lib + pkgsBuildHost + rust + stdenv; + }); +in + +craneLib.buildPackage rec { + inherit + (craneLib.crateNameFromCargoToml { + cargoToml = "${inputs.self}/Cargo.toml"; + }) + pname + version; + + src = let filter = inputs.nix-filter.lib; in filter { + root = inputs.self; + + # Keep sorted + include = [ + "Cargo.lock" + "Cargo.toml" + "src" + ]; + }; + + # This is redundant with CI + doCheck = false; + + nativeBuildInputs = [ + # bindgen needs the build platform's libclang. Apparently due to "splicing + # weirdness", pkgs.rustPlatform.bindgenHook on its own doesn't quite do the + # right thing here. + pkgsBuildHost.rustPlatform.bindgenHook + ]; + + inherit env; + + passthru = { + inherit env; + }; + + meta.mainProgram = pname; +} diff --git a/nix/pkgs/oci-image/default.nix b/nix/pkgs/oci-image/default.nix new file mode 100644 index 00000000..8b359ce4 --- /dev/null +++ b/nix/pkgs/oci-image/default.nix @@ -0,0 +1,25 @@ +# Keep sorted +{ default +, dockerTools +, lib +, tini +}: + +dockerTools.buildImage { + name = default.pname; + tag = "next"; + copyToRoot = [ + dockerTools.caCertificates + ]; + config = { + # Use the `tini` init system so that signals (e.g. ctrl+c/SIGINT) + # are handled as expected + Entrypoint = [ + "${lib.getExe' tini "tini"}" + "--" + ]; + Cmd = [ + "${lib.getExe default}" + ]; + }; +} diff --git a/nix/shell.nix b/nix/shell.nix new file mode 100644 index 00000000..8cfc1c44 --- /dev/null +++ b/nix/shell.nix @@ -0,0 +1,55 @@ +# Keep sorted +{ cargo-deb +, default +, engage +, go +, inputs +, jq +, lychee +, mdbook +, mkShell +, olm +, system +, toolchain +}: + +mkShell { + env = default.env // { + # Rust Analyzer needs to be able to find the path to default crate + # sources, and it can read this environment variable to do so. The + # `rust-src` component is required in order for this to work. + RUST_SRC_PATH = "${toolchain}/lib/rustlib/src/rust/library"; + }; + + # Development tools + nativeBuildInputs = default.nativeBuildInputs ++ [ + # Always use nightly rustfmt because most of its options are unstable + # + # This needs to come before `toolchain` in this list, otherwise + # `$PATH` will have stable rustfmt instead. + inputs.fenix.packages.${system}.latest.rustfmt + + # Keep sorted + engage + toolchain + + # Needed for producing Debian packages + cargo-deb + + # Needed for our script for Complement + jq + + # Needed for Complement + go + olm + + # Needed for our script for Complement + jq + + # Needed for finding broken markdown links + lychee + + # Useful for editing the book locally + mdbook + ]; +}