diff --git a/flake.lock b/flake.lock index 31a85584..6d665aa5 100644 --- a/flake.lock +++ b/flake.lock @@ -1,94 +1,28 @@ { "nodes": { - "attic": { - "inputs": { - "crane": "crane", - "flake-compat": "flake-compat", - "flake-parts": "flake-parts", - "nix-github-actions": "nix-github-actions", - "nixpkgs": "nixpkgs", - "nixpkgs-stable": "nixpkgs-stable" - }, + "advisory-db": { + "flake": false, "locked": { - "lastModified": 1758711588, - "narHash": "sha256-0nZlCCDC5PfndsQJXXtcyrtrfW49I3KadGMDlutzaGU=", - "owner": "zhaofengli", - "repo": "attic", - "rev": "12cbeca141f46e1ade76728bce8adc447f2166c6", + "lastModified": 1760533899, + "narHash": "sha256-wEBOgeHURiBQQnzdCKl30IDnSBqbEXw0Sfnm+an0JmE=", + "owner": "rustsec", + "repo": "advisory-db", + "rev": "218a772dc1f031aa64e0c575a92c8e04439674d5", "type": "github" }, "original": { - "owner": "zhaofengli", - "ref": "main", - "repo": "attic", - "type": "github" - } - }, - "cachix": { - "inputs": { - "devenv": "devenv", - "flake-compat": "flake-compat_2", - "git-hooks": "git-hooks", - "nixpkgs": "nixpkgs_2" - }, - "locked": { - "lastModified": 1756385612, - "narHash": "sha256-+NU5MMhuPHHRyvZZWNFG7zt+leRSPsJu1MwhOUzkPUk=", - "owner": "cachix", - "repo": "cachix", - "rev": "dc24688cd67518c3711d511fa369c0f5a131063a", - "type": "github" - }, - "original": { - "owner": "cachix", - "ref": "master", - "repo": "cachix", - "type": "github" - } - }, - "cachix_2": { - "inputs": { - "devenv": [ - "cachix", - "devenv" - ], - "flake-compat": [ - "cachix", - "devenv" - ], - "git-hooks": [ - "cachix", - "devenv", - "git-hooks" - ], - "nixpkgs": [ - "cachix", - "devenv", - "nixpkgs" - ] - }, - "locked": { - "lastModified": 1748883665, - "narHash": "sha256-R0W7uAg+BLoHjMRMQ8+oiSbTq8nkGz5RDpQ+ZfxxP3A=", - "owner": "cachix", - "repo": "cachix", - "rev": "f707778d902af4d62d8dd92c269f8e70de09acbe", - "type": "github" - }, - "original": { - "owner": "cachix", - "ref": "latest", - "repo": "cachix", + "owner": "rustsec", + "repo": "advisory-db", "type": "github" } }, "crane": { "locked": { - "lastModified": 1751562746, - "narHash": "sha256-smpugNIkmDeicNz301Ll1bD7nFOty97T79m4GUMUczA=", + "lastModified": 1758686891, + "narHash": "sha256-Lq8JkwjSzv80T1i8KCvqAch75zwD98UKA+4atZjvfZ0=", "owner": "ipetkov", "repo": "crane", - "rev": "aed2020fd3dc26e1e857d4107a5a67a33ab6c1fd", + "rev": "02063302383f43237602f5aea5a67766b08e4787", "type": "github" }, "original": { @@ -97,53 +31,6 @@ "type": "github" } }, - "crane_2": { - "locked": { - "lastModified": 1759893430, - "narHash": "sha256-yAy4otLYm9iZ+NtQwTMEbqHwswSFUbhn7x826RR6djw=", - "owner": "ipetkov", - "repo": "crane", - "rev": "1979a2524cb8c801520bd94c38bb3d5692419d93", - "type": "github" - }, - "original": { - "owner": "ipetkov", - "ref": "master", - "repo": "crane", - "type": "github" - } - }, - "devenv": { - "inputs": { - "cachix": "cachix_2", - "flake-compat": [ - "cachix", - "flake-compat" - ], - "git-hooks": [ - "cachix", - "git-hooks" - ], - "nix": "nix", - "nixpkgs": [ - "cachix", - "nixpkgs" - ] - }, - "locked": { - "lastModified": 1754404745, - "narHash": "sha256-BdbW/iTImczgcuATgQIa9sPGuYIBxVq2xqcvICsa2AQ=", - "owner": "cachix", - "repo": "devenv", - "rev": "6563b21105168f90394dfaf58284b078af2d7275", - "type": "github" - }, - "original": { - "owner": "cachix", - "repo": "devenv", - "type": "github" - } - }, "fenix": { "inputs": { "nixpkgs": [ @@ -161,44 +48,11 @@ }, "original": { "owner": "nix-community", - "ref": "main", "repo": "fenix", "type": "github" } }, "flake-compat": { - "flake": false, - "locked": { - "lastModified": 1747046372, - "narHash": "sha256-CIVLLkVgvHYbgI2UpXvIIBJ12HWgX+fjA8Xf8PUmqCY=", - "owner": "edolstra", - "repo": "flake-compat", - "rev": "9100a0f413b0c601e0533d1d94ffd501ce2e7885", - "type": "github" - }, - "original": { - "owner": "edolstra", - "repo": "flake-compat", - "type": "github" - } - }, - "flake-compat_2": { - "flake": false, - "locked": { - "lastModified": 1747046372, - "narHash": "sha256-CIVLLkVgvHYbgI2UpXvIIBJ12HWgX+fjA8Xf8PUmqCY=", - "owner": "edolstra", - "repo": "flake-compat", - "rev": "9100a0f413b0c601e0533d1d94ffd501ce2e7885", - "type": "github" - }, - "original": { - "owner": "edolstra", - "repo": "flake-compat", - "type": "github" - } - }, - "flake-compat_3": { "flake": false, "locked": { "lastModified": 1747046372, @@ -217,17 +71,14 @@ }, "flake-parts": { "inputs": { - "nixpkgs-lib": [ - "attic", - "nixpkgs" - ] + "nixpkgs-lib": "nixpkgs-lib" }, "locked": { - "lastModified": 1751413152, - "narHash": "sha256-Tyw1RjYEsp5scoigs1384gIg6e0GoBVjms4aXFfRssQ=", + "lastModified": 1756770412, + "narHash": "sha256-+uWLQZccFHwqpGqr2Yt5VsW/PbeJVTn9Dk6SHWhNRPw=", "owner": "hercules-ci", "repo": "flake-parts", - "rev": "77826244401ea9de6e3bac47c2db46005e1f30b5", + "rev": "4524271976b625a4a605beefd893f270620fd751", "type": "github" }, "original": { @@ -236,214 +87,13 @@ "type": "github" } }, - "flake-parts_2": { - "inputs": { - "nixpkgs-lib": [ - "cachix", - "devenv", - "nix", - "nixpkgs" - ] - }, - "locked": { - "lastModified": 1733312601, - "narHash": "sha256-4pDvzqnegAfRkPwO3wmwBhVi/Sye1mzps0zHWYnP88c=", - "owner": "hercules-ci", - "repo": "flake-parts", - "rev": "205b12d8b7cd4802fbcb8e8ef6a0f1408781a4f9", - "type": "github" - }, - "original": { - "owner": "hercules-ci", - "repo": "flake-parts", - "type": "github" - } - }, - "flake-utils": { - "inputs": { - "systems": "systems" - }, - "locked": { - "lastModified": 1731533236, - "narHash": "sha256-l0KFg5HjrsfsO/JpG+r7fRrqm12kzFHyUHqHCVpMMbI=", - "owner": "numtide", - "repo": "flake-utils", - "rev": "11707dc2f618dd54ca8739b309ec4fc024de578b", - "type": "github" - }, - "original": { - "owner": "numtide", - "ref": "main", - "repo": "flake-utils", - "type": "github" - } - }, - "git-hooks": { - "inputs": { - "flake-compat": [ - "cachix", - "flake-compat" - ], - "gitignore": "gitignore", - "nixpkgs": [ - "cachix", - "nixpkgs" - ] - }, - "locked": { - "lastModified": 1750779888, - "narHash": "sha256-wibppH3g/E2lxU43ZQHC5yA/7kIKLGxVEnsnVK1BtRg=", - "owner": "cachix", - "repo": "git-hooks.nix", - "rev": "16ec914f6fb6f599ce988427d9d94efddf25fe6d", - "type": "github" - }, - "original": { - "owner": "cachix", - "repo": "git-hooks.nix", - "type": "github" - } - }, - "gitignore": { - "inputs": { - "nixpkgs": [ - "cachix", - "git-hooks", - "nixpkgs" - ] - }, - "locked": { - "lastModified": 1709087332, - "narHash": "sha256-HG2cCnktfHsKV0s4XW83gU3F57gaTljL9KNSuG6bnQs=", - "owner": "hercules-ci", - "repo": "gitignore.nix", - "rev": "637db329424fd7e46cf4185293b9cc8c88c95394", - "type": "github" - }, - "original": { - "owner": "hercules-ci", - "repo": "gitignore.nix", - "type": "github" - } - }, - "nix": { - "inputs": { - "flake-compat": [ - "cachix", - "devenv", - "flake-compat" - ], - "flake-parts": "flake-parts_2", - "git-hooks-nix": [ - "cachix", - "devenv", - "git-hooks" - ], - "nixpkgs": [ - "cachix", - "devenv", - "nixpkgs" - ], - "nixpkgs-23-11": [ - "cachix", - "devenv" - ], - "nixpkgs-regression": [ - "cachix", - "devenv" - ] - }, - "locked": { - "lastModified": 1752773918, - "narHash": "sha256-dOi/M6yNeuJlj88exI+7k154z+hAhFcuB8tZktiW7rg=", - "owner": "cachix", - "repo": "nix", - "rev": "031c3cf42d2e9391eee373507d8c12e0f9606779", - "type": "github" - }, - "original": { - "owner": "cachix", - "ref": "devenv-2.30", - "repo": "nix", - "type": "github" - } - }, - "nix-filter": { - "locked": { - "lastModified": 1757882181, - "narHash": "sha256-+cCxYIh2UNalTz364p+QYmWHs0P+6wDhiWR4jDIKQIU=", - "owner": "numtide", - "repo": "nix-filter", - "rev": "59c44d1909c72441144b93cf0f054be7fe764de5", - "type": "github" - }, - "original": { - "owner": "numtide", - "ref": "main", - "repo": "nix-filter", - "type": "github" - } - }, - "nix-github-actions": { - "inputs": { - "nixpkgs": [ - "attic", - "nixpkgs" - ] - }, - "locked": { - "lastModified": 1737420293, - "narHash": "sha256-F1G5ifvqTpJq7fdkT34e/Jy9VCyzd5XfJ9TO8fHhJWE=", - "owner": "nix-community", - "repo": "nix-github-actions", - "rev": "f4158fa080ef4503c8f4c820967d946c2af31ec9", - "type": "github" - }, - "original": { - "owner": "nix-community", - "repo": "nix-github-actions", - "type": "github" - } - }, "nixpkgs": { "locked": { - "lastModified": 1751949589, - "narHash": "sha256-mgFxAPLWw0Kq+C8P3dRrZrOYEQXOtKuYVlo9xvPntt8=", + "lastModified": 1758427187, + "narHash": "sha256-pHpxZ/IyCwoTQPtFIAG2QaxuSm8jWzrzBGjwQZIttJc=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "9b008d60392981ad674e04016d25619281550a9d", - "type": "github" - }, - "original": { - "owner": "NixOS", - "ref": "nixpkgs-unstable", - "repo": "nixpkgs", - "type": "github" - } - }, - "nixpkgs-stable": { - "locked": { - "lastModified": 1751741127, - "narHash": "sha256-t75Shs76NgxjZSgvvZZ9qOmz5zuBE8buUaYD28BMTxg=", - "owner": "NixOS", - "repo": "nixpkgs", - "rev": "29e290002bfff26af1db6f64d070698019460302", - "type": "github" - }, - "original": { - "owner": "NixOS", - "ref": "nixos-25.05", - "repo": "nixpkgs", - "type": "github" - } - }, - "nixpkgs_2": { - "locked": { - "lastModified": 1754214453, - "narHash": "sha256-Q/I2xJn/j1wpkGhWkQnm20nShYnG7TI99foDBpXm1SY=", - "owner": "NixOS", - "repo": "nixpkgs", - "rev": "5b09dc45f24cf32316283e62aec81ffee3c3e376", + "rev": "554be6495561ff07b6c724047bdd7e0716aa7b46", "type": "github" }, "original": { @@ -453,7 +103,7 @@ "type": "github" } }, - "nixpkgs_3": { + "nixpkgs-lib": { "locked": { "lastModified": 1760504863, "narHash": "sha256-h13YFQMi91nXkkRoJMIfezorz5SbD6849jw5L0fjK4I=", @@ -463,22 +113,19 @@ "type": "github" }, "original": { - "owner": "NixOS", - "ref": "nixpkgs-unstable", - "repo": "nixpkgs", + "owner": "nix-community", + "repo": "nixpkgs.lib", "type": "github" } }, "root": { "inputs": { - "attic": "attic", - "cachix": "cachix", - "crane": "crane_2", + "advisory-db": "advisory-db", + "crane": "crane", "fenix": "fenix", - "flake-compat": "flake-compat_3", - "flake-utils": "flake-utils", - "nix-filter": "nix-filter", - "nixpkgs": "nixpkgs_3" + "flake-compat": "flake-compat", + "flake-parts": "flake-parts", + "nixpkgs": "nixpkgs" } }, "rust-analyzer-src": { @@ -497,21 +144,6 @@ "repo": "rust-analyzer", "type": "github" } - }, - "systems": { - "locked": { - "lastModified": 1681028828, - "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=", - "owner": "nix-systems", - "repo": "default", - "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e", - "type": "github" - }, - "original": { - "owner": "nix-systems", - "repo": "default", - "type": "github" - } } }, "root": "root", diff --git a/flake.nix b/flake.nix index 967381cf..7d68191a 100644 --- a/flake.nix +++ b/flake.nix @@ -1,351 +1,41 @@ { + description = "A nix flake for the continuwuity project"; + inputs = { - attic.url = "github:zhaofengli/attic?ref=main"; - cachix.url = "github:cachix/cachix?ref=master"; - crane = { - url = "github:ipetkov/crane?ref=master"; - }; + # basics + flake-parts.url = "github:hercules-ci/flake-parts"; + nixpkgs.url = "github:NixOS/nixpkgs/nixos-unstable"; + + # for rust via nix + crane.url = "github:ipetkov/crane"; fenix = { - url = "github:nix-community/fenix?ref=main"; + url = "github:nix-community/fenix"; inputs.nixpkgs.follows = "nixpkgs"; }; + + # for vuln checks + advisory-db = { + url = "github:rustsec/advisory-db"; + flake = false; + }; + + # for default.nix flake-compat = { url = "github:edolstra/flake-compat?ref=master"; flake = false; }; - flake-utils.url = "github:numtide/flake-utils?ref=main"; - nix-filter.url = "github:numtide/nix-filter?ref=main"; - nixpkgs.url = "github:NixOS/nixpkgs?ref=nixpkgs-unstable"; + }; outputs = - inputs: - inputs.flake-utils.lib.eachDefaultSystem ( - system: - let - pkgsHost = import inputs.nixpkgs { - inherit system; - }; - - fnx = inputs.fenix.packages.${system}; - # The Rust toolchain to use - toolchain = fnx.combine [ - (fnx.fromToolchainFile { - file = ./rust-toolchain.toml; - - # See also `rust-toolchain.toml` - sha256 = "sha256-+9FmLhAOezBZCOziO0Qct1NOrfpjNsXxc/8I0c7BdKE="; - }) - fnx.complete.rustfmt - ]; - - mkScope = - pkgs: - pkgs.lib.makeScope pkgs.newScope (self: { - inherit pkgs inputs; - craneLib = (inputs.crane.mkLib pkgs).overrideToolchain (_: toolchain); - main = self.callPackage ./pkg/nix/pkgs/main { }; - liburing = pkgs.liburing.overrideAttrs { - # Tests weren't building - outputs = [ - "out" - "dev" - "man" - ]; - buildFlags = [ "library" ]; - }; - rocksdb = - (pkgs.rocksdb_9_10.override { - # Override the liburing input for the build with our own so - # we have it built with the library flag - inherit (self) liburing; - }).overrideAttrs - (old: { - src = pkgsHost.fetchFromGitea { - domain = "forgejo.ellis.link"; - owner = "continuwuation"; - repo = "rocksdb"; - rev = "10.5.fb"; - sha256 = "sha256-X4ApGLkHF9ceBtBg77dimEpu720I79ffLoyPa8JMHaU="; - }; - version = "v10.5.fb"; - cmakeFlags = - pkgs.lib.subtractLists [ - # No real reason to have snappy or zlib, no one uses this - "-DWITH_SNAPPY=1" - "-DZLIB=1" - "-DWITH_ZLIB=1" - # We don't need to use ldb or sst_dump (core_tools) - "-DWITH_CORE_TOOLS=1" - # We don't need to build rocksdb tests - "-DWITH_TESTS=1" - # We use rust-rocksdb via C interface and don't need C++ RTTI - "-DUSE_RTTI=1" - # This doesn't exist in RocksDB, and USE_SSE is deprecated for - # PORTABLE=$(march) - "-DFORCE_SSE42=1" - # PORTABLE will get set in main/default.nix - "-DPORTABLE=1" - ] old.cmakeFlags - ++ [ - # No real reason to have snappy, no one uses this - "-DWITH_SNAPPY=0" - "-DZLIB=0" - "-DWITH_ZLIB=0" - # We don't need to use ldb or sst_dump (core_tools) - "-DWITH_CORE_TOOLS=0" - # We don't need trace tools - "-DWITH_TRACE_TOOLS=0" - # We don't need to build rocksdb tests - "-DWITH_TESTS=0" - # We use rust-rocksdb via C interface and don't need C++ RTTI - "-DUSE_RTTI=0" - ]; - - # outputs has "tools" which we don't need or use - outputs = [ "out" ]; - - # preInstall hooks has stuff for messing with ldb/sst_dump which we don't need or use - preInstall = ""; - - # We have this already at https://forgejo.ellis.link/continuwuation/rocksdb/commit/a935c0273e1ba44eacf88ce3685a9b9831486155 - # Unsetting this so we don't have to revert it and make this nix exclusive - patches = [ ]; - - postPatch = '' - # Fix gcc-13 build failures due to missing and - # includes, fixed upstream since 8.x - sed -e '1i #include ' -i db/compaction/compaction_iteration_stats.h - sed -e '1i #include ' -i table/block_based/data_block_hash_index.h - sed -e '1i #include ' -i util/string_util.h - sed -e '1i #include ' -i include/rocksdb/utilities/checkpoint.h - ''; - }); - }); - - scopeHost = mkScope pkgsHost; - mkCrossScope = - crossSystem: - let - pkgsCrossStatic = - (import inputs.nixpkgs { - inherit system; - crossSystem = { - config = crossSystem; - }; - }).pkgsStatic; - in - mkScope pkgsCrossStatic; - - in - { - packages = - { - default = scopeHost.main.override { - disable_features = [ - # Don't include experimental features - "experimental" - # jemalloc profiling/stats features are expensive and shouldn't - # be expected on non-debug builds. - "jemalloc_prof" - "jemalloc_stats" - # This is non-functional on nix for some reason - "hardened_malloc" - # conduwuit_mods is a development-only hot reload feature - "conduwuit_mods" - ]; - }; - default-debug = scopeHost.main.override { - profile = "dev"; - # Debug build users expect full logs - disable_release_max_log_level = true; - disable_features = [ - # Don't include experimental features - "experimental" - # This is non-functional on nix for some reason - "hardened_malloc" - # conduwuit_mods is a development-only hot reload feature - "conduwuit_mods" - ]; - }; - # Just a test profile used for things like CI and complement - default-test = scopeHost.main.override { - profile = "test"; - disable_release_max_log_level = true; - disable_features = [ - # Don't include experimental features - "experimental" - # this is non-functional on nix for some reason - "hardened_malloc" - # conduwuit_mods is a development-only hot reload feature - "conduwuit_mods" - ]; - }; - all-features = scopeHost.main.override { - all_features = true; - disable_features = [ - # Don't include experimental features - "experimental" - # jemalloc profiling/stats features are expensive and shouldn't - # be expected on non-debug builds. - "jemalloc_prof" - "jemalloc_stats" - # This is non-functional on nix for some reason - "hardened_malloc" - # conduwuit_mods is a development-only hot reload feature - "conduwuit_mods" - ]; - }; - all-features-debug = scopeHost.main.override { - profile = "dev"; - all_features = true; - # Debug build users expect full logs - disable_release_max_log_level = true; - disable_features = [ - # Don't include experimental features - "experimental" - # This is non-functional on nix for some reason - "hardened_malloc" - # conduwuit_mods is a development-only hot reload feature - "conduwuit_mods" - ]; - }; - hmalloc = scopeHost.main.override { features = [ "hardened_malloc" ]; }; - } - // builtins.listToAttrs ( - builtins.concatLists ( - builtins.map - ( - crossSystem: - let - binaryName = "static-${crossSystem}"; - scopeCrossStatic = mkCrossScope crossSystem; - in - [ - # An output for a statically-linked binary - { - name = binaryName; - value = scopeCrossStatic.main; - } - - # An output for a statically-linked binary with x86_64 haswell - # target optimisations - { - name = "${binaryName}-x86_64-haswell-optimised"; - value = scopeCrossStatic.main.override { - x86_64_haswell_target_optimised = - if (crossSystem == "x86_64-linux-gnu" || crossSystem == "x86_64-linux-musl") then true else false; - }; - } - - # An output for a statically-linked unstripped debug ("dev") binary - { - name = "${binaryName}-debug"; - value = scopeCrossStatic.main.override { - profile = "dev"; - # debug build users expect full logs - disable_release_max_log_level = true; - }; - } - - # An output for a statically-linked unstripped debug binary with the - # "test" profile (for CI usage only) - { - name = "${binaryName}-test"; - value = scopeCrossStatic.main.override { - profile = "test"; - disable_release_max_log_level = true; - disable_features = [ - # dont include experimental features - "experimental" - # this is non-functional on nix for some reason - "hardened_malloc" - # conduwuit_mods is a development-only hot reload feature - "conduwuit_mods" - ]; - }; - } - - # An output for a statically-linked binary with `--all-features` - { - name = "${binaryName}-all-features"; - value = scopeCrossStatic.main.override { - all_features = true; - disable_features = [ - # dont include experimental features - "experimental" - # jemalloc profiling/stats features are expensive and shouldn't - # be expected on non-debug builds. - "jemalloc_prof" - "jemalloc_stats" - # this is non-functional on nix for some reason - "hardened_malloc" - # conduwuit_mods is a development-only hot reload feature - "conduwuit_mods" - ]; - }; - } - - # An output for a statically-linked binary with `--all-features` and with x86_64 haswell - # target optimisations - { - name = "${binaryName}-all-features-x86_64-haswell-optimised"; - value = scopeCrossStatic.main.override { - all_features = true; - disable_features = [ - # dont include experimental features - "experimental" - # jemalloc profiling/stats features are expensive and shouldn't - # be expected on non-debug builds. - "jemalloc_prof" - "jemalloc_stats" - # this is non-functional on nix for some reason - "hardened_malloc" - # conduwuit_mods is a development-only hot reload feature - "conduwuit_mods" - ]; - x86_64_haswell_target_optimised = - if (crossSystem == "x86_64-linux-gnu" || crossSystem == "x86_64-linux-musl") then true else false; - }; - } - - # An output for a statically-linked unstripped debug ("dev") binary with `--all-features` - { - name = "${binaryName}-all-features-debug"; - value = scopeCrossStatic.main.override { - profile = "dev"; - all_features = true; - # debug build users expect full logs - disable_release_max_log_level = true; - disable_features = [ - # dont include experimental features - "experimental" - # this is non-functional on nix for some reason - "hardened_malloc" - # conduwuit_mods is a development-only hot reload feature - "conduwuit_mods" - ]; - }; - } - - # An output for a statically-linked binary with hardened_malloc - { - name = "${binaryName}-hmalloc"; - value = scopeCrossStatic.main.override { - features = [ "hardened_malloc" ]; - }; - } - ] - ) - [ - #"x86_64-apple-darwin" - #"aarch64-apple-darwin" - "x86_64-linux-gnu" - "x86_64-linux-musl" - "aarch64-linux-musl" - ] - ) - ); - } - ); + inputs@{ flake-parts, ... }: + flake-parts.lib.mkFlake { inherit inputs; } { + imports = [ ./nix ]; + systems = [ + # good support + "x86_64-linux" + # support untested but theoretically there + "aarch64-linux" + ]; + }; } diff --git a/nix/checks/default.nix b/nix/checks/default.nix new file mode 100644 index 00000000..04678942 --- /dev/null +++ b/nix/checks/default.nix @@ -0,0 +1,103 @@ +{ inputs, ... }: +{ + perSystem = + { + self', + lib, + pkgs, + ... + }: + let + uwulib = inputs.self.uwulib.init pkgs; + + commonAttrs = (uwulib.build.commonAttrs { }) // { + buildInputs = [ + pkgs.liburing + self'.packages.rust-jemalloc-sys-unprefixed' + self'.packages.rocksdbAllFeatures + ]; + nativeBuildInputs = [ + pkgs.pkg-config + # bindgen needs the build platform's libclang. Apparently due to "splicing + # weirdness", pkgs.rustPlatform.bindgenHook on its own doesn't quite do the + # right thing here. + pkgs.rustPlatform.bindgenHook + ]; + env = { + LIBCLANG_PATH = lib.makeLibraryPath [ pkgs.llvmPackages.libclang.lib ]; + LD_LIBRARY_PATH = lib.makeLibraryPath [ + pkgs.liburing + self'.packages.rust-jemalloc-sys-unprefixed' + self'.packages.rocksdbAllFeatures + ]; + } + // uwulib.environment.buildPackageEnv + // { + ROCKSDB_INCLUDE_DIR = "${self'.packages.rocksdbAllFeatures}/include"; + ROCKSDB_LIB_DIR = "${self'.packages.rocksdbAllFeatures}/lib"; + }; + }; + cargoArtifacts = self'.packages.continuwuity-all-features-deps; + in + { + # taken from + # + # https://crane.dev/examples/quick-start.html + checks = { + continuwuity-all-features-build = self'.packages.continuwuity-all-features-bin; + + continuwuity-all-features-clippy = uwulib.build.craneLibForChecks.cargoClippy ( + commonAttrs + // { + inherit cargoArtifacts; + cargoClippyExtraArgs = "-- --deny warnings"; + } + ); + + continuwuity-all-features-docs = uwulib.build.craneLibForChecks.cargoDoc ( + commonAttrs + // { + inherit cargoArtifacts; + # This can be commented out or tweaked as necessary, e.g. set to + # `--deny rustdoc::broken-intra-doc-links` to only enforce that lint + env.RUSTDOCFLAGS = "--deny warnings"; + } + ); + + # Check formatting + continuwuity-all-features-fmt = uwulib.build.craneLibForChecks.cargoFmt { + src = uwulib.build.src; + }; + + continuwuity-all-features-toml-fmt = uwulib.build.craneLibForChecks.taploFmt { + src = pkgs.lib.sources.sourceFilesBySuffices uwulib.build.src [ ".toml" ]; + # taplo arguments can be further customized below as needed + # taploExtraArgs = "--config ./taplo.toml"; + }; + + # Audit dependencies + continuwuity-all-features-audit = uwulib.build.craneLibForChecks.cargoAudit { + inherit (inputs) advisory-db; + src = uwulib.build.src; + }; + + # Audit licenses + continuwuity-all-features-deny = uwulib.build.craneLibForChecks.cargoDeny { + src = uwulib.build.src; + }; + + # Run tests with cargo-nextest + # Consider setting `doCheck = false` on `continuwuity-all-features` if you do not want + # the tests to run twice + continuwuity-all-features-nextest = uwulib.build.craneLibForChecks.cargoNextest ( + commonAttrs + // { + inherit cargoArtifacts; + partitions = 1; + partitionType = "count"; + cargoNextestPartitionsExtraArgs = "--no-tests=pass"; + } + ); + }; + }; +} diff --git a/nix/default.nix b/nix/default.nix new file mode 100644 index 00000000..a642a1ff --- /dev/null +++ b/nix/default.nix @@ -0,0 +1,8 @@ +{ + imports = [ + ./checks + ./packages + ./shells + ./tests + ]; +} diff --git a/nix/packages/continuwuity/default.nix b/nix/packages/continuwuity/default.nix new file mode 100644 index 00000000..b47f6ab6 --- /dev/null +++ b/nix/packages/continuwuity/default.nix @@ -0,0 +1,60 @@ +{ inputs, ... }: +{ + perSystem = + { + self', + lib, + pkgs, + ... + }: + let + uwulib = inputs.self.uwulib.init pkgs; + in + { + packages = + lib.pipe + [ + # this is the default variant + { + variantName = "default"; + commonAttrsArgs.profile = "release"; + rocksdb = self'.packages.rocksdb; + features = { }; + } + # this is the variant with all features enabled (liburing + jemalloc) + { + variantName = "all-features"; + commonAttrsArgs.profile = "release"; + rocksdb = self'.packages.rocksdb.override { + enableJemalloc = true; + enableLiburing = true; + }; + features = { + enabledFeatures = "all"; + disabledFeatures = uwulib.features.defaultDisabledFeatures ++ [ "bindgen-static" ]; + }; + } + ] + [ + (builtins.map (cfg: rec { + deps = { + name = "continuwuity-${cfg.variantName}-deps"; + value = uwulib.build.buildDeps { + features = uwulib.features.calcFeatures cfg.features; + inherit (cfg) commonAttrsArgs rocksdb; + }; + }; + bin = { + name = "continuwuity-${cfg.variantName}-bin"; + value = uwulib.build.buildPackage { + deps = self'.packages.${deps.name}; + features = uwulib.features.calcFeatures cfg.features; + inherit (cfg) commonAttrsArgs rocksdb; + }; + }; + })) + (builtins.concatMap builtins.attrValues) + builtins.listToAttrs + ]; + }; +} diff --git a/nix/packages/default.nix b/nix/packages/default.nix new file mode 100644 index 00000000..737bae80 --- /dev/null +++ b/nix/packages/default.nix @@ -0,0 +1,9 @@ +{ + imports = [ + ./continuwuity + ./jemalloc.nix + ./rocksdb + ./rust.nix + ./uwulib + ]; +} diff --git a/nix/packages/jemalloc.nix b/nix/packages/jemalloc.nix new file mode 100644 index 00000000..958ef5e8 --- /dev/null +++ b/nix/packages/jemalloc.nix @@ -0,0 +1,26 @@ +{ + perSystem = + { + pkgs, + ... + }: + { + # we disable some unused features here. The code won't compile without these + # + # > : Invalid conf pair: prof_active:false + # > error: test failed, to rerun pass `-p conduwuit --lib` + # > + # > Caused by: + # > process didn't exit successfully: `/build/source/target/release/deps/conduwuit-67fbd204f38e8c35` (signal: 11, SIGSEGV: invalid memory reference) + packages.rust-jemalloc-sys-unprefixed' = pkgs.rust-jemalloc-sys-unprefixed.overrideAttrs (old: { + configureFlags = + old.configureFlags + ++ + # we dont need docs + [ "--disable-doc" ] + ++ + # we dont need cxx/C++ integration + [ "--disable-cxx" ]; + }); + }; +} diff --git a/nix/packages/rocksdb/default.nix b/nix/packages/rocksdb/default.nix new file mode 100644 index 00000000..a5f241a9 --- /dev/null +++ b/nix/packages/rocksdb/default.nix @@ -0,0 +1,15 @@ +{ + perSystem = + { + self', + pkgs, + ... + }: + { + packages = { + rocksdb = pkgs.callPackage ./package.nix { + rust-jemalloc-sys-unprefixed = self'.packages.rust-jemalloc-sys-unprefixed'; + }; + }; + }; +} diff --git a/nix/packages/rocksdb/package.nix b/nix/packages/rocksdb/package.nix new file mode 100644 index 00000000..40bd0935 --- /dev/null +++ b/nix/packages/rocksdb/package.nix @@ -0,0 +1,88 @@ +{ + lib, + stdenv, + + rocksdb, + liburing, + rust-jemalloc-sys-unprefixed, + + enableJemalloc ? false, + enableLiburing ? false, + + fetchFromGitea, + + ... +}: +let + notDarwin = !stdenv.hostPlatform.isDarwin; +in +(rocksdb.override { + # Override the liburing input for the build with our own so + # we have it built with the library flag + inherit liburing; + jemalloc = rust-jemalloc-sys-unprefixed; + + # rocksdb fails to build with prefixed jemalloc, which is required on + # darwin due to [1]. In this case, fall back to building rocksdb with + # libc malloc. This should not cause conflicts, because all of the + # jemalloc symbols are prefixed. + # + # [1]: https://github.com/tikv/jemallocator/blob/ab0676d77e81268cd09b059260c75b38dbef2d51/jemalloc-sys/src/env.rs#L17 + enableJemalloc = enableJemalloc && notDarwin; + + # for some reason enableLiburing in nixpkgs rocksdb is default true + # which breaks Darwin entirely + enableLiburing = enableLiburing && notDarwin; +}).overrideAttrs + (old: { + src = fetchFromGitea { + domain = "forgejo.ellis.link"; + owner = "continuwuation"; + repo = "rocksdb"; + rev = "10.5.fb"; + sha256 = "sha256-X4ApGLkHF9ceBtBg77dimEpu720I79ffLoyPa8JMHaU="; + }; + version = "10.5.fb"; + cmakeFlags = + lib.subtractLists (builtins.map (flag: lib.cmakeBool flag true) [ + # No real reason to have snappy or zlib, no one uses this + "WITH_SNAPPY" + "ZLIB" + "WITH_ZLIB" + # We don't need to use ldb or sst_dump (core_tools) + "WITH_CORE_TOOLS" + # We don't need to build rocksdb tests + "WITH_TESTS" + # We use rust-rocksdb via C interface and don't need C++ RTTI + "USE_RTTI" + # This doesn't exist in RocksDB, and USE_SSE is deprecated for + # PORTABLE=$(march) + "FORCE_SSE42" + ]) old.cmakeFlags + ++ (builtins.map (flag: lib.cmakeBool flag false) [ + # No real reason to have snappy, no one uses this + "WITH_SNAPPY" + "ZLIB" + "WITH_ZLIB" + # We don't need to use ldb or sst_dump (core_tools) + "WITH_CORE_TOOLS" + # We don't need trace tools + "WITH_TRACE_TOOLS" + # We don't need to build rocksdb tests + "WITH_TESTS" + # We use rust-rocksdb via C interface and don't need C++ RTTI + "USE_RTTI" + ]); + + enableLiburing = enableLiburing && notDarwin; + + # outputs has "tools" which we don't need or use + outputs = [ "out" ]; + + # preInstall hooks has stuff for messing with ldb/sst_dump which we don't need or use + preInstall = ""; + + # We have this already at https://forgejo.ellis.link/continuwuation/rocksdb/commit/a935c0273e1ba44eacf88ce3685a9b9831486155 + # Unsetting `patches` so we don't have to revert it and make this nix exclusive + patches = [ ]; + }) diff --git a/nix/packages/rust.nix b/nix/packages/rust.nix new file mode 100644 index 00000000..e9b5d6ec --- /dev/null +++ b/nix/packages/rust.nix @@ -0,0 +1,31 @@ +{ inputs, ... }: +{ + perSystem = + { + system, + ... + }: + { + packages = + let + fnx = inputs.fenix.packages.${system}; + + stable = fnx.fromToolchainFile { + file = inputs.self + "/rust-toolchain.toml"; + + # See also `rust-toolchain.toml` + sha256 = "sha256-+9FmLhAOezBZCOziO0Qct1NOrfpjNsXxc/8I0c7BdKE="; + }; + in + { + # used for building nix stuff (doesn't include rustfmt overhead) + build-toolchain = stable; + # used for dev shells + dev-toolchain = fnx.combine [ + stable + # use the nightly rustfmt because we use nightly features + fnx.complete.rustfmt + ]; + }; + }; +} diff --git a/nix/packages/uwulib/build.nix b/nix/packages/uwulib/build.nix new file mode 100644 index 00000000..7d72b0b4 --- /dev/null +++ b/nix/packages/uwulib/build.nix @@ -0,0 +1,108 @@ +args@{ pkgs, inputs, ... }: +let + inherit (pkgs) lib; + uwuenv = import ./environment.nix args; + selfpkgs = inputs.self.packages.${pkgs.stdenv.system}; +in +rec { + # basic, very minimal instance of the crane library with a minimal rust toolchain + craneLib = (inputs.crane.mkLib pkgs).overrideToolchain (_: selfpkgs.build-toolchain); + # the checks require more rust toolchain components, hence we have this separate instance of the crane library + craneLibForChecks = (inputs.crane.mkLib pkgs).overrideToolchain (_: selfpkgs.dev-toolchain); + + # meta information (name, version, etc) of the rust crate based on the Cargo.toml + crateInfo = craneLib.crateNameFromCargoToml { cargoToml = "${inputs.self}/Cargo.toml"; }; + + src = + let + # see https://crane.dev/API.html#cranelibfiltercargosources + # + # we need to keep the `web` directory which would be filtered out by the regular source filtering function + # + # https://crane.dev/API.html#cranelibcleancargosource + isWebTemplate = path: _type: builtins.match ".*src/web.*" path != null; + isRust = craneLib.filterCargoSources; + isNix = path: _type: builtins.match ".+/nix.*" path != null; + webOrRustNotNix = p: t: !(isNix p t) && (isWebTemplate p t || isRust p t); + in + lib.cleanSourceWith { + src = inputs.self; + filter = webOrRustNotNix; + name = "source"; + }; + + # common attrs that are shared between building continuwuity's deps and the package itself + commonAttrs = + { + profile ? "dev", + ... + }: + { + inherit (crateInfo) + pname + version + ; + inherit src; + + # this prevents unnecessary rebuilds + strictDeps = true; + + dontStrip = profile == "dev" || profile == "test"; + dontPatchELF = profile == "dev" || profile == "test"; + + doCheck = true; + + nativeBuildInputs = [ + # bindgen needs the build platform's libclang. Apparently due to "splicing + # weirdness", pkgs.rustPlatform.bindgenHook on its own doesn't quite do the + # right thing here. + pkgs.rustPlatform.bindgenHook + ]; + }; + + makeRocksDBEnv = + { rocksdb }: + { + ROCKSDB_INCLUDE_DIR = "${rocksdb}/include"; + ROCKSDB_LIB_DIR = "${rocksdb}/lib"; + }; + + # function that builds the continuwuity dependencies derivation + buildDeps = + { + rocksdb, + features, + commonAttrsArgs, + }: + craneLib.buildDepsOnly ( + (commonAttrs commonAttrsArgs) + // { + env = uwuenv.buildDepsOnlyEnv // (makeRocksDBEnv { inherit rocksdb; }); + inherit (features) cargoExtraArgs; + } + + ); + + # function that builds the continuwuity package + buildPackage = + { + deps, + rocksdb, + features, + commonAttrsArgs, + }: + let + rocksdbEnv = makeRocksDBEnv { inherit rocksdb; }; + in + craneLib.buildPackage ( + (commonAttrs commonAttrsArgs) + // { + cargoArtifacts = deps; + doCheck = true; + env = uwuenv.buildPackageEnv // rocksdbEnv; + passthru.env = uwuenv.buildPackageEnv // rocksdbEnv; + meta.mainProgram = crateInfo.pname; + inherit (features) cargoExtraArgs; + } + ); +} diff --git a/nix/packages/uwulib/default.nix b/nix/packages/uwulib/default.nix new file mode 100644 index 00000000..0510fe0b --- /dev/null +++ b/nix/packages/uwulib/default.nix @@ -0,0 +1,10 @@ +{ inputs, ... }: +{ + flake.uwulib = { + init = pkgs: { + features = import ./features.nix { inherit pkgs inputs; }; + environment = import ./environment.nix { inherit pkgs inputs; }; + build = import ./build.nix { inherit pkgs inputs; }; + }; + }; +} diff --git a/nix/packages/uwulib/environment.nix b/nix/packages/uwulib/environment.nix new file mode 100644 index 00000000..ec140cf9 --- /dev/null +++ b/nix/packages/uwulib/environment.nix @@ -0,0 +1,18 @@ +args@{ pkgs, inputs, ... }: +let + uwubuild = import ./build.nix args; +in +rec { + buildDepsOnlyEnv = { + # https://crane.dev/faq/rebuilds-bindgen.html + NIX_OUTPATH_USED_AS_RANDOM_SEED = "aaaaaaaaaa"; + CARGO_PROFILE = "release"; + } + // uwubuild.craneLib.mkCrossToolchainEnv (p: pkgs.clangStdenv); + + buildPackageEnv = { + GIT_COMMIT_HASH = inputs.self.rev or inputs.self.dirtyRev or ""; + GIT_COMMIT_HASH_SHORT = inputs.self.shortRev or inputs.self.dirtyShortRev or ""; + } + // buildDepsOnlyEnv; +} diff --git a/nix/packages/uwulib/features.nix b/nix/packages/uwulib/features.nix new file mode 100644 index 00000000..d85985e8 --- /dev/null +++ b/nix/packages/uwulib/features.nix @@ -0,0 +1,77 @@ +{ pkgs, inputs, ... }: +let + inherit (pkgs) lib; +in +rec { + defaultDisabledFeatures = [ + # dont include experimental features + "experimental" + # jemalloc profiling/stats features are expensive and shouldn't + # be expected on non-debug builds. + "jemalloc_prof" + "jemalloc_stats" + # this is non-functional on nix for some reason + "hardened_malloc" + # conduwuit_mods is a development-only hot reload feature + "conduwuit_mods" + # we don't want to enable this feature set by default but be more specific about it + "full" + ]; + # We perform default-feature unification in nix, because some of the dependencies + # on the nix side depend on feature values. + calcFeatures = + { + tomlPath ? "${inputs.self}/src/main", + # either a list of feature names or a string "all" which enables all non-default features + enabledFeatures ? [ ], + disabledFeatures ? defaultDisabledFeatures, + default_features ? true, + disable_release_max_log_level ? false, + }: + let + # simple helper to get the contents of a Cargo.toml file in a nix format + getToml = path: lib.importTOML "${path}/Cargo.toml"; + + # get all the features except for the default features + allFeatures = lib.pipe tomlPath [ + getToml + (manifest: manifest.features) + lib.attrNames + (lib.remove "default") + ]; + + # get just the default enabled features + allDefaultFeatures = lib.pipe tomlPath [ + getToml + (manifest: manifest.features.default) + ]; + + # depending on the value of enabledFeatures choose just a set or all non-default features + # + # - [ list of features ] -> choose exactly the features listed + # - "all" -> choose all non-default features + additionalFeatures = if enabledFeatures == "all" then allFeatures else enabledFeatures; + + # unification with default features (if enabled) + features = lib.unique (additionalFeatures ++ lib.optionals default_features allDefaultFeatures); + + # prepare the features that are subtracted from the set + disabledFeatures' = + disabledFeatures ++ lib.optionals disable_release_max_log_level [ "release_max_log_level" ]; + + # construct the final feature set + finalFeatures = lib.subtractLists disabledFeatures' features; + in + { + # final feature set, useful for querying it + features = finalFeatures; + + # crane flag with the relevant features + cargoExtraArgs = builtins.concatStringsSep " " [ + "--no-default-features" + "--locked" + (lib.optionalString (finalFeatures != [ ]) "--features") + (builtins.concatStringsSep "," finalFeatures) + ]; + }; +} diff --git a/nix/shells/default.nix b/nix/shells/default.nix new file mode 100644 index 00000000..1524441d --- /dev/null +++ b/nix/shells/default.nix @@ -0,0 +1,25 @@ +{ inputs, ... }: +{ + perSystem = + { + self', + lib, + pkgs, + ... + }: + let + uwulib = inputs.self.uwulib.init pkgs; + in + { + # basic nix shell containing all things necessary to build continuwuity in all flavors manually (on x86_64-linux) + devShells.default = uwulib.build.craneLib.devShell { + packages = [ + pkgs.pkg-config + pkgs.liburing + self'.packages.rust-jemalloc-sys-unprefixed' + self'.packages.rocksdbAllFeatures + ]; + env.LIBCLANG_PATH = lib.makeLibraryPath [ pkgs.llvmPackages.libclang.lib ]; + }; + }; +} diff --git a/nix/tests/default.nix b/nix/tests/default.nix new file mode 100644 index 00000000..9855dcc1 --- /dev/null +++ b/nix/tests/default.nix @@ -0,0 +1,124 @@ +{ + perSystem = + { + self', + lib, + pkgs, + ... + }: + { + # run some nixos tests as checks + checks = lib.pipe self'.packages [ + # we take all packages (names) + builtins.attrNames + # we filter out all packages that end with `-bin` (which we are interested in for testing) + (builtins.filter (lib.hasSuffix "-bin")) + # for each of these binaries we built the basic nixos test + # + # this test was initially yoinked from + # + # https://github.com/NixOS/nixpkgs/blob/960ce26339661b1b69c6f12b9063ca51b688615f/nixos/tests/matrix/continuwuity.nix + (builtins.map (name: { + name = "test-${name}"; + value = pkgs.testers.runNixOSTest { + inherit name; + + nodes = { + continuwuity = { + services.matrix-continuwuity = { + enable = true; + package = self'.packages.${name}; + settings.global = { + server_name = name; + address = [ "0.0.0.0" ]; + allow_registration = true; + yes_i_am_very_very_sure_i_want_an_open_registration_server_prone_to_abuse = true; + }; + extraEnvironment.RUST_BACKTRACE = "yes"; + }; + networking.firewall.allowedTCPPorts = [ 6167 ]; + }; + client = + { pkgs, ... }: + { + environment.systemPackages = [ + (pkgs.writers.writePython3Bin "do_test" { libraries = [ pkgs.python3Packages.matrix-nio ]; } '' + import asyncio + import nio + + + async def main() -> None: + # Connect to continuwuity + client = nio.AsyncClient("http://continuwuity:6167", "alice") + + # Register as user alice + response = await client.register("alice", "my-secret-password") + + # Log in as user alice + response = await client.login("my-secret-password") + + # Create a new room + response = await client.room_create(federate=False) + print("Matrix room create response:", response) + assert isinstance(response, nio.RoomCreateResponse) + room_id = response.room_id + + # Join the room + response = await client.join(room_id) + print("Matrix join response:", response) + assert isinstance(response, nio.JoinResponse) + + # Send a message to the room + response = await client.room_send( + room_id=room_id, + message_type="m.room.message", + content={ + "msgtype": "m.text", + "body": "Hello continuwuity!" + } + ) + print("Matrix room send response:", response) + assert isinstance(response, nio.RoomSendResponse) + + # Sync responses + response = await client.sync(timeout=30000) + print("Matrix sync response:", response) + assert isinstance(response, nio.SyncResponse) + + # Check the message was received by continuwuity + last_message = response.rooms.join[room_id].timeline.events[-1].body + assert last_message == "Hello continuwuity!" + + # Leave the room + response = await client.room_leave(room_id) + print("Matrix room leave response:", response) + assert isinstance(response, nio.RoomLeaveResponse) + + # Close the client + await client.close() + + + if __name__ == "__main__": + asyncio.run(main()) + '') + ]; + }; + }; + + testScript = '' + start_all() + + with subtest("start continuwuity"): + continuwuity.wait_for_unit("continuwuity.service") + continuwuity.wait_for_open_port(6167) + + with subtest("ensure messages can be exchanged"): + client.succeed("do_test >&2") + ''; + + }; + })) + builtins.listToAttrs + ]; + }; +} diff --git a/pkg/nix/pkgs/main/cross-compilation-env.nix b/pkg/nix/pkgs/main/cross-compilation-env.nix deleted file mode 100644 index 3e993bba..00000000 --- a/pkg/nix/pkgs/main/cross-compilation-env.nix +++ /dev/null @@ -1,83 +0,0 @@ -{ lib -, pkgsBuildHost -, rust -, stdenv -}: - -lib.optionalAttrs stdenv.hostPlatform.isStatic - { - ROCKSDB_STATIC = ""; - } -// -{ - CARGO_BUILD_RUSTFLAGS = - lib.concatStringsSep - " " - (lib.optionals - stdenv.hostPlatform.isStatic - [ "-C" "relocation-model=static" ] - ++ lib.optionals - (stdenv.buildPlatform.config != stdenv.hostPlatform.config) - [ - "-l" - "c" - - "-l" - "stdc++" - - "-L" - "${stdenv.cc.cc.lib}/${stdenv.hostPlatform.config}/lib" - ] - ); -} - - # What follows is stolen from [here][0]. Its purpose is to properly - # configure compilers and linkers for various stages of the build, and - # even covers the case of build scripts that need native code compiled and - # run on the build platform (I think). - # - # [0]: https://github.com/NixOS/nixpkgs/blob/nixpkgs-unstable/pkgs/build-support/rust/lib/default.nix#L48-L68 - // -( - let - inherit (rust.lib) envVars; - in - lib.optionalAttrs - (stdenv.targetPlatform.rust.rustcTarget - != stdenv.hostPlatform.rust.rustcTarget) - ( - let - inherit (stdenv.targetPlatform.rust) cargoEnvVarTarget; - in - { - "CC_${cargoEnvVarTarget}" = envVars.ccForTarget; - "CXX_${cargoEnvVarTarget}" = envVars.cxxForTarget; - "CARGO_TARGET_${cargoEnvVarTarget}_LINKER" = envVars.ccForTarget; - } - ) - // - ( - let - inherit (stdenv.hostPlatform.rust) cargoEnvVarTarget rustcTarget; - in - { - "CC_${cargoEnvVarTarget}" = envVars.ccForHost; - "CXX_${cargoEnvVarTarget}" = envVars.cxxForHost; - "CARGO_TARGET_${cargoEnvVarTarget}_LINKER" = envVars.ccForHost; - CARGO_BUILD_TARGET = rustcTarget; - } - ) - // - ( - let - inherit (stdenv.buildPlatform.rust) cargoEnvVarTarget; - in - { - "CC_${cargoEnvVarTarget}" = envVars.ccForBuild; - "CXX_${cargoEnvVarTarget}" = envVars.cxxForBuild; - "CARGO_TARGET_${cargoEnvVarTarget}_LINKER" = envVars.ccForBuild; - HOST_CC = "${pkgsBuildHost.stdenv.cc}/bin/cc"; - HOST_CXX = "${pkgsBuildHost.stdenv.cc}/bin/c++"; - } - ) -) diff --git a/pkg/nix/pkgs/main/default.nix b/pkg/nix/pkgs/main/default.nix deleted file mode 100644 index 781530ba..00000000 --- a/pkg/nix/pkgs/main/default.nix +++ /dev/null @@ -1,224 +0,0 @@ -# Dependencies (keep sorted) -{ craneLib -, inputs -, jq -, lib -, libiconv -, liburing -, pkgsBuildHost -, rocksdb -, removeReferencesTo -, rust -, rust-jemalloc-sys -, stdenv - - # Options (keep sorted) -, all_features ? false -, default_features ? true - # default list of disabled features -, disable_features ? [ - # dont include experimental features - "experimental" - # jemalloc profiling/stats features are expensive and shouldn't - # be expected on non-debug builds. - "jemalloc_prof" - "jemalloc_stats" - # this is non-functional on nix for some reason - "hardened_malloc" - # conduwuit_mods is a development-only hot reload feature - "conduwuit_mods" - ] -, disable_release_max_log_level ? false -, features ? [ ] -, profile ? "release" - # rocksdb compiled with -march=haswell and target-cpu=haswell rustflag - # haswell is pretty much any x86 cpu made in the last 12 years, and - # supports modern CPU extensions that rocksdb can make use of. - # disable if trying to make a portable x86_64 build for very old hardware -, x86_64_haswell_target_optimised ? false -}: - -let - # We perform default-feature unification in nix, because some of the dependencies - # on the nix side depend on feature values. - crateFeatures = path: - let manifest = lib.importTOML "${path}/Cargo.toml"; in - lib.remove "default" (lib.attrNames manifest.features); - crateDefaultFeatures = path: - (lib.importTOML "${path}/Cargo.toml").features.default; - allDefaultFeatures = crateDefaultFeatures "${inputs.self}/src/main"; - allFeatures = crateFeatures "${inputs.self}/src/main"; - features' = lib.unique - (features ++ - lib.optionals default_features allDefaultFeatures ++ - lib.optionals all_features allFeatures); - disable_features' = disable_features ++ lib.optionals disable_release_max_log_level [ "release_max_log_level" ]; - features'' = lib.subtractLists disable_features' features'; - - featureEnabled = feature: builtins.elem feature features''; - - enableLiburing = featureEnabled "io_uring" && !stdenv.hostPlatform.isDarwin; - - # This derivation will set the JEMALLOC_OVERRIDE variable, causing the - # tikv-jemalloc-sys crate to use the nixpkgs jemalloc instead of building it's - # own. In order for this to work, we need to set flags on the build that match - # whatever flags tikv-jemalloc-sys was going to use. These are dependent on - # which features we enable in tikv-jemalloc-sys. - rust-jemalloc-sys' = (rust-jemalloc-sys.override { - # tikv-jemalloc-sys/unprefixed_malloc_on_supported_platforms feature - unprefixed = true; - }).overrideAttrs (old: { - configureFlags = old.configureFlags ++ - # we dont need docs - [ "--disable-doc" ] ++ - # we dont need cxx/C++ integration - [ "--disable-cxx" ] ++ - # tikv-jemalloc-sys/profiling feature - lib.optional (featureEnabled "jemalloc_prof") "--enable-prof" ++ - # tikv-jemalloc-sys/stats feature - (if (featureEnabled "jemalloc_stats") then [ "--enable-stats" ] else [ "--disable-stats" ]); - }); - - buildDepsOnlyEnv = - let - rocksdb' = (rocksdb.override { - jemalloc = lib.optional (featureEnabled "jemalloc") rust-jemalloc-sys'; - # rocksdb fails to build with prefixed jemalloc, which is required on - # darwin due to [1]. In this case, fall back to building rocksdb with - # libc malloc. This should not cause conflicts, because all of the - # jemalloc symbols are prefixed. - # - # [1]: https://github.com/tikv/jemallocator/blob/ab0676d77e81268cd09b059260c75b38dbef2d51/jemalloc-sys/src/env.rs#L17 - enableJemalloc = featureEnabled "jemalloc" && !stdenv.hostPlatform.isDarwin; - - # for some reason enableLiburing in nixpkgs rocksdb is default true - # which breaks Darwin entirely - inherit enableLiburing; - }).overrideAttrs (old: { - inherit enableLiburing; - cmakeFlags = (if x86_64_haswell_target_optimised then - (lib.subtractLists [ - # dont make a portable build if x86_64_haswell_target_optimised is enabled - "-DPORTABLE=1" - ] - old.cmakeFlags - ++ [ "-DPORTABLE=haswell" ]) else [ "-DPORTABLE=1" ] - ) - ++ old.cmakeFlags; - - # outputs has "tools" which we dont need or use - outputs = [ "out" ]; - - # preInstall hooks has stuff for messing with ldb/sst_dump which we dont need or use - preInstall = ""; - }); - in - { - # https://crane.dev/faq/rebuilds-bindgen.html - NIX_OUTPATH_USED_AS_RANDOM_SEED = "aaaaaaaaaa"; - - CARGO_PROFILE = profile; - ROCKSDB_INCLUDE_DIR = "${rocksdb'}/include"; - ROCKSDB_LIB_DIR = "${rocksdb'}/lib"; - } - // - (import ./cross-compilation-env.nix { - # Keep sorted - inherit - lib - pkgsBuildHost - rust - stdenv; - }); - - buildPackageEnv = { - GIT_COMMIT_HASH = inputs.self.rev or inputs.self.dirtyRev or ""; - GIT_COMMIT_HASH_SHORT = inputs.self.shortRev or inputs.self.dirtyShortRev or ""; - } // buildDepsOnlyEnv // { - # Only needed in static stdenv because these are transitive dependencies of rocksdb - CARGO_BUILD_RUSTFLAGS = buildDepsOnlyEnv.CARGO_BUILD_RUSTFLAGS - + lib.optionalString (enableLiburing && stdenv.hostPlatform.isStatic) - " -L${lib.getLib liburing}/lib -luring" - + lib.optionalString x86_64_haswell_target_optimised - " -Ctarget-cpu=haswell"; - }; - - - - commonAttrs = { - inherit - (craneLib.crateNameFromCargoToml { - cargoToml = "${inputs.self}/Cargo.toml"; - }) - pname - version; - - src = let filter = inputs.nix-filter.lib; in filter { - root = inputs.self; - - # Keep sorted - include = [ - ".cargo" - "Cargo.lock" - "Cargo.toml" - "src" - "xtask" - ]; - }; - - doCheck = true; - - cargoExtraArgs = "--no-default-features --locked " - + lib.optionalString - (features'' != [ ]) - "--features " + (builtins.concatStringsSep "," features''); - - dontStrip = profile == "dev" || profile == "test"; - dontPatchELF = profile == "dev" || profile == "test"; - - buildInputs = lib.optional (featureEnabled "jemalloc") rust-jemalloc-sys' - # needed to build Rust applications on macOS - ++ lib.optionals stdenv.hostPlatform.isDarwin [ - # https://github.com/NixOS/nixpkgs/issues/206242 - # ld: library not found for -liconv - libiconv - # https://stackoverflow.com/questions/69869574/properly-adding-darwin-apple-sdk-to-a-nix-shell - # https://discourse.nixos.org/t/compile-a-rust-binary-on-macos-dbcrossbar/8612 - pkgsBuildHost.darwin.apple_sdk.frameworks.Security - ]; - - nativeBuildInputs = [ - # bindgen needs the build platform's libclang. Apparently due to "splicing - # weirdness", pkgs.rustPlatform.bindgenHook on its own doesn't quite do the - # right thing here. - pkgsBuildHost.rustPlatform.bindgenHook - - # We don't actually depend on `jq`, but crane's `buildPackage` does, but - # its `buildDepsOnly` doesn't. This causes those two derivations to have - # differing values for `NIX_CFLAGS_COMPILE`, which contributes to spurious - # rebuilds of bindgen and its depedents. - jq - ]; - }; -in - -craneLib.buildPackage (commonAttrs // { - cargoArtifacts = craneLib.buildDepsOnly (commonAttrs // { - env = buildDepsOnlyEnv; - }); - - doCheck = true; - - cargoExtraArgs = "--no-default-features --locked " - + lib.optionalString - (features'' != [ ]) - "--features " + (builtins.concatStringsSep "," features''); - - env = buildPackageEnv; - - passthru = { - env = buildPackageEnv; - }; - - meta.mainProgram = commonAttrs.pname; -})