rust: update cargo builder to fetch registry dynamically
The biggest benefit is that we no longer have to update the registry package. This means that just about any cargo package can be built by nix. No longer does `cargo update` need to be feared because it will update to packages newer then what is available in nixpkgs. Instead of fetching the cargo registry this bundles all the source code into a "vendor/" folder. This also uses the new --frozen and --locked flags which is nice. Currently cargo-vendor only provides binaries for Linux and macOS 64-bit. This can be solved by building it for the other architectures and uploading it somewhere (like the NixOS cache). This also has the downside that it requires a change to everyone's deps hash. And if the old one is used because it was cached it will fail to build as it will attempt to use the old version. For this reason the attribute has been renamed to `cargoSha256`. Authors: * Kevin Cox <kevincox@kevincox.ca> * Jörg Thalheim <Mic92@users.noreply.github.com> * zimbatm <zimbatm@zimbatm.com>
This commit is contained in:
Kevin Cox
@@ -0,0 +1,32 @@
|
||||
{ fetchurl, stdenv }:
|
||||
let
|
||||
inherit (stdenv) system;
|
||||
|
||||
version = "0.1.12";
|
||||
|
||||
hashes = {
|
||||
x86_64-linux = "1hxlavcxy374yypfamlkygjg662lhll8j434qcvdawkvlidg5ii5";
|
||||
x86_64-darwin = "1jkvhh710gwjnnjx59kaplx2ncfvkx9agfa76rr94sbjqq4igddm";
|
||||
};
|
||||
hash = hashes. ${system} or (throw "missing bootstrap hash for platform ${system}");
|
||||
|
||||
platforms = {
|
||||
x86_64-linux = "x86_64-unknown-linux-musl";
|
||||
x86_64-darwin = "x86_64-apple-darwin";
|
||||
};
|
||||
platform = platforms . ${system};
|
||||
|
||||
in stdenv.mkDerivation {
|
||||
name = "cargo-vendor-${version}";
|
||||
|
||||
src = fetchurl {
|
||||
url = "https://github.com/alexcrichton/cargo-vendor/releases/download/${version}/cargo-vendor-${version}-${platform}.tar.gz";
|
||||
sha256 = hash;
|
||||
};
|
||||
|
||||
phases = "unpackPhase installPhase";
|
||||
|
||||
installPhase = ''
|
||||
install -Dm755 cargo-vendor $out/bin/cargo-vendor
|
||||
'';
|
||||
}
|
||||
@@ -1,10 +1,14 @@
|
||||
{ stdenv, callPackage, path, cacert, git, rust, rustRegistry }:
|
||||
|
||||
{ fetchurl, stdenv, path, cacert, git, rust }:
|
||||
let
|
||||
rustRegistry' = rustRegistry;
|
||||
cargoVendor = import ./cargo-vendor.nix {
|
||||
inherit fetchurl stdenv;
|
||||
};
|
||||
|
||||
fetchcargo = import ./fetchcargo.nix {
|
||||
inherit stdenv cacert git rust cargoVendor;
|
||||
};
|
||||
in
|
||||
{ name, depsSha256
|
||||
, rustRegistry ? rustRegistry'
|
||||
{ name, cargoSha256
|
||||
, src ? null
|
||||
, srcs ? null
|
||||
, sourceRoot ? null
|
||||
@@ -18,17 +22,13 @@ in
|
||||
let
|
||||
lib = stdenv.lib;
|
||||
|
||||
fetchDeps = import ./fetchcargo.nix {
|
||||
inherit stdenv cacert git rust rustRegistry;
|
||||
};
|
||||
|
||||
cargoDeps = fetchDeps {
|
||||
cargoDeps = fetchcargo {
|
||||
inherit name src srcs sourceRoot cargoUpdateHook;
|
||||
sha256 = depsSha256;
|
||||
sha256 = cargoSha256;
|
||||
};
|
||||
|
||||
in stdenv.mkDerivation (args // {
|
||||
inherit cargoDeps rustRegistry;
|
||||
inherit cargoDeps;
|
||||
|
||||
patchRegistryDeps = ./patch-registry-deps;
|
||||
|
||||
@@ -43,71 +43,24 @@ in stdenv.mkDerivation (args // {
|
||||
postUnpack = ''
|
||||
eval "$cargoDepsHook"
|
||||
|
||||
echo "Using cargo deps from $cargoDeps"
|
||||
mkdir .cargo
|
||||
cat >.cargo/config <<-EOF
|
||||
[source.crates-io]
|
||||
registry = 'https://github.com/rust-lang/crates.io-index'
|
||||
replace-with = 'vendored-sources'
|
||||
|
||||
cp -a "$cargoDeps" deps
|
||||
chmod +w deps -R
|
||||
|
||||
# It's OK to use /dev/null as the URL because by the time we do this, cargo
|
||||
# won't attempt to update the registry anymore, so the URL is more or less
|
||||
# irrelevant
|
||||
|
||||
cat <<EOF > deps/config
|
||||
[registry]
|
||||
index = "file:///dev/null"
|
||||
[source.vendored-sources]
|
||||
directory = '$cargoDeps'
|
||||
EOF
|
||||
|
||||
export CARGO_HOME="$(realpath deps)"
|
||||
export RUST_LOG=${logLevel}
|
||||
export SSL_CERT_FILE=${cacert}/etc/ssl/certs/ca-bundle.crt
|
||||
|
||||
# Let's find out which $indexHash cargo uses for file:///dev/null
|
||||
(cd $sourceRoot && cargo fetch &>/dev/null) || true
|
||||
cd deps
|
||||
indexHash="$(basename $(echo registry/index/*))"
|
||||
|
||||
echo "Using indexHash '$indexHash'"
|
||||
|
||||
rm -rf -- "registry/cache/$indexHash" \
|
||||
"registry/index/$indexHash"
|
||||
|
||||
mv registry/cache/HASH "registry/cache/$indexHash"
|
||||
|
||||
echo "Using rust registry from $rustRegistry"
|
||||
ln -s "$rustRegistry" "registry/index/$indexHash"
|
||||
|
||||
# Retrieved the Cargo.lock file which we saved during the fetch
|
||||
cd ..
|
||||
mv deps/Cargo.lock $sourceRoot/
|
||||
|
||||
(
|
||||
cd $sourceRoot
|
||||
|
||||
cargo fetch
|
||||
cargo clean
|
||||
)
|
||||
'' + (args.postUnpack or "");
|
||||
|
||||
prePatch = ''
|
||||
# Patch registry dependencies, using the scripts in $patchRegistryDeps
|
||||
(
|
||||
set -euo pipefail
|
||||
|
||||
cd $NIX_BUILD_TOP/deps/registry/src/*
|
||||
|
||||
for script in $patchRegistryDeps/*; do
|
||||
# Run in a subshell so that directory changes and shell options don't
|
||||
# affect any following commands
|
||||
|
||||
( . $script)
|
||||
done
|
||||
)
|
||||
'' + (args.prePatch or "");
|
||||
|
||||
buildPhase = with builtins; args.buildPhase or ''
|
||||
runHook preBuild
|
||||
echo "Running cargo build --release ${concatStringsSep " " cargoBuildFlags}"
|
||||
cargo build --release ${concatStringsSep " " cargoBuildFlags}
|
||||
cargo build --release --frozen ${concatStringsSep " " cargoBuildFlags}
|
||||
runHook postBuild
|
||||
'';
|
||||
|
||||
@@ -126,4 +79,6 @@ in stdenv.mkDerivation (args // {
|
||||
find target/release -maxdepth 1 -executable -exec cp "{}" $out/bin \;
|
||||
runHook postInstall
|
||||
'';
|
||||
|
||||
passthru = { inherit cargoDeps; };
|
||||
})
|
||||
|
||||
@@ -1,209 +0,0 @@
|
||||
# copied from libgit2 source code 'repo-template.h'
|
||||
makeGitTemplate() {
|
||||
local target="$1"
|
||||
mkdir -p -m777 "$target/info" "$target/pack" "$target/objects" "$target/refs"
|
||||
mkdir -p -m777 "$target/refs/heads" "$target/refs/tags" "$target/objects/info" "$target/objects/pack"
|
||||
cat <<'EOF' > "$target/description"
|
||||
Unnamed repository; edit this file 'description' to name the repository.
|
||||
EOF
|
||||
chmod 666 "$target/description"
|
||||
cat <<'EOF' > "$target/info/exclude"
|
||||
# File patterns to ignore; see `git help ignore` for more information.
|
||||
# Lines that start with '#' are comments.
|
||||
EOF
|
||||
}
|
||||
|
||||
fetchCargoDeps() {
|
||||
src=$(realpath $1)
|
||||
out=$(realpath $2)
|
||||
|
||||
echo "Fetching $src to $out"
|
||||
|
||||
mkdir $out
|
||||
|
||||
# Configure git template dir to make libgit2 more deterministic
|
||||
#
|
||||
# Without a template dir, libgit2 defaults to /usr/share/git-core/templates,
|
||||
# which can vary between systems if sandboxed builds aren't used.
|
||||
#
|
||||
# Note: we explictly set --tmpdir for mktemp here to make it more friendly
|
||||
# for nix-shell users, where $TMPDIR is not necessarily set to NIX_BUILD_TOP
|
||||
echo "Setting up git templatedir"
|
||||
export GIT_TEMPLATE_DIR="$(mktemp -d --tmpdir=$NIX_BUILD_TOP git-template.XXX)"
|
||||
makeGitTemplate "$GIT_TEMPLATE_DIR"
|
||||
export XDG_CONFIG_HOME="$(mktemp -d --tmpdir=$NIX_BUILD_TOP home.XXX)"
|
||||
mkdir -p $XDG_CONFIG_HOME/git
|
||||
cat <<EOF > $XDG_CONFIG_HOME/git/config
|
||||
[init]
|
||||
templatedir = $GIT_TEMPLATE_DIR
|
||||
EOF
|
||||
|
||||
# Configure cargo to fetch from a local copy of the crates.io registry
|
||||
|
||||
echo "Using rust registry from $rustRegistry"
|
||||
|
||||
cat <<EOF > $out/config
|
||||
[registry]
|
||||
index = "file://$rustRegistry"
|
||||
EOF
|
||||
|
||||
export CARGO_HOME=$out
|
||||
cd $src
|
||||
|
||||
if [[ ! -f Cargo.lock ]]; then
|
||||
echo
|
||||
echo "ERROR: The Cargo.lock file doesn't exist"
|
||||
echo
|
||||
echo "Cargo.lock is needed to make sure that depsSha256 doesn't change"
|
||||
echo "when the registry is updated."
|
||||
echo
|
||||
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# We need to do the following string replacement so that 'cargo fetch'
|
||||
# doesn't ignore the versions specified in Cargo.lock
|
||||
substituteInPlace Cargo.lock \
|
||||
--replace "registry+https://github.com/rust-lang/crates.io-index" \
|
||||
"registry+file://$rustRegistry"
|
||||
|
||||
# Do any possible 'cargo update -p <pkgName> --precise <version>' ad-hoc updates
|
||||
eval "$cargoUpdateHook"
|
||||
|
||||
# Do the fetch
|
||||
cargo fetch --verbose
|
||||
|
||||
# Now that we have fetched everything, let's make the output deterministic
|
||||
|
||||
# Cargo uses the following directory structure for fetched data, where
|
||||
# $indexHash is a hash of the registry index URL:
|
||||
#
|
||||
#
|
||||
# /config:
|
||||
#
|
||||
# Cargo config file. We'll delete this because it's not deterministic,
|
||||
# and instead recreate it just before running 'cargo build'.
|
||||
#
|
||||
# /registry/cache/$indexHash/:
|
||||
#
|
||||
# This is where tarballs of registry package dependencies are kept
|
||||
# We'll need to keep this, but make sure $indexHash is a fixed name.
|
||||
#
|
||||
# /registry/index/$indexHash/:
|
||||
#
|
||||
# A copy of the registry index is kept here. We can delete this, and
|
||||
# instead, just before running 'cargo build', we'll symlink this
|
||||
# directory to our static copy of the registry in the Nix store.
|
||||
#
|
||||
# /registry/src/$indexHash/{pkgName-pkgVersion}/:
|
||||
#
|
||||
# Here cargo keeps extracted sources of the cached tarballs.
|
||||
# We'll just delete this because cargo will re-populate them from the
|
||||
# tarballs.
|
||||
#
|
||||
# /git/db/{domain-hash}/:
|
||||
#
|
||||
# Here cargo keeps the `.git` directories of git dependencies.
|
||||
# We'll need to keep these, but make them deterministic.
|
||||
#
|
||||
# /git/checkouts/{domain-hash}/{branchName}/:
|
||||
#
|
||||
# Here cargo keeps checked-out sources of the git dependencies.
|
||||
# We can delete this, because cargo will re-populate them from the above
|
||||
# `.git` directories.
|
||||
#
|
||||
# Let's start
|
||||
|
||||
# Remove cargo config file, which points to the ever-changing registry
|
||||
rm $out/config
|
||||
|
||||
# Save the Cargo.lock file into the output, so that we don't have to do another
|
||||
# 'cargo update' during the build (which would try to access the network) for
|
||||
# any ad-hoc package updates (through $cargoUpdateHook).
|
||||
#
|
||||
# We need to replace the rustRegistry URL with something deterministic.
|
||||
# Since the URL won't actually be accessed anymore, it's fine to use /dev/null.
|
||||
|
||||
substituteInPlace Cargo.lock \
|
||||
--replace "registry+file://$rustRegistry" \
|
||||
"registry+file:///dev/null"
|
||||
mv Cargo.lock $out/
|
||||
|
||||
|
||||
# Let's replace $indexHash with something more deterministic
|
||||
mv $out/registry/cache/* $out/registry/cache/HASH
|
||||
|
||||
# The registry index changes all the time, so it's not deterministic
|
||||
# We'll symlink it before running 'cargo build'
|
||||
rm -rf $out/registry/index/*
|
||||
|
||||
# Make git DBs deterministic
|
||||
# TODO: test with git submodules
|
||||
[[ ! -d $out/git/checkouts ]] || (cd $out/git/checkouts && for name in *; do
|
||||
revs=""
|
||||
cd "$out/git/checkouts/$name"
|
||||
while read dir; do
|
||||
# extract substring: [dir = "./xxx/yyy/.git"] => [branch = "xxx/yyy"]
|
||||
branch="${dir:2:$((${#dir}-7))}"
|
||||
|
||||
cd "$out/git/checkouts/$name/$branch"
|
||||
rev="$(git rev-parse HEAD)"
|
||||
revs="$rev $revs"
|
||||
done < <(find . -type d -name .git -print)
|
||||
|
||||
echo "List of revs to keep for git db $name: $revs"
|
||||
|
||||
(
|
||||
# The following code was adapted from nix-prefetch-git
|
||||
|
||||
cd "$out/git/db/$name"
|
||||
|
||||
export GIT_DIR=.
|
||||
|
||||
# Remove all remote branches
|
||||
git branch -r | while read branch; do
|
||||
git branch -rD "$branch" >&2
|
||||
done
|
||||
|
||||
# Remove all tags
|
||||
git tag | while read tag; do
|
||||
git tag -d "$tag" >&2
|
||||
done
|
||||
|
||||
# Remove all local branches
|
||||
branchrefs=()
|
||||
eval "$(git for-each-ref --shell --format='branchrefs+=(%(refname))' refs/heads/)"
|
||||
|
||||
for branchref in "${branchrefs[@]}"; do
|
||||
git update-ref -d "$branchref" >&2
|
||||
done
|
||||
|
||||
# Create ad-hoc branches for the revs we need
|
||||
echo "$revs" | while read -d " " rev; do
|
||||
echo "Creating git branch b_$rev $rev"
|
||||
git branch b_$rev $rev
|
||||
done
|
||||
|
||||
# Remove files that have timestamps or otherwise have non-deterministic
|
||||
# properties.
|
||||
rm -rf logs/ hooks/ index FETCH_HEAD ORIG_HEAD refs/remotes/origin/HEAD config
|
||||
|
||||
# Do a full repack. Must run single-threaded, or else we lose determinism.
|
||||
git config pack.threads 1
|
||||
git repack -A -d -f
|
||||
rm -f config
|
||||
|
||||
# Garbage collect unreferenced objects.
|
||||
git gc --prune=all
|
||||
)
|
||||
done)
|
||||
|
||||
# Remove unneeded outputs
|
||||
[[ ! -d $out/registry/src ]] || rm -rf $out/registry/src
|
||||
[[ ! -d $out/git/checkouts ]] || rm -rf $out/git/checkouts
|
||||
|
||||
# XXX: provide some debugging output to see find out why we are seeing
|
||||
# sporadic hash mismatches
|
||||
find $out ! -type f
|
||||
find $out -type f -exec sha256sum {} +
|
||||
}
|
||||
@@ -1,19 +1,30 @@
|
||||
{ stdenv, cacert, git, rust, rustRegistry }:
|
||||
{ stdenv, cacert, git, rust, cargoVendor }:
|
||||
{ name ? "cargo-deps", src, srcs, sourceRoot, sha256, cargoUpdateHook ? "" }:
|
||||
|
||||
stdenv.mkDerivation {
|
||||
name = "${name}-fetch";
|
||||
buildInputs = [ rust.cargo rust.rustc git ];
|
||||
inherit src srcs sourceRoot rustRegistry cargoUpdateHook;
|
||||
name = "${name}-vendor";
|
||||
buildInputs = [ cacert cargoVendor git rust.cargo ];
|
||||
inherit src srcs sourceRoot;
|
||||
|
||||
phases = "unpackPhase installPhase";
|
||||
|
||||
installPhase = ''
|
||||
source ${./fetch-cargo-deps}
|
||||
if [[ ! -f Cargo.lock ]]; then
|
||||
echo
|
||||
echo "ERROR: The Cargo.lock file doesn't exist"
|
||||
echo
|
||||
echo "Cargo.lock is needed to make sure that cargoSha256 doesn't change"
|
||||
echo "when the registry is updated."
|
||||
echo
|
||||
|
||||
exit 1
|
||||
fi
|
||||
|
||||
export SSL_CERT_FILE=${cacert}/etc/ssl/certs/ca-bundle.crt
|
||||
export CARGO_HOME=$(mktemp -d cargo-home.XXX)
|
||||
|
||||
fetchCargoDeps . "$out"
|
||||
cargo vendor
|
||||
|
||||
cp -ar vendor $out
|
||||
'';
|
||||
|
||||
outputHashAlgo = "sha256";
|
||||
|
||||
Reference in New Issue
Block a user