Merge pull request #26805 from obsidiansystems/cross-elegant

Make cross compilation elegant
This commit is contained in:
John Ericson
2017-12-30 22:58:02 -05:00
committed by GitHub
88 changed files with 1059 additions and 699 deletions
+60 -16
View File
@@ -14,11 +14,27 @@ rec {
mkDerivation =
{ name ? ""
, nativeBuildInputs ? []
, buildInputs ? []
# These types of dependencies are all exhaustively documented in
# the "Specifying Dependencies" section of the "Standard
# Environment" chapter of the Nixpkgs manual.
, propagatedNativeBuildInputs ? []
, propagatedBuildInputs ? []
# TODO(@Ericson2314): Stop using legacy dep attribute names
# host offset -> target offset
, depsBuildBuild ? [] # -1 -> -1
, depsBuildBuildPropagated ? [] # -1 -> -1
, nativeBuildInputs ? [] # -1 -> 0 N.B. Legacy name
, propagatedNativeBuildInputs ? [] # -1 -> 0 N.B. Legacy name
, depsBuildTarget ? [] # -1 -> 1
, depsBuildTargetPropagated ? [] # -1 -> 1
, depsHostHost ? [] # 0 -> 0
, depsHostHostPropagated ? [] # 0 -> 0
, buildInputs ? [] # 0 -> 1 N.B. Legacy name
, propagatedBuildInputs ? [] # 0 -> 1 N.B. Legacy name
, depsTargetTarget ? [] # 1 -> 1
, depsTargetTargetPropagated ? [] # 1 -> 1
, configureFlags ? []
, # Target is not included by default because most programs don't care.
@@ -56,15 +72,35 @@ rec {
inherit erroneousHardeningFlags hardeningDisable hardeningEnable supportedHardeningFlags;
})
else let
dependencies = map lib.chooseDevOutputs [
(map (drv: drv.nativeDrv or drv) nativeBuildInputs
++ lib.optional separateDebugInfo ../../build-support/setup-hooks/separate-debug-info.sh
++ lib.optional stdenv.hostPlatform.isWindows ../../build-support/setup-hooks/win-dll-link.sh)
(map (drv: drv.crossDrv or drv) buildInputs)
dependencies = map (map lib.chooseDevOutputs) [
[
(map (drv: drv.__spliced.buildBuild or drv) depsBuildBuild)
(map (drv: drv.nativeDrv or drv) nativeBuildInputs
++ lib.optional separateDebugInfo ../../build-support/setup-hooks/separate-debug-info.sh
++ lib.optional stdenv.hostPlatform.isWindows ../../build-support/setup-hooks/win-dll-link.sh)
(map (drv: drv.__spliced.buildTarget or drv) depsBuildTarget)
]
[
(map (drv: drv.__spliced.hostHost or drv) depsHostHost)
(map (drv: drv.crossDrv or drv) buildInputs)
]
[
(map (drv: drv.__spliced.targetTarget or drv) depsTargetTarget)
]
];
propagatedDependencies = map lib.chooseDevOutputs [
(map (drv: drv.nativeDrv or drv) propagatedNativeBuildInputs)
(map (drv: drv.crossDrv or drv) propagatedBuildInputs)
propagatedDependencies = map (map lib.chooseDevOutputs) [
[
(map (drv: drv.__spliced.buildBuild or drv) depsBuildBuildPropagated)
(map (drv: drv.nativeDrv or drv) propagatedNativeBuildInputs)
(map (drv: drv.__spliced.buildTarget or drv) depsBuildTargetPropagated)
]
[
(map (drv: drv.__spliced.hostHost or drv) depsHostHostPropagated)
(map (drv: drv.crossDrv or drv) propagatedBuildInputs)
]
[
(map (drv: drv.__spliced.targetTarget or drv) depsTargetTargetPropagated)
]
];
outputs' =
@@ -105,11 +141,19 @@ rec {
userHook = config.stdenv.userHook or null;
__ignoreNulls = true;
nativeBuildInputs = lib.elemAt dependencies 0;
buildInputs = lib.elemAt dependencies 1;
depsBuildBuild = lib.elemAt (lib.elemAt dependencies 0) 0;
nativeBuildInputs = lib.elemAt (lib.elemAt dependencies 0) 1;
depsBuildTarget = lib.elemAt (lib.elemAt dependencies 0) 2;
depsHostBuild = lib.elemAt (lib.elemAt dependencies 1) 0;
buildInputs = lib.elemAt (lib.elemAt dependencies 1) 1;
depsTargetTarget = lib.elemAt (lib.elemAt dependencies 2) 0;
propagatedNativeBuildInputs = lib.elemAt propagatedDependencies 0;
propagatedBuildInputs = lib.elemAt propagatedDependencies 1;
depsBuildBuildPropagated = lib.elemAt (lib.elemAt propagatedDependencies 0) 0;
propagatedNativeBuildInputs = lib.elemAt (lib.elemAt propagatedDependencies 0) 1;
depsBuildTargetPropagated = lib.elemAt (lib.elemAt propagatedDependencies 0) 2;
depsHostBuildPropagated = lib.elemAt (lib.elemAt propagatedDependencies 1) 0;
propagatedBuildInputs = lib.elemAt (lib.elemAt propagatedDependencies 1) 1;
depsTargetTargetPropagated = lib.elemAt (lib.elemAt propagatedDependencies 2) 0;
# This parameter is sometimes a string, sometimes null, and sometimes a list, yuck
configureFlags = let inherit (lib) optional elem; in
+261 -75
View File
@@ -300,11 +300,83 @@ runHook preHook
runHook addInputsHook
# Recursively find all build inputs.
# Package accumulators
# shellcheck disable=SC2034
declare -a pkgsBuildBuild pkgsBuildHost pkgsBuildTarget
declare -a pkgsHostHost pkgsHostTarget
declare -a pkgsTargetTarget
declare -ra pkgBuildAccumVars=(pkgsBuildBuild pkgsBuildHost pkgsBuildTarget)
declare -ra pkgHostAccumVars=(pkgsHostHost pkgsHostTarget)
declare -ra pkgTargetAccumVars=(pkgsTargetTarget)
declare -ra pkgAccumVarVars=(pkgBuildAccumVars pkgHostAccumVars pkgTargetAccumVars)
# Hooks
declare -a envBuildBuildHooks envBuildHostHooks envBuildTargetHooks
declare -a envHostHostHooks envHostTargetHooks
declare -a envTargetTargetHooks
declare -ra pkgBuildHookVars=(envBuildBuildHook envBuildHostHook envBuildTargetHook)
declare -ra pkgHostHookVars=(envHostHostHook envHostTargetHook)
declare -ra pkgTargetHookVars=(envTargetTargetHook)
declare -ra pkgHookVarVars=(pkgBuildHookVars pkgHostHookVars pkgTargetHookVars)
# Add env hooks for all sorts of deps with the specified host offset.
addEnvHooks() {
local depHostOffset="$1"
shift
local pkgHookVarsSlice="${pkgHookVarVars[$depHostOffset + 1]}[@]"
local pkgHookVar
for pkgHookVar in "${!pkgHookVarsSlice}"; do
eval "${pkgHookVar}s"'+=("$@")'
done
}
# Propagated dep files
declare -ra propagatedBuildDepFiles=(
propagated-build-build-deps
propagated-native-build-inputs # Legacy name for back-compat
propagated-build-target-deps
)
declare -ra propagatedHostDepFiles=(
propagated-host-host-deps
propagated-build-inputs # Legacy name for back-compat
)
declare -ra propagatedTargetDepFiles=(
propagated-target-target-deps
)
declare -ra propagatedDepFilesVars=(
propagatedBuildDepFiles
propagatedHostDepFiles
propagatedTargetDepFiles
)
# Platform offsets: build = -1, host = 0, target = 1
declare -ra allPlatOffsets=(-1 0 1)
# Mutually-recursively find all build inputs. See the dependency section of the
# stdenv chapter of the Nixpkgs manual for the specification this algorithm
# implements.
findInputs() {
local pkg="$1"; shift
local var="$1"; shift
local propagatedBuildInputsFiles=("$@")
local -r pkg="$1"
local -ri hostOffset="$2"
local -ri targetOffset="$3"
# Sanity check
(( "$hostOffset" <= "$targetOffset" )) || exit -1
local varVar="${pkgAccumVarVars[$hostOffset + 1]}"
local varRef="$varVar[\$targetOffset - \$hostOffset]"
local var="${!varRef}"
unset -v varVar varRef
# TODO(@Ericson2314): Restore using associative array once Darwin
# nix-shell doesn't use impure bash. This should replace the O(n)
@@ -324,21 +396,106 @@ findInputs() {
exit 1
fi
local file
for file in "${propagatedBuildInputsFiles[@]}"; do
file="$pkg/nix-support/$file"
[[ -f "$file" ]] || continue
# The current package's host and target offset together
# provide a <=-preserving homomorphism from the relative
# offsets to current offset
function mapOffset() {
local -ri inputOffset="$1"
if (( "$inputOffset" <= 0 )); then
local -ri outputOffset="$inputOffset + $hostOffset"
else
local -ri outputOffset="$inputOffset - 1 + $targetOffset"
fi
echo "$outputOffset"
}
local pkgNext
for pkgNext in $(< "$file"); do
findInputs "$pkgNext" "$var" "${propagatedBuildInputsFiles[@]}"
# Host offset relative to that of the package whose immediate
# dependencies we are currently exploring.
local -i relHostOffset
for relHostOffset in "${allPlatOffsets[@]}"; do
# `+ 1` so we start at 0 for valid index
local files="${propagatedDepFilesVars[$relHostOffset + 1]}"
# Host offset relative to the package currently being
# built---as absolute an offset as will be used.
local -i hostOffsetNext
hostOffsetNext="$(mapOffset relHostOffset)"
# Ensure we're in bounds relative to the package currently
# being built.
[[ "${allPlatOffsets[*]}" = *"$hostOffsetNext"* ]] || continue
# Target offset relative to the *host* offset of the package
# whose immediate dependencies we are currently exploring.
local -i relTargetOffset
for relTargetOffset in "${allPlatOffsets[@]}"; do
(( "$relHostOffset" <= "$relTargetOffset" )) || continue
local fileRef="${files}[$relTargetOffset - $relHostOffset]"
local file="${!fileRef}"
unset -v fileRef
# Target offset relative to the package currently being
# built.
local -i targetOffsetNext
targetOffsetNext="$(mapOffset relTargetOffset)"
# Once again, ensure we're in bounds relative to the
# package currently being built.
[[ "${allPlatOffsets[*]}" = *"$targetOffsetNext"* ]] || continue
[[ -f "$pkg/nix-support/$file" ]] || continue
local pkgNext
for pkgNext in $(< "$pkg/nix-support/$file"); do
findInputs "$pkgNext" "$hostOffsetNext" "$targetOffsetNext"
done
done
done
}
# Make sure all are at least defined as empty
: ${depsBuildBuild=} ${depsBuildBuildPropagated=}
: ${nativeBuildInputs=} ${propagatedNativeBuildInputs=} ${defaultNativeBuildInputs=}
: ${depsBuildTarget=} ${depsBuildTargetPropagated=}
: ${depsHostHost=} ${depsHostHostPropagated=}
: ${buildInputs=} ${propagatedBuildInputs=} ${defaultBuildInputs=}
: ${depsTargetTarget=} ${depsTargetTargetPropagated=}
for pkg in $depsBuildBuild $depsBuildBuildPropagated; do
findInputs "$pkg" -1 -1
done
for pkg in $nativeBuildInputs $propagatedNativeBuildInputs; do
findInputs "$pkg" -1 0
done
for pkg in $depsBuildTarget $depsBuildTargetPropagated; do
findInputs "$pkg" -1 1
done
for pkg in $depsHostHost $depsHostHostPropagated; do
findInputs "$pkg" 0 0
done
for pkg in $buildInputs $propagatedBuildInputs ; do
findInputs "$pkg" 0 1
done
for pkg in $depsTargetTarget $depsTargetTargetPropagated; do
findInputs "$pkg" 1 1
done
# Default inputs must be processed last
for pkg in $defaultNativeBuildInputs; do
findInputs "$pkg" -1 0
done
for pkg in $defaultBuildInputs; do
findInputs "$pkg" 0 1
done
# Add package to the future PATH and run setup hooks
activatePackage() {
local pkg="$1"
local -ri hostOffset="$2"
local -ri targetOffset="$3"
# Sanity check
(( "$hostOffset" <= "$targetOffset" )) || exit -1
if [ -f "$pkg" ]; then
local oldOpts="$(shopt -po nounset)"
@@ -347,11 +504,18 @@ activatePackage() {
eval "$oldOpts"
fi
if [ -d "$pkg/bin" ]; then
# Only dependencies whose host platform is guaranteed to match the
# build platform are included here. That would be `depsBuild*`,
# and legacy `nativeBuildInputs`, in general. If we aren't cross
# compiling, however, everything can be put on the PATH. To ease
# the transition, we do include everything in thatcase.
#
# TODO(@Ericson2314): Don't special-case native compilation
if [[ ( -z "${crossConfig-}" || "$hostOffset" -le -1 ) && -d "$pkg/bin" ]]; then
addToSearchPath _PATH "$pkg/bin"
fi
if [ -f "$pkg/nix-support/setup-hook" ]; then
if [[ -f "$pkg/nix-support/setup-hook" ]]; then
local oldOpts="$(shopt -po nounset)"
set +u
source "$pkg/nix-support/setup-hook"
@@ -359,67 +523,72 @@ activatePackage() {
fi
}
declare -a nativePkgs crossPkgs
if [ -z "${crossConfig:-}" ]; then
# Not cross-compiling - both buildInputs (and variants like propagatedBuildInputs)
# are handled identically to nativeBuildInputs
for i in ${nativeBuildInputs:-} ${buildInputs:-} \
${defaultNativeBuildInputs:-} ${defaultBuildInputs:-} \
${propagatedNativeBuildInputs:-} ${propagatedBuildInputs:-}; do
findInputs "$i" nativePkgs propagated-native-build-inputs propagated-build-inputs
done
else
for i in ${nativeBuildInputs:-} ${defaultNativeBuildInputs:-} ${propagatedNativeBuildInputs:-}; do
findInputs "$i" nativePkgs propagated-native-build-inputs
done
for i in ${buildInputs:-} ${defaultBuildInputs:-} ${propagatedBuildInputs:-}; do
findInputs "$i" crossPkgs propagated-build-inputs
done
fi
_activatePkgs() {
local -i hostOffset targetOffset
local pkg
for i in ${nativePkgs+"${nativePkgs[@]}"} ${crossPkgs+"${crossPkgs[@]}"}; do
activatePackage "$i"
done
for hostOffset in "${allPlatOffsets[@]}"; do
local pkgsVar="${pkgAccumVarVars[$hostOffset + 1]}"
for targetOffset in "${allPlatOffsets[@]}"; do
(( "$hostOffset" <= "$targetOffset" )) || continue
local pkgsRef="${pkgsVar}[$targetOffset - $hostOffset]"
local pkgsSlice="${!pkgsRef}[@]"
for pkg in ${!pkgsSlice+"${!pkgsSlice}"}; do
activatePackage "$pkg" "$hostOffset" "$targetOffset"
done
done
done
}
# Run the package setup hooks and build _PATH
_activatePkgs
# Set the relevant environment variables to point to the build inputs
# found above.
#
# These `depOffset`s tell the env hook what sort of dependency
# (ignoring propagatedness) is being passed to the env hook. In a real
# language, we'd append a closure with this information to the
# relevant env hook array, but bash doesn't have closures, so it's
# easier to just pass this in.
# These `depOffset`s, beyond indexing the arrays, also tell the env
# hook what sort of dependency (ignoring propagatedness) is being
# passed to the env hook. In a real language, we'd append a closure
# with this information to the relevant env hook array, but bash
# doesn't have closures, so it's easier to just pass this in.
_addToEnv() {
local -i depHostOffset depTargetOffset
local pkg
_addToNativeEnv() {
local pkg="$1"
if [[ -n "${crossConfig:-}" ]]; then
local -i depOffset=-1
else
local -i depOffset=0
fi
# Run the package-specific hooks set by the setup-hook scripts.
runHook envHook "$pkg"
for depHostOffset in "${allPlatOffsets[@]}"; do
local hookVar="${pkgHookVarVars[$depHostOffset + 1]}"
local pkgsVar="${pkgAccumVarVars[$depHostOffset + 1]}"
for depTargetOffset in "${allPlatOffsets[@]}"; do
(( "$depHostOffset" <= "$depTargetOffset" )) || continue
local hookRef="${hookVar}[$depTargetOffset - $depHostOffset]"
if [[ -z "${crossConfig-}" ]]; then
# Apply environment hooks to all packages during native
# compilation to ease the transition.
#
# TODO(@Ericson2314): Don't special-case native compilation
for pkg in \
${pkgsBuildBuild+"${pkgsBuildBuild[@]}"} \
${pkgsBuildHost+"${pkgsBuildHost[@]}"} \
${pkgsBuildTarget+"${pkgsBuildTarget[@]}"} \
${pkgsHostHost+"${pkgsHostHost[@]}"} \
${pkgsHostTarget+"${pkgsHostTarget[@]}"} \
${pkgsTargetTarget+"${pkgsTargetTarget[@]}"}
do
runHook "${!hookRef}" "$pkg"
done
else
local pkgsRef="${pkgsVar}[$depTargetOffset - $depHostOffset]"
local pkgsSlice="${!pkgsRef}[@]"
for pkg in ${!pkgsSlice+"${!pkgsSlice}"}; do
runHook "${!hookRef}" "$pkg"
done
fi
done
done
}
# Old bash empty array hack
for i in ${nativePkgs+"${nativePkgs[@]}"}; do
_addToNativeEnv "$i"
done
_addToCrossEnv() {
local pkg="$1"
local -i depOffset=0
# Run the package-specific hooks set by the setup-hook scripts.
runHook crossEnvHook "$pkg"
}
# Old bash empty array hack
for i in ${crossPkgs+"${crossPkgs[@]}"}; do
_addToCrossEnv "$i"
done
# Run the package-specific hooks set by the setup-hook scripts.
_addToEnv
_addRpathPrefix "$out"
@@ -882,6 +1051,7 @@ installPhase() {
# propagated-build-inputs.
fixupPhase() {
# Make sure everything is writable so "strip" et al. work.
local output
for output in $outputs; do
if [ -e "${!output}" ]; then chmod -R u+w "${!output}"; fi
done
@@ -895,19 +1065,35 @@ fixupPhase() {
done
# Propagate build inputs and setup hook into the development output.
# Propagate dependencies & setup hook into the development output.
declare -ra flatVars=(
# Build
depsBuildBuildPropagated
propagatedNativeBuildInputs
depsBuildTargetPropagated
# Host
depsHostHostPropagated
propagatedBuildInputs
# Target
depsTargetTargetPropagated
)
declare -ra flatFiles=(
"${propagatedBuildDepFiles[@]}"
"${propagatedHostDepFiles[@]}"
"${propagatedTargetDepFiles[@]}"
)
local propagatedInputsIndex
for propagatedInputsIndex in "${!flatVars[@]}"; do
local propagatedInputsSlice="${flatVars[$propagatedInputsIndex]}[@]"
local propagatedInputsFile="${flatFiles[$propagatedInputsIndex]}"
[[ "${!propagatedInputsSlice}" ]] || continue
if [ -n "${propagatedBuildInputs:-}" ]; then
mkdir -p "${!outputDev}/nix-support"
# shellcheck disable=SC2086
printWords $propagatedBuildInputs > "${!outputDev}/nix-support/propagated-build-inputs"
fi
if [ -n "${propagatedNativeBuildInputs:-}" ]; then
mkdir -p "${!outputDev}/nix-support"
# shellcheck disable=SC2086
printWords $propagatedNativeBuildInputs > "${!outputDev}/nix-support/propagated-native-build-inputs"
fi
printWords ${!propagatedInputsSlice} > "${!outputDev}/nix-support/$propagatedInputsFile"
done
if [ -n "${setupHook:-}" ]; then
@@ -193,6 +193,7 @@ rec {
nuke-refs $out/bin/*
nuke-refs $out/lib/*
nuke-refs $out/libexec/gcc/*/*/*
nuke-refs $out/lib/gcc/*/*/*
mkdir $out/.pack
mv $out/* $out/.pack