public/nixpkgs
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge branch 'master' into docker-dirlinks

Browse Source
This commit is contained in:
Ryan Trinkle
2017-11-03 10:53:00 -04:00
committed by GitHub
parent dce2c258ac 74260a4922
commit ded1281f45
8350 changed files with 251021 additions and 185547 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkgs/build-support/docker/default.nix
+89 -14
View File
@@ -6,9 +6,12 @@
findutils,
go,
jshon,
jq,
lib,
pkgs,
pigz,
nixUnstable,
perl,
runCommand,
rsync,
shadow,
@@ -26,7 +29,7 @@
rec {
examples = import ./examples.nix {
inherit pkgs buildImage pullImage shadowSetup;
inherit pkgs buildImage pullImage shadowSetup buildImageWithNixDb;
};
pullImage = callPackage ./pull.nix {};
@@ -42,7 +45,7 @@ rec {
cp ${./tarsum.go} tarsum.go
export GOPATH=$(pwd)
mkdir src
ln -sT ${docker.src}/pkg/tarsum src/tarsum
ln -sT ${docker.src}/components/engine/pkg/tarsum src/tarsum
go build
cp tarsum $out
@@ -82,7 +85,7 @@ rec {
export PATH=${shadow}/bin:$PATH
mkdir -p /etc/pam.d
if [[ ! -f /etc/passwd ]]; then
echo "root:x:0:0::/root:/bin/sh" > /etc/passwd
echo "root:x:0:0::/root:${stdenv.shell}" > /etc/passwd
echo "root:!x:::::::" > /etc/shadow
fi
if [[ ! -f /etc/group ]]; then
@@ -225,6 +228,19 @@ rec {
${text}
'';
nixRegistration = contents: runCommand "nix-registration" {
buildInputs = [ nixUnstable perl ];
# For obtaining the closure of `contents'.
exportReferencesGraph =
let contentsList = if builtins.isList contents then contents else [ contents ];
in map (x: [("closure-" + baseNameOf x) x]) contentsList;
}
''
mkdir $out
printRegistration=1 perl ${pkgs.pathsFromGraph} closure-* > $out/db.dump
perl ${pkgs.pathsFromGraph} closure-* > $out/storePaths
'';
# Create a "layer" (set of files).
mkPureLayer = {
# Name of the layer
@@ -238,11 +254,10 @@ rec {
# into directories.
keepContentsDirlinks ? false,
# Additional commands to run on the layer before it is tar'd up.
extraCommands ? ""
extraCommands ? "", uid ? 0, gid ? 0
}:
runCommand "docker-layer-${name}" {
inherit baseJson contents extraCommands;
buildInputs = [ jshon rsync ];
}
''
@@ -257,6 +272,8 @@ rec {
echo "No contents to add to layer."
fi
chmod ug+w layer
if [[ -n $extraCommands ]]; then
(cd layer; eval "$extraCommands")
fi
@@ -264,7 +281,7 @@ rec {
# Tar up the layer and throw it into 'layer.tar'.
echo "Packing layer..."
mkdir $out
tar -C layer --mtime="@$SOURCE_DATE_EPOCH" -cf $out/layer.tar .
tar -C layer --mtime="@$SOURCE_DATE_EPOCH" --owner=${toString uid} --group=${toString gid} -cf $out/layer.tar .
# Compute a checksum of the tarball.
echo "Computing layer checksum..."
@@ -320,6 +337,8 @@ rec {
echo "Adding $item..."
rsync -a${if keepContentsDirlinks then "K" else "k"} --chown=0:0 $item/ layer/
done
chmod ug+w layer
'';
postMount = ''
@@ -387,11 +406,13 @@ rec {
# Docker config; e.g. what command to run on the container.
config ? null,
# Optional bash script to run on the files prior to fixturizing the layer.
extraCommands ? "",
extraCommands ? "", uid ? 0, gid ? 0,
# Optional bash script to run as root on the image when provisioning.
runAsRoot ? null,
# Size of the virtual machine disk to provision when building the image.
diskSize ? 1024,
# Time of creation of the image.
created ? "1970-01-01T00:00:01Z",
}:
let
@@ -399,17 +420,16 @@ rec {
# Create a JSON blob of the configuration. Set the date to unix zero.
baseJson = writeText "${baseName}-config.json" (builtins.toJSON {
created = "1970-01-01T00:00:01Z";
inherit created config;
architecture = "amd64";
os = "linux";
config = config;
});
layer =
if runAsRoot == null
then mkPureLayer {
name = baseName;
inherit baseJson contents keepContentsDirlinks extraCommands;
inherit baseJson contents keepContentsDirlinks extraCommands uid gid;
} else mkRootLayer {
name = baseName;
inherit baseJson fromImage fromImageName fromImageTag
@@ -417,9 +437,10 @@ rec {
extraCommands;
};
result = runCommand "docker-image-${baseName}.tar.gz" {
buildInputs = [ jshon pigz coreutils findutils ];
imageName = name;
imageTag = tag;
buildInputs = [ jshon pigz coreutils findutils jq ];
# Image name and tag must be lowercase
imageName = lib.toLower name;
imageTag = lib.toLower tag;
inherit fromImage baseJson;
layerClosure = writeReferencesToFile layer;
passthru.buildArgs = args;
@@ -443,6 +464,9 @@ rec {
if [[ -n "$fromImage" ]]; then
echo "Unpacking base image..."
tar -C image -xpf "$fromImage"
# Do not import the base image configuration and manifest
chmod a+w image image/*.json
rm -f image/*.json
if [[ -z "$fromImageName" ]]; then
fromImageName=$(jshon -k < image/repositories|head -n1)
@@ -501,6 +525,24 @@ rec {
# Use the temp folder we've been working on to create a new image.
mv temp image/$layerID
# Create image json and image manifest
imageJson=$(cat ${baseJson} | jq ". + {\"rootfs\": {\"diff_ids\": [], \"type\": \"layers\"}}")
manifestJson=$(jq -n "[{\"RepoTags\":[\"$imageName:$imageTag\"]}]")
currentID=$layerID
while [[ -n "$currentID" ]]; do
layerChecksum=$(sha256sum image/$currentID/layer.tar | cut -d ' ' -f1)
imageJson=$(echo "$imageJson" | jq ".history |= [{\"created\": \"${created}\"}] + .")
imageJson=$(echo "$imageJson" | jq ".rootfs.diff_ids |= [\"sha256:$layerChecksum\"] + .")
manifestJson=$(echo "$manifestJson" | jq ".[0].Layers |= [\"$currentID/layer.tar\"] + .")
currentID=$(cat image/$currentID/json | (jshon -e parent -u 2>/dev/null || true))
done
imageJsonChecksum=$(echo "$imageJson" | sha256sum | cut -d ' ' -f1)
echo "$imageJson" > "image/$imageJsonChecksum.json"
manifestJson=$(echo "$manifestJson" | jq ".[0].Config = \"$imageJsonChecksum.json\"")
echo "$manifestJson" > image/manifest.json
# Store the json under the name image/repositories.
jshon -n object \
-n object -s "$layerID" -i "$imageTag" \
@@ -510,11 +552,44 @@ rec {
chmod -R a-w image
echo "Cooking the image..."
tar -C image --mtime="@$SOURCE_DATE_EPOCH" -c . | pigz -nT > $out
tar -C image --mtime="@$SOURCE_DATE_EPOCH" --owner=0 --group=0 --xform s:'./':: -c . | pigz -nT > $out
echo "Finished."
'';
in
result;
# Build an image and populate its nix database with the provided
# contents. The main purpose is to be able to use nix commands in
# the container.
# Be careful since this doesn't work well with multilayer.
buildImageWithNixDb = args@{ contents ? null, extraCommands ? "", ... }:
buildImage (args // {
extraCommands = ''
echo "Generating the nix database..."
echo "Warning: only the database of the deepest Nix layer is loaded."
echo " If you want to use nix commands in the container, it would"
echo " be better to only have one layer that contains a nix store."
# This requires Nix 1.12 or higher
export NIX_REMOTE=local?root=$PWD
${nixUnstable}/bin/nix-store --load-db < ${nixRegistration contents}/db.dump
# We fill the store in order to run the 'verify' command that
# generates hash and size of output paths.
# Note when Nix 1.12 is be the stable one, the database dump
# generated by the exportReferencesGraph function will
# contains sha and size. See
# https://github.com/NixOS/nix/commit/c2b0d8749f7e77afc1c4b3e8dd36b7ee9720af4a
storePaths=$(cat ${nixRegistration contents}/storePaths)
echo "Copying everything to /nix/store (will take a while)..."
cp -prd $storePaths nix/store/
${nixUnstable}/bin/nix-store --verify --check-contents
mkdir -p nix/var/nix/gcroots/docker/
for i in ${lib.concatStringsSep " " contents}; do
ln -s $i nix/var/nix/gcroots/docker/$(basename $i)
done;
'' + extraCommands;
});
}
pkgs/build-support/docker/examples.nix
+22 -11
View File
@@ -7,7 +7,7 @@
# $ nix-build '<nixpkgs>' -A dockerTools.examples.redis
# $ docker load < result
{ pkgs, buildImage, pullImage, shadowSetup }:
{ pkgs, buildImage, pullImage, shadowSetup, buildImageWithNixDb }:
rec {
# 1. basic example
@@ -83,16 +83,12 @@ rec {
};
# 4. example of pulling an image. could be used as a base for other images
#
# ***** Currently broken, getting 404s. Perhaps the docker API has changed?
#
#
# debian = pullImage {
# imageName = "debian";
# imageTag = "jessie";
# # this hash will need change if the tag is updated at docker hub
# sha256 = "18kd495lc2k35h03bpcbdjnix17nlqbwf6nmq3sb161blf0dk14q";
# };
nixFromDockerHub = pullImage {
imageName = "nixos/nix";
imageTag = "1.11";
# this hash will need change if the tag is updated at docker hub
sha256 = "0nncn9pn5miygan51w34c2p9qssi96jgsaqv44dxxdprc8pg0g83";
};
# 5. example of multiple contents, emacs and vi happily coexisting
editors = buildImage {
@@ -105,4 +101,19 @@ rec {
pkgs.nano
];
};
# 5. nix example to play with the container nix store
# docker run -it --rm nix nix-store -qR $(nix-build '<nixpkgs>' -A nix)
nix = buildImageWithNixDb {
name = "nix";
contents = [
# nix-store uses cat program to display results as specified by
# the image env variable NIX_PAGER.
pkgs.coreutils
pkgs.nix
];
config = {
Env = [ "NIX_PAGER=cat" ];
};
};
}
pkgs/build-support/docker/pull.nix
+24 -33
View File
@@ -1,41 +1,32 @@
{ stdenv, lib, curl, jshon, python, runCommand }:
# Inspired and simplified version of fetchurl.
{ stdenv, lib, docker, vmTools, utillinux, curl, kmod, dhcp, cacert, e2fsprogs }:
let
nameReplace = name: builtins.replaceStrings ["/" ":"] ["-" "-"] name;
in
# For simplicity we only support sha256.
{ imageName, imageTag ? "latest", imageId ? "${imageName}:${imageTag}"
, sha256, name ? (nameReplace "docker-image-${imageName}-${imageTag}.tar") }:
let
pullImage = vmTools.runInLinuxVM (
stdenv.mkDerivation {
inherit name imageId;
# Currently only registry v1 is supported, compatible with Docker Hub.
certs = "${cacert}/etc/ssl/certs/ca-bundle.crt";
{ imageName, imageTag ? "latest", imageId ? null
, sha256, name ? "${imageName}-${imageTag}"
, indexUrl ? "https://index.docker.io"
, registryVersion ? "v1"
, curlOpts ? "" }:
builder = ./pull.sh;
assert registryVersion == "v1";
buildInputs = [ curl utillinux docker kmod dhcp cacert e2fsprogs ];
let layer = stdenv.mkDerivation {
inherit name imageName imageTag imageId
indexUrl registryVersion curlOpts;
outputHashAlgo = "sha256";
outputHash = sha256;
builder = ./pull.sh;
detjson = ./detjson.py;
impureEnvVars = lib.fetchers.proxyImpureEnvVars;
buildInputs = [ curl jshon python ];
preVM = vmTools.createEmptyImage {
size = 2048;
fullName = "${name}-disk";
};
outputHashAlgo = "sha256";
outputHash = sha256;
outputHashMode = "recursive";
impureEnvVars = lib.fetchers.proxyImpureEnvVars ++ [
# This variable allows the user to pass additional options to curl
"NIX_CURL_FLAGS"
];
# Doing the download on a remote machine just duplicates network
# traffic, so don't do that.
preferLocalBuild = true;
};
in runCommand "${name}.tar.gz" {} ''
tar -C ${layer} -czf $out .
''
QEMU_OPTS = "-netdev user,id=net0 -device virtio-net-pci,netdev=net0";
});
in
pullImage
pkgs/build-support/docker/pull.sh
+29 -79
View File
@@ -1,86 +1,36 @@
# Reference: docker src contrib/download-frozen-image.sh
source $stdenv/setup
# Curl flags to handle redirects, not use EPSV, handle cookies for
# servers to need them during redirects, and work on SSL without a
# certificate (this isn't a security problem because we check the
# cryptographic hash of the output anyway).
curl=$(command -v curl)
curl() {
[[ -n ${token:-} ]] && set -- -H "Authorization: Token $token" "$@"
$curl \
--location --max-redirs 20 \
--retry 3 \
--fail \
--disable-epsv \
--cookie-jar cookies \
--insecure \
$curlOpts \
$NIX_CURL_FLAGS \
"$@"
}
mkdir -p /var/lib/docker
mkfs.ext4 /dev/vda
mount -t ext4 /dev/vda /var/lib/docker
fetchLayer() {
local url="$1"
local dest="$2"
local curlexit=18;
modprobe virtio_net
dhclient eth0
# if we get error code 18, resume partial download
while [ $curlexit -eq 18 ]; do
# keep this inside an if statement, since on failure it doesn't abort the script
if curl -C - "$url" --output "$dest"; then
return 0
else
curlexit=$?;
fi
done
mkdir -p /etc/ssl/certs/
cp "$certs" "/etc/ssl/certs/"
return $curlexit
}
headers=$(curl -o /dev/null -D- -H 'X-Docker-Token: true' \
"$indexUrl/$registryVersion/repositories/$imageName/images")
header() {
grep $1 <<< "$headers" | tr -d '\r' | cut -d ' ' -f 2
}
# this only takes the first endpoint, more may be provided
# https://docs.docker.com/v1.6/reference/api/docker-io_api/
if ! registryUrl=$(header X-Docker-Endpoints); then
echo "error: index returned no endpoint"
exit 1
fi
baseUrl="https://$registryUrl/$registryVersion"
token="$(header X-Docker-Token || true)";
if [ -z "$imageId" ]; then
imageId="$(curl "$baseUrl/repositories/$imageName/tags/$imageTag")"
imageId="${imageId//\"/}"
if [ -z "$imageId" ]; then
echo "error: no image ID found for ${imageName}:${imageTag}"
exit 1
# from https://github.com/tianon/cgroupfs-mount/blob/master/cgroupfs-mount
mount -t tmpfs -o uid=0,gid=0,mode=0755 cgroup /sys/fs/cgroup
cd /sys/fs/cgroup
for sys in $(awk '!/^#/ { if ($4 == 1) print $1 }' /proc/cgroups); do
mkdir -p $sys
if ! mountpoint -q $sys; then
if ! mount -n -t cgroup -o $sys cgroup $sys; then
rmdir $sys || true
fi
echo "found image ${imageName}:${imageTag}@$imageId"
fi
mkdir -p $out
jshon -n object \
-n object -s "$imageId" -i "$imageTag" \
-i "$imageName" > $out/repositories
curl "$baseUrl/images/$imageId/ancestry" -o ancestry.json
layerIds=$(jshon -a -u < ancestry.json)
for layerId in $layerIds; do
echo "fetching layer $layerId"
mkdir "$out/$layerId"
echo '1.0' > "$out/$layerId/VERSION"
curl "$baseUrl/images/$layerId/json" | python $detjson > "$out/$layerId/json"
fetchLayer "$baseUrl/images/$layerId/layer" "$out/$layerId/layer.tar"
fi
done
# run docker daemon
dockerd -H tcp://127.0.0.1:5555 -H unix:///var/run/docker.sock &
until docker ps 2>/dev/null; do
printf '.'
sleep 1
done
rm -r $out
docker pull ${imageId}
docker save ${imageId} > $out