Update AMI generator

The EBS and S3 (instance-store) AMIs are now created from the same image. HVM instance-store AMIs are also generated. Disk image generation has been factored out into a function (nixos/lib/make-disk-image.nix) that can be used to build other kinds of images.
2015-09-27 21:06:40 +02:00
parent efed00b55e
commit aeb31b97ad
13 changed files with 499 additions and 411 deletions
@@ -1,5 +0,0 @@
-{ modulesPath, ...}:
-{
-  imports = [ "${modulesPath}/virtualisation/amazon-init.nix" ];
-  services.journald.rateLimitBurst = 0;
-}
@@ -1,5 +0,0 @@
-{ config, pkgs, ...}:
-{
-  imports = [ ./amazon-base-config.nix ];
-  ec2.hvm = true;
-}
@@ -1,34 +0,0 @@
-{ config, pkgs, lib, ...}:
-let
-  cloudUtils = pkgs.fetchurl {
-    url = "https://launchpad.net/cloud-utils/trunk/0.27/+download/cloud-utils-0.27.tar.gz";
-    sha256 = "16shlmg36lidp614km41y6qk3xccil02f5n3r4wf6d1zr5n4v8vd";
-  };
-  growpart = pkgs.stdenv.mkDerivation {
-    name = "growpart";
-    src = cloudUtils;
-    buildPhase = ''
-      cp bin/growpart $out
-      sed -i 's|awk|gawk|' $out
-      sed -i 's|sed|gnused|' $out
-    '';
-    dontInstall = true;
-    dontPatchShebangs = true;
-  };
-in
-{
-  imports = [ ./amazon-base-config.nix ];
-  ec2.hvm = true;
-  boot.loader.grub.device = lib.mkOverride 0 "/dev/xvdg";
-  boot.kernelParams = [ "console=ttyS0" ];
-
-  boot.initrd.extraUtilsCommands = ''
-    copy_bin_and_libs ${pkgs.gawk}/bin/gawk
-    copy_bin_and_libs ${pkgs.gnused}/bin/sed
-    copy_bin_and_libs ${pkgs.utillinux}/sbin/sfdisk
-    cp -v ${growpart} $out/bin/growpart
-  '';
-  boot.initrd.postDeviceCommands = ''
-    [ -e /dev/xvda ] && [ -e /dev/xvda1 ] && TMPDIR=/run sh $(type -P growpart) /dev/xvda 1
-  '';
-}
@@ -0,0 +1,27 @@
+{ config, lib, pkgs, ... }:
+
+with lib;
+
+{
+
+  imports =
+    [ ../../../modules/installer/cd-dvd/channel.nix
+      ../../../modules/virtualisation/amazon-image.nix
+    ];
+
+  system.build.amazonImage = import ../../../lib/make-disk-image.nix {
+    inherit pkgs lib config;
+    partitioned = config.ec2.hvm;
+    diskSize = 8192;
+    configFile = pkgs.writeText "configuration.nix"
+      ''
+        {
+          imports = [ <nixpkgs/nixos/modules/virtualisation/amazon-image.nix> ];
+          ${optionalString config.ec2.hvm ''
+            ec2.hvm = true;
+          ''}
+        }
+      '';
+  };
+
+}
@@ -0,0 +1,217 @@
+#! /bin/sh -e
+
+set -o pipefail
+#set -x
+
+stateDir=${TMPDIR:-/tmp}/ec2-image
+echo "keeping state in $stateDir"
+mkdir -p $stateDir
+
+version=$(nix-instantiate --eval --strict '<nixpkgs>' -A lib.nixpkgsVersion | sed s/'"'//g)
+echo "NixOS version is $version"
+
+rm -f ec2-amis.nix
+
+
+for type in hvm pv; do
+    link=$stateDir/$type
+    imageFile=$link/nixos.img
+    system=x86_64-linux
+    arch=x86_64
+
+    # Build the image.
+    if ! [ -L $link ]; then
+        if [ $type = pv ]; then hvmFlag=false; else hvmFlag=true; fi
+
+        echo "building image type '$type'..."
+        nix-build -o $link \
+            '<nixpkgs/nixos>' \
+            -A config.system.build.amazonImage \
+            --arg configuration "{ imports = [ <nixpkgs/nixos/maintainers/scripts/ec2/amazon-image.nix> ]; ec2.hvm = $hvmFlag; }"
+    fi
+
+    for store in ebs s3; do
+
+        bucket=nixos-amis
+        bucketDir="$version-$type-$store"
+
+        prevAmi=
+        prevRegion=
+
+        #for region in eu-west-1 eu-central-1 us-east-1 us-west-1 us-west-2 ap-southeast-1 ap-southeast-2 ap-northeast-1 sa-east-1; do
+        for region in eu-west-1 us-east-1; do
+
+            name=nixos-$version-$arch-$type-$store
+            description="NixOS $system $version ($type-$store)"
+
+            amiFile=$stateDir/$region.$type.$store.ami-id
+
+            if ! [ -e $amiFile ]; then
+
+                echo "doing $name in $region..."
+
+                if [ -n "$prevAmi" ]; then
+                    ami=$(ec2-copy-image \
+                        --region "$region" \
+                        --source-region "$prevRegion" --source-ami-id "$prevAmi" \
+                        --name "$name" --description "$description" | cut -f 2)
+                else
+
+                    if [ $store = s3 ]; then
+
+                        # Bundle the image.
+                        imageDir=$stateDir/$type-bundled
+
+                        if ! [ -d $imageDir ]; then
+                            rm -rf $imageDir.tmp
+                            mkdir -p $imageDir.tmp
+                            ec2-bundle-image \
+                                -d $imageDir.tmp \
+                                -i $imageFile --arch $arch \
+                                --user "$AWS_ACCOUNT" -c "$EC2_CERT" -k "$EC2_PRIVATE_KEY"
+                            mv $imageDir.tmp $imageDir
+                        fi
+
+                        # Upload the bundle to S3.
+                        if ! [ -e $imageDir/uploaded ]; then
+                            echo "uploading bundle to S3..."
+                            ec2-upload-bundle \
+                                -m $imageDir/nixos.img.manifest.xml \
+                                -b "$bucket/$bucketDir" \
+                                -a "$EC2_ACCESS_KEY" -s "$EC2_SECRET_KEY" \
+                                --location EU
+                            touch $imageDir/uploaded
+                        fi
+
+                        extraFlags="$bucket/$bucketDir/nixos.img.manifest.xml"
+
+                    else
+
+                        # Convert the image to vhd format so we don't have
+                        # to upload a huge raw image.
+                        vhdFile=$stateDir/$type.vhd
+                        if ! [ -e $vhdFile ]; then
+                            qemu-img convert -O vpc $imageFile $vhdFile.tmp
+                            mv $vhdFile.tmp $vhdFile
+                        fi
+
+                        taskId=$(cat $stateDir/$region.$type.task-id 2> /dev/null || true)
+                        volId=$(cat $stateDir/$region.$type.vol-id 2> /dev/null || true)
+                        snapId=$(cat $stateDir/$region.$type.snap-id 2> /dev/null || true)
+
+                        # Import the VHD file.
+                        if [ -z "$snapId" -a -z "$volId" -a -z "$taskId" ]; then
+                            echo "importing $vhdFile..."
+                            taskId=$(ec2-import-volume $vhdFile --no-upload -f vhd \
+                                -o "$EC2_ACCESS_KEY" -w "$EC2_SECRET_KEY" \
+                                --region "$region" -z "${region}a" \
+                                --bucket "$bucket" --prefix "$bucketDir/" \
+                                | tee /dev/stderr \
+                                | sed 's/.*\(import-vol-[0-9a-z]\+\).*/\1/ ; t ; d')
+                            echo -n "$taskId" > $stateDir/$region.$type.task-id
+                        fi
+
+                        if [ -z "$snapId" -a -z "$volId" ]; then
+                            ec2-resume-import  $vhdFile -t "$taskId" --region "$region" \
+                                -o "$EC2_ACCESS_KEY" -w "$EC2_SECRET_KEY"
+                        fi
+
+                        # Wait for the volume creation to finish.
+                        if [ -z "$snapId" -a -z "$volId" ]; then
+                            echo "waiting for import to finish..."
+                            while true; do
+                                volId=$(ec2-describe-conversion-tasks "$taskId" --region "$region" | sed 's/.*VolumeId.*\(vol-[0-9a-f]\+\).*/\1/ ; t ; d')
+                                if [ -n "$volId" ]; then break; fi
+                                sleep 10
+                            done
+
+                            echo -n "$volId" > $stateDir/$region.$type.vol-id
+                        fi
+
+                        # Delete the import task.
+                        if [ -n "$volId" -a -n "$taskId" ]; then
+                            echo "removing import task..."
+                            ec2-delete-disk-image -t "$taskId" --region "$region" -o "$EC2_ACCESS_KEY" -w "$EC2_SECRET_KEY" || true
+                            rm -f $stateDir/$region.$type.task-id
+                        fi
+
+                        # Create a snapshot.
+                        if [ -z "$snapId" ]; then
+                            echo "creating snapshot..."
+                            snapId=$(ec2-create-snapshot "$volId" --region "$region" | cut -f 2)
+                            echo -n "$snapId" > $stateDir/$region.$type.snap-id
+                            ec2-create-tags "$snapId" -t "Name=$description" --region "$region"
+                        fi
+
+                        # Wait for the snapshot to finish.
+                        echo "waiting for snapshot to finish..."
+                        while true; do
+                            status=$(ec2-describe-snapshots "$snapId" --region "$region" | head -n1 | cut -f 4)
+                            if [ "$status" = completed ]; then break; fi
+                            sleep 10
+                        done
+
+                        # Delete the volume.
+                        if [ -n "$volId" ]; then
+                            echo "deleting volume..."
+                            ec2-delete-volume "$volId" --region "$region" || true
+                            rm -f $stateDir/$region.$type.vol-id
+                        fi
+
+                        extraFlags="-b /dev/sda1=$snapId:20:true:gp2"
+
+                        if [ $type = pv ]; then
+                            extraFlags+=" --root-device-name=/dev/sda1"
+                        fi
+
+                        extraFlags+=" -b /dev/sdb=ephemeral0 -b /dev/sdc=ephemeral1 -b /dev/sdd=ephemeral2 -b /dev/sde=ephemeral3"
+                    fi
+
+                    # Register the AMI.
+                    if [ $type = pv ]; then
+                        kernel=$(ec2-describe-images -o amazon --filter "manifest-location=*pv-grub-hd0_1.04-$arch*" --region "$region" | cut -f 2)
+                        [ -n "$kernel" ]
+                        echo "using PV-GRUB kernel $kernel"
+                        extraFlags+=" --virtualization-type paravirtual --kernel $kernel"
+                    else
+                        extraFlags+=" --virtualization-type hvm"
+                    fi
+
+                    set -x
+                    ami=$(ec2-register \
+                        -n "$name" \
+                        -d "$description" \
+                        --region "$region" \
+                        --architecture "$arch" \
+                        $extraFlags | cut -f 2)
+                fi
+
+                echo -n "$ami" > $amiFile
+                echo "created AMI $ami of type '$type' in $region..."
+
+            else
+                ami=$(cat $amiFile)
+            fi
+
+            echo "waiting for AMI..."
+            while true; do
+                status=$(ec2-describe-images "$ami" --region "$region" | head -n1 | cut -f 5)
+                if [ "$status" = available ]; then break; fi
+                sleep 10
+            done
+
+            ec2-modify-image-attribute \
+                --region "$region" "$ami" -l -a all
+
+            echo "region = $region, type = $type, store = $store, ami = $ami"
+            if [ -z "$prevAmi" ]; then
+                prevAmi="$ami"
+                prevRegion="$region"
+            fi
+
+            echo "  \"15.09\".$region.$type-$store = \"$ami\";" >> ec2-amis.nix
+        done
+
+    done
+
+done
@@ -1,216 +0,0 @@
-#! /usr/bin/env python
-
-import os
-import sys
-import time
-import argparse
-import nixops.util
-from nixops import deployment
-from boto.ec2.blockdevicemapping import BlockDeviceMapping, BlockDeviceType
-import boto.ec2
-from nixops.statefile import StateFile, get_default_state_file
-
-parser = argparse.ArgumentParser(description='Create an EBS-backed NixOS AMI')
-parser.add_argument('--region', dest='region', required=True, help='EC2 region to create the image in')
-parser.add_argument('--channel', dest='channel', default="14.12", help='Channel to use')
-parser.add_argument('--keep', dest='keep', action='store_true', help='Keep NixOps machine after use')
-parser.add_argument('--hvm', dest='hvm', action='store_true', help='Create HVM image')
-parser.add_argument('--key', dest='key_name', action='store_true', help='Keypair used for HVM instance creation', default="rob")
-args = parser.parse_args()
-
-instance_type = "m3.medium" if args.hvm else "m1.small"
-
-if args.hvm:
-    virtualization_type = "hvm"
-    root_block = "/dev/sda1"
-    image_type = 'hvm'
-else:
-    virtualization_type = "paravirtual"
-    root_block = "/dev/sda"
-    image_type = 'ebs'
-
-ebs_size = 20
-
-# Start a NixOS machine in the given region.
-f = open("ebs-creator-config.nix", "w")
-f.write('''{{
-  resources.ec2KeyPairs.keypair.accessKeyId = "lb-nixos";
-  resources.ec2KeyPairs.keypair.region = "{0}";
-
-  machine =
-    {{ pkgs, ... }}:
-    {{
-      deployment.ec2.accessKeyId = "lb-nixos";
-      deployment.ec2.region = "{0}";
-      deployment.ec2.blockDeviceMapping."/dev/xvdg".size = pkgs.lib.mkOverride 10 {1};
-    }};
-}}
-'''.format(args.region, ebs_size))
-f.close()
-
-db = StateFile(get_default_state_file())
-try:
-    depl = db.open_deployment("ebs-creator")
-except Exception:
-    depl = db.create_deployment()
-    depl.name = "ebs-creator"
-depl.logger.set_autoresponse("y")
-depl.nix_exprs = [os.path.abspath("./ebs-creator.nix"), os.path.abspath("./ebs-creator-config.nix")]
-if not args.keep: depl.destroy_resources()
-depl.deploy(allow_reboot=True)
-
-m = depl.machines['machine']
-
-# Do the installation.
-device="/dev/xvdg"
-if args.hvm:
-    m.run_command('parted -s /dev/xvdg -- mklabel msdos')
-    m.run_command('parted -s /dev/xvdg -- mkpart primary ext2 1M -1s')
-    device="/dev/xvdg1"
-
-m.run_command("if mountpoint -q /mnt; then umount /mnt; fi")
-m.run_command("mkfs.ext4 -L nixos {0}".format(device))
-m.run_command("mkdir -p /mnt")
-m.run_command("mount {0} /mnt".format(device))
-m.run_command("touch /mnt/.ebs")
-m.run_command("mkdir -p /mnt/etc/nixos")
-
-m.run_command("nix-channel --add https://nixos.org/channels/nixos-{} nixos".format(args.channel))
-m.run_command("nix-channel --update")
-
-version = m.run_command("nix-instantiate --eval-only -A lib.nixpkgsVersion '<nixpkgs>'", capture_stdout=True).split(' ')[0].replace('"','').strip()
-print >> sys.stderr, "NixOS version is {0}".format(version)
-if args.hvm:
-    m.upload_file("./amazon-base-config.nix", "/mnt/etc/nixos/amazon-base-config.nix")
-    m.upload_file("./amazon-hvm-config.nix", "/mnt/etc/nixos/configuration.nix")
-    m.upload_file("./amazon-hvm-install-config.nix", "/mnt/etc/nixos/amazon-hvm-install-config.nix")
-    m.run_command("NIXOS_CONFIG=/etc/nixos/amazon-hvm-install-config.nix nixos-install")
-else:
-    m.upload_file("./amazon-base-config.nix", "/mnt/etc/nixos/configuration.nix")
-    m.run_command("nixos-install")
-
-m.run_command("umount /mnt")
-
-if args.hvm:
-    ami_name = "nixos-{0}-x86_64-hvm".format(version)
-    description = "NixOS {0} (x86_64; EBS root; hvm)".format(version)
-else:
-    ami_name = "nixos-{0}-x86_64-ebs".format(version)
-    description = "NixOS {0} (x86_64; EBS root)".format(version)
-
-
-# Wait for the snapshot to finish.
-def check():
-    status = snapshot.update()
-    print >> sys.stderr, "snapshot status is {0}".format(status)
-    return status == '100%'
-
-m.connect()
-volume = m._conn.get_all_volumes([], filters={'attachment.instance-id': m.resource_id, 'attachment.device': "/dev/sdg"})[0]
-
-# Create a snapshot.
-snapshot = volume.create_snapshot(description=description)
-print >> sys.stderr, "created snapshot {0}".format(snapshot.id)
-
-nixops.util.check_wait(check, max_tries=120)
-
-m._conn.create_tags([snapshot.id], {'Name': ami_name})
-
-if not args.keep: depl.destroy_resources()
-
-# Register the image.
-aki = m._conn.get_all_images(filters={'manifest-location': 'ec2*pv-grub-hd0_1.03-x86_64*'})[0]
-print >> sys.stderr, "using kernel image {0} - {1}".format(aki.id, aki.location)
-
-block_map = BlockDeviceMapping()
-block_map[root_block] = BlockDeviceType(snapshot_id=snapshot.id, delete_on_termination=True, size=ebs_size, volume_type="gp2")
-block_map['/dev/sdb'] = BlockDeviceType(ephemeral_name="ephemeral0")
-block_map['/dev/sdc'] = BlockDeviceType(ephemeral_name="ephemeral1")
-block_map['/dev/sdd'] = BlockDeviceType(ephemeral_name="ephemeral2")
-block_map['/dev/sde'] = BlockDeviceType(ephemeral_name="ephemeral3")
-
-common_args = dict(
-        name=ami_name,
-        description=description,
-        architecture="x86_64",
-        root_device_name=root_block,
-        block_device_map=block_map,
-        virtualization_type=virtualization_type,
-        delete_root_volume_on_termination=True
-        )
-if not args.hvm:
-    common_args['kernel_id']=aki.id
-
-ami_id = m._conn.register_image(**common_args)
-
-print >> sys.stderr, "registered AMI {0}".format(ami_id)
-
-print >> sys.stderr, "sleeping a bit..."
-time.sleep(30)
-
-print >> sys.stderr, "setting image name..."
-m._conn.create_tags([ami_id], {'Name': ami_name})
-
-print >> sys.stderr, "making image public..."
-image = m._conn.get_all_images(image_ids=[ami_id])[0]
-image.set_launch_permissions(user_ids=[], group_names=["all"])
-
-# Do a test deployment to make sure that the AMI works.
-f = open("ebs-test.nix", "w")
-f.write(
-    '''
-    {{
-      network.description = "NixOS EBS test";
-
-      resources.ec2KeyPairs.keypair.accessKeyId = "lb-nixos";
-      resources.ec2KeyPairs.keypair.region = "{0}";
-
-      machine = {{ config, pkgs, resources, ... }}: {{
-        deployment.targetEnv = "ec2";
-        deployment.ec2.accessKeyId = "lb-nixos";
-        deployment.ec2.region = "{0}";
-        deployment.ec2.instanceType = "{2}";
-        deployment.ec2.keyPair = resources.ec2KeyPairs.keypair.name;
-        deployment.ec2.securityGroups = [ "public-ssh" ];
-        deployment.ec2.ami = "{1}";
-      }};
-    }}
-    '''.format(args.region, ami_id, instance_type))
-f.close()
-
-test_depl = db.create_deployment()
-test_depl.auto_response = "y"
-test_depl.name = "ebs-creator-test"
-test_depl.nix_exprs = [os.path.abspath("./ebs-test.nix")]
-test_depl.deploy(create_only=True)
-test_depl.machines['machine'].run_command("nixos-version")
-
-# Log the AMI ID.
-f = open("ec2-amis.nix".format(args.region, image_type), "w")
-f.write("{\n")
-
-for dest in [ 'us-east-1', 'us-west-1', 'us-west-2', 'eu-west-1', 'eu-central-1', 'ap-southeast-1', 'ap-southeast-2', 'ap-northeast-1', 'sa-east-1']:
-    copy_image = None
-    if args.region != dest:
-        try:
-            print >> sys.stderr, "copying image from region {0} to {1}".format(args.region, dest)
-            conn = boto.ec2.connect_to_region(dest)
-            copy_image = conn.copy_image(args.region, ami_id, ami_name, description=None, client_token=None)
-        except :
-            print >> sys.stderr, "FAILED!"
-
-        # Log the AMI ID.
-        if copy_image != None:
-            f.write('  "{0}"."{1}".{2} = "{3}";\n'.format(args.channel,dest,"hvm" if args.hvm else "ebs",copy_image.image_id))
-    else:
-        f.write('  "{0}"."{1}".{2} = "{3}";\n'.format(args.channel,args.region,"hvm" if args.hvm else "ebs",ami_id))
-
-
-f.write("}\n")
-f.close()
-
-if not args.keep:
-    test_depl.logger.set_autoresponse("y")
-    test_depl.destroy_resources()
-    test_depl.delete()
-
@@ -1,53 +0,0 @@
-#! /bin/sh -e
-
-export NIXOS_CONFIG=$(dirname $(readlink -f $0))/amazon-base-config.nix
-
-version=$(nix-instantiate --eval-only '<nixpkgs/nixos>' -A config.system.nixosVersion | sed s/'"'//g)
-echo "NixOS version is $version"
-
-buildAndUploadFor() {
-    system="$1"
-    arch="$2"
-
-    echo "building $system image..."
-    nix-build '<nixpkgs/nixos>' \
-        -A config.system.build.amazonImage --argstr system "$system" -o ec2-ami
-
-    ec2-bundle-image -i ./ec2-ami/nixos.img --user "$AWS_ACCOUNT" --arch "$arch" \
-        -c "$EC2_CERT" -k "$EC2_PRIVATE_KEY"
-
-    for region in eu-west-1; do
-        echo "uploading $system image for $region..."
-
-        name=nixos-$version-$arch-s3
-        bucket="$(echo $name-$region | tr '[A-Z]_' '[a-z]-')"
-
-        if [ "$region" = eu-west-1 ]; then s3location=EU;
-        elif [ "$region" = us-east-1 ]; then s3location=US;
-        else s3location="$region"
-        fi
-
-        ec2-upload-bundle -b "$bucket" -m /tmp/nixos.img.manifest.xml \
-            -a "$EC2_ACCESS_KEY" -s "$EC2_SECRET_KEY" --location "$s3location" \
-            --url http://s3.amazonaws.com
-
-        kernel=$(ec2-describe-images -o amazon --filter "manifest-location=*pv-grub-hd0_1.04-$arch*" --region "$region" | cut -f 2)
-        echo "using PV-GRUB kernel $kernel"
-
-        ami=$(ec2-register "$bucket/nixos.img.manifest.xml" -n "$name" -d "NixOS $system r$revision" -O "$EC2_ACCESS_KEY" -W "$EC2_SECRET_KEY" \
-            --region "$region" --kernel "$kernel" | cut -f 2)
-
-        echo "AMI ID is $ami"
-
-        echo "  \"14.12\".\"$region\".s3 = \"$ami\";" >> ec2-amis.nix
-
-        ec2-modify-image-attribute --region "$region" "$ami" -l -a all -O "$EC2_ACCESS_KEY" -W "$EC2_SECRET_KEY"
-
-        for cp_region in us-east-1 us-west-1 us-west-2 eu-central-1 ap-southeast-1 ap-southeast-2 ap-northeast-1 sa-east-1; do
-          new_ami=$(aws ec2 copy-image --source-image-id $ami --source-region $region --region $cp_region --name "$name" | json ImageId)
-          echo "  \"14.12\".\"$cp_region\".s3 = \"$new_ami\";" >> ec2-amis.nix  
-        done
-    done
-}
-
-buildAndUploadFor x86_64-linux x86_64
@@ -1,13 +0,0 @@
-{
-  network.description = "NixOS EBS creator";
-
-  machine =
-    { config, pkgs, resources, ... }:
-    { deployment.targetEnv = "ec2";
-      deployment.ec2.instanceType = "c3.large";
-      deployment.ec2.securityGroups = [ "public-ssh" ];
-      deployment.ec2.ebsBoot = false;
-      deployment.ec2.keyPair = resources.ec2KeyPairs.keypair.name;
-      environment.systemPackages = [ pkgs.parted ];
-    };
-}