peteris.rocks

Script to launch Amazon EC2 Spot instances

Quickly launch and provision an Amazon EC2 spot instance with a simple Python script

Last updated on

This script will

Install

First, install the dependencies. This is for Ubuntu Server 14.04.

sudo apt-get install -y python python-pip python-dev libffi-dev libssl-dev
sudo pip install boto paramiko

Create a new file (e.g. launch-spot.py) and give it exec permissions (chmod +x launch-spot.py).

#!/usr/bin/python2.7 -u

# pip install boto paramiko

import argparse
import boto, boto.ec2, boto.ec2.blockdevicemapping, boto.manage
import paramiko
import os, sys, time

#boto.set_stream_logger('boto')

def launch_spot_instance(id, profile, spot_wait_sleep=5, instance_wait_sleep=3):
  ec2 = boto.ec2.connect_to_region(profile['region'])

  if not 'key_pair' in profile:
    profile['key_pair'] = ('KP-' + id, 'KP-' + id + '.pem')
    try:
      print >> sys.stderr, 'Creating key pair...',
      keypair = ec2.create_key_pair('KP-' + id)
      keypair.save('.')
      print >> sys.stderr, 'created'
    except boto.exception.EC2ResponseError as e:
      if e.code == 'InvalidKeyPair.Duplicate':
        print >> sys.stderr, 'already exists'
      else:
        raise e

  if not 'security_group' in profile:
    try:
      print >> sys.stderr, 'Creating security group...',
      sc = ec2.create_security_group('SG-' + id, 'Security Group for ' + id)
      for proto, fromport, toport, ip in profile['firewall']:
        sc.authorize(proto, fromport, toport, ip)
      profile['security_group'] = (sc.id, sc.name)
      print >> sys.stderr, 'created'
    except boto.exception.EC2ResponseError as e:
      if e.code == 'InvalidGroup.Duplicate':
        print >> sys.stderr, 'already exists'
        sc = ec2.get_all_security_groups(groupnames=['SG-' + id])[0]
        profile['security_group'] = (sc.id, sc.name)
      else:
        raise e

  existing_requests = ec2.get_all_spot_instance_requests(filters={'launch.group-id': profile['security_group'][0], 'state': ['open', 'active']})
  if existing_requests:
    if len(existing_requests) > 1:
      raise Exception('Too many existing spot requests')
    print >> sys.stderr, 'Reusing existing spot request'
    spot_req_id = existing_requests[0].id
  else:
    bdm = boto.ec2.blockdevicemapping.BlockDeviceMapping()
    bdm['/dev/sda1'] = boto.ec2.blockdevicemapping.BlockDeviceType(volume_type='gp2', size=profile['disk_size'], delete_on_termination=profile['disk_delete_on_termination'])
    bdm['/dev/sdb'] = boto.ec2.blockdevicemapping.BlockDeviceType(ephemeral_name='ephemeral0')
    print >> sys.stderr, 'Requesting spot instance'
    spot_reqs = ec2.request_spot_instances(
      price=profile['price'], image_id=profile['image_id'], instance_type=profile['type'], placement=profile['region'] + profile['availability_zone'],
      security_groups=[profile['security_group'][1]], key_name=profile['key_pair'][0], block_device_map=bdm)
    spot_req_id = spot_reqs[0].id

  print >> sys.stderr, 'Waiting for launch',
  instance_id = None
  spot_tag_added = False
  while not instance_id:
    spot_req = ec2.get_all_spot_instance_requests(request_ids=[spot_req_id])[0]
    if not spot_tag_added:
      spot_req.add_tag('Name', id)
      spot_tag_added = True
    if spot_req.state == 'failed':
      raise Exception('Spot request failed')
    instance_id = spot_req.instance_id
    if not instance_id:
      print >> sys.stderr, '.',
      time.sleep(spot_wait_sleep)
  print >> sys.stderr

  print >> sys.stderr, 'Retrieving instance by id'
  reservations = ec2.get_all_instances(instance_ids=[instance_id])
  instance = reservations[0].instances[0]
  instance.add_tag('Name', id)
  print >> sys.stderr, 'Got instance: ' + str(instance.id) +  ' [' + instance.state + ']'
  print >> sys.stderr, 'Waiting for instance to boot',
  while not instance.state in ['running', 'terminated', 'shutting-down']:
    print >> sys.stderr, '.',
    time.sleep(instance_wait_sleep)
    instance.update()
  print >> sys.stderr
  if instance.state != 'running':
    raise Exception('Instance was terminated')
  return instance

def connect_to_instance(ip, username, key_filename, timeout=10):
  print >> sys.stderr, 'Connecting to SSH [' + ip + '] ',
  client = paramiko.SSHClient()
  client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
  retries = 0
  while retries < 30:
    try:
      print >> sys.stderr, '.',
      client.connect(ip, username=username, key_filename=key_filename, timeout=timeout)
      break
    except:
      retries += 1
  print >> sys.stderr
  return client

def setup_instance(id, instance, file, user_name, key_name):
  script = open(file, 'r').read().replace('\r', '')

  client = connect_to_instance(instance.ip_address, user_name, key_name)
  session = client.get_transport().open_session()
  session.set_combine_stderr(True)

  print >> sys.stderr, 'Running script: ' + os.path.relpath(file, os.getcwd())
  session.exec_command(script)
  stdout = session.makefile()
  try:
    for line in stdout:
      print line.rstrip()
  except (KeyboardInterrupt, SystemExit):
    print >> sys.stderr, 'Ctrl-C, stopping'
  client.close()
  exit_code = session.recv_exit_status()
  print >> sys.stderr, 'Exit code: ' + str(exit_code)
  return exit_code == 0

if __name__ == '__main__':

  profiles = {
    '15G': {
      'region': 'eu-west-1',
      'availability_zone': 'a',
      'price': '0.05',
      'type': 'r3.large',
      'image_id': 'ami-ed82e39e',
      'username': 'ubuntu',
      #'key_pair': ('AWS-EU', 'eu-key.pem'),
      'disk_size': 20,
      'disk_delete_on_termination': True,
      'scripts': [],
      'firewall': [ ('tcp', 22, 22, '0.0.0.0/0') ]
    }
  }

  parser = argparse.ArgumentParser(description='Launch spot instance')
  parser.add_argument('-n', '--name', help='Name', required=True)
  parser.add_argument('-p', '--profile', help='Profile', default=profiles.keys()[0], choices=profiles.keys())
  parser.add_argument('-s', '--script', help='Script path', action='append', default=[])
  parser.add_argument('-i', '--interactive', help='Connect to SSH', action='store_true')
  args = parser.parse_args()

  profile = profiles[args.profile]

  try:
    instance = launch_spot_instance(args.name, profile)
  except boto.exception.NoAuthHandlerFound:
    print >> sys.stderr, 'Error: No credentials found, try setting the AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY environment variables'
    sys.exit(1)

  for script in profile['scripts'] + args.script:
    if not setup_instance(id=args.name, instance=instance, file=script, user_name=profile['username'], key_name=profile['key_pair'][1]):
      break

  if args.interactive:
    print 'ssh ' + profile['username'] + '@' + instance.ip_address + ' -i ' + profile['key_pair'][1] + ' -oStrictHostKeyChecking=no'

The script is less than 200 lines long and should be readable from top to bottom.

Use

Set your Amazon AWS access keys as environment variables.

Or you can read the boto documentation to find out how to store the keys in configuration files.

export AWS_ACCESS_KEY_ID="XXXXXXXXXXXXXXXXXXXX"
export AWS_SECRET_ACCESS_KEY="XXXXXXXXXXXXXXXXXXXXXXXXXX"

Then launch a new spot instance like this

$ ./launch-spot.py -n test -p 15G
Creating key pair... created
Creating security group... created
Requesting spot instance
Waiting for launch . . . . . .
Retrieving instance by id
Got instance: i-15fc09d5 [pending]
Waiting for instance to boot . . . .

-n stands for --name and it is how you can identify this spot instance.

-p stands for --profile and is a collection of settings for launching this instance (region, availability zone, instance type, max spot price, AMI, SSH username, disk size, security group rules, etc.)

The profiles are hard coded in the script. Feel free to modify the script to load them from an external configuration file (in JSON perhaps).

Currently, there is just one profile named 15G

profiles = {
  '15G': {
    'region': 'eu-west-1',
    'availability_zone': 'a',
    'price': '0.05',
    'type': 'r3.large',
    'image_id': 'ami-ed82e39e',
    'username': 'ubuntu',
    #'key_pair': ('AWS-EU', 'eu-key.pem'),
    'disk_size': 20,
    'disk_delete_on_termination': True,
    'scripts': [],
    'firewall': [ ('tcp', 22, 22, '0.0.0.0/0') ]
  }
}

You can run the script again and as long as you use the same name it will resume.

$ ./launch-spot.py -n test -p 15G
Creating key pair... already exists
Creating security group... already exists
Reusing existing spot request
Waiting for launch
Retrieving instance by id
Got instance: i-15fc09d5 [running]
Waiting for instance to boot

To run a script on this instance via SSH use one or more -s arguments.

For example, if you have test.sh

#!/bin/bash
touch /tmp/i-was-here
ls -l /tmp/i-was-here

this will be the output of the script

$ ./launch-spot.py -n test -p 15G -s test.sh
Creating key pair... already exists
Creating security group... already exists
Reusing existing spot request
Waiting for launch
Retrieving instance by id
Got instance: i-15fc09d5 [running]
Waiting for instance to boot
Connecting to SSH [54.74.149.116]  .
Running script: test.sh
-rw-rw-r-- 1 ubuntu ubuntu 0 Oct 10 11:53 /tmp/i-was-here
Exit code: 0

If you want to connect to this instance manually, you can use the -i flag:

$ ./launch-spot.py -n test -p 15G -i
Creating key pair... already exists
Creating security group... already exists
Reusing existing spot request
Waiting for launch
Retrieving instance by id
Got instance: i-15fc09d5 [running]
Waiting for instance to boot
ssh [email protected] -i KP-test.pem -oStrictHostKeyChecking=no

You can copy & paste the last line in your terminal and connect to the instance.

As you can see, if you haven't specified key_pair in the profile, a new key pair will be created with the name KP-test and it will be saved as KP-test.pem.

If there's no security_group in the profile, a new one will be created with the name SG-test and the rules from the firewall profile setting will be applied. At a minimum, SSH from your IP should be allowed.

If you want to stop the spot instance, simply run sudo halt on the instance. The instance, the instance request and disks (if auto terminate is on) will automatically be shut down and terminated.

Scripting

If you use a simple bash script, I suggest using one like this. set -e will stop the execution if an error occurs. If the script finished successfully, it will not be run again.

#!/bin/bash

set -e

if [ ! -f /var/setup.done ]; then

    ...

    sudo touch /var/setup.done

fi

If you share a provisioning script with Vagrant and AWS, you can use this line to check if you are running on AWS EC2 or locally on Vagrant.

export AWS=`curl -s -m 1 http://169.254.169.254/latest/meta-data/instance-id 2> /dev/null`
export VAGRANT=`ls /vagrant 2> /dev/null | head -n 1`

Amazon Ubuntu mirrors and apt-get can be slow and flaky. Especially if you hit CTRL+C while the script is executing apt-get and then run it again.

# this may or may not prevent "hash sum mismatch" errors
sudo rm -rf /var/lib/apt/lists/*
sudo apt-get clean
# amazon mirrors are very slow
sudo sed -i 's/us-east-1.ec2.archive.ubuntu.com/us.archive.ubuntu.com/g' /etc/apt/sources.list
sudo sed -i 's/eu-west-1.ec2.archive.ubuntu.com/ie.archive.ubuntu.com/g' /etc/apt/sources.list
# stay up to date
sudo apt-get update
# in case previous apt-get was interrupted
sudo dpkg --configure -a

Create a file system on the instance disk and move /home, /tmp and /swapfile to it. This can be useful on large SSD backed instance disks.

export AWS=`curl -s -m 1 http://169.254.169.254/latest/meta-data/instance-id 2> /dev/null`
export AWS_INSTANCE_STORE=1

if [ "$AWS" ] && [ "$AWS_INSTANCE_STORE" -e "1" ] && [ ! -d /mnt/instance ]; then
  # aws instance store
  sudo mkfs -t ext4 /dev/xvdb
  sudo mkdir -p /mnt/instance
  sudo mount /dev/xvdb /mnt/instance
  sudo chown -R $USER:$USER /mnt/instance

  # don't use root disk
  [ ! -d /mnt/instance/home ] && sudo mv /home/ubuntu /mnt/instance/home && sudo ln -s /mnt/instance/home /home/ubuntu
  [ ! -d /mnt/instance/tmp ] && sudo mv /tmp/ /mnt/instance/ && sudo ln -s /mnt/instance/tmp /tmp
  [ ! -f /mnt/instance/swapfile ] && sudo touch /mnt/instance/swapfile && sudo ln -s /mnt/instance/swapfile /swapfile
fi

Add swap.

export SWAP="4G"

if [ "$SWAP" ]; then
  sudo fallocate -l $SWAP /swapfile && sudo mkswap /swapfile && sudo swapon /swapfile
  sudo sysctl vm.swappiness=1 && sudo sysctl vm.vfs_cache_pressure=50
fi

Keep /tmp in RAM.

export TMPFS=0
export TMPFS_SWAP="5G"

if [ "$TMPFS" ] && [ "$TMPFS" -eq "1" ]; then
  sudo mount -o defaults,noatime,nosuid,nodev,noexec,mode=1777,size=500G -t tmpfs tmpfs /tmp
  [ "$TMPFS_SWAP" ] && sudo fallocate -l $TMPFS_SWAP /swapfile.tmpfs && sudo mkswap /swapfile.tmpfs && sudo swapon /swapfile.tmpfs
fi

To upload/download files to S3 on AWS and use local files in /vagrant when running in Vagrant, you could do something like this.

export AWS=`curl -s -m 1 http://169.254.169.254/latest/meta-data/instance-id 2> /dev/null`
export AWS_ACCESS_KEY_ID="XXXXXXXXXXXXXXXXXXXX"
export AWS_SECRET_ACCESS_KEY="XXXXXXXXXXXXXXXXXXXXXXXXXX"
export VAGRANT=`ls /vagrant 2> /dev/null | head -n 1`

sudo apt-get install -y python-pip
sudo pip install awscli
[ "$AWS_ACCESS_KEY_ID" ] && [ "$AWS_SECRET_ACCESS_KEY" ] && echo -e "$AWS_ACCESS_KEY_ID\n$AWS_SECRET_ACCESS_KEY\n\n" | aws configure

if [ "$VAGRANT" ]; then
  cp /vagrant/file.tar.gz file.tar.gz
elif [ "$AWS" ]; then
  aws s3 cp s3://bucket/file.tar.gz file.tar.gz
fi

...

if [ "$VAGRANT" ]; then
  cp result.tar.gz /vagrant/result.tar.gz
elif [ "$AWS" ]; then
  aws s3 cp result.tar.gz s3://bucket/result.tar.gz --storage-class REDUCED_REDUNDANCY
fi