[libvirt] [PATCH sandbox v5 06/20] Image: Add download function
Cedric Bosdonnat
cbosdonnat at suse.com
Wed Sep 9 11:50:11 UTC 2015
On Tue, 2015-09-08 at 17:29 +0100, Daniel P. Berrange wrote:
> From: Eren Yagdiran <erenyagdiran at gmail.com>
>
> Refactor download function from virt-sandbox-image to use
> the newly introduced Source abstract class. The docker-specific
> download code is moved to a new DockerSource class.
>
> Signed-off-by: Daniel P. Berrange <berrange at redhat.com>
> ---
> libvirt-sandbox/image/cli.py | 204 ++++---------------------
> libvirt-sandbox/image/sources/DockerSource.py | 209 ++++++++++++++++++++++++++
> libvirt-sandbox/image/sources/Makefile.am | 1 +
> libvirt-sandbox/image/sources/Source.py | 15 ++
> 4 files changed, 257 insertions(+), 172 deletions(-)
> create mode 100644 libvirt-sandbox/image/sources/DockerSource.py
>
> diff --git a/libvirt-sandbox/image/cli.py b/libvirt-sandbox/image/cli.py
> index de34321..7af617e 100755
> --- a/libvirt-sandbox/image/cli.py
> +++ b/libvirt-sandbox/image/cli.py
> @@ -69,176 +69,6 @@ def debug(msg):
> def info(msg):
> sys.stdout.write(msg)
>
> -def get_url(server, path, headers):
> - url = "https://" + server + path
> - debug(" Fetching %s..." % url)
> - req = urllib2.Request(url=url)
> -
> - if json:
> - req.add_header("Accept", "application/json")
> -
> - for h in headers.keys():
> - req.add_header(h, headers[h])
> -
> - return urllib2.urlopen(req)
> -
> -def get_json(server, path, headers):
> - try:
> - res = get_url(server, path, headers)
> - data = json.loads(res.read())
> - debug("OK\n")
> - return (data, res)
> - except Exception, e:
> - debug("FAIL %s\n" % str(e))
> - raise
> -
> -def save_data(server, path, headers, dest, checksum=None, datalen=None):
> - try:
> - res = get_url(server, path, headers)
> -
> - csum = None
> - if checksum is not None:
> - csum = hashlib.sha256()
> -
> - pattern = [".", "o", "O", "o"]
> - patternIndex = 0
> - donelen = 0
> -
> - with open(dest, "w") as f:
> - while 1:
> - buf = res.read(1024*64)
> - if not buf:
> - break
> - if csum is not None:
> - csum.update(buf)
> - f.write(buf)
> -
> - if datalen is not None:
> - donelen = donelen + len(buf)
> - debug("\x1b[s%s (%5d Kb of %5d Kb)\x1b8" % (
> - pattern[patternIndex], (donelen/1024), (datalen/1024)
> - ))
> - patternIndex = (patternIndex + 1) % 4
> -
> - debug("\x1b[K")
> - if csum is not None:
> - csumstr = "sha256:" + csum.hexdigest()
> - if csumstr != checksum:
> - debug("FAIL checksum '%s' does not match '%s'" % (csumstr, checksum))
> - os.remove(dest)
> - raise IOError("Checksum '%s' for data does not match '%s'" % (csumstr, checksum))
> - debug("OK\n")
> - return res
> - except Exception, e:
> - debug("FAIL %s\n" % str(e))
> - raise
> -
> -
> -def download_template(name, server, destdir):
> - tag = "latest"
> -
> - offset = name.find(':')
> - if offset != -1:
> - tag = name[offset + 1:]
> - name = name[0:offset]
> -
> - # First we must ask the index server about the image name. THe
> - # index server will return an auth token we can use when talking
> - # to the registry server. We need this token even when anonymous
> - try:
> - (data, res) = get_json(server, "/v1/repositories/" + name + "/images",
> - {"X-Docker-Token": "true"})
> - except urllib2.HTTPError, e:
> - raise ValueError(["Image '%s' does not exist" % name])
> -
> - registryserver = res.info().getheader('X-Docker-Endpoints')
> - token = res.info().getheader('X-Docker-Token')
> - checksums = {}
> - for layer in data:
> - pass
> - # XXX Checksums here don't appear to match the data in
> - # image download later. Find out what these are sums of
> - #checksums[layer["id"]] = layer["checksum"]
> -
> - # Now we ask the registry server for the list of tags associated
> - # with the image. Tags usually reflect some kind of version of
> - # the image, but they aren't officially "versions". There is
> - # always a "latest" tag which is the most recent upload
> - #
> - # We must pass in the auth token from the index server. This
> - # token can only be used once, and we're given a cookie back
> - # in return to use for later RPC calls.
> - (data, res) = get_json(registryserver, "/v1/repositories/" + name + "/tags",
> - { "Authorization": "Token " + token })
> -
> - cookie = res.info().getheader('Set-Cookie')
> -
> - if not tag in data:
> - raise ValueError(["Tag '%s' does not exist for image '%s'" % (tag, name)])
> - imagetagid = data[tag]
> -
> - # Only base images are self-contained, most images reference one
> - # or more parents, in a linear stack. Here we are getting the list
> - # of layers for the image with the tag we used.
> - (data, res) = get_json(registryserver, "/v1/images/" + imagetagid + "/ancestry",
> - { "Authorization": "Token " + token })
> -
> - if data[0] != imagetagid:
> - raise ValueError(["Expected first layer id '%s' to match image id '%s'",
> - data[0], imagetagid])
> -
> - try:
> - createdFiles = []
> - createdDirs = []
> -
> - for layerid in data:
> - templatedir = destdir + "/" + layerid
> - if not os.path.exists(templatedir):
> - os.mkdir(templatedir)
> - createdDirs.append(templatedir)
> -
> - jsonfile = templatedir + "/template.json"
> - datafile = templatedir + "/template.tar.gz"
> -
> - if not os.path.exists(jsonfile) or not os.path.exists(datafile):
> - # The '/json' URL gives us some metadata about the layer
> - res = save_data(registryserver, "/v1/images/" + layerid + "/json",
> - { "Authorization": "Token " + token }, jsonfile)
> - createdFiles.append(jsonfile)
> - layersize = int(res.info().getheader("Content-Length"))
> -
> - datacsum = None
> - if layerid in checksums:
> - datacsum = checksums[layerid]
> -
> - # and the '/layer' URL is the actual payload, provided
> - # as a tar.gz archive
> - save_data(registryserver, "/v1/images/" + layerid + "/layer",
> - { "Authorization": "Token " + token }, datafile, datacsum, layersize)
> - createdFiles.append(datafile)
> -
> - # Strangely the 'json' data for a layer doesn't include
> - # its actual name, so we save that in a json file of our own
> - index = {
> - "name": name,
> - }
> -
> - indexfile = destdir + "/" + imagetagid + "/index.json"
> - with open(indexfile, "w") as f:
> - f.write(json.dumps(index))
> - except Exception, e:
> - for f in createdFiles:
> - try:
> - os.remove(f)
> - except:
> - pass
> - for d in createdDirs:
> - try:
> - os.rmdir(d)
> - except:
> - pass
> -
> -
> def delete_template(name, destdir):
> imageusage = {}
> imageparent = {}
> @@ -342,8 +172,16 @@ def create_template(name, imagepath, format, destdir):
> parentImage = templateImage
>
> def download(args):
> - info("Downloading %s from %s to %s\n" % (args.template, default_index_server, default_template_dir))
> - download_template(args.template, default_index_server, default_template_dir)
> + try:
> + dynamic_source_loader(args.source).download_template(templatename=args.template,
> + templatedir=args.template_dir,
> + registry=args.registry,
> + username=args.username,
> + password=args.password)
> + except IOError,e:
> + print "Source %s cannot be found in given path" %args.source
> + except Exception,e:
> + print "Download Error %s" % str(e)
>
> def delete(args):
> info("Deleting %s from %s\n" % (args.template, default_template_dir))
> @@ -357,10 +195,32 @@ def requires_template(parser):
> parser.add_argument("template",
> help=_("name of the template"))
>
> +def requires_source(parser):
> + parser.add_argument("-s","--source",
> + default="docker",
> + help=_("name of the template"))
> +
> +def requires_auth_conn(parser):
> + parser.add_argument("-r","--registry",
> + help=_("Url of the custom registry"))
This wording really sounds docker-specific. The registry word only fits
docker terminology, would surely not apply to virt-builder or appc case.
Something like "images storage" would may be more generic. The problem
is that "repository" has a special meaning in the docker terminology.
> + parser.add_argument("-u","--username",
> + help=_("Username for the custom registry"))
> + parser.add_argument("-p","--password",
> + help=_("Password for the custom registry"))
> +
> +def requires_template_dir(parser):
> + global default_template_dir
> + parser.add_argument("-t","--template-dir",
> + default=default_template_dir,
> + help=_("Template directory for saving templates"))
> +
> def gen_download_args(subparser):
> parser = subparser.add_parser("download",
> help=_("Download template data"))
> requires_template(parser)
> + requires_source(parser)
> + requires_auth_conn(parser)
> + requires_template_dir(parser)
> parser.set_defaults(func=download)
>
> def gen_delete_args(subparser):
> diff --git a/libvirt-sandbox/image/sources/DockerSource.py b/libvirt-sandbox/image/sources/DockerSource.py
> new file mode 100644
> index 0000000..37b40dc
> --- /dev/null
> +++ b/libvirt-sandbox/image/sources/DockerSource.py
> @@ -0,0 +1,209 @@
> +#!/usr/bin/python
> +# -*- coding: utf-8 -*-
> +#
> +# Copyright (C) 2015 Universitat Politècnica de Catalunya.
> +#
> +# This library is free software; you can redistribute it and/or
> +# modify it under the terms of the GNU Lesser General Public
> +# License as published by the Free Software Foundation; either
> +# version 2.1 of the License, or (at your option) any later version.
> +#
> +# This library is distributed in the hope that it will be useful,
> +# but WITHOUT ANY WARRANTY; without even the implied warranty of
> +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
> +# Lesser General Public License for more details.
> +#
> +# You should have received a copy of the GNU Lesser General Public
> +# License along with this library; if not, write to the Free Software
> +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
> +#
> +# Author: Eren Yagdiran <erenyagdiran at gmail.com>
> +#
> +
> +from Source import Source
> +import urllib2
> +import sys
> +import json
> +import traceback
> +import os
> +import subprocess
> +import shutil
> +
> +class DockerSource(Source):
> +
> + www_auth_username = None
> + www_auth_password = None
> +
> + def __init__(self):
> + self.default_index_server = "index.docker.io"
> +
> + def _check_cert_validate(self):
> + major = sys.version_info.major
> + SSL_WARNING = "SSL certificates couldn't be validated by default. You need to have 2.7.9/3.4.3 or higher"
> + SSL_WARNING +="\nSee https://bugs.python.org/issue22417\n"
> + py2_7_9_hexversion = 34015728
> + py3_4_3_hexversion = 50594800
> + if (major == 2 and sys.hexversion < py2_7_9_hexversion) or (major == 3 and sys.hexversion < py3_4_3_hexversion):
> + sys.stderr.write(SSL_WARNING)
> +
> + def download_template(self, templatename, templatedir,
> + registry=None, username=None, password=None):
> + if registry is None:
> + registry = self.default_index_server
> +
> + if username is not None:
> + self.www_auth_username = username
> + self.www_auth_password = password
> +
> + self._check_cert_validate()
> + tag = "latest"
> + offset = templatename.find(':')
> + if offset != -1:
> + tag = templatename[offset + 1:]
> + templatename = templatename[0:offset]
> + try:
> + (data, res) = self._get_json(registry, "/v1/repositories/" + templatename + "/images",
> + {"X-Docker-Token": "true"})
> + except urllib2.HTTPError, e:
> + raise ValueError(["Image '%s' does not exist" % templatename])
> +
> + registryendpoint = res.info().getheader('X-Docker-Endpoints')
> + token = res.info().getheader('X-Docker-Token')
> + checksums = {}
> + for layer in data:
> + pass
> + (data, res) = self._get_json(registryendpoint, "/v1/repositories/" + templatename + "/tags",
> + { "Authorization": "Token " + token })
> +
> + cookie = res.info().getheader('Set-Cookie')
> +
> + if not tag in data:
> + raise ValueError(["Tag '%s' does not exist for image '%s'" % (tag, templatename)])
> + imagetagid = data[tag]
> +
> + (data, res) = self._get_json(registryendpoint, "/v1/images/" + imagetagid + "/ancestry",
> + { "Authorization": "Token "+token })
> +
> + if data[0] != imagetagid:
> + raise ValueError(["Expected first layer id '%s' to match image id '%s'",
> + data[0], imagetagid])
> +
> + try:
> + createdFiles = []
> + createdDirs = []
> +
> + for layerid in data:
> + layerdir = templatedir + "/" + layerid
> + if not os.path.exists(layerdir):
> + os.makedirs(layerdir)
> + createdDirs.append(layerdir)
> +
> + jsonfile = layerdir + "/template.json"
> + datafile = layerdir + "/template.tar.gz"
> +
> + if not os.path.exists(jsonfile) or not os.path.exists(datafile):
> + res = self._save_data(registryendpoint, "/v1/images/" + layerid + "/json",
> + { "Authorization": "Token " + token }, jsonfile)
> + createdFiles.append(jsonfile)
> +
> + layersize = int(res.info().getheader("Content-Length"))
> +
> + datacsum = None
> + if layerid in checksums:
> + datacsum = checksums[layerid]
> +
> + self._save_data(registryendpoint, "/v1/images/" + layerid + "/layer",
> + { "Authorization": "Token "+token }, datafile, datacsum, layersize)
> + createdFiles.append(datafile)
> +
> + index = {
> + "name": templatename,
> + }
> +
> + indexfile = templatedir + "/" + imagetagid + "/index.json"
> + print("Index file " + indexfile)
> + with open(indexfile, "w") as f:
> + f.write(json.dumps(index))
> + except Exception as e:
> + traceback.print_exc()
> + for f in createdFiles:
> + try:
> + os.remove(f)
> + except:
> + pass
> + for d in createdDirs:
> + try:
> + shutil.rmtree(d)
> + except:
> + pass
> + def _save_data(self,server, path, headers, dest, checksum=None, datalen=None):
> + try:
> + res = self._get_url(server, path, headers)
> +
> + csum = None
> + if checksum is not None:
> + csum = hashlib.sha256()
> +
> + pattern = [".", "o", "O", "o"]
> + patternIndex = 0
> + donelen = 0
> +
> + with open(dest, "w") as f:
> + while 1:
> + buf = res.read(1024*64)
> + if not buf:
> + break
> + if csum is not None:
> + csum.update(buf)
> + f.write(buf)
> +
> + if datalen is not None:
> + donelen = donelen + len(buf)
> + debug("\x1b[s%s (%5d Kb of %5d Kb)\x1b8" % (
> + pattern[patternIndex], (donelen/1024), (datalen/1024)
> + ))
> + patternIndex = (patternIndex + 1) % 4
> +
> + debug("\x1b[K")
> + if csum is not None:
> + csumstr = "sha256:" + csum.hexdigest()
> + if csumstr != checksum:
> + debug("FAIL checksum '%s' does not match '%s'" % (csumstr, checksum))
> + os.remove(dest)
> + raise IOError("Checksum '%s' for data does not match '%s'" % (csumstr, checksum))
> + debug("OK\n")
> + return res
> + except Exception, e:
> + debug("FAIL %s\n" % str(e))
> + raise
> +
> + def _get_url(self,server, path, headers):
> + url = "https://" + server + path
> + debug("Fetching %s..." % url)
> +
> + req = urllib2.Request(url=url)
> + if json:
> + req.add_header("Accept", "application/json")
> + for h in headers.keys():
> + req.add_header(h, headers[h])
> +
> + #www Auth header starts
> + if self.www_auth_username is not None:
> + base64string = base64.encodestring('%s:%s' % (self.www_auth_username, self.www_auth_password)).replace('\n', '')
> + req.add_header("Authorization", "Basic %s" % base64string)
> + #www Auth header finish
> +
> + return urllib2.urlopen(req)
> +
> + def _get_json(self,server, path, headers):
> + try:
> + res = self._get_url(server, path, headers)
> + data = json.loads(res.read())
> + debug("OK\n")
> + return (data, res)
> + except Exception, e:
> + debug("FAIL %s\n" % str(e))
> + raise
> +
> +def debug(msg):
> + sys.stderr.write(msg)
> diff --git a/libvirt-sandbox/image/sources/Makefile.am b/libvirt-sandbox/image/sources/Makefile.am
> index 48d0f33..069557d 100644
> --- a/libvirt-sandbox/image/sources/Makefile.am
> +++ b/libvirt-sandbox/image/sources/Makefile.am
> @@ -3,6 +3,7 @@ pythonimagedir = $(pythondir)/libvirt_sandbox/image/sources
> pythonimage_DATA = \
> __init__.py \
> Source.py \
> + DockerSource.py \
> $(NULL)
>
> EXTRA_DIST = $(pythonimage_DATA)
> diff --git a/libvirt-sandbox/image/sources/Source.py b/libvirt-sandbox/image/sources/Source.py
> index f12b0eb..81f5176 100644
> --- a/libvirt-sandbox/image/sources/Source.py
> +++ b/libvirt-sandbox/image/sources/Source.py
> @@ -31,3 +31,18 @@ class Source():
> __metaclass__ = ABCMeta
> def __init__(self):
> pass
> +
> + @abstractmethod
> + def download_template(self, templatename, templatedir,
> + registry=None, username=None, password=None):
> + """
> + :param templatename: name of the template image to download
> + :param templatedir: local directory path in which to store the template
> + :param registry: optional hostname of image registry server
> + :param username: optional username to authenticate against registry server
> + :param password: optional password to authenticate against registry server
> +
> + Download a template from the registry, storing it in the local
> + filesystem
> + """
> + pass
ACK, but may need some thinking on the "Registry" word.
--
Cedric
More information about the libvir-list
mailing list