[virt-tools-list] [virt-bootstrap] [PATCH v7 17/26] Add virt-builder source
Cedric Bosdonnat
cbosdonnat at suse.com
Tue Aug 29 07:42:48 UTC 2017
On Sat, 2017-08-26 at 21:42 +0100, Radostin Stoyanov wrote:
> Add implementation for virt-builder source which aims to create
> container root file system from VM image build with virt-builder.
>
> Usage examples:
> $ virt-bootstrap virt-builder://fedora-25 /tmp/foo
> $ virt-bootstrap virt-builder://ubuntu-16.04 /tmp/bar --root-password secret
> $ virt-bootstrap virt-builder://fedora-25 /tmp/foo -f qcow2 --idmap 0:1000:10
> $ sudo virt-bootstrap virt-builder://fedora-25 /tmp/foo --idmap 0:1000:10
>
> Tests are also introduced along with the implementation. They cover
> creation of root file system and UID/GID mapping for 'dir' and 'qcow2'
> output format by mocking the build_image() method to avoid the time
> consuming call to virt-builder which might also require network
> connection with function which creates dummy disk image.
> Setting root password is handled by virt-builder and hence the
> introduced test only ensures that the password string is passed
> correctly.
> ---
> src/virtBootstrap/sources/__init__.py | 1 +
> src/virtBootstrap/sources/virt_builder_source.py | 154 ++++++++++++++
> src/virtBootstrap/virt_bootstrap.py | 2 +-
> tests/virt_builder_source.py | 243 +++++++++++++++++++++++
> 4 files changed, 399 insertions(+), 1 deletion(-)
> create mode 100644 src/virtBootstrap/sources/virt_builder_source.py
> create mode 100644 tests/virt_builder_source.py
>
> diff --git a/src/virtBootstrap/sources/__init__.py b/src/virtBootstrap/sources/__init__.py
> index e891e9b..be6b25c 100644
> --- a/src/virtBootstrap/sources/__init__.py
> +++ b/src/virtBootstrap/sources/__init__.py
> @@ -24,3 +24,4 @@ sources - Class definitions which process container image or
>
> from virtBootstrap.sources.file_source import FileSource
> from virtBootstrap.sources.docker_source import DockerSource
> +from virtBootstrap.sources.virt_builder_source import VirtBuilderSource
> diff --git a/src/virtBootstrap/sources/virt_builder_source.py b/src/virtBootstrap/sources/virt_builder_source.py
> new file mode 100644
> index 0000000..0343091
> --- /dev/null
> +++ b/src/virtBootstrap/sources/virt_builder_source.py
> @@ -0,0 +1,154 @@
> +# -*- coding: utf-8 -*-
> +# Authors: Radostin Stoyanov <rstoyanov1 at gmail.com>
> +#
> +# Copyright (C) 2017 Radostin Stoyanov
> +#
> +# This program is free software: you can redistribute it and/or modify
> +# it under the terms of the GNU General Public License as published by
> +# the Free Software Foundation, either version 3 of the License, or
> +# (at your option) any later version.
> +
> +# This program is distributed in the hope that it will be useful,
> +# but WITHOUT ANY WARRANTY; without even the implied warranty of
> +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
> +# GNU General Public License for more details.
> +
> +# You should have received a copy of the GNU General Public License
> +# along with this program. If not, see <http://www.gnu.org/licenses/>.
> +
> +"""
> +VirtBuilderSource aim is to extract the root file system from VM image
> +build with virt-builder from template.
> +"""
> +
> +import os
> +import logging
> +import subprocess
> +import tempfile
> +
> +import guestfs
> +from virtBootstrap import utils
> +
> +
> +# pylint: disable=invalid-name
> +# Create logger
> +logger = logging.getLogger(__name__)
> +
> +
> +class VirtBuilderSource(object):
> + """
> + Extract root file system from image build with virt-builder.
> + """
> + def __init__(self, **kwargs):
> + """
> + Create container rootfs by building VM from virt-builder template
> + and extract the rootfs.
> +
> + @param uri: Template name
> + @param fmt: Format used to store the output [dir, qcow2]
> + @param uid_map: Mappings for UID of files in rootfs
> + @param gid_map: Mappings for GID of files in rootfs
> + @param root_password: Root password to set in rootfs
> + @param progress: Instance of the progress module
> + """
> + # Parsed URIs:
> + # - "virt-builder:///<template>"
> + # - "virt-builder://<template>"
> + # - "virt-builder:/<template>"
> + self.template = kwargs['uri'].netloc or kwargs['uri'].path[1:]
> + self.output_format = kwargs.get('fmt', utils.DEFAULT_OUTPUT_FORMAT)
> + self.uid_map = kwargs.get('uid_map', [])
> + self.gid_map = kwargs.get('gid_map', [])
> + self.root_password = kwargs.get('root_password', None)
> + self.progress = kwargs['progress'].update_progress
> +
> + def build_image(self, output_file):
> + """
> + Build VM from virt-builder template
> + """
> + cmd = ['virt-builder', self.template,
> + '-o', output_file,
> + '--no-network',
> + '--delete', '/dev/*',
> + '--delete', '/boot/*',
> + # Comment out all lines in fstab
> + '--edit', '/etc/fstab:s/^/#/']
> + if self.root_password is not None:
> + cmd += ['--root-password', "password:%s" % self.root_password]
> + self.run_buider(cmd)
> +
> + def run_buider(self, cmd):
typo: s/run_buider/run_builder/
> + """
> + Execute virt-builder command
> + """
> + subprocess.check_call(cmd)
> +
> + def unpack(self, dest):
> + """
> + Build image and extract root file system
> +
> + @param dest: Directory path where output files will be stored.
> + """
> +
> + with tempfile.NamedTemporaryFile(prefix='bootstrap_') as tmp_file:
> + if self.output_format == 'dir':
> + self.progress("Building image", value=0, logger=logger)
> + self.build_image(tmp_file.name)
> + self.progress("Extracting rootfs", value=50, logger=logger)
> + g = guestfs.GuestFS(python_return_dict=True)
> + g.add_drive_opts(tmp_file.name, readonly=False, format='raw')
> + g.launch()
> +
> + # Get the device with file system
> + root_dev = g.inspect_os()
> + if not root_dev:
> + raise Exception("No file system was found")
> + g.mount(root_dev[0], '/')
> +
> + # Extract file system to destination directory
> + g.copy_out('/', dest)
> +
> + g.umount('/')
> + g.shutdown()
> +
> + self.progress("Extraction completed successfully!",
> + value=100, logger=logger)
> + logger.info("Files are stored in: %s", dest)
> +
> + elif self.output_format == 'qcow2':
> + output_file = os.path.join(dest, 'layer-0.qcow2')
> +
> + self.progress("Building image", value=0, logger=logger)
> + self.build_image(tmp_file.name)
> +
> + self.progress("Extracting rootfs", value=50, logger=logger)
> + g = guestfs.GuestFS(python_return_dict=True)
> + g.add_drive_opts(tmp_file.name, readonly=True, format='raw')
> + # Create qcow2 disk image
> + g.disk_create(
> + filename=output_file,
> + format='qcow2',
> + size=os.path.getsize(tmp_file.name)
> + )
> + g.add_drive_opts(output_file, readonly=False, format='qcow2')
> + g.launch()
> + # Get the device with file system
> + root_dev = g.inspect_os()
> + if not root_dev:
> + raise Exception("No file system was found")
> + output_dev = g.list_devices()[1]
> + # Copy the file system to the new qcow2 disk
> + g.copy_device_to_device(root_dev[0], output_dev, sparse=True)
> + g.shutdown()
> +
> + # UID/GID mapping
> + if self.uid_map or self.gid_map:
> + logger.info("Mapping UID/GID")
> + utils.map_id_in_image(1, dest, self.uid_map, self.gid_map)
> +
> + self.progress("Extraction completed successfully!", value=100,
> + logger=logger)
> + logger.info("Image is stored in: %s", output_file)
> +
> + else:
> + raise Exception("Unknown format:" + self.output_format)
> diff --git a/src/virtBootstrap/virt_bootstrap.py b/src/virtBootstrap/virt_bootstrap.py
> index e387842..fe95f5e 100755
> --- a/src/virtBootstrap/virt_bootstrap.py
> +++ b/src/virtBootstrap/virt_bootstrap.py
> @@ -62,7 +62,7 @@ def get_source(source_type):
> Get object which match the source type
> """
> try:
> - class_name = "%sSource" % source_type.capitalize()
> + class_name = "%sSource" % source_type.title().replace('-', '')
> clazz = getattr(sources, class_name)
> return clazz
> except Exception:
> diff --git a/tests/virt_builder_source.py b/tests/virt_builder_source.py
> new file mode 100644
> index 0000000..3cd812b
> --- /dev/null
> +++ b/tests/virt_builder_source.py
> @@ -0,0 +1,243 @@
> +# -*- coding: utf-8 -*-
> +# Authors: Radostin Stoyanov <rstoyanov1 at gmail.com>
> +#
> +# Copyright (C) 2017 Radostin Stoyanov
> +#
> +# This program is free software: you can redistribute it and/or modify
> +# it under the terms of the GNU General Public License as published by
> +# the Free Software Foundation, either version 3 of the License, or
> +# (at your option) any later version.
> +
> +# This program is distributed in the hope that it will be useful,
> +# but WITHOUT ANY WARRANTY; without even the implied warranty of
> +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
> +# GNU General Public License for more details.
> +
> +# You should have received a copy of the GNU General Public License
> +# along with this program. If not, see <http://www.gnu.org/licenses/>.
> +
> +
> +"""
> +Tests which aim to exercise the extraction of root file system from disk image
> +created with virt-builder.
> +
> +Brief description of these tests:
> +1. Create dummy root file system on raw disk image.
> +2. Create index file of local repository for virt-builder.
> +3. Call bootstrap() with modified virt-builder commnad to use local repository
> + as source.
> +4. Check the result.
> +"""
> +
> +import copy
> +import platform
> +import os
> +import shutil
> +import tempfile
> +import unittest
> +import subprocess
> +
> +import guestfs
> +
> +from . import virt_bootstrap
> +from . import mock
> +from . import DEFAULT_FILE_MODE
> +from . import ROOTFS_TREE
> +from . import Qcow2ImageAccessor
> +from . import NOT_ROOT
> +
> +
> +# pylint: disable=invalid-name, too-many-instance-attributes
> +class TestVirtBuilderSource(Qcow2ImageAccessor):
> + """
> + Test cases for virt-builder source.
> + """
> +
> + def create_local_repository(self):
> + """
> + Create raw disk image with dummy root file system and index file which
> + contains the metadata used by virt-builder.
> + """
> + g = guestfs.GuestFS(python_return_dict=True)
> + g.disk_create(
> + self.image['path'],
> + format=self.image['format'],
> + size=self.image['size']
> + )
> + g.add_drive(
> + self.image['path'],
> + readonly=False,
> + format=self.image['format']
> + )
> + g.launch()
> + g.mkfs('ext2', '/dev/sda')
> + g.mount('/dev/sda', '/')
> + for user in self.rootfs_tree:
> + usr_uid = self.rootfs_tree[user]['uid']
> + usr_gid = self.rootfs_tree[user]['gid']
> +
> + for member in self.rootfs_tree[user]['dirs']:
> + dir_name = '/' + member
> + g.mkdir_p(dir_name)
> + g.chown(usr_uid, usr_gid, dir_name)
> +
> + for member in self.rootfs_tree[user]['files']:
> + if isinstance(member, tuple):
> + m_name, m_permissions, m_data = member
> + file_name = '/' + m_name
> + g.write(file_name, m_data)
> + g.chmod(m_permissions & 0o777, file_name)
> + else:
> + file_name = '/' + member
> + g.touch(file_name)
> + g.chmod(DEFAULT_FILE_MODE & 0o777, file_name)
> +
> + g.chown(usr_uid, usr_gid, file_name)
> +
> + # Create index file
> + with open(self.repo_index, 'w') as index_file:
> + index_file.write(
> + '[{template}]\n'
> + 'name=Test\n'
> + 'arch={arch}\n'
> + 'file={filename}\n' # Relative (not real) path must be used.
> + 'format={format}\n'
> + 'expand=/dev/sda\n'
> + 'size={size}\n'.format(**self.image)
> + # The new line at the end of the index file is required.
> + # Otherwise, virt-builder will return "syntax error".
> + )
> +
> + def setUp(self):
> + self.rootfs_tree = copy.deepcopy(ROOTFS_TREE)
> +
> + self.fmt = None
> + self.uid_map = None
> + self.gid_map = None
> + self.root_password = None
> + self.checked_members = set()
> +
> + self.dest_dir = tempfile.mkdtemp('_bootstrap_dest')
> + self.repo_dir = tempfile.mkdtemp('_local_builder_repo')
> + # Set permissions for tmp directories to avoid
> + # "Permission denied" errors from Libvirt.
> + os.chmod(self.repo_dir, 0o755)
> + os.chmod(self.dest_dir, 0o755)
> + self.repo_index = os.path.join(self.repo_dir, 'index')
> +
> + self.image = {
> + 'template': 'test',
> + 'filename': 'test.img',
> + 'path': os.path.join(self.repo_dir, 'test.img'),
> + 'format': 'raw',
> + 'size': (1 * 1024 * 1024),
> + 'arch': platform.processor(),
> + }
> + self.create_local_repository()
> +
> + def mocked_run_builder(self, cmd):
> + """
> + Modify the virt-builder command to use the dummy disk image
> + and capture the 'stdout'.
> + """
> + subprocess.check_call(
> + cmd + [
> + '--source',
> + 'file://%s' % self.repo_index,
> + '--no-check-signature',
> + '--no-cache'
> + ],
> + stdout=subprocess.PIPE
> + )
> +
> + def tearDown(self):
> + """
> + Clean up
> + """
> + shutil.rmtree(self.repo_dir)
> + shutil.rmtree(self.dest_dir)
> +
> + def call_bootstrap(self):
> + """
> + Mock out run_builder() with mocked_run_builder() and
> + call bootstrap() method from virtBootstrap.
> + """
> + # By default virt-builder sets random root password which leads to
> + # modification in /etc/shadow file. If we don't test this we simplify
> + # the test by not adding shadow file in our dummy root file system.
> + if not self.root_password:
> + self.rootfs_tree['root']['files'] = ['etc/hosts', 'etc/fstab']
> +
> + target = ('virtBootstrap.sources.VirtBuilderSource.run_buider')
typo again: s/run_buider/run_builder/
> + with mock.patch(target) as m_run_builder:
> + m_run_builder.side_effect = self.mocked_run_builder
> +
> + virt_bootstrap.bootstrap(
> + progress_cb=mock.Mock(),
> + uri='virt-builder://%s' % self.image['template'],
> + dest=self.dest_dir,
> + fmt=self.fmt,
> + gid_map=self.gid_map,
> + uid_map=self.uid_map,
> + root_password=self.root_password
> + )
> +
> + def test_dir_extract_rootfs(self):
> + """
> + Ensures that the root file system is extracted correctly.
> + """
> + self.fmt = 'dir'
> + self.call_bootstrap()
> + self.check_rootfs(skip_ownership=NOT_ROOT)
> +
> + @unittest.skipIf(NOT_ROOT, "Root privileges required")
> + def test_dir_ownership_mapping(self):
> + """
> + Ensures that UID/GID mapping is applied to extracted root file system.
> + """
> + self.fmt = 'dir'
> + self.gid_map = [[1000, 2000, 10], [0, 1000, 10], [500, 500, 10]]
> + self.uid_map = [[1000, 2000, 10], [0, 1000, 10], [500, 500, 10]]
> + self.call_bootstrap()
> + self.apply_mapping()
> + self.check_rootfs()
> +
> + def test_dir_setting_root_password(self):
> + """
> + Ensures that password for root is set correctly.
> + """
> + self.root_password = 'my secret root password'
> + self.fmt = 'dir'
> + self.call_bootstrap()
> + self.validate_shadow_file()
> +
> + def test_qcow2_build_image(self):
> + """
> + Ensures that the root file system is copied correctly within single
> + partition qcow2 image.
> + """
> + self.fmt = 'qcow2'
> + self.call_bootstrap()
> + self.check_qcow2_images(self.get_image_path())
> +
> + def test_qcow2_ownership_mapping(self):
> + """
> + Ensures that UID/GID mapping is applied in qcow2 image "layer-1.qcow2".
> + """
> + self.fmt = 'qcow2'
> + self.gid_map = [[1000, 2000, 10], [0, 1000, 10], [500, 500, 10]]
> + self.uid_map = [[1000, 2000, 10], [0, 1000, 10], [500, 500, 10]]
> + self.call_bootstrap()
> + self.apply_mapping()
> + self.check_qcow2_images(self.get_image_path(1))
> +
> + def test_qcow2_setting_root_password(self):
> + """
> + Ensures that the root password is set in the shadow file of
> + "layer-1.qcow2"
> + """
> + self.fmt = 'qcow2'
> + self.root_password = "My secret password"
> + self.call_bootstrap()
> + self.check_image = self.validate_shadow_file_in_image
> + self.check_qcow2_images(self.get_image_path())
ACK with the typos fixed
--
Cedric
More information about the virt-tools-list
mailing list