[virt-tools-list] [virt-bootstrap] [PATCH v6 17/26] Add virt-builder source
Radostin Stoyanov
rstoyanov1 at gmail.com
Thu Aug 17 09:39:55 UTC 2017
Add implementation for virt-builder source which aims to create
container root file system from VM image build with virt-builder.
Usage examples:
virt-bootstrap virt-builder://fedora-25 /tmp/foo
virt-bootstrap virt-builder://ubuntu-16.04 /tmp/bar --root-password secret
virt-bootstrap virt-builder://fedora-25 /tmp/foo -f qcow2 --idmap 0:1000:10
sudo virt-bootstrap virt-builder://fedora-25 /tmp/foo --idmap 0:1000:10
Tests are also introduced along with the implementation. They cover
creation of root file system and UID/GID mapping for 'dir' and 'qcow2'
output format by mocking the build_image() method to avoid the time
consuming call to virt-builder which might also require network
connection with function which creates dummy disk image.
Setting root password is handled by virt-builder and hence the
introduced test only ensures that the password string is passed
correctly.
---
src/virtBootstrap/sources/__init__.py | 1 +
src/virtBootstrap/sources/virt_builder_source.py | 148 +++++++++++++++
src/virtBootstrap/virt_bootstrap.py | 4 +-
tests/__init__.py | 23 +++
tests/file_source.py | 28 +--
tests/virt_builder_source.py | 228 +++++++++++++++++++++++
6 files changed, 406 insertions(+), 26 deletions(-)
create mode 100644 src/virtBootstrap/sources/virt_builder_source.py
create mode 100644 tests/virt_builder_source.py
diff --git a/src/virtBootstrap/sources/__init__.py b/src/virtBootstrap/sources/__init__.py
index e891e9b..be6b25c 100644
--- a/src/virtBootstrap/sources/__init__.py
+++ b/src/virtBootstrap/sources/__init__.py
@@ -24,3 +24,4 @@ sources - Class definitions which process container image or
from virtBootstrap.sources.file_source import FileSource
from virtBootstrap.sources.docker_source import DockerSource
+from virtBootstrap.sources.virt_builder_source import VirtBuilderSource
diff --git a/src/virtBootstrap/sources/virt_builder_source.py b/src/virtBootstrap/sources/virt_builder_source.py
new file mode 100644
index 0000000..780ffb1
--- /dev/null
+++ b/src/virtBootstrap/sources/virt_builder_source.py
@@ -0,0 +1,148 @@
+# -*- coding: utf-8 -*-
+# Authors: Radostin Stoyanov <rstoyanov1 at gmail.com>
+#
+# Copyright (C) 2017 Radostin Stoyanov
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+"""
+VirtBuilderSource aim is to extract the root file system from VM image
+build with virt-builder from template.
+"""
+
+import os
+import logging
+import subprocess
+import tempfile
+
+import guestfs
+from virtBootstrap import utils
+
+
+# pylint: disable=invalid-name
+# Create logger
+logger = logging.getLogger(__name__)
+
+
+class VirtBuilderSource(object):
+ """
+ Extract root file system from image build with virt-builder.
+ """
+ def __init__(self, **kwargs):
+ """
+ Create container rootfs by building VM from virt-builder template
+ and extract the rootfs.
+
+ @param uri: Template name
+ @param fmt: Format used to store the output [dir, qcow2]
+ @param uid_map: Mappings for UID of files in rootfs
+ @param gid_map: Mappings for GID of files in rootfs
+ @param root_password: Root password to set in rootfs
+ @param progress: Instance of the progress module
+ """
+ # Parsed URIs:
+ # - "virt-builder:///<template>"
+ # - "virt-builder://<template>"
+ # - "virt-builder:/<template>"
+ self.template = kwargs['uri'].netloc or kwargs['uri'].path[1:]
+ self.output_format = kwargs.get('fmt', utils.DEFAULT_OUTPUT_FORMAT)
+ self.uid_map = kwargs.get('uid_map', [])
+ self.gid_map = kwargs.get('gid_map', [])
+ self.root_password = kwargs.get('root_password', None)
+ self.progress = kwargs['progress'].update_progress
+
+ def build_image(self, output_file):
+ """
+ Build VM from virt-builder template
+ """
+ cmd = ['virt-builder', self.template,
+ '-o', output_file,
+ '--no-network',
+ '--delete', '/dev/*',
+ '--delete', '/boot/*',
+ # Comment out all lines in fstab
+ '--edit', '/etc/fstab:s/^/#/']
+ if self.root_password is not None:
+ cmd += ['--root-password', "password:%s" % self.root_password]
+ subprocess.check_call(cmd)
+
+ def unpack(self, dest):
+ """
+ Build image and extract root file system
+
+ @param dest: Directory path where output files will be stored.
+ """
+
+ with tempfile.NamedTemporaryFile(prefix='bootstrap_') as tmp_file:
+ if self.output_format == 'dir':
+ self.progress("Building image", value=0, logger=logger)
+ self.build_image(tmp_file.name)
+ self.progress("Extracting rootfs", value=50, logger=logger)
+ g = guestfs.GuestFS(python_return_dict=True)
+ g.add_drive_opts(tmp_file.name, readonly=False, format='raw')
+ g.launch()
+
+ # Get the device with file system
+ root_dev = g.inspect_os()
+ if not root_dev:
+ raise Exception("No file system was found")
+ g.mount(root_dev[0], '/')
+
+ # Extract file system to destination directory
+ g.copy_out('/', dest)
+
+ g.umount('/')
+ g.shutdown()
+
+ self.progress("Extraction completed successfully!",
+ value=100, logger=logger)
+ logger.info("Files are stored in: %s", dest)
+
+ elif self.output_format == 'qcow2':
+ output_file = os.path.join(dest, 'layer-0.qcow2')
+
+ self.progress("Building image", value=0, logger=logger)
+ self.build_image(tmp_file.name)
+
+ self.progress("Extracting rootfs", value=50, logger=logger)
+ g = guestfs.GuestFS(python_return_dict=True)
+ g.add_drive_opts(tmp_file.name, readonly=True, format='raw')
+ # Create qcow2 disk image
+ g.disk_create(
+ filename=output_file,
+ format='qcow2',
+ size=os.path.getsize(tmp_file.name)
+ )
+ g.add_drive_opts(output_file, readonly=False, format='qcow2')
+ g.launch()
+ # Get the device with file system
+ root_dev = g.inspect_os()
+ if not root_dev:
+ raise Exception("No file system was found")
+ output_dev = g.list_devices()[1]
+ # Copy the file system to the new qcow2 disk
+ g.copy_device_to_device(root_dev[0], output_dev, sparse=True)
+ g.shutdown()
+
+ # UID/GID mapping
+ if self.uid_map or self.gid_map:
+ logger.info("Mapping UID/GID")
+ utils.map_id_in_image(1, dest, self.uid_map, self.gid_map)
+
+ self.progress("Extraction completed successfully!", value=100,
+ logger=logger)
+ logger.info("Image is stored in: %s", output_file)
+
+ else:
+ raise Exception("Unknown format:" + self.output_format)
diff --git a/src/virtBootstrap/virt_bootstrap.py b/src/virtBootstrap/virt_bootstrap.py
index e387842..68e6516 100755
--- a/src/virtBootstrap/virt_bootstrap.py
+++ b/src/virtBootstrap/virt_bootstrap.py
@@ -62,7 +62,7 @@ def get_source(source_type):
Get object which match the source type
"""
try:
- class_name = "%sSource" % source_type.capitalize()
+ class_name = "%sSource" % source_type.title().replace('-', '')
clazz = getattr(sources, class_name)
return clazz
except Exception:
@@ -179,6 +179,8 @@ def main():
docker://docker.io/fedora
docker://privateregistry:5000/image
file:///path/to/local/rootfs.tar.xz
+ virt-builder://fedora-25
+ virt-builder://ubuntu-16.04
----------------------------------------
''')))
diff --git a/tests/__init__.py b/tests/__init__.py
index 7a53c38..8888a0d 100644
--- a/tests/__init__.py
+++ b/tests/__init__.py
@@ -29,6 +29,8 @@ import tarfile
import unittest
import passlib.hosts
+import guestfs
+
try:
import mock
except ImportError:
@@ -434,3 +436,24 @@ class BuildTarFiles(unittest.TestCase):
self.check_image_content(g, user, 'dirs', g.is_dir)
# Check files
self.check_image_content(g, user, 'files', g.is_file)
+
+ def check_qcow2_image(self, image_path):
+ """
+ Ensures that qcow2 images contain all files.
+ """
+ g = guestfs.GuestFS(python_return_dict=True)
+ g.add_drive_opts(image_path, readonly=True)
+ g.launch()
+ g.mount('/dev/sda', '/')
+ self.check_image(g)
+ g.umount('/')
+ g.shutdown()
+
+ def get_image_path(self, n=0):
+ """
+ Returns the path where the qcow2 image will be stored.
+ """
+ return os.path.join(
+ self.dest_dir,
+ "layer-%d.qcow2" % n
+ )
diff --git a/tests/file_source.py b/tests/file_source.py
index 8851c3f..895ba9e 100644
--- a/tests/file_source.py
+++ b/tests/file_source.py
@@ -24,7 +24,6 @@ with FileSource.
import os
import unittest
-import guestfs
from . import virt_bootstrap
from . import BuildTarFiles
@@ -38,27 +37,6 @@ class Qcow2BuildImage(BuildTarFiles):
works as expected.
"""
- def check_qcow2_images(self, image_path):
- """
- Ensures that qcow2 images contain all files.
- """
- g = guestfs.GuestFS(python_return_dict=True)
- g.add_drive_opts(image_path, readonly=True)
- g.launch()
- g.mount('/dev/sda', '/')
- self.check_image(g)
- g.umount('/')
- g.shutdown()
-
- def get_image_path(self, n=0):
- """
- Returns the path where the qcow2 image will be stored.
- """
- return os.path.join(
- self.dest_dir,
- "layer-%d.qcow2" % n
- )
-
def runTest(self):
"""
Create qcow2 image from each dummy tarfile.
@@ -69,7 +47,7 @@ class Qcow2BuildImage(BuildTarFiles):
fmt='qcow2',
progress_cb=mock.Mock()
)
- self.check_qcow2_images(self.get_image_path())
+ self.check_qcow2_image(self.get_image_path())
class Qcow2OwnershipMapping(Qcow2BuildImage):
@@ -91,7 +69,7 @@ class Qcow2OwnershipMapping(Qcow2BuildImage):
gid_map=self.gid_map
)
self.apply_mapping()
- self.check_qcow2_images(self.get_image_path(1))
+ self.check_qcow2_image(self.get_image_path(1))
class Qcow2SettingRootPassword(Qcow2BuildImage):
@@ -112,7 +90,7 @@ class Qcow2SettingRootPassword(Qcow2BuildImage):
root_password=self.root_password
)
self.check_image = self.validate_shadow_file_in_image
- self.check_qcow2_images(self.get_image_path(1))
+ self.check_qcow2_image(self.get_image_path(1))
@unittest.skipIf(os.geteuid() != 0, "Root privileges required")
diff --git a/tests/virt_builder_source.py b/tests/virt_builder_source.py
new file mode 100644
index 0000000..4fe7713
--- /dev/null
+++ b/tests/virt_builder_source.py
@@ -0,0 +1,228 @@
+# -*- coding: utf-8 -*-
+# Authors: Radostin Stoyanov <rstoyanov1 at gmail.com>
+#
+# Copyright (C) 2017 Radostin Stoyanov
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+
+"""
+Regression tests which aim to excercise the creation of root file system
+with VirtBuilderSource.
+"""
+
+import os
+import unittest
+
+import guestfs
+
+from . import BuildTarFiles
+from . import virt_bootstrap
+from . import mock
+
+
+# pylint: disable=invalid-name
+class DirExtractRootFS(BuildTarFiles):
+ """
+ This test is replacing the method build_image() from VirtBuilderSource
+ with a function which generates gummy disk image.
+
+ It ensures that all files and directories from the created root file
+ systems is extracted correctly.
+ """
+ def create_tar_files(self):
+ """
+ Don't need to build tar files for these tests.
+ """
+ pass
+
+ def create_dummy_disk(self, output_file):
+ """
+ Create raw disk image with dummy root file system
+ """
+ g = guestfs.GuestFS(python_return_dict=True)
+ g.disk_create(output_file, format='raw', size=(1 * 1024 * 1024))
+ g.add_drive(output_file, readonly=False, format='raw')
+ g.launch()
+ g.mkfs('ext3', '/dev/sda')
+ g.mount('/dev/sda', '/')
+ for user in self.rootfs_tree:
+
+ usr_uid = self.rootfs_tree[user]['uid']
+ usr_gid = self.rootfs_tree[user]['gid']
+ usr_dirs = self.rootfs_tree[user]['dirs']
+ usr_files = self.rootfs_tree[user]['files']
+
+ for member in usr_dirs:
+ dir_name = '/' + member
+ g.mkdir_p(dir_name)
+ g.chown(usr_uid, usr_gid, dir_name)
+
+ for member in usr_files:
+ if isinstance(member, tuple):
+ file_name = '/' + member[0]
+ g.write(file_name, member[2])
+ g.chmod(member[1] & 0o777, file_name)
+ else:
+ file_name = '/' + member
+ g.touch(file_name)
+ g.chmod(0o755 & 0o777, file_name)
+ g.chown(usr_uid, usr_gid, file_name)
+
+ def runTest(self):
+ """
+ Mock the function build_image() and call bootstrap().
+ Then check the extracted root file system.
+ """
+ build_image = 'virtBootstrap.sources.VirtBuilderSource.build_image'
+ with mock.patch(build_image) as m_build_image:
+ m_build_image.side_effect = self.create_dummy_disk
+ virt_bootstrap.bootstrap(
+ uri='virt-builder://foobar',
+ dest=self.dest_dir,
+ fmt='dir',
+ progress_cb=mock.Mock()
+ )
+ self.check_rootfs(skip_ownership=(os.geteuid() != 0))
+
+
+ at unittest.skipIf(os.geteuid() != 0, "Root privileges required")
+class DirOwnershipMapping(DirExtractRootFS):
+ """
+ Ensures that UID/GID mapping for extracted root file system are applied
+ correctly.
+ """
+ def runTest(self):
+ """
+ Mock the function build_image() and call bootstrap() with uid/gid
+ mapping values.
+ Then check the ownership of the extracted root file system.
+ """
+ self.uid_map = [[1000, 2000, 10], [0, 1000, 10], [500, 500, 10]]
+ self.gid_map = [[1000, 2000, 10], [0, 1000, 10], [500, 500, 10]]
+ build_image = 'virtBootstrap.sources.VirtBuilderSource.build_image'
+ with mock.patch(build_image) as m_build_image:
+ m_build_image.side_effect = self.create_dummy_disk
+ virt_bootstrap.bootstrap(
+ progress_cb=mock.Mock(),
+ uri='virt-builder://foobar',
+ dest=self.dest_dir,
+ fmt='dir',
+ uid_map=self.uid_map,
+ gid_map=self.gid_map
+ )
+ self.apply_mapping()
+ self.check_rootfs(skip_ownership=(os.geteuid() != 0))
+
+
+class DirSettingRootPassword(DirExtractRootFS):
+ """
+ The root password is set by virt-builder in this test we do not call
+ virt-builder as this is time consuming job and might require network
+ connection to download the VM template.
+
+ Instead we only check if the root password is passed to virt-builder
+ and if virt-bootstrap extracts the shadow file correctly.
+ """
+ def verify_virt_builder_cmd(self, cmd):
+ """
+ Ensures that virt-builder is called with valid command and the root
+ password is passed.
+ """
+ self.assertEqual(
+ 'virt-builder',
+ cmd[0],
+ "virt-builder command does not start with 'virt-builder'"
+ )
+ self.assertIn(
+ '--root-password',
+ cmd,
+ "The flag '--root-password' is missing in virt-builder command"
+ )
+ self.assertEqual(
+ 'password:%s' % self.root_password,
+ cmd[cmd.index('--root-password') + 1],
+ "Root password doesn't match"
+ )
+ self.create_dummy_disk(cmd[cmd.index('-o') + 1])
+
+ def runTest(self):
+ """
+ Mock the function subprocess.check_call() and call bootstrap().
+ Then check the extracted root file system.
+ """
+ self.root_password = "Root password"
+ with mock.patch('subprocess.check_call') as m_check_call:
+ m_check_call.side_effect = self.verify_virt_builder_cmd
+ virt_bootstrap.bootstrap(
+ uri='virt-builder://foobar',
+ dest=self.dest_dir,
+ fmt='dir',
+ root_password=self.root_password,
+ progress_cb=mock.Mock()
+ )
+ m_check_call.assert_called_once()
+ self.validate_shadow_file(
+ os.path.join(self.dest_dir, 'etc/shadow'),
+ skip_ownership=(os.geteuid() != 0),
+ skip_hash=True
+ )
+
+
+class Qcow2BuildImage(DirExtractRootFS):
+ """
+ Ensures that the file system is copied correctly to the output qcow2 image.
+ """
+ def runTest(self):
+ """
+ Mock the function build_image() and call bootstrap().
+ Then check the content of the new image.
+ """
+ build_image = 'virtBootstrap.sources.VirtBuilderSource.build_image'
+ with mock.patch(build_image) as m_build_image:
+ m_build_image.side_effect = self.create_dummy_disk
+ virt_bootstrap.bootstrap(
+ progress_cb=mock.Mock(),
+ uri='virt-builder://foobar',
+ dest=self.dest_dir,
+ fmt='qcow2'
+ )
+ self.check_qcow2_image(self.get_image_path())
+
+
+class Qcow2OwnershipMapping(DirExtractRootFS):
+ """
+ Ensures that UID/GID mapping is applied correctly with qcow2 conversion.
+ """
+ def runTest(self):
+ """
+ Mock the function build_image() and call bootstrap() with uid/gid
+ mapping values.
+ Then check the ownership of the extracted root file system.
+ """
+ self.uid_map = [[1000, 2000, 10], [0, 1000, 10], [500, 500, 10]]
+ self.gid_map = [[1000, 2000, 10], [0, 1000, 10], [500, 500, 10]]
+ build_image = 'virtBootstrap.sources.VirtBuilderSource.build_image'
+ with mock.patch(build_image) as m_build_image:
+ m_build_image.side_effect = self.create_dummy_disk
+ virt_bootstrap.bootstrap(
+ uri='virt-builder://foobar',
+ dest=self.dest_dir,
+ fmt='qcow2',
+ uid_map=self.uid_map,
+ gid_map=self.gid_map,
+ progress_cb=mock.Mock()
+ )
+ self.apply_mapping()
+ self.check_qcow2_image(self.get_image_path(1))
--
2.13.5
More information about the virt-tools-list
mailing list