[Date Prev][Date Next]   [Thread Prev][Thread Next]   [Thread Index] [Date Index] [Author Index]

[PATCH 7/7] livecd: improve livecd install speed by about 20%: turboLiveInst/genMinInstDelta



the final piece, livecd generating an optomized image that the prior anaconda patch can take advantage of during a livecd install.

see...

https://www.redhat.com/archives/fedora-livecd-list/2007-September/msg00101.html

for all the good details...

Enjoy...

-dmc/jdog



diff -Naur livecd.3.resize2fsToMinimal_implicitsize/README livecd.turboliveinst/README
--- livecd.3.resize2fsToMinimal_implicitsize/README	2007-09-17 16:47:31.000000000 +0000
+++ livecd.turboliveinst/README	2007-09-14 17:13:34.000000000 +0000
@@ -74,6 +74,13 @@
 
  o Unmounts the installation root
 
+ o Runs resize2fs to minimize and unminimize the ext3 file to remove data
+   from deleted files
+
+ o Runs resize2fs to minimize on a devicemapper snapshot, to generate a 
+   small minimized delta image file which can be used by anaconda to 
+   reduce installation time by not copying unused data to disk
+
  o Creates a squashfs file system containing only the ext3 file (compression)
 
  o Configures the boot loader
diff -Naur livecd.3.resize2fsToMinimal_implicitsize/creator/isotostick.sh livecd.turboliveinst/creator/isotostick.sh
--- livecd.3.resize2fsToMinimal_implicitsize/creator/isotostick.sh	2007-09-14 05:49:42.000000000 +0000
+++ livecd.turboliveinst/creator/isotostick.sh	2007-09-14 16:00:12.000000000 +0000
@@ -195,6 +195,10 @@
 elif [ -f $CDMNT/ext3fs.img ]; then
     cp $CDMNT/ext3fs.img $USBMNT/LiveOS/ext3fs.img || exitclean 
 fi
+if [ -f $CDMNT/osmin.gz ]; then
+    cp $CDMNT/osmin.gz $USBMNT/LiveOS/osmin.gz || exitclean
+fi
+
 cp $CDMNT/isolinux/* $USBMNT/$SYSLINUXPATH
 
 echo "Updating boot config file"
diff -Naur livecd.3.resize2fsToMinimal_implicitsize/creator/livecd-creator livecd.turboliveinst/creator/livecd-creator
--- livecd.3.resize2fsToMinimal_implicitsize/creator/livecd-creator	2007-09-17 17:09:57.000000000 +0000
+++ livecd.turboliveinst/creator/livecd-creator	2007-09-14 17:40:21.000000000 +0000
@@ -343,6 +343,7 @@
 
         self.image_size = 4096 # in megabytes
         self.blocksize = 4096 # in kilobytes
+        self.minimized_image_size = 0 # in kilobytes
 
     def _getRequiredPackages(self):
         return []
@@ -934,7 +935,7 @@
         for line in output.split("\n"):
             if line.startswith(field + ":"):
                 return line[len(field) + 1:].strip()
-
+        
         raise KeyError("Failed to find field '%s' in output" % field)
 
     def getBlockCountOfExt2FS(self, filesystem):
@@ -942,7 +943,7 @@
                                   stdout=subprocess.PIPE,
                                   stderr=open('/dev/null', 'w')
                                   ).communicate()[0]
-
+        
         return int(self.parseField(output, "Block count"))
 
     def resize2fs(self, image, n_blocks):
@@ -990,10 +991,88 @@
         os.ftruncate(fd, min_blocks * self.blocksize)
         os.close(fd)
 
-        print >> sys.stderr, "Installation target minimized to %dK" % (min_blocks * self.blocksize / 1024L)
+        self.minimized_image_size = min_blocks * self.blocksize / 1024L
+        print >> sys.stderr, "Installation target minimized to %dK" % (self.minimized_image_size)
 
         self.resize2fs(image, n_blocks)
 
+
+    #
+    # genMinInstDelta: generates an osmin overlay file to sit alongside
+    #                  os.img.  liveinst may then detect the existence of
+    #                  osmin, and use it to create a minimized os.img
+    #                  which can be installed more quickly, and to smaller
+    #                  destination volumes.
+    #
+    def genMinInstDelta(self):
+        # create the sparse file for the minimized overlay
+        fd = os.open("%s/out/osmin" %(self.build_dir,),
+                     os.O_WRONLY | os.O_CREAT)
+        off = long(64L * 1024L * 1024L)
+        os.lseek(fd, off, 0)
+        os.write(fd, '\x00')
+        os.close(fd)
+        
+        # associate os image with loop device
+        osloop = LoopbackMount("%s/data/os.img" %(self.build_dir,),
+                               "not_going_to_actually_get_mounted")
+        osloop.loopsetup()
+        
+        # associate overlay with loop device
+        minloop = LoopbackMount("%s/out/osmin" %(self.build_dir,),
+                                "not_going_to_actually_get_mounted")
+        minloop.loopsetup()
+        
+        # create a snapshot device
+        rc = subprocess.call(["/sbin/dmsetup",
+                              "--table",
+                              "0 %d snapshot %s %s p 8"
+                              %(self.image_size * 1024L * 2L,
+                                osloop.loopdev, minloop.loopdev),
+                              "create",
+                              "livecd-creator-%d" %(os.getpid(),) ])
+        if rc != 0:
+            raise InstallationError("Could not create genMinInstDelta snapshot device")
+        # resize snapshot device back to minimal (self.minimized_image_size)
+        rc = subprocess.call(["/sbin/resize2fs",
+                              "/dev/mapper/livecd-creator-%d" %(os.getpid(),),
+                              "%dK" %(self.minimized_image_size,)])
+        if rc != 0:
+            raise InstallationError("Could not shrink ext3fs image")
+
+        # calculate how much delta data to keep
+        dmsetupOutput = subprocess.Popen(['/sbin/dmsetup', 'status',
+                                          "livecd-creator-%d" %(os.getpid(),)],
+                                         stdout=subprocess.PIPE,
+                                         stderr=open('/dev/null', 'w')
+                                         ).communicate()[0]
+
+        try:
+            minInstDeltaDataLength = int((dmsetupOutput.split()[3]).split('/')[0])
+            print >> sys.stderr, "genMinInstDelta data length is %d 512 byte sectors" % (minInstDeltaDataLength)
+        except ValueError:
+            raise InstallationError("Could not calculate amount of data used by genMinInstDelta")            
+        
+        # tear down snapshot and loop devices
+        rc = subprocess.call(["/sbin/dmsetup", "remove",
+                              "livecd-creator-%d" %(os.getpid(),) ])
+        if rc != 0:
+            raise InstallationError("Could not remove genMinInstDelta snapshot device")
+        osloop.lounsetup()
+        minloop.lounsetup()
+
+        # truncate the unused excess portion of the sparse file
+        fd = os.open("%s/out/osmin" %(self.build_dir,), os.O_WRONLY )
+        os.ftruncate(fd, minInstDeltaDataLength * 512)
+        os.close(fd)
+
+        # the delta data is *extremely* compressible (e.g. 1.2M->7kb)
+        rc = subprocess.call(["/usr/bin/gzip", "osmin"],
+                             cwd="%s/out" %(self.build_dir,),
+                             env={"PWD": "%s/out" %(self.build_dir,)})
+        if rc != 0:
+            raise InstallationError("Could not compress genMinInstDelta data")
+
     def package(self):
         self.createSquashFS()
         self.createIso()
@@ -1316,7 +1395,6 @@
     parser.add_option("-s", "--skip-compression", action="store_true", dest="skip_compression",
                       help=optparse.SUPPRESS_HELP)
 
-
     (options, args) = parser.parse_args()
     if not options.kscfg or not os.path.isfile(options.kscfg):
         raise Usage("Kickstart config '%s' does not exist" %(options.kscfg,))
@@ -1388,7 +1466,10 @@
 
         target.cleanupDeleted()
 
+        target.genMinInstDelta()
+
         target.package()
+
     except InstallationError, e:
         print >> sys.stderr, "Error creating Live CD : %s" % e
         target.teardown()
diff -Naur livecd.3.resize2fsToMinimal_implicitsize/creator/mayflower livecd.turboliveinst/creator/mayflower
--- livecd.3.resize2fsToMinimal_implicitsize/creator/mayflower	2007-09-14 05:49:42.000000000 +0000
+++ livecd.turboliveinst/creator/mayflower	2007-09-14 18:28:20.000000000 +0000
@@ -625,6 +625,24 @@
     mount -n -o ro,remount /sysroot
 }
 
+modprobe loop max_loop=128
+
+# we might have a genMinInstDelta delta file for anaconda to take advantage of
+if [ -e /sysroot/LiveOS/osmin.gz ]; then
+  mknod /dev/loop118 b 7 118
+  # note: osmin.gz should typically only be about 7kb. 
+  dd if=/sysroot/LiveOS/osmin.gz of=/osmin.gz bs=512 2> /dev/null
+  # pad to at least next sector boundry
+  dd if=/dev/zero of=/osmin.gz bs=512 count=1 oflag=append conv=notrunc 2> /dev/null
+  losetup /dev/loop118 /osmin.gz
+elif [ -e /sysroot/osmin.gz ] ; then
+  mknod /dev/loop118 b 7 118
+  dd if=/sysroot/osmin.gz of=/osmin.gz bs=512 2> /dev/null
+  # pad to at least next sector boundry
+  dd if=/dev/zero of=/osmin.gz bs=512 count=1 oflag=append conv=notrunc 2> /dev/null
+  losetup /dev/loop118 /osmin.gz
+fi
+
 # we might have an uncompressed embedded ext3  to use as rootfs (uncompressed live)
 #
 if [ -e /sysroot/LiveOS/ext3fs.img ]; then
@@ -638,13 +656,11 @@
         echo "setting up embedded ext3 fs "
     fi
 
-    mknod /dev/loop118 b 7 118
     mknod /dev/loop119 b 7 119
     mknod /dev/loop120 b 7 120
     mknod /dev/loop121 b 7 121
     mkdir -p /dev/mapper
     mknod /dev/mapper/control c 10 63
-    modprobe loop max_loop=128
     modprobe dm_snapshot
 
     losetup /dev/loop121 \$EXT3FS
@@ -667,13 +683,11 @@
         echo "setting up embedded squash -> ext3 fs "
     fi
 
-    mknod /dev/loop118 b 7 118
     mknod /dev/loop119 b 7 119
     mknod /dev/loop120 b 7 120
     mknod /dev/loop121 b 7 121
     mkdir -p /dev/mapper
     mknod /dev/mapper/control c 10 63
-    modprobe loop max_loop=128
     modprobe dm_snapshot
 
     if [ "\$live_ram" == "1" ] ; then

[Date Prev][Date Next]   [Thread Prev][Thread Next]   [Thread Index] [Date Index] [Author Index]