[Date Prev][Date Next]   [Thread Prev][Thread Next]   [Thread Index] [Date Index] [Author Index]

[Libguestfs] [PATCH v3] New APIs: compress-out, compress-device-out.



This third version changes the API again to allow you to select both
the compression program (from {compress,gzip,bzip2,xz,lzop}) and
optionally the compression level.

Some more timings are below.  'lzop' is generally much faster than
other methods, although there is still > 50% penalty for using lzop on
a fairly full filesystem.

Interestingly, 'lzop -1' which is supposed to be fast, is slower than
the default level.

Rich.

100 MB empty filesystem.
Straight download was 0.65 seconds.

$ ./run ./fish/guestfish -N fs \
    time compress-device-out gzip /dev/vda1 >(wc -c) level:1
elapsed time: 0.92 seconds
462390

$ ./run ./fish/guestfish -N fs \
    time compress-device-out gzip /dev/vda1 >(wc -c) level:9
elapsed time: 1.14 seconds
107903

$ ./run ./fish/guestfish -N fs \
    time compress-device-out lzop /dev/vda1 >(wc -c)
elapsed time: 0.73 seconds
428023

$ ./run ./fish/guestfish -N fs \
    time compress-device-out lzop /dev/vda1 >(wc -c) level:1
elapsed time: 0.84 seconds
428125

$ ./run ./fish/guestfish -N fs \
    time compress-device-out bzip2 /dev/vda1 >(wc -c)
elapsed time: 1.80 seconds
2801

$ ./run ./fish/guestfish -N fs \
    time compress-device-out xz /dev/vda1 >(wc -c)
elapsed time: 9.17 seconds
18164

Real filesystem image ~5.5GB in size, 59% in use.
Straight download was 78.74 seconds.

$ ./run ./fish/guestfish --ro -a /dev/vg_pin/F16x64 \
    run : time compress-device-out lzop /dev/vg_f16x64/lv_root >(wc -c)
elapsed time: 123.17 seconds
2488500746


-- 
Richard Jones, Virtualization Group, Red Hat http://people.redhat.com/~rjones
libguestfs lets you edit virtual machines.  Supports shell scripting,
bindings from many languages.  http://libguestfs.org
>From 628466d65b80ded83d93a2d30b2bad5db1d904c7 Mon Sep 17 00:00:00 2001
From: "Richard W.M. Jones" <rjones redhat com>
Date: Wed, 28 Sep 2011 11:14:06 +0100
Subject: [PATCH] New APIs: compress-out, compress-device-out.

These APIs let you copy compressed files or devices out from the disk
image.

Compression is useful for large images which are mostly zeroes.  We
cannot currently do sparseness detection, and compression gives us a
form of zero detection for free.

Example usage:

$ guestfish --ro -a /dev/vg_pin/F16x64 -i \
    compress-out gzip /etc/passwd /tmp/passwd.gz
$ file -z /tmp/passwd.gz
/tmp/passwd.gz: ASCII text (gzip compressed data, was "passwd", from
Unix, last modified: Sun Aug 28 14:40:46 2011)
---
 appliance/packagelist.in       |    1 +
 daemon/Makefile.am             |    1 +
 daemon/compress.c              |  207 ++++++++++++++++++++++++++++++++++++++++
 generator/generator_actions.ml |   27 +++++
 po/POTFILES.in                 |    1 +
 src/MAX_PROC_NR                |    2 +-
 6 files changed, 238 insertions(+), 1 deletions(-)
 create mode 100644 daemon/compress.c

diff --git a/appliance/packagelist.in b/appliance/packagelist.in
index 05afa94..387cae1 100644
--- a/appliance/packagelist.in
+++ b/appliance/packagelist.in
@@ -106,6 +106,7 @@ gzip
 jfsutils
 lsof
 lvm2
+lzop
 module-init-tools
 net-tools
 /*
diff --git a/daemon/Makefile.am b/daemon/Makefile.am
index 1664af0..e23ce86 100644
--- a/daemon/Makefile.am
+++ b/daemon/Makefile.am
@@ -94,6 +94,7 @@ guestfsd_SOURCES = \
 	checksum.c \
 	cmp.c \
 	command.c \
+	compress.c \
 	cpmv.c \
 	daemon.h \
 	dd.c \
diff --git a/daemon/compress.c b/daemon/compress.c
new file mode 100644
index 0000000..d08d3cc
--- /dev/null
+++ b/daemon/compress.c
@@ -0,0 +1,207 @@
+/* libguestfs - the guestfsd daemon
+ * Copyright (C) 2011 Red Hat Inc.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ */
+
+#include <config.h>
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <fcntl.h>
+
+#include "guestfs_protocol.h"
+#include "daemon.h"
+#include "actions.h"
+
+/* Has one FileOut parameter. */
+static int
+do_compressX_out (const char *file, const char *filter, int is_device)
+{
+  int r;
+  FILE *fp;
+  char *cmd;
+  char buf[GUESTFS_MAX_CHUNK_SIZE];
+
+  /* The command will look something like:
+   *   gzip -c /sysroot%s     # file
+   * or:
+   *   gzip -c < %s           # device
+   *
+   * We have to quote the file or device name.
+   *
+   * The unnecessary redirect for devices is there because lzop
+   * unhelpfully refuses to compress anything that isn't a regular
+   * file.
+   */
+  if (!is_device) {
+    if (asprintf_nowarn (&cmd, "%s %R", filter, file) == -1) {
+      reply_with_perror ("asprintf");
+      return -1;
+    }
+  } else {
+    if (asprintf_nowarn (&cmd, "%s < %Q", filter, file) == -1) {
+      reply_with_perror ("asprintf");
+      return -1;
+    }
+  }
+
+  if (verbose)
+    fprintf (stderr, "%s\n", cmd);
+
+  fp = popen (cmd, "r");
+  if (fp == NULL) {
+    reply_with_perror ("%s", cmd);
+    free (cmd);
+    return -1;
+  }
+  free (cmd);
+
+  /* Now we must send the reply message, before the file contents.  After
+   * this there is no opportunity in the protocol to send any error
+   * message back.  Instead we can only cancel the transfer.
+   */
+  reply (NULL, NULL);
+
+  while ((r = fread (buf, 1, sizeof buf, fp)) > 0) {
+    if (send_file_write (buf, r) < 0) {
+      pclose (fp);
+      return -1;
+    }
+  }
+
+  if (ferror (fp)) {
+    perror (file);
+    send_file_end (1);		/* Cancel. */
+    pclose (fp);
+    return -1;
+  }
+
+  if (pclose (fp) != 0) {
+    perror (file);
+    send_file_end (1);		/* Cancel. */
+    return -1;
+  }
+
+  if (send_file_end (0))	/* Normal end of file. */
+    return -1;
+
+  return 0;
+}
+
+#define CHECK_SUPPORTED(prog)                                           \
+  if (!prog_exists (prog)) {                                            \
+    /* note: substring "not supported" must appear in this error */     \
+    reply_with_error ("compression type %s is not supported", prog);    \
+    return -1;                                                          \
+  }
+
+static int
+get_filter (const char *ctype, int level, char *ret, size_t n)
+{
+  if (STREQ (ctype, "compress")) {
+    CHECK_SUPPORTED ("compress");
+    if (level != -1) {
+      reply_with_error ("compress: cannot use optional level parameter with this compression type");
+      return -1;
+    }
+    snprintf (ret, n, "compress -c");
+    return 0;
+  }
+  else if (STREQ (ctype, "gzip")) {
+    CHECK_SUPPORTED ("gzip");
+    if (level == -1)
+      snprintf (ret, n, "gzip -c");
+    else if (level >= 1 && level <= 9)
+      snprintf (ret, n, "gzip -c -%d", level);
+    else {
+      reply_with_error ("gzip: incorrect value for level parameter");
+      return -1;
+    }
+    return 0;
+  }
+  else if (STREQ (ctype, "bzip2")) {
+    CHECK_SUPPORTED ("bzip2");
+    if (level == -1)
+      snprintf (ret, n, "bzip2 -c");
+    else if (level >= 1 && level <= 9)
+      snprintf (ret, n, "bzip2 -c -%d", level);
+    else {
+      reply_with_error ("bzip2: incorrect value for level parameter");
+      return -1;
+    }
+    return 0;
+  }
+  else if (STREQ (ctype, "xz")) {
+    CHECK_SUPPORTED ("xz");
+    if (level == -1)
+      snprintf (ret, n, "xz -c");
+    else if (level >= 0 && level <= 9)
+      snprintf (ret, n, "xz -c -%d", level);
+    else {
+      reply_with_error ("xz: incorrect value for level parameter");
+      return -1;
+    }
+    return 0;
+  }
+  else if (STREQ (ctype, "lzop")) {
+    CHECK_SUPPORTED ("lzop");
+    if (level == -1)
+      snprintf (ret, n, "lzop -c");
+    else if (level >= 1 && level <= 9)
+      snprintf (ret, n, "lzop -c -%d", level);
+    else {
+      reply_with_error ("lzop: incorrect value for level parameter");
+      return -1;
+    }
+    return 0;
+  }
+
+  reply_with_error ("unknown compression type");
+  return -1;
+}
+
+/* Has one FileOut parameter. */
+/* Takes optional arguments, consult optargs_bitmask. */
+int
+do_compress_out (const char *ctype, const char *file, int level)
+{
+  char filter[64];
+
+  if (!(optargs_bitmask & GUESTFS_COMPRESS_OUT_LEVEL_BITMASK))
+    level = -1;
+
+  if (get_filter (ctype, level, filter, sizeof filter) == -1)
+    return -1;
+
+  return do_compressX_out (file, filter, 0);
+}
+
+/* Has one FileOut parameter. */
+/* Takes optional arguments, consult optargs_bitmask. */
+int
+do_compress_device_out (const char *ctype, const char *file, int level)
+{
+  char filter[64];
+
+  if (!(optargs_bitmask & GUESTFS_COMPRESS_DEVICE_OUT_LEVEL_BITMASK))
+    level = -1;
+
+  if (get_filter (ctype, level, filter, sizeof filter) == -1)
+    return -1;
+
+  return do_compressX_out (file, filter, 1);
+}
diff --git a/generator/generator_actions.ml b/generator/generator_actions.ml
index c3d74f5..da49716 100644
--- a/generator/generator_actions.ml
+++ b/generator/generator_actions.ml
@@ -6146,6 +6146,33 @@ C<path> does not exist, then a new file is created.
 
 See also C<guestfs_write>.");
 
+  ("compress_out", (RErr, [String "ctype"; Pathname "file"; FileOut "zfile"], [Int "level"]), 291, [],
+   [],
+   "output compressed file",
+   "\
+This command compresses C<file> and writes it out to the local
+file C<zfile>.
+
+The compression program used is controlled by the C<ctype> parameter.
+Currently this includes: C<compress>, C<gzip>, C<bzip2>, C<xz> or C<lzop>.
+Some compression types may not be supported by particular builds of
+libguestfs, in which case you will get an error containing the
+substring \"not supported\".
+
+The optional C<level> parameter controls compression level.  The
+meaning and default for this parameter depends on the compression
+program being used.");
+
+  ("compress_device_out", (RErr, [String "ctype"; Device "device"; FileOut "zdevice"], [Int "level"]), 292, [],
+   [],
+   "output compressed device",
+   "\
+This command compresses C<device> and writes it out to the local
+file C<zdevice>.
+
+The C<ctype> and optional C<level> parameters have the same meaning
+as in C<guestfs_compress_out>.");
+
 ]
 
 let all_functions = non_daemon_functions @ daemon_functions
diff --git a/po/POTFILES.in b/po/POTFILES.in
index df54873..effc9ea 100644
--- a/po/POTFILES.in
+++ b/po/POTFILES.in
@@ -11,6 +11,7 @@ daemon/btrfs.c
 daemon/checksum.c
 daemon/cmp.c
 daemon/command.c
+daemon/compress.c
 daemon/cpmv.c
 daemon/dd.c
 daemon/debug.c
diff --git a/src/MAX_PROC_NR b/src/MAX_PROC_NR
index 8408670..f20bd4e 100644
--- a/src/MAX_PROC_NR
+++ b/src/MAX_PROC_NR
@@ -1 +1 @@
-290
+292
-- 
1.7.6


[Date Prev][Date Next]   [Thread Prev][Thread Next]   [Thread Index] [Date Index] [Author Index]