rpms/yum-presto/devel total-progress+stat-fixes.patch, NONE, 1.1 yum-presto.spec, 1.12, 1.13
James Antill
james at fedoraproject.org
Tue Apr 28 18:47:10 UTC 2009
- Previous message (by thread): rpms/R-BufferedMatrixMethods/devel .cvsignore, 1.5, 1.6 R-BufferedMatrixMethods.spec, 1.7, 1.8 sources, 1.5, 1.6
- Next message (by thread): rpms/syslinux/devel sources,1.26,1.27 syslinux.spec,1.58,1.59
- Messages sorted by:
[ date ]
[ thread ]
[ subject ]
[ author ]
Author: james
Update of /cvs/pkgs/rpms/yum-presto/devel
In directory cvs1.fedora.phx.redhat.com:/tmp/cvs-serv17715
Modified Files:
yum-presto.spec
Added Files:
total-progress+stat-fixes.patch
Log Message:
* Tue Apr 28 2009 James Antill <james at fedoraproject.org> - 0.4.5-6
- Add total progress data to delta downloads
- Fix stats. to account for the fact not everything has a delta
- Allow delta downloads to be configurable on savings
total-progress+stat-fixes.patch:
--- NEW FILE total-progress+stat-fixes.patch ---
commit 2324e07a8e9b6599f4e67df3301c2e1d75e8e669
Author: James Antill <james at and.org>
Date: Mon Apr 27 23:54:45 2009 -0400
Give total download progress stats.
diff --git a/yum-presto/presto.py b/yum-presto/presto.py
index 1e2c4ee..e754dd1 100644
--- a/yum-presto/presto.py
+++ b/yum-presto/presto.py
@@ -263,6 +263,7 @@ def downloadPkgs(conduit, presto, download_pkgs=None):
curthread.start()
remote_pkgs = []
+ remote_size = 0
if download_pkgs is None:
download_pkgs = conduit.getDownloadPackages()
@@ -298,6 +299,7 @@ def downloadPkgs(conduit, presto, download_pkgs=None):
if cursize >= totsize:
os.unlink(deltapath)
+ remote_size += totsize
remote_pkgs.append( (po, delta) )
else:
# Deltarpm is local and good, put it in the rebuild thread.
@@ -309,10 +311,21 @@ def downloadPkgs(conduit, presto, download_pkgs=None):
rebuild_size += po.size
continue
else:
+ remote_size += long(delta['size'])
remote_pkgs.append( (po, delta) )
+ # This is kind of a hack and does nothing in non-Fedora versions,
+ # we'll fix it one way or anther soon.
+ if (hasattr(urlgrabber.progress, 'text_meter_total_size') and
+ len(remote_pkgs) > 1):
+ urlgrabber.progress.text_meter_total_size(remote_size)
+
+ if remote_size:
+ conduit.info(2, "Download delta size: %s" % format_number(remote_size))
+
# now we need to do downloads
i = 0
+ local_size = 0
for (po, delta) in remote_pkgs:
i += 1
# FIXME: verifyChecksum should handle the urlgrabber objects...
@@ -327,20 +340,30 @@ def downloadPkgs(conduit, presto, download_pkgs=None):
# FIXME: this should be moved into _getFile
dirstat = os.statvfs(deltadir)
- if (dirstat.f_bavail * dirstat.f_bsize) <= (long(po.size) +
- long(delta['size'])):
+ delta_size = long(delta['size'])
+ if (dirstat.f_bavail * dirstat.f_bsize) <= (long(po.size) + delta_size):
adderror(po, 'Insufficient space in download directory %s '
'to download' % (deltadir,))
continue
+
+ if hasattr(urlgrabber.progress, 'text_meter_total_size'):
+ urlgrabber.progress.text_meter_total_size(remote_size, local_size)
try:
- text = "(%s/%s): %s" % (i, len(remote_pkgs),
- os.path.basename(delta['filename']))
+ if i == 1 and not local_size and remote_size == delta_size:
+ text = os.path.basename(delta['filename'])
+ else:
+ text = "(%s/%s): %s" % (i, len(remote_pkgs),
+ os.path.basename(delta['filename']))
deltafile = po.repo._getFile(url=po.basepath,
relative=delta['filename'],
local=deltapath,
checkfunc=checkfunc,
text=text,
cache=po.repo.cache)
+ local_size += delta_size
+ if hasattr(urlgrabber.progress, 'text_meter_total_size'):
+ urlgrabber.progress.text_meter_total_size(remote_size,
+ local_size)
except yum.Errors.RepoError, e:
adderror(po, str(e))
else:
commit da53d1654fd22a25e654166d0e8692f845f1a7b9
Author: James Antill <james at and.org>
Date: Tue Apr 28 00:09:56 2009 -0400
Fix stats. to account for non-deltas, output "still to download" number
diff --git a/yum-presto/presto.py b/yum-presto/presto.py
index e754dd1..b3a7979 100644
--- a/yum-presto/presto.py
+++ b/yum-presto/presto.py
@@ -151,8 +151,6 @@ class ReconstructionThread(threading.Thread):
def getDelta(po, presto, conduit):
"""Does the package have a reasonable delta for us to use?"""
- global complete_download_size
- global actual_download_size
rpmdb = conduit.getRpmDB()
# local packages don't make sense to use a delta for...
@@ -188,8 +186,6 @@ def getDelta(po, presto, conduit):
cursize = os.stat(local)[6]
totsize = long(po.size)
if po.verifyLocalPkg(): # we've got it.
- actual_download_size -= cursize
- complete_download_size -= cursize
conduit.info(5, "Already have package for %s.%s."
% (po.name, po.arch))
return None
@@ -265,6 +261,8 @@ def downloadPkgs(conduit, presto, download_pkgs=None):
remote_pkgs = []
remote_size = 0
+ must_still_download_pkgs = 0
+
if download_pkgs is None:
download_pkgs = conduit.getDownloadPackages()
@@ -273,6 +271,7 @@ def downloadPkgs(conduit, presto, download_pkgs=None):
for po in download_pkgs:
delta = getDelta(po, presto, conduit)
if delta is None:
+ must_still_download_pkgs += po.size
continue
# verify the delta if it already exists
@@ -412,7 +411,7 @@ def downloadPkgs(conduit, presto, download_pkgs=None):
conduit.info(2, curthread.messages[:-1])
curthread.messages = ""
- return errors
+ return (errors, must_still_download_pkgs)
class DeltaInfo(object):
"""Base Delta rpm info object"""
@@ -544,7 +543,12 @@ def predownload_hook(conduit):
download_pkgs.sort()
# Download deltarpms
- problems = downloadPkgs(conduit, pinfo, download_pkgs)
+ (problems, more) = downloadPkgs(conduit, pinfo, download_pkgs)
+
+ # Remove from "our" size, the size which still has to be got via. pkg
+ # downloads
+ complete_download_size -= more
+ actual_download_size -= more
# If 'exitondownloaderror' is on, exit
if conduit.confBool('main', 'exitondownloaderror') and \
@@ -556,7 +560,15 @@ def predownload_hook(conduit):
errstring += ' %s: %s\n' % (key, error)
raise PluginYumExit(errstring)
-def postdownload_hook(conduit):
+ xpostdownload_hook(conduit)
+ if more:
+ conduit.info(2, "Package(s) data still to download: %s" %
+ format_number(more))
+
+# Output stats. about delta downloads ... if we don't have deltas for
+# everything (pretty common), we want to output these before we start
+# downloading the pkgs themselves. So don't do it postdownload, but after above.
+def xpostdownload_hook(conduit):
global complete_download_size
global actual_download_size
commit df4bb5ed90395fef64ea8aa46180a40ddc6e56da
Author: James Antill <james at and.org>
Date: Tue Apr 28 00:18:18 2009 -0400
Add bad delta paths to must_still_download_pkgs, save some stat()s
diff --git a/yum-presto/presto.py b/yum-presto/presto.py
index b3a7979..c39c8aa 100644
--- a/yum-presto/presto.py
+++ b/yum-presto/presto.py
@@ -214,7 +214,8 @@ def getDelta(po, presto, conduit):
# we just want to use the smallest delta
if bestdelta and delta['size'] >= bestdelta['size']:
continue
-
+
+ # FIXME: This probably takes a while ... need to do something. progress?
if not verifyDelta(delta['sequence'], po.arch):
continue
@@ -240,6 +241,19 @@ def _safe_stat_size(fname):
except:
return 0
+def _get_delta_path(cache, po, delta):
+ # verify the delta if it already exists
+ deltadir = os.path.join(po.repo.cachedir, 'deltas')
+
+ if deltadir not in cache and not os.path.isdir(deltadir):
+ try:
+ os.mkdir(deltadir)
+ except OSError:
+ return None
+
+ cache.add(deltadir)
+ return os.path.join(deltadir, os.path.basename(delta['filename']))
+
def downloadPkgs(conduit, presto, download_pkgs=None):
"""download list of package objects handed to you, return errors"""
global process_lock
@@ -268,21 +282,18 @@ def downloadPkgs(conduit, presto, download_pkgs=None):
# see which deltas we need to download; if the delta is already
# downloaded, we can start it reconstructing in the background
+ deltapath_cache = set()
for po in download_pkgs:
delta = getDelta(po, presto, conduit)
if delta is None:
must_still_download_pkgs += po.size
continue
- # verify the delta if it already exists
- deltadir = os.path.join(po.repo.cachedir, 'deltas')
- if not os.path.isdir(deltadir):
- try:
- os.mkdir(deltadir)
- except OSError:
- continue
- deltapath = os.path.join(deltadir,
- os.path.basename(delta['filename']))
+ deltapath = _get_delta_path(deltapath_cache, po, delta)
+ if not deltapath:
+ must_still_download_pkgs += po.size
+ continue
+
if os.path.exists(deltapath):
try:
conduit._base.verifyChecksum(deltapath, delta['checksum_type'],
commit 795bc6addc0d8cfb764fd8dec297e9e0a0476c7e
Author: James Antill <james at and.org>
Date: Tue Apr 28 13:20:17 2009 -0400
Make delta.size an int as everyone assumes it anyway
diff --git a/yum-presto/presto.py b/yum-presto/presto.py
index c39c8aa..7e59dc3 100644
--- a/yum-presto/presto.py
+++ b/yum-presto/presto.py
@@ -184,7 +184,7 @@ def getDelta(po, presto, conduit):
local = po.localPkg()
if os.path.exists(local):
cursize = os.stat(local)[6]
- totsize = long(po.size)
+ totsize = po.size
if po.verifyLocalPkg(): # we've got it.
conduit.info(5, "Already have package for %s.%s."
% (po.name, po.arch))
@@ -305,7 +305,7 @@ def downloadPkgs(conduit, presto, download_pkgs=None):
raise yum.Errors.RepoError, msg
else:
cursize = os.stat(deltapath)[6]
- totsize = long(delta['size'])
+ totsize = delta['size']
if cursize >= totsize:
os.unlink(deltapath)
@@ -317,11 +317,11 @@ def downloadPkgs(conduit, presto, download_pkgs=None):
# HACK: Use the download progress, at least we get something
cb = po.repo.callback
po.returnChecksums() # Magic, see below
- queue.put((conduit, po, deltapath, long(delta['size'])))
+ queue.put((conduit, po, deltapath, delta['size']))
rebuild_size += po.size
continue
else:
- remote_size += long(delta['size'])
+ remote_size += delta['size']
remote_pkgs.append( (po, delta) )
# This is kind of a hack and does nothing in non-Fedora versions,
@@ -350,7 +350,7 @@ def downloadPkgs(conduit, presto, download_pkgs=None):
# FIXME: this should be moved into _getFile
dirstat = os.statvfs(deltadir)
- delta_size = long(delta['size'])
+ delta_size = delta['size']
if (dirstat.f_bavail * dirstat.f_bsize) <= (long(po.size) + delta_size):
adderror(po, 'Insufficient space in download directory %s '
'to download' % (deltadir,))
@@ -383,7 +383,7 @@ def downloadPkgs(conduit, presto, download_pkgs=None):
# from the SQL db in this thread, so that the other thread can call
# .verifyLocalPkg() without having to hit sqlite.
po.returnChecksums()
- queue.put((conduit, po, deltafile, long(delta['size'])))
+ queue.put((conduit, po, deltafile, delta['size']))
rebuild_size += po.size
if errors.has_key(po):
@@ -441,11 +441,14 @@ class DeltaInfo(object):
self.checksum_type = x.get("type")
setattr(self, x.tag, x.text)
+ # Things expect/assume this, might as well help them:
+ self.size = long(self.size)
+
def evr(self):
return "%s:%s-%s" % (self.epoch, self.version, self.release)
def __str__(self):
- return "filename: %s, sequence: %s, size: %s, checksum (%s) = %s" \
+ return "filename: %s, sequence: %s, size: %d, checksum (%s) = %s" \
% (self.filename, self.sequence, self.size,
self.checksum_type, self.checksum)
@@ -544,7 +547,7 @@ def predownload_hook(conduit):
continue
repos.add(po.repo)
download_pkgs.append(po)
- complete_download_size += int(po.size)
+ complete_download_size += po.size
actual_download_size = complete_download_size
if not download_pkgs:
commit 7c3036b9ba90024d52cfd24392c27d9b170665c6
Author: James Antill <james at and.org>
Date: Tue Apr 28 13:34:39 2009 -0400
Allow download vs. rebuild tradeoff, to be configurable
diff --git a/yum-presto/presto.conf b/yum-presto/presto.conf
index a066321..ecf41c2 100644
--- a/yum-presto/presto.conf
+++ b/yum-presto/presto.conf
@@ -6,3 +6,12 @@ enabled=1
# This defaults to yum's keepcache option, if not set.
# keepdeltas = false
+# This lets you change if the delta is downloaded given it's relative size vs.
+# the pkg. Eg. the default:
+#
+# minimum_percentage = 95
+#
+# ...means that given a pkg of 100M, a delta of 95M (or less) would be
+# downloaded instead but a delta of 96M would be skipped in favour of the pkg.
+# Percentage of 0 means never use the delta, percentage of 100 means always
+# use it (assuming the delta is never bigger than the pkg).
diff --git a/yum-presto/presto.py b/yum-presto/presto.py
index 7e59dc3..ee7af65 100644
--- a/yum-presto/presto.py
+++ b/yum-presto/presto.py
@@ -149,7 +149,7 @@ class ReconstructionThread(threading.Thread):
self.lock.release()
-def getDelta(po, presto, conduit):
+def getDelta(po, presto, conduit, conf_minimum_percentage=100):
"""Does the package have a reasonable delta for us to use?"""
rpmdb = conduit.getRpmDB()
@@ -211,6 +211,11 @@ def getDelta(po, presto, conduit):
continue
delta = deltas[evr]
+ # If the delta isn't small enough, ignore it
+ povsize = (po.size * conf_minimum_percentage) / 100
+ if povsize < delta['size']:
+ continue
+
# we just want to use the smallest delta
if bestdelta and delta['size'] >= bestdelta['size']:
continue
@@ -265,6 +270,13 @@ def downloadPkgs(conduit, presto, download_pkgs=None):
rebuild_size = 0 # Size of packages we are going to rebuild.
cb = None
+ conf_mp = conduit.confInt('main', 'minimum_percentage',
+ default=95)
+ if conf_mp > 100:
+ conf_mp = 100
+ if conf_mp < 0:
+ conf_mp = 0
+
# Set up thread for applying drpms
queue = Queue.Queue(0)
process_lock = thread.allocate_lock()
@@ -284,7 +296,7 @@ def downloadPkgs(conduit, presto, download_pkgs=None):
# downloaded, we can start it reconstructing in the background
deltapath_cache = set()
for po in download_pkgs:
- delta = getDelta(po, presto, conduit)
+ delta = getDelta(po, presto, conduit, conf_mp)
if delta is None:
must_still_download_pkgs += po.size
continue
commit bc5f307377ee973e9def78db88f8e1e9699595c9
Author: James Antill <james at and.org>
Date: Tue Apr 28 14:21:03 2009 -0400
"from urlgrabber.progress import" != "import urlgrabber.progress"
diff --git a/yum-presto/presto.py b/yum-presto/presto.py
index ee7af65..17502bb 100644
--- a/yum-presto/presto.py
+++ b/yum-presto/presto.py
@@ -37,6 +37,7 @@ import yum.Errors
import yum.misc
from urlgrabber.grabber import URLGrabError
from urlgrabber.progress import format_number
+import urlgrabber.progress
complete_download_size = 0
actual_download_size = 0
Index: yum-presto.spec
===================================================================
RCS file: /cvs/pkgs/rpms/yum-presto/devel/yum-presto.spec,v
retrieving revision 1.12
retrieving revision 1.13
diff -u -p -r1.12 -r1.13
--- yum-presto.spec 26 Apr 2009 17:06:47 -0000 1.12
+++ yum-presto.spec 28 Apr 2009 18:46:39 -0000 1.13
@@ -3,12 +3,13 @@
Summary: Presto plugin for yum
Name: yum-presto
Version: 0.4.5
-Release: 5%{?dist}
+Release: 6%{?dist}
License: GPLv2+
Group: Development/Tools
Source: http://www.lesbg.com/jdieter/presto/%{name}-%{version}.tar.bz2
Patch0: speedup.patch
Patch1: cleanup.patch
+Patch2: total-progress+stat-fixes.patch
URL: http://www.lesbg.com/jdieter/presto/
BuildRoot: %{_tmppath}/%{name}-%{version}-%{release}-root-%(%{__id_u} -n)
BuildArch: noarch
@@ -30,6 +31,7 @@ foo-1.1 package from your installed foo-
%patch0 -p2
%patch1 -p2
+%patch2 -p2
%build
@@ -50,6 +52,11 @@ rm -rf $RPM_BUILD_ROOT
%config(noreplace) %{_sysconfdir}/yum/pluginconf.d/presto.conf
%changelog
+* Tue Apr 28 2009 James Antill <james at fedoraproject.org> - 0.4.5-6
+- Add total progress data to delta downloads
+- Fix stats. to account for the fact not everything has a delta
+- Allow delta downloads to be configurable on savings
+
* Sun Apr 26 2009 James Antill <james at fedoraproject.org> - 0.4.5-5
- Added cleanup patch from upstream.
- Adds progress for rebuilding.
- Previous message (by thread): rpms/R-BufferedMatrixMethods/devel .cvsignore, 1.5, 1.6 R-BufferedMatrixMethods.spec, 1.7, 1.8 sources, 1.5, 1.6
- Next message (by thread): rpms/syslinux/devel sources,1.26,1.27 syslinux.spec,1.58,1.59
- Messages sorted by:
[ date ]
[ thread ]
[ subject ]
[ author ]
More information about the fedora-extras-commits
mailing list