extras-buildsys/utils extras-push-new,1.24,1.24.2.1
Michael Schwendt (mschwendt)
fedora-extras-commits at redhat.com
Fri Jun 9 08:08:18 UTC 2006
Author: mschwendt
Update of /cvs/fedora/extras-buildsys/utils
In directory cvs-int.fedora.redhat.com:/tmp/cvs-serv2436
Modified Files:
Tag: push-new-tmpdir_20060609
extras-push-new
Log Message:
No longer sign and move the original rpms in the plague-results
repository. Create temporary copies and work on them.
No longer clean up plague-results directory either. Mark unneeded
build job results and successfully pushed builds as "PUSHED".
TODO: They can be cleaned up after some safe time in order to not
break the plague-results repo.
Run sync and repoclosure only in -f force mode or after any changes
have been noticed.
Previous production version of push script is tagged: push-new-production-3
(rev 1.24)
Ville's hardlink development is still in HEAD (rev 1.25)
Index: extras-push-new
===================================================================
RCS file: /cvs/fedora/extras-buildsys/utils/extras-push-new,v
retrieving revision 1.24
retrieving revision 1.24.2.1
diff -u -r1.24 -r1.24.2.1
--- extras-push-new 2 Jun 2006 22:54:38 -0000 1.24
+++ extras-push-new 9 Jun 2006 08:08:16 -0000 1.24.2.1
@@ -21,6 +21,7 @@
import rpmUtils
import shutil
import string
+import tempfile
DEBUG = False
@@ -95,7 +96,7 @@
def run_and_check(cmd):
- debugprint(cmd)
+ debugprint('DEBUG: %s' % cmd)
if DEBUG:
return
result = os.system(cmd)
@@ -165,6 +166,16 @@
# ====================================================================
+def list_repo_root(repodir):
+ """returns list of directory names in repository root directory"""
+ d = os.listdir(repodir)
+ try:
+ d.remove('repodata')
+ except:
+ pass
+ return d
+
+
def find_srcrpms(rootpath):
"""returns a list of source rpm path names in given tree"""
srcrpms = []
@@ -176,15 +187,27 @@
return srcrpms
-def prune_needsign_tree(repodir):
- # The root dir contains a directory for every pushed package "name".
- pkgnames = os.listdir(repodir)
+def mark_pkg_pushed(pkgrelroot):
+ fname = os.path.join(pkgrelroot,'PUSHED')
+ if DEBUG:
+ print 'Creating %s' % fname
+ return
try:
- pkgnames.remove('repodata')
- except:
- pass
+ f = open(fname,'w')
+ except IOError, (errno, strerr):
+ print 'ERROR: %s: %s' % (strerr,fname)
+ sys.exit(errno) # fatal for our needs
+ f.close()
+
+
+def prune_needsign_tree(repodir):
+ # Currently, this only marks unneeded package build job directories
+ # as PUSHED.
+ #
+ # TODO: Make this remove sufficiently old PUSHED directories.
- for name in pkgnames:
+ # The root dir contains a directory for every pushed package "name".
+ for name in list_repo_root(repodir):
pkgroot = os.path.join(repodir,name)
if not os.path.isdir(pkgroot):
continue
@@ -214,30 +237,45 @@
(nextn,nexta,nexte,nextv,nextr) = naevr(nextsrcrpm)
# 1 means: e,v,r is higher than nexte,nextv,nextr
if rpmUtils.miscutils.compareEVR((e,v,r),(nexte,nextv,nextr)) == 1:
- debugprint('Removing: %s' % nextrelroot)
- if not DEBUG:
- shutil.rmtree(nextrelroot)
+ debugprint('Ignoring: %s' % nextrelroot)
+ mark_pkg_pushed(nextrelroot)
else:
- debugprint('Removing: %s' % relroot)
- if not DEBUG:
- shutil.rmtree(relroot)
+ debugprint('Ignoring: %s' % relroot)
+ mark_pkg_pushed(relroot)
# Make this the next newest package for ongoing comparison.
relroot = nextrelroot
(n,a,e,v,r) = (nextn,nexta,nexte,nextv,nextr)
-# ====================================================================
-# (previously in extras-sign-move.py)
+def debugcopy2(src,dest):
+ """shutil.copy2 a file, but don't overwrite destination"""
+ if os.path.exists(dest):
+ debugprint('WARNING: %s already exists, ignoring new one' % dest)
+ return
+ debugprint('Copying %s to %s' % (src,dest))
+ if not DEBUG:
+ shutil.copy2(src,dest)
+
+
+def debugmove(src,dest):
+ """shutil.move a file, but don't overwrite destination"""
+ if os.path.exists(dest):
+ debugprint('WARNING: %s already exists, ignoring new one' % dest)
+ return
+ debugprint('Copying %s to %s' % (src,dest))
+ if not DEBUG:
+ shutil.move(src,dest)
+
# get the path to where to look for the packages to be signed
# get the list of stuff there (logs, md5sums, specs, rpms)
-# list the rpms and pause
+# list the rpms
+# copy the rpms to a tmp directory
# sign the rpms
# collect list of srpms and get package n-v-r from them
-# move pkgs into repo
+# move/copy pkgs into repo
-
-def find_files(path):
+def find_files(repodir):
"""returns a dict of filetypes and paths to those files"""
filedict = {}
filedict['srpm'] = []
@@ -247,27 +285,43 @@
filedict['md5sum'] = []
filedict['debuginfo'] = []
filedict['other'] = []
- for root, dirs, files in os.walk(path):
- for file in files:
- # match the files to what list they should be in
- if file.endswith('.log'):
- which = 'log'
- elif file.endswith('.rpm'):
- if file.find('debuginfo') != -1:
- which = 'debuginfo'
- elif file.endswith('.src.rpm'):
- which = 'srpm'
+ filedict['pkgrelroot'] = []
+
+ # The root dir contains a directory for every pushed package "name".
+ for name in list_repo_root(repodir):
+ pkgroot = os.path.join(repodir,name)
+ if not os.path.isdir(pkgroot):
+ continue
+ # Every version-release is stored in an own sub-dir.
+ # We want only those which are not marked as PUSHED yet.
+ for pkgrel in os.listdir(pkgroot):
+ pkgrelroot = os.path.join(pkgroot,pkgrel)
+ if os.path.isfile( os.path.join(pkgrelroot,'PUSHED') ):
+ continue
+ filedict['pkgrelroot'].append(pkgrelroot)
+
+ for path in filedict['pkgrelroot']:
+ for root, dirs, files in os.walk(path):
+ for file in files:
+ # match the files to what list they should be in
+ if file.endswith('.log'):
+ which = 'log'
+ elif file.endswith('.rpm'):
+ if file.find('debuginfo') != -1:
+ which = 'debuginfo'
+ elif file.endswith('.src.rpm'):
+ which = 'srpm'
+ else:
+ which = 'rpm'
+ elif file.endswith('.spec'):
+ which = 'spec'
+ elif file.endswith('.md5sum'):
+ which = 'md5sum'
else:
- which = 'rpm'
- elif file.endswith('.spec'):
- which = 'spec'
- elif file.endswith('.md5sum'):
- which = 'md5sum'
- else:
- which = 'other'
+ which = 'other'
- fullfile = os.path.join(root, file)
- filedict[which].append(fullfile)
+ fullfile = os.path.join(root, file)
+ filedict[which].append(fullfile)
return filedict
@@ -291,7 +345,7 @@
numfiles = len(filelist)
if numfiles < 1:
print "No packages to sign"
- return errno.EINVAL
+ return 0
while numfiles > 0:
if numfiles > 256:
@@ -310,7 +364,7 @@
return 0
-def sign_move(dist):
+def copy_sign_move(dist):
runfilename = getrunfilename(dist)
try:
rundirfile = open(runfilename,'a')
@@ -320,7 +374,14 @@
distdir = '%s-%s-%s' % (distro, dist, project)
needsign = os.path.join(stagesdir, distdir)
-
+ destdir = os.path.join(treedir, dist)
+ if not os.path.exists(destdir):
+ for arch in archdict[dist]:
+ if not DEBUG:
+ os.makedirs(destdir + '/' + arch + '/debug')
+ if not DEBUG:
+ os.makedirs(destdir + '/' + 'SRPMS')
+
repolockname = os.path.join(needsign,REPO_LOCKFILE_NAME)
repolock = LockFile(name=repolockname,blocking=True)
try:
@@ -334,52 +395,72 @@
prune_needsign_tree(needsign)
files = find_files(needsign)
+
+ repolock.unlock()
+
rpms = files['rpm'] + files['srpm'] + files['debuginfo']
+ print 'Total packages found: %d' % len(rpms)
+ if len(rpms) == 0:
+ print '%s: nothing to do' % dist
+ return 0
rpms.sort()
for item in rpms:
print ' %s' % os.path.basename(item)
- destdir = os.path.join(treedir, dist)
- if not os.path.exists(destdir):
- for arch in archdict[dist]:
+ print 'Copying packages to temporary working directory:'
+ if not DEBUG:
+ try:
+ signtmpdir = tempfile.mkdtemp('','.push',treedir)
+ if signtmpdir == treedir: # paranoid, should never happen
+ raise Exception
+ except:
+ return errno.EIO
+ else:
+ signtmpdir = os.path.join(treedir,'.push.tmp')
+ relocrpms = [] # just the list of target files for gpg signing
+ for src in rpms:
+ dst = signtmpdir + src
+ relocrpms.append(dst)
+ (destpath,destfile) = os.path.split(dst)
+ try:
if not DEBUG:
- os.makedirs(destdir + '/' + arch + '/debug')
- if not DEBUG:
- os.makedirs(destdir + '/' + 'SRPMS')
+ if not os.path.exists(destpath):
+ os.makedirs(destpath)
+ debugcopy2(signtmpdir+src,dst,'')
+ except:
+ print 'ERROR: Creating temporary working copy failed.'
+ debugprint('Removing tree %s' % signtmpdir)
+ if not DEBUG:
+ shutil.rmtree(signtmpdir)
+ sys.exit(errno.EIO)
- rpms = files['rpm'] + files['srpm'] + files['debuginfo']
- if len(rpms) == 0:
- print '%s: nothing to sign' % dist
- repolock.unlock()
- return 0
+ # At this point, we have copies of all rpms below signtmpdir.
+ # We prefix every previously found file with signtmpdir, so
+ # that we work on the temporary copy and not in the original
+ # plague-results directory.
print "Signing Packages:"
if not DEBUG:
- result = sign_pkgs(rpms)
+ result = sign_pkgs(relocrpms)
if result:
print "Error signing packages"
- repolock.unlock()
+ debugprint('Removing tree %s' % signtmpdir)
+ if not DEBUG:
+ shutil.rmtree(signtmpdir)
return result
- print "Moving Packages into Place"
-
- infolist = []
+ print "Copying packages into place:"
+ infolist = [] # list of src.rpm NEVR for build report
+
for package in files['srpm']:
(n,a,e,v,r) = naevr(package)
+ infolist.append('%s-%s-%s' % (n,v,r))
pkg_fn = os.path.basename(package)
srpmloc = os.path.join(destdir, 'SRPMS', pkg_fn)
- debugprint('Moving %s to %s' % (package, srpmloc))
- if os.path.exists(srpmloc):
- debugprint('%s already exists, ignoring new one' % srpmloc)
- if not DEBUG:
- os.unlink(package)
- else:
- infolist.append('%s-%s-%s' % (n,v,r))
- if not DEBUG:
- shutil.move(package, srpmloc)
+ debugmove(signtmpdir+package,srpmloc)
- # go through each package and move it to the right arch location.
- # if it is a noarch package, copy2 it to all arch locations and unlink it
+ # go through each package and move it to the right arch location
+ # if it is a noarch package, copy2 it to all arch locations
# if it is listed in copydict, copy2 it around as appropriate
# if it is a debuginfo package, move it into the 'debug' dir for that arch
@@ -395,25 +476,14 @@
if package in files['debuginfo']:
arch = '%s/debug' % arch
rpmloc = os.path.join(destdir, arch, pkg_fn)
- debugprint("Copying %s to %s" % (package, rpmloc))
- if os.path.exists(rpmloc):
- debugprint('%s already exists, ignoring new one' % rpmloc)
- elif not DEBUG:
- shutil.copy2(package, rpmloc)
+ debugcopy2(signtmpdir+package,rpmloc)
if a == 'noarch':
for arch in archdict[dist]:
if package in files['debuginfo']:
arch = '%s/debug' % arch
rpmloc = os.path.join(destdir, arch, pkg_fn)
- debugprint("Copying %s to %s" % (package, rpmloc))
- if os.path.exists(rpmloc):
- debugprint('%s already exists, ignoring new one' % rpmloc)
- elif not DEBUG:
- shutil.copy2(package, rpmloc)
-
- if not DEBUG:
- os.unlink(package)
+ debugcopy2(signtmpdir+package,rpmloc)
continue
elif a in ['i386', 'i486', 'i586', 'i686', 'athlon']:
@@ -439,41 +509,21 @@
continue
rpmloc = os.path.join(destdir, arch, pkg_fn)
- debugprint('Moving %s to %s' % (package, rpmloc))
- if os.path.exists(rpmloc):
- debugprint('%s already exists, ignoring new one' % rpmloc)
- if not DEBUG:
- os.unlink(package)
- elif not DEBUG:
- shutil.move(package, rpmloc)
+ debugmove(signtmpdir+package,rpmloc)
- # Store list of move packages, so it can be mailed later.
+ # Store list of new packages on disk, so it can be mailed later.
for add in infolist:
add += '\n'
rundirfile.write( add )
rundirfile.close()
- # clean up the crap
- print "Cleaning Up"
- for file in files['log'] + files['md5sum'] + files['spec']:
- debugprint('Removing %s' % file)
- if not DEBUG:
- os.unlink(file)
-
- # clean up empty dirs, too.
- for root, dirs, files in os.walk(needsign,topdown=False):
- if 'repodata' in dirs:
- dirs.remove('repodata')
- for dir in dirs:
- path = os.path.join(root,dir)
- try:
- if not DEBUG:
- os.rmdir(path)
- debugprint('Removing directory %s' % path)
- except:
- pass
+ # Mark successfully signed packages as PUSHED.
+ for path in files['pkgrelroot']:
+ mark_pkg_pushed(path)
- repolock.unlock()
+ debugprint('Removing tree %s' % signtmpdir)
+ if not DEBUG:
+ shutil.rmtree(signtmpdir)
return(0)
# ====================================================================
@@ -570,10 +620,16 @@
print 'ERROR: script locked via lockfile %s - it seems to be running already' % lockfile
sys.exit(errno.EPERM)
+ changecount = {} # released build requests per dist
+ totalchanges = 0
for dist in diststopush:
- result = sign_move(dist)
+ result = copy_sign_move(dist)
if result:
sys.exit(result)
+ # len(getlinesfromrunfile(dist)) is the number of packages
+ # per dist which have been pushed.
+ changecount[dist] = len( getlinesfromrunfile(dist) )
+ totalchanges += changecount[dist]
# Option -f re-runs repobuild/repoview for all dists, even if
# no new packages have been pushed for a dist.
@@ -581,27 +637,25 @@
if opts.force:
changed = alldists
- # len(getlinesfromrunfile(dist)) is the number of packages
- # per dist which have been pushed.
-
for dist in changed:
- if opts.force or len( getlinesfromrunfile(dist) ):
+ if opts.force or changecount[dist]:
run_and_check('extras-repobuild.py %s' % dist)
- if opts.doublesync:
+ if opts.doublesync and totalchanges:
run_and_check('extras-sync')
for dist in changed:
- if opts.force or len( getlinesfromrunfile(dist) ):
+ if opts.force or changecount[dist]:
if opts.repoview:
run_and_check('extras-repoview.py %s' % dist)
- run_and_check('extras-sync')
+ if opts.force or totalchanges:
+ run_and_check('extras-sync')
if opts.mail:
email_list( diststopush )
- if opts.repoclosure:
+ if opts.repoclosure and (opts.force or totalchanges):
run_and_check('nohup rc-run-all.py &')
lock.unlock()
More information about the fedora-extras-commits
mailing list