extras-buildsys/utils extras-push-new,1.24,1.24.2.1

Michael Schwendt (mschwendt) fedora-extras-commits at redhat.com
Fri Jun 9 08:08:18 UTC 2006


Author: mschwendt

Update of /cvs/fedora/extras-buildsys/utils
In directory cvs-int.fedora.redhat.com:/tmp/cvs-serv2436

Modified Files:
      Tag: push-new-tmpdir_20060609
	extras-push-new 
Log Message:
No longer sign and move the original rpms in the plague-results
repository. Create temporary copies and work on them.

No longer clean up plague-results directory either. Mark unneeded
build job results and successfully pushed builds as "PUSHED".
TODO: They can be cleaned up after some safe time in order to not
break the plague-results repo.

Run sync and repoclosure only in -f force mode or after any changes
have been noticed.

Previous production version of push script is tagged: push-new-production-3
(rev 1.24)

Ville's hardlink development is still in HEAD (rev 1.25)




Index: extras-push-new
===================================================================
RCS file: /cvs/fedora/extras-buildsys/utils/extras-push-new,v
retrieving revision 1.24
retrieving revision 1.24.2.1
diff -u -r1.24 -r1.24.2.1
--- extras-push-new	2 Jun 2006 22:54:38 -0000	1.24
+++ extras-push-new	9 Jun 2006 08:08:16 -0000	1.24.2.1
@@ -21,6 +21,7 @@
 import rpmUtils
 import shutil
 import string
+import tempfile
 
 DEBUG = False
 
@@ -95,7 +96,7 @@
         
 
 def run_and_check(cmd):
-    debugprint(cmd)
+    debugprint('DEBUG: %s' % cmd)
     if DEBUG:
         return
     result = os.system(cmd)
@@ -165,6 +166,16 @@
 
 # ====================================================================
 
+def list_repo_root(repodir):
+    """returns list of directory names in repository root directory"""
+    d = os.listdir(repodir)
+    try:
+        d.remove('repodata')
+    except:
+        pass
+    return d
+
+
 def find_srcrpms(rootpath):
     """returns a list of source rpm path names in given tree"""
     srcrpms = []
@@ -176,15 +187,27 @@
     return srcrpms
 
 
-def prune_needsign_tree(repodir):
-    # The root dir contains a directory for every pushed package "name".
-    pkgnames = os.listdir(repodir)
+def mark_pkg_pushed(pkgrelroot):
+    fname = os.path.join(pkgrelroot,'PUSHED')
+    if DEBUG:
+        print 'Creating %s' % fname
+        return
     try:
-        pkgnames.remove('repodata')
-    except:
-        pass
+        f = open(fname,'w')
+    except IOError, (errno, strerr):
+        print 'ERROR: %s: %s' % (strerr,fname)
+        sys.exit(errno)  # fatal for our needs
+    f.close()
+
+
+def prune_needsign_tree(repodir):
+    # Currently, this only marks unneeded package build job directories
+    # as PUSHED.
+    #
+    # TODO: Make this remove sufficiently old PUSHED directories.
 
-    for name in pkgnames:
+    # The root dir contains a directory for every pushed package "name".
+    for name in list_repo_root(repodir):
         pkgroot = os.path.join(repodir,name)
         if not os.path.isdir(pkgroot):
             continue
@@ -214,30 +237,45 @@
             (nextn,nexta,nexte,nextv,nextr) = naevr(nextsrcrpm)
             # 1 means: e,v,r is higher than nexte,nextv,nextr
             if rpmUtils.miscutils.compareEVR((e,v,r),(nexte,nextv,nextr)) == 1:
-                debugprint('Removing: %s' % nextrelroot)
-                if not DEBUG:
-                    shutil.rmtree(nextrelroot)
+                debugprint('Ignoring: %s' % nextrelroot)
+                mark_pkg_pushed(nextrelroot)
             else:
-                debugprint('Removing: %s' % relroot)
-                if not DEBUG:
-                    shutil.rmtree(relroot)
+                debugprint('Ignoring: %s' % relroot)
+                mark_pkg_pushed(relroot)
                 # Make this the next newest package for ongoing comparison.
                 relroot = nextrelroot
                 (n,a,e,v,r) = (nextn,nexta,nexte,nextv,nextr)
 
 
-# ====================================================================
-# (previously in extras-sign-move.py)
+def debugcopy2(src,dest):
+    """shutil.copy2 a file, but don't overwrite destination"""
+    if os.path.exists(dest):
+        debugprint('WARNING: %s already exists, ignoring new one' % dest)
+        return
+    debugprint('Copying %s to %s' % (src,dest))
+    if not DEBUG:
+        shutil.copy2(src,dest)
+
+
+def debugmove(src,dest):
+    """shutil.move a file, but don't overwrite destination"""
+    if os.path.exists(dest):
+        debugprint('WARNING: %s already exists, ignoring new one' % dest)
+        return
+    debugprint('Copying %s to %s' % (src,dest))
+    if not DEBUG:
+        shutil.move(src,dest)
+
 
 # get the path to where to look for the packages to be signed
 # get the list of stuff there (logs, md5sums, specs, rpms)
-# list the rpms and pause
+# list the rpms
+# copy the rpms to a tmp directory
 # sign the rpms
 # collect list of srpms and get package n-v-r from them
-# move pkgs into repo
+# move/copy pkgs into repo
 
-        
-def find_files(path):
+def find_files(repodir):
     """returns a dict of filetypes and paths to those files"""
     filedict = {}
     filedict['srpm'] = []
@@ -247,27 +285,43 @@
     filedict['md5sum'] = []
     filedict['debuginfo'] = []
     filedict['other'] = []
-    for root, dirs, files in os.walk(path):
-        for file in files:
-            # match the files to what list they should be in
-            if file.endswith('.log'):
-                which = 'log'
-            elif file.endswith('.rpm'):
-                if file.find('debuginfo') != -1:
-                    which = 'debuginfo' 
-                elif file.endswith('.src.rpm'):
-                    which = 'srpm'
+    filedict['pkgrelroot'] = []
+
+    # The root dir contains a directory for every pushed package "name".
+    for name in list_repo_root(repodir):
+        pkgroot = os.path.join(repodir,name)
+        if not os.path.isdir(pkgroot):
+            continue
+        # Every version-release is stored in an own sub-dir.
+        # We want only those which are not marked as PUSHED yet.
+        for pkgrel in os.listdir(pkgroot):
+            pkgrelroot = os.path.join(pkgroot,pkgrel)
+            if os.path.isfile( os.path.join(pkgrelroot,'PUSHED') ):
+                continue
+            filedict['pkgrelroot'].append(pkgrelroot)
+
+    for path in filedict['pkgrelroot']:
+        for root, dirs, files in os.walk(path):
+            for file in files:
+                # match the files to what list they should be in
+                if file.endswith('.log'):
+                    which = 'log'
+                elif file.endswith('.rpm'):
+                    if file.find('debuginfo') != -1:
+                        which = 'debuginfo' 
+                    elif file.endswith('.src.rpm'):
+                        which = 'srpm'
+                    else:
+                        which = 'rpm'
+                elif file.endswith('.spec'):
+                    which = 'spec'
+                elif file.endswith('.md5sum'):
+                    which = 'md5sum'
                 else:
-                    which = 'rpm'
-            elif file.endswith('.spec'):
-                which = 'spec'
-            elif file.endswith('.md5sum'):
-                which = 'md5sum'
-            else:
-                which = 'other'
+                    which = 'other'
 
-            fullfile = os.path.join(root, file)
-            filedict[which].append(fullfile)
+                fullfile = os.path.join(root, file)
+                filedict[which].append(fullfile)
     return filedict
 
 
@@ -291,7 +345,7 @@
     numfiles = len(filelist)
     if numfiles < 1:
         print "No packages to sign"
-        return errno.EINVAL
+        return 0
 
     while numfiles > 0:
         if numfiles > 256:
@@ -310,7 +364,7 @@
     return 0
 
     
-def sign_move(dist):
+def copy_sign_move(dist):
     runfilename = getrunfilename(dist)
     try:
         rundirfile = open(runfilename,'a')
@@ -320,7 +374,14 @@
 
     distdir = '%s-%s-%s' % (distro, dist, project)
     needsign = os.path.join(stagesdir, distdir)
-    
+    destdir = os.path.join(treedir, dist)
+    if not os.path.exists(destdir):
+        for arch in archdict[dist]:
+            if not DEBUG:
+                os.makedirs(destdir + '/' + arch + '/debug')
+        if not DEBUG:
+            os.makedirs(destdir + '/' + 'SRPMS')
+
     repolockname = os.path.join(needsign,REPO_LOCKFILE_NAME)
     repolock = LockFile(name=repolockname,blocking=True)
     try:
@@ -334,52 +395,72 @@
         
     prune_needsign_tree(needsign)
     files = find_files(needsign)
+
+    repolock.unlock()
+
     rpms = files['rpm'] + files['srpm'] + files['debuginfo']    
+    print 'Total packages found: %d' % len(rpms)
+    if len(rpms) == 0:
+        print '%s: nothing to do' % dist
+        return 0
     rpms.sort()
     for item in rpms:
         print ' %s' % os.path.basename(item)
 
-    destdir = os.path.join(treedir, dist)
-    if not os.path.exists(destdir):
-        for arch in archdict[dist]:
+    print 'Copying packages to temporary working directory:'
+    if not DEBUG:
+        try:
+            signtmpdir = tempfile.mkdtemp('','.push',treedir)
+            if signtmpdir == treedir: # paranoid, should never happen
+                raise Exception
+        except:
+            return errno.EIO
+    else:
+        signtmpdir = os.path.join(treedir,'.push.tmp')
+    relocrpms = []  # just the list of target files for gpg signing
+    for src in rpms:
+        dst = signtmpdir + src
+        relocrpms.append(dst)
+        (destpath,destfile) = os.path.split(dst)
+        try:
             if not DEBUG:
-                os.makedirs(destdir + '/' + arch + '/debug')
-        if not DEBUG:
-            os.makedirs(destdir + '/' + 'SRPMS')
+                if not os.path.exists(destpath):
+                    os.makedirs(destpath)
+            debugcopy2(signtmpdir+src,dst,'')
+        except:
+            print 'ERROR: Creating temporary working copy failed.'
+            debugprint('Removing tree %s' % signtmpdir)
+            if not DEBUG:
+                shutil.rmtree(signtmpdir)
+            sys.exit(errno.EIO)
 
-    rpms = files['rpm'] + files['srpm'] + files['debuginfo']
-    if len(rpms) == 0:
-        print '%s: nothing to sign' % dist
-        repolock.unlock()
-        return 0
+    # At this point, we have copies of all rpms below signtmpdir.
+    # We prefix every previously found file with signtmpdir, so
+    # that we work on the temporary copy and not in the original
+    # plague-results directory.
 
     print "Signing Packages:"
     if not DEBUG:
-        result = sign_pkgs(rpms)
+        result = sign_pkgs(relocrpms)
         if result:
             print "Error signing packages"
-            repolock.unlock()
+            debugprint('Removing tree %s' % signtmpdir)
+            if not DEBUG:
+                shutil.rmtree(signtmpdir)
             return result
 
-    print "Moving Packages into Place"
-
-    infolist = []
+    print "Copying packages into place:"
+    infolist = []  # list of src.rpm NEVR for build report
+    
     for package in files['srpm']:
         (n,a,e,v,r) = naevr(package)
+        infolist.append('%s-%s-%s' % (n,v,r))
         pkg_fn = os.path.basename(package)
         srpmloc = os.path.join(destdir, 'SRPMS', pkg_fn)
-        debugprint('Moving %s to %s' % (package, srpmloc))
-        if os.path.exists(srpmloc):
-            debugprint('%s already exists, ignoring new one' % srpmloc)
-            if not DEBUG:
-                os.unlink(package)
-        else:
-            infolist.append('%s-%s-%s' % (n,v,r))
-            if not DEBUG:
-                shutil.move(package, srpmloc)
+        debugmove(signtmpdir+package,srpmloc)
 
-    # go through each package and move it to the right arch location.
-    # if it is a noarch package, copy2 it to all arch locations and unlink it
+    # go through each package and move it to the right arch location
+    # if it is a noarch package, copy2 it to all arch locations
     # if it is listed in copydict, copy2 it around as appropriate
     # if it is a debuginfo package, move it into the 'debug' dir for that arch
 
@@ -395,25 +476,14 @@
                 if package in files['debuginfo']:
                     arch = '%s/debug' % arch
                 rpmloc = os.path.join(destdir, arch, pkg_fn)
-                debugprint("Copying %s to %s" % (package, rpmloc))
-                if os.path.exists(rpmloc):
-                    debugprint('%s already exists, ignoring new one' % rpmloc)
-                elif not DEBUG:
-                    shutil.copy2(package, rpmloc)
+                debugcopy2(signtmpdir+package,rpmloc)
 
         if a == 'noarch':
             for arch in archdict[dist]:
                 if package in files['debuginfo']:
                     arch = '%s/debug' % arch
                 rpmloc = os.path.join(destdir, arch, pkg_fn)
-                debugprint("Copying %s to %s" % (package, rpmloc))
-                if os.path.exists(rpmloc):
-                    debugprint('%s already exists, ignoring new one' % rpmloc)
-                elif not DEBUG:
-                    shutil.copy2(package, rpmloc)
-            
-            if not DEBUG:
-                os.unlink(package)
+                debugcopy2(signtmpdir+package,rpmloc)
             continue
             
         elif a in ['i386', 'i486', 'i586', 'i686', 'athlon']:
@@ -439,41 +509,21 @@
             continue
 
         rpmloc = os.path.join(destdir, arch, pkg_fn)
-        debugprint('Moving %s to %s' % (package, rpmloc))        
-        if os.path.exists(rpmloc):
-            debugprint('%s already exists, ignoring new one' % rpmloc)
-            if not DEBUG:
-                os.unlink(package)
-        elif not DEBUG:
-            shutil.move(package, rpmloc)
+        debugmove(signtmpdir+package,rpmloc)
 
-    # Store list of move packages, so it can be mailed later.
+    # Store list of new packages on disk, so it can be mailed later.
     for add in infolist:
         add += '\n'
         rundirfile.write( add )
     rundirfile.close()
     
-    # clean up the crap
-    print "Cleaning Up"
-    for file in files['log'] + files['md5sum'] + files['spec']:
-        debugprint('Removing %s' % file)
-        if not DEBUG:
-            os.unlink(file)
-        
-    # clean up empty dirs, too.
-    for root, dirs, files in os.walk(needsign,topdown=False):
-        if 'repodata' in dirs:
-            dirs.remove('repodata')
-        for dir in dirs:
-            path = os.path.join(root,dir)
-            try:
-                if not DEBUG:
-                    os.rmdir(path)
-                debugprint('Removing directory %s' % path)
-            except:
-                pass
+    # Mark successfully signed packages as PUSHED.
+    for path in files['pkgrelroot']:
+        mark_pkg_pushed(path)
 
-    repolock.unlock()
+    debugprint('Removing tree %s' % signtmpdir)
+    if not DEBUG:
+        shutil.rmtree(signtmpdir)
     return(0)
 
 # ====================================================================
@@ -570,10 +620,16 @@
         print 'ERROR: script locked via lockfile %s - it seems to be running already' % lockfile
         sys.exit(errno.EPERM)
 
+    changecount = {}  # released build requests per dist
+    totalchanges = 0
     for dist in diststopush:
-        result = sign_move(dist)
+        result = copy_sign_move(dist)
         if result:
             sys.exit(result)
+        # len(getlinesfromrunfile(dist)) is the number of packages
+        # per dist which have been pushed.
+        changecount[dist] = len( getlinesfromrunfile(dist) )
+        totalchanges += changecount[dist]
 
     # Option -f re-runs repobuild/repoview for all dists, even if
     # no new packages have been pushed for a dist.
@@ -581,27 +637,25 @@
     if opts.force:
         changed = alldists
 
-    # len(getlinesfromrunfile(dist)) is the number of packages
-    # per dist which have been pushed.
-
     for dist in changed:
-        if opts.force or len( getlinesfromrunfile(dist) ):
+        if opts.force or changecount[dist]:
             run_and_check('extras-repobuild.py %s' % dist)
 
-    if opts.doublesync:
+    if opts.doublesync and totalchanges:
         run_and_check('extras-sync')
 
     for dist in changed:
-        if opts.force or len( getlinesfromrunfile(dist) ):
+        if opts.force or changecount[dist]:
             if opts.repoview:
                 run_and_check('extras-repoview.py %s' % dist)
 
-    run_and_check('extras-sync')
+    if opts.force or totalchanges:
+        run_and_check('extras-sync')
 
     if opts.mail:
         email_list( diststopush )
 
-    if opts.repoclosure:
+    if opts.repoclosure and (opts.force or totalchanges):
         run_and_check('nohup rc-run-all.py &')
 
     lock.unlock()




More information about the fedora-extras-commits mailing list