extras-buildsys/utils extras-push-new,NONE,1.1
Michael Schwendt (mschwendt)
fedora-extras-commits at redhat.com
Sat May 6 12:56:43 UTC 2006
- Previous message (by thread): rpms/syslog-ng/FC-5 .cvsignore, 1.5, 1.6 sources, 1.5, 1.6 syslog-ng.spec, 1.12, 1.13
- Next message (by thread): rpms/syslog-ng/FC-3 .cvsignore, 1.5, 1.6 sources, 1.5, 1.6 syslog-ng.spec, 1.8, 1.9
- Messages sorted by:
[ date ]
[ thread ]
[ subject ]
[ author ]
Author: mschwendt
Update of /cvs/fedora/extras-buildsys/utils
In directory cvs-int.fedora.redhat.com:/tmp/cvs-serv7584
Added Files:
extras-push-new
Log Message:
Add a extras-push-new script, which combines the scripts extras-push, extras-push-all and extras-sign-move.py and avoids some of the problems we've experienced.
--- NEW FILE extras-push-new ---
#!/usr/bin/python -t
# -*- mode: Python; indent-tabs-mode: nil; -*-
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Library General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
import fcntl
import os, sys
import rpmUtils
import shutil
import string
DEBUG = False
signersgid = 100581 # extras_signers group
distro = 'fedora'
project = 'extras'
stagesdir = '/srv/rpmbuild/repodir'
treedir = '/srv/rpmbuild/%s/tree/%s' % (project, project)
# Where we store mails to be sent, lockfiles.
rundir = '/srv/rpmbuild/extras-push-rundir'
cachedir = '/tmp/repomd-cache'
compsname = 'comps.xml'
project_hr = 'Fedora Extras'
mail_from = 'buildsys at fedoraproject.org'
mail_to = 'fedora-extras-list at redhat.com'
mail_footer = """
For more information about the built packages please see the repository
or the fedora Info Feed: http://fedoraproject.org/infofeed/
"""
config = { 'repoview' : True,
'repoclosure' : False,
'gidcheck' : True,
'setumask' : True,
'mail' : True
}
alldists = [ 'development', '5', '4', '3' ]
archdict = { '3' : ['x86_64', 'i386'],
'4' : ['ppc','x86_64', 'i386'],
'5' : ['ppc','x86_64', 'i386'],
'development' : ['ppc','x86_64','i386'],
}
# Note: *no* depsolving is done for these.
copydict = { 'wine.i386': ['x86_64'],
'wine-arts.i386': ['x86_64'],
'wine-capi.i386': ['x86_64'],
'wine-cms.i386': ['x86_64'],
'wine-debuginfo.i386': ['x86_64'],
'wine-devel.i386': ['x86_64'],
'wine-esd.i386': ['x86_64'],
'wine-jack.i386': ['x86_64'],
'wine-ldap.i386': ['x86_64'],
'wine-nas.i386': ['x86_64'],
'wine-tools.i386': ['x86_64'],
'wine-twain.i386': ['x86_64'],
# lcms for wine-cms in FE-3 and FE-4
'lcms.i386': ['x86_64'],
'lcms-debuginfo.i386': ['x86_64'],
}
ts = rpmUtils.transaction.initReadOnlyTransaction()
def debugprint(msg):
if DEBUG:
print msg
def run_and_check(cmd):
debugprint(cmd)
if DEBUG:
return
result = os.system(cmd)
if result != 0:
print >> sys.stderr, 'Error running command: %s ' % cmd
sys.exit(result)
class LockFile:
def __init__(self, name):
self.name = name
def lock(self):
if DEBUG:
return
self.file = open(self.name,'w')
try:
self.rc = fcntl.flock(self.file, fcntl.LOCK_EX | fcntl.LOCK_NB)
except IOError:
print 'ERROR: script locked via lockfile %s - it seems to be running already' % name
sys.exit(11)
def unlock(self):
if DEBUG:
return
fcntl.flock(self.file, fcntl.LOCK_UN)
self.file.close()
# ====================================================================
# (previously in extras-sign-move.py)
# get the path to where to look for the packages to be signed
# get the list of stuff there (logs, md5sums, specs, rpms)
# list the rpms and pause
# sign the rpms
# collect list of srpms and get package n-v-r from them
# move pkgs into repo
def find_files(path):
"""returns a dict of filetypes and paths to those files"""
filedict = {}
filedict['srpm'] = []
filedict['log'] = []
filedict['rpm'] = []
filedict['spec'] = []
filedict['md5sum'] = []
filedict['debuginfo'] = []
filedict['other'] = []
for root, dirs, files in os.walk(path):
for file in files:
# match the files to what list they should be in
if file.endswith('.log'):
which = 'log'
elif file.endswith('.rpm'):
if file.find('debuginfo') != -1:
which = 'debuginfo'
elif file.endswith('.src.rpm'):
which = 'srpm'
else:
which = 'rpm'
elif file.endswith('.spec'):
which = 'spec'
elif file.endswith('.md5sum'):
which = 'md5sum'
else:
which = 'other'
fullfile = os.path.join(root, file)
filedict[which].append(fullfile)
return filedict
def naevr(pkg):
"""return nevra from the package srpm"""
hdr = rpmUtils.miscutils.hdrFromPackage(ts, pkg)
name = hdr['name']
ver = hdr['version']
rel = hdr['release']
arch = hdr['arch']
epoch = hdr['epoch']
if epoch is None:
epoch = 0
return (name, arch, epoch, ver, rel)
def sign_pkgs(filelist):
"""gpg sign all the rpms"""
numfiles = len(filelist)
if numfiles < 1:
print "No packages to sign"
return False
while numfiles > 0:
if numfiles > 256:
files = filelist[:256]
del filelist[:256]
else:
files = filelist
filelist = []
foo = string.join(files)
result = os.system('echo %s | xargs rpm --resign' % foo)
if result != 0:
return False
numfiles = len(filelist)
return True
def sign_move(dist):
runfilename = os.path.join(rundir, dist)
if DEBUG:
runfilename += '.DEBUG'
try:
rundirfile = open(runfilename,'a')
except:
print 'ERROR: Could not write to %s' % runfilename
return 11
distdir = '%s-%s-%s' % (distro, dist, project)
needsign = os.path.join(stagesdir, distdir)
files = find_files(needsign)
rpms = files['rpm'] + files['srpm'] + files['debuginfo']
rpms.sort()
for item in rpms:
print ' %s' % os.path.basename(item)
destdir = os.path.join(treedir, dist)
if not os.path.exists(destdir):
for arch in archdict[dist]:
if not DEBUG:
os.makedirs(destdir + '/' + arch + '/debug')
if not DEBUG:
os.makedirs(destdir + '/' + 'SRPMS')
rpms = files['rpm'] + files['srpm'] + files['debuginfo']
if len(rpms) == 0:
print 'nothing to sign'
return(4)
print "Signing Packages:"
if not DEBUG:
result = sign_pkgs(rpms)
if not result:
print "Error signing packages"
return(2)
print "Moving Packages into Place"
infolist = []
for package in files['srpm']:
(n,a,e,v,r) = naevr(package)
pkg_fn = os.path.basename(package)
srpmloc = os.path.join(destdir, 'SRPMS', pkg_fn)
debugprint('Moving %s to %s' % (package, srpmloc))
if os.path.exists(srpmloc):
debugprint('%s already exists, ignoring new one' % srpmloc)
if not DEBUG:
os.unlink(package)
else:
infolist.append('%s-%s-%s' % (n,v,r))
if not DEBUG:
shutil.move(package, srpmloc)
# go through each package and move it to the right arch location.
# if it is a noarch package, copy2 it to all arch locations and unlink it
# if it is listed in copydict, copy2 it around as appropriate
# if it is a debuginfo package, move it into the 'debug' dir for that arch
for package in files['rpm'] + files['debuginfo']:
pkg_fn = os.path.basename(package)
(n,a,e,v,r) = naevr(package)
copyarchs = copydict.get('%s.%s' % (n, a))
if copyarchs:
for arch in copyarchs:
if not arch in archdict[dist]:
continue
if package in files['debuginfo']:
arch = '%s/debug' % arch
rpmloc = os.path.join(destdir, arch, pkg_fn)
debugprint("Copying %s to %s" % (package, rpmloc))
if os.path.exists(rpmloc):
debugprint('%s already exists, ignoring new one' % rpmloc)
elif not DEBUG:
shutil.copy2(package, rpmloc)
if a == 'noarch':
for arch in archdict[dist]:
if package in files['debuginfo']:
arch = '%s/debug' % arch
rpmloc = os.path.join(destdir, arch, pkg_fn)
debugprint("Copying %s to %s" % (package, rpmloc))
if os.path.exists(rpmloc):
debugprint('%s already exists, ignoring new one' % rpmloc)
elif not DEBUG:
shutil.copy2(package, rpmloc)
if not DEBUG:
os.unlink(package)
continue
elif a in ['i386', 'i486', 'i586', 'i686', 'athlon']:
if package in files['debuginfo']:
arch = 'i386/debug'
else:
arch = 'i386'
elif a in ['x86_64', 'ia32e', 'amd64']:
if package in files['debuginfo']:
arch = 'x86_64/debug'
else:
arch = 'x86_64'
elif a in ['ppc', 'ppc64', 'ppc32']:
if package in files['debuginfo']:
arch = 'ppc/debug'
else:
arch = 'ppc'
else:
print 'Unknown arch %s' % a
continue
rpmloc = os.path.join(destdir, arch, pkg_fn)
debugprint('Moving %s to %s' % (package, rpmloc))
if os.path.exists(rpmloc):
debugprint('%s already exists, ignoring new one' % rpmloc)
if not DEBUG:
os.unlink(package)
elif not DEBUG:
shutil.move(package, rpmloc)
# Store list of move packages, so it can be mailed later.
for add in infolist:
add += '\n'
rundirfile.write( add )
rundirfile.close()
# clean up the crap
print "Cleaning Up"
for file in files['log'] + files['md5sum'] + files['spec']:
debugprint('removing %s' % file)
if not DEBUG:
os.unlink(file)
# FIXME clean up empty dirs, too.
return(0)
# ====================================================================
import smtplib
from email.MIMEText import MIMEText
def email_list(pkglist, dist):
"""email mailing list with the new package listing"""
uniqued = rpmUtils.miscutils.unique(pkglist)
uniqued.sort()
output = "\nPackages built and released for %s %s: %s \n\n" % (project_hr, dist, len(uniqued))
for pkg in uniqued:
add = '%s\n' % (pkg)
output = output + add
output = output + mail_footer
msg = MIMEText(output)
subject = '%s %s Package Build Report' % (project_hr, dist)
msg['Subject'] = subject
msg['From'] = mail_from
msg['To'] = mail_to
debugprint( msg.as_string() )
if DEBUG:
return
s = smtplib.SMTP()
s.connect()
s.sendmail(mail_from, [mail_to], msg.as_string())
s.close()
# ====================================================================
def getlinesfromrunfile(dist):
runfilename = os.path.join(rundir, dist)
if DEBUG:
runfilename += '.DEBUG'
pkglist = []
if os.path.isfile(runfilename):
try:
rundirfile = open(runfilename,'r')
for line in rundirfile:
pkglist.append( line.rstrip() )
rundirfile.close()
except IOError:
print 'ERROR: Could not open %s' % runfilename
sys.exit(5)
return pkglist
def emptyrunfile(dist):
runfilename = os.path.join(rundir, dist)
if DEBUG:
runfilename += '.DEBUG'
if not os.path.isfile(runfilename):
return
try:
os.remove(runfilename)
except:
print 'ERROR: Could not remove %s' % runfilename
pass
if __name__ == '__main__':
if config['gidcheck']:
if os.getgid() != signersgid:
print 'ERROR: Change into extras_signers group!'
sys.exit(13)
if config['setumask']:
os.umask(0002)
if len(sys.argv) < 2:
print 'SYNTAX: %s <dist> [dist]...' % sys.argv[0]
sys.exit(22)
dists = [] # the list of distribution release we want to push
distdict = {}
for d in sys.argv[1:]:
if d == 'all':
dists = alldists
continue
if d not in alldists:
print "ERROR: No Distribution named '%s' found" % d
sys.exit(1)
if not distdict.has_key(d):
dists.append(d)
distdict[d] = True
debugprint(dists)
# If we are called as 'extras-sign-move.py' we simulate old
# behaviour and only do: sign, move, repobuild, sync
if sys.argv[0].endswith('extras-sign-move.py'):
config['repoview'] = False
if not os.path.exists(rundir):
os.makedirs(rundir)
lockfile = os.path.join(rundir,'extras-push.lock')
lock = LockFile(lockfile)
lock.lock()
for dist in dists:
result = sign_move(dist)
if result and result != 4: # 4 = no packages to sign
sys.exit(result)
for dist in dists:
# Process all dists for which we find a run-file which contains >= 1 pkg names.
pkglist = getlinesfromrunfile(dist)
if len(pkglist):
run_and_check('extras-repobuild.py %s' % dist)
if config['repoview']:
run_and_check('extras-repoview.py %s' % dist)
if config['repoclosure']:
pass
run_and_check('extras-sync')
if config['mail']:
for dist in dists:
email_list( getlinesfromrunfile(dist), dist )
emptyrunfile(dist)
lock.unlock()
sys.exit(0)
- Previous message (by thread): rpms/syslog-ng/FC-5 .cvsignore, 1.5, 1.6 sources, 1.5, 1.6 syslog-ng.spec, 1.12, 1.13
- Next message (by thread): rpms/syslog-ng/FC-3 .cvsignore, 1.5, 1.6 sources, 1.5, 1.6 syslog-ng.spec, 1.8, 1.9
- Messages sorted by:
[ date ]
[ thread ]
[ subject ]
[ author ]
More information about the fedora-extras-commits
mailing list