[Date Prev][Date Next] [Thread Prev][Thread Next]
[Thread Index]
[Date Index]
[Author Index]
fedora-updates-system/updatessystem Comet.py, NONE, 1.1 metadata.py, NONE, 1.1 buildsys.py, 1.3, 1.4 controllers.py, 1.7, 1.8 json.py, 1.2, 1.3 mail.py, 1.1, 1.2 model.py, 1.5, 1.6 new.py, 1.5, 1.6 push.py, 1.2, 1.3 release.py, 1.2, 1.3 rpc.py, 1.1, 1.2 util.py, 1.1, 1.2
- From: Luke Macken (lmacken) <fedora-extras-commits redhat com>
- To: fedora-extras-commits redhat com
- Subject: fedora-updates-system/updatessystem Comet.py, NONE, 1.1 metadata.py, NONE, 1.1 buildsys.py, 1.3, 1.4 controllers.py, 1.7, 1.8 json.py, 1.2, 1.3 mail.py, 1.1, 1.2 model.py, 1.5, 1.6 new.py, 1.5, 1.6 push.py, 1.2, 1.3 release.py, 1.2, 1.3 rpc.py, 1.1, 1.2 util.py, 1.1, 1.2
- Date: Sun, 31 Dec 2006 04:10:44 -0500
Author: lmacken
Update of /cvs/fedora/fedora-updates-system/updatessystem
In directory cvs-int.fedora.redhat.com:/tmp/cvs-serv8229/updatessystem
Modified Files:
buildsys.py controllers.py json.py mail.py model.py new.py
push.py release.py rpc.py util.py
Added Files:
Comet.py metadata.py
Log Message:
- Add GPL headers
- README updates
- Disable transactions in our development environment
- Write mail sending module using TurboMail and some other hackery
- Initialize the updates-stage in our model.py
- Write push code and metadata generation
- Started writing the xmlrpc server code
- Create rpm_fileheader and sha1sum functions in util.py
- Elaborate configuration
- Change logging format
- Start writing a few more testcases
- Wrote comet-based (async httpxmlrequest) Push Console
- Add RPM spec file
***** Error reading new file: [Errno 2] No such file or directory: 'Comet.py'
***** Error reading new file: [Errno 2] No such file or directory: 'metadata.py'
Index: buildsys.py
===================================================================
RCS file: /cvs/fedora/fedora-updates-system/updatessystem/buildsys.py,v
retrieving revision 1.3
retrieving revision 1.4
diff -u -r1.3 -r1.4
--- buildsys.py 7 Dec 2006 07:29:47 -0000 1.3
+++ buildsys.py 31 Dec 2006 09:10:14 -0000 1.4
@@ -1,4 +1,16 @@
# $Id$
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; version 2 of the License.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Library General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
import os
import util
@@ -32,13 +44,19 @@
"""
def get_source_path(self, update):
- return os.path.join(config.get('build_dir'), *util.get_nvr(update.nvr))
+ """
+ Return the path to the built package. For the LocalTest repository
+ we'll return the following:
+ build_dir/package/version/release
+ """
+ build_dir = config.get('build_dir')
+ assert build_dir
+ return os.path.join(build_dir, *util.get_nvr(update.nvr))
def get_srpm_path(self, update):
srpm = os.path.join(self.get_source_path(update), "src",
"%s.src.rpm" % update.nvr)
if not os.path.isfile(srpm):
- print "Cannot find", srpm
raise SRPMNotFound
return srpm
Index: controllers.py
===================================================================
RCS file: /cvs/fedora/fedora-updates-system/updatessystem/controllers.py,v
retrieving revision 1.7
retrieving revision 1.8
diff -u -r1.7 -r1.8
--- controllers.py 7 Dec 2006 07:29:47 -0000 1.7
+++ controllers.py 31 Dec 2006 09:10:14 -0000 1.8
@@ -1,4 +1,16 @@
# $Id$
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; version 2 of the License.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Library General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
import mail
import util
@@ -7,12 +19,12 @@
from new import NewUpdateController, update_form
from push import PushController
+from admin import AdminController
from model import Package, PackageUpdate, Release, Bugzilla, CVE
from sqlite import IntegrityError
from buildsys import SRPMNotFound
from sqlobject import SQLObjectNotFound
-from elementtree import ElementTree as ET
-from updatessystem import json
+#from updatessystem import json
from turbogears import (controllers, expose, validate, redirect, identity,
paginate, flash, error_handler)
@@ -31,8 +43,9 @@
@expose(template="updatessystem.templates.login")
def login(self, forward_url=None, previous_url=None, *args, **kw):
"""
- TODO: fedora account system integration
+ TODO: fedora account system integration
"""
+
if not identity.current.anonymous \
and identity.was_login_attempted() \
and not identity.get_identity_errors():
@@ -83,13 +96,19 @@
flash("Update %s not found" % update)
raise redirect("/list")
if up.pushed:
- # TODO: Removing pushed updates should essentially unpush?
- # - remove from update stage
- # - regenerate metadata (remove extended metadata)
- # - remove from database
- # - sync to mirrors
- flash("Cannot delete %s; it has already been pushed" % update)
+ # Removing pushed updates should essentially unpush
+ # - remove from update stage (DONE)
+ # - regenerate metadata (DONE)
+ # - remove extended metadata TODO
+ # - remove from database (DONE)
+ # - sync to mirrors TODO
+ push.unpush_update(up)
+ push.generate_metadata(up)
+ up.destroySelf()
+ mail.send_admin('deleted', up)
+ flash("Deleted and unpushed %s" % update)
else:
+ mail.send_admin('deleted', up)
up.destroySelf()
flash("%s deleted" % update)
raise redirect("/list")
@@ -109,8 +128,8 @@
'type' : up.type,
'embargo' : up.embargo,
'notes' : up.notes,
- 'bugs' : string.join([str(bug.bz_id) for bug in up.bugs]),
- 'cves' : string.join([cve.cve_id for cve in up.cves]),
+ 'bugs' : up.get_bugstring(),
+ 'cves' : up.get_cvestring(),
'edited' : up.nvr
}
return dict(form=update_form, values=values, action='/save')
@@ -132,7 +151,6 @@
package = Package.byName(name)
p = PackageUpdate(package=package, release=release,
submitter='FIXME', **kw)
- p.build_filelist()
# TODO: we should eventually put all of this stuff into a
# FancyValidator for the Package field
@@ -142,7 +160,8 @@
except SQLObjectNotFound:
flash("Package %s not found" % name)
raise redirect('/new')
- except IntegrityError:
+ except IntegrityError, e:
+ log.debug(e)
flash("Update for %s already exists" % kw['nvr'])
raise redirect('/new')
except SRPMNotFound:
@@ -164,6 +183,9 @@
bz = Bugzilla.byBz_id(int(bug))
except SQLObjectNotFound:
bz = Bugzilla(bz_id=int(bug))
+ except ValueError:
+ flash("Invalid bug number")
+ raise redirect('/')
p.addBugzilla(bz)
for cve_id in cves.replace(',', ' ').split():
cve = None
@@ -173,5 +195,11 @@
cve = CVE(cve_id=cve_id)
p.addCVE(cve)
- flash("Update successfully %s" % (edited and 'edited' or 'added'))
+ if edited:
+ flash("Update successfully edited")
+ mail.send_admin('edited', p)
+ else:
+ flash("Update successfully added")
+ mail.send_admin('new', p)
+
raise redirect('/show/%s' % p.nvr)
Index: json.py
===================================================================
RCS file: /cvs/fedora/fedora-updates-system/updatessystem/json.py,v
retrieving revision 1.2
retrieving revision 1.3
diff -u -r1.2 -r1.3
--- json.py 7 Dec 2006 07:19:41 -0000 1.2
+++ json.py 31 Dec 2006 09:10:14 -0000 1.3
@@ -8,6 +8,19 @@
# return [obj.val1, obj.val2]
# The goal is to break your objects down into simple values:
# lists, dicts, numbers and strings
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; version 2 of the License.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Library General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
from turbojson.jsonify import jsonify
from turbojson.jsonify import jsonify_sqlobject
Index: mail.py
===================================================================
RCS file: /cvs/fedora/fedora-updates-system/updatessystem/mail.py,v
retrieving revision 1.1
retrieving revision 1.2
diff -u -r1.1 -r1.2
--- mail.py 3 Dec 2006 19:51:51 -0000 1.1
+++ mail.py 31 Dec 2006 09:10:14 -0000 1.2
@@ -1,44 +1,103 @@
# $Id$
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; version 2 of the License.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Library General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
-"""
- Module to handle sending email.
-
- TODO: Maybe use TurboMail instead? We need to get it into extras first
- http://cheeseshop.python.org/pypi/TurboMail
-"""
-
-import smtplib
+import re
+import turbomail
from turbogears import config
-release_team = config.get('release_team_addr')
+release_team = config.get('release_team_address')
+from_addr = config.get('from_address')
-new_update_mail = """\
-From: %(from)s
-To: %(to)s
-Subject: [Fedora Update] %(sec)s[push] %(package)s
-
-%(from)s has requested the pushing of %(package)s as a
-%(testing)s update for %(release)s.
-
-Execute the following command to sign this package:
-
-%(signcmd)s
-
-To push this update, please go to the following URL:
-http://porkchop.devel.redhat.com/fedora-updates/push.py
-"""
-
-def send_mail(to, msg):
- conn = smtplib.SMTP(config.get('mail_server', 'localhost'))
- conn.sendmail(config.get('from_addr'), [to], msg)
- conn.quit()
-
-def new_update(update):
- send_mail(release_team, new_update_mail % {
- 'from' : update.submitter,
- 'to' : release_team,
- 'sec' : (update.security) and '[SECURITY]' or '',
- 'package' : update.nvr,
- 'testing' : (update.testing) and 'Testing' or 'Final',
- 'signcmd' : 'TODO'
- })
+# this should probably go into model.PackageUpdate.__str__
+def update_str(update):
+ return """\
+ Package: %(package)s
+ Type: %(type)s
+ Bugs: %(bugs)s
+ CVES: %(cves)s
+ Notes: %(notes)s
+ """ % ({
+ 'package' : update.nvr,
+ 'type' : update.type,
+ 'notes' : update.notes,
+ 'email' : update.submitter,
+ 'bugs' : update.get_bugstring(),
+ 'cves' : update.get_cvestring()
+ })
+
+messages = {
+ 'new' : {
+ 'subject' : '[Fedora Update] [new] %(package)s',
+ 'body' : """\
+%(email)s has submitted a new update for %(release)s
+
+%(updatestr)s
+""",
+ 'fields' : lambda x: {
+ 'email' : x.submitter,
+ 'release' : x.release.long_name,
+ 'updatestr' : update_str(x)
+ }
+ },
+
+ 'deleted' : {
+ 'subject' : '[Fedora Update] [deleted] %(package)s',
+ 'body' : """\
+%(email)s has deleted the %(package)s update for %(release}s
+""",
+ 'fields' : lambda x: {
+ 'email' : x.submitter,
+ 'package' : x.nvr,
+ 'release' : x.release.long_name
+ }
+ },
+
+ 'edited' : {
+ 'subject' : '[Fedora Update] [edited] %(package)s',
+ 'body' : """\
+%(email)s has edited the %(package)s update for %(release)s
+
+%(updatestr)s
+""",
+ 'fields' : lambda x: {
+ 'package' : x.nvr,
+ 'email' : x.submitter,
+ 'release' : x.release.long_name,
+ 'updatestr' : update_str(x)
+ }
+ },
+
+ 'pushed' : {
+ 'subject' : '[Fedora Update] [pushed] %(package)s',
+ 'body' : """\
+%(package)s has been successfully pushed for %(release)s.
+""",
+ 'fields' : lambda x: {
+ 'package' : x.nvr,
+ 'release' : x.release.long_name
+ }
+ }
+}
+
+def send(to, msg_type, update):
+ """ Send an update notification email to a given recipient """
+ message = turbomail.Message(from_addr, to, messages[msg_type]['subject'] %
+ {'package': update.nvr})
+ message.plain = messages[msg_type]['body'] % \
+ messages[msg_type]['fields'](update)
+ turbomail.enqueue(message)
+
+def send_admin(msg_type, update):
+ """ Send an update notification to the admins/release team. """
+ send(release_team, msg_type, update)
Index: model.py
===================================================================
RCS file: /cvs/fedora/fedora-updates-system/updatessystem/model.py,v
retrieving revision 1.5
retrieving revision 1.6
diff -u -r1.5 -r1.6
--- model.py 7 Dec 2006 07:19:41 -0000 1.5
+++ model.py 31 Dec 2006 09:10:14 -0000 1.6
@@ -1,12 +1,27 @@
-#!/usr/bin/env python
+#!/usr/bin/python -tt
# $Id$
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; version 2 of the License.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Library General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
+
+import logging
from sqlobject import *
from datetime import datetime
-from turbogears import identity
+from turbogears import identity, config
from turbogears.database import PackageHub
+log = logging.getLogger(__name__)
hub = PackageHub("updatessystem")
__connection__ = hub
@@ -43,7 +58,7 @@
class PackageUpdate(SQLObject):
""" This class defines an update in our system. """
- nvr = UnicodeCol(notNone=True, alternateID=True)
+ nvr = UnicodeCol(notNone=True, alternateID=True, unique=True)
date_submitted = DateTimeCol(default=datetime.now, notNone=True)
date_modified = DateTimeCol(default=None)
package = ForeignKey('Package')
@@ -64,16 +79,37 @@
archived_mail = UnicodeCol(default=None)
needs_push = BoolCol(default=False)
comments = MultipleJoin('Comment', joinColumn='update_id')
+
+ ## TODO: create File table ?
filelist = PickleCol(default={}) # { 'arch' : [file1, file2, ..] }
- def build_filelist(self):
+ def _set_nvr(self, nvr):
+ """ Called when the a PackageUpdate is created. Here we do some
+ initialization.
+ """
+ from buildsys import SRPMNotFound
+ self._SO_set_nvr(nvr)
+ try:
+ self._build_filelist()
+ except SRPMNotFound:
+ log.error("Unable to build filelist for %s" % nvr)
+
+ def get_bugstring(self):
+ import string
+ return string.join([str(bug.bz_id) for bug in self.bugs])
+
+ def get_cvestring(self):
+ import string
+ return string.join([cve.cve_id for cve in self.cves])
+
+ def _build_filelist(self):
""" Build and store the filelist for this update. """
import os, util
from os.path import isdir, join, basename
from buildsys import buildsys
- print "Building filelist"
+ log.debug("Building filelist for %s" % self.nvr)
filelist = {}
- filelist['src'] = buildsys.get_srpm_path(self)
+ filelist['SRPMS'] = [buildsys.get_srpm_path(self)]
sourcepath = buildsys.get_source_path(self)
for arch in self.release.arches:
filelist[arch.name] = []
@@ -94,13 +130,12 @@
if arch in multilib.arches and \
self.release in multilib.releases:
filelist[arch.name].append(join(path, file))
- print file
+ log.debug(" * %s" % file)
except SQLObjectNotFound:
continue
except IndexError:
- print "Unknown file:", file
+ log.debug("Unknown file: %s" % file)
continue
- print "\n", filelist
self.filelist = filelist
class Comment(SQLObject):
@@ -116,14 +151,34 @@
class Bugzilla(SQLObject):
""" Table of Bugzillas that we know about. """
- bz_id = IntCol(alternateID=True)
- title = UnicodeCol(default=None)
- updates = RelatedJoin("PackageUpdate")
-
- # TODO: automatically fetch bugzilla title upon creation
+ bz_id = IntCol(alternateID=True)
+ title = UnicodeCol(default=None)
+ updates = RelatedJoin("PackageUpdate")
+ severity = UnicodeCol(default=None)
+
+ _bz_server = config.get("bz_server")
+ _default_closemsg = config.get("bz_default_closemsg")
+ #_bz_server = 'https://bugzilla.redhat.com/bugzilla/xmlrpc.cgi'
+ #_default_closemsg = "%(package)s has been released for %(release)s. If problems still persist, please make note of it in this bug report."
+
+ def _set_bz_id(self, bz_id):
+ self._SO_set_bz_id(bz_id)
+ #self._fetch_details()
- _bz_server = 'https://bugzilla.redhat.com/bugzilla/xmlrpc.cgi'
- _default_closemsg = "%(package)s has been released for %(release)s. If problems still persist, please make note of it in this bug report."
+ def _fetch_details(self):
+ import xmlrpclib
+ try:
+ log.debug("Fetching bugzilla title for bug #%d" % self.bz_id)
+ log.debug("Using bugzilla server %s" % self._bz_server)
+ server = xmlrpclib.Server(self._bz_server)
+ #me = User.by_user_name('updatesys')
+ #bug = server.bugzilla.getBugSimple(self.bz_id, me.email_address,
+ # me.password)
+ self.severity = bug['bug_severity']
+ self.title = bug['short_desc']
+ except Exception, e:
+ log.error("Unable to fetch Bugzilla title")
+ log.debug("Exception: %s" % e)
def _check_security(self):
""" Check if the bugzilla is a security bug:
@@ -245,12 +300,14 @@
Release.dropTable(ifExists=True)
Package.dropTable(ifExists=True)
Arch.dropTable(ifExists=True)
+ Group.dropTable(ifExists=True)
Multilib.dropTable(ifExists=True)
hub.commit()
Release.createTable(ifNotExists=True)
Package.createTable(ifNotExists=True)
Arch.createTable(ifNotExists=True)
Multilib.createTable(ifNotExists=True)
+ Group.createTable(ifNotExists=True)
arches = {
# arch subarches
@@ -266,7 +323,7 @@
'ppc' : ['ppc64', 'ppc64iseries']
}
- print "\nInitializing Arch tables"
+ print "Initializing Arch tables..."
for arch in arches.keys():
a = Arch(name=arch, subarches=arches[arch], compatarches=biarches[arch])
print a
@@ -288,10 +345,10 @@
## Import old biarch package lists
from deprecated.biarch import biarch
- num_multilib = 0
- print "\nInitializing Release table..."
+ print "\nInitializing Release table and multilib packages..."
for release in releases:
+ num_multilib = 0
rel = Release(name=release['name'], long_name=release['long_name'],
codename=release['codename'])
map(rel.addArch, release['arches'])
@@ -310,7 +367,50 @@
## TODO: Add packages from pkgdb ?
print "\nAdding packages..."
- pkg = Package(name='foobar')
+ pkg = Package(name='gaim')
print pkg
+ ##
+ ## Initialize the updates-stage
+ ##
+ import os
+ import sys
+ from os.path import join, isdir
+ from turbogears import config
+ sys.path.append('/usr/share/createrepo')
+ import genpkgmetadata
+
+ stage_dir = config.get('stage_dir')
+ print "\nInitializing the staging directory"
+
+ def mkmetadatadir(dir):
+ print dir
+ os.mkdir(dir)
+ genpkgmetadata.main(['-q', str(dir)])
+
+ if isdir(stage_dir):
+ import shutil
+ olddir = stage_dir + '.old'
+ if isdir(olddir):
+ shutil.rmtree(olddir)
+ print "Moving existing stage_dir to stage_dir.old"
+ shutil.move(stage_dir, olddir)
+
+ os.mkdir(stage_dir)
+ os.mkdir(join(stage_dir, 'testing'))
+ for release in Release.select():
+ for status in ('', 'testing'):
+ dir = join(stage_dir, status, release.name[-1])
+ os.mkdir(dir)
+ mkmetadatadir(join(dir, 'SRPMS'))
+ for arch in release.arches:
+ mkmetadatadir(join(dir, arch.name))
+ mkmetadatadir(join(dir, arch.name, 'debug'))
+
+ ##
+ ## Create the admin group
+ ##
+ #admin = Group(display_name='Administrators', group_name='admin')
+ #print "\nCreating admin group\n", admin
+
hub.commit()
Index: new.py
===================================================================
RCS file: /cvs/fedora/fedora-updates-system/updatessystem/new.py,v
retrieving revision 1.5
retrieving revision 1.6
diff -u -r1.5 -r1.6
--- new.py 7 Dec 2006 07:19:41 -0000 1.5
+++ new.py 31 Dec 2006 09:10:14 -0000 1.6
@@ -1,4 +1,16 @@
# $Id$
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; version 2 of the License.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Library General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
import util
import logging
@@ -10,10 +22,11 @@
error_handler, redirect)
from turbogears.widgets import (WidgetsList, TextField, SingleSelectField,
- CheckBox, TextArea, CalendarDatePicker,
+ CheckBox, TextArea, CalendarDateTimePicker,
TableForm, HiddenField)
log = logging.getLogger(__name__)
+
update_types = ('security', 'bugfix', 'enhancement')
def get_releases():
@@ -25,8 +38,9 @@
validator=validators.OneOf(get_releases()))
testing = CheckBox(validator=validators.Bool)
type = SingleSelectField(options=update_types,
- validator=validators.OneOf(update_types))
- embargo = CalendarDatePicker(validator=validators.DateConverter)
+ validator=validators.OneOf(update_types),
+ attrs={'onChange': 'newType()'})
+ embargo = CalendarDateTimePicker(validator=validators.DateTimeConverter)
bugs = TextField(validator=validators.String)
cves = TextField(validator=validators.String)
notes = TextArea(validator=validators.String)
Index: push.py
===================================================================
RCS file: /cvs/fedora/fedora-updates-system/updatessystem/push.py,v
retrieving revision 1.2
retrieving revision 1.3
diff -u -r1.2 -r1.3
--- push.py 7 Dec 2006 07:19:41 -0000 1.2
+++ push.py 31 Dec 2006 09:10:14 -0000 1.3
@@ -1,44 +1,174 @@
# $Id$
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; version 2 of the License.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Library General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
import os
+import mail
import logging
+import cherrypy
+from Comet import comet
from model import PackageUpdate
from buildsys import buildsys
from turbogears import controllers, expose, flash, redirect, config
+from os.path import isfile, isdir, basename, join
+
+import sys
+sys.path.append('/usr/share/createrepo')
+import genpkgmetadata
+
log = logging.getLogger(__name__)
+class RepositoryLocked(Exception):
+ pass
+
class PushController(controllers.Controller):
+ def __init__(self):
+ self.createrepo_cache = config.get('createrepo_cache_dir')
+ self.stage_dir = config.get('stage_dir')
+ self.lockfile = join(self.stage_dir, '.lock')
+ if isfile(self.lockfile):
+ log.debug("Removing stale lockfile")
+ self._unlock_repo()
+
+ def _lock_repo(self):
+ if isfile(self.lockfile):
+ raise RepositoryLocked
+ lock = file(self.lockfile, 'w')
+ lock.close()
+
+ def _unlock_repo(self):
+ if isfile(self.lockfile):
+ os.unlink(self.lockfile)
+
@expose(template='updatessystem.templates.push')
def index(self):
updates = PackageUpdate.select(PackageUpdate.q.needs_push == True)
return dict(updates=updates)
- @expose()
- def submit(self, updates, **kw):
+ @expose(template='updatessystem.templates.pushconsole')
+ def console(self, updates, **kw):
if not updates:
flash("No updates selected for pushing")
- redirect("/push")
+ raise redirect("/push")
if not isinstance(updates, list):
updates = [updates]
- for nvr in updates:
- log.debug("Pushing %s" % nvr)
- update = PackageUpdate.byNvr(nvr)
- self.push_update(update)
+ #cherrypy.response.headers['Content-Type'] = 'text/plain'
+ log.debug("Setting updates in session: %s" % updates)
+ cherrypy.session['topush'] = updates
+ return dict()
+
+ @expose()
+ def push_updates(self):
+ """
+ This method is called by the pushconsole template.
+ It returns a generator that spits out the push results. We're using
+ comet here, so the results will be pushed out to the client
+ asynchronously.
+ """
+ @comet(content_type='text/plain')
+ def _do_push():
+ yield "Starting push\n"
+ try:
+ self._lock_repo()
+ yield "Acquired lock for repository"
+ except RepositoryLocked:
+ err = "Unable to acquire lock for repository"
+ log.debug(err)
+ yield err
+ return
+
+ line = '=' * 120
+ for package in cherrypy.session['topush']:
+ update = PackageUpdate.byNvr(package)
+ yield "%s\nPushing %s\n%s" % (line, update.nvr, line)
+ try:
+ for output in self.push_update(update):
+ log.debug(output)
+ yield output
+ for output in self.generate_metadata(update):
+ yield output
+ yield "Sending notification to %s\n" % update.submitter
+ except Exception, e:
+ log.error("Exception during push: %s" % e)
+ yield "ERROR: Exception thrown during push: %s" % e
+ # TODO: enable me when we have identities working
+ #mail.send(update.submitter, 'pushed', update)
+ update.needs_push = False
+ update.pushed = True
+ self._unlock_repo()
+ yield "Pushing Complete!\n" + line
+
+ return _do_push()
- flash("Pushing Complete!")
- redirect("/list")
def get_dest_path(self, update, arch):
""" Return the destination path for a given update """
- return os.path.join(config.get('stage_dir'),
- update.testing and 'testing' or '',
- update.release.name[-1], arch)
+ return join(self.stage_dir, update.testing and 'testing' or '',
+ update.release.name[-1], arch)
def push_update(self, update):
- # go through update.filelist, and push to buildsys.get_dest_path.
- # TODO: should we put this logic inside of the PackageUpdate model,
- # so each update knows how to push itself ?
- pass
+ """ Go through update.filelist, and push to buildsys.get_dest_path. """
+ log.debug("Pushing update %s" % update.nvr)
+ try:
+ for arch in update.filelist.keys():
+ dest = self.get_dest_path(update, arch)
+ for file in update.filelist[arch]:
+ filename = basename(file)
+ if filename.find('debuginfo') != -1:
+ destfile = join(dest, 'debug', filename)
+ else:
+ destfile = join(dest, filename)
+ if isfile(destfile):
+ yield "Removing already pushed file: %s" % filename
+ os.unlink(destfile)
+ yield "Pushing %s" % (filename)
+ os.link(file, destfile)
+ except Exception, e:
+ yield "Caught the following exception during push: %s" % e
+ for msg in self.unpush_update(update):
+ yield msg
+
+ def unpush_update(self, update):
+ """
+ Remove all files for a given update that may or may not exist in the
+ updates stage.
+ """
+ yield "Unpushing %s" % update.nvr
+ for arch in update.filelist.keys():
+ deset = self.get_dest_path(update, arch)
+ for file in update.filelist[arch]:
+ destfile = join(dest, basename(file))
+ if isfile(destfile):
+ yield "Deleting %s" % destfile
+ os.unlink(destfile)
+
+ def generate_metadata(self, update):
+ """ Generate the repomd for all repos that this update effects """
+ for arch in update.filelist.keys():
+ repo = self.get_dest_path(update, arch)
+ cache_dir = join(self.createrepo_cache, 'fc%s-%s-%s' %
+ (update.release.name[-1], arch, update.testing and
+ 'testing' or 'final'))
+ yield "Generating metadata for %s" % repo
+ genpkgmetadata.main(['--cachedir', str(cache_dir), '-p', '-q',
+ str(repo)])
+ debugrepo = join(repo, 'debug')
+ if isdir(debugrepo):
+ genpkgmetadata.main(['--cachedir', str(cache_dir), '-p', '-q',
+ str(debugrepo)])
+
+## Allow us to return a generator for streamed responses
+cherrypy.config.update({'/push/push_updates':{'stream_response':True}})
Index: release.py
===================================================================
RCS file: /cvs/fedora/fedora-updates-system/updatessystem/release.py,v
retrieving revision 1.2
retrieving revision 1.3
diff -u -r1.2 -r1.3
--- release.py 3 Dec 2006 19:51:51 -0000 1.2
+++ release.py 31 Dec 2006 09:10:14 -0000 1.3
@@ -1,5 +1,18 @@
# $Id$
# Release information about Fedora Updates System
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; version 2 of the License.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Library General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
version = "1.0"
@@ -7,10 +20,6 @@
# long_description = "More description about your plan"
authors = "Luke Macken"
email = "lmacken fedoraproject org"
-
-# copyright = "Vintage 2006 - a good year indeed"
-
-# if it's open source, you might want to specify these
-# url = "http://yourcool.site/";
-# download_url = "http://yourcool.site/download";
-# license = "MIT"
+copyright = "Red Hat, Inc. 2006"
+url = "http://fedoraproject.org/wiki/Infrastructure/UpdatesSystem";
+license = "GPL"
Index: rpc.py
===================================================================
RCS file: /cvs/fedora/fedora-updates-system/updatessystem/rpc.py,v
retrieving revision 1.1
retrieving revision 1.2
diff -u -r1.1 -r1.2
--- rpc.py 3 Dec 2006 19:51:51 -0000 1.1
+++ rpc.py 31 Dec 2006 09:10:14 -0000 1.2
@@ -1,32 +1,68 @@
-#!/usr/bin/env python
+#!/usr/bin/python -tt
# $Id$
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; version 2 of the License.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Library General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
+import logging
import turbogears
import SocketServer
import SimpleXMLRPCServer
from model import PackageUpdate, Package
+from threading import Thread
+from turbogears import config
from turbogears.database import PackageHub
hub = PackageHub("updatessystem")
__connection__ = hub
+log = logging.getLogger(__name__)
+
+global rpc_instance
+rpc_instance = None
+
class SimpleThreadedXMLRPCServer(SocketServer.ThreadingMixIn,
SimpleXMLRPCServer.SimpleXMLRPCServer):
pass
-class UpdatesRPC:
+class UpdatesRPC(Thread):
- def __init__(self):
+ def __init__(self, root):
+ Thread.__init__(self)
+ print "UpdatesRPC.__init__"
turbogears.update_config(configfile='dev.cfg',
modulename='updatessystem.config')
+ self.root = root
+ self.rpc_port = config.get('rpc_port')
+
+ @staticmethod
+ def get_instance(root):
+ """ Static method to return the singleton for our XML-RPC server """
+ print "get_instance"
+ global rpc_instance
+ if not rpc_instance:
+ rpc_instance = UpdatesRPC(root)
+ rpc_instance.start()
+ return rpc_instance
+
+ def run(self):
+ print "Starting XML-RPC server on port %d" % self.rpc_port
+ #log.debug("Starting XML-RPC server on port %d" % self.rpc_port)
+ server = SimpleThreadedXMLRPCServer(("localhost", self.rpc_port))
+ server.register_instance(UpdatesRPC())
+ server.serve_forever()
+ log.debug("UpdatesRPC thread shutting down")
def submit(self, pkg, rel, bugs, cves, notes, security=False,
embargo=None, testing=False):
return "DONE"
-
-if __name__ == "__main__":
- print "Starting XML-RPC server"
- server = SimpleThreadedXMLRPCServer(("localhost", 8001))
- server.register_instance(UpdatesRPC())
- server.serve_forever()
Index: util.py
===================================================================
RCS file: /cvs/fedora/fedora-updates-system/updatessystem/util.py,v
retrieving revision 1.1
retrieving revision 1.2
diff -u -r1.1 -r1.2
--- util.py 22 Nov 2006 07:24:54 -0000 1.1
+++ util.py 31 Dec 2006 09:10:14 -0000 1.2
@@ -1,8 +1,42 @@
# $Id$
+# Random functions that don't fit elsewhere.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; version 2 of the License.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Library General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
-"""
- Random functions that don't fit elsewhere.
-"""
+def rpm_fileheader(pkgpath):
+ import os
+ import rpm
+ is_oldrpm = hasattr(rpm, 'opendb')
+ fd = os.open(pkgpath,0)
+ try:
+ if is_oldrpm:
+ h = rpm.headerFromPackage(fd)[0]
+ else:
+ ts = rpm.TransactionSet()
+ #ts.setVSFlags(~rpm._RPMVSF_NOSIGNATURES)
+ h = ts.hdrFromFdno(fd)
+ del ts
+ finally:
+ os.close(fd)
+ return h
+
+def sha1sum(file):
+ import sha
+ fd = open(file)
+ hash = sha.new(fd.read())
+ fd.close()
+ return hash.hexdigest()
def get_nvr(nvr):
""" Return the [ name, version, release ] a given name-ver-rel. """
[Date Prev][Date Next] [Thread Prev][Thread Next]
[Thread Index]
[Date Index]
[Author Index]