[PATCH koji] Added support for building from git repositories
Enrico Scholz
enrico.scholz at informatik.tu-chemnitz.de
Sat Sep 15 19:31:58 UTC 2007
This patch adds support for building from git:// respositories. It is used
like
koji call build 'git://<REPOSITORY>?<PACKAGE>#<TAG>' <TARGET>
The repository specified above must have the following layout:
<BASE_DIR>/<REPOSITORY>
|- common/
| `- .git/
|- ...
|- <PACKAGE>
| `- .git/
|- ...
This means that every package is hold in an own git repository. There must
be a 'common' repository which contains files required for building the
srpm.
There is no explicit support for branches; the patch just checks out the
given tag and builds from it. Perhaps it can be enhanced to check whether
branch-point is some child of the tag...
TODO: the git and cvs handlers share some common, non trivial code. This
should be generalized.
TODO: use a separate user account for the 'git' and 'make srpm' operations;
currently they are executed under the uid 'kojid' is running (which is
'root). Hence, a '%(/sbin/killall5)' in the spec file can bring down
the box.
Signed-off-by: Enrico Scholz <enrico.scholz at informatik.tu-chemnitz.de>
---
builder/kojid | 130 +++++++++++++++++++++++++++++++++++++++++++-
cli/koji | 9 ++-
koji.spec | 2 -
koji/__init__.py | 11 ++++
www/kojiweb/taskinfo.chtml | 7 ++
www/kojiweb/tasks.chtml | 1
6 files changed, 153 insertions(+), 7 deletions(-)
diff --git a/builder/kojid b/builder/kojid
index 5940954..bbb9774 100755
--- a/builder/kojid
+++ b/builder/kojid
@@ -1396,7 +1396,7 @@ class ChainBuildTask(BaseTaskHandler):
subtasks = []
build_tasks = []
for src in build_level:
- if src.startswith('cvs://'):
+ if src.startswith('cvs://') or src.startswith('git://'):
task_id = session.host.subtask(method='build',
arglist=[src, target, opts],
parent=self.id)
@@ -1437,8 +1437,10 @@ class BuildTask(BaseTaskHandler):
raise koji.BuildError, "arch_override is only allowed for scratch builds"
task_info = session.getTaskInfo(self.id)
# only allow admins to perform non-scratch builds from srpm
- if not src.startswith('cvs://') and not opts.get('scratch') \
- and not 'admin' in session.getUserPerms(task_info['owner']):
+ if not src.startswith('cvs://') and \
+ not src.startswith('git://') and \
+ not opts.get('scratch') and \
+ not 'admin' in session.getUserPerms(task_info['owner']):
raise koji.BuildError, "only admins may peform non-scratch builds from srpm"
target_info = session.getBuildTarget(target)
if not target_info:
@@ -1496,6 +1498,8 @@ class BuildTask(BaseTaskHandler):
if isinstance(src,str):
if src.startswith('cvs://'):
return self.getSRPMFromCVS(src)
+ if src.startswith('git://'):
+ return self.getSRPMFromGit(src)
else:
#assume this is a path under uploads
return src
@@ -1514,6 +1518,17 @@ class BuildTask(BaseTaskHandler):
srpm = result['srpm']
return srpm
+ def getSRPMFromGit(self, url):
+ #TODO - allow different ways to get the srpm
+ task_id = session.host.subtask(method='buildSRPMFromGit',
+ arglist=[url],
+ label='srpm',
+ parent=self.id)
+ # wait for subtask to finish
+ result = self.wait(task_id)[task_id]
+ srpm = result['srpm']
+ return srpm
+
def readSRPMHeader(self, srpm):
#srpm arg should be a path relative to <BASEDIR>/work
global options
@@ -1767,6 +1782,115 @@ class TagBuildTask(BaseTaskHandler):
exctype, value = sys.exc_info()[:2]
session.host.tagNotification(False, tag_id, fromtag, build_id, user_id, ignore_success, "%s: %s" % (exctype, value))
raise e
+
+
+class BuildSRPMFromGitTask(BaseTaskHandler):
+
+ Methods = ['buildSRPMFromGit']
+ _taskWeight = 0.75
+
+ def spec_sanity_checks(self, filename):
+ spec = open(filename).read()
+ for tag in ("Packager", "Distribution", "Vendor"):
+ if re.match("%s:" % tag, spec, re.M):
+ raise koji.BuildError, "%s is not allowed to be set in spec file" % tag
+ for tag in ("packager", "distribution", "vendor"):
+ if re.match("%%define\s+%s\s+" % tag, spec, re.M):
+ raise koji.BuildError, "%s is not allowed to be defined in spec file" % tag
+
+ def handler(self,url):
+ if not url.startswith('git://'):
+ raise koji.BuildError("invalid git URL: %s" % url)
+
+ # Hack it because it refuses to parse it properly otherwise
+ scheme, netloc, path, params, query, fragment = urlparse.urlparse('http'+url[3:])
+ if not (netloc and path and fragment and query):
+ raise koji.BuildError("invalid git URL: %s" % url)
+
+ # Steps:
+ # 1. GIT clone into tempdir
+ # 3. Run 'make srpm'
+
+ gitdir = self.workdir + '/git'
+ koji.ensuredir(gitdir)
+ logfile = self.workdir + "/srpm.log"
+ uploadpath = self.getUploadDir()
+ sourcedir = '%s/%s' % (gitdir, query)
+
+ cmd = ['git', 'clone', 'git://%s%s/%s' % (netloc, path, query), query]
+ if log_output(cmd[0], cmd, logfile, uploadpath, cwd=gitdir, logerror=1, append=1):
+ output = "(none)"
+ try:
+ output = open(logfile).read()
+ except IOError:
+ pass
+ raise koji.BuildError, "Error with clone 'git://%s%s/%s: %s" % (netloc, path, query, output)
+
+ cmd = ['git', 'clone', 'git://%s%s/common' % (netloc, path)]
+ self.logger.debug("executing: %s" % cmd)
+ if log_output(cmd[0], cmd, logfile, uploadpath, cwd=gitdir, logerror=1, append=1):
+ output = "(none)"
+ try:
+ output = open(logfile).read()
+ except IOError:
+ pass
+ raise koji.BuildError, "Error with clone 'git://%s%s/common: %s" % (netloc, path, output)
+
+ cmd = ['git', 'ls-files', '-t', '-s']
+ log_output(cmd[0], cmd, logfile, uploadpath, cwd='%s/common' % gitdir, logerror=1, append=1)
+
+ cmd = ['git', 'checkout', '-b', 'kojibuild', fragment]
+ self.logger.debug("executing: %s" % cmd)
+ if log_output(cmd[0], cmd, logfile, uploadpath, cwd=sourcedir, logerror=1, append=1):
+ output = "(none)"
+ try:
+ output = open(logfile).read()
+ except IOError:
+ pass
+ raise koji.BuildError, "Error with checking out tag '%s' from git://%s%s/%s: %s" % (fragment, netloc, path, query, output)
+
+ cmd = ['git', 'ls-files', '-t', '-s']
+ log_output(cmd[0], cmd, logfile, uploadpath, cwd=sourcedir, logerror=1, append=1)
+
+ spec_files = glob.glob("%s/*.spec" % sourcedir)
+ if len(spec_files) == 0:
+ raise koji.BuildError("No spec file found")
+ elif len(spec_files) > 1:
+ raise koji.BuildError("Multiple spec files found: %s" % spec_files)
+ spec_file = spec_files[0]
+
+ # Run spec file sanity checks. Any failures will throw a BuildError
+ self.spec_sanity_checks(spec_file)
+
+ #build srpm
+ cmd = ['make', '-C', sourcedir, 'srpm', '_KOJI=1.2.2', '_KOJI_TAG=%s' % fragment]
+ self.logger.debug("executing: %s" % cmd)
+ if log_output(cmd[0], cmd, logfile, uploadpath, cwd=gitdir, logerror=1, append=1):
+ raise koji.BuildError, "Error building SRPM"
+
+ srpms = glob.glob('%s/*.src.rpm' % sourcedir)
+ if len(srpms) == 0:
+ raise koji.BuildError, "No srpms found in %s" % sourcedir
+ elif len(srpms) > 1:
+ raise koji.BuildError, "Multiple srpms found in %s: %s" % (sourcedir, ", ".join(srpms))
+ else:
+ srpm = srpms[0]
+
+ # check srpm name
+ h = koji.get_rpm_header(srpm)
+ name = h[rpm.RPMTAG_NAME]
+ version = h[rpm.RPMTAG_VERSION]
+ release = h[rpm.RPMTAG_RELEASE]
+ srpm_name = "%(name)s-%(version)s-%(release)s.src.rpm" % locals()
+ if srpm_name != os.path.basename(srpm):
+ raise koji.BuildError, 'srpm name mismatch: %s != %s' % (srpm_name, os.path.basename(srpm))
+
+ #upload srpm and return
+ self.uploadFile(srpm)
+ return {
+ 'srpm' : "%s/%s" % (uploadpath, srpm_name),
+ 'log' : "%s/srpm.log" % uploadpath,
+ }
class BuildSRPMFromCVSTask(BaseTaskHandler):
diff --git a/cli/koji b/cli/koji
index 718b6e5..4f05500 100755
--- a/cli/koji
+++ b/cli/koji
@@ -669,7 +669,7 @@ def handle_build(options, session, args):
if build_opts.background:
#relative to koji.PRIO_DEFAULT
priority = 5
- if not source.startswith('cvs://'):
+ if not (source.startswith('cvs://') or source.startswith('git://')):
# only allow admins to perform non-scratch builds from srpm
if not opts['scratch'] and not session.hasPerm('admin'):
parser.error(_("builds from srpm must use --scratch"))
@@ -741,7 +741,7 @@ def handle_chain_build(options, session, args):
if build_level:
src_list.append(build_level)
build_level = []
- elif src.startswith('cvs://'):
+ elif src.startswith('cvs://') or src.startswith('git://'):
build_level.append(src)
elif '/' not in src and len(src.split('-')) >= 3:
# quick check that it looks like a N-V-R
@@ -2432,8 +2432,13 @@ def _parseTaskParams(session, method, task_id):
if method == 'buildFromCVS':
lines.append("CVS URL: %s" % params[0])
lines.append("Build Target: %s" % params[1])
+ elif method == 'buildFromGit':
+ lines.append("GIT URL: %s" % params[0])
+ lines.append("Build Target: %s" % params[1])
elif method == 'buildSRPMFromCVS':
lines.append("CVS URL: %s" % params[0])
+ elif method == 'buildSRPMFromGit':
+ lines.append("GIT URL: %s" % params[0])
elif method == 'multiArchBuild':
lines.append("SRPM: %s/work/%s" % (options.topdir, params[0]))
lines.append("Build Target: %s" % params[1])
diff --git a/koji.spec b/koji.spec
index f14bb6e..7d4d643 100644
--- a/koji.spec
+++ b/koji.spec
@@ -49,7 +49,7 @@ Requires(post): /sbin/service
Requires(preun): /sbin/chkconfig
Requires(preun): /sbin/service
Requires(pre): /usr/sbin/useradd
-Requires: cvs
+Requires: cvs git-core make
Requires: rpm-build
Requires: redhat-rpm-config
Requires: createrepo >= 0.4.10
diff --git a/koji/__init__.py b/koji/__init__.py
index d808126..5789220 100644
--- a/koji/__init__.py
+++ b/koji/__init__.py
@@ -1517,6 +1517,11 @@ def taskLabel(taskInfo):
if source.startswith('cvs://'):
source = source[source.rfind('/') + 1:]
source = source.replace('#', ':')
+ elif source.startswith('git://'):
+ source = source[source.rfind('?') + 1:]
+ tmp = source[:source.find('#')]
+ source = source[source.find('#') + 1:]
+ source = '%s:%s' % (tmp, source[source.rfind('/') + 1:])
else:
source = os.path.basename(source)
extra = '%s, %s' % (target, source)
@@ -1526,6 +1531,12 @@ def taskLabel(taskInfo):
url = url[url.rfind('/') + 1:]
url = url.replace('#', ':')
extra = url
+ elif method == 'buildSRPMFromGit':
+ if taskInfo.has_key('request'):
+ url = taskInfo['request'][0]
+ url = url[url.rfind('/') + 1:]
+ url = url.replace('#', ':')
+ extra = url
elif method == 'buildArch':
if taskInfo.has_key('request'):
srpm, tagID, arch = taskInfo['request'][:3]
diff --git a/www/kojiweb/taskinfo.chtml b/www/kojiweb/taskinfo.chtml
index 76df4bc..dd5cf72 100644
--- a/www/kojiweb/taskinfo.chtml
+++ b/www/kojiweb/taskinfo.chtml
@@ -67,6 +67,11 @@
<strong>Build Target:</strong> <a href="buildtargetinfo?name=$params[1]">$params[1]</a>
#elif $task.method == 'buildSRPMFromCVS'
<strong>CVS URL:</strong> $params[0]
+ #elif $task.method == 'buildFromGit'
+ <strong>GIT URL:</strong> $params[0]<br/>
+ <strong>Build Target:</strong> <a href="buildtargetinfo?name=$params[1]">$params[1]</a>
+ #elif $task.method == 'buildSRPMFromGit'
+ <strong>GIT URL:</strong> $params[0]
#elif $task.method == 'multiArchBuild'
<strong>SRPM:</strong> $params[0]<br/>
<strong>Build Target:</strong> <a href="buildtargetinfo?name=$params[1]">$params[1]</a><br/>
@@ -276,7 +281,7 @@ $cgi.escape($result.faultString.strip())
<a href="getfile?taskID=$task.id&name=$urllib.quote($filename)">$filename</a><br/>
#end for
#if $task.state not in ($koji.TASK_STATES.CLOSED, $koji.TASK_STATES.CANCELED, $koji.TASK_STATES.FAILED) and \
- $task.method in ('buildSRPMFromCVS', 'buildArch', 'createrepo')
+ $task.method in ('buildSRPMFromCVS', 'buildSRPMFromGit', 'buildArch', 'createrepo')
<br/>
<a href="watchlogs?taskID=$task.id">Watch logs</a>
#end if
diff --git a/www/kojiweb/tasks.chtml b/www/kojiweb/tasks.chtml
index 70de06c..32ca6ad 100644
--- a/www/kojiweb/tasks.chtml
+++ b/www/kojiweb/tasks.chtml
@@ -104,6 +104,7 @@ All
<option value="all" #if $method == 'all' then 'selected="selected"' else ''#>all</option>
<option value="build" #if $method == 'build' then 'selected="selected"' else ''#>build</option>
<option value="buildSRPMFromCVS" #if $method == 'buildSRPMFromCVS' then 'selected="selected"' else ''#>buildSRPMFromCVS</option>
+ <option value="buildSRPMFromGit" #if $method == 'buildSRPMFromGit' then 'selected="selected"' else ''#>buildSRPMFromGit</option>
<option value="buildArch" #if $method == 'buildArch' then 'selected="selected"' else ''#>buildArch</option>
<option value="buildNotification" #if $method == 'buildNotification' then 'selected="selected"' else ''#>buildNotification</option>
<option value="tagBuild" #if $method == 'tagBuild' then 'selected="selected"' else ''#>tagBuild</option>
More information about the Fedora-buildsys-list
mailing list