[Date Prev][Date Next]   [Thread Prev][Thread Next]   [Thread Index] [Date Index] [Author Index]

[Pulp-list] modelchanges diff



I attached the modelchanges diff from jmatthews's branch vs master.

Note:

* The PackageVersionApi create now takes name in an effort to consider dropping "Package" class

-    def create(self, packageid, epoch, version, release, arch):
+    def create(self, name, epoch, version, release, arch, description,
+            checksum_type, checksum, filename):


what I would do is take it further and merge PackageVersion and Package class into one object in our API and just call it a Package.

* Addition of sha256 calcs to our datamodel in repo_sync.import_package

Mike
--
Mike McCune
mmccune AT redhat.com
Red Hat Engineering       | Portland, OR
Systems Management        | 650.254.4248
diff --git a/playpen/mongodb/display_pkgs.py b/playpen/mongodb/display_pkgs.py
new file mode 100755
index 0000000..9e56c71
--- /dev/null
+++ b/playpen/mongodb/display_pkgs.py
@@ -0,0 +1,35 @@
+#!/usr/bin/env python
+import time
+from pymongo import Connection
+from pymongo.son_manipulator import AutoReference, NamespaceInjector
+from optparse import OptionParser
+
+import pulp.util
+from pulp.api.package_version import PackageVersionApi
+
+if __name__ == "__main__":
+    
+    package_id = "pulp-test-package"
+    checksum = "6bce3f26e1fc0fc52ac996f39c0d0e14fc26fb8077081d5b4dbfb6431b08aa9f"
+    checksum_type = "sha256"
+    filename = "pulp-test-package-0.3.1-1.fc11.x86_64.rpm"
+
+
+    config = pulp.util.loadConfig("../../etc/pulp.ini")
+    pvApi = PackageVersionApi(config)
+
+    found = pvApi.packageversion(filename=filename, checksum_type=checksum_type, checksum=checksum)
+    print "Lookup for %s, %s, %s yielded %s" % (filename, checksum_type, checksum, found)
+
+    db = pvApi.objectdb
+    print "db = %s" % (db)
+    found = db.find({"filename":filename})
+    print "Search for all PV's with %s: %s" % (filename, found)
+    for f in found:
+        print f
+    found = db.find()
+    print "%s PV objects found with an open search" % (found.count())
+
+    found = pvApi.packageversion()
+    print "search with empty searchDict returned %s results" % (found.count())
+
diff --git a/src/pulp/api/base.py b/src/pulp/api/base.py
index 1b9a123..7ae0fb9 100644
--- a/src/pulp/api/base.py
+++ b/src/pulp/api/base.py
@@ -26,7 +26,8 @@ class BaseApi(object):
 
         # Mongo DB
         self.connection = pymongo.Connection()
-        self.db = self.connection._database
+        #self.db = self.connection._database
+        self.db = self.connection._database_model_changes
         # Inject the collection's namespace into each object
         self.db.add_son_manipulator(NamespaceInjector())
         # Provides auto-referencing/auto-dereferencing ability
diff --git a/src/pulp/api/package_version.py b/src/pulp/api/package_version.py
index be076b3..ab0c364 100644
--- a/src/pulp/api/package_version.py
+++ b/src/pulp/api/package_version.py
@@ -23,29 +23,62 @@ class PackageVersionApi(BaseApi):
 
     def __init__(self, config):
         BaseApi.__init__(self, config)
+        self.objectdb.ensure_index([('name', pymongo.DESCENDING), 
+            ('epoch', pymongo.DESCENDING), 
+            ('version', pymongo.DESCENDING),
+            ('release', pymongo.DESCENDING),
+            ('arch', pymongo.DESCENDING), 
+            ('filename', pymongo.DESCENDING),
+            ('checksum', pymongo.DESCENDING)], 
+            unique=True, background=True)
 
     def _get_unique_indexes(self):
         return []
 
     def _get_indexes(self):
-        return ["packageid"]
+        return ["name", "filename", "checksum", "epoch", "version", "release",
+                "arch", "description"]
 
     def _getcollection(self):
         return self.db.packageversions
 
-    def create(self, packageid, epoch, version, release, arch):
+    def create(self, name, epoch, version, release, arch, description, 
+            checksum_type, checksum, filename):
         """
         Create a new PackageVersion object and return it
         """
-        pv = model.PackageVersion(packageid, epoch, version, release, arch)
+        pv = model.PackageVersion(name, epoch, version, release, arch, description,
+                checksum_type, checksum, filename)
         self.objectdb.insert(pv)
         return pv
-        
-    def packageversion(self, id, filter=None):
+
+    def delete(self, object):
+        """
+        Delete package version object based on "_id" key
+        """
+        self.objectdb.remove({"_id":object["_id"]})
+
+    def packageversion(self, name=None, epoch=None, version=None, release=None, arch=None, 
+            filename=None, checksum_type=None, checksum=None):
         """
-        Return a single PackageVersion object
+        Return a list of all package version objects matching search terms
         """
-        return self.objectdb.find_one({'id': id})
+        searchDict = {}
+        if name:
+            searchDict['name'] = name
+        if epoch:
+            searchDict['epoch'] = epoch
+        if version:
+            searchDict['version'] = version
+        if release:
+            searchDict['release'] = release
+        if arch:
+            searchDict['arch'] = arch
+        if filename:
+            searchDict['filename'] = filename
+        if checksum_type and checksum:
+            searchDict['checksum.%s' % checksum_type] = checksum
+        return self.objectdb.find(searchDict)
 
     def packageversions(self):
         """
diff --git a/src/pulp/api/repo.py b/src/pulp/api/repo.py
index 3ee8858..f2e1015 100644
--- a/src/pulp/api/repo.py
+++ b/src/pulp/api/repo.py
@@ -29,7 +29,7 @@ from grinder.RepoFetch import YumRepoGrinder
 from pulp import model
 from pulp import repo_sync, upload
 from pulp.api.base import BaseApi
-from pulp.api.package import PackageApi
+#from pulp.api.package import PackageApi
 from pulp.api.package_version import PackageVersionApi
 from pulp.api.package_group import PackageGroupApi
 from pulp.api.package_group_category import PackageGroupCategoryApi
@@ -46,13 +46,19 @@ class RepoApi(BaseApi):
     def __init__(self, config):
         BaseApi.__init__(self, config)
 
-        self.packageApi = PackageApi(config)
+        #self.packageApi = PackageApi(config)
         self.packageVersionApi = PackageVersionApi(config)
         self.packageGroupApi = PackageGroupApi(config)
         self.packageGroupCategoryApi = PackageGroupCategoryApi(config)
 
         # TODO: Extract this to a config
         self.localStoragePath = config.get('paths', 'local_storage')
+   
+    def _get_indexes(self):
+        return ["packages", "packagegroups", "packagegroupcategories"]
+
+    def _get_unique_indexes(self):
+        return ["id"]
 
     def _getcollection(self):
         return self.db.repos
@@ -71,11 +77,128 @@ class RepoApi(BaseApi):
         
     def packages(self, id):
         """
-        Return list of Package objects in this Repo
+        Return dictionary of PackageVersion objects in this Repo, key is package name
         """
         repo = self.repository(id)
+        if (repo == None):
+            raise PulpException("No Repo with id: %s found" % id)
         return repo['packages']
     
+    def packageversions(self, repoid, name):
+        """
+        Return list of PackageVersions objects for this repo and package name
+        """
+        repo = self.repository(repoid)
+        if (repo == None):
+            raise PulpException("No Repo with id: %s found" % repoid)
+        if not repo["packages"].has_key(name):
+            return None
+        return repo["packages"][name]
+
+    def add_package_version(self, repoid, pv):
+        """
+        Adds the passed in package version to this repo
+        """
+        repo = self.repository(repoid)
+        if (repo == None):
+            raise PulpException("No Repo with id: %s found" % repoid)
+        if not repo["packages"].has_key(pv['name']):
+            repo["packages"][pv['name']] = []
+        # TODO:  We might want to restrict PackageVersions we add to only
+        #        allow 1 NEVRA per repo and require filename to be unique
+        for item in repo["packages"][pv['name']]:
+            if item['_id'] == pv['_id']:
+                # No need to update repo, this PackageVersion is already under this repo
+                return
+        # Note:  A DBRef() for the objects '_id' is what's added in mongo
+        #        This is a reference to the PackageVersion collection's object
+        repo["packages"][pv['name']].append(pv)
+        self.update(repo)
+
+    def remove_package_version(self, repoid, pv):
+        repo = self.repository(repoid)
+        if (repo == None):
+            raise PulpException("No Repo with id: %s found" % repoid)
+        if not repo["packages"].has_key(pv['name']):
+            raise PulpException("No Package with name: %s found in repo: %s" %
+                    (pv['name'], repoid))
+        for item in repo["packages"][pv['name']]:
+            if item['name'] == pv['name'] and \
+                item['version'] == pv['version'] and \
+                item['epoch'] == pv['epoch'] and \
+                item['release'] == pv['release'] and \
+                item['arch'] == pv['arch']:
+                    repo['packages'][pv['name']].remove(item)
+                    if len(repo['packages'][pv['name']]) == 0:
+                        # list is empty now, so cleanup and remove 
+                        # it from the packages
+                        del repo['packages'][pv['name']]
+        self.update(repo)
+
+    def remove_packagegroup(self, repoid, groupid):
+        """
+        Remove a packagegroup from a repo
+        """
+        repo = self.repository(repoid)
+        if (repo == None):
+            raise PulpException("No Repo with id: %s found" % repoid)
+        if repo['packagegroups'].has_key(groupid):
+            del repo['packagegroups'][groupid]
+        self.update(repo)
+
+    def update_packagegroup(self, repoid, pg):
+        """
+        Save the passed in PackageGroup to this repo
+        """
+        repo = self.repository(repoid)
+        if (repo == None):
+            raise PulpException("No Repo with id: %s found" % repoid)
+        repo['packagegroups'][pg['id']] = pg
+        self.update(repo)
+
+    def update_packagegroups(self, repoid, pglist):
+        """
+        Save the list of passed in PackageGroup objects to this repo
+        """
+        repo = self.repository(repoid)
+        if (repo == None):
+            raise PulpException("No Repo with id: %s found" % repoid)
+        for item in pglist:
+            repo['packagegroups'][item['id']] = item
+        self.update(repo)
+
+    def translate_packagegroup(self, obj):
+        """
+        Translate a SON Document to an object that yum.comps.Comps can work with
+        """
+        # Main reason for doing this is that yum.comps expects the passed in 
+        # object to support dot notation references, the returned SON document
+        # does not support this, so yum.comps isn't able to read the info 
+        #TODO: More work is needed in this method before output of groups will work
+        pg = model.PackageGroup(obj['id'], obj['name'], obj['description'], 
+                user_visible=obj['user_visible'], display_order=obj['display_order'],
+                default=obj['default'], langonly=obj['langonly'])
+        pg.groupid = obj['id']  
+        pg.translated_name = {}
+        for key in obj['translated_name']:
+            pg.translated_name[key] = obj['translated_name'][key]
+        pg.translated_description = {}
+        for key in obj['translated_description']:
+            pg.translated_description[key] = obj['translated_description']
+        pg.mandatory_packages = {}
+        for pkgname in obj['mandatory_package_names']:
+            pg.mandatory_packages[pkgname] = 1 
+        pg.optional_packages = {}
+        for pkgname in obj['optional_package_names']:
+            pg.optional_packages[pkgname] = 1
+        pg.default_packages = {}
+        for pkgname in obj['default_package_names']:
+            pg.default_packages[pkgname] = 1
+        pg.conditional_packages = {}
+        for key in obj['conditional_package_names']:
+            pg.conditional_packages[key] = obj['conditional_package_names'][key]
+        return pg
+
     def packagegroups(self, id):
         """
         Return list of PackageGroup objects in this Repo
@@ -83,13 +206,98 @@ class RepoApi(BaseApi):
         repo = self.repository(id)
         return repo['packagegroups']
     
+    def packagegroup(self, repoid, groupid):
+        """
+        Return a PackageGroup from this Repo
+        """
+        repo = self.repository(repoid)
+        if not repo['packagegroups'].has_key(groupid):
+            return None
+        return repo['packagegroups'][groupid]
+
+    def remove_packagegroupcategory(self, repoid, categoryid):
+        """
+        Remove a packagegroupcategory from a repo
+        """
+        repo = self.repository(repoid)
+        if (repo == None):
+            raise PulpException("No Repo with id: %s found" % repoid)
+        if repo['packagegroupcategories'].has_key(categoryid):
+            del repo['packagegroupcategories'][categoryid]
+        self.update(repo)
+    
+    def update_packagegroupcategory(self, repoid, pgc):
+        """
+        Save the passed in PackageGroupCategory to this repo
+        """
+        repo = self.repository(repoid)
+        if (repo == None):
+            raise PulpException("No Repo with id: %s found" % repoid)
+        repo['packagegroupcategories'][pgc['id']] = pgc
+        self.update(repo)
+    
+    def update_packagegroupcategories(self, repoid, pgclist):
+        """
+        Save the list of passed in PackageGroupCategory objects to this repo
+        """
+        repo = self.repository(repoid)
+        if (repo == None):
+            raise PulpException("No Repo with id: %s found" % repoid)
+        for item in pgclist:
+            repo['packagegroupcategories'][item['id']] = item
+        self.update(repo)
+
+    def translate_packagegroupcategory(self, obj):
+        """
+        Translate a SON Document to an object that yum.comps.Comps can work with
+        """
+        #TODO: More work is needed in this method before output of categories will work
+        pgc = model.PackageGroupCategory(obj['id'], obj['name'], obj['description'], 
+                display_order=obj['display_order'])
+        pgc.categoryid = obj['id']
+        pgc.translated_name = {}
+        for key in obj['translated_name']:
+            pgc.translated_name[key] = obj['translated_name'][key]
+        pgc.translated_description = {}
+        for key in obj['translated_description']:
+            pgc.translated_description[key] = obj['translated_description'][key]
+        pgc._groups = {}
+        for groupid in obj['packagegroupids']:
+            pgc._groups[groupid] = groupid
+        return pgc
+
+    def packagegroups(self, id):
+        """
+        Return list of PackageGroup objects in this Repo
+        """
+        repo = self.repository(id)
+        return repo['packagegroups']
+
+    def packagegroup(self, repoid, groupid):
+        """
+        Return a PackageGroup from this Repo
+        """
+        repo = self.repository(repoid)
+        if not repo['packagegroups'].has_key(groupid):
+            return None
+        return repo['packagegroups'][groupid]
+
     def packagegroupcategories(self, id):
         """
         Return list of PackageGroupCategory objects in this Repo
         """
         repo = self.repository(id)
         return repo['packagegroupcategories']
-    
+
+    def packagegroupcategory(self, repoid, categoryid):
+        """
+        Return a PackageGroupCategory object from this Repo
+        """
+        repo = self.repository(repoid)
+        if not repo['packagegroupcategories'].has_key(categoryid):
+            return None
+        return repo['packagegroupcategories'][categoryid]
+
     def create(self, id, name, arch, feed):
         """
         Create a new Repository object and return it
diff --git a/src/pulp/model.py b/src/pulp/model.py
index 9384727..ce4d4a3 100644
--- a/src/pulp/model.py
+++ b/src/pulp/model.py
@@ -64,30 +64,35 @@ class RepoSource(Base):
 
 
 class Package(Base):
-    def __init__(self, packageid, description):
-        #TODO: move 'description' to PackageVersion
+    def __init__(self, repoid, packageid):
         #TODO: Consider getting rid of 'package', we might not need it
+        self.repoid = repoid
         self.packageid = packageid
-        self.description = description
         self.versions = []
 
 class PackageVersion(Base):
-    def __init__(self, packageid, epoch, version, release, arch):
-        self.packageid = packageid
+    def __init__(self, name, epoch, version, release, arch, description, 
+            checksum_type, checksum, filename):
+        #TODO: Can we enforce unique indexes between keys?
+        self.name = name
         self.epoch = epoch
         self.version = version
         self.release = release
         self.arch = arch
-        #TODO: add support for 'filename' and 'checksum' to constructor, apis, and tests
-        #self.filename = ""
-        #self.checksum = {}
+        self.description = description
+        self.filename = filename
+        self.checksum = {checksum_type:checksum}
+        # Add gpg keys
         self.requires = []
         self.provides = []
 
 class PackageGroup(Base):
-    def __init__(self, groupid, name, description, user_visible=False, 
+    """
+    Class represents a yum.comps.Group
+    """
+    def __init__(self, id, name, description, user_visible=False, 
             display_order=1024, default=True, langonly=None):
-        self.groupid = groupid
+        self.id = id
         self.name = name
         self.description = description
         self.user_visible = user_visible
@@ -102,8 +107,8 @@ class PackageGroup(Base):
         self.translated_description = {}
 
 class PackageGroupCategory(Base):
-    def __init__(self, categoryid, name, description, display_order=99):
-        self.categoryid = categoryid
+    def __init__(self, id, name, description, display_order=99):
+        self.id = id
         self.name = name
         self.description = description
         self.display_order = display_order
diff --git a/src/pulp/repo_sync.py b/src/pulp/repo_sync.py
index f5264ea..1fa9cf1 100644
--- a/src/pulp/repo_sync.py
+++ b/src/pulp/repo_sync.py
@@ -17,6 +17,7 @@
 import gzip
 import logging
 import os
+import time
 import traceback
 from urlparse import urlparse
 
@@ -60,94 +61,87 @@ class BaseSynchronizer(object):
         self.package_group_api = PackageGroupApi(config)
 
     def add_packages_from_dir(self, dir, repo):
-
         dir_list = os.listdir(dir)
         package_count = 0
+        startTime = time.time()
         for fname in dir_list:
             self.import_package(dir + fname, repo)
             package_count = package_count + 1
-        log.debug("read [%s] packages" % package_count)
-        self._read_comps_xml(dir, repo)
+        endTime = time.time()
+        log.debug("Repo: %s read [%s] packages took %s seconds" % 
+                (repo['id'], package_count, endTime - startTime))
+        # TODO: Parse repomd.xml and lookup name for groups element
+        compsfile = None
+        compspath = os.path.join(dir, 'repodata/comps.xml')
+        if os.path.isfile(compspath):
+            compsfile = open(compspath, "r")
+        else:
+            compspath = os.path.join(dir, 'repodata/comps.xml.gz')
+            if os.path.isfile(compspath):
+                compsfile = gzip.open(compspath, 'r')
+        if compsfile:
+            repo['comps_xml_path'] = compspath
+            self.import_groups_data(compsfile, repo)
+            log.debug("Loaded comps info from %s" % (compspath))
 
     def import_package(self, pkg_path, repo):
-        packages = repo['packages']
         if (pkg_path.endswith(".rpm")):
             try:
+                file_name = os.path.basename(pkg_path)
                 info = pulp.util.get_rpm_information(pkg_path)
-                p = self.package_api.package(info['name'])
-                if not p:
-                    p = self.package_api.create(info['name'], info['description'])
-
-                pv = self.package_version_api.packageversion_by_ivera(p['packageid'],
-                                                                      info['version'],
-                                                                      info['epoch'],
-                                                                      info['release'],
-                                                                      info['arch'],)
-                if not pv:
-                    pv = self.package_version_api.create(p["packageid"], info['epoch'], 
-                                              info['version'], info['release'], info['arch'])
+                if not repo["packages"].has_key(info['name']):
+                    repo["packages"][info['name']] = []
+                hashtype = "sha256"
+                checksum = pulp.util.getFileChecksum(hashtype=hashtype, 
+                        filename=pkg_path)
+                found = self.package_version_api.packageversion(name=info['name'], 
+                        epoch=info['epoch'], version=info['version'], 
+                        release=info['release'], arch=info['arch'],filename=file_name, 
+                        checksum_type=hashtype, checksum=checksum)
+                if found.count() == 1:
+                    pv = found[0]
+                else:
+                    pv = self.package_version_api.create(info['name'], info['epoch'],
+                        info['version'], info['release'], info['arch'], info['description'],
+                        "sha256", checksum, file_name)
                     for dep in info['requires']:
                         pv.requires.append(dep)
                     for dep in info['provides']:
                         pv.provides.append(dep)
                     self.package_version_api.update(pv)
-
-                p["versions"].append(pv)
-                self.package_api.update(p)
-                packages[p["packageid"]] = p
-                log.debug("Repo <%s> added package <%s> with %s versions" %
-                          (repo["id"], p["packageid"], len(p["versions"])))
+                #TODO:  Ensure we don't add duplicate pv's to the 'packages' list
+                repo['packages'][info['name']].append(pv)
             except Exception, e:
-                log.debug("Exception = %s" % (traceback.format_exc()))
+                log.debug("%s" % (traceback.format_exc()))
                 log.error("error reading package %s" % (pkg_path))
 
-    def _read_comps_xml(self, dir, repo):
+    def import_groups_data(self, compsfile, repo):
         """
         Reads a comps.xml or comps.xml.gz under repodata from dir
         Loads PackageGroup and Category info our db
         """
-
-        compspath = os.path.join(dir, 'repodata/comps.xml')
-        compsxml = None
-        if os.path.isfile(compspath):
-            compsxml = open(compspath, "r")
-        else:
-            compspath = os.path.join(dir, 'repodata/comps.xml.gz')
-            if os.path.isfile(compspath):
-                compsxml = gzip.open(compspath, 'r')
-    
-        if not compsxml:
-            log.info("Not able to find a comps.xml(.gz) to read")
-            return False
-
-        log.info("Reading comps info from %s" % (compspath))
-        repo['comps_xml_path'] = compspath
         try:
             comps = yum.comps.Comps()
-            comps.add(compsxml)
+            comps.add(compsfile)
             for c in comps.categories:
-                ctg = self.package_group_category_api.create(c.categoryid, c.name,
-                                                          c.description, c.display_order)
+                ctg = model.PackageGroupCategory(c.categoryid, c.name,
+                    c.description, c.display_order)
                 groupids = [grp for grp in c.groups]
-                ctg.packagegroupids.extend(groupids)
-                ctg.translated_name = c.translated_name
-                ctg.translated_description = c.translated_description
-                self.package_group_category_api.update(ctg)
-                repo['packagegroupcategories'][ctg.categoryid] = ctg
-
+                ctg['packagegroupids'].extend(groupids)
+                ctg['translated_name'] = c.translated_name
+                ctg['translated_description'] = c.translated_description
+                repo['packagegroupcategories'][ctg['id']] = ctg
             for g in comps.groups:
-                grp = self.package_group_api.create(g.groupid, g.name, g.description,
-                                              g.user_visible, g.display_order, g.default, g.langonly)
+                grp = model.PackageGroup(g.groupid, g.name, g.description,
+                    g.user_visible, g.display_order, g.default, g.langonly)
                 grp.mandatory_package_names.extend(g.mandatory_packages.keys())
                 grp.optional_package_names.extend(g.optional_packages.keys())
                 grp.default_package_names.extend(g.default_packages.keys())
                 grp.conditional_package_names = g.conditional_packages
                 grp.translated_name = g.translated_name
                 grp.translated_description = g.translated_description
-                self.package_group_api.update(grp)
-                repo['packagegroups'][grp.groupid] = grp
-            log.info("Comps info added from %s" % (compspath))
-        except yum.comps.CompsException:
+                repo['packagegroups'][grp['id']] = grp
+        except yum.Errors.CompsException:
             log.error("Unable to parse comps info for %s" % (compspath))
             return False
         return True
diff --git a/test/common/large_load.py b/test/common/large_load.py
index 413cd0d..8098380 100644
--- a/test/common/large_load.py
+++ b/test/common/large_load.py
@@ -87,9 +87,9 @@ class LargeLoad(unittest.TestCase):
             # self.capi.update(c)
             if (i % 100 == 0):
                 print "created [%s] consumers" % i
-                p = Package(TEST_PACKAGE_ID, 'random package to be found')
+                p = Package(repo["id"], TEST_PACKAGE_ID, 'random package to be found')
                 c.packageids.append(p.id)
-                # self.capi.update(c)
+                #self.capi.update(c)
             last_desc = c.description
             last_id = c.id
             consumers.append(c)
@@ -117,6 +117,11 @@ class LargeLoad(unittest.TestCase):
         cwithp = ll.capi.consumerswithpackage(TEST_PACKAGE_ID)
         print "Found [%s] consumers with packageid: [%s]" % (len(cwithp), TEST_PACKAGE_ID)
 
+
+print "This has not been updated with the branch modelchanges yet"
+print "More work is needed on the consumer side"
+sys.exit(1)
+
 parser = optparse.OptionParser()
 parser.add_option('--dirlist', dest='dirlist', 
                  action='store', help='File containing list of directories containing the repos you wish to use for this test')

diff --git a/test/unit/test_api.py b/test/unit/test_api.py
index eaa16d6..7eae1eb 100644
--- a/test/unit/test_api.py
+++ b/test/unit/test_api.py
@@ -53,7 +53,7 @@ from testutil import load_test_config
 class TestApi(unittest.TestCase):
     def clean(self):
         self.rapi.clean()
-        self.papi.clean()
+        #self.papi.clean()
         self.capi.clean()
         self.pvapi.clean()
         self.pgapi.clean()
@@ -63,7 +63,8 @@ class TestApi(unittest.TestCase):
         config = load_test_config()
 
         self.rapi = RepoApi(config)
-        self.papi = PackageApi(config)
+        self.rapi.localStoragePath = "/tmp"
+        #self.papi = PackageApi(config)
         self.capi = ConsumerApi(config)
         self.pvapi = PackageVersionApi(config)
         self.pgapi = PackageGroupApi(config)
@@ -154,8 +155,10 @@ class TestApi(unittest.TestCase):
     def test_repo_packages(self):
         repo = self.rapi.create('some-id','some name', \
             'i386', 'yum:http://example.com')
-        package = Package('test_repo_packages','test package')
-        repo['packages'][package["packageid"]] = package
+        pv = self.create_package_version('test_repo_packages')
+        # package = PackageVersion('test_repo_packages','test package')
+        self.rapi.add_package_version(repo["id"], pv)
+        # repo['packages'][package["packageid"]] = package
         self.rapi.update(repo)
         
         found = self.rapi.repository('some-id')
@@ -170,42 +173,33 @@ class TestApi(unittest.TestCase):
                 'test-group-description')
         package = Package('test_repo_packages','test package')
         pkggroup.default_package_names.append(package["packageid"])
-        repo['packagegroups'][pkggroup["groupid"]] = pkggroup
+        repo['packagegroups'][pkggroup["id"]] = pkggroup
         repo['packages'][package["packageid"]] = package
+        
         self.rapi.update(repo)
         
         found = self.rapi.repository('some-id')
-        packages = found['packages']
-        assert(packages != None)
-        assert(packages['test_repo_packages'] != None)
         assert(found['packagegroups'] != None)
-        print "test_repo_package_groups found['packagegroups'] = %s" % (found['packagegroups'])
-        assert(pkggroup.groupid in found['packagegroups'])
+        assert(pkggroup['id'] in found['packagegroups'])
     
     def test_repo_package_group_categories(self):
-        repo = self.rapi.create('some-id','some name', \
+        repo = self.rapi.create('some-id_pkg_group_categories','some name', \
             'i386', 'yum:http://example.com')
-        package = Package('test_repo_packages','test package')
         pkggroup = PackageGroup('test-group-id', 'test-group-name', 
                 'test-group-description')
-        pkggroup.default_package_names.append(package["packageid"])
+        pkggroup.default_package_names.append("test-package-name")
         ctg = PackageGroupCategory('test-group-cat-id', 'test-group-cat-name',
                 'test-group-cat-description')
         ctg.packagegroupids = pkggroup.id
-        repo['packagegroupcategories'][ctg.categoryid] = ctg
-        repo['packagegroups'][pkggroup.groupid] = pkggroup
-        repo['packages'][package["packageid"]] = package
+        repo['packagegroupcategories'][ctg.id] = ctg
+        repo['packagegroups'][pkggroup.id] = pkggroup
         self.rapi.update(repo)
         
-        found = self.rapi.repository('some-id')
-        packages = found['packages']
-        assert(packages != None)
-        assert(packages['test_repo_packages'] != None)
+        found = self.rapi.repository('some-id_pkg_group_categories')
         assert(found['packagegroups'] != None)
-        print "test_repo_package_groups found['packagegroups'] = %s" % (found['packagegroups'])
-        assert(pkggroup.groupid in found['packagegroups'])
+        assert(pkggroup['id'] in found['packagegroups'])
         assert(found['packagegroupcategories'] != None)
-        assert(ctg.categoryid in found['packagegroupcategories'])
+        assert(ctg['id'] in found['packagegroupcategories'])
     
     def test_consumer_create(self):
         c = self.capi.create('test-consumer', 'some consumer desc')
@@ -235,8 +229,11 @@ class TestApi(unittest.TestCase):
             
     def test_consumerwithpackage(self):
         c = self.capi.create('test-consumer', 'some consumer desc')
-        package = Package('test_consumerwithpackage','test package search')
-        c.packageids.append(package["packageid"])
+        repo = self.rapi.create('some-id', 'some name',
+                'i386', 'yum:http://example.com')
+        test_pkg_name = "test_consumerwithpackage"
+        #TODO: The consumer model/api needs to be updated, it's not setup to handle
+        #       tracking a packageversion
         for i in range(10):
             package = Package(random_string(), random_string())
             c.packageids.append(package["packageid"])
@@ -257,7 +254,7 @@ class TestApi(unittest.TestCase):
         assert(parsed != None)
         print parsed
     
-    def test_sync_two_repos_share_common_package(self):
+    def test_sync_two_repos_same_nevra_different_checksum(self):
         """
         Sync 2 repos that have a package with same NEVRA 
         but different checksum
@@ -273,10 +270,9 @@ class TestApi(unittest.TestCase):
         repo_a = self.rapi.create(repo_name_a,'some name', 'x86_64', 
                                   'local:file://%s' % datadir_a)
         repo_b = self.rapi.create(repo_name_b,'some name', 'x86_64', 
-                                  'local:file://%s' % datadir_b)
-        self.rapi.sync(repo_a.id)
-        self.rapi.sync(repo_b.id)
-
+                                'local:file://%s' % datadir_b)
+        self.rapi.sync(repo_a["id"])
+        self.rapi.sync(repo_b["id"])
         # Look up each repo from API
         found_a = self.rapi.repository(repo_a.id)
         found_b = self.rapi.repository(repo_b.id)
@@ -286,15 +282,14 @@ class TestApi(unittest.TestCase):
         assert (found_b["packages"].has_key(test_pkg_name))
 
         # Grab the associated package version (there should only be 1)
-        # Ensure that the package versions have different md5sums, but all other
+        # Ensure that the package versions have different checksums, but all other
         # keys are identical
-        assert (len(found_a["packages"][test_pkg_name]["versions"]) == 1)
-        assert (len(found_b["packages"][test_pkg_name]["versions"]) == 1)
-        pkgVerA = found_a["packages"][test_pkg_name]["versions"][0]
-        pkgVerB = found_a["packages"][test_pkg_name]["versions"][0]
-        for key in ['epoch', 'version', 'release', 'arch']:
+        assert (len(found_a["packages"][test_pkg_name]) == 1)
+        assert (len(found_b["packages"][test_pkg_name]) == 1)
+        pkgVerA = found_a["packages"][test_pkg_name][0]
+        pkgVerB = found_b["packages"][test_pkg_name][0]
+        for key in ['epoch', 'version', 'release', 'arch', 'filename', 'name']:
             assert (pkgVerA[key] == pkgVerB[key])
-
         #TODO:
         # Add test to compare checksum when it's implemented in PackageVersion
         # verify the checksums are different
@@ -304,12 +299,12 @@ class TestApi(unittest.TestCase):
         Sync 2 repos that share a common package, same NEVRA
         same checksum
         """
-        test_pkg_name = "pulp-test-package-same-nevra"
+        test_pkg_name = "pulp-test-package"
         my_dir = os.path.abspath(os.path.dirname(__file__))
         repo_name_a = "test_two_repos_share_common_pkg_repo_A"
         repo_name_b = "test_two_repos_share_common_pkg_repo_B"
-        datadir_a = my_dir + "/data/sameNEVRA_differentChecksums/A/repo/"
-        datadir_b = my_dir + "/data/sameNEVRA_differentChecksums/B/repo/"
+        datadir_a = my_dir + "/data/sameNEVRA_sameChecksums/A/repo/"
+        datadir_b = my_dir + "/data/sameNEVRA_sameChecksums/B/repo/"
         # Create & Sync Repos
         repo_a = self.rapi.create(repo_name_a,'some name', 'x86_64', 
                                 'local:file://%s' % datadir_a)
@@ -327,14 +322,14 @@ class TestApi(unittest.TestCase):
         # Ensure that the package versions have different md5sums, but all other
         # keys are identical
 
-        # BELOW TEST Needs more changes to model/sync code before it can pass
-        #assert (len(found_a["packages"][test_pkg_name]["versions"]) == 1)
-        #assert (len(found_b["packages"][test_pkg_name]["versions"]) == 1)
-        #pkgVerA = found_a["packages"][test_pkg_name]["versions"][0]
-        #pkgVerB = found_a["packages"][test_pkg_name]["versions"][0]
+        assert (len(found_a["packages"][test_pkg_name]) == 1)
+        assert (len(found_b["packages"][test_pkg_name]) == 1)
+        pkgVerA = found_a["packages"][test_pkg_name][0]
+        pkgVerB = found_b["packages"][test_pkg_name][0]
         # Ensure that the 2 PackageVersions instances actually point 
         # to the same single instance
-        #assert(pkgVerA['_id'] == pkgVerB['_id'])
+        assert(repo_a['_id'] != repo_b['_id'])
+        assert(pkgVerA['_id'] == pkgVerB['_id'])
     
     def test_sync(self):
         repo = self.rapi.create('some-id','some name', 'i386', 
@@ -352,7 +347,9 @@ class TestApi(unittest.TestCase):
         dirList = os.listdir(self.rapi.localStoragePath + '/' + repo.id)
         assert(len(dirList) > 0)
         found = self.rapi.repository(repo.id)
+        print "found = ", found
         packages = found['packages']
+        print "packages = ", packages
         assert(packages != None)
         assert(len(packages) > 0)
         
@@ -369,27 +366,84 @@ class TestApi(unittest.TestCase):
         assert(len(packages) > 0)
         print packages
         p = packages.values()[0]
-        assert(p['versions'] != None)
+        assert(p != None)
         # versions = p['versions']
         
-    def test_package_versions(self):
-        p = self.papi.create('some-package-id', 'some package desc')
-        pv = self.pvapi.create(p.packageid, 0, '1.2.3', '1', 'i386')
-        p.versions.append(pv)
-        self.papi.update(p)
+    def create_package_version(self, name): 
+        test_pkg_name = name
+        test_epoch = "1"
+        test_version = "1.2.3"
+        test_release = "1.el5"
+        test_arch = "x86_64"
+        test_description = "test description text"
+        test_checksum_type = "sha256"
+        test_checksum = "9d05cc3dbdc94150966f66d76488a3ed34811226735e56dc3e7a721de194b42e"
+        test_filename = "test-filename-1.2.3-1.el5.x86_64.rpm"
+        pv = self.pvapi.create(name=test_pkg_name, epoch=test_epoch, version=test_version, 
+                release=test_release, arch=test_arch, description=test_description, 
+                checksum_type="sha256", checksum=test_checksum, filename=test_filename)
+        return pv
         
-        found = self.papi.package(p.packageid)
-        versions = found['versions']
-        assert(versions != None)
-        assert(versions[0]['packageid'] == p.packageid)
-        print found
-        
-    def test_packages(self):
-        p = self.papi.create('some-package-id', 'some package desc')
-        packages = self.papi.packages()
-        print "packages: %s" % packages
-        assert(len(packages) > 0)
-    
+    def test_package_versions(self):
+        repo = self.rapi.create('some-id','some name',
+            'i386', 'yum:http://example.com')
+        repo = self.rapi.repository(repo["id"])
+        test_pkg_name = "test_package_versions_name"
+        test_epoch = "1"
+        test_version = "1.2.3"
+        test_release = "1.el5"
+        test_arch = "x86_64"
+        test_description = "test description text"
+        test_checksum_type = "sha256"
+        test_checksum = "9d05cc3dbdc94150966f66d76488a3ed34811226735e56dc3e7a721de194b42e"
+        test_filename = "test-filename-1.2.3-1.el5.x86_64.rpm"
+        pv = self.pvapi.create(name=test_pkg_name, epoch=test_epoch, version=test_version, 
+                release=test_release, arch=test_arch, description=test_description, 
+                checksum_type="sha256", checksum=test_checksum, filename=test_filename)
+        # Add this package version to the repo
+        self.rapi.add_package_version(repo["id"], pv)
+        # Lookup repo and confirm new package version was added
+        repo = self.rapi.repository(repo["id"])
+        self.assertTrue(repo["packages"].has_key(test_pkg_name))
+        self.assertTrue(len(repo["packages"][test_pkg_name]) == 1)
+        saved_pkg = repo["packages"][test_pkg_name][0]
+        self.assertTrue(saved_pkg['name'] == test_pkg_name)
+        self.assertTrue(saved_pkg['epoch'] == test_epoch)
+        self.assertTrue(saved_pkg['version'] == test_version)
+        self.assertTrue(saved_pkg['release'] == test_release)
+        self.assertTrue(saved_pkg['arch'] == test_arch)
+        self.assertTrue(saved_pkg['description'] == test_description)
+        self.assertTrue(saved_pkg['checksum'].has_key(test_checksum_type))
+        self.assertTrue(saved_pkg['checksum'][test_checksum_type] == test_checksum)
+        self.assertTrue(saved_pkg['filename'] == test_filename)
+        # Verify we can find this package version through repo api calls
+        pkgs = self.rapi.packages(repo['id'])
+        self.assertTrue(pkgs.has_key(test_pkg_name))
+        self.assertTrue(len(pkgs[test_pkg_name]) == 1)
+        self.assertTrue(pkgs[test_pkg_name][0]['filename'] == test_filename)
+        pkgs = self.rapi.packageversions(repo['id'], test_pkg_name)
+        self.assertTrue(len(pkgs) == 1)
+        self.assertTrue(pkgs[0]['filename'] == test_filename)
+
+        # Remove package version from repo
+        self.rapi.remove_package_version(repo['id'], pv)
+        repo = self.rapi.repository(repo['id'])
+        self.assertTrue(not repo["packages"].has_key(test_pkg_name))
+        # Verify package version from repo
+        found = self.pvapi.packageversion(name=test_pkg_name, epoch=test_epoch, 
+                version=test_version, release=test_release, arch=test_arch, 
+                filename=test_filename, checksum_type=test_checksum_type,
+                checksum=test_checksum)
+        self.assertTrue(found.count() == 1)
+        # Remove from PackageVersion collection
+        self.pvapi.delete(found[0])
+        # Verify it's deleted
+        found = self.pvapi.packageversion(name=test_pkg_name, epoch=test_epoch, 
+                version=test_version, release=test_release, arch=test_arch, 
+                filename=test_filename, checksum_type=test_checksum_type,
+                checksum=test_checksum)
+        self.assertTrue(found.count() == 0)
+
     def test_package_groups(self):
         pkggroup = self.pgapi.create('test-pkg-group-id', 'test-pkg-group-name', 
                 'test-pkg-group-description')
diff --git a/test/unit/test_comps.py b/test/unit/test_comps.py
new file mode 100644
index 0000000..a868805
--- /dev/null
+++ b/test/unit/test_comps.py
@@ -0,0 +1,192 @@
+#!/usr/bin/python
+#
+# Copyright (c) 2010 Red Hat, Inc.
+#
+#
+# This software is licensed to you under the GNU General Public License,
+# version 2 (GPLv2). There is NO WARRANTY for this software, express or
+# implied, including the implied warranties of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE. You should have received a copy of GPLv2
+# along with this software; if not, see
+# http://www.gnu.org/licenses/old-licenses/gpl-2.0.txt.
+#
+# Red Hat trademarks are not licensed under GPLv2. No permission is
+# granted to use or replicate Red Hat trademarks that are incorporated
+# in this software or its documentation.
+#
+import sys
+import os
+srcdir = os.path.abspath(os.path.dirname(__file__)) + "/../../src"
+sys.path.append(srcdir)
+import unittest
+import logging
+
+import yum
+
+import pulp.util
+import pulp.model
+from pulp.api.repo import RepoApi
+from pulp.repo_sync import BaseSynchronizer
+
+class TestComps(unittest.TestCase):
+
+    def setUp(self):
+        config_file = os.path.join(srcdir, "../etc/pulp/pulp.ini")
+        self.config = pulp.util.loadConfig(config_file)
+        self.rapi = RepoApi(self.config)
+        self.rapi.clean()
+
+    def tearDown(self):
+        self.rapi.clean()
+
+
+    def test_import_groups_data(self):
+        repo = self.rapi.create('test_import_groups_data_id',
+                'test_import_groups_data_id', 'i386', 
+                'yum:http://example.com/')
+        # Parse existing comps.xml
+        compspath = "./data/rhel-i386-server-5/comps.xml"
+        compsfile = open(compspath)
+        base = BaseSynchronizer(self.config)
+        base.import_groups_data(compsfile, repo)
+        # 'repo' object should now contain groups/categories
+        # we need to save it to the db so we can query from it
+        self.rapi.update(repo)
+        # Testing for expected values
+        found = self.rapi.packagegroup(repo['id'], "web-server")
+        self.assertTrue(found != None)
+        self.assertTrue("httpd" in found['mandatory_package_names'])
+        self.assertTrue("mod_auth_kerb" in found['optional_package_names'])
+        self.assertTrue("mod_auth_mysql" in found['optional_package_names'])
+        self.assertTrue("crypto-utils" in found['default_package_names'])
+        self.assertTrue("distcache" in found['default_package_names'])
+        # PackageGroupCategory, look up expected values,
+        found = self.rapi.packagegroupcategory(repo['id'], "BAD_VALUE_NOT_IN_CATEGORY")
+        self.assertTrue(found == None)
+        found = self.rapi.packagegroupcategory(repo['id'], "development")
+        self.assertTrue(found != None)
+
+    def test_basic_comps(self):
+        repo = self.rapi.create('test_comps_id','test_comps_name', 
+            'i386', 'yum:http://example.com/')
+        grp = pulp.model.PackageGroup("groupid1", "groupname1", 
+            "description", "user_visible", "display_order", "default"
+            "langonly")
+        grp['mandatory_package_names'] = ["mandatory_package_name1"]
+        grp['optional_package_names'] = ["optional_package_name1"]
+        grp['default_package_names'] = ["default_package_name1"]
+        grp['conditional_package_names'] = {"pkg1":"value pkg1"}
+        grp['translated_name'] = {"a":"value"}
+        grp['translated_description'] = {"b":"value"}
+        self.rapi.update_packagegroup(repo['id'], grp)
+        found = self.rapi.packagegroup(repo['id'], grp['id'])
+        self.assertTrue(found != None)
+        self.assertTrue(found['name'] == 'groupname1')
+        self.assertTrue("mandatory_package_name1" in found['mandatory_package_names'])
+
+        ctg = pulp.model.PackageGroupCategory("categoryid1", 
+                    "categoryname", "description", "display_order")
+        ctg['packagegroupids'] = ["groupid1"]
+        ctg['translated_name'] = {"a":"name"}
+        ctg['translated_description'] = {"b":"description"}
+        self.rapi.update_packagegroupcategory(repo["id"], ctg)
+        found = self.rapi.packagegroupcategory(repo["id"], ctg["id"]) 
+        self.assertTrue(found != None)
+        self.assertTrue(found["name"] == "categoryname")
+        self.assertTrue("groupid1" in found["packagegroupids"])
+
+    def broken_intend_this_to_run_full_read_write_out_to_xml(self):
+        """
+        Test full cycle of Groups/Categories, import a comps.xml, parse it
+        modify the entries, then write them out to XML
+        """
+        #TODO: Writing to XML is broken 
+        # Parse existing comps.xml
+        compsPath = "./data/rhel-i386-server-5/comps.xml"
+        comps = yum.comps.Comps()
+        comps.add(compsPath)
+        self.assertTrue(len(comps.get_groups()) != 0)
+        self.assertTrue(len(comps.get_categories()) != 0)
+        # Create Groups/Categories from parsed data
+        repo = self.rapi.create('test_comps_id','test_comps_name', 
+                'i386', 'yum:http://example.com/')
+        found = self.rapi.packagegroups(repo['id'])
+        self.assertTrue(len(found) == 0)
+        found = self.rapi.packagegroupcategories(repo['id'])
+        self.assertTrue(len(found) == 0)
+
+        grp_list = []
+        groupids = []
+        for g in comps.get_groups():
+            grp = pulp.model.PackageGroup(g.groupid, g.name, 
+                    g.description, g.user_visible, g.display_order, g.default, 
+                    g.langonly)
+            grp['mandatory_package_names'].extend(g.mandatory_packages.keys())
+            grp['optional_package_names'].extend(g.optional_packages.keys())
+            grp['default_package_names'].extend(g.default_packages.keys())
+            grp['conditional_package_names'] = g.conditional_packages
+            grp['translated_name'] = g.translated_name
+            grp['translated_description'] = g.translated_description
+            grp_list.append(grp)
+            groupids.append(grp['id'])
+        self.rapi.update_packagegroups(repo['id'], grp_list)
+        ctg_list = []
+        categoryids = []
+        for c in comps.get_categories():
+            ctg = pulp.model.PackageGroupCategory(c.categoryid, 
+                    c.name, c.description, c.display_order)
+            groupids = [grp for grp in c.groups]
+            ctg['packagegroupids'].extend(groupids)
+            ctg['translated_name'] = c.translated_name
+            ctg['translated_description'] = c.translated_description
+            ctg_list.append(ctg)
+            categoryids.append(ctg['id'])
+        self.rapi.update_packagegroupcategories(repo['id'], ctg_list)
+        # Lookup data from API calls
+        found = self.rapi.packagegroups(repo['id'])
+        self.assertTrue(len(found) > 0)
+        found = self.rapi.packagegroupcategories(repo['id'])
+        self.assertTrue(len(found) > 0)
+        # PackageGroup, look up expected values, 
+        # good values come from known data in rhel-5 comps.xml
+        found = self.rapi.packagegroup(repo['id'], "BAD_VALUE_NOT_IN_GROUP")
+        self.assertTrue(found == None)
+        found = self.rapi.packagegroup(repo['id'], "web-server")
+        self.assertTrue(found != None)
+        self.assertTrue("httpd" in found['mandatory_package_names'])
+        self.assertTrue("mod_auth_kerb" in found['optional_package_names'])
+        self.assertTrue("mod_auth_mysql" in found['optional_package_names'])
+        self.assertTrue("crypto-utils" in found['default_package_names'])
+        self.assertTrue("distcache" in found['default_package_names'])
+        # PackageGroupCategory, look up expected values,
+        found = self.rapi.packagegroupcategory(repo['id'], "BAD_VALUE_NOT_IN_CATEGORY")
+        self.assertTrue(found == None)
+        found = self.rapi.packagegroupcategory(repo['id'], "development")
+        self.assertTrue(found != None)
+        # Test Removal
+        self.rapi.remove_packagegroup(repo['id'], "web-server")
+        found = self.rapi.packagegroup(repo['id'], "web-server")
+        self.assertTrue(found == None)
+        self.rapi.remove_packagegroupcategory(repo['id'], "development")
+        found = self.rapi.packagegroupcategory(repo['id'], "development")
+        self.assertTrue(found == None)
+
+        newComps = yum.comps.Comps()
+        # Look up categories from a repo
+        ctgs = self.rapi.packagegroupcategories(repo["id"])
+        grps = self.rapi.packagegroups(repo["id"])
+
+        for cid in ctgs:
+            category = self.rapi.translate_packagegroupcategory(ctgs[cid])
+            newComps.add_category(category)
+        for gid in grps:
+            pkggrp = self.rapi.translate_packagegroup(grps[gid])
+            newComps.add_group(pkggrp)
+        # Write back to xml
+        xml = newComps.xml()
+        print "Generated XML = %s" % (xml)
+        self.assertTrue(True)
+
+
+
+

[Date Prev][Date Next]   [Thread Prev][Thread Next]   [Thread Index] [Date Index] [Author Index]