[Date Prev][Date Next]   [Thread Prev][Thread Next]   [Thread Index] [Date Index] [Author Index]

Re: Improving availability and guaranteeing integrity in ISO downloads




On 9-jun-2007, at 14:39, Matt Domsch wrote:

On Sat, Jun 09, 2007 at 08:25:12AM -0400, Jesse Keating wrote:
On Friday 08 June 2007 18:31:53 Anthony Bryan wrote:
What can we do to make this happen? Is this the type of thing that's
easier for the maintainer of MirrorManager to add, or should we supply
a patch?

It would be best to open a dialog with Matt Domsch about this (:

I replied on fedora-infrastructure-list just a bit ago.  I'm happy to
see this functionality get added to mirrormanager, and am happy to
review patches as well. :-)  The trick will be having a python module
that, given a list of URL/country tuples, generates the XML pages.
Which probably isn't that hard, really.

Thanks,
Matt

File attached :-)

My Python skills suck, so feel free to improve it. It doesn't implement all fields in the MetaLink specification, but it should do the job.

#!/usr/bin/python
# vim: ts=4 sw=4 et
"""
    This module creates a metalink file from a list of urls
    Author: ruben fedoraproject org
    License: GPL

    Metalink, an Open Standard, makes downloading easier by
    bundling the various ways (FTP/HTTP/P2P) to get files into
    one format for easier downloads, as well as bundling
    checksums to automatically verify completed downloads.
    Currently the following applications support this file:

  - aria2 (Unix) [ http://aria2.sourceforge.net/ ]
  - GetRight 6 (Windows) [ http://www.getright.com/ ]
  - Speed Download (Mac) [ http://www.yazsoft.com/ ]
  - wxDownload Fast (All)[ http://dfast.sourceforge.net/ ]

  For further information please visit http://www.metalinker.org/
"""
import xml.dom.minidom
import xml.dom.ext
import sys

class MetaLink(object):
    """
    Create a metalink file from MetaLinkFile objects
    """

    def __init__(self):
        self.__doc = xml.dom.minidom.Document()
        self.__root_element = self.__doc.createElementNS('http://www.metalinker.org/', 'metalink')
        self.__doc.appendChild(self.__root_element)
        self.__files_element = self.__doc.createElement('files')
        self.__root_element.appendChild(self.__files_element)


    def getorigin(self):
        return self.__root_element.getAttribute('origin')
    def setorigin(self, origin):
        self.__root_element.setAttribute('origin', origin)
    origin = property(getorigin, setorigin, None,
        """
        The original location of this .metalink file.
        If type is dynamic then this is the location where an updated version of the file will be found
        """
    )

    def gettype(self):
        return self.__root_element.getAttribute('type')
    def settype(self, type):
        self.__root_element.setAttribute('type', type)
    type = property(gettype, settype, None,
        """
        type = dynamic or static. Static files are not updated
        """
    )

    def getpubdate(self):
        return self.__root_element.getAttribute('pubdate')
    def setpubdate(self, pubdate):
        self.__root_element.setAttribute('pubdate', pubdate)
    pubdate = property(getpubdate, setpubdate, None,
        """
        pubdate = Original date and time of publishing of the .metalink file.
        All Metalink date-times conform to the Date and Time Specification of RFC 822,
        with the exception that the year may be expressed with two characters or four characters (four preferred).
        Example: Mon, 15 May 2006 00:00:01 GMT
        """
    )

    def getgenerator(self):
        return self.__root_element.getAttribute('generator')
    def setgenerator(self, generator):
        self.__root_element.setAttribute('generator', generator)
    generator = property(getgenerator, setgenerator, None,
        """
        The application used to generate the .metalink file
        """
    )

    def getrefreshdate(self):
        return self.__root_element.getAttribute('refreshdate')
    def setrefreshdate(self, refreshdate):
        self.__root_element.setAttribute('refreshdate', refreshdate)
    refreshdate = property(getrefreshdate, setrefreshdate, None,
        """
        The date and time when a dynamic file has been updated
        """
    )

    def getpublisher(self):
        name = self.__root_element.getElementsByTagName('publisher')[0].childNodes[0].childNodes[0].data
        url = self.__root_element.getElementsByTagName('publisher')[0].childNodes[1].childNodes[0].data
        return (name, url)
    def setpublisher(self, publisher):
        name, url = publisher
        publisher_element = self.__doc.createElement('publisher')
        self.__root_element.insertBefore(publisher_element, self.__files_element)

        publisher_name = self.__doc.createElement('name')
        publisher_element.appendChild(publisher_name)
        text = self.__doc.createTextNode(name)
        publisher_name.appendChild(text)

        publisher_url = self.__doc.createElement('url')
        publisher_element.appendChild(publisher_url)
        text = self.__doc.createTextNode(url)
        publisher_url.appendChild(text)
    publisher = property(getpublisher, setpublisher, None,
        """
        This is the publisher of the file the Metalink points to
        """
    )

    def getlicense(self):
        name = self.__root_element.getElementsByTagName('license')[0].childNodes[0].childNodes[0].data
        url = self.__root_element.getElementsByTagName('license')[0].childNodes[1].childNodes[0].data
        return (name, url)
    def setlicense(self, license):
        name, url = license
        license_element = self.__doc.createElement('license')
        self.__root_element.insertBefore(license_element, self.__files_element)

        license_name = self.__doc.createElement('name')
        license_element.appendChild(license_name)
        text = self.__doc.createTextNode(name)
        license_name.appendChild(text)

        license_url = self.__doc.createElement('url')
        license_element.appendChild(license_url)
        text = self.__doc.createTextNode(url)
        license_url.appendChild(text)
    license = property(getlicense, setlicense, None,
        """
        The license the file was released under. Such as: Shareware, Commercial, GPL, BSD, Creative Commons, etc
        """
    )

    
    def addfile(self,MetaLinkFile):
        """
        Add a MetaLinkFile to the file list
        """
        file_element = self.__doc.createElement('file')

        # filename
        file_element.setAttribute('name', MetaLinkFile.name)
        self.__files_element.appendChild(file_element)

        # size
        size_element = self.__doc.createElement('size')
        file_element.appendChild(size_element)
        text = self.__doc.createTextNode(MetaLinkFile.size)
        size_element.appendChild(text)    

        # verification hash
        verification_element = self.__doc.createElement('verifaction');
        file_element.appendChild(verification_element)
        hash_element = self.__doc.createElement('hash')
        hash_element.setAttribute('type', MetaLinkFile.hash[0])
        verification_element.appendChild(hash_element)
        chksum_element = self.__doc.createTextNode(MetaLinkFile.hash[1])
        hash_element.appendChild(chksum_element)

        # resources
        resources_element = self.__doc.createElement('resources')
        file_element.appendChild(resources_element)
        for url in MetaLinkFile.urls:
            url_element = self.__doc.createElement('url')        
            url_element.setAttribute('type', url['type'])
            # optional elements
            if url.has_key('preference'):
                url_element.setAttribute('preference', url['preference'])
            if url.has_key('location'):
                url_element.setAttribute('location', url['location'])
            resources_element.appendChild(url_element)
            link_element = self.__doc.createTextNode(url['url'])
            url_element.appendChild(link_element)

    def writexml(self, writer):
        """
        Write the metalink file to a writer (anything which accepts write())
        """
        self.__doc.writexml(writer, addindent="    ", newl='\n', encoding='UTF-8')


class MetaLinkFile(object):
    """
    Represents a file with a couple of different urls
    """
    def __init__(self):
        self.__urls = []

    def getname(self):
        return self.__name
    def setname(self, name):
        self.__name = name
    filename = property(getname, setname, None,
        """
        Name of the file
        """
    )

    def gethash(self):
        return self.__hash
    def sethash(self, hash):
        self.__hash = hash
    hash = property(gethash, sethash, None,
        """
        Tuple of type and chksum, i.e ('sha1' , '1234')
        """
    )

    def getsize(self):
        return self.__size
    def setsize(self, size):
        self.__size = size
    size = property(getsize, setsize, None,
        """
        Filesize in bytes
        """
    )

    def addurl(self, url):
        """
        Accepts a dict with type, url, location and preference keys
        """
        if url['url'].endswith('.torrent'):
            url['type'] = 'bittorrent'
        elif url['url'].startswith('ftp://'):
            url['type'] = 'ftp'
        elif url['url'].startswith('http://'):
            url['type'] = 'http'
        elif url['url'].startswith('rsync://'):
            url['type'] = 'rsync'
        elif url['url'].startswith('magnet:'):
            url['type'] = 'magnet'
        elif url['url'].startswith('ed2k://'):
            url['type'] = 'ed2k'
        else:
            raise NotImplementedError, "Unknown url type"
        self.__urls.append(url)

    def geturls(self):
        return self.__urls
    def seturls(self, urls):
        self.__urls = urls
    urls = property(geturls, seturls)
    

if __name__ == '__main__':
    ml = MetaLink()
    ml.origin = 'http://mirrors.fedoraproject.org/F-7-i386-DVD.iso.metalink'
    ml.publisher = ('Fedora', 'http://fedoraproject.org')
    ml.license = ('Fedora 7 EULA', 'http://fedoraproject.org/wiki/Legal/Licenses/EULA7/')
    ml.type = 'static'
    ml.pubdate = '2007-05-27-16:42:00'
    ml.refreshdate = '2007-05-27-16:42:00'
    ml.generator = 'Fedora MetaLink Generator'

    ml_file = MetaLinkFile()
    ml_file.name = "F-7-i386-DVD.iso"
    ml_file.hash = ('sha1', '96b13dbbc9f3bc569ddad9745f64b9cdb43ea9ae')
    ml_file.size = "28311552"
    ml_file.addurl({'location': 'AT', 'url': 'ftp://ftp.uni-klu.ac.at/linux/fedora/linux/core/releases/7/Fedora/i386/iso/F-7-i386-DVD.iso.torrent'})
    ml_file.addurl({'location': 'AT', 'url': 'http://ftp.univie.ac.at/systems/linux/fedora/releases/7/Fedora/i386/iso/F-7-i386-DVD.iso'})
    ml_file.addurl({'location': 'AU', 'url': 'http://mirror.optus.net/fedora/linux/releases/7/Fedora/i386/iso/F-7-i386-DVD.iso', 'preference': '100'})     

    ml.addfile(ml_file)

    ml.writexml(sys.stdout)


[Date Prev][Date Next]   [Thread Prev][Thread Next]   [Thread Index] [Date Index] [Author Index]