aptsources/distinfo.py

#  distinfo.py - provide meta information for distro repositories
#
#  Copyright (c) 2005 Gustavo Noronha Silva <[email protected]>
#  Copyright (c) 2006-2007 Sebastian Heinlein <[email protected]>
#
#  Authors: Gustavo Noronha Silva <[email protected]>
#           Sebastian Heinlein <[email protected]>
#
#  This program is free software; you can redistribute it and/or
#  modify it under the terms of the GNU General Public License as
#  published by the Free Software Foundation; either version 2 of the
#  License, or (at your option) any later version.
#
#  This program is distributed in the hope that it will be useful,
#  but WITHOUT ANY WARRANTY; without even the implied warranty of
#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#  GNU General Public License for more details.
#
#  You should have received a copy of the GNU General Public License
#  along with this program; if not, write to the Free Software
#  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
#  USA

from __future__ import print_function

import csv
import errno
import logging
import os
from subprocess import Popen, PIPE
import re

import apt_pkg

from apt_pkg import gettext as _


def _expand_template(template, csv_path):
    """Expand the given template.

    A template file consists of a header, followed by paragraphs
    of templated suites, followed by a footer. A templated suite
    is any paragraph where the Suite field contains {.

    This function expands all templated suites using the information
    found in the CSV file supplied by distro-info-data.

    It yields lines of template info.
    """

    known_suites = set()

    # Copy out any header, and gather all hardcoded suites
    with apt_pkg.TagFile(template) as tmpl:
        for section in tmpl:
            if "X-Exclude-Suites" in section:
                known_suites.update(section["X-Exclude-Suites"].split(", "))
            if "Suite" in section:
                if "{" in section["Suite"]:
                    break

                known_suites.add(section["Suite"])

            yield from str(section).splitlines()
        else:
            # We did not break, so we did copy all of them
            return

        for section in tmpl:
            if "Suite" in section:
                known_suites.add(section["Suite"])

    with open(csv_path) as csv_object:
        releases = reversed(list(csv.DictReader(csv_object)))

    # Perform template substitution on the middle of the list
    for rel in releases:
        if rel["series"] in known_suites:
            continue
        yield ""
        rel["version"] = rel["version"].replace(" LTS", "")
        with apt_pkg.TagFile(template) as tmpl:
            for section in tmpl:
                # Only work on template sections, this skips head and tails
                if "Suite" not in section or "{" not in section["Suite"]:
                    continue
                if "X-Version" in section:
                    # Version requirements. Maybe should be made nicer
                    ver = rel["version"]
                    if any(
                            (field.startswith("le") and
                             apt_pkg.version_compare(field[3:], ver) < 0) or
                            (field.startswith("ge") and
                             apt_pkg.version_compare(field[3:], ver) > 0)
                            for field in section["X-Version"].split(", ")):
                        continue

                for line in str(section).format(**rel).splitlines():
                    if line.startswith("X-Version"):
                        continue
                    yield line

    # Copy out remaining suites
    with apt_pkg.TagFile(template) as tmpl:
        # Skip the head again, we don't want to copy it twice
        for section in tmpl:
            if "Suite" in section and "{" in section["Suite"]:
                break

        for section in tmpl:
            # Ignore any template parts and copy the rest out,
            # this is the inverse of the template substitution loop
            if "Suite" in section and "{" in section["Suite"]:
                continue

            yield from str(section).splitlines()


class Template(object):

    def __init__(self):
        self.name = None
        self.child = False
        self.parents = []          # ref to parent template(s)
        self.match_name = None
        self.description = None
        self.base_uri = None
        self.type = None
        self.components = []
        self.children = []
        self.match_uri = None
        self.mirror_set = {}
        self.distribution = None
        self.available = True
        self.official = True

    def has_component(self, comp):
        ''' Check if the distribution provides the given component '''
        return comp in (c.name for c in self.components)

    def is_mirror(self, url):
        ''' Check if a given url of a repository is a valid mirror '''
        proto, hostname, dir = split_url(url)
        if hostname in self.mirror_set:
            return self.mirror_set[hostname].has_repository(proto, dir)
        else:
            return False


class Component(object):

    def __init__(self, name, desc=None, long_desc=None, parent_component=None):
        self.name = name
        self.description = desc
        self.description_long = long_desc
        self.parent_component = parent_component

    def get_parent_component(self):
        return self.parent_component

    def set_parent_component(self, parent):
        self.parent_component = parent

    def get_description(self):
        if self.description_long is not None:
            return self.description_long
        elif self.description is not None:
            return self.description
        else:
            return None

    def set_description(self, desc):
        self.description = desc

    def set_description_long(self, desc):
        self.description_long = desc

    def get_description_long(self):
        return self.description_long


class Mirror(object):
    ''' Storage for mirror related information '''

    def __init__(self, proto, hostname, dir, location=None):
        self.hostname = hostname
        self.repositories = []
        self.add_repository(proto, dir)
        self.location = location

    def add_repository(self, proto, dir):
        self.repositories.append(Repository(proto, dir))

    def get_repositories_for_proto(self, proto):
        return [r for r in self.repositories if r.proto == proto]

    def has_repository(self, proto, dir):
        if dir is None:
            return False
        for r in self.repositories:
            if r.proto == proto and dir in r.dir:
                return True
        return False

    def get_repo_urls(self):
        return [r.get_url(self.hostname) for r in self.repositories]

    def get_location(self):
        return self.location

    def set_location(self, location):
        self.location = location


class Repository(object):

    def __init__(self, proto, dir):
        self.proto = proto
        self.dir = dir

    def get_info(self):
        return self.proto, self.dir

    def get_url(self, hostname):
        return "%s://%s/%s" % (self.proto, hostname, self.dir)


def split_url(url):
    ''' split a given URL into the protocoll, the hostname and the dir part '''
    split = re.split(":*\\/+", url, maxsplit=2)
    while len(split) < 3:
        split.append(None)
    return split


class DistInfo(object):

    def __init__(self, dist=None, base_dir="/usr/share/python-apt/templates"):
        self.metarelease_uri = ''
        self.templates = []
        self.arch = apt_pkg.config.find("APT::Architecture")

        location = None
        match_loc = re.compile(r"^#LOC:(.+)$")
        match_mirror_line = re.compile(
            r"^(#LOC:.+)|(((http)|(ftp)|(rsync)|(file)|(mirror)|(https))://"
            r"[A-Za-z0-9/\.:\-_@]+)$")
        #match_mirror_line = re.compile(r".+")

        if not dist:
            try:
                dist = Popen(["lsb_release", "-i", "-s"],
                             universal_newlines=True,
                             stdout=PIPE).communicate()[0].strip()
            except (OSError, IOError) as exc:
                if exc.errno != errno.ENOENT:
                    logging.warning(
                        'lsb_release failed, using defaults:' % exc)
                dist = "Debian"

        self.dist = dist

        map_mirror_sets = {}

        dist_fname = "%s/%s.info" % (base_dir, dist)
        csv_fname = "/usr/share/distro-info/{}.csv".format(dist.lower())

        template = None
        component = None
        for line in _expand_template(dist_fname, csv_fname):
            tokens = line.split(':', 1)
            if len(tokens) < 2:
                continue
            field = tokens[0].strip()
            value = tokens[1].strip()
            if field == 'ChangelogURI':
                self.changelogs_uri = _(value)
            elif field == 'MetaReleaseURI':
                self.metarelease_uri = value
            elif field == 'Suite':
                self.finish_template(template, component)
                component = None
                template = Template()
                template.name = value
                template.distribution = dist
                template.match_name = "^%s$" % value
            elif field == 'MatchName':
                template.match_name = value
            elif field == 'ParentSuite':
                template.child = True
                for nanny in self.templates:
                    # look for parent and add back ref to it
                    if nanny.name == value:
                        template.parents.append(nanny)
                        nanny.children.append(template)
            elif field == 'Available':
                template.available = apt_pkg.string_to_bool(value)
            elif field == 'Official':
                template.official = apt_pkg.string_to_bool(value)
            elif field == 'RepositoryType':
                template.type = value
            elif field == 'BaseURI' and not template.base_uri:
                template.base_uri = value
            elif field == 'BaseURI-%s' % self.arch:
                template.base_uri = value
            elif field == 'MatchURI' and not template.match_uri:
                template.match_uri = value
            elif field == 'MatchURI-%s' % self.arch:
                template.match_uri = value
            elif (field == 'MirrorsFile' or
                  field == 'MirrorsFile-%s' % self.arch):
                # Make the path absolute.
                value = os.path.isabs(value) and value or \
                        os.path.abspath(os.path.join(base_dir, value))
                if value not in map_mirror_sets:
                    mirror_set = {}
                    try:
                        with open(value) as value_f:
                            mirror_data = list(filter(
                                match_mirror_line.match,
                                [x.strip() for x in value_f]))
                    except Exception:
                        print("WARNING: Failed to read mirror file")
                        mirror_data = []
                    for line in mirror_data:
                        if line.startswith("#LOC:"):
                            location = match_loc.sub(r"\1", line)
                            continue
                        (proto, hostname, dir) = split_url(line)
                        if hostname in mirror_set:
                            mirror_set[hostname].add_repository(proto, dir)
                        else:
                            mirror_set[hostname] = Mirror(
                                proto, hostname, dir, location)
                    map_mirror_sets[value] = mirror_set
                template.mirror_set = map_mirror_sets[value]
            elif field == 'Description':
                template.description = _(value)
            elif field == 'Component':
                if (component and not
                        template.has_component(component.name)):
                    template.components.append(component)
                component = Component(value)
            elif field == 'CompDescription':
                component.set_description(_(value))
            elif field == 'CompDescriptionLong':
                component.set_description_long(_(value))
            elif field == 'ParentComponent':
                component.set_parent_component(value)
        self.finish_template(template, component)
        template = None
        component = None

    def finish_template(self, template, component):
        " finish the current tempalte "
        if not template:
            return
        # reuse some properties of the parent template
        if template.match_uri is None and template.child:
            for t in template.parents:
                if t.match_uri:
                    template.match_uri = t.match_uri
                    break
        if template.mirror_set == {} and template.child:
            for t in template.parents:
                if t.match_uri:
                    template.mirror_set = t.mirror_set
                    break
        if component and not template.has_component(component.name):
            template.components.append(component)
            component = None
        # the official attribute is inherited
        for t in template.parents:
            template.official = t.official
        self.templates.append(template)


if __name__ == "__main__":
    d = DistInfo("Ubuntu", "/usr/share/python-apt/templates")
    logging.info(d.changelogs_uri)
    for template in d.templates:
        logging.info("\nSuite: %s" % template.name)
        logging.info("Desc: %s" % template.description)
        logging.info("BaseURI: %s" % template.base_uri)
        logging.info("MatchURI: %s" % template.match_uri)
        if template.mirror_set != {}:
            logging.info("Mirrors: %s" % list(template.mirror_set.keys()))
        for comp in template.components:
            logging.info(" %s -%s -%s" % (comp.name,
                                          comp.description,
                                          comp.description_long))
        for child in template.children:
            logging.info("  %s" % child.description)