starting to work with pacman version numbers, added preliminary version of the dependency integrity linter check

10 files changed, 229 insertions, 20 deletions

diff --git a/README.rst b/README.rst
index 285dd3d..5bcb9c8 100644
--- a/README.rst
+++ b/README.rst
@@ -191,3 +191,17 @@ pkgfile_invalid_signature
 this check validates the package signature against the pacman keyrings. It
 reports an issue whenever a package is signed by an unknown key, that is not
 part of the keyring, or by a key that has expired.
+
+package dependency checks
+-------------------------
+
+a number of checks verifying the satisfiability of dependencies in the repos
+
+unsatisfiable_depends
+~~~~~~~~~~~~~~~~~~~~~
+
+for the list of entries in the repo.db's check that all entries in the
+depends() array of the package are satisfiable with the provides() entries of
+the packages in the repositories core, extra, community, and the ones
+configured in CONFIG.parabola.repos. This check reports an issue whenever a
+depends() entry is found that is not satisfiable.
diff --git a/parabola_repolint/__main__.py b/parabola_repolint/__main__.py
index 317495d..7900fc9 100644
--- a/parabola_repolint/__main__.py
+++ b/parabola_repolint/__main__.py
@@ -72,6 +72,7 @@ def checked_main(args):
     linter.run_checks()
 
     res = linter.format()
+    logging.info(res)
 
     if CONFIG.notify.etherpad_url:
         etherpad_replace(res)
diff --git a/parabola_repolint/linter_checks/dependencies.py b/parabola_repolint/linter_checks/dependencies.py
new file mode 100644
index 0000000..a5e76c2
--- /dev/null
+++ b/parabola_repolint/linter_checks/dependencies.py
@@ -0,0 +1,88 @@
+'''
+linter checks for repo dependency integrity
+'''
+
+import operator
+
+from parabola_repolint.repocache import PkgVersion
+from parabola_repolint.linter import LinterIssue, LinterCheckBase, LinterCheckType
+
+
+class UnsatisfiableDepends(LinterCheckBase):
+    '''
+  for the list of entries in the repo.db's check that all entries in the
+  depends() array of the package are satisfiable with the provides() entries of
+  the packages in the repositories core, extra, community, and the ones
+  configured in CONFIG.parabola.repos. This check reports an issue whenever a
+  depends() entry is found that is not satisfiable.
+'''
+
+    name = 'unsatisfiable_depends'
+    check_type = LinterCheckType.PKGENTRY
+
+    header = 'repo.db entries with unsatisfiable depends'
+
+    def check(self, pkgentry):
+        ''' run the check '''
+        repos = list(self._cache.repos.values()) + list(self._cache.arch_repos.values())
+        missing = []
+
+        for depend in pkgentry.depends:
+            matches = self._repos_contain_depends(depend, repos, pkgentry.arch)
+            if not matches:
+                missing.append(depend)
+
+        if missing:
+            raise LinterIssue('%s (%s)', pkgentry, ','.join(missing))
+
+    def _repos_contain_depends(self, depend, repos, arch):
+        ''' test whether a dependency is provided by the list of repos '''
+        version = None
+        splits = ['==', '>=', '<=', '>', '<', '=']
+        for split in splits:
+            if split in depend:
+                depend, version = depend.split(split)
+                version = (split, PkgVersion(version))
+                break
+
+        candidates = []
+        for repo in repos:
+            candidates += repo.provides_cache.get(arch, {}).get(depend, [])
+
+        matches = []
+        for candidate in candidates:
+            if self._candidate_contains_depends(depend, candidate, version):
+                matches.append(candidate)
+
+        return matches
+
+    def _candidate_contains_depends(self, depend, candidate, version):
+        ''' test whether a dependency is provided by a pkgentry '''
+        if version is None:
+            return True
+
+        provides = candidate.provides.union([candidate.pkgname])
+        pversion = candidate.pkgver
+
+        for provide in provides:
+            assert '<' not in provide
+            assert '>' not in provide
+
+            if '=' not in provide:
+                continue
+
+            provide, _pversion = provide.split('=')
+            if provide == depend:
+                pversion = PkgVersion(_pversion)
+                break
+
+        operators = {
+            '==': operator.eq,
+            '=':  operator.eq,
+            '>=': operator.ge,
+            '<=': operator.le,
+            '>':  operator.gt,
+            '<':  operator.lt,
+        }
+
+        return operators[version[0]](pversion, version[1])
diff --git a/parabola_repolint/linter_checks/package_signature.py b/parabola_repolint/linter_checks/package_signature.py
index 18edd23..7c62adc 100644
--- a/parabola_repolint/linter_checks/package_signature.py
+++ b/parabola_repolint/linter_checks/package_signature.py
@@ -9,7 +9,6 @@ import datetime
 from parabola_repolint.linter import LinterIssue, LinterCheckBase, LinterCheckType
 
 
-# pylint: disable=no-self-use
 class SigningKeyExpiry(LinterCheckBase):
     '''
   for the list of signing keys and subkeys in parabola.gpg that are used to sign
@@ -25,6 +24,7 @@ class SigningKeyExpiry(LinterCheckBase):
 
     header = 'signing keys expired or about to expire'
 
+    # pylint: disable=no-self-use
     def check(self, key):
         ''' run the check '''
         if not key['packages']:
@@ -48,7 +48,6 @@ class SigningKeyExpiry(LinterCheckBase):
             raise LinterIssue('%s: %s (signed %i)', key['keyid'], reason, len(key['packages']))
 
 
-# pylint: disable=no-self-use
 class MasterKeyExpiry(LinterCheckBase):
     '''
   for the list of master keys in parabola.gpg, check whether they are expired, or
@@ -63,6 +62,7 @@ class MasterKeyExpiry(LinterCheckBase):
 
     header = 'master keys expired or about to expire'
 
+    # pylint: disable=no-self-use
     def check(self, key):
         ''' run the check '''
         if not key['expires']:
@@ -82,7 +82,6 @@ class MasterKeyExpiry(LinterCheckBase):
             raise LinterIssue('%s: %s', key['keyid'], reason)
 
 
-# pylint: disable=no-self-use
 class PkgEntrySignatureMismatch(LinterCheckBase):
     '''
   for the list of entries in the repo.db's, check whether the signature stored in
@@ -97,6 +96,7 @@ class PkgEntrySignatureMismatch(LinterCheckBase):
 
     header = 'repo.db entries with mismatched signing keys'
 
+    # pylint: disable=no-self-use
     def check(self, pkgentry):
         ''' run the check '''
         if not pkgentry.pkgfile:
@@ -127,7 +127,6 @@ class PkgEntrySignatureMismatch(LinterCheckBase):
             raise LinterIssue('%s: %s != %s', pkgentry, key1, key2)
 
 
-# pylint: disable=no-self-use
 class PkgFileInvalidSignature(LinterCheckBase):
     '''
   this check validates the package signature against the pacman keyrings. It
diff --git a/parabola_repolint/linter_checks/package_validity.py b/parabola_repolint/linter_checks/package_validity.py
index 469f93f..3a3b10e 100644
--- a/parabola_repolint/linter_checks/package_validity.py
+++ b/parabola_repolint/linter_checks/package_validity.py
@@ -7,7 +7,6 @@ import hashlib
 from parabola_repolint.linter import LinterIssue, LinterCheckBase, LinterCheckType
 
 
-# pylint: disable=no-self-use
 class PkgFileMissingBuildinfo(LinterCheckBase):
     '''
   for the list of built packages in the repos, check whether each has an embedded
@@ -21,6 +20,7 @@ class PkgFileMissingBuildinfo(LinterCheckBase):
 
     header = 'built packages with no .BUILDINFO file'
 
+    # pylint: disable=no-self-use
     def check(self, pkgentry):
         ''' run the check '''
         if not pkgentry.pkgfile:
@@ -32,7 +32,6 @@ class PkgFileMissingBuildinfo(LinterCheckBase):
             raise LinterIssue('%s (built %s)', pkgfile, builddate)
 
 
-# pylint: disable=no-self-use
 class PkgFileMissingPkginfo(LinterCheckBase):
     '''
   for the list of built packages in the repos, check whether each has an embedded
@@ -45,6 +44,7 @@ class PkgFileMissingPkginfo(LinterCheckBase):
 
     header = 'built packages with no .PKGINFO file'
 
+    # pylint: disable=no-self-use
     def check(self, pkgfile):
         ''' run the check '''
         if not pkgfile.pkginfo:
@@ -66,6 +66,7 @@ class PkgFileBadPkgbuildDigest(LinterCheckBase):
 
     header = 'built packages with mismatched PKGBUILD digests'
 
+    # pylint: disable=no-self-use
     def check(self, pkgentry):
         ''' run the check '''
         if not pkgentry.pkgfile:
diff --git a/parabola_repolint/linter_checks/pkgbuild_validity.py b/parabola_repolint/linter_checks/pkgbuild_validity.py
index 9289120..5396be9 100644
--- a/parabola_repolint/linter_checks/pkgbuild_validity.py
+++ b/parabola_repolint/linter_checks/pkgbuild_validity.py
@@ -9,7 +9,6 @@ from parabola_repolint.config import CONFIG
 KNOWN_ARCHES = CONFIG.parabola.arches
 
 
-# pylint: disable=no-self-use
 class InvalidPkgbuild(LinterCheckBase):
     '''
   this check tests for syntactical problems with the PKGBUILD file itself,
@@ -23,13 +22,13 @@ class InvalidPkgbuild(LinterCheckBase):
 
     header = 'invalid PKGBUILDs'
 
+    # pylint: disable=no-self-use
     def check(self, pkgbuild):
         ''' run the check '''
         if not pkgbuild.valid:
             raise LinterIssue('%s', pkgbuild)
 
 
-# pylint: disable=no-self-use
 class UnsupportedArches(LinterCheckBase):
     '''
   this check tests for PKGBUILD files that list archictectures in the `arch`
@@ -46,6 +45,7 @@ class UnsupportedArches(LinterCheckBase):
 
     header = 'PKGBUILDs with unsupported arches'
 
+    # pylint: disable=no-self-use
     def check(self, pkgbuild):
         ''' run the check '''
         if not pkgbuild.valid:
diff --git a/parabola_repolint/linter_checks/repo_integrity.py b/parabola_repolint/linter_checks/repo_integrity.py
index 8f4e26e..5ba4ac6 100644
--- a/parabola_repolint/linter_checks/repo_integrity.py
+++ b/parabola_repolint/linter_checks/repo_integrity.py
@@ -5,7 +5,6 @@ these are linter checks for PKGBUILD / .pkg.tar.xz / repo.db entry integrity
 from parabola_repolint.linter import LinterIssue, LinterCheckBase, LinterCheckType
 
 
-# pylint: disable=no-self-use
 class PkgBuildMissingPkgEntries(LinterCheckBase):
     '''
   for the list of packages produced by the pkgbuild for the supported arches,
@@ -19,6 +18,7 @@ class PkgBuildMissingPkgEntries(LinterCheckBase):
 
     header = 'PKGBUILDs with missing entries in repo.db'
 
+    # pylint: disable=no-self-use
     def check(self, pkgbuild):
         ''' run the check '''
         missing = []
@@ -30,7 +30,6 @@ class PkgBuildMissingPkgEntries(LinterCheckBase):
             raise LinterIssue('%s (%s)', pkgbuild, ','.join(missing))
 
 
-# pylint: disable=no-self-use
 class PkgBuildDuplicatePkgEntries(LinterCheckBase):
     '''
   for the list of packages produced by the pkgbuild for the supported arches,
@@ -44,6 +43,7 @@ class PkgBuildDuplicatePkgEntries(LinterCheckBase):
 
     header = 'PKGBUILDs with duplicate entries in repo.db'
 
+    # pylint: disable=no-self-use
     def check(self, pkgbuild):
         ''' run the check '''
         duplicate = []
@@ -57,7 +57,6 @@ class PkgBuildDuplicatePkgEntries(LinterCheckBase):
             raise LinterIssue('%s (%s)', pkgbuild, ','.join(duplicate))
 
 
-# pylint: disable=no-self-use
 class PkgBuildMissingPkgFiles(LinterCheckBase):
     '''
   for the list of packages produced by the pkgbuild for the supported arches,
@@ -70,6 +69,7 @@ class PkgBuildMissingPkgFiles(LinterCheckBase):
 
     header = 'PKGBUILDs with missing built packages'
 
+    # pylint: disable=no-self-use
     def check(self, pkgbuild):
         ''' run the check '''
         missing = []
@@ -81,7 +81,6 @@ class PkgBuildMissingPkgFiles(LinterCheckBase):
             raise LinterIssue('%s (%s)', pkgbuild, ','.join(missing))
 
 
-# pylint: disable=no-self-use
 class PkgEntryMissingPkgbuild(LinterCheckBase):
     '''
   for the list of entries in a repo.db, check whether a valid PKGBUILD exists
@@ -94,13 +93,13 @@ class PkgEntryMissingPkgbuild(LinterCheckBase):
 
     header = 'repo.db entries with no valid PKGBUILD'
 
+    # pylint: disable=no-self-use
     def check(self, pkgentry):
         ''' run the check '''
         if not pkgentry.pkgbuilds:
             raise LinterIssue('%s', pkgentry)
 
 
-# pylint: disable=no-self-use
 class PkgEntryDuplicatePkgbuilds(LinterCheckBase):
     '''
   for the list of entries in a repo.db, check whether more than one valid
@@ -113,6 +112,7 @@ class PkgEntryDuplicatePkgbuilds(LinterCheckBase):
 
     header = 'repo.db entries with duplicate PKGBUILDs'
 
+    # pylint: disable=no-self-use
     def check(self, pkgentry):
         ''' run the check '''
         if len(pkgentry.pkgbuilds) > 1:
@@ -122,7 +122,6 @@ class PkgEntryDuplicatePkgbuilds(LinterCheckBase):
             raise LinterIssue('%s (%s)', pkgentry, ','.join(duplicates))
 
 
-# pylint: disable=no-self-use
 class PkgEntryMissingPkgFile(LinterCheckBase):
     '''
   for the list of entries in a repo.db, check wether a built package exists that
@@ -135,13 +134,13 @@ class PkgEntryMissingPkgFile(LinterCheckBase):
 
     header = 'repo.db entries with no valid built package'
 
+    # pylint: disable=no-self-use
     def check(self, pkgentry):
         ''' run the check '''
         if not pkgentry.pkgfile:
             raise LinterIssue('%s', pkgentry)
 
 
-# pylint: disable=no-self-use
 class PkgFileMissingPkgbuild(LinterCheckBase):
     '''
   for the list of built packages, check whether a valid PKGBUILD exists that
@@ -154,6 +153,7 @@ class PkgFileMissingPkgbuild(LinterCheckBase):
 
     header = 'built packages with no valid PKGBUILD'
 
+    # pylint: disable=no-self-use
     def check(self, pkgfile):
         ''' run the check '''
         if not pkgfile.pkgbuilds:
@@ -161,7 +161,6 @@ class PkgFileMissingPkgbuild(LinterCheckBase):
             raise LinterIssue('%s (built %s)', pkgfile, builddate)
 
 
-# pylint: disable=no-self-use
 class PkgFileDuplicatePkgbuilds(LinterCheckBase):
     '''
   for the list of built packages, check whether more than one valid PKGBUILD
@@ -174,6 +173,7 @@ class PkgFileDuplicatePkgbuilds(LinterCheckBase):
 
     header = 'built packages with duplicate PKGBUILDs'
 
+    # pylint: disable=no-self-use
     def check(self, pkgfile):
         ''' run the check '''
         if len(pkgfile.pkgbuilds) > 1:
@@ -183,7 +183,6 @@ class PkgFileDuplicatePkgbuilds(LinterCheckBase):
             raise LinterIssue('%s (%s)', pkgfile, ','.join(duplicates))
 
 
-# pylint: disable=no-self-use
 class PkgFileMissingPkgEntry(LinterCheckBase):
     '''
   for the list of built packages, check wether a repo.db entry exists that refers
@@ -196,6 +195,7 @@ class PkgFileMissingPkgEntry(LinterCheckBase):
 
     header = 'built packages without a referring repo.db entry'
 
+    # pylint: disable=no-self-use
     def check(self, pkgfile):
         ''' run the check '''
         if not pkgfile.pkgentries:
@@ -203,7 +203,6 @@ class PkgFileMissingPkgEntry(LinterCheckBase):
             raise LinterIssue('%s (built %s)', pkgfile, builddate)
 
 
-# pylint: disable=no-self-use
 class PkgFileDuplicatePkgEntries(LinterCheckBase):
     '''
   for the list of built packages, check that at most one repo.db entry exists
@@ -216,6 +215,7 @@ class PkgFileDuplicatePkgEntries(LinterCheckBase):
 
     header = 'built packages with duplicate referring repo.db entries'
 
+    # pylint: disable=no-self-use
     def check(self, pkgfile):
         ''' run the check '''
         if len(pkgfile.pkgentries) > 1:
diff --git a/parabola_repolint/linter_checks/repo_redundancy.py b/parabola_repolint/linter_checks/repo_redundancy.py
index 4450e36..5d034a0 100644
--- a/parabola_repolint/linter_checks/repo_redundancy.py
+++ b/parabola_repolint/linter_checks/repo_redundancy.py
@@ -5,7 +5,6 @@ these are checks for things in the repo that are redundant and can go away.
 from parabola_repolint.linter import LinterIssue, LinterCheckBase, LinterCheckType
 
 
-# pylint: disable=no-self-use
 class RedundantPkgEntryPCR(LinterCheckBase):
     '''
   for the list of entries in the parabola repos, check whether package is
diff --git a/parabola_repolint/repocache.py b/parabola_repolint/repocache.py
index dae9bbb..3f27938 100644
--- a/parabola_repolint/repocache.py
+++ b/parabola_repolint/repocache.py
@@ -10,12 +10,75 @@ import logging
 import datetime
 
 import sh
+from pyalpm import vercmp
 from xdg import BaseDirectory
 
 from parabola_repolint.config import CONFIG
 from parabola_repolint.gnupg import GPG_PACMAN
 
 
+class PkgVersion():
+    ''' represent a package version number and its components '''
+
+    def __init__(self, pkgver):
+        ''' constructor '''
+        self._version_str = pkgver
+
+        # as per libalpm/version.c:parseEVR
+        self._epoch = '0'
+        self._pkgrel = None
+
+        if ':' in pkgver:
+            self._epoch, pkgver = pkgver.split(':', 1)
+        if '-' in pkgver:
+            pkgver, self._pkgrel = pkgver.rsplit('-', 1)
+
+        self._pkgver = pkgver
+
+    @property
+    def epoch(self):
+        ''' produce the epoch part of the version '''
+        return self._epoch
+
+    @property
+    def pkgver(self):
+        ''' produce the pkgver part of the version '''
+        return self._pkgver
+
+    @property
+    def pkgrel(self):
+        ''' produce the pkgrel part of the version '''
+        return self._pkgrel
+
+    def _cmp(self, other):
+        ''' compare two version strings and indicate their relationship '''
+        return vercmp(str(self), str(other))
+
+    def __eq__(self, other):
+        ''' indicate whether two versions can be considered equal '''
+        return self._cmp(other) == 0
+
+    def __lt__(self, other):
+        ''' indicate whether one version is less than another '''
+        return self._cmp(other) < 0
+
+    def __gt__(self, other):
+        ''' indicate whether one version is greater than another '''
+        return self._cmp(other) > 0
+
+    def __le__(self, other):
+        ''' indicate whether one version is less or equal to another '''
+        return self._cmp(other) <= 0
+
+    def __ge__(self, other):
+        ''' indicate whether one version is greater or equal to another '''
+        return self._cmp(other) >= 0
+
+    def __repr__(self):
+        ''' produce the original string representation of the version '''
+        return self._version_str
+
+
 BUILDINFO_VALUE = [
     'format',
     'pkgname',
@@ -301,11 +364,26 @@ class PkgEntry():
         return self._data['NAME']
 
     @property
+    def pkgver(self):
+        ''' produce the pkgver of the package '''
+        return PkgVersion(self._data['VERSION'])
+
+    @property
     def pgpsig(self):
         ''' produce the base64 encoded pgp signature of the package '''
         return self._data['PGPSIG']
 
     @property
+    def provides(self):
+        ''' produce the names provided by the package '''
+        return set(self._data.get('PROVIDES', '').split())
+
+    @property
+    def depends(self):
+        ''' produce the install time dependencies of the package '''
+        return set(self._data.get('DEPENDS', '').split())
+
+    @property
     def arch(self):
         ''' produce the architecture of the package '''
         return self._repoarch
@@ -589,6 +667,7 @@ class Repo():
 
         self._pkgentries = []
         self._pkgentries_cache = {}
+        self._provides_cache = {}
         self._load_pkgentries()
 
         logging.info('%s pkgentries: %i', name, len(self._pkgentries))
@@ -596,6 +675,8 @@ class Repo():
             out.write(json.dumps(self._pkgentries, indent=4, sort_keys=True, default=str))
         with open(os.path.join(self._pkgentries_dir, '.pkgentries_cache'), 'w') as out:
             out.write(json.dumps(self._pkgentries_cache, indent=4, sort_keys=True, default=str))
+        with open(os.path.join(self._pkgentries_dir, '.provides_cache'), 'w') as out:
+            out.write(json.dumps(self._provides_cache, indent=4, sort_keys=True, default=str))
 
         self._pkgfiles = []
         self._load_pkgfiles()
@@ -630,6 +711,11 @@ class Repo():
         return self._pkgentries_cache
 
     @property
+    def provides_cache(self):
+        ''' produce the list of pkgentries by provides entries '''
+        return self._provides_cache
+
+    @property
     def pkgfiles(self):
         ''' produce the list of pkg.tar.xz files in the repo '''
         return self._pkgfiles
@@ -701,6 +787,21 @@ class Repo():
                     self._pkgentries_cache[pkgentry.arch][pkgentry.pkgname] = []
                 self._pkgentries_cache[pkgentry.arch][pkgentry.pkgname].append(pkgentry)
 
+                for provides in pkgentry.provides.union([pkgentry.pkgname]):
+                    if pkgentry.arch not in self._provides_cache:
+                        self._provides_cache[pkgentry.arch] = {}
+
+                    splits = ['==', '>=', '<=', '>', '<', '=']
+                    for split in splits:
+                        if split in provides:
+                            provides = provides.split(split)[0]
+                            break
+
+                    if provides not in self._provides_cache[pkgentry.arch]:
+                        self._provides_cache[pkgentry.arch][provides] = []
+                    self._provides_cache[pkgentry.arch][provides].append(pkgentry)
+
+
     def _load_pkgfiles(self):
         ''' load the pkg.tar.xz files from the repo '''
         i = 0
@@ -724,7 +825,7 @@ class Repo():
         return '[%s]' % self._name
 
 
-ARCH_REPOS = {'core', 'extra', 'community'}
+ARCH_REPOS = ['core', 'extra', 'community']
 
 
 class RepoCache():
@@ -763,6 +864,11 @@ class RepoCache():
         return [p for r in self._repos.values() for p in r.pkgfiles]
 
     @property
+    def repos(self):
+        ''' produce repo objects for the parabola repos under test '''
+        return self._repos
+
+    @property
     def arch_repos(self):
         ''' produce repo objects for core, extra and community '''
         return self._arch_repos
diff --git a/setup.py b/setup.py
index 89e79e8..27f7a3b 100644
--- a/setup.py
+++ b/setup.py
@@ -25,6 +25,7 @@ setup(
     },
 
     install_requires=[
+        'pyalpm',
         'pyyaml',
         'sh',
         'pyxdg',