sectracker.analyzers.vulnerabilities(): extract fixed package information

git-svn-id: svn+ssh://svn.debian.org/svn/secure-testing@14659 e39458fd-73e7-0310-bf30-c45bca0a0e42

3 files changed, 148 insertions, 5 deletions

diff --git a/doc/python-format.txt b/doc/python-format.txt
index b0f0a91613..a2d2edff90 100644
--- a/doc/python-format.txt
+++ b/doc/python-format.txt
@@ -111,19 +111,28 @@ These act just as flags; no additional data is present.
 
 # Derived vulnerability information
 
-These are contained in a list of info objects:
+sectracker.analyzers.vulnerabilities() computes fixed versions for
+bug/package pairs.  These are returned in a list of vulnerability
+objects:
 
-* info.bug: name of the bug (potentially auto-generated)
+* vuln.bug: name of the bug (potentially auto-generated)
 
-* info.package: name of the package
+* vuln.package: name of the package
 
-* info.fixed: fixed version in unstable (a string), or None (no fix
+* vuln.fixed: fixed version in unstable (a string), or None (no fix
   available) or True (all versions fixed)
 
-* info.fixed_other: a tuple, containing other fixed versions (which
+* vuln.fixed_other: a tuple, containing other fixed versions (which
   are less than the unfixed unstable version, but nevertheless known
   not to be vulnerable)
 
 In itself, this data is not very illuminating, but comparision with
 other information sources can be used to detect vulnerable installed
 packages, generate bug and distribution overview pages etc.
+
+This computation is in a separate pass because packages are sometimes
+propagated between releases/distributions in the Debian archive.  The
+returned data only contains plain versions, disregarding the source,
+so further processing can correctly handle package propagation (in the
+sense that if a bug was fixed in one place, all propagated copies are
+also fixed).
diff --git a/lib/python/sectracker/analyzers.py b/lib/python/sectracker/analyzers.py
index c3fdd365bc..03b5ad1abd 100644
--- a/lib/python/sectracker/analyzers.py
+++ b/lib/python/sectracker/analyzers.py
@@ -18,6 +18,8 @@
 import apt_pkg as _apt_pkg
 import re as _re
 
+from sectracker.xcollections import namedtuple as _namedtuple
+
 # vercmp is the Debian version comparison algorithm
 _apt_pkg.init()
 try:
@@ -96,3 +98,129 @@ def copysources(bugdb, diag):
                     else:
                         result[target] = set((copy_source,))
     return result
+
+Vulnerability = _namedtuple("Vulnerability", "bug package fixed fixed_other")
+
+def vulnerabilities(bugdb, copysrc, versions, diag):
+    """Determine vulnerable versions.
+
+    Returns named tuples with fields "bug", "package", "fixed",
+    "fixed_other"."""
+
+    assert "sid" in versions # should come from extractversions()
+
+    def buildpackages1(bug):
+        packages = {}
+        for ann in bug.annotations:
+            if ann.type == "package":
+                if ann.package not in packages:
+                    packages[ann.package] = {}
+                pkg = packages[ann.package]
+                pkg[ann.release] = (bug, ann)
+        return packages
+
+    def buildpackages(bug):
+        packages = buildpackages1(bug)
+        if bug.header.name not in copysrc:
+            return packages
+        copiers = [buildpackages1(bugdb[b])
+                   for b in copysrc[bug.header.name]]
+        for c in copiers:
+            for pname, creleases in c.items():
+                if pname not in packages:
+                    packages[pname] = creleases
+                    continue
+                preleases = packages[pname]
+                for rel, cbugann in creleases.items():
+                    if rel in preleases:
+                        pbug, pann = preleases[rel]
+                        cbug, cann = cbugann
+                        if pbug is bug:
+                            # Never override annotations in the CVE file.
+                            continue
+                        diag.warning("annotation on %s overridden"
+                                     % pbug.header.name,
+                                     file=pbug.file, line=pann.line)
+                        diag.warning("  by annotation on %s via %s" 
+                                     % (cbug.header.name, bug.header.name),
+                                     file=cbug.file, line=cann.line)
+                    preleases[rel] = cbugann
+        return packages
+
+    def latentlyvulnerable(packages):
+        for pname, preleases in packages.items():
+            if None not in preleases:
+                diag.warning("package %s is latently vulnerable in unstable"
+                             % pname,
+                             file=bug.file, line=bug.header.line)
+                for (pbug, pann) in preleases.values():
+                    diag.warning("%s vulnerability in %s"
+                                 % (pname, pann.release),
+                                 file=pbug.file, line=pann.line)
+
+    def convertversion(ann):
+        # None: unfixed
+        # version-string: fixed in that version
+        # True: never vulnerable
+        if ann.urgency == "unimportant" or ann.kind == "not-affected":
+            return True
+        ver = ann.version
+        if ver is not None:
+            return ver
+        return None
+        
+    def extractunstable(preleases):
+        if None not in preleases:
+            return None
+        return convertversion(preleases[None][1])
+
+    def getversions(pname, version_items=versions.items()):
+        # FIXME: extractversions() should return flipped nested
+        # dictionary, to make the following easier.
+        for rel, pkgs in version_items:
+            if rel == "sid":
+                continue
+            if pname in pkgs:
+                for ver in pkgs[pname]:
+                    yield rel, ver
+
+    result = []
+    for bug in bugdb.values():
+        if _re_source.match(bug.header.name):
+            # Copy sources are dealt with by copying their
+            # annotations.
+            continue
+
+        packages = buildpackages(bug)
+        latentlyvulnerable(packages)
+
+        for pname, preleases in packages.items():
+            unstable_fixed = extractunstable(preleases)
+            if unstable_fixed is True:
+                # unstable was never vulnerable, which overrides
+                # all other annoations
+                continue
+
+            other_versions = set()
+            for rel, ver in getversions(pname):
+                if unstable_fixed is not None \
+                        and vercmp(ver, unstable_fixed) >= 0:
+                    # This version is already covered by the
+                    # unstable fix.
+                    continue
+                if rel in preleases:
+                    relver = convertversion(preleases[rel][1])
+                    if relver is None:
+                        continue
+                    if relver is True:
+                        # FIXME? should not happen because the
+                        # vulnerable must have been present in
+                        # unstable at some point
+                        other_versions.add(ver)
+                        continue
+                    if vercmp(ver, relver) >= 0:
+                        continue
+                    other_versions.add(ver)
+            result.append(Vulnerability(bug.header.name, pname,
+                                        unstable_fixed, other_versions))
+    return result
diff --git a/lib/python/sectracker_test/test_analyzers.py b/lib/python/sectracker_test/test_analyzers.py
index ef4f89f6a4..6fb3d048f3 100644
--- a/lib/python/sectracker_test/test_analyzers.py
+++ b/lib/python/sectracker_test/test_analyzers.py
@@ -49,6 +49,12 @@ copysrc = copysources(bugdb, diag)
 assert "CVE-2008-0225" in copysrc
 assert "DSA-1472-1" in copysrc["CVE-2008-0225"]
 
+# vulnerabilities
+vdb = vulnerabilities(bugdb, copysrc, rpv, diag)
+if False:
+    for v in vdb:
+        print v
+
 for err in diag.messages():
     print "%s:%d: %s: %s" % (err.file, err.line, err.level, err.message)
 assert not diag.messages()