Overhaul the source-package page

This commit addresses a long-standing bug where resolved bugs
disappear completely.  In addition, lts/security archives are no
longer shown separately, and no-dsa is marked explicitly.  The package
vulnerability state is taken from the database, so it is hopefully
quite accurate.

Remove security_db.DB.getBugsForSourcePackage() and replace it with a
global function security_db.getBugsForSourcePackage().  Add additional
named tuples BugForSourcePackage, BugForSourcePackageRelease,
BugsForSourcePackage_internal.

Add yellow CSS style.



git-svn-id: svn+ssh://svn.debian.org/svn/secure-testing@34502 e39458fd-73e7-0310-bf30-c45bca0a0e42

3 files changed, 146 insertions, 73 deletions

diff --git a/bin/tracker_service.py b/bin/tracker_service.py
index a5ae76c60f..db20d60a4b 100644
--- a/bin/tracker_service.py
+++ b/bin/tracker_service.py
@@ -581,6 +581,7 @@ to improve our documentation and procedures, so feedback is welcome.""")])])
 
     def page_source_package(self, path, params, url):
         pkg = path[0]
+        data = security_db.getBugsForSourcePackage(self.db.cursor(), pkg)
 
         def gen_versions():
             for (release, version) in self.db.getSourcePackageVersions(
@@ -590,31 +591,26 @@ to improve our documentation and procedures, so feedback is welcome.""")])])
             for bug in lst:
                 yield self.make_xref(url, bug.bug), bug.description
 
-        suites = ()
-        for (release, version) in self.db.getSourcePackageVersions(
-            self.db.cursor(), pkg):
-                if release not in suites:
-                    suites = suites + (release,)
+        def format_summary_entry(per_release):
+            if per_release is None:
+                return self.make_purple('unknown')
+            if per_release.vulnerable == 1:
+                if per_release.state == 'no-dsa':
+                    return self.make_mouseover(
+                        (self.make_yellow('vulnerable (no DSA)'),),
+                        text=per_release.reason)
+                else:
+                    return self.make_red('vulnerable')
+            if per_release.vulnerable == 2:
+                return self.make_purple('undetermined')
+            assert per_release.vulnerable == 0
+            return self.make_green('fixed')
 
         def gen_summary(bugs):
             for bug in bugs:
-                status = {}
-                for (package, releases, version, vulnerable) \
-                        in self.db.getSourcePackages(self.db.cursor(), bug.bug):
-                    for release in releases:
-                        if package == pkg:
-                            if vulnerable == 1:
-                                status[release] = self.make_red('vulnerable')
-                            elif vulnerable == 2:
-                                status[release] = self.make_purple('undetermined')
-                            else:
-                                status[release] = self.make_green('fixed')
-                status_row = ()
-                for release in suites:
-                    if release in status:
-                        status_row = status_row + (status[release],)
-                    else:
-                        status_row = status_row + (self.make_purple('unknown'),)
+                status_row = tuple(
+                    format_summary_entry(bug.releases.get(rel, None))
+                    for rel in data.all_releases)
                 yield (self.make_xref(url, bug.bug),) + status_row \
                     + (bug.description,)
 
@@ -632,30 +628,21 @@ to improve our documentation and procedures, so feedback is welcome.""")])])
              make_table(gen_versions(), title=H2('Available versions'), caption=('Release', 'Version')),
 
              make_table(
-                 gen_summary(
-                     # open issues
-                     self.db.getBugsForSourcePackage(
-                         self.db.cursor(), pkg, True, False),
-                 ),
+                 gen_summary(data.open),
                  title=H2('Open issues'),
-                 caption=('Bug',) + suites + ('Description',),
+                 caption=('Bug',) + data.all_releases + ('Description',),
                  replacement='No known open issues.'
              ),
 
 
              make_table(
-                 gen_summary(
-                     # open unimportant isues
-                     self.db.getBugsForSourcePackage(
-                         self.db.cursor(), pkg, True, True),
-                 ),
+                 gen_summary(data.unimportant),
                  title=H2('Open unimportant issues'),
-                 caption=('Bug',) + suites + ('Description',),
+                 caption=('Bug',) + data.all_releases + ('Description',),
                  replacement='No known unimportant issues.'
              ),
 
-             make_table(gen_bug_list(self.db.getBugsForSourcePackage
-                                     (self.db.cursor(), pkg, False, True)),
+             make_table(gen_bug_list(data.resolved),
                         title=H2('Resolved issues'),
                         caption=('Bug', 'Description'),
                         replacement='No known resolved issues.'),
@@ -1607,12 +1594,18 @@ Debian bug number.'''),
     def make_red(self, contents):
         return SPAN(contents, _class="red")
 
+    def make_yellow(self, contents):
+        return SPAN(contents, _class="yellow")
+
     def make_purple(self, contents):
 	return SPAN(contents, _class="purple")
 
     def make_green(self, contents):
         return SPAN(contents, _class="green")
 
+    def make_mouseover(self, contents, text):
+        return tag("SPAN", contents, title=text)
+
     def make_dangerous(self, contents):
         return SPAN(contents, _class="dangerous")
 
diff --git a/lib/python/security_db.py b/lib/python/security_db.py
index 2844b2809f..c203125f3e 100644
--- a/lib/python/security_db.py
+++ b/lib/python/security_db.py
@@ -26,12 +26,15 @@ The data is kept in a SQLite 3 database.
 FIXME: Document the database schema once it is finished.
 """
 
+from apt_pkg import version_compare
 import apsw
 import base64
 import bugs
+from collections import namedtuple
 import cPickle
 import cStringIO
 import glob
+import itertools
 import os
 import os.path
 import re
@@ -39,8 +42,6 @@ import sys
 import types
 import zlib
 
-from collections import namedtuple
-
 import debian_support
 import dist_config
 
@@ -100,15 +101,121 @@ class SchemaMismatch(Exception):
 
     The caller is expected to remove and regenerate the database."""
 
-    def getBugsForSourcePackage(self, cursor, pkg, vulnerable, unimportant):
-        """Returns a generator for a list of (BUG, DESCRIPTION) pairs
-        which have the requested status.  Only bugs affecting supported
-        releases are returned."""
-
-# Returned by DB.getBugsForSourcePackage().
+# Returned by getBugsForSourcePackage().
+# all/open/unimportant/resolved are sequences of BugForSourcePackage.
 BugsForSourcePackage = namedtuple(
     "BugsForSourcePackage",
-    "bug description")
+    "all_releases all open unimportant resolved")
+
+# Returned by getBugsForSourcePackage().  releases is a sequence of
+# BugForSourcePackageRelease.  global_state is the aggregated state
+# across all releases (open/resolved/unimportant).
+BugForSourcePackage = namedtuple(
+    "BugForSourcePackage",
+    "bug description global_state releases")
+
+# Returned by getBugsForSourcePackage(). release, subrelease, version
+# come from the source_packages table.  vulnerable comes from
+# source_package_status.  state is open/no-dsa/resolved/unimportant
+# and inferred from vulnerable and package_notes_nodsa.
+BugForSourcePackageRelease = namedtuple(
+    "BugForSourcePackageRelease",
+    "release subrelease version vulnerable state reason")
+
+# Internally used by getBugsForSourcePackage().
+BugsForSourcePackage_internal = namedtuple(
+    "BugsForSourcePackage_internal",
+    "bug_name description release subrelease version vulnerable urgency")
+BugsForSourcePackage_query = \
+"""SELECT bugs.name AS bug_name, bugs.description AS description,
+    sp.release AS release, sp.subrelease AS subrelease, sp.version AS version,
+    st.vulnerable AS vulnerable, st.urgency AS urgency
+  FROM bugs
+  JOIN source_package_status st ON (bugs.name = st.bug_name)
+  JOIN source_packages sp ON (st.package = sp.rowid)
+  WHERE sp.name = ?
+  AND (bugs.name LIKE 'CVE-%' OR bugs.name LIKE 'TEMP-%')
+  ORDER BY bugs.name DESC, sp.release"""
+# Sort order is important for the groupby operation below.
+
+def getBugsForSourcePackage(cursor, pkg):
+    data = [BugsForSourcePackage_internal(*row) for row in
+            cursor.execute(BugsForSourcePackage_query, (pkg,))]
+    # Filter out special releases such as backports.
+    data = [row for row in data
+            if debian_support.internRelease(row.release) is not None]
+    # Obtain the set of releases actually in used, by canonical order.
+    all_releases = tuple(sorted(set(row.release for row in data),
+                                   key = debian_support.internRelease))
+    # dict from (bug_name, release) to the no-dsa reason/comment string.
+    no_dsas = {}
+    for bug_name, release, reason in cursor.execute(
+            """SELECT bug_name, release, comment FROM package_notes_nodsa
+            WHERE package = ?""", (pkg,)):
+        no_dsas[(bug_name, release)] = reason
+
+    all_bugs = []
+    # Group by bug name.
+    for bug_name, data in itertools.groupby(data,
+                                            lambda row: row.bug_name):
+        data = tuple(data)
+        description = data[0].description
+        open_seen = False
+        unimportant_seen = False
+        releases = {}
+        # Group by release.
+        for release, data1 in itertools.groupby(data, lambda row: row.release):
+            data1 = tuple(data1)
+            # The best row is the row with the highest version number.
+            # If there is a tie, the empty subrelease row wins.
+            best_row = data1[0]
+            for row in data1[1:]:
+                cmpresult = version_compare(row.version, best_row.version)
+                if cmpresult > 0 \
+                   or (cmpresult == 0 and row.subrelease == ''):
+                    best_row = row
+            reason = None
+
+            # Compute state.  Update state-seen flags for global state
+            # determination.
+            if best_row.vulnerable:
+                if best_row.urgency == 'unimportant':
+                    state = 'unimportant'
+                    unimportant_seen = True
+                else:
+                    open_seen = True
+                    reason = no_dsas.get((bug_name, best_row.release), None)
+                    if reason is not None:
+                        state = 'no-dsa'
+                    else:
+                        state = 'open'
+            else:
+                state = 'resolved'
+
+            bug = BugForSourcePackageRelease(
+                best_row.release, best_row.subrelease, best_row.version,
+                best_row.vulnerable, state, reason)
+            releases[best_row.release] = bug
+
+        # Compute global_state.
+        if open_seen:
+            global_state = 'open'
+        elif unimportant_seen:
+            global_state = 'unimportant'
+        else:
+            global_state = 'resolved'
+
+        all_bugs.append(BugForSourcePackage(bug_name, description,
+                                            global_state, releases))
+
+    # Split all_bugs into per-state sequences.
+    per_state = {'all_releases': all_releases,
+                 'all': all_bugs}
+    for state in ("open", "unimportant", "resolved"):
+        per_state[state] = tuple(bug for bug in all_bugs
+                                 if bug.global_state == state)
+
+    return BugsForSourcePackage(**per_state)
 
 # Returned by DB.getDSAsForSourcePackage().
 DSAsForSourcePackage = namedtuple(
@@ -1735,34 +1842,6 @@ class DB:
             (pkg,))
         return flag
 
-    def getBugsForSourcePackage(self, cursor, pkg, vulnerable, unimportant):
-        """Returns a generator for BugsForSourcePackage named tuples which
-        have the requested status.  Only bugs affecting supported
-        releases are returned.
-        """
-        for row in cursor.execute(
-            """SELECT DISTINCT name, description
-            FROM (SELECT bugs.name AS name, bugs.description AS description,
-            MAX(st.vulnerable
-            AND COALESCE((SELECT st2.vulnerable FROM source_packages AS sp2,
-            source_package_status AS st2
-            WHERE sp2.name = sp.name AND sp2.release = sp.release
-            AND ( sp2.subrelease = 'security' OR sp2.subrelease = 'lts' ) AND sp2.archive = sp.archive
-            AND st2.package = sp2.rowid AND st2.bug_name = st.bug_name
-            ORDER BY st2.vulnerable DESC), 1)) AS vulnerable,
-            st.urgency = 'unimportant' OR NOT vulnerable AS unimportant
-            FROM source_packages AS sp, source_package_status AS st, bugs
-            WHERE sp.name = ?
-            AND sp.release IN ('squeeze', 'wheezy', 'jessie', 'stretch', 'sid')
-            AND sp.subrelease <> 'security' AND sp.subrelease <> 'lts'
-            AND st.package = sp.rowid
-            AND bugs.name = st.bug_name
-            AND (bugs.name LIKE 'CVE-%' OR bugs.name LIKE 'TEMP-%')
-            GROUP BY bugs.name, bugs.description, sp.name)
-            WHERE vulnerable = ? AND unimportant = ?
-            ORDER BY name DESC""", (pkg, vulnerable, unimportant)):
-            yield BugsForSourcePackage(*row)
-
     def getDSAsForSourcePackage(self, cursor, package):
         for row in cursor.execute(
             """SELECT bugs.name, bugs.description
diff --git a/static/style.css b/static/style.css
index c3850a0ab3..fb6d52085d 100644
--- a/static/style.css
+++ b/static/style.css
@@ -202,6 +202,7 @@ label[rel="extra"]:last-child {
 }
 
 span.red { color: red; }
+span.yellow { color: #c0c000; }
 span.purple { color: purple; }
 span.green { color: green; }
 span.dangerous { color: orange; }