diff options
Diffstat (limited to 'lib')
| -rw-r--r-- | lib/debian-releases.mk | 9 | ||||
| -rw-r--r-- | lib/python/sectracker/analyzers.py | 6 | ||||
| -rw-r--r-- | lib/python/sectracker/parsers.py | 92 | ||||
| -rw-r--r-- | lib/python/sectracker/xpickle.py | 2 | ||||
| -rw-r--r-- | lib/python/sectracker_test/test_parsers.py | 55 | ||||
| -rw-r--r-- | lib/python/security_db.py | 94 |
6 files changed, 180 insertions, 78 deletions
diff --git a/lib/debian-releases.mk b/lib/debian-releases.mk index eb03a874da..52e238adee 100644 --- a/lib/debian-releases.mk +++ b/lib/debian-releases.mk @@ -7,7 +7,7 @@ endef MAIN_RELEASES = $(call get_config, '.distributions | to_entries[] | select(.value.release) | .key') SECURITY_RELEASES = $(filter-out sid, $(MAIN_RELEASES)) -BACKPORT_RELEASES = $(SECURITY_RELEASES) +BACKPORT_RELEASES = $(filter-out buster, $(SECURITY_RELEASES)) # Define the variables for the release on the main mirror define add_main_release = @@ -15,6 +15,11 @@ $(1)_MIRROR = $$(MIRROR) $(1)_DIST = $(1) $(1)_ARCHS = $(call get_config, '.distributions.$(1).architectures[]') $(1)_RELEASE = $(1) +ifneq (,$(filter jessie stretch buster bullseye,$(1))) +$(1)_SECTIONS = main contrib non-free +else +$(1)_SECTIONS = main contrib non-free non-free-firmware +endif $(1)_SUBRELEASE = RELEASES += $(1) endef @@ -34,6 +39,7 @@ $(1)_security_DIST = $(1)-security endif $(1)_security_ARCHS = $$($(1)_ARCHS) $(1)_security_RELEASE = $(1) +$(1)_security_SECTIONS = $$($(1)_SECTIONS) $(1)_security_SUBRELEASE = security RELEASES += $(1)_security endef @@ -45,6 +51,7 @@ $(1)_backports_MIRROR = $$(MIRROR) $(1)_backports_DIST = $(1)-backports $(1)_backports_ARCHS = $$($(1)_ARCHS) $(1)_backports_RELEASE = $(1)-backports +$(1)_backports_SECTIONS = $$($(1)_SECTIONS) $(1)_backports_SUBRELEASE = RELEASES += $(1)_backports endef diff --git a/lib/python/sectracker/analyzers.py b/lib/python/sectracker/analyzers.py index 386af47974..641227dd1a 100644 --- a/lib/python/sectracker/analyzers.py +++ b/lib/python/sectracker/analyzers.py @@ -30,14 +30,14 @@ def mergelists(listfiles, diag): in diag.""" result = {} for listfile in listfiles: - for bug in listfile.list: + for bug in listfile: header = bug.header name = header.name if name in result: diag.error("duplicate bug %r" % name, - file=header.file, line=header.header.line) + file=bug.file, line=header.line) diag.error("location of previous bug", - file=result[name].file, line=result[name].line) + file=result[name].file, line=result[name].header.line) continue result[name] = bug return result diff --git a/lib/python/sectracker/parsers.py b/lib/python/sectracker/parsers.py index 26bf59bf10..6b42d4a417 100644 --- a/lib/python/sectracker/parsers.py +++ b/lib/python/sectracker/parsers.py @@ -17,7 +17,11 @@ # along with this program; if not, write to the Free Software # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA +from dataclasses import dataclass +import typing +import traceback import re +import sys from sys import intern import debian_support @@ -26,7 +30,9 @@ from collections import namedtuple as _namedtuple import sectracker.xpickle as _xpickle import sectracker.diagnostics -FORMAT = "4" +FORMAT = "5" + +_debug_enabled = False def _sortedtuple(seq): l = list(seq) @@ -71,15 +77,41 @@ def sourcepackages(name, f): data[pkg_name] = SourcePackage(pkg_name, pkg_version, pkg_binary) return data -FlagAnnotation = _namedtuple("FlagAnnotation", "line type") -StringAnnotation = _namedtuple("StringAnnotation", - "line type description") -XrefAnnotation = _namedtuple("XrefAnnotation", "line type bugs") -PackageAnnotation = _namedtuple( - "PackageAnnotation", - "line type release package kind version description flags") -PackageBugAnnotation = _namedtuple("PackageBugAnnotation", "bug") -PackageUrgencyAnnotation = _namedtuple("PackageUrgencyAnnotation", "severity") +@dataclass +class FlagAnnotation: + line: int + type: str + +@dataclass +class StringAnnotation: + line: int + type: str + description: str + +@dataclass +class XrefAnnotation: + line: int + type: str + bugs: typing.List[str] + +@dataclass +class PackageAnnotation: + line: int + type: str + release: str + package: str + kind: str + version: str + description: str + flags: list + +@dataclass +class PackageBugAnnotation: + bug: int + +@dataclass +class PackageUrgencyAnnotation: + severity: str def _annotationdispatcher(): # Parser for inner annotations, like (bug #1345; low) @@ -156,7 +188,7 @@ def _annotationdispatcher(): ) elif kind in pseudo_struct: flags = parseinner(diag, inner) - if kind == "itp" and not inner[1]: + if kind == "itp" and not [flag for flag in flags if isinstance(flag, PackageBugAnnotation)]: diag.error("<itp> needs Debian bug reference") return PackageAnnotation( line=diag.line(), @@ -169,14 +201,14 @@ def _annotationdispatcher(): flags=flags, ) else: - diag.error("invalid pseudo-version: " + repr(version)) + diag.error("invalid pseudo-version: " + repr(kind)) return None @_regexpcase.rule(r'\{(.*)\}') def xref(groups, diag): - x = tuple(groups[0].strip().split()) + x = groups[0].strip().split() if x: - return XrefAnnotation(diag.line(), "xref", x) + return XrefAnnotation(line=diag.line(), type="xref", bugs=list(x)) else: diag.error("empty cross-reference") return None @@ -191,9 +223,17 @@ def _annotationdispatcher(): default=lambda text, diag: diag.error("invalid annotation")) _annotationdispatcher = _annotationdispatcher() -List = _namedtuple("List", "list messages") -Bug = _namedtuple("Bug", "file header annotations") -Header = _namedtuple("Header", "line name description") +@dataclass +class Header: + line: int + name: str + description: str + +@dataclass +class Bug: + file: str + header: Header + annotations: list # TODO: use a list of annotations def _parselist(path, f, parseheader, finish): lineno = 0 @@ -248,7 +288,13 @@ def _parselist(path, f, parseheader, finish): if header is not None: bugs.append(finish(header, headerlineno, anns, diag)) - return List(tuple(bugs), diag.messages()) + + if _debug_enabled: + for m in diag.messages(): + sys.stderr.write(str(m) + "\n") + print("%s:%d: %s: %s" % (m.file, m.line, m.level, m.message)) + + return bugs @_xpickle.loader("CVE" + FORMAT) def cvelist(path, f): @@ -268,7 +314,7 @@ def cvelist(path, f): return (name, desc) def finish(header, headerlineno, anns, diag): name, desc = header - return Bug(path, Header(headerlineno, name, desc), tuple(anns)) + return Bug(path, Header(headerlineno, name, desc), list(anns)) return _parselist(path, f, parseheader, finish) def writecvelist(data, f): @@ -348,7 +394,7 @@ def dsalist(path, f): def finish(header, headerlineno, anns, diag): d, m, y, name, desc = header _checkrelease(anns, diag, "DSA") - return Bug(path, Header(headerlineno, name, None), tuple(anns)) + return Bug(path, Header(headerlineno, name, None), list(anns)) return _parselist(path, f, parseheader, finish) @_xpickle.loader("DTSA" + FORMAT) @@ -365,7 +411,7 @@ def dtsalist(path, f): def finish(header, headerlineno, anns, diag): d, m, y, name, desc = header _checkrelease(anns, diag, "DTSA") - return Bug(path, Header(headerlineno, name, None), tuple(anns)) + return Bug(path, Header(headerlineno, name, None), list(anns)) return _parselist(path, f, parseheader, finish) @_xpickle.loader("DLA" + FORMAT) @@ -381,7 +427,7 @@ def dlalist(path, f): def finish(header, headerlineno, anns, diag): d, m, y, name, desc = header _checkrelease(anns, diag, "DLA") - return Bug(path, Header(headerlineno, name, None), tuple(anns)) + return Bug(path, Header(headerlineno, name, None), list(anns)) return _parselist(path, f, parseheader, finish) @_xpickle.loader("EXT" + FORMAT) @@ -397,5 +443,5 @@ def extadvlist(path, f): def finish(header, headerlineno, anns, diag): d, m, y, name, desc = header _checkrelease(anns, diag, "EXT") - return Bug(path, Header(headerlineno, name, None), tuple(anns)) + return Bug(path, Header(headerlineno, name, None), list(anns)) return _parselist(path, f, parseheader, finish) diff --git a/lib/python/sectracker/xpickle.py b/lib/python/sectracker/xpickle.py index d3324825ce..13fa8bb82e 100644 --- a/lib/python/sectracker/xpickle.py +++ b/lib/python/sectracker/xpickle.py @@ -63,7 +63,7 @@ def _wraploader(typ, parser): try: with open(path + EXTENSION, "rb") as f: return (_pickle.load(f), True) - except (EOFError, IOError, _pickle.PickleError): + except (AttributeError, EOFError, IOError, _pickle.PickleError): return (None, False) def check(data, st): diff --git a/lib/python/sectracker_test/test_parsers.py b/lib/python/sectracker_test/test_parsers.py index 8cdd141a47..4c724ebced 100644 --- a/lib/python/sectracker_test/test_parsers.py +++ b/lib/python/sectracker_test/test_parsers.py @@ -25,85 +25,78 @@ assert "bash" in o assert o["bash"].name == "bash" assert "bash" in o["bash"].binary -safeunlink("../../data/CVE/list" + EXTENSION) -o = cvelist("../../data/CVE/list") -for err in o.messages: - print("%s:%d: %s: %s" % (err.file, err.line, err.level, err.message)) +p._debug_enabled = True safeunlink("../../data/DSA/list" + EXTENSION) -o = dsalist("../../data/DSA/list") -for err in o.messages: - print("%s:%d: %s: %s" % (err.file, err.line, err.level, err.message)) +dsalist("../../data/DSA/list") safeunlink("../../data/DTSA/list" + EXTENSION) -o = dtsalist("../../data/DTSA/list") -for err in o.messages: - print("%s:%d: %s: %s" % (err.file, err.line, err.level, err.message)) +dtsalist("../../data/DTSA/list") safeunlink("../../data/DLA/list" + EXTENSION) -o = dlalist("../../data/DLA/list") -for err in o.messages: - print("%s:%d: %s: %s" % (err.file, err.line, err.level, err.message)) +dlalist("../../data/DLA/list") Message = sectracker.diagnostics.Message for (line, res, xmsgs) in [ (' - foo <unfixed>', PackageAnnotation(17, "package", None, "foo", "unfixed", None, - None, None, (), False), ()), + None, []), ()), (' - foo', PackageAnnotation(17, "package", None, "foo", "unfixed", None, - None, None, (), False), ()), + None, []), ()), (' [lenny] - foo <unfixed>', PackageAnnotation(17, "package", "lenny", "foo", "unfixed", None, - None, None, (), False), ()), + None, []), ()), (' [lenny] - foo <undetermined> (bug #1234)', PackageAnnotation(17, "package", "lenny", "foo", "undetermined", - None, None, None, (1234,), False), ()), + None, None, [PackageBugAnnotation(1234)]), ()), (' [lenny] - foo <itp> (bug #1234)', PackageAnnotation(17, "package", "lenny", "foo", "itp", None, - None, None, (1234,), False), ()), + None, [PackageBugAnnotation(1234)]), ()), (' [lenny] - foo <itp>', PackageAnnotation(17, "package", "lenny", "foo", "itp", None, - None, None, (), False), + None, []), (Message("CVE", 17, "error", "<itp> needs Debian bug reference"),)), (' [lenny] - foo 1.0', PackageAnnotation(17, "package", "lenny", "foo", "fixed", "1.0" , - None, None, (), False), ()), + None, []), ()), (' [lenny] - foo <unfixed> (bug filed)', PackageAnnotation(17, "package", "lenny", "foo", "unfixed", None, - None, None, (), True), ()), + None, []), + (Message("CVE", 17, "error", + "invalid inner annotation: 'bug filed'"),)), (' [lenny] - foo <unfixed> (bug filed; bug #1234)', PackageAnnotation(17, "package", "lenny", "foo", "unfixed", None, - None, None, (1234,), False), + None, [PackageBugAnnotation(1234)]), (Message("CVE", 17, "error", - "'bug filed' and bug numbers listed"),)), + "invalid inner annotation: 'bug filed'"),)), (' [lenny] - foo <unfixed> (low)', PackageAnnotation(17, "package", "lenny", "foo", "unfixed", None, - None, "low", (), False), ()), + None, [PackageUrgencyAnnotation("low")]), ()), (' [lenny] - foo <unfixed> (low; low)', PackageAnnotation(17, "package", "lenny", "foo", "unfixed", None, - None, "low", (), False), - (Message("CVE", 17, "error", "duplicate flag: 'low'"),)), + None, [PackageUrgencyAnnotation("low")]), + (Message("CVE", 17, "error", "duplicate urgency: 'low'"),)), (' [lenny] - foo <unfixed> (bug #1234; garbled)', PackageAnnotation(17, "package", "lenny", "foo", "unfixed", None, - None, None, (1234,), False), + None, [PackageBugAnnotation(1234)]), (Message("CVE", 17, "error", "invalid inner annotation: 'garbled'"),)), (' [lenny] - foo <no-dsa> (explanation goes here)', PackageAnnotation(17, "package", "lenny", "foo", "no-dsa", None, - "explanation goes here", None, (), False), ()), + "explanation goes here", []), ()), (' [lenny] - foo <end-of-life> (explanation goes here)', PackageAnnotation(17, "package", "lenny", "foo", "end-of-life", - None, "explanation goes here", None, (), False), + None, "explanation goes here", []), ()), (' [lenny] - foo <not-affected> (explanation goes here)', PackageAnnotation(17, "package", "lenny", "foo", "not-affected", None, - "explanation goes here", None, (), False), ()), + "explanation goes here", []), ()), ('\t{CVE-2009-1234 CVE-2009-1235}', XrefAnnotation(17, "xref", - tuple("CVE-2009-1234 CVE-2009-1235".split())), + ["CVE-2009-1234", "CVE-2009-1235"]), ()), ('\t{}', None, (Message("CVE", 17, "error", "empty cross-reference"),)), diff --git a/lib/python/security_db.py b/lib/python/security_db.py index c4f163cb7a..d02c803d56 100644 --- a/lib/python/security_db.py +++ b/lib/python/security_db.py @@ -420,6 +420,10 @@ class DB: cursor.execute( "CREATE TABLE removed_packages (name TEXT NOT NULL PRIMARY KEY)") + # This table is used to keep the list of source packages, for which the filing of a bug is not required. + cursor.execute( + "CREATE TABLE ignored_packages (name TEXT NOT NULL PRIMARY KEY)") + cursor.execute( """CREATE TABLE nvd_data (cve_name TEXT NOT NULL PRIMARY KEY, @@ -908,19 +912,29 @@ class DB: def clear_db(cleared=[False]): # Avoid clearing the database multiple times. if cleared[0]: + if self.verbose: + print(" finished (already cleared)") return else: + if self.verbose: + print(" clearing database") cleared[0] = True - cursor.execute("DELETE FROM debian_bugs") - cursor.execute("DELETE FROM bugs") - cursor.execute("DELETE FROM package_notes") - cursor.execute("DELETE FROM bugs_notes") - cursor.execute("DELETE FROM bugs_xref") - cursor.execute("DELETE FROM package_notes_nodsa") - cursor.execute("DELETE FROM removed_packages") - cursor.execute("DELETE FROM next_point_update") + tables = ['debian_bugs', 'bugs', 'package_notes', 'bugs_notes', 'bugs_xref', 'package_notes_nodsa', 'ignored_packages', 'removed_packages', 'next_point_update'] + # clean up all tables + for table in tables: + # check first, whether the table exists + try: + cursor.execute(f"SELECT * FROM {table} LIMIT 1") + except: + # table does not exist + if self.verbose: + print(f"Table {table} does not exist") + continue + if self.verbose: + print (f"Clearing table {table}") + cursor.execute(f"DELETE FROM {table}") # The *_status tables are regenerated anyway, no need to # delete them here. @@ -953,21 +967,34 @@ class DB: "SELECT inodeprint FROM inodeprints WHERE file = ?", (filename,)): if old_print == current_print: + if self.verbose: + print(" unchanged: " + repr(filename)) return False else: + if self.verbose: + print(" changed: " + repr(filename)) + print(f" old: {old_print}, new: {current_print}") return True return True source_removed_packages = '/packages/removed-packages' + source_ignored_unreported = '/packages/ignored-debian-bug-packages' sources = self.getSources() source_paths = [src["path"] for src in sources] - unchanged = True - for filename in source_paths + [source_removed_packages]: + changed_source = None + for filename in source_paths + [source_removed_packages, source_ignored_unreported]: if has_changed(path + filename): - unchanged = False + if self.verbose: + print(" changed: " + repr(path + filename)) + print (" clearing database") + changed_source = path + filename break - if unchanged: + + if changed_source: + if self.verbose: + print(f" clearing database, because some files have changed (at least {changed_source})") + else: if self.verbose: print(" finished (no changes)") return @@ -975,6 +1002,8 @@ class DB: clear_db() def read_one(source): + if self.verbose: + print(" reading " + repr(source.name)) filename = source.name current_print = self.filePrint(filename) @@ -989,9 +1018,16 @@ class DB: cls = getattr(bugs, cls) read_one(cls(path + srcpath)) + # Read list of packages, which were removed from the status/unreported if self.verbose: print(" update removed packages") - self.readRemovedPackages(cursor, path + source_removed_packages) + self.readRemovedAndIgnoredPackages(cursor, path + source_removed_packages, table = "removed_packages") + + # Read list of packages, which should be ignored for the status/unreported + if self.verbose: + print(" update ignored packages") + self.readRemovedAndIgnoredPackages(cursor, path + source_ignored_unreported, table = "ignored_packages") + errors = [] @@ -1966,9 +2002,15 @@ class DB: ORDER BY bug""", (bug, bug, bug, bug)): yield bug_name - def readRemovedPackages(self, cursor, filename): - """Reads a file of removed packages and stores it in the database. - The original contents of the removed_packages table is preserved.""" + def readRemovedAndIgnoredPackages(self, cursor, filename, table='removed_packages'): + """Reads a file of removed or ignored packages and stores it in the database. + For that the table parameter must be set to 'removed_packages'. + This is the default value. + The original contents of the removed_packages table is preserved. + + This function also reads the file of packages, where filing debian bugs is being ignored + and stores it in the database. + """ f = open(filename) @@ -1989,8 +2031,21 @@ class DB: else: raise ValueError("not a package: " + repr(line)) + # check, if {table} exists, otherwise create it + cursor.execute( + f"CREATE TABLE IF NOT EXISTS {table} (name TEXT NOT NULL PRIMARY KEY)") + + # Add packages into the table cursor.executemany( - "INSERT OR IGNORE INTO removed_packages (name) VALUES (?)", gen()) + f"INSERT OR IGNORE INTO {table} (name) VALUES (?)", gen()) + + + # Add file print to database for removed packages + current_print = self.filePrint(filename) + cursor.execute( + """INSERT OR REPLACE INTO inodeprints (inodeprint, file) + VALUES (?, ?)""", (current_print, filename)) + def getUnknownPackages(self, cursor): """Returns a generator for a list of unknown packages. @@ -2030,7 +2085,7 @@ class DB: st.bug_name > 'TEMP-' AND st.bug_name LIKE 'TEMP-%' ORDER BY st.bug_name""",(vulnerability,))) - def getUnreportedVulnerabilities(self, cursor=None): + def getUnreportedVulnerabilities(self, cursor=None, show_ignored=False): """Returns a list of pairs (BUG_NAME, DESCRIPTION) of vulnerabilities which are unfixed in unstable and lack a filed bug. """ @@ -2039,7 +2094,7 @@ class DB: last_bug = None result = [] for bug, pkg in cursor.execute( -"""SELECT DISTINCT source_package_status.bug_name, source_packages.name +f"""SELECT DISTINCT source_package_status.bug_name, source_packages.name FROM source_packages JOIN source_package_status ON source_packages.rowid = source_package_status.package @@ -2052,6 +2107,7 @@ class DB: AND package_notes.urgency <> 'unimportant' AND package_notes.rowid NOT IN (SELECT note FROM debian_bugs) AND source_package_status.vulnerable + AND ({show_ignored} OR NOT EXISTS (SELECT * FROM ignored_packages WHERE ignored_packages.name = source_packages.name)) ORDER BY source_package_status.bug_name, source_packages.name"""): if last_bug is None or last_bug != bug: last_bug = bug |