From 7c1acf0e789a6d29f02eedf19d067238df3bd0fd Mon Sep 17 00:00:00 2001 From: Emilio Pozuelo Monfort Date: Tue, 26 Nov 2019 13:05:56 +0100 Subject: gen-DSA: get distro info from config.json --- bin/gen-DSA | 30 ++++++++++++++++++++---------- doc/README.releases | 1 - 2 files changed, 20 insertions(+), 11 deletions(-) diff --git a/bin/gen-DSA b/bin/gen-DSA index 3b73dfbaaa..8ef6fcb6b4 100755 --- a/bin/gen-DSA +++ b/bin/gen-DSA @@ -27,10 +27,20 @@ case "$(basename "$0")" in ;; esac -OLDOLDSTABLE=jessie -OLDSTABLE=stretch -STABLE=buster -TESTING=bullseye +if ! which jq >/dev/null 2>&1 ; then + echo "error: jq is needed to parse distributions, please install it" + exit 1 +fi + +RELEASES=`jq -r '.distributions | to_entries[] | select(.value.release) | .value.release | ascii_upcase' data/config.json` +CODENAMES=`jq -r '.distributions | to_entries[] | select(.value.release) | .key' data/config.json` + +while read dist; do + read codename + eval $dist=$codename +done << EOF +`jq -r '.distributions | to_entries[] | select(.value.release) | (.value.release | ascii_upcase), .key' data/config.json` +EOF NAME_SPACING=24 DATE_SPACING=22 @@ -335,15 +345,15 @@ setvar PACKAGE setvar CVE "$CVE_LIST" setvar ${IDMODE}ID "$DAID" setvar BUGNUM -setvar OLDOLDSTABLE -setvar OLDSTABLE -setvar STABLE -setvar TESTING setvar SPACEDDATE setvar DATE setvar TEXT "${TEXT:-$IDMODE text goes here}" -for dist in $OLDOLDSTABLE $OLDSTABLE $STABLE $TESTING UNSTABLE; do +for dist in $RELEASES; do + setvar $dist +done + +for dist in $CODENAMES; do version="$(eval 'printf "%s" "$'"$dist"_VERSION'"')" if $save && [ -z "$version" ] && grep -q "${dist}_VERSION" "$tmpf"; then printf "Enter $dist's version [unset]: " @@ -377,7 +387,7 @@ EOF printf "\t{%s}\n" "$CVE" >> $daid_entry fi - for dist in $OLDOLDSTABLE $OLDSTABLE $STABLE; do + for dist in $CODENAMES; do version="$(eval 'printf "%s" "$'"$dist"_VERSION'"')" [ -z "$version" ] || \ printf "\t[%s] - %s %s\n" "$dist" "$PACKAGE" "$version" >> $daid_entry diff --git a/doc/README.releases b/doc/README.releases index 995fdd8a20..60c7d6ec40 100644 --- a/doc/README.releases +++ b/doc/README.releases @@ -4,7 +4,6 @@ General ------- [ ] Update doc/DSA.template -[ ] Update bin/gen-DSA [ ] bin/add-dsa-needed.sh [ ] bin/tracker_data.py [ ] Update security-team.debian.org pages -- cgit v1.2.3 From d919142735a57c70d5bda0dd5d91e8e3ee046d22 Mon Sep 17 00:00:00 2001 From: Emilio Pozuelo Monfort Date: Wed, 27 Nov 2019 13:45:27 +0100 Subject: config.py: add python module to read config.json --- lib/python/config.py | 52 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 52 insertions(+) create mode 100644 lib/python/config.py diff --git a/lib/python/config.py b/lib/python/config.py new file mode 100644 index 0000000000..61f633ee28 --- /dev/null +++ b/lib/python/config.py @@ -0,0 +1,52 @@ +# config.py -- methods to read global configuration from data/config.json +# Copyright (C) 2019 Emilio Pozuelo Monfort +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + +import json +import os + +_config = None + +def get_config(): + global _config + if not _config: + d = os.path.dirname(os.path.abspath(__file__)) + + with open(d + '/../../data/config.json') as f: + config = json.load(f) + + _config = config['distributions'] + + return _config + +def get_supported_releases(): + config = get_config() + + return [d for d in config.keys() if 'release' in config[d]] + +def get_release_codename(release, suffix=''): + config = get_config() + + for r in config.keys(): + if 'release' in config[r] and config[r]['release'] == release: + return r + suffix + + return None + +def get_release_alias(codename): + config = get_config() + + return config[codename]['release'] -- cgit v1.2.3 From db63d4c8654c3d809caced8e5354f9655a00ae7c Mon Sep 17 00:00:00 2001 From: Emilio Pozuelo Monfort Date: Wed, 27 Nov 2019 14:10:29 +0100 Subject: tracker_service: don't register oldoldstable when not supported --- bin/tracker_service.py | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/bin/tracker_service.py b/bin/tracker_service.py index d45d83b6a1..bcc6770cea 100755 --- a/bin/tracker_service.py +++ b/bin/tracker_service.py @@ -3,6 +3,7 @@ import sys sys.path.insert(0,'../lib/python') import bugs +import config import re import security_db from web_support import * @@ -142,8 +143,9 @@ class TrackerService(webservice_base_class): self.register('*', self.page_object) self.register('redirect/*', self.page_redirect) self.register('source-package/*', self.page_source_package) - self.register('status/release/oldoldstable', - self.page_status_release_oldoldstable) + if config.get_release_codename('oldoldstable'): + self.register('status/release/oldoldstable', + self.page_status_release_oldoldstable) self.register('status/release/oldstable', self.page_status_release_oldstable) self.register('status/release/stable', self.page_status_release_stable) @@ -151,8 +153,9 @@ class TrackerService(webservice_base_class): self.page_status_release_stable_backports) self.register('status/release/oldstable-backports', self.page_status_release_oldstable_backports) - self.register('status/release/oldoldstable-backports', - self.page_status_release_oldoldstable_backports) + if config.get_release_codename('oldoldstable'): + self.register('status/release/oldoldstable-backports', + self.page_status_release_oldoldstable_backports) self.register('status/release/testing', self.page_status_release_testing) self.register('status/release/unstable', @@ -250,10 +253,12 @@ aware of and/or help us improve the quality of this information by """, 'Vulnerable packages in the oldstable suite'), ('status/release/oldstable-backports', 'Vulnerable packages in backports for oldstable'), - ('status/release/oldoldstable', - 'Vulnerable packages in the oldoldstable suite'), - ('status/release/oldoldstable-backports', - 'Vulnerable packages in backports for oldoldstable'), + config.get_release_codename('oldoldstable') and + ('status/release/oldoldstable', + 'Vulnerable packages in the oldoldstable suite') or ('', ''), + config.get_release_codename('oldoldstable') and + ('status/release/oldoldstable-backports', + 'Vulnerable packages in backports for oldoldstable') or ('', ''), ('status/dtsa-candidates', "Candidates for DTSAs"), ('status/todo', 'TODO items'), ('status/undetermined', 'Packages that may be vulnerable but need to be checked (undetermined issues)'), -- cgit v1.2.3 From 6d67d31ff23f13a44e2f01d3aa4ccef91cbd3b8c Mon Sep 17 00:00:00 2001 From: Emilio Pozuelo Monfort Date: Wed, 27 Nov 2019 14:11:49 +0100 Subject: tracker_service: don't hardcode backport codenames --- bin/tracker_service.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/bin/tracker_service.py b/bin/tracker_service.py index bcc6770cea..fbc9eb4c32 100755 --- a/bin/tracker_service.py +++ b/bin/tracker_service.py @@ -887,19 +887,19 @@ to improve our documentation and procedures, so feedback is welcome.""")])]) return self.page_status_release_unstable_like( path, params, url, title='Vulnerable source packages among backports for stable', - rel='buster-backports') + rel=config.get_release_codename('stable', '-backports')) def page_status_release_oldstable_backports(self, path, params, url): return self.page_status_release_unstable_like( path, params, url, title='Vulnerable source packages among backports for oldstable', - rel='stretch-backports') + rel=config.get_release_codename('oldstable', '-backports')) def page_status_release_oldoldstable_backports(self, path, params, url): return self.page_status_release_unstable_like( path, params, url, title='Vulnerable source packages among backports for oldoldstable', - rel='jessie-backports') + rel=config.get_release_codename('oldoldstable', '-backports')) def page_status_dtsa_candidates(self, path, params, url): -- cgit v1.2.3 From 651299e86218da37f577fa2aa980a7a1b75b9f9f Mon Sep 17 00:00:00 2001 From: Emilio Pozuelo Monfort Date: Wed, 27 Nov 2019 14:13:00 +0100 Subject: tracker_service: don't hardcode codenames in db queries --- bin/tracker_service.py | 23 +++++++++++------------ 1 file changed, 11 insertions(+), 12 deletions(-) diff --git a/bin/tracker_service.py b/bin/tracker_service.py index fbc9eb4c32..e4d76af029 100755 --- a/bin/tracker_service.py +++ b/bin/tracker_service.py @@ -914,18 +914,19 @@ to improve our documentation and procedures, so feedback is welcome.""")])]) (SELECT testing.version_id < stable.version_id FROM source_packages AS testing, source_packages AS stable WHERE testing.name = testing_status.package - AND testing.release = 'bullseye' + AND testing.release = ? AND testing.subrelease = '' AND testing.archive = testing_status.section AND stable.name = testing_status.package - AND stable.release = 'buster' + AND stable.release = ? AND stable.subrelease = 'security' AND stable.archive = testing_status.section), (SELECT range_remote FROM nvd_data WHERE cve_name = bug) FROM testing_status WHERE (NOT unstable_vulnerable) - AND (NOT testing_security_fixed)"""): + AND (NOT testing_security_fixed)""", + (config.get_release_codename('testing'), config.get_release_codename('stable'))): if bf.urgencyFiltered(urgency, vulnerable): continue if bf.remoteFiltered(remote): @@ -999,14 +1000,13 @@ checker to find out why they have not entered testing yet."""), old_pkg = '' old_dsc = '' last_displayed = '' - releases = ('sid', 'bullseye', 'buster', 'stretch', 'jessie') + releases = config.get_supported_releases() for (pkg_name, bug_name, release, desc) in self.db.cursor().execute( """SELECT DISTINCT sp.name, st.bug_name, sp.release, bugs.description FROM source_package_status AS st, source_packages AS sp, bugs WHERE st.vulnerable == 2 AND sp.rowid = st.package - AND ( sp.release = ? OR sp.release = ? OR sp.release = ? - OR sp.release = ? OR sp.release = ? ) + AND sp.release IN (""" + ",".join("?" * len(releases)) + """) AND sp.subrelease = '' AND st.bug_name == bugs.name ORDER BY sp.name, st.bug_name""", releases): @@ -1044,14 +1044,14 @@ checker to find out why they have not entered testing yet."""), old_dsc = '' old_name = '' last_displayed = '' - releases = ('sid', 'bullseye', 'buster', 'stretch', 'jessie') + releases = config.get_supported_releases() for (pkg_name, bug_name, release, desc) in self.db.cursor().execute( """SELECT DISTINCT sp.name, st.bug_name, sp.release, bugs.description FROM source_package_status AS st, source_packages AS sp, bugs WHERE st.vulnerable > 0 AND sp.rowid = st.package - AND ( sp.release = ? OR sp.release = ? OR sp.release = ? - OR sp.release = ? OR sp.release = ? ) AND st.urgency == 'unimportant' + AND sp.release IN (""" + ",".join("?" * len(releases)) + """) + AND st.urgency == 'unimportant' AND sp.subrelease = '' AND st.bug_name == bugs.name ORDER BY sp.name, st.bug_name""", releases): @@ -1330,7 +1330,7 @@ Debian bug number.'''), urgency = defaultdict(lambda: defaultdict(dict)) nodsa = defaultdict(lambda: defaultdict(dict)) nodsa_reason = defaultdict(lambda: defaultdict(dict)) - supported_releases = ('sid', 'bullseye', 'buster', 'stretch', 'jessie') + supported_releases = config.get_supported_releases() for (pkg, issue, desc, debianbug, release, subrelease, db_version, db_fixed_version, db_status, db_urgency, db_remote, db_nodsa, db_nodsa_reason) in self.db.cursor().execute( """SELECT sp.name, st.bug_name, (SELECT cve_desc FROM nvd_data @@ -1355,8 +1355,7 @@ Debian bug number.'''), FROM source_package_status AS st, source_packages AS sp, bugs WHERE sp.rowid = st.package AND st.bug_name = bugs.name AND ( st.bug_name LIKE 'CVE-%' OR st.bug_name LIKE 'TEMP-%' ) - AND ( sp.release = ? OR sp.release = ? OR sp.release = ? - OR sp.release = ? OR sp.release = ? ) + AND sp.release IN (""" + ",".join("?" * len(supported_releases)) + """) ORDER BY sp.name, st.bug_name, sp.release, sp.subrelease""" , supported_releases): ### to ease debugging...: -- cgit v1.2.3 From d75d1c94897671e1d0581df7cac9f662d071bf6d Mon Sep 17 00:00:00 2001 From: Emilio Pozuelo Monfort Date: Wed, 27 Nov 2019 14:19:13 +0100 Subject: security_db: don't hardcode codenames in calls to _calcTesting() --- lib/python/security_db.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/lib/python/security_db.py b/lib/python/security_db.py index fd90ab8b21..c0ab95c869 100644 --- a/lib/python/security_db.py +++ b/lib/python/security_db.py @@ -43,6 +43,7 @@ import sys import types import zlib +import config import debian_support import dist_config @@ -1280,10 +1281,13 @@ class DB: "SELECT name FROM bugs WHERE NOT not_for_us"): self._calcUnstable(c, bug_name) - self._calcTesting(c, bug_name, 'testing', 'bullseye') - self._calcTesting(c, bug_name, 'stable', 'buster') - self._calcTesting(c, bug_name, 'oldstable', 'stretch') - self._calcTesting(c, bug_name, 'oldoldstable', 'jessie') + + for release in config.get_supported_releases(): + if release == 'sid': + continue + + alias = config.get_release_alias(release) + self._calcTesting(c, bug_name, alias, release) return result -- cgit v1.2.3 From ff29884c959977149ee22b347fd1e90bfba1831a Mon Sep 17 00:00:00 2001 From: Emilio Pozuelo Monfort Date: Wed, 27 Nov 2019 14:20:15 +0100 Subject: security_db: don't hardcode codenames in calls to gen_release --- lib/python/security_db.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/python/security_db.py b/lib/python/security_db.py index c0ab95c869..7beef42a82 100644 --- a/lib/python/security_db.py +++ b/lib/python/security_db.py @@ -1736,7 +1736,7 @@ class DB: store_value('release/1/' + release, '\n'.join(result)) - for release in ('sid', 'jessie', 'stretch', 'buster', 'bullseye'): + for release in config.get_supported_releases(): gen_release(release) result = result_start -- cgit v1.2.3 From 09233fbd81e2ad7cbece5e472c54790e90c7e7ea Mon Sep 17 00:00:00 2001 From: Emilio Pozuelo Monfort Date: Wed, 27 Nov 2019 14:23:12 +0100 Subject: security_db: take the sid value in calculateDebsecan0 When the release is sid, just pass 'sid' rather than the empty string to change that afterwards. --- lib/python/security_db.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/lib/python/security_db.py b/lib/python/security_db.py index 7beef42a82..a66ab1fbab 100644 --- a/lib/python/security_db.py +++ b/lib/python/security_db.py @@ -1456,12 +1456,10 @@ class DB: c.execute("""INSERT INTO vulnlist SELECT bug_name, package, id FROM package_notes WHERE release = ''""") - if release: + if release != 'sid': c.execute("""INSERT OR REPLACE INTO vulnlist SELECT bug_name, package, id FROM package_notes WHERE release = ?""", (release,)) - else: - release = 'sid' urgency_to_flag = {'low' : 'L', 'medium' : 'M', 'high' : 'H', 'not yet assigned' : ' '} @@ -1749,7 +1747,7 @@ class DB: def calculateDebsecan(self): """Calculate all debsecan data.""" - for release in ('', 'jessie', 'stretch', 'buster', 'bullseye'): + for release in ('sid', 'jessie', 'stretch', 'buster', 'bullseye'): self.calculateDebsecan0(release) self.calculateDebsecan1() -- cgit v1.2.3 From aa57cee8dfcb9d527b04e211a51aba47b585cb22 Mon Sep 17 00:00:00 2001 From: Emilio Pozuelo Monfort Date: Wed, 27 Nov 2019 14:26:32 +0100 Subject: security_db: don't hardcode release codenames in calculateDebsecan --- lib/python/security_db.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/python/security_db.py b/lib/python/security_db.py index a66ab1fbab..b929320c6b 100644 --- a/lib/python/security_db.py +++ b/lib/python/security_db.py @@ -1747,7 +1747,7 @@ class DB: def calculateDebsecan(self): """Calculate all debsecan data.""" - for release in ('sid', 'jessie', 'stretch', 'buster', 'bullseye'): + for release in config.get_supported_releases(): self.calculateDebsecan0(release) self.calculateDebsecan1() -- cgit v1.2.3 From f0ffe0da1a980a4cfb53fb4d7af41b7d15b34788 Mon Sep 17 00:00:00 2001 From: Emilio Pozuelo Monfort Date: Wed, 27 Nov 2019 14:34:15 +0100 Subject: security_db: don't hardcode releases in db queries --- lib/python/security_db.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/lib/python/security_db.py b/lib/python/security_db.py index b929320c6b..8ba681ab82 100644 --- a/lib/python/security_db.py +++ b/lib/python/security_db.py @@ -1780,13 +1780,16 @@ class DB: """A generator which returns tuples (RELEASE-LIST, VERSION), the available versions of the source package pkg.""" + releases = config.get_supported_releases() + values = [pkg] + releases + for (release, version) in cursor.execute( """SELECT release_name(release, subrelease, archive) AS release, version FROM source_packages WHERE name = ? - AND release IN ('jessie', 'stretch', 'buster', 'bullseye', 'sid') + AND release IN (""" + ",".join("?" * len(releases)) + """) GROUP BY release, version - ORDER BY release_to_number(release), subrelease_to_number(subrelease), version COLLATE version""", (pkg,)): + ORDER BY release_to_number(release), subrelease_to_number(subrelease), version COLLATE version""", values): yield release, version def getBinaryPackageVersions(self, cursor, pkg): @@ -1832,6 +1835,9 @@ class DB: RELEASE-LIST, VERSION, VULNERABLE-FLAG) of source packages which are related to the given bug.""" + releases = config.get_supported_releases() + values = [bug] + releases + for (package, releases, version, vulnerable) in cursor.execute( """SELECT package, string_list(release), version, vulnerable FROM (SELECT p.name AS package, @@ -1839,10 +1845,10 @@ class DB: p.version AS version, s.vulnerable AS vulnerable FROM source_package_status AS s, source_packages AS p WHERE s.bug_name = ? AND p.rowid = s.package - AND release in ('jessie', 'stretch', 'buster', 'bullseye', 'sid')) + AND release in (""" + ",".join("?" * len(releases)) + """)) GROUP BY package, version, vulnerable ORDER BY package, releasepart_to_number(release), subreleasepart_to_number(release), version COLLATE version""", - (bug,)): + values): yield package, releases.split(', '), version, vulnerable def getBugsFromDebianBug(self, cursor, number): -- cgit v1.2.3 From 327d1fd90823452db992a334fdf47eec890c0ceb Mon Sep 17 00:00:00 2001 From: Emilio Pozuelo Monfort Date: Wed, 27 Nov 2019 14:40:51 +0100 Subject: security_db: don't hardcode release codenames in _initViews --- lib/python/security_db.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/lib/python/security_db.py b/lib/python/security_db.py index 8ba681ab82..44170c455e 100644 --- a/lib/python/security_db.py +++ b/lib/python/security_db.py @@ -495,7 +495,11 @@ class DB: AND sp.release = 'bullseye' AND sp.subrelease = '' ORDER BY sp.name, st.urgency, st.bug_name""") - for (name, nickname) in (('stable', 'buster'), ('oldstable', 'stretch'), ('oldoldstable', 'jessie'),): + releases = (('stable', config.get_release_codename('stable')), + ('oldstable', config.get_release_codename('oldstable')), + ('oldoldstable', config.get_release_codename('oldoldstable'))) + + for (name, nickname) in releases: cursor.execute( """CREATE TEMPORARY VIEW %s_status AS SELECT DISTINCT sp.name AS package, st.bug_name AS bug, -- cgit v1.2.3 From 6ccb0b4eba0a824a5fc919061b176d6d25dae626 Mon Sep 17 00:00:00 2001 From: Emilio Pozuelo Monfort Date: Wed, 27 Nov 2019 14:42:09 +0100 Subject: security_db: remove unused getEffectiveVersion method --- lib/python/security_db.py | 54 ----------------------------------------------- 1 file changed, 54 deletions(-) diff --git a/lib/python/security_db.py b/lib/python/security_db.py index 44170c455e..d62ca6283b 100644 --- a/lib/python/security_db.py +++ b/lib/python/security_db.py @@ -45,7 +45,6 @@ import zlib import config import debian_support -import dist_config class InsertError(Exception): """Class for capturing insert errors. @@ -2038,59 +2037,6 @@ class DB: ORDER BY n.package"""): yield (package, bugs.split(','), map(int, debian_bugs.split(','))) - def getEffectiveVersion(self, release, pkg, purpose, cache=None, cursor=None): - """Retrieve the effective version of a source package in a release. - - The effective version is the version that matches the recommended - sources.list file for the intended purpose. For suitable values - of purpose, see dist_config. - """ - # The cache is structured as a (RELEASE, PACKAGE) => VAL - # dict, where VAL is either a dict PURPOSE => VERSION, - # a VERSION, or None. - if cache is not None: - sp = (release, pkg) - if sp in cache: - d = cache[sp] - if d.__class__ == dict: - return d.get(purpose, None) - else: - return d - - if cursor is None: - cursor = self.cursor() - - rel = dist_config.releases[release] - purposes = rel['purpose'] - results = {} - - Version = debian_support.Version - for (part, ver) in cursor.execute( - """SELECT DISTINCT subrelease, version FROM source_packages - WHERE release = ? AND name = ?""", (str(release), pkg)): - ver = Version(ver) - for (purpose, permitted) in purposes.items(): - if part not in permitted: - continue - if purpose in results: - oldver = results[purpose] - if ver <= oldver: - continue - results[purpose] = ver - - if cache is not None: - vers = set(map(str, results.values())) - l = len(vers) - if l == 1: - for r in vers: - cache[sp] = Version(r) - elif l == 0: - cache[sp] = None - else: - cache[sp] = results - - return results.get(purpose, None) - def check(self, cursor=None): """Runs a simple consistency check and prints the results.""" -- cgit v1.2.3 From ac98735a16590eab2f1b7065e5cdefab5d75157a Mon Sep 17 00:00:00 2001 From: Emilio Pozuelo Monfort Date: Wed, 27 Nov 2019 14:42:38 +0100 Subject: dist_config.py: remove unused file --- doc/README.releases | 1 - lib/python/dist_config.py | 97 ----------------------------------------------- 2 files changed, 98 deletions(-) delete mode 100644 lib/python/dist_config.py diff --git a/doc/README.releases b/doc/README.releases index 60c7d6ec40..e142ddaacb 100644 --- a/doc/README.releases +++ b/doc/README.releases @@ -14,7 +14,6 @@ Security Tracker code See https://bugs.debian.org/783491 [ ] bin/tracker_service.py [ ] lib/python/debian_support.py -[ ] lib/python/dist_config.py [ ] lib/python/security_db.py [ ] Makefile diff --git a/lib/python/dist_config.py b/lib/python/dist_config.py deleted file mode 100644 index 107f63a088..0000000000 --- a/lib/python/dist_config.py +++ /dev/null @@ -1,97 +0,0 @@ -# dist_config.py -- describe how the Debian package database is assembled -# Copyright (C) 2008 Florian Weimer -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; either version 2 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program; if not, write to the Free Software -# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA - -""" -This Python moule describes how different views of the Debian package -database are assembled from a set of on-disk files. - -Each view is labeled by a purpose. Currently defined purposes are: - - overview: Used to generate the release overview web page. This - should not contain vulnerabilities which the security team - considers processed. - - debsecan: Used to generate the "fix is available" data for debsecan. - This should reflect the recommended set of sources.list - entries for the release. -""" - -###################################################################### -# Configuration section -###################################################################### - -def apply_config(): - # Invoked at the end of the file. Edit this to suit your needs. - - common_archs = 'amd64,armel,i386,mips,mipsel,powerpc'.split(',') - squeeze_archs = common_archs + ['s390','ia64','kfreebsd-amd64','kfreebsd-i386','sparc' ] - wheezy_archs = [ 'amd64','armel','armhf','i386' ] - jessie_archs = [ 'amd64','armel','armhf','i386' ] - stretch_archs = [ 'amd64','arm64','armel','armhf','i386','mips','mips64el','mipsel','ppc64el','s390x' ] - buster_archs = [ 'amd64','arm64','armel','armhf','i386','mips','mips64el','mipsel','ppc64el','s390x' ] - bullseye_archs = [ 'amd64','arm64','armel','armhf','i386','mips64el','mipsel','ppc64el','s390x' ] - sid_archs = [ 'amd64','arm64','armel','armhf','i386','mips64el','mipsel','ppc64el','s390x' ] - - add_release(name='squeeze', - architectures=squeeze_archs, - ) - - add_release(name='wheezy', - architectures=wheezy_archs, - ) - - add_release(name='jessie', - architectures=jessie_archs, - ) - - add_release(name='stretch', - architectures=stretch_archs, - ) - - add_release(name='buster', - architectures=buster_archs, - ) - - add_release(name='bullseye', - architectures=bullseye_archs, - ) - - add_release(name='sid', - architectures=sid_archs, - ) - -###################################################################### -# Support routines -###################################################################### - -releases = {} - -def add_release(name, architectures, - debsecan_part=('', 'security'), - overview_part=('', 'security', 'proposed-updates')): - import debian_support - name = debian_support.internRelease(name) - if name in releases: - raise ValueError("duplicate release", name) - releases[name] = {'architectures' : architectures, - 'purpose' : {'debsecan' : debsecan_part, - 'overview' : overview_part}} - -# Run the code in the configuration section - -apply_config() -del apply_config -- cgit v1.2.3 From 0606b911eca9144b864ff11485e660f8266e2644 Mon Sep 17 00:00:00 2001 From: Emilio Pozuelo Monfort Date: Thu, 28 Nov 2019 11:13:02 +0100 Subject: security_db: don't hardcode the testing suite codename --- lib/python/security_db.py | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/lib/python/security_db.py b/lib/python/security_db.py index d62ca6283b..910ba62375 100644 --- a/lib/python/security_db.py +++ b/lib/python/security_db.py @@ -464,6 +464,7 @@ class DB: """) def _initViews(self, cursor): + testing = config.get_release_codename('testing') cursor.execute( """CREATE TEMPORARY VIEW testing_status AS SELECT DISTINCT sp.name AS package, st.bug_name AS bug, @@ -479,7 +480,7 @@ class DB: COALESCE((SELECT NOT vulnerable FROM source_packages AS tsecp, source_package_status AS tsecst WHERE tsecp.name = sp.name - AND tsecp.release = 'bullseye' AND tsecp.subrelease = 'security' + AND tsecp.release = '%s' AND tsecp.subrelease = 'security' AND tsecp.archive = sp.archive AND tsecst.bug_name = st.bug_name AND tsecst.package = tsecp.rowid), 0) AS testing_security_fixed, @@ -488,11 +489,12 @@ class DB: (EXISTS (SELECT * FROM package_notes_nodsa AS pnd WHERE pnd.bug_name = st.bug_name AND pnd.package = sp.name - AND pnd.release = 'bullseye')) AS no_dsa + AND pnd.release = '%s')) AS no_dsa FROM source_package_status AS st, source_packages AS sp WHERE st.vulnerable > 0 AND sp.rowid = st.package - AND sp.release = 'bullseye' AND sp.subrelease = '' - ORDER BY sp.name, st.urgency, st.bug_name""") + AND sp.release = '%s' AND sp.subrelease = '' + ORDER BY sp.name, st.urgency, st.bug_name""" + % (testing, testing, testing)) releases = (('stable', config.get_release_codename('stable')), ('oldstable', config.get_release_codename('oldstable')), @@ -1144,7 +1146,7 @@ class DB: """Calculate vulnerable packages. To each package note, a release-specific vulnerability status - is attached. Currently, only bullseye/testing is processed. + is attached. Currently, only testing is processed. Returns a list strings describing inconsistencies. """ @@ -1160,17 +1162,18 @@ class DB: # The following does not work because stable->security -> # testing -> unstable propagation is no longer available. if False: - # Ignore bullseye/testing because stable issues may be + # Ignore testing because stable issues may be # fast-tracked into testing, bypassing unstable. + testing = config.get_release_codename('testing') for (bug_name, pkg_name, rel, unstable_ver, rel_ver) \ in list(cursor.execute( """SELECT a.bug_name, a.package, b.release, a.fixed_version, b.fixed_version FROM package_notes a, package_notes b WHERE a.bug_name = b.bug_name AND a.package = b.package - AND a.release = '' AND b.release NOT IN ('', 'bullseye') + AND a.release = '' AND b.release NOT IN ('', '%s') AND a.fixed_version IS NOT NULL - AND a.fixed_version_id < b.fixed_version_id""")): + AND a.fixed_version_id < b.fixed_version_id""" % (testing,))): b = bugs.BugFromDB(cursor, bug_name) result.append("%s:%d: inconsistent versions for package %s" % (b.source_file, b.source_line, pkg_name)) -- cgit v1.2.3 From f93ecf042f524aed8edb1fedef6c8cb880f64d7e Mon Sep 17 00:00:00 2001 From: Emilio Pozuelo Monfort Date: Thu, 28 Nov 2019 11:15:26 +0100 Subject: security_db: drop squeeze workarounds --- lib/python/security_db.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/lib/python/security_db.py b/lib/python/security_db.py index 910ba62375..bc5f8a07da 100644 --- a/lib/python/security_db.py +++ b/lib/python/security_db.py @@ -746,9 +746,6 @@ class DB: if unchanged: continue - if release == 'squeeze-lts': - release = 'squeeze' - subrelease = 'lts' cursor.execute( """DELETE FROM source_packages WHERE release = ? AND subrelease = ? AND archive = ?""", @@ -809,9 +806,6 @@ class DB: raise ValueError("invalid file name: " + repr(filename)) (release, subrelease, archive, architecture) = match.groups() - if release == 'squeeze-lts': - release = 'squeeze' - subrelease = 'lts' (unch, parsed) = self._parseFile(cursor, filename) unchanged = unchanged and unch for name in parsed.keys(): -- cgit v1.2.3 From 466d09f104bd99309aff43fd6340fe120250dd6b Mon Sep 17 00:00:00 2001 From: Emilio Pozuelo Monfort Date: Mon, 2 Dec 2019 12:38:32 +0100 Subject: config: add a method to get all releases --- lib/python/config.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/lib/python/config.py b/lib/python/config.py index 61f633ee28..c445dadb6c 100644 --- a/lib/python/config.py +++ b/lib/python/config.py @@ -15,6 +15,8 @@ # along with this program; if not, write to the Free Software # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA +# TODO: the OrderedDict use can be dropped once we use Python 3 (>= 3.7) +from collections import OrderedDict import json import os @@ -26,7 +28,7 @@ def get_config(): d = os.path.dirname(os.path.abspath(__file__)) with open(d + '/../../data/config.json') as f: - config = json.load(f) + config = json.load(f, object_pairs_hook=OrderedDict) _config = config['distributions'] @@ -37,6 +39,11 @@ def get_supported_releases(): return [d for d in config.keys() if 'release' in config[d]] +def get_all_releases(): + config = get_config() + + return config.keys() + def get_release_codename(release, suffix=''): config = get_config() -- cgit v1.2.3 From af4d74e90f5ada987660f5ca3dacee0bdfebbe58 Mon Sep 17 00:00:00 2001 From: Emilio Pozuelo Monfort Date: Mon, 2 Dec 2019 12:40:19 +0100 Subject: debian_support: don't hardcode release names --- doc/README.releases | 1 - lib/python/debian_support.py | 5 +++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/doc/README.releases b/doc/README.releases index e142ddaacb..5dbca5084b 100644 --- a/doc/README.releases +++ b/doc/README.releases @@ -13,7 +13,6 @@ Security Tracker code --------------------- See https://bugs.debian.org/783491 [ ] bin/tracker_service.py -[ ] lib/python/debian_support.py [ ] lib/python/security_db.py [ ] Makefile diff --git a/lib/python/debian_support.py b/lib/python/debian_support.py index 84f66815c1..d405440e9f 100644 --- a/lib/python/debian_support.py +++ b/lib/python/debian_support.py @@ -37,6 +37,8 @@ except ImportError: import apt_pkg apt_pkg.init() +import config + # Timeout for downloads. TIMEOUT = 30 @@ -194,8 +196,7 @@ class Release(PseudoEnum): pass def listReleases(): releases = {} - rels = ("experimental", # For use in [brackets] in the list files. - "potato", "woody", "sarge", "etch", "lenny", "squeeze", "wheezy", "jessie", "stretch", "buster", "bullseye", "sid") + rels = ["experimental"] + config.get_all_releases() for r in range(len(rels)): releases[rels[r]] = Release(rels[r], r) Release.releases = releases -- cgit v1.2.3 From 78da3d92c25541f2891062fe9066320477c79847 Mon Sep 17 00:00:00 2001 From: Emilio Pozuelo Monfort Date: Mon, 2 Dec 2019 12:41:35 +0100 Subject: security_db: don't hardcode release names --- doc/README.releases | 1 - lib/python/security_db.py | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/doc/README.releases b/doc/README.releases index 5dbca5084b..b2d66779d4 100644 --- a/doc/README.releases +++ b/doc/README.releases @@ -13,7 +13,6 @@ Security Tracker code --------------------- See https://bugs.debian.org/783491 [ ] bin/tracker_service.py -[ ] lib/python/security_db.py [ ] Makefile Security Tracker host diff --git a/lib/python/security_db.py b/lib/python/security_db.py index bc5f8a07da..a4281274a7 100644 --- a/lib/python/security_db.py +++ b/lib/python/security_db.py @@ -588,7 +588,7 @@ class DB: return -1 self.db.createscalarfunction("subreleasepart_to_number", subreleasepart_to_number, 1) - releases = ['potato', 'woody', 'sarge', 'etch', 'lenny', 'squeeze', 'wheezy', 'jessie', 'stretch', 'buster', 'bullseye', 'sid'] + releases = config.get_all_releases() def release_to_number(u): try: return releases.index(u) -- cgit v1.2.3 From 91c961eef4c0a8847d26f6043884748f504fb49a Mon Sep 17 00:00:00 2001 From: Emilio Pozuelo Monfort Date: Mon, 2 Dec 2019 13:05:20 +0100 Subject: tracker_service: unify *stable methods --- bin/tracker_service.py | 44 +++++++++++++++----------------------------- 1 file changed, 15 insertions(+), 29 deletions(-) diff --git a/bin/tracker_service.py b/bin/tracker_service.py index e4d76af029..7a1080743f 100755 --- a/bin/tracker_service.py +++ b/bin/tracker_service.py @@ -145,17 +145,17 @@ class TrackerService(webservice_base_class): self.register('source-package/*', self.page_source_package) if config.get_release_codename('oldoldstable'): self.register('status/release/oldoldstable', - self.page_status_release_oldoldstable) + self.page_status_release_stable_like) self.register('status/release/oldstable', - self.page_status_release_oldstable) - self.register('status/release/stable', self.page_status_release_stable) + self.page_status_release_stable_like) + self.register('status/release/stable', self.page_status_release_stable_like) self.register('status/release/stable-backports', - self.page_status_release_stable_backports) + self.page_status_release_backports_like) self.register('status/release/oldstable-backports', - self.page_status_release_oldstable_backports) + self.page_status_release_backports_like) if config.get_release_codename('oldoldstable'): self.register('status/release/oldoldstable-backports', - self.page_status_release_oldoldstable_backports) + self.page_status_release_backports_like) self.register('status/release/testing', self.page_status_release_testing) self.register('status/release/unstable', @@ -755,14 +755,10 @@ to improve our documentation and procedures, so feedback is welcome.""")])]) for this vulnerability.'''), self.nvd_text]) - def page_status_release_stable(self, path, params, url): - return self.page_status_release_stable_oldstable_oldoldstable('stable', params, url) - def page_status_release_oldstable(self, path, params, url): - return self.page_status_release_stable_oldstable_oldoldstable('oldstable', - params, url) - def page_status_release_oldoldstable(self, path, params, url): - return self.page_status_release_stable_oldstable_oldoldstable('oldoldstable', - params, url) + def page_status_release_stable_like(self, path, params, url): + release = os.path.basename(url.path_info) + + return self.page_status_release_stable_oldstable_oldoldstable(release, params, url) def page_status_release_testing(self, path, params, url): bf = BugFilter(params) @@ -883,24 +879,14 @@ to improve our documentation and procedures, so feedback is welcome.""")])]) title='Vulnerable source packages in the unstable suite', rel='sid') - def page_status_release_stable_backports(self, path, params, url): - return self.page_status_release_unstable_like( - path, params, url, - title='Vulnerable source packages among backports for stable', - rel=config.get_release_codename('stable', '-backports')) + def page_status_release_backports_like(self, path, params, url): + release = os.path.basename(url.path_info) + release = release.split("-")[0] - def page_status_release_oldstable_backports(self, path, params, url): return self.page_status_release_unstable_like( path, params, url, - title='Vulnerable source packages among backports for oldstable', - rel=config.get_release_codename('oldstable', '-backports')) - - def page_status_release_oldoldstable_backports(self, path, params, url): - return self.page_status_release_unstable_like( - path, params, url, - title='Vulnerable source packages among backports for oldoldstable', - rel=config.get_release_codename('oldoldstable', '-backports')) - + title='Vulnerable source packages among backports for ' + release, + rel=config.get_release_codename(release, '-backports')) def page_status_dtsa_candidates(self, path, params, url): bf = BugFilter(params,nonodsa=True,noignored=True,nopostponed=True) -- cgit v1.2.3 From 64c7b21ff126cb926f82598bf1eded54cc9d8ddb Mon Sep 17 00:00:00 2001 From: Emilio Pozuelo Monfort Date: Mon, 2 Dec 2019 14:02:49 +0100 Subject: tracker_service: dynamically register stable releases --- bin/tracker_service.py | 56 ++++++++++++++++++++++++-------------------------- 1 file changed, 27 insertions(+), 29 deletions(-) diff --git a/bin/tracker_service.py b/bin/tracker_service.py index 7a1080743f..75f9eca426 100755 --- a/bin/tracker_service.py +++ b/bin/tracker_service.py @@ -139,23 +139,24 @@ class TrackerService(webservice_base_class): self.json_data = None # the JSON dump itself self.json_timestamp = None # timestamp of JSON generation self.json_last_modified = None + + self.stable_releases = config.get_supported_releases() + self.stable_releases.remove(config.get_release_codename('testing')) + self.stable_releases.remove('sid') + self.stable_releases.reverse() + self.register('', self.page_home) self.register('*', self.page_object) self.register('redirect/*', self.page_redirect) self.register('source-package/*', self.page_source_package) - if config.get_release_codename('oldoldstable'): - self.register('status/release/oldoldstable', + + for release in self.stable_releases: + alias = config.get_release_alias(release) + self.register('status/release/' + alias, self.page_status_release_stable_like) - self.register('status/release/oldstable', - self.page_status_release_stable_like) - self.register('status/release/stable', self.page_status_release_stable_like) - self.register('status/release/stable-backports', - self.page_status_release_backports_like) - self.register('status/release/oldstable-backports', - self.page_status_release_backports_like) - if config.get_release_codename('oldoldstable'): - self.register('status/release/oldoldstable-backports', + self.register('status/release/' + alias + '-backports', self.page_status_release_backports_like) + self.register('status/release/testing', self.page_status_release_testing) self.register('status/release/unstable', @@ -216,6 +217,16 @@ class TrackerService(webservice_base_class): else: return RedirectResult(url.scriptRelativeFull(query)) + def gen_stable_links(): + links = [] + for release in self.stable_releases: + alias = config.get_release_alias(release) + links.append(('status/release/' + alias, + 'Vulnerable packages in the ' + alias + ' suite')) + links.append(('status/release/' + alias + '-backports', + 'Vulnerable packages in backports for ' + alias)) + return links + return self.create_page( url, 'Security Bug Tracker', [P( @@ -241,25 +252,12 @@ aware of and/or help us improve the quality of this information by """, NAV(make_menu( url.scriptRelative, - ('status/release/unstable', + *[('status/release/unstable', 'Vulnerable packages in the unstable suite'), ('status/release/testing', - 'Vulnerable packages in the testing suite'), - ('status/release/stable', - 'Vulnerable packages in the stable suite'), - ('status/release/stable-backports', - 'Vulnerable packages in backports for stable'), - ('status/release/oldstable', - 'Vulnerable packages in the oldstable suite'), - ('status/release/oldstable-backports', - 'Vulnerable packages in backports for oldstable'), - config.get_release_codename('oldoldstable') and - ('status/release/oldoldstable', - 'Vulnerable packages in the oldoldstable suite') or ('', ''), - config.get_release_codename('oldoldstable') and - ('status/release/oldoldstable-backports', - 'Vulnerable packages in backports for oldoldstable') or ('', ''), - ('status/dtsa-candidates', "Candidates for DTSAs"), + 'Vulnerable packages in the testing suite')] + + gen_stable_links() + + [('status/dtsa-candidates', "Candidates for DTSAs"), ('status/todo', 'TODO items'), ('status/undetermined', 'Packages that may be vulnerable but need to be checked (undetermined issues)'), ('status/unimportant', 'Packages that have open unimportant issues'), @@ -278,7 +276,7 @@ aware of and/or help us improve the quality of this information by """, 'Covered Debian releases and architectures'), ('data/json', 'All information in JSON format') - )), + ])), self.make_search_button(url), P("""(You can enter CVE names, Debian bug numbers and package -- cgit v1.2.3 From e39e6c99b9b1f3d250cf808c58702d012616a811 Mon Sep 17 00:00:00 2001 From: Emilio Pozuelo Monfort Date: Mon, 2 Dec 2019 14:05:26 +0100 Subject: tracker_service: simplify stable-like callbacks And take the file out of README.releases. --- bin/tracker_service.py | 9 ++------- doc/README.releases | 1 - 2 files changed, 2 insertions(+), 8 deletions(-) diff --git a/bin/tracker_service.py b/bin/tracker_service.py index 75f9eca426..44a2186ca1 100755 --- a/bin/tracker_service.py +++ b/bin/tracker_service.py @@ -696,8 +696,8 @@ to improve our documentation and procedures, so feedback is welcome.""")])]) replacement='No known security announcements.') ]) - def page_status_release_stable_oldstable_oldoldstable(self, release, params, url): - assert release in ('stable', 'oldstable', 'oldoldstable',) + def page_status_release_stable_like(self, path, params, url): + release = os.path.basename(url.path_info) bf = BugFilter(params) @@ -753,11 +753,6 @@ to improve our documentation and procedures, so feedback is welcome.""")])]) for this vulnerability.'''), self.nvd_text]) - def page_status_release_stable_like(self, path, params, url): - release = os.path.basename(url.path_info) - - return self.page_status_release_stable_oldstable_oldoldstable(release, params, url) - def page_status_release_testing(self, path, params, url): bf = BugFilter(params) diff --git a/doc/README.releases b/doc/README.releases index b2d66779d4..6754de0359 100644 --- a/doc/README.releases +++ b/doc/README.releases @@ -12,7 +12,6 @@ General Security Tracker code --------------------- See https://bugs.debian.org/783491 -[ ] bin/tracker_service.py [ ] Makefile Security Tracker host -- cgit v1.2.3 From 4720933b22a790188f76b20ab59faab37c5fc3e1 Mon Sep 17 00:00:00 2001 From: Emilio Pozuelo Monfort Date: Mon, 2 Dec 2019 17:17:19 +0100 Subject: Makefile: remove update-$(alias) targets They are aliases for the real update-$(dist) targets, and are unused as we mostly use update-packages target, which directly calls the update-$(dist) ones. --- Makefile | 18 ------------------ 1 file changed, 18 deletions(-) diff --git a/Makefile b/Makefile index 545a293228..cf6cc968c5 100644 --- a/Makefile +++ b/Makefile @@ -78,30 +78,12 @@ endef $(foreach release,$(RELEASES),$(eval $(call add_update_rule,$(release)))) # Define some common aliases -.PHONY: update-unstable update-testing update-stable update-oldstable update-oldoldstable -.PHONY: update-testing-security update-stable-security update-oldstable-security update-oldoldstable-security .PHONY: update-main update-security update-backports -update-unstable: update-sid -update-testing: update-$(TESTING) -update-testing-security: update-$(TESTING)_security -update-stable: update-$(STABLE) -update-stable-security: update-$(STABLE)_security -update-oldstable: update-$(OLDSTABLE) -update-oldstable-security: update-$(OLDSTABLE)_security -ifeq ($(OLDOLDSTABLE),) -update-oldoldstable: -update-oldoldstable-security: -else -update-oldoldstable: update-$(OLDOLDSTABLE) -update-oldoldstable-security: update-$(OLDOLDSTABLE)_security -endif update-main: $(foreach release,$(MAIN_RELEASES),update-$(release)) update-security: $(foreach release,$(SECURITY_RELEASES),update-$(release)_security) update-backports: $(foreach release,$(BACKPORT_RELEASES),update-$(release)_backports) supported-update-targets: - @echo -n "unstable testing stable oldstable oldoldstable " - @echo -n "testing-security stable-security oldstable-security oldoldstable-security " @echo -n "main security backports " @echo -n "$(RELEASES) " @echo -n "packages lists nvd" -- cgit v1.2.3 From 95d941e6e825a7e5a78933fd6c6db0b08cf3f65f Mon Sep 17 00:00:00 2001 From: Emilio Pozuelo Monfort Date: Mon, 2 Dec 2019 17:22:18 +0100 Subject: Don't hardcode architecture list in the Makefile Move it to config.json instead and grab it from there. --- Makefile | 6 ------ data/config.json | 5 +++++ lib/debian-releases.mk | 6 +++++- 3 files changed, 10 insertions(+), 7 deletions(-) diff --git a/Makefile b/Makefile index cf6cc968c5..0641a3b20a 100644 --- a/Makefile +++ b/Makefile @@ -10,12 +10,6 @@ TESTING = bullseye MIRROR = http://debian.csail.mit.edu/debian SECURITY_MIRROR = http://security.debian.org/debian-security -jessie_ARCHS = amd64 armel armhf i386 -stretch_ARCHS = amd64 arm64 armel armhf i386 mips mips64el mipsel ppc64el s390x -buster_ARCHS = amd64 arm64 armel armhf i386 mips mips64el mipsel ppc64el s390x -bullseye_ARCHS = amd64 arm64 armel armhf i386 mips64el mipsel ppc64el s390x -sid_ARCHS = amd64 arm64 armel armhf i386 mips64el mipsel ppc64el s390x - # The rest of the file should not need to be edited # Include the definitions of the releases to be fetched diff --git a/data/config.json b/data/config.json index 524f31de8d..59660ee0ed 100644 --- a/data/config.json +++ b/data/config.json @@ -59,6 +59,7 @@ "jessie-proposed-updates" ] }, + "architectures": [ "amd64", "armel", "armhf", "i386" ], "release": "oldoldstable" }, "stretch": { @@ -71,6 +72,7 @@ "stretch-proposed-updates" ] }, + "architectures": [ "amd64", "arm64", "armel", "armhf", "i386", "mips", "mips64el", "mipsel", "ppc64el", "s390x" ], "release": "oldstable" }, "buster": { @@ -83,6 +85,7 @@ "buster-proposed-updates" ] }, + "architectures": [ "amd64", "arm64", "armel", "armhf", "i386", "mips", "mips64el", "mipsel", "ppc64el", "s390x" ], "release": "stable" }, "bullseye": { @@ -95,6 +98,7 @@ "bullseye-proposed-updates" ] }, + "architectures": [ "amd64", "arm64", "armel", "armhf", "i386", "mips64el", "mipsel", "ppc64el", "s390x" ], "release": "testing" }, "bookworm": { @@ -114,6 +118,7 @@ "sid" ] }, + "architectures": [ "amd64", "arm64", "armel", "armhf", "i386", "mips64el", "mipsel", "ppc64el", "s390x" ], "release": "unstable" } }, diff --git a/lib/debian-releases.mk b/lib/debian-releases.mk index c868d6edfd..d09ac33be0 100644 --- a/lib/debian-releases.mk +++ b/lib/debian-releases.mk @@ -1,6 +1,10 @@ # This file defines the variables describing all Debian repositories # that need to be fetched in the "update-packages" process +define get_config = +$(shell jq -r $(1) 'data/config.json') +endef + # backports suites only have Sources.xz and respective Packages.xz # available. # Cf. as well https://bugs.debian.org/664866 @@ -12,7 +16,7 @@ MAIN_RELEASES := $(SECURITY_RELEASES) sid define add_main_release = $(1)_MIRROR = $$(MIRROR) $(1)_DIST = $(1) -$(1)_ARCHS ?= amd64 arm64 armel armhf i386 mips64el mipsel ppc64el s390x +$(1)_ARCHS = $(call get_config, '.distributions.$(1).architectures[]') $(1)_RELEASE = $(1) $(1)_SUBRELEASE = RELEASES += $(1) -- cgit v1.2.3 From c00c77714887ab7d1c62fc9a2d0008191d202874 Mon Sep 17 00:00:00 2001 From: Emilio Pozuelo Monfort Date: Mon, 2 Dec 2019 17:28:09 +0100 Subject: Makefile: don't hardcode Debian releases --- Makefile | 9 --------- doc/README.releases | 7 ++----- lib/debian-releases.mk | 4 ++-- 3 files changed, 4 insertions(+), 16 deletions(-) diff --git a/Makefile b/Makefile index 0641a3b20a..d58e9b0e7e 100644 --- a/Makefile +++ b/Makefile @@ -1,17 +1,8 @@ PYTHON_MODULES = $(wildcard lib/python/*.py) -# The following variables need to be kept up-to-date and can be adjusted -# currently unsupported releases can be commented out -OLDOLDSTABLE = jessie -OLDSTABLE = stretch -STABLE = buster -TESTING = bullseye - MIRROR = http://debian.csail.mit.edu/debian SECURITY_MIRROR = http://security.debian.org/debian-security -# The rest of the file should not need to be edited - # Include the definitions of the releases to be fetched include lib/*-releases.mk diff --git a/doc/README.releases b/doc/README.releases index 6754de0359..4dc50b1029 100644 --- a/doc/README.releases +++ b/doc/README.releases @@ -1,5 +1,7 @@ Checklist to perform when a new stable release is announced =========================================================== +See https://bugs.debian.org/783491 + General ------- @@ -9,11 +11,6 @@ General [ ] Update security-team.debian.org pages [ ] Update support information in static/distributions.json -Security Tracker code ---------------------- -See https://bugs.debian.org/783491 -[ ] Makefile - Security Tracker host --------------------- [ ] Check /srv/security-tracker.debian.org/website/bin diff --git a/lib/debian-releases.mk b/lib/debian-releases.mk index d09ac33be0..ecb72a23a6 100644 --- a/lib/debian-releases.mk +++ b/lib/debian-releases.mk @@ -9,8 +9,8 @@ endef # available. # Cf. as well https://bugs.debian.org/664866 #BACKPORT_RELEASES := $(OLDSTABLE) $(STABLE) -SECURITY_RELEASES := $(OLDOLDSTABLE) $(OLDSTABLE) $(STABLE) $(TESTING) -MAIN_RELEASES := $(SECURITY_RELEASES) sid +MAIN_RELEASES = $(call get_config, '.distributions | to_entries[] | select(.value.release) | .key') +SECURITY_RELEASES = $(filter-out sid, $(MAIN_RELEASES)) # Define the variables for the release on the main mirror define add_main_release = -- cgit v1.2.3 From 3ac09e46f236b87dc1ad2ace0bdaad54fef34aec Mon Sep 17 00:00:00 2001 From: Emilio Pozuelo Monfort Date: Mon, 2 Dec 2019 17:31:18 +0100 Subject: README.releases: update config.json on new releases --- doc/README.releases | 1 + 1 file changed, 1 insertion(+) diff --git a/doc/README.releases b/doc/README.releases index 4dc50b1029..9ce9731614 100644 --- a/doc/README.releases +++ b/doc/README.releases @@ -8,6 +8,7 @@ General [ ] Update doc/DSA.template [ ] bin/add-dsa-needed.sh [ ] bin/tracker_data.py +[ ] data/config.json [ ] Update security-team.debian.org pages [ ] Update support information in static/distributions.json -- cgit v1.2.3 From 83b92a8d31e616343a3e9b88ace75e8cca25a438 Mon Sep 17 00:00:00 2001 From: Emilio Pozuelo Monfort Date: Tue, 3 Dec 2019 13:25:04 +0100 Subject: lts-needs-forward-port: take releases from config.py --- bin/lts-needs-forward-port.py | 28 ++++++++++++++++++++-------- 1 file changed, 20 insertions(+), 8 deletions(-) diff --git a/bin/lts-needs-forward-port.py b/bin/lts-needs-forward-port.py index 4277a832bc..5a0770bb29 100755 --- a/bin/lts-needs-forward-port.py +++ b/bin/lts-needs-forward-port.py @@ -18,21 +18,33 @@ import argparse import collections +import os import sys -from tracker_data import TrackerData, RELEASES +from tracker_data import TrackerData + +def setup_path(): + dirname = os.path.dirname + base = dirname(dirname(os.path.realpath(sys.argv[0]))) + sys.path.insert(0, os.path.join(base, "lib", "python")) + +setup_path() +import config + +lts = config.get_supported_releases()[0] +next_lts = config.get_supported_releases()[1] +oldstable = config.get_release_codename('oldstable') -# lts is currently jessie, next_lts stretch LIST_NAMES = ( ('needs_fix_in_next_lts', - ('Issues that are unfixed in {next_lts} but fixed in {lts}' - ).format(**RELEASES)), + ('Issues that are unfixed in {} but fixed in {}' + ).format(next_lts, lts)), ('needs_review_in_next_lts', - ('Issues that are no-dsa in {next_lts} but fixed in {lts}' - ).format(**RELEASES)), + ('Issues that are no-dsa in {} but fixed in {}' + ).format(next_lts, lts)), ('fixed_via_pu_in_oldstable', - ('Issues that will be fixed via p-u in {oldstable}' - ).format(**RELEASES)), + ('Issues that will be fixed via p-u in {}' + ).format(oldstable)), ) -- cgit v1.2.3 From 95cacee6c89aaa218bcce4e23b3e61afa33bd290 Mon Sep 17 00:00:00 2001 From: Emilio Pozuelo Monfort Date: Tue, 3 Dec 2019 13:27:32 +0100 Subject: lts-needs-forward-port: add FIXME about looking at stable pu list --- bin/lts-needs-forward-port.py | 1 + 1 file changed, 1 insertion(+) diff --git a/bin/lts-needs-forward-port.py b/bin/lts-needs-forward-port.py index 5a0770bb29..62f8cc8936 100755 --- a/bin/lts-needs-forward-port.py +++ b/bin/lts-needs-forward-port.py @@ -76,6 +76,7 @@ def main(): if status_in_lts.status == 'resolved': # Package will be updated via the next oldstable # point release + # FIXME: when lts == oldstable, this should look at the stable pu list if (issue.name in tracker.oldstable_point_update and pkg in tracker.oldstable_point_update[issue.name]): add_to_list('fixed_via_pu_in_oldstable', pkg, issue) -- cgit v1.2.3 From 6ebadf9eadee2816703a443fc5b5569eaf75c860 Mon Sep 17 00:00:00 2001 From: Emilio Pozuelo Monfort Date: Tue, 3 Dec 2019 13:32:39 +0100 Subject: lts-bts: get LTS release from config.py --- bin/lts-bts | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/bin/lts-bts b/bin/lts-bts index 98df374c03..da9365721c 100755 --- a/bin/lts-bts +++ b/bin/lts-bts @@ -11,7 +11,15 @@ import sys import tempfile import warnings -from tracker_data import TrackerData, RELEASES +from tracker_data import TrackerData + +def setup_path(): + dirname = os.path.dirname + base = dirname(dirname(os.path.realpath(sys.argv[0]))) + sys.path.insert(0, os.path.join(base, "lib", "python")) + +setup_path() +import config from jinja2 import Template @@ -103,7 +111,7 @@ def main(): cc = 'debian-lts@lists.debian.org' team = 'lts' - release = RELEASES['lts'] + release = config.get_supported_releases()[0] # Basic check instructions = "packages/{}.txt".format(args.package) -- cgit v1.2.3 From c8c344738e5067611d58c842e6d496f4c947f2ac Mon Sep 17 00:00:00 2001 From: Emilio Pozuelo Monfort Date: Tue, 3 Dec 2019 13:39:42 +0100 Subject: lts-cve-triage: take lts releases from config.py --- bin/lts-cve-triage.py | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/bin/lts-cve-triage.py b/bin/lts-cve-triage.py index 9cb6306983..39b8b7f0a3 100755 --- a/bin/lts-cve-triage.py +++ b/bin/lts-cve-triage.py @@ -15,13 +15,26 @@ # You should have received a copy of the GNU General Public License # along with this file. If not, see . +import os import sys import argparse import collections -from tracker_data import TrackerData, RELEASES +from tracker_data import TrackerData from unsupported_packages import UnsupportedPackages, LimitedSupportPackages +def setup_path(): + dirname = os.path.dirname + base = dirname(dirname(os.path.realpath(sys.argv[0]))) + sys.path.insert(0, os.path.join(base, "lib", "python")) + +setup_path() +import config + +RELEASES = { + 'lts': config.get_supported_releases()[0], + 'next_lts': config.get_supported_releases()[1], +} def colored(x, *args, **kwargs): return x -- cgit v1.2.3 From 02357bdb0344913c35d3f1f55e6654183c0d4b73 Mon Sep 17 00:00:00 2001 From: Emilio Pozuelo Monfort Date: Tue, 3 Dec 2019 13:49:07 +0100 Subject: Call TrackerData's Issue::get_status() with release codenames --- bin/lts-cve-triage.py | 4 ++-- bin/lts-needs-forward-port.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/bin/lts-cve-triage.py b/bin/lts-cve-triage.py index 39b8b7f0a3..2191475a6b 100755 --- a/bin/lts-cve-triage.py +++ b/bin/lts-cve-triage.py @@ -113,8 +113,8 @@ for pkg in tracker.iterate_packages(): continue for issue in tracker.iterate_pkg_issues(pkg): - status_in_lts = issue.get_status('lts') - status_in_next_lts = issue.get_status('next_lts') + status_in_lts = issue.get_status([RELEASES['lts']) + status_in_next_lts = issue.get_status(RELEASES['next_lts']) if status_in_lts.status in ('not-affected', 'resolved'): continue diff --git a/bin/lts-needs-forward-port.py b/bin/lts-needs-forward-port.py index 62f8cc8936..7a4d24c1d4 100755 --- a/bin/lts-needs-forward-port.py +++ b/bin/lts-needs-forward-port.py @@ -67,8 +67,8 @@ def main(): for pkg in tracker.iterate_packages(): for issue in tracker.iterate_pkg_issues(pkg): - status_in_lts = issue.get_status('lts') - status_in_next_lts = issue.get_status('next_lts') + status_in_lts = issue.get_status(lts) + status_in_next_lts = issue.get_status(next_lts) if status_in_lts.status in ('not-affected', 'open'): continue -- cgit v1.2.3 From 9915494c9fd883c49e9044e75a744e750ed4be5e Mon Sep 17 00:00:00 2001 From: Emilio Pozuelo Monfort Date: Tue, 3 Dec 2019 13:49:53 +0100 Subject: tracker_data: remove RELEASES array This also removes the normalize_release method, and Issue::get_status() no longer supports passing aliases such as 'stable' or 'lts'. --- bin/tracker_data.py | 22 ---------------------- doc/README.releases | 1 - 2 files changed, 23 deletions(-) diff --git a/bin/tracker_data.py b/bin/tracker_data.py index 13eab0f4b8..b5f15c3976 100644 --- a/bin/tracker_data.py +++ b/bin/tracker_data.py @@ -21,27 +21,6 @@ import subprocess import requests import six -RELEASES = { - 'oldoldstable': 'jessie', - 'oldstable': 'stretch', - 'stable': 'buster', - 'testing': 'bullseye', - 'unstable': 'sid', - 'experimental': 'experimental', - # LTS specific aliases - 'lts': 'jessie', - 'next_lts': 'stretch', -} - - -def normalize_release(release): - if release in RELEASES: - return RELEASES[release] - elif release in RELEASES.values(): - return release - else: - raise ValueError("Unknown release: {}".format(release)) - class TrackerData(object): DATA_URL = "https://security-tracker.debian.org/tracker/data/json" @@ -189,7 +168,6 @@ class Issue(object): self.data = data def get_status(self, release): - release = normalize_release(release) data = self.data['releases'].get(release) if data is None: status = 'not-affected' diff --git a/doc/README.releases b/doc/README.releases index 9ce9731614..3305f3d7bc 100644 --- a/doc/README.releases +++ b/doc/README.releases @@ -7,7 +7,6 @@ General [ ] Update doc/DSA.template [ ] bin/add-dsa-needed.sh -[ ] bin/tracker_data.py [ ] data/config.json [ ] Update security-team.debian.org pages [ ] Update support information in static/distributions.json -- cgit v1.2.3 From 5c860cef30051f557bb167af3222f4c5ec61c9f9 Mon Sep 17 00:00:00 2001 From: Emilio Pozuelo Monfort Date: Tue, 3 Dec 2019 14:54:55 +0100 Subject: security_db: don't hardcode the list of supported releases At times there will just be two, so get that list from the config. --- lib/python/security_db.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/lib/python/security_db.py b/lib/python/security_db.py index a4281274a7..f77710ef5e 100644 --- a/lib/python/security_db.py +++ b/lib/python/security_db.py @@ -496,11 +496,12 @@ class DB: ORDER BY sp.name, st.urgency, st.bug_name""" % (testing, testing, testing)) - releases = (('stable', config.get_release_codename('stable')), - ('oldstable', config.get_release_codename('oldstable')), - ('oldoldstable', config.get_release_codename('oldoldstable'))) + releases = config.get_supported_releases() + releases.remove(config.get_release_codename('testing')) + releases.remove('sid') - for (name, nickname) in releases: + for release in releases: + alias = config.get_release_alias(release) cursor.execute( """CREATE TEMPORARY VIEW %s_status AS SELECT DISTINCT sp.name AS package, st.bug_name AS bug, @@ -527,7 +528,7 @@ class DB: AND secst.bug_name = st.bug_name AND secst.package = secp.rowid), 0) ORDER BY sp.name, urgency_to_number(urgency), st.bug_name""" - % (name, nickname, nickname, nickname, nickname)) + % (alias, release, release, release, release)) cursor.execute( """CREATE TEMPORARY VIEW debian_cve AS -- cgit v1.2.3