From 4d0646300dacb41ec837baf2b4cc13268fcf177c Mon Sep 17 00:00:00 2001
From: Florian Weimer <fw@deneb.enyo.de>
Date: Thu, 20 Nov 2008 15:15:24 +0000
Subject: bin/update-nvd: Remove duplicates before loading NVD data

git-svn-id: svn+ssh://svn.debian.org/svn/secure-testing@10437 e39458fd-73e7-0310-bf30-c45bca0a0e42
---
 bin/update-nvd | 8 ++++++++
 1 file changed, 8 insertions(+)

(limited to 'bin/update-nvd')

diff --git a/bin/update-nvd b/bin/update-nvd
index 88fe39450b..281c047dd9 100644
--- a/bin/update-nvd
+++ b/bin/update-nvd
@@ -34,6 +34,14 @@ for name in sys.argv[1:]:
     data += nvd.parse(f)
     f.close()
 
+# For some reason, NVD adds duplicates, so we need to get rid of them.
+# Sort afterwords to increase locality in the insert process.
+deduplicate = {}
+for x in data:
+    deduplicate[x[0]] = x
+data = deduplicate.values()
+data.sort()
+
 cursor = db.writeTxn()
 if incremental:
     db.updateNVD(cursor, data)
-- 
cgit v1.2.3