GNOME.org
 

[gnome-build-meta/jjardon/cve_report_3_32] Add CVE check job

commit c7300da81043794ac6cbad03808954392a400710
Author: Javier Jardón <jjardon gnome org>
Date:   Wed Mar 27 16:14:15 2019 +0000

    Add CVE check job
    
    This is all taken from freedesktop-sdk
    
    Eventually most of the scripts will be converted to buildstream plugin
    so It's easier to reuse

 .gitlab-ci.yml                     |  27 +++++++++
 elements/platform-manifest.bst     |   8 +++
 elements/sdk-manifest.bst          |   8 +++
 project.conf                       |   1 +
 utils/generate-cve-report.py       |  73 +++++++++++++++++++++++++
 utils/update-local-cve-database.py | 109 +++++++++++++++++++++++++++++++++++++
 6 files changed, 226 insertions(+)
---
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index eaa42121..0f3fbec6 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -14,6 +14,7 @@ variables:
   DOCKER_AARCH64: "${DOCKER_REGISTRY}/aarch64:e132e7bf9180b30c6ad0f4b057442cc2f2a0aa86"
 
 stages:
+  - reports
   - track
   - build
   - prepare_flatpak
@@ -200,6 +201,32 @@ before_script:
 # "Real" CI jobs
 #
 
+cve_report:
+  <<: *x86_64
+  stage: reports
+  script:
+    - pip3 install --user lxml
+
+    - ${BST} track --deps all platform-manifest.bst sdk-manifest.bst
+    - ${BST} build platform-manifest.bst
+    - ${BST} build sdk-manifest.bst
+
+    - ${BST} checkout platform-manifest.bst platform-manifest/
+    - ${BST} checkout sdk-manifest.bst sdk-manifest/
+
+    - mkdir -p "${XDG_CACHE_HOME}/cve"
+    - cd "${XDG_CACHE_HOME}/cve"
+    - python3 "${CI_PROJECT_DIR}/utils/update-local-cve-database.py"
+
+    - mkdir -p "${CI_PROJECT_DIR}/cve-reports"
+    - python3 "${CI_PROJECT_DIR}/utils/generate-cve-report.py" 
"${CI_PROJECT_DIR}/sdk-manifest/usr/manifest.json" "${CI_PROJECT_DIR}/cve-reports/sdk.html"
+    - python3 "${CI_PROJECT_DIR}/utils/generate-cve-report.py" 
"${CI_PROJECT_DIR}/platform-manifest/usr/manifest.json" "${CI_PROJECT_DIR}/cve-reports/platform.html"
+  artifacts:
+    paths:
+      - "${CI_PROJECT_DIR}/cve-reports"
+  only:
+    - gnome-3-32
+
 track:
   <<: *x86_64
   stage: track
diff --git a/elements/platform-manifest.bst b/elements/platform-manifest.bst
new file mode 100644
index 00000000..4afeac16
--- /dev/null
+++ b/elements/platform-manifest.bst
@@ -0,0 +1,8 @@
+kind: collect_manifest
+
+depends:
+  - filename: sdk-platform.bst
+    type: build
+
+config:
+  path: "/usr/manifest.json"
diff --git a/elements/sdk-manifest.bst b/elements/sdk-manifest.bst
new file mode 100644
index 00000000..436ba15f
--- /dev/null
+++ b/elements/sdk-manifest.bst
@@ -0,0 +1,8 @@
+kind: collect_manifest
+
+depends:
+  - filename: sdk.bst
+    type: build
+
+config:
+  path: "/usr/manifest.json"
diff --git a/project.conf b/project.conf
index 71fffeb6..e526855c 100644
--- a/project.conf
+++ b/project.conf
@@ -295,3 +295,4 @@ plugins:
   elements:
     flatpak_image: 0
     flatpak_repo: 0
+    collect_manifest: 0
diff --git a/utils/generate-cve-report.py b/utils/generate-cve-report.py
new file mode 100644
index 00000000..70c1b21e
--- /dev/null
+++ b/utils/generate-cve-report.py
@@ -0,0 +1,73 @@
+"""Generate and HTML output with all current CVEs for a given manifest.
+
+Usage:
+  python3 generate-cve-report.py path/to/manifest.json output.html
+
+This requires you to run update-local-cve-database.py first in the
+same current directory.
+"""
+
+import json
+import sys
+import sqlite3
+
+conn = sqlite3.connect('data-2.db')
+c = conn.cursor()
+
+with open(sys.argv[1], 'rb') as f:
+    manifest = json.load(f)
+
+with open(sys.argv[2], 'w') as out:
+    out.write('<!DOCTYPE html>\n')
+    out.write('<html><head><title>Report</title></head><body><table>\n')
+
+    entries = []
+    for module in manifest["modules"]:
+        name = module["name"]
+        sources = module["sources"]
+        cpe = module["x-cpe"]
+        product = cpe["product"]
+        version = cpe.get("version")
+        vendor = cpe.get("vendor")
+        patches = cpe.get("patches", [])
+        ignored = cpe.get("ignored", [])
+        if not version:
+            print("{} is missing a version".format(name))
+            continue
+
+        if vendor:
+            c.execute("""SELECT cve.id, cve.summary, cve.score FROM cve, product_vuln
+                             WHERE cve.id=product_vuln.cve_id
+                               AND product_vuln.name=?
+                               AND product_vuln.version=?
+                               AND product_vuln.vendor=?""",
+                      (product, version, vendor))
+        else:
+            c.execute("""SELECT cve.id, cve.summary, cve.score FROM cve, product_vuln
+                             WHERE cve.id=product_vuln.cve_id
+                               AND product_vuln.name=?
+                               AND product_vuln.version=?""",
+                      (product, version))
+        while True:
+            row = c.fetchone()
+            if row is None:
+                break
+            if row[0] in patches or row[0] in ignored:
+                continue
+            entries.append((row[0], name, version, row[1], row[2]))
+
+    def by_score(entry, ):
+        ID, name, version, summary, score = entry
+        try:
+            return float(score)
+        except ValueError:
+            return float("inf")
+
+    for ID, name, version, summary, score in sorted(entries, key=by_score, reverse=True):
+        out.write('<tr>')
+        out.write('<td><a href="https://nvd.nist.gov/vuln/detail/{ID}";>{ID}</a></td>'.format(ID=ID))
+        for d in [name, version, summary, score]:
+            out.write('<td>{}</td>'.format(d))
+        out.write('</tr>\n')
+
+    out.write('</table></html>\n')
diff --git a/utils/update-local-cve-database.py b/utils/update-local-cve-database.py
new file mode 100644
index 00000000..5af766a8
--- /dev/null
+++ b/utils/update-local-cve-database.py
@@ -0,0 +1,109 @@
+"""Downloads CVE database and store it as sqlite3 database.
+
+This tool does not take parameter. It will create files in the current
+directory:
+ - data.db: The sqlite3 database itself.
+ - nvdcve-2.0-*.xml.gz: The cached raw XML databases from the CVE database.
+
+Do not remove nvdcve-2.0-*.xml.gz files unless you remove
+data.db. data.db contains etags, and files would not be downloaded
+again if files are just removed.
+
+Files are not downloaded if not modified. But we still verify with the
+remote database we have the latest version of the files.
+"""
+
+import gzip
+import sys
+import sqlite3
+import datetime
+import itertools
+import urllib.request
+import urllib.parse
+from contextlib import contextmanager
+
+from lxml import etree as ET
+
+namespaces = {
+    "feed": "http://scap.nist.gov/schema/feed/vulnerability/2.0";,
+    "vuln": "http://scap.nist.gov/schema/vulnerability/0.4";,
+    "cvss": "http://scap.nist.gov/schema/cvss-v2/0.2";,
+}
+
+def extract_vulns(tree):
+    for entry in tree.iterfind("feed:entry", namespaces=namespaces):
+        cve_id = entry.find("vuln:cve-id", namespaces=namespaces).text
+        summary = entry.find("vuln:summary", namespaces=namespaces).text
+        score = entry.find("vuln:cvss/cvss:base_metrics/cvss:score", namespaces=namespaces)
+        yield cve_id, summary, score.text if score is not None else None
+
+def extract_product_vulns(tree):
+    for entry in tree.iterfind("feed:entry", namespaces=namespaces):
+        cve_id = entry.find("vuln:cve-id", namespaces=namespaces).text
+        for vuln_software in entry.iterfind("vuln:vulnerable-software-list", namespaces=namespaces):
+            for product in vuln_software.iterfind("vuln:product", namespaces=namespaces):
+                product_name = product.text
+                try:
+                    vendor, name, version = product_name.split(':')[2:5]
+                except ValueError:
+                    continue
+                yield cve_id, vendor, name, version
+
+def ensure_tables(c):
+    c.execute("""CREATE TABLE IF NOT EXISTS etags
+                 (year TEXT UNIQUE, etag TEXT)""")
+    c.execute("""CREATE TABLE IF NOT EXISTS cve
+                 (id TEXT UNIQUE, summary TEXT, score TEXT)""")
+    c.execute("""CREATE TABLE IF NOT EXISTS product_vuln
+                 (cve_id TEXT, name TEXT, vendor TEXT, version TEXT,
+                  UNIQUE(cve_id, name, vendor, version))""")
+
+def update_year(c, year):
+    url = 'https://nvd.nist.gov/feeds/xml/cve/2.0/nvdcve-2.0-{}.xml.gz'.format(year)
+    c.execute("SELECT etag FROM etags WHERE year=?", (year,))
+    row = c.fetchone()
+    if row is not None:
+        etag = row[0]
+    else:
+        etag = None
+
+    request = urllib.request.Request(url)
+    if etag is not None:
+        request.add_header('If-None-Match', etag)
+    try:
+        with urllib.request.urlopen(request) as resp:
+            new_etag = resp.getheader('ETag')
+            assert new_etag is not None
+            if new_etag is not None:
+                c.execute("INSERT OR REPLACE INTO etags (year, etag) VALUES (?, ?)", (year, new_etag))
+            with open('nvdcve-2.0-{}.xml.gz'.format(year), 'wb') as f:
+                while True:
+                    buf = resp.read(4096)
+                    if not buf:
+                        print("Downloaded {}".format(f.name))
+                        break
+                    f.write(buf)
+    except urllib.error.HTTPError as error:
+        if error.code != 304:
+            raise
+        print("Cached {}".format('nvdcve-2.0-{}.xml.gz'.format(year)))
+
+    with gzip.open('nvdcve-2.0-{}.xml.gz'.format(year)) as f:
+        tree = ET.parse(f)
+        for cve_id, summary, score in extract_vulns(tree):
+            c.execute("INSERT OR REPLACE INTO cve (id, summary, score) VALUES (?, ?, ?)", (cve_id, summary, 
score))
+
+        for cve_id, vendor, name, version in extract_product_vulns(tree):
+            c.execute("INSERT OR REPLACE INTO product_vuln (cve_id, name, vendor, version) VALUES (?, ?, ?, 
?)", (cve_id, name, vendor, version))
+
+if __name__ == '__main__':
+    conn = sqlite3.connect('data-2.db')
+    c = conn.cursor()
+    try:
+        ensure_tables(c)
+        for year in range(2002, datetime.datetime.now().year + 1):
+            update_year(c, str(year))
+        update_year(c, 'Modified')
+        conn.commit()
+    finally:
+        conn.close()
[Date Prev][Date Next]   [Thread Prev][Thread Next]   [Thread Index] [Date Index] [Author Index]