Modularize vulnerability importers

scripts/build.py

@@ -11,9 +11,14 @@ import shutil
 import sqlite3
 import subprocess
 import tempfile
+import sys
 from datetime import datetime, timezone
 from pathlib import Path
 
+# Make the top-level importers package importable when invoked as scripts/build.py.
+sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
+from importers import IMPORTERS
+
 CVE = re.compile(r'^CVE-\d{4}-\d{4,}$', re.I)
 FILES = ('vulnerabilities','aliases','affected_packages','cpe_matches')
 
@@ -22,14 +27,6 @@ def canonical(obj):
     return json.dumps(obj, ensure_ascii=False, sort_keys=True, separators=(',', ':'))
 
 
-def sha(s):
-    return hashlib.sha256(s.encode('utf-8')).hexdigest()[:32]
-
-
-def date(value):
-    return value if isinstance(value, str) and value else None
-
-
 def get_git_revision(path):
     return subprocess.check_output(['git', '-C', str(path), 'rev-parse', 'HEAD'], text=True).strip()
 
@@ -38,133 +35,38 @@ def input_files(repo):
     d = repo / 'data'
     if not d.exists():
         return
+
+    # Nur tatsächliche CVE-Datensätze einlesen
+    if repo.name == 'vuln-cve':
+        yield from sorted(d.rglob('CVE-*.json'))
+        return
+
+    # Andere Quellen unverändert verarbeiten
     yield from sorted(d.rglob('*.json'))
 
 
-def upsert_vuln(db, vid, published=None, modified=None, state=None, summary=None):
-    if not vid or not (CVE.match(vid) or vid.startswith(('GHSA-','OSV-','PYSEC-','RUSTSEC-','GO-','DSA-','USN-'))):
-        return
-    db.execute('''INSERT INTO vulnerabilities(id,published,modified,state,summary) VALUES(?,?,?,?,?)
-      ON CONFLICT(id) DO UPDATE SET
-      published=COALESCE(excluded.published,vulnerabilities.published),
-      modified=COALESCE(excluded.modified,vulnerabilities.modified),
-      state=COALESCE(excluded.state,vulnerabilities.state),
-      summary=COALESCE(excluded.summary,vulnerabilities.summary)''',
-      (vid, published, modified, state, summary[:500] if isinstance(summary,str) else None))
-
-
-def process_cve(db, obj):
-    m = obj.get('cveMetadata')
-    if not isinstance(m,dict):
-        return False
-    vid=m.get('cveId')
-    if not isinstance(vid,str) or not CVE.match(vid):
-        return False
-    upsert_vuln(db,vid,m.get('datePublished'),m.get('dateUpdated'),m.get('state'))
-    return True
-
-
-def walk_cpe(node, allow_simple=True):
-    """Only emit independent leaf matches for simple non-negated OR nodes.
-    Complex AND/NOT expressions must remain unsupported rather than cause false positives.
-    """
-    if not isinstance(node,dict) or node.get('negate'):
-        return
-    if node.get('operator','OR').upper() != 'OR':
-        return
-    matches=node.get('cpeMatch') or []
-    if isinstance(matches,list):
-        for match in matches:
-            if isinstance(match,dict):
-                yield match
-    for child in node.get('children') or []:
-        yield from walk_cpe(child)
-
-
-def process_nvd(db,obj):
-    cve=obj.get('cve')
-    if not isinstance(cve,dict):
-        return False
-    vid=cve.get('id')
-    if not isinstance(vid,str) or not CVE.match(vid):
-        return False
-    upsert_vuln(db,vid,cve.get('published'),cve.get('lastModified'),cve.get('vulnStatus'))
-    db.execute('DELETE FROM cpe_matches WHERE vulnerability_id=? AND source=?',(vid,'vuln-nvd'))
-    for node in cve.get('configurations') or []:
-        # NVD root configurations may contain nodes; a configuration with several
-        # nodes can be an AND and is not safe to flatten.
-        if not isinstance(node,dict) or node.get('operator','OR').upper() != 'OR' or node.get('negate'):
-            continue
-        for n in node.get('nodes') or []:
-            for match in walk_cpe(n):
-                criteria=match.get('criteria')
-                if not isinstance(criteria,str):
-                    continue
-                vals=(criteria, int(bool(match.get('vulnerable',False))),
-                      match.get('versionStartIncluding'), match.get('versionStartExcluding'),
-                      match.get('versionEndIncluding'),match.get('versionEndExcluding'))
-                key=sha(canonical((vid,vals)))
-                db.execute('''INSERT OR REPLACE INTO cpe_matches
-                  (id,vulnerability_id,criteria,vulnerable,version_start_including,version_start_excluding,version_end_including,version_end_excluding,source)
-                  VALUES(?,?,?,?,?,?,?,?,?)''',(key,vid,*vals,'vuln-nvd'))
-    return True
-
-
-def process_osv(db,obj):
-    vid=obj.get('id')
-    if not isinstance(vid,str) or not vid:
-        return False
-    # Only capture OSV ids with a known prefix or valid CVE.
-    if not (CVE.match(vid) or vid.startswith(('GHSA-','OSV-','PYSEC-','RUSTSEC-','GO-','DSA-','USN-'))):
-        return False
-    upsert_vuln(db,vid,date(obj.get('published')),date(obj.get('modified')),
-                'WITHDRAWN' if obj.get('withdrawn') else None,obj.get('summary'))
-    db.execute('DELETE FROM affected_packages WHERE vulnerability_id=? AND source=?',(vid,'vuln-osv'))
-    db.execute('DELETE FROM aliases WHERE vulnerability_id=?',(vid,))
-    for a in obj.get('aliases') or []:
-        if isinstance(a,str) and a:
-            db.execute('INSERT OR IGNORE INTO aliases(alias,vulnerability_id) VALUES(?,?)',(a,vid))
-    for i,a in enumerate(obj.get('affected') or []):
-        if not isinstance(a,dict):
-            continue
-        pkg=a.get('package') or {}
-        if not isinstance(pkg,dict):
-            continue
-        eco=pkg.get('ecosystem'); name=pkg.get('name')
-        ranges=a.get('ranges') or []
-        versions=a.get('versions') or []
-        if not isinstance(ranges,list) or not isinstance(versions,list):
-            continue
-        if not isinstance(eco,str) or not isinstance(name,str):
-            continue
-        # Preserve version/range event semantics; no lexicographic comparisons.
-        key=sha(canonical((vid,i,eco,name,ranges,versions)))
-        db.execute('INSERT OR REPLACE INTO affected_packages VALUES(?,?,?,?,?,?,?)',
-                   (key,vid,eco,name,canonical(ranges),canonical(versions),'vuln-osv'))
-    return True
-
-
-def stream_source(db,name,path):
-    count=0; bad=0
+def stream_source(db, name, path):
+    importer = IMPORTERS[name]
+    count = 0
+    bad = 0
+    scanned = 0
     for file in input_files(path):
+        scanned += 1
         try:
-            with file.open('r',encoding='utf-8') as f:
-                obj=json.load(f)
-            if name=='vuln-cve':
-                accepted=process_cve(db,obj)
-            elif name=='vuln-nvd':
-                accepted=process_nvd(db,obj)
-            elif name=='vuln-osv':
-                accepted=process_osv(db,obj)
-            else:
-                accepted=False
-            count+=int(accepted)
-        except (ValueError,UnicodeError,TypeError,sqlite3.Error) as exc:
-            bad+=1
-            if bad<=5:
-                print(f'INVALID {name} {file}: {exc}',flush=True)
+            with file.open('r', encoding='utf-8') as stream:
+                obj = json.load(stream)
+            count += int(bool(importer(db, obj)))
+        except (ValueError, UnicodeError, TypeError, KeyError, sqlite3.Error) as exc:
+            bad += 1
+            if bad <= 5:
+                print(f'INVALID {name} {file}: {exc}', flush=True)
     if bad:
         raise RuntimeError(f'{name}: {bad} invalid records; database not published')
+    if name == 'vuln-nvd' and count == 0:
+        raise RuntimeError('vuln-nvd: no usable documents; check source data and JSON shape')
+    if name in ('vuln-cve', 'vuln-osv') and count == 0:
+        raise RuntimeError(f'{name}: no usable documents; refusing to publish')
+    print(f'{name}: scanned={scanned} accepted={count}', flush=True)
     return count
 
 
@@ -197,34 +99,79 @@ def compress(src,out):
     subprocess.run(['zstd','-q','-f','-T1','-19',str(src),'-o',str(out)],check=True)
 
 
+def build_signature(source_revs, schema_path):
+    """Hash source revisions plus ALL executable importer logic and schema."""
+    root = Path(__file__).resolve().parents[1]
+    h = hashlib.sha256()
+    inputs = sorted((root / 'scripts').glob('*.py'))
+    inputs += sorted((root / 'importers').glob('*.py'))
+    inputs += [schema_path.resolve()]
+    for path in inputs:
+        if not path.is_file():
+            raise RuntimeError(f'Missing build input: {path}')
+        logical_name = path.relative_to(root).as_posix() if path.is_relative_to(root) else 'sql/schema.sql'
+        h.update(logical_name.encode('utf-8') + b'\0')
+        h.update(hashlib.sha256(path.read_bytes()).digest())
+    h.update(canonical(source_revs).encode('utf-8'))
+    return h.hexdigest()
+
+
+def existing_release_ok(output, manifest):
+    """Only skip if current release artifacts exist and match recorded hash."""
+    dbfile, packed = output / 'master.db', output / 'master.db.zst'
+    if not dbfile.is_file() or not packed.is_file() or not manifest.get('master_sha256'):
+        return False
+    h = hashlib.sha256()
+    with packed.open('rb') as stream:
+        for block in iter(lambda: stream.read(1024 * 1024), b''):
+            h.update(block)
+    return h.hexdigest() == manifest['master_sha256']
+
+
 def main():
     ap=argparse.ArgumentParser()
     ap.add_argument('--sources',type=Path,required=True)
     ap.add_argument('--output',type=Path,required=True)
     ap.add_argument('--schema',type=Path,default=Path(__file__).resolve().parents[1]/'sql/schema.sql')
+    ap.add_argument('--force', action='store_true', help='Force rebuild even when input signature is unchanged')
     args=ap.parse_args()
     args.output.mkdir(parents=True,exist_ok=True)
     active=[(p.name,p) for p in sorted(args.sources.iterdir()) if p.is_dir() and (p/'.git').exists()]
     if not active:
         raise SystemExit('No source repositories checked out; nothing published')
     # Fail closed if expected core sources are missing.
-    for name in ('vuln-cve','vuln-nvd','vuln-osv'):
+    for name in IMPORTERS:
         if name not in [n for n,_ in active]:
             raise SystemExit(f'Missing core source: {name}')
     source_revs={name:get_git_revision(path) for name,path in active}
-    old_db=args.output/'master.db'
+    signature = build_signature(source_revs, args.schema)
+    manifest_path = args.output / 'manifest.json'
+    previous_manifest = None
+    if manifest_path.is_file():
+        try:
+            previous_manifest = json.loads(manifest_path.read_text(encoding='utf-8'))
+        except (ValueError, UnicodeError) as exc:
+            raise RuntimeError(f'Invalid existing manifest, refusing to overwrite: {exc}') from exc
+    if (not args.force and previous_manifest
+            and previous_manifest.get('build_signature') == signature
+            and existing_release_ok(args.output, previous_manifest)):
+        print(f"NO_CHANGES: version={previous_manifest['version']} inputs unchanged; master and delta retained", flush=True)
+        return
+    if not args.force and previous_manifest and previous_manifest.get('build_signature') == signature:
+        print('WARNING: unchanged inputs but missing/invalid release artifact; rebuilding', flush=True)
+    old_db=args.output/'master.db' 
     with tempfile.TemporaryDirectory(prefix='vuln-build-',dir=args.output) as td:
         tmp=Path(td); new=tmp/'master.db'; db=sqlite3.connect(new)
         db.executescript(args.schema.read_text(encoding='utf-8'))
         stats={}
         for name,path in active:
-            if name in ('vuln-cve','vuln-nvd','vuln-osv'):
+            if name in IMPORTERS:
                 stats[name]=stream_source(db,name,path)
             else:
                 stats[name]=0
-            db.execute('INSERT INTO sources VALUES(?,?,?,?)',(name,source_revs[name],int(name in ('vuln-cve','vuln-nvd','vuln-osv')),stats[name]))
+            db.execute('INSERT INTO sources VALUES(?,?,?,?)',(name,source_revs[name],int(name in IMPORTERS),stats[name]))
             print(f'{name}: indexed={stats[name]} revision={source_revs[name][:12]}',flush=True)
-        if not any(stats.get(n) for n in ('vuln-cve','vuln-nvd','vuln-osv')):
+        if not any(stats.get(n) for n in IMPORTERS):
             raise RuntimeError('Core source indexes are empty; refusing to publish')
         old_version=None
         if old_db.is_file():
@@ -238,8 +185,10 @@ def main():
         assert db.execute('PRAGMA integrity_check').fetchone()[0]=='ok'
         db.close()
         manifest={'version':version,'schema_version':1,'previous':old_version,'source_commits':source_revs,
+                  'build_signature':signature,
                   'indexed_documents':stats,'format':'sqlite+zstd','delta_format':'jsonl+zstd',
-                  'note':'Only CVE/NVD/OSV indexed in v1; no complete version-matching engine'}
+                  'note':'CVE/NVD/OSV v1 index; AND/NOT CPE conditions are not evaluated as independent matches',
+                  'active_importers':sorted(IMPORTERS)}
         compress(new,tmp/'master.db.zst')
         if old_version:
             with sqlite3.connect(new) as nd:

scripts/run.sh

@@ -1,33 +1,12 @@
 #!/usr/bin/env bash
+# Run on the existing host-mode Gitea runner.
 set -Eeuo pipefail
 umask 077
-: "${CACHE_ROOT:=/vuln-cache/vuln-builder}"
-: "${GITEA_BASE_URL:=https://git.apex-project.de}"
-: "${GITEA_JOB_TOKEN:?GITEA_JOB_TOKEN missing}"
-mkdir -p "$CACHE_ROOT" "$CACHE_ROOT/sources" "$CACHE_ROOT/releases"
-exec 9>"$CACHE_ROOT/.build.lock"
-/usr/bin/flock -w 1800 9 || { echo 'Another builder run in progress' >&2; exit 1; }
-export GIT_CONFIG_COUNT=1 GIT_CONFIG_KEY_0=http.extraHeader
-GIT_CONFIG_VALUE_0="Authorization: Basic $(printf 'oauth2:%s' "$GITEA_JOB_TOKEN" | base64 | tr -d '\n')"
-export GIT_CONFIG_VALUE_0
-# Configurable list; extra feeds tracked in manifest but not yet indexed.
-SOURCES="${VULN_SOURCES:-vuln-cve vuln-nvd vuln-osv vuln-cpe vuln-exploits vuln-microsoft vuln-redhat vuln-oracle vuln-bsi vuln-debian vuln-ubuntu vuln-alpine vuln-suse vuln-suse-vex vuln-almalinux vuln-rocky}"
-for name in $SOURCES; do
-  [[ "$name" =~ ^vuln-[a-z0-9-]+$ ]] || { echo "Invalid source name: $name" >&2; exit 1; }
-  repo="$CACHE_ROOT/sources/$name"
-  url="${GITEA_BASE_URL%/}/VULN/$name.git"
-  if [ ! -d "$repo/.git" ]; then
-    echo "Cloning $name"
-    git clone --quiet --depth 1 --single-branch --branch main "$url" "$repo"
-  else
-    echo "Updating $name"
-    git -C "$repo" fetch --quiet --depth 1 origin main
-    git -C "$repo" reset --hard -q FETCH_HEAD
-    git -C "$repo" clean -fdq
-  fi
-done
-unset GIT_CONFIG_COUNT GIT_CONFIG_KEY_0 GIT_CONFIG_VALUE_0
-cd "$(dirname "$0")/.."
-python3 scripts/build.py --sources "$CACHE_ROOT/sources" --output "$CACHE_ROOT/releases"
-echo 'Release files:'
-ls -lh "$CACHE_ROOT/releases"/*.zst "$CACHE_ROOT/releases"/manifest.json
+ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
+CACHE_ROOT="${CACHE_ROOT:-/opt/gitea-builder/cache/vuln-builder}"
+export CACHE_ROOT
+mkdir -p "$CACHE_ROOT" "$CACHE_ROOT/releases"
+exec 9>"$CACHE_ROOT/.builder.lock"
+flock -w 600 9 || { echo 'Another builder is running' >&2; exit 1; }
+python3 "$ROOT/scripts/sync_sources.py"
+python3 "$ROOT/scripts/build.py" --sources "$CACHE_ROOT/sources" --output "$CACHE_ROOT/releases" "$@"

scripts/sync_sources.py

@@ -0,0 +1,120 @@
+
+#!/usr/bin/env python3
+
+import json
+import os
+import re
+import subprocess
+import urllib.request
+from pathlib import Path
+
+GITEA_URL = "https://git.apex-project.de"
+SOURCES_URL = (
+    f"{GITEA_URL}/VULN/vuln-status/"
+    "raw/branch/main/sources.json"
+)
+
+CACHE_ROOT = Path(
+    os.getenv("CACHE_ROOT", "/vuln-cache/vuln-builder")
+)
+SOURCES_DIR = CACHE_ROOT / "sources"
+
+TOKEN = os.getenv("SOURCE_READ_TOKEN", "")
+REPO_PATTERN = re.compile(r"^[a-zA-Z0-9_.-]+$")
+
+
+def load_sources():
+    request = urllib.request.Request(SOURCES_URL)
+
+    if TOKEN:
+        request.add_header("Authorization", f"token {TOKEN}")
+
+    with urllib.request.urlopen(request, timeout=60) as response:
+        config = json.load(response)
+
+    if not isinstance(config.get("sources"), list):
+        raise ValueError("Ungültige sources.json")
+
+    if not REPO_PATTERN.fullmatch(config.get("owner", "")):
+        raise ValueError("Ungültiger Repository-Owner")
+
+    if config.get("branch") != "main":
+        raise ValueError("Unerwarteter Git-Branch")
+
+    return config
+
+
+def git(*args, cwd=None):
+    subprocess.run(
+        ["git", *args],
+        cwd=cwd,
+        check=True,
+        env={
+            **os.environ,
+            "GIT_TERMINAL_PROMPT": "0"
+        }
+    )
+
+
+def sync_repository(owner, repo, branch):
+    if not REPO_PATTERN.fullmatch(repo):
+        raise ValueError(f"Ungültiger Repository-Name: {repo}")
+
+    path = SOURCES_DIR / repo
+    remote = f"{GITEA_URL}/{owner}/{repo}.git"
+
+    if not (path / ".git").is_dir():
+        git(
+            "clone",
+            "--depth", "1",
+            "--branch", branch,
+            remote,
+            str(path)
+        )
+    else:
+        git("fetch", "--depth", "1", "origin", branch, cwd=path)
+        git("reset", "--hard", "FETCH_HEAD", cwd=path)
+
+    result = subprocess.check_output(
+        ["git", "rev-parse", "HEAD"],
+        cwd=path,
+        text=True
+    ).strip()
+
+    return {
+        "repo": repo,
+        "commit": result,
+        "path": str(path)
+    }
+
+
+def main():
+    config = load_sources()
+    SOURCES_DIR.mkdir(parents=True, exist_ok=True)
+
+    result = []
+
+    for source in config["sources"]:
+        repo = source["repo"]
+        print(f"Synchronisiere {repo}", flush=True)
+
+        info = sync_repository(
+            config["owner"],
+            repo,
+            config["branch"]
+        )
+
+        result.append(info)
+
+    manifest = CACHE_ROOT / "source-manifest.json"
+
+    manifest.write_text(
+        json.dumps(result, indent=2),
+        encoding="utf-8"
+    )
+
+    print(f"{len(result)} Repositories synchronisiert")
+
+
+if __name__ == "__main__":
+    main()