diff --git a/scripts/build.py b/scripts/build.py index e2fb518..6905791 100644 --- a/scripts/build.py +++ b/scripts/build.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -"""Conservative SQLite index from CVEProject, NVD and OSV raw JSON. +"""Conservative SQLite index from CVEProject, NVD, OSV and Red Hat CSAF raw JSON. Do not mistake this index for a complete vulnerability matching engine. """ import argparse @@ -11,25 +11,24 @@ import shutil import sqlite3 import subprocess import tempfile +import sys from datetime import datetime, timezone from pathlib import Path +# Make the top-level importers package importable when invoked as scripts/build.py. +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) +from importers import IMPORTERS + CVE = re.compile(r'^CVE-\d{4}-\d{4,}$', re.I) -FILES = ('vulnerabilities','aliases','affected_packages','cpe_matches') +FILES = ('vulnerabilities','aliases','affected_packages','cpe_nodes','cpe_matches', + 'redhat_advisories','redhat_advisory_cves','sources','meta') +SCHEMA_VERSION = 3 def canonical(obj): return json.dumps(obj, ensure_ascii=False, sort_keys=True, separators=(',', ':')) -def sha(s): - return hashlib.sha256(s.encode('utf-8')).hexdigest()[:32] - - -def date(value): - return value if isinstance(value, str) and value else None - - def get_git_revision(path): return subprocess.check_output(['git', '-C', str(path), 'rev-parse', 'HEAD'], text=True).strip() @@ -44,168 +43,90 @@ def input_files(repo): yield from sorted(d.rglob('CVE-*.json')) return + if repo.name == 'vuln-redhat': + yield from sorted((d / 'csaf').rglob('*.json')) + return + # Andere Quellen unverändert verarbeiten yield from sorted(d.rglob('*.json')) -def upsert_vuln(db, vid, published=None, modified=None, state=None, summary=None): - if not vid or not (CVE.match(vid) or vid.startswith(('GHSA-','OSV-','PYSEC-','RUSTSEC-','GO-','DSA-','USN-'))): - return - db.execute('''INSERT INTO vulnerabilities(id,published,modified,state,summary) VALUES(?,?,?,?,?) - ON CONFLICT(id) DO UPDATE SET - published=COALESCE(excluded.published,vulnerabilities.published), - modified=COALESCE(excluded.modified,vulnerabilities.modified), - state=COALESCE(excluded.state,vulnerabilities.state), - summary=COALESCE(excluded.summary,vulnerabilities.summary)''', - (vid, published, modified, state, summary[:500] if isinstance(summary,str) else None)) - - -def process_cve(db, obj): - if not isinstance(obj, dict): - raise ValueError( - f"Unexpected CVE JSON type: {type(obj).__name__}; " - f"sample={str(obj)[:300]}" - ) - m = obj.get('cveMetadata') - if not isinstance(m,dict): - return False - vid=m.get('cveId') - if not isinstance(vid,str) or not CVE.match(vid): - return False - upsert_vuln(db,vid,m.get('datePublished'),m.get('dateUpdated'),m.get('state')) - return True - - -def walk_cpe(node, allow_simple=True): - """Only emit independent leaf matches for simple non-negated OR nodes. - Complex AND/NOT expressions must remain unsupported rather than cause false positives. - """ - if not isinstance(node,dict) or node.get('negate'): - return - if node.get('operator','OR').upper() != 'OR': - return - matches=node.get('cpeMatch') or [] - if isinstance(matches,list): - for match in matches: - if isinstance(match,dict): - yield match - for child in node.get('children') or []: - yield from walk_cpe(child) - - -def process_nvd(db,obj): - # NVD mirror stores the CVE object directly; API responses wrap it in {'cve': ...}. - if not isinstance(obj,dict): - return False - cve=obj.get('cve') if isinstance(obj.get('cve'),dict) else obj - vid=cve.get('id') - if not isinstance(vid,str) or not CVE.match(vid): - return False - upsert_vuln(db,vid,cve.get('published'),cve.get('lastModified'),cve.get('vulnStatus')) - db.execute('DELETE FROM cpe_matches WHERE vulnerability_id=? AND source=?',(vid,'vuln-nvd')) - for node in cve.get('configurations') or []: - # NVD root configurations may contain nodes; a configuration with several - # nodes can be an AND and is not safe to flatten. - if not isinstance(node,dict) or node.get('operator','OR').upper() != 'OR' or node.get('negate'): - continue - for n in node.get('nodes') or []: - for match in walk_cpe(n): - criteria=match.get('criteria') - if not isinstance(criteria,str): - continue - vals=(criteria, int(bool(match.get('vulnerable',False))), - match.get('versionStartIncluding'), match.get('versionStartExcluding'), - match.get('versionEndIncluding'),match.get('versionEndExcluding')) - key=sha(canonical((vid,vals))) - db.execute('''INSERT OR REPLACE INTO cpe_matches - (id,vulnerability_id,criteria,vulnerable,version_start_including,version_start_excluding,version_end_including,version_end_excluding,source) - VALUES(?,?,?,?,?,?,?,?,?)''',(key,vid,*vals,'vuln-nvd')) - return True - - -def process_osv(db,obj): - vid=obj.get('id') - if not isinstance(vid,str) or not vid: - return False - # Only capture OSV ids with a known prefix or valid CVE. - if not (CVE.match(vid) or vid.startswith(('GHSA-','OSV-','PYSEC-','RUSTSEC-','GO-','DSA-','USN-'))): - return False - upsert_vuln(db,vid,date(obj.get('published')),date(obj.get('modified')), - 'WITHDRAWN' if obj.get('withdrawn') else None,obj.get('summary')) - db.execute('DELETE FROM affected_packages WHERE vulnerability_id=? AND source=?',(vid,'vuln-osv')) - db.execute('DELETE FROM aliases WHERE vulnerability_id=?',(vid,)) - for a in obj.get('aliases') or []: - if isinstance(a,str) and a: - db.execute('INSERT OR IGNORE INTO aliases(alias,vulnerability_id) VALUES(?,?)',(a,vid)) - for i,a in enumerate(obj.get('affected') or []): - if not isinstance(a,dict): - continue - pkg=a.get('package') or {} - if not isinstance(pkg,dict): - continue - eco=pkg.get('ecosystem'); name=pkg.get('name') - ranges=a.get('ranges') or [] - versions=a.get('versions') or [] - if not isinstance(ranges,list) or not isinstance(versions,list): - continue - if not isinstance(eco,str) or not isinstance(name,str): - continue - # Preserve version/range event semantics; no lexicographic comparisons. - key=sha(canonical((vid,i,eco,name,ranges,versions))) - db.execute('INSERT OR REPLACE INTO affected_packages VALUES(?,?,?,?,?,?,?)', - (key,vid,eco,name,canonical(ranges),canonical(versions),'vuln-osv')) - return True - - -def stream_source(db,name,path): - count=0; bad=0 +def stream_source(db, name, path): + importer = IMPORTERS[name] + count = 0 + bad = 0 + scanned = 0 for file in input_files(path): + scanned += 1 try: - with file.open('r',encoding='utf-8') as f: - obj=json.load(f) - if name=='vuln-cve': - accepted=process_cve(db,obj) - elif name=='vuln-nvd': - accepted=process_nvd(db,obj) - elif name=='vuln-osv': - accepted=process_osv(db,obj) - else: - accepted=False - count+=int(accepted) - except (ValueError,UnicodeError,TypeError,sqlite3.Error) as exc: - bad+=1 - if bad<=5: - print(f'INVALID {name} {file}: {exc}',flush=True) + with file.open('r', encoding='utf-8') as stream: + obj = json.load(stream) + count += int(bool(importer(db, obj))) + except (ValueError, UnicodeError, TypeError, KeyError, sqlite3.Error) as exc: + bad += 1 + if bad <= 5: + print(f'INVALID {name} {file}: {exc}', flush=True) if bad: raise RuntimeError(f'{name}: {bad} invalid records; database not published') if name == 'vuln-nvd' and count == 0: raise RuntimeError('vuln-nvd: no usable documents; check source data and JSON shape') + if name in ('vuln-cve', 'vuln-osv', 'vuln-redhat') and count == 0: + raise RuntimeError(f'{name}: no usable documents; refusing to publish') + print(f'{name}: scanned={scanned} accepted={count}', flush=True) return count def dump_table(db,table): # memory efficient cursor; a canonical row object for deltas + # Tables added by new importers do not exist in the previous SQLite master. + if db.execute("SELECT 1 FROM sqlite_master WHERE type='table' AND name=?", (table,)).fetchone() is None: + return cols=[r[1] for r in db.execute(f'PRAGMA table_info({table})')] for row in db.execute(f'SELECT * FROM {table} ORDER BY 1'): yield row[0],dict(zip(cols,row)) -def make_delta(old_path,new_db,path): - old=sqlite3.connect(old_path) - with path.open('w',encoding='utf-8') as f: - for table in FILES: - prev=dump_table(old,table); curr=dump_table(new_db,table) - a=next(prev,None); b=next(curr,None) - while a is not None or b is not None: - if a is not None and (b is None or a[0]"$CACHE_ROOT/.builder.lock" flock -w 600 9 || { echo 'Another builder is running' >&2; exit 1; } python3 "$ROOT/scripts/sync_sources.py" python3 "$ROOT/scripts/build.py" --sources "$CACHE_ROOT/sources" --output "$CACHE_ROOT/releases" "$@" - diff --git a/scripts/sync_sources.py b/scripts/sync_sources.py index 5d2213a..726a524 100644 --- a/scripts/sync_sources.py +++ b/scripts/sync_sources.py @@ -2,6 +2,7 @@ #!/usr/bin/env python3 import json +from datetime import datetime, timezone import os import re import subprocess @@ -84,7 +85,8 @@ def sync_repository(owner, repo, branch): return { "repo": repo, "commit": result, - "path": str(path) + "path": str(path), + "synced_at": datetime.now(timezone.utc).isoformat(timespec="seconds").replace("+00:00", "Z") }