Initial CVE mirror setup

2026-11-10 17:48:35 +01:00
commit f9e656cf85
7 changed files with 146 additions and 0 deletions
@@ -0,0 +1 @@
+*.json text eol=lf
@@ -0,0 +1,39 @@
+name: Sync CVE List V5
+on:
+  schedule:
+    - cron: '17 * * * *'
+  workflow_dispatch:
+permissions:
+  contents: write
+concurrency:
+  group: cve-mirror
+  cancel-in-progress: false
+jobs:
+  mirror:
+    runs-on: [self-hosted, linux, x64, vulnerability]
+    timeout-minutes: 240
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          persist-credentials: true
+          fetch-depth: 1
+      - name: Verify tools
+        run: |
+          git --version
+          rsync --version | head -1
+          python3 --version
+      - name: Download, verify, and mirror
+        run: python3 scripts/sync.py --since-year 2017
+      - name: Validate mirror
+        run: python3 scripts/validate.py
+      - name: Commit and push changed records
+        run: |
+          git config user.name 'vulnerability-bot'
+          git config user.email 'vulnerability-bot@users.noreply.github.com'
+          git add -A data state
+          if git diff --cached --quiet; then
+            echo 'No changes'
+            exit 0
+          fi
+          git commit -m "Sync CVE records ($(date -u +%Y-%m-%dT%H:%M:%SZ))"
+          git push origin HEAD:main
@@ -0,0 +1,4 @@
+__pycache__/
+*.pyc
+.cache/
+.work/
@@ -0,0 +1,16 @@
+# vuln-cve
+
+Mirror der offiziellen [CVE List V5](https://github.com/CVEProject/cvelistV5), gefiltert auf CVE-ID-Jahre ab 2017. Alle Records werden unverändert unter `data/YYYY/<bucket>/CVE-....json` versioniert. Der Upstream-Git-Commit und der letzte Sync stehen in `state/sync.json`.
+
+## Einrichtung
+
+1. GitHub-Repository `vuln-cve` erstellen, Branch `main`.
+2. Linux Self-hosted Runner mit Labels `self-hosted`, `linux`, `x64`, `vulnerability` registrieren. `git`, `rsync` und `python3` installieren.
+3. In GitHub Actions Schreibzugriff für `GITHUB_TOKEN` zulassen. Branch-Regeln müssen Bot-Pushes erlauben.
+4. Workflow `Sync CVE List V5` manuell starten; danach läuft er stündlich.
+
+Lokal: `python3 scripts/sync.py --since-year 2017 && python3 scripts/validate.py`.
+
+**Kapazität:** der erste Sync enthält sehr viele kleine Dateien und benötigt viel Speicher, I/O und Git-Objekte. GitHub-Datei-/Repository-Grenzen beachten; bei wachsender Datenmenge in Jahres-Repositories aufteilen. Das Skript führt absichtlich einen vollständigen Upstream-Abgleich statt einer potentiell unvollständigen `delta.json`-Auswertung durch.
+
+**Sicherheit:** Repository ausschließlich mit vertrauenswürdigen Runnern aktualisieren. `rsync --delete` löscht nur Inhalte unter `data/`, wenn der Upstream mindestens 1.000 gültige CVE-Records enthielt und *alle* Quelldateien validiert wurden. Der Sync bewahrt nur CVE-Jahrgänge ab 2017 auf. Historische Änderungen vor dem ersten Import sind nicht enthalten; Git dokumentiert Änderungen ab dem ersten eigenen Commit.
@@ -0,0 +1,70 @@
+#!/usr/bin/env python3
+"""Mirror official CVE List V5 current snapshot (2017+) into this repository."""
+import argparse
+import datetime as dt
+import json
+import os
+from pathlib import Path
+import re
+import shutil
+import subprocess
+import tempfile
+
+UPSTREAM = 'https://github.com/CVEProject/cvelistV5.git'
+ROOT = Path(__file__).resolve().parents[1]
+
+def run(*args, cwd=None):
+    return subprocess.check_output(args, cwd=cwd, text=True).strip()
+
+def main():
+    ap = argparse.ArgumentParser()
+    ap.add_argument('--since-year', type=int, default=2017)
+    ap.add_argument('--source-url', default=UPSTREAM)
+    ap.add_argument('--minimum-records', type=int, default=1000)
+    args = ap.parse_args()
+    now_year = dt.datetime.now(dt.timezone.utc).year
+    if not 1999 <= args.since_year <= now_year:
+        ap.error('invalid since-year')
+    if not shutil.which('git') or not shutil.which('rsync'):
+        raise SystemExit('git and rsync are required')
+    source_dir = ROOT / 'data'
+    state_dir = ROOT / 'state'
+    source_dir.mkdir(exist_ok=True)
+    state_dir.mkdir(exist_ok=True)
+    with tempfile.TemporaryDirectory(prefix='vuln-cve-') as tmp:
+        clone = Path(tmp) / 'source'
+        subprocess.run(['git', 'clone', '--depth=1', '--filter=blob:none', '--sparse', args.source_url, str(clone)], check=True)
+        years = [f'cves/{y}' for y in range(args.since_year, now_year + 1)]
+        subprocess.run(['git', '-C', str(clone), 'sparse-checkout', 'set', *years], check=True)
+        revision = run('git', 'rev-parse', 'HEAD', cwd=clone)
+        record_root = clone / 'cves'
+        candidates = list(record_root.rglob('CVE-*.json'))
+        if len(candidates) < args.minimum_records:
+            raise RuntimeError(f'Upstream has only {len(candidates)} CVE records: refusing destructive synchronization')
+        # Validate JSON structure and identifier across the entire source before deleting local entries.
+        validated = 0
+        for p in candidates:
+            if not re.fullmatch(r'CVE-\d{4}-\d{4,}', p.stem):
+                raise RuntimeError(f'Unexpected file name: {p}')
+            try:
+                with p.open(encoding='utf-8') as handle:
+                    record = json.load(handle)
+                if record.get('cveMetadata', {}).get('cveId') != p.stem:
+                    raise ValueError('metadata CVE ID mismatch')
+            except Exception as exc:
+                raise RuntimeError(f'Invalid upstream CVE record {p}: {exc}') from exc
+            validated += 1
+        # Delete files not present upstream, including records withdrawn or removed.
+        subprocess.run(['rsync', '-a', '--delete', '--exclude=.gitkeep', str(record_root) + '/', str(source_dir) + '/'], check=True)
+        state = {
+            'source': args.source_url,
+            'upstream_commit': revision,
+            'since_year': args.since_year,
+            'records': validated,
+            'synced_at_utc': dt.datetime.now(dt.timezone.utc).isoformat(),
+        }
+        (state_dir / 'sync.json').write_text(json.dumps(state, ensure_ascii=False, indent=2) + '\n', encoding='utf-8')
+        print(f'Mirrored {validated} CVE records at {revision}')
+
+if __name__ == '__main__':
+    main()
@@ -0,0 +1,16 @@
+#!/usr/bin/env python3
+"""Validate mirrored records and the sync manifest."""
+import json
+from pathlib import Path
+
+root = Path(__file__).resolve().parents[1]
+state = json.loads((root / 'state/sync.json').read_text())
+files = list((root / 'data').rglob('CVE-*.json'))
+if not files or len(files) != state['records']:
+    raise SystemExit(f'Record count mismatch: expected {state["records"]}, got {len(files)}')
+for path in files:
+    with path.open(encoding='utf-8') as f:
+        record = json.load(f)
+    if record.get('cveMetadata', {}).get('cveId') != path.stem:
+        raise SystemExit(f'Invalid record {path}')
+print(f'Validated {len(files)} CVE records')