commit f9e656cf856fc2411c13e9bcb053bdcbc4fd435f Author: root Date: Tue Nov 10 17:48:35 2026 +0100 Initial CVE mirror setup diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000000..4ee01478dc --- /dev/null +++ b/.gitattributes @@ -0,0 +1 @@ +*.json text eol=lf diff --git a/.github/workflows/update.yml b/.github/workflows/update.yml new file mode 100644 index 0000000000..fab58c4ca6 --- /dev/null +++ b/.github/workflows/update.yml @@ -0,0 +1,39 @@ +name: Sync CVE List V5 +on: + schedule: + - cron: '17 * * * *' + workflow_dispatch: +permissions: + contents: write +concurrency: + group: cve-mirror + cancel-in-progress: false +jobs: + mirror: + runs-on: [self-hosted, linux, x64, vulnerability] + timeout-minutes: 240 + steps: + - uses: actions/checkout@v4 + with: + persist-credentials: true + fetch-depth: 1 + - name: Verify tools + run: | + git --version + rsync --version | head -1 + python3 --version + - name: Download, verify, and mirror + run: python3 scripts/sync.py --since-year 2017 + - name: Validate mirror + run: python3 scripts/validate.py + - name: Commit and push changed records + run: | + git config user.name 'vulnerability-bot' + git config user.email 'vulnerability-bot@users.noreply.github.com' + git add -A data state + if git diff --cached --quiet; then + echo 'No changes' + exit 0 + fi + git commit -m "Sync CVE records ($(date -u +%Y-%m-%dT%H:%M:%SZ))" + git push origin HEAD:main diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000000..4943878ad2 --- /dev/null +++ b/.gitignore @@ -0,0 +1,4 @@ +__pycache__/ +*.pyc +.cache/ +.work/ diff --git a/README.md b/README.md new file mode 100644 index 0000000000..b1217a31c4 --- /dev/null +++ b/README.md @@ -0,0 +1,16 @@ +# vuln-cve + +Mirror der offiziellen [CVE List V5](https://github.com/CVEProject/cvelistV5), gefiltert auf CVE-ID-Jahre ab 2017. Alle Records werden unverändert unter `data/YYYY//CVE-....json` versioniert. Der Upstream-Git-Commit und der letzte Sync stehen in `state/sync.json`. + +## Einrichtung + +1. GitHub-Repository `vuln-cve` erstellen, Branch `main`. +2. Linux Self-hosted Runner mit Labels `self-hosted`, `linux`, `x64`, `vulnerability` registrieren. `git`, `rsync` und `python3` installieren. +3. In GitHub Actions Schreibzugriff für `GITHUB_TOKEN` zulassen. Branch-Regeln müssen Bot-Pushes erlauben. +4. Workflow `Sync CVE List V5` manuell starten; danach läuft er stündlich. + +Lokal: `python3 scripts/sync.py --since-year 2017 && python3 scripts/validate.py`. + +**Kapazität:** der erste Sync enthält sehr viele kleine Dateien und benötigt viel Speicher, I/O und Git-Objekte. GitHub-Datei-/Repository-Grenzen beachten; bei wachsender Datenmenge in Jahres-Repositories aufteilen. Das Skript führt absichtlich einen vollständigen Upstream-Abgleich statt einer potentiell unvollständigen `delta.json`-Auswertung durch. + +**Sicherheit:** Repository ausschließlich mit vertrauenswürdigen Runnern aktualisieren. `rsync --delete` löscht nur Inhalte unter `data/`, wenn der Upstream mindestens 1.000 gültige CVE-Records enthielt und *alle* Quelldateien validiert wurden. Der Sync bewahrt nur CVE-Jahrgänge ab 2017 auf. Historische Änderungen vor dem ersten Import sind nicht enthalten; Git dokumentiert Änderungen ab dem ersten eigenen Commit. diff --git a/data/.gitkeep b/data/.gitkeep new file mode 100644 index 0000000000..e69de29bb2 diff --git a/scripts/sync.py b/scripts/sync.py new file mode 100644 index 0000000000..66541e0068 --- /dev/null +++ b/scripts/sync.py @@ -0,0 +1,70 @@ +#!/usr/bin/env python3 +"""Mirror official CVE List V5 current snapshot (2017+) into this repository.""" +import argparse +import datetime as dt +import json +import os +from pathlib import Path +import re +import shutil +import subprocess +import tempfile + +UPSTREAM = 'https://github.com/CVEProject/cvelistV5.git' +ROOT = Path(__file__).resolve().parents[1] + +def run(*args, cwd=None): + return subprocess.check_output(args, cwd=cwd, text=True).strip() + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument('--since-year', type=int, default=2017) + ap.add_argument('--source-url', default=UPSTREAM) + ap.add_argument('--minimum-records', type=int, default=1000) + args = ap.parse_args() + now_year = dt.datetime.now(dt.timezone.utc).year + if not 1999 <= args.since_year <= now_year: + ap.error('invalid since-year') + if not shutil.which('git') or not shutil.which('rsync'): + raise SystemExit('git and rsync are required') + source_dir = ROOT / 'data' + state_dir = ROOT / 'state' + source_dir.mkdir(exist_ok=True) + state_dir.mkdir(exist_ok=True) + with tempfile.TemporaryDirectory(prefix='vuln-cve-') as tmp: + clone = Path(tmp) / 'source' + subprocess.run(['git', 'clone', '--depth=1', '--filter=blob:none', '--sparse', args.source_url, str(clone)], check=True) + years = [f'cves/{y}' for y in range(args.since_year, now_year + 1)] + subprocess.run(['git', '-C', str(clone), 'sparse-checkout', 'set', *years], check=True) + revision = run('git', 'rev-parse', 'HEAD', cwd=clone) + record_root = clone / 'cves' + candidates = list(record_root.rglob('CVE-*.json')) + if len(candidates) < args.minimum_records: + raise RuntimeError(f'Upstream has only {len(candidates)} CVE records: refusing destructive synchronization') + # Validate JSON structure and identifier across the entire source before deleting local entries. + validated = 0 + for p in candidates: + if not re.fullmatch(r'CVE-\d{4}-\d{4,}', p.stem): + raise RuntimeError(f'Unexpected file name: {p}') + try: + with p.open(encoding='utf-8') as handle: + record = json.load(handle) + if record.get('cveMetadata', {}).get('cveId') != p.stem: + raise ValueError('metadata CVE ID mismatch') + except Exception as exc: + raise RuntimeError(f'Invalid upstream CVE record {p}: {exc}') from exc + validated += 1 + # Delete files not present upstream, including records withdrawn or removed. + subprocess.run(['rsync', '-a', '--delete', '--exclude=.gitkeep', str(record_root) + '/', str(source_dir) + '/'], check=True) + state = { + 'source': args.source_url, + 'upstream_commit': revision, + 'since_year': args.since_year, + 'records': validated, + 'synced_at_utc': dt.datetime.now(dt.timezone.utc).isoformat(), + } + (state_dir / 'sync.json').write_text(json.dumps(state, ensure_ascii=False, indent=2) + '\n', encoding='utf-8') + print(f'Mirrored {validated} CVE records at {revision}') + +if __name__ == '__main__': + main() diff --git a/scripts/validate.py b/scripts/validate.py new file mode 100644 index 0000000000..b6be029d2c --- /dev/null +++ b/scripts/validate.py @@ -0,0 +1,16 @@ +#!/usr/bin/env python3 +"""Validate mirrored records and the sync manifest.""" +import json +from pathlib import Path + +root = Path(__file__).resolve().parents[1] +state = json.loads((root / 'state/sync.json').read_text()) +files = list((root / 'data').rglob('CVE-*.json')) +if not files or len(files) != state['records']: + raise SystemExit(f'Record count mismatch: expected {state["records"]}, got {len(files)}') +for path in files: + with path.open(encoding='utf-8') as f: + record = json.load(f) + if record.get('cveMetadata', {}).get('cveId') != path.stem: + raise SystemExit(f'Invalid record {path}') +print(f'Validated {len(files)} CVE records')