From 55381d81e74848e9106e3da5100bd04f519181ae Mon Sep 17 00:00:00 2001 From: root Date: Tue, 10 Nov 2026 20:55:39 +0100 Subject: [PATCH] Initial KEV and EPSS mirror --- .gitea/workflows/update.yml | 71 ++++++++++++++++++++++++++++++++ .gitignore | 5 +++ README.md | 24 +++++++++++ data/.gitkeep | 0 scripts/sync.py | 80 +++++++++++++++++++++++++++++++++++++ scripts/validate.py | 11 +++++ state/.gitkeep | 0 7 files changed, 191 insertions(+) create mode 100644 .gitea/workflows/update.yml create mode 100644 .gitignore create mode 100644 README.md create mode 100644 data/.gitkeep create mode 100644 scripts/sync.py create mode 100644 scripts/validate.py create mode 100644 state/.gitkeep diff --git a/.gitea/workflows/update.yml b/.gitea/workflows/update.yml new file mode 100644 index 0000000..3eeff45 --- /dev/null +++ b/.gitea/workflows/update.yml @@ -0,0 +1,71 @@ +name: vuln-exploits Synchronisation (lokaler Git-Cache) + +on: + schedule: + - cron: "15 16 * * *" + workflow_dispatch: + +permissions: + contents: write + +concurrency: + group: vuln-exploits-sync + cancel-in-progress: false + +jobs: + mirror: + runs-on: vuln-runner + timeout-minutes: 360 + steps: + - name: Rohdaten aktualisieren + shell: bash + env: + GITEA_JOB_TOKEN: ${{ secrets.GITEA_TOKEN }} + CACHE_ROOT: /var/lib/act_runner/cache/vuln-exploits + run: | + set -Eeuo pipefail + umask 077 + : "${GITEA_JOB_TOKEN:?GITEA_TOKEN fehlt}" + mkdir -p "$CACHE_ROOT" + exec 9>"$CACHE_ROOT/.sync.lock" + flock -w 300 9 || { echo "Bereits ein Lauf aktiv" >&2; exit 1; } + + ASKPASS="$(mktemp "$CACHE_ROOT/.askpass.XXXXXX")" + trap 'rm -f "$ASKPASS"' EXIT + cat > "$ASKPASS" <<'ASKPASS_SCRIPT' + #!/bin/sh + case "$1" in + *Username*|*username*) printf '%s\n' 'oauth2' ;; + *Password*|*password*) printf '%s\n' "$GITEA_JOB_TOKEN" ;; + *) exit 1 ;; + esac + ASKPASS_SCRIPT + chmod 700 "$ASKPASS" + export GIT_ASKPASS="$ASKPASS" GIT_TERMINAL_PROMPT=0 + REMOTE="https://git.apex-project.de/VULN/vuln-exploits.git" + REPO="$CACHE_ROOT/repo" + if [ ! -d "$REPO/.git" ]; then + [ ! -e "$REPO" ] || { echo "Unvollstaendiger Cache: $REPO" >&2; exit 1; } + git -c credential.helper= clone --branch main --single-branch --depth 1 "$REMOTE" "$REPO" + else + # Never reset an unfinished local commit or changes after a failed push. + if [ -n "$(git -C "$REPO" status --porcelain)" ] || [ "$(git -C "$REPO" rev-list --count origin/main..HEAD)" != 0 ]; then + echo "Lokale Aenderungen/Commits vorhanden: manuell pruefen, nicht ueberschreiben" >&2 + exit 1 + fi + git -C "$REPO" remote set-url origin "$REMOTE" + git -C "$REPO" -c credential.helper= fetch --no-tags --depth 1 origin main + git -C "$REPO" checkout -B main FETCH_HEAD + fi + cd "$REPO" + python3 scripts/sync.py + python3 scripts/validate.py + git config user.name "Vulnerability Bot" + git config user.email "vuln-bot@users.noreply.local" + git add -A -- data/ state/ + if git diff --cached --quiet; then + echo "Keine neuen Daten" + exit 0 + fi + git commit -m "vuln-exploits update $(date -u +%Y-%m-%dT%H:%M:%SZ)" + git -c credential.helper= push origin HEAD:main diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..1a6b2a7 --- /dev/null +++ b/.gitignore @@ -0,0 +1,5 @@ +__pycache__/ +*.pyc +*.tmp +.sync.lock +.DS_Store diff --git a/README.md b/README.md new file mode 100644 index 0000000..60820d7 --- /dev/null +++ b/README.md @@ -0,0 +1,24 @@ +# vuln-exploits – CISA KEV + FIRST EPSS + +Git-Mirror der **originalen** CISA-KEV-JSON-Datei und des täglichen FIRST-EPSS-CSV (gzip). + +## Installation +1. Leeres Gitea-Repository `VULN/vuln-exploits` erstellen und diese Dateien pushen. +2. `sudo install -d -o act_runner -g act_runner -m 700 /var/lib/act_runner/cache/vuln-exploits` +3. Bestehender Host-Runner mit Label `vuln-runner` und Schreibrecht des Job-Tokens. +4. Workflow `.gitea/workflows/update.yml` über Actions starten. + +## Dateistruktur +- `data/cisa-kev/catalog.json`: kompletter KEV-Katalog, wird bei Änderungen ersetzt. +- `data/epss/current.csv.gz`: aktuellster vollständiger EPSS-Snapshot, täglich ersetzt. +- `state/sync.json`: Statistiken, letzter erfolgreicher Lauf. +- Optional: `EPSS_DAILY_HISTORY=1` im Workflow-Environment ergänzen, um täglich ein historisches gzip zu speichern (kostet Git-Speicher). + +## Betrieb +- Täglich 16:15 UTC geplant, damit die EPSS-Tagesdatei üblicherweise veröffentlicht ist. +- Downloads werden bei jedem Lauf erneut ausgeführt. Bei EPSS ändert sich der Gesamtdatensatz täglich: Ein Git-Cache vermeidet den erneuten Download des eigenen Repositories, nicht den EPSS-Tagesdownload. +- Download und Validierung müssen erfolgreich sein, bevor Dateien ersetzt werden. + +Quellen: +- https://www.cisa.gov/sites/default/files/feeds/known_exploited_vulnerabilities.json +- https://www.first.org/epss/data diff --git a/data/.gitkeep b/data/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/scripts/sync.py b/scripts/sync.py new file mode 100644 index 0000000..5312a67 --- /dev/null +++ b/scripts/sync.py @@ -0,0 +1,80 @@ +#!/usr/bin/env python3 +"""Store raw CISA KEV JSON and FIRST EPSS daily compressed CSV in Git.""" +import csv +import gzip +import io +import json +import os +import re +import tempfile +import time +import urllib.error +import urllib.request +from datetime import datetime, timezone +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +SOURCES = { + 'kev': 'https://www.cisa.gov/sites/default/files/feeds/known_exploited_vulnerabilities.json', + 'epss': 'https://epss.empiricalsecurity.com/epss_scores-current.csv.gz', +} + +def download(url): + req = urllib.request.Request(url, headers={'User-Agent': 'vuln-exploits-mirror/1.0'}) + for retry in range(5): + try: + with urllib.request.urlopen(req, timeout=180) as resp: + return resp.read() + except urllib.error.HTTPError as exc: + if exc.code not in (408, 429, 500, 502, 503, 504) or retry == 4: + raise + except (urllib.error.URLError, TimeoutError): + if retry == 4: + raise + time.sleep(3 * 2**retry) + +def atomic_write(path, content): + path.parent.mkdir(parents=True, exist_ok=True) + if path.exists() and path.read_bytes() == content: + return False + temp = path.with_name(path.name + '.tmp') + try: + temp.write_bytes(content) + os.replace(temp, path) + finally: + temp.unlink(missing_ok=True) + return True + +def main(): + kev = download(SOURCES['kev']) + data = json.loads(kev) + records = data.get('vulnerabilities') + if not isinstance(records, list) or len(records) < 1000 or data.get('count') != len(records): + raise ValueError('Unexpected or truncated CISA KEV JSON') + if not all(re.fullmatch(r'CVE-\d{4}-\d{4,}', r.get('cveID','')) for r in records): + raise ValueError('CISA KEV contains invalid CVE IDs') + epss = download(SOURCES['epss']) + with gzip.GzipFile(fileobj=io.BytesIO(epss)) as zipped: + csv_text = io.TextIOWrapper(zipped, encoding='utf-8-sig', newline='') + reader = csv.reader(line for line in csv_text if not line.startswith('#')) + header = next(reader) + if header != ['cve', 'epss', 'percentile']: + raise ValueError(f'Unexpected EPSS CSV header: {header}') + rows = sum(1 for row in reader if len(row) == 3 and re.fullmatch(r'CVE-\d{4}-\d{4,}', row[0])) + if rows < 10000: + raise ValueError(f'EPSS snapshot too small: {rows}') + # Only update files after both datasets validated. + date = datetime.now(timezone.utc).strftime('%Y-%m-%d') + output = ROOT/'data' + changed_kev = atomic_write(output/'cisa-kev'/'catalog.json', kev) + changed_epss = atomic_write(output/'epss'/'current.csv.gz', epss) + # Daily snapshots are optional: keeping all of them significantly grows Git. + if os.getenv('EPSS_DAILY_HISTORY', '0') == '1': + atomic_write(output/'epss'/'history'/f'{date}.csv.gz', epss) + state = {'version': 1, 'last_success': datetime.now(timezone.utc).isoformat(), 'kev_count': len(records), 'epss_count': rows, + 'epss_history_enabled': os.getenv('EPSS_DAILY_HISTORY', '0') == '1'} + atomic_write(ROOT/'state'/'sync.json', (json.dumps(state, indent=2, sort_keys=True)+'\n').encode()) + print(f'KEV={len(records)} changed={changed_kev}, EPSS={rows} changed={changed_epss}') + +if __name__ == '__main__': + main() diff --git a/scripts/validate.py b/scripts/validate.py new file mode 100644 index 0000000..c9546f3 --- /dev/null +++ b/scripts/validate.py @@ -0,0 +1,11 @@ +#!/usr/bin/env python3 +import gzip +import json +from pathlib import Path +root = Path(__file__).resolve().parents[1] +state = json.loads((root/'state/sync.json').read_text()) +kev = json.loads((root/'data/cisa-kev/catalog.json').read_text()) +assert kev['count'] == len(kev['vulnerabilities']) == state['kev_count'] +with gzip.open(root/'data/epss/current.csv.gz', 'rt', encoding='utf-8') as file: + assert any('cve,epss,percentile' in line for _, line in zip(range(10), file)) +print('CISA KEV and EPSS validated') diff --git a/state/.gitkeep b/state/.gitkeep new file mode 100644 index 0000000..e69de29