Initial KEV and EPSS mirror

.gitea/workflows/update.yml

@@ -0,0 +1,71 @@
+name: vuln-exploits Synchronisation (lokaler Git-Cache)
+
+on:
+  schedule:
+    - cron: "15 16 * * *"
+  workflow_dispatch:
+
+permissions:
+  contents: write
+
+concurrency:
+  group: vuln-exploits-sync
+  cancel-in-progress: false
+
+jobs:
+  mirror:
+    runs-on: vuln-runner
+    timeout-minutes: 360
+    steps:
+      - name: Rohdaten aktualisieren
+        shell: bash
+        env:
+          GITEA_JOB_TOKEN: ${{ secrets.GITEA_TOKEN }}
+          CACHE_ROOT: /var/lib/act_runner/cache/vuln-exploits
+        run: |
+          set -Eeuo pipefail
+          umask 077
+          : "${GITEA_JOB_TOKEN:?GITEA_TOKEN fehlt}"
+          mkdir -p "$CACHE_ROOT"
+          exec 9>"$CACHE_ROOT/.sync.lock"
+          flock -w 300 9 || { echo "Bereits ein Lauf aktiv" >&2; exit 1; }
+
+          ASKPASS="$(mktemp "$CACHE_ROOT/.askpass.XXXXXX")"
+          trap 'rm -f "$ASKPASS"' EXIT
+          cat > "$ASKPASS" <<'ASKPASS_SCRIPT'
+          #!/bin/sh
+          case "$1" in
+            *Username*|*username*) printf '%s\n' 'oauth2' ;;
+            *Password*|*password*) printf '%s\n' "$GITEA_JOB_TOKEN" ;;
+            *) exit 1 ;;
+          esac
+          ASKPASS_SCRIPT
+          chmod 700 "$ASKPASS"
+          export GIT_ASKPASS="$ASKPASS" GIT_TERMINAL_PROMPT=0
+          REMOTE="https://git.apex-project.de/VULN/vuln-exploits.git"
+          REPO="$CACHE_ROOT/repo"
+          if [ ! -d "$REPO/.git" ]; then
+            [ ! -e "$REPO" ] || { echo "Unvollstaendiger Cache: $REPO" >&2; exit 1; }
+            git -c credential.helper= clone --branch main --single-branch --depth 1 "$REMOTE" "$REPO"
+          else
+            # Never reset an unfinished local commit or changes after a failed push.
+            if [ -n "$(git -C "$REPO" status --porcelain)" ] || [ "$(git -C "$REPO" rev-list --count origin/main..HEAD)" != 0 ]; then
+              echo "Lokale Aenderungen/Commits vorhanden: manuell pruefen, nicht ueberschreiben" >&2
+              exit 1
+            fi
+            git -C "$REPO" remote set-url origin "$REMOTE"
+            git -C "$REPO" -c credential.helper= fetch --no-tags --depth 1 origin main
+            git -C "$REPO" checkout -B main FETCH_HEAD
+          fi
+          cd "$REPO"
+          python3 scripts/sync.py
+          python3 scripts/validate.py
+          git config user.name "Vulnerability Bot"
+          git config user.email "vuln-bot@users.noreply.local"
+          git add -A -- data/ state/
+          if git diff --cached --quiet; then
+            echo "Keine neuen Daten"
+            exit 0
+          fi
+          git commit -m "vuln-exploits update $(date -u +%Y-%m-%dT%H:%M:%SZ)"
+          git -c credential.helper= push origin HEAD:main

.gitignore

@@ -0,0 +1,5 @@
+__pycache__/
+*.pyc
+*.tmp
+.sync.lock
+.DS_Store

README.md

@@ -0,0 +1,24 @@
+# vuln-exploits – CISA KEV + FIRST EPSS
+
+Git-Mirror der **originalen** CISA-KEV-JSON-Datei und des täglichen FIRST-EPSS-CSV (gzip).
+
+## Installation
+1. Leeres Gitea-Repository `VULN/vuln-exploits` erstellen und diese Dateien pushen.
+2. `sudo install -d -o act_runner -g act_runner -m 700 /var/lib/act_runner/cache/vuln-exploits`
+3. Bestehender Host-Runner mit Label `vuln-runner` und Schreibrecht des Job-Tokens.
+4. Workflow `.gitea/workflows/update.yml` über Actions starten.
+
+## Dateistruktur
+- `data/cisa-kev/catalog.json`: kompletter KEV-Katalog, wird bei Änderungen ersetzt.
+- `data/epss/current.csv.gz`: aktuellster vollständiger EPSS-Snapshot, täglich ersetzt.
+- `state/sync.json`: Statistiken, letzter erfolgreicher Lauf.
+- Optional: `EPSS_DAILY_HISTORY=1` im Workflow-Environment ergänzen, um täglich ein historisches gzip zu speichern (kostet Git-Speicher).
+
+## Betrieb
+- Täglich 16:15 UTC geplant, damit die EPSS-Tagesdatei üblicherweise veröffentlicht ist.
+- Downloads werden bei jedem Lauf erneut ausgeführt. Bei EPSS ändert sich der Gesamtdatensatz täglich: Ein Git-Cache vermeidet den erneuten Download des eigenen Repositories, nicht den EPSS-Tagesdownload.
+- Download und Validierung müssen erfolgreich sein, bevor Dateien ersetzt werden.
+
+Quellen:
+- https://www.cisa.gov/sites/default/files/feeds/known_exploited_vulnerabilities.json
+- https://www.first.org/epss/data

scripts/sync.py

@@ -0,0 +1,80 @@
+#!/usr/bin/env python3
+"""Store raw CISA KEV JSON and FIRST EPSS daily compressed CSV in Git."""
+import csv
+import gzip
+import io
+import json
+import os
+import re
+import tempfile
+import time
+import urllib.error
+import urllib.request
+from datetime import datetime, timezone
+from pathlib import Path
+
+ROOT = Path(__file__).resolve().parents[1]
+SOURCES = {
+    'kev': 'https://www.cisa.gov/sites/default/files/feeds/known_exploited_vulnerabilities.json',
+    'epss': 'https://epss.empiricalsecurity.com/epss_scores-current.csv.gz',
+}
+
+def download(url):
+    req = urllib.request.Request(url, headers={'User-Agent': 'vuln-exploits-mirror/1.0'})
+    for retry in range(5):
+        try:
+            with urllib.request.urlopen(req, timeout=180) as resp:
+                return resp.read()
+        except urllib.error.HTTPError as exc:
+            if exc.code not in (408, 429, 500, 502, 503, 504) or retry == 4:
+                raise
+        except (urllib.error.URLError, TimeoutError):
+            if retry == 4:
+                raise
+        time.sleep(3 * 2**retry)
+
+def atomic_write(path, content):
+    path.parent.mkdir(parents=True, exist_ok=True)
+    if path.exists() and path.read_bytes() == content:
+        return False
+    temp = path.with_name(path.name + '.tmp')
+    try:
+        temp.write_bytes(content)
+        os.replace(temp, path)
+    finally:
+        temp.unlink(missing_ok=True)
+    return True
+
+def main():
+    kev = download(SOURCES['kev'])
+    data = json.loads(kev)
+    records = data.get('vulnerabilities')
+    if not isinstance(records, list) or len(records) < 1000 or data.get('count') != len(records):
+        raise ValueError('Unexpected or truncated CISA KEV JSON')
+    if not all(re.fullmatch(r'CVE-\d{4}-\d{4,}', r.get('cveID','')) for r in records):
+        raise ValueError('CISA KEV contains invalid CVE IDs')
+    epss = download(SOURCES['epss'])
+    with gzip.GzipFile(fileobj=io.BytesIO(epss)) as zipped:
+        csv_text = io.TextIOWrapper(zipped, encoding='utf-8-sig', newline='')
+        reader = csv.reader(line for line in csv_text if not line.startswith('#'))
+        header = next(reader)
+        if header != ['cve', 'epss', 'percentile']:
+            raise ValueError(f'Unexpected EPSS CSV header: {header}')
+        rows = sum(1 for row in reader if len(row) == 3 and re.fullmatch(r'CVE-\d{4}-\d{4,}', row[0]))
+        if rows < 10000:
+            raise ValueError(f'EPSS snapshot too small: {rows}')
+    # Only update files after both datasets validated.
+    date = datetime.now(timezone.utc).strftime('%Y-%m-%d')
+    output = ROOT/'data'
+    changed_kev = atomic_write(output/'cisa-kev'/'catalog.json', kev)
+    changed_epss = atomic_write(output/'epss'/'current.csv.gz', epss)
+    # Daily snapshots are optional: keeping all of them significantly grows Git.
+    if os.getenv('EPSS_DAILY_HISTORY', '0') == '1':
+        atomic_write(output/'epss'/'history'/f'{date}.csv.gz', epss)
+    state = {'version': 1, 'last_success': datetime.now(timezone.utc).isoformat(), 'kev_count': len(records), 'epss_count': rows,
+             'epss_history_enabled': os.getenv('EPSS_DAILY_HISTORY', '0') == '1'}
+    atomic_write(ROOT/'state'/'sync.json', (json.dumps(state, indent=2, sort_keys=True)+'\n').encode())
+    print(f'KEV={len(records)} changed={changed_kev}, EPSS={rows} changed={changed_epss}')
+
+if __name__ == '__main__':
+    main()

scripts/validate.py

@@ -0,0 +1,11 @@
+#!/usr/bin/env python3
+import gzip
+import json
+from pathlib import Path
+root = Path(__file__).resolve().parents[1]
+state = json.loads((root/'state/sync.json').read_text())
+kev = json.loads((root/'data/cisa-kev/catalog.json').read_text())
+assert kev['count'] == len(kev['vulnerabilities']) == state['kev_count']
+with gzip.open(root/'data/epss/current.csv.gz', 'rt', encoding='utf-8') as file:
+    assert any('cve,epss,percentile' in line for _, line in zip(range(10), file))
+print('CISA KEV and EPSS validated')