Initial KEV and EPSS mirror
This commit is contained in:
root
@@ -0,0 +1,71 @@
|
||||
name: vuln-exploits Synchronisation (lokaler Git-Cache)
|
||||
|
||||
on:
|
||||
schedule:
|
||||
- cron: "15 16 * * *"
|
||||
workflow_dispatch:
|
||||
|
||||
permissions:
|
||||
contents: write
|
||||
|
||||
concurrency:
|
||||
group: vuln-exploits-sync
|
||||
cancel-in-progress: false
|
||||
|
||||
jobs:
|
||||
mirror:
|
||||
runs-on: vuln-runner
|
||||
timeout-minutes: 360
|
||||
steps:
|
||||
- name: Rohdaten aktualisieren
|
||||
shell: bash
|
||||
env:
|
||||
GITEA_JOB_TOKEN: ${{ secrets.GITEA_TOKEN }}
|
||||
CACHE_ROOT: /var/lib/act_runner/cache/vuln-exploits
|
||||
run: |
|
||||
set -Eeuo pipefail
|
||||
umask 077
|
||||
: "${GITEA_JOB_TOKEN:?GITEA_TOKEN fehlt}"
|
||||
mkdir -p "$CACHE_ROOT"
|
||||
exec 9>"$CACHE_ROOT/.sync.lock"
|
||||
flock -w 300 9 || { echo "Bereits ein Lauf aktiv" >&2; exit 1; }
|
||||
|
||||
ASKPASS="$(mktemp "$CACHE_ROOT/.askpass.XXXXXX")"
|
||||
trap 'rm -f "$ASKPASS"' EXIT
|
||||
cat > "$ASKPASS" <<'ASKPASS_SCRIPT'
|
||||
#!/bin/sh
|
||||
case "$1" in
|
||||
*Username*|*username*) printf '%s\n' 'oauth2' ;;
|
||||
*Password*|*password*) printf '%s\n' "$GITEA_JOB_TOKEN" ;;
|
||||
*) exit 1 ;;
|
||||
esac
|
||||
ASKPASS_SCRIPT
|
||||
chmod 700 "$ASKPASS"
|
||||
export GIT_ASKPASS="$ASKPASS" GIT_TERMINAL_PROMPT=0
|
||||
REMOTE="https://git.apex-project.de/VULN/vuln-exploits.git"
|
||||
REPO="$CACHE_ROOT/repo"
|
||||
if [ ! -d "$REPO/.git" ]; then
|
||||
[ ! -e "$REPO" ] || { echo "Unvollstaendiger Cache: $REPO" >&2; exit 1; }
|
||||
git -c credential.helper= clone --branch main --single-branch --depth 1 "$REMOTE" "$REPO"
|
||||
else
|
||||
# Never reset an unfinished local commit or changes after a failed push.
|
||||
if [ -n "$(git -C "$REPO" status --porcelain)" ] || [ "$(git -C "$REPO" rev-list --count origin/main..HEAD)" != 0 ]; then
|
||||
echo "Lokale Aenderungen/Commits vorhanden: manuell pruefen, nicht ueberschreiben" >&2
|
||||
exit 1
|
||||
fi
|
||||
git -C "$REPO" remote set-url origin "$REMOTE"
|
||||
git -C "$REPO" -c credential.helper= fetch --no-tags --depth 1 origin main
|
||||
git -C "$REPO" checkout -B main FETCH_HEAD
|
||||
fi
|
||||
cd "$REPO"
|
||||
python3 scripts/sync.py
|
||||
python3 scripts/validate.py
|
||||
git config user.name "Vulnerability Bot"
|
||||
git config user.email "vuln-bot@users.noreply.local"
|
||||
git add -A -- data/ state/
|
||||
if git diff --cached --quiet; then
|
||||
echo "Keine neuen Daten"
|
||||
exit 0
|
||||
fi
|
||||
git commit -m "vuln-exploits update $(date -u +%Y-%m-%dT%H:%M:%SZ)"
|
||||
git -c credential.helper= push origin HEAD:main
|
||||
@@ -0,0 +1,5 @@
|
||||
__pycache__/
|
||||
*.pyc
|
||||
*.tmp
|
||||
.sync.lock
|
||||
.DS_Store
|
||||
@@ -0,0 +1,24 @@
|
||||
# vuln-exploits – CISA KEV + FIRST EPSS
|
||||
|
||||
Git-Mirror der **originalen** CISA-KEV-JSON-Datei und des täglichen FIRST-EPSS-CSV (gzip).
|
||||
|
||||
## Installation
|
||||
1. Leeres Gitea-Repository `VULN/vuln-exploits` erstellen und diese Dateien pushen.
|
||||
2. `sudo install -d -o act_runner -g act_runner -m 700 /var/lib/act_runner/cache/vuln-exploits`
|
||||
3. Bestehender Host-Runner mit Label `vuln-runner` und Schreibrecht des Job-Tokens.
|
||||
4. Workflow `.gitea/workflows/update.yml` über Actions starten.
|
||||
|
||||
## Dateistruktur
|
||||
- `data/cisa-kev/catalog.json`: kompletter KEV-Katalog, wird bei Änderungen ersetzt.
|
||||
- `data/epss/current.csv.gz`: aktuellster vollständiger EPSS-Snapshot, täglich ersetzt.
|
||||
- `state/sync.json`: Statistiken, letzter erfolgreicher Lauf.
|
||||
- Optional: `EPSS_DAILY_HISTORY=1` im Workflow-Environment ergänzen, um täglich ein historisches gzip zu speichern (kostet Git-Speicher).
|
||||
|
||||
## Betrieb
|
||||
- Täglich 16:15 UTC geplant, damit die EPSS-Tagesdatei üblicherweise veröffentlicht ist.
|
||||
- Downloads werden bei jedem Lauf erneut ausgeführt. Bei EPSS ändert sich der Gesamtdatensatz täglich: Ein Git-Cache vermeidet den erneuten Download des eigenen Repositories, nicht den EPSS-Tagesdownload.
|
||||
- Download und Validierung müssen erfolgreich sein, bevor Dateien ersetzt werden.
|
||||
|
||||
Quellen:
|
||||
- https://www.cisa.gov/sites/default/files/feeds/known_exploited_vulnerabilities.json
|
||||
- https://www.first.org/epss/data
|
||||
@@ -0,0 +1,80 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Store raw CISA KEV JSON and FIRST EPSS daily compressed CSV in Git."""
|
||||
import csv
|
||||
import gzip
|
||||
import io
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import tempfile
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
SOURCES = {
|
||||
'kev': 'https://www.cisa.gov/sites/default/files/feeds/known_exploited_vulnerabilities.json',
|
||||
'epss': 'https://epss.empiricalsecurity.com/epss_scores-current.csv.gz',
|
||||
}
|
||||
|
||||
def download(url):
|
||||
req = urllib.request.Request(url, headers={'User-Agent': 'vuln-exploits-mirror/1.0'})
|
||||
for retry in range(5):
|
||||
try:
|
||||
with urllib.request.urlopen(req, timeout=180) as resp:
|
||||
return resp.read()
|
||||
except urllib.error.HTTPError as exc:
|
||||
if exc.code not in (408, 429, 500, 502, 503, 504) or retry == 4:
|
||||
raise
|
||||
except (urllib.error.URLError, TimeoutError):
|
||||
if retry == 4:
|
||||
raise
|
||||
time.sleep(3 * 2**retry)
|
||||
|
||||
def atomic_write(path, content):
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
if path.exists() and path.read_bytes() == content:
|
||||
return False
|
||||
temp = path.with_name(path.name + '.tmp')
|
||||
try:
|
||||
temp.write_bytes(content)
|
||||
os.replace(temp, path)
|
||||
finally:
|
||||
temp.unlink(missing_ok=True)
|
||||
return True
|
||||
|
||||
def main():
|
||||
kev = download(SOURCES['kev'])
|
||||
data = json.loads(kev)
|
||||
records = data.get('vulnerabilities')
|
||||
if not isinstance(records, list) or len(records) < 1000 or data.get('count') != len(records):
|
||||
raise ValueError('Unexpected or truncated CISA KEV JSON')
|
||||
if not all(re.fullmatch(r'CVE-\d{4}-\d{4,}', r.get('cveID','')) for r in records):
|
||||
raise ValueError('CISA KEV contains invalid CVE IDs')
|
||||
epss = download(SOURCES['epss'])
|
||||
with gzip.GzipFile(fileobj=io.BytesIO(epss)) as zipped:
|
||||
csv_text = io.TextIOWrapper(zipped, encoding='utf-8-sig', newline='')
|
||||
reader = csv.reader(line for line in csv_text if not line.startswith('#'))
|
||||
header = next(reader)
|
||||
if header != ['cve', 'epss', 'percentile']:
|
||||
raise ValueError(f'Unexpected EPSS CSV header: {header}')
|
||||
rows = sum(1 for row in reader if len(row) == 3 and re.fullmatch(r'CVE-\d{4}-\d{4,}', row[0]))
|
||||
if rows < 10000:
|
||||
raise ValueError(f'EPSS snapshot too small: {rows}')
|
||||
# Only update files after both datasets validated.
|
||||
date = datetime.now(timezone.utc).strftime('%Y-%m-%d')
|
||||
output = ROOT/'data'
|
||||
changed_kev = atomic_write(output/'cisa-kev'/'catalog.json', kev)
|
||||
changed_epss = atomic_write(output/'epss'/'current.csv.gz', epss)
|
||||
# Daily snapshots are optional: keeping all of them significantly grows Git.
|
||||
if os.getenv('EPSS_DAILY_HISTORY', '0') == '1':
|
||||
atomic_write(output/'epss'/'history'/f'{date}.csv.gz', epss)
|
||||
state = {'version': 1, 'last_success': datetime.now(timezone.utc).isoformat(), 'kev_count': len(records), 'epss_count': rows,
|
||||
'epss_history_enabled': os.getenv('EPSS_DAILY_HISTORY', '0') == '1'}
|
||||
atomic_write(ROOT/'state'/'sync.json', (json.dumps(state, indent=2, sort_keys=True)+'\n').encode())
|
||||
print(f'KEV={len(records)} changed={changed_kev}, EPSS={rows} changed={changed_epss}')
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,11 @@
|
||||
#!/usr/bin/env python3
|
||||
import gzip
|
||||
import json
|
||||
from pathlib import Path
|
||||
root = Path(__file__).resolve().parents[1]
|
||||
state = json.loads((root/'state/sync.json').read_text())
|
||||
kev = json.loads((root/'data/cisa-kev/catalog.json').read_text())
|
||||
assert kev['count'] == len(kev['vulnerabilities']) == state['kev_count']
|
||||
with gzip.open(root/'data/epss/current.csv.gz', 'rt', encoding='utf-8') as file:
|
||||
assert any('cve,epss,percentile' in line for _, line in zip(range(10), file))
|
||||
print('CISA KEV and EPSS validated')
|
||||
Reference in New Issue
Block a user