commit 54c164197de14474f54ee34d1f0cb16713281275 Author: root Date: Tue Nov 10 21:44:49 2026 +0100 Initial BSI CSAF mirror diff --git a/.gitea/workflows/update.yml b/.gitea/workflows/update.yml new file mode 100644 index 0000000..d7f27fd --- /dev/null +++ b/.gitea/workflows/update.yml @@ -0,0 +1,47 @@ +name: BSI WID CSAF Mirror + +on: + workflow_dispatch: + schedule: + - cron: '23 */4 * * *' + +permissions: + contents: write + +concurrency: + group: vuln-bsi-sync + cancel-in-progress: false + +jobs: + mirror: + runs-on: vuln-runner + timeout-minutes: 360 + steps: + - name: Sync BSI CSAF using persistent Git checkout + env: + GITEA_JOB_TOKEN: ${{ secrets.GITEA_TOKEN }} + CACHE_ROOT: /var/lib/act_runner/cache/vuln-bsi + REPO_URL: https://git.apex-project.de/VULN/vuln-bsi.git + run: | + set -Eeuo pipefail + # Workflow runs from Gitea's job workspace; cache repo may not exist yet. + # On first run, fetch script from repository using authenticated Git. + umask 077 + tmpdir="$(mktemp -d)" + trap 'rm -rf "$tmpdir"' EXIT + cat > "$tmpdir/askpass" <<'EOF' + #!/bin/sh + case "$1" in + *Username*) printf '%s\n' 'oauth2' ;; + *Password*) printf '%s\n' "$GITEA_JOB_TOKEN" ;; + esac + EOF + chmod 700 "$tmpdir/askpass" + export GIT_ASKPASS="$tmpdir/askpass" GIT_TERMINAL_PROMPT=0 + if [ ! -f "$CACHE_ROOT/repo/scripts/run-cached.sh" ]; then + git clone --depth 1 --branch main "$REPO_URL" "$tmpdir/bootstrap" + exec_path="$tmpdir/bootstrap/scripts/run-cached.sh" + else + exec_path="$CACHE_ROOT/repo/scripts/run-cached.sh" + fi + bash "$exec_path" diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..7b7c296 --- /dev/null +++ b/.gitignore @@ -0,0 +1,4 @@ +__pycache__/ +*.pyc +.venv/ +*.tmp diff --git a/README.md b/README.md new file mode 100644 index 0000000..061ab56 --- /dev/null +++ b/README.md @@ -0,0 +1,49 @@ +# vuln-bsi — BSI WID CSAF WHITE Mirror + +Spiegelt die öffentlich zugänglichen CSAF-Dokumente des BSI WID / CERT-Bund ab **2017** in das eigene Gitea-Repository. + +Quelle: https://wid.cert-bund.de/.well-known/csaf/white/index.txt + +## Einrichtung + +1. Leeres Repository `VULN/vuln-bsi` in Gitea erstellen; Inhalt dieses Ordners auf `main` pushen. +2. Auf der Gitea-VM den Cache anlegen: + +```bash +sudo install -d -o act_runner -g act_runner -m 700 /var/lib/act_runner/cache/vuln-bsi +sudo apt update && sudo apt install -y git python3 util-linux ca-certificates +``` + +3. Actions aktivieren, Runner-Label `vuln-runner` bereitstellen und Workflow manuell starten. +4. Der Gitea-Job-Token wird über `secrets.GITEA_TOKEN` verwendet. `permissions: contents: write` muss auf der Instanz unterstützt und erlaubt sein. + +Automatisch: `23 */4 * * *` (UTC). Ein vollständiger Erstlauf kann eine längere Zeit und einige GB Git-Speicher benötigen. + +## Datenstruktur + +- `data/index.txt`: unveränderter offizieller Index +- `data/csaf/YYYY/*.json`: vollständige CSAF-Rohdateien +- `state/sync.json`: Verzeichnisliste, Prüf-Cursor, letzter erfolgreicher Lauf + +## Synchronisationslogik + +- Bei Erstlauf alle Einträge ab 2017 herunterladen. +- Später neue und fehlende Dateien sowie 300 rotierende Altdateien abrufen; Änderungen werden an den Datei-Inhalten erkannt. +- Dateien, die aus dem Index verschwinden, erst nach vollständigem erfolgreichem Abruf entfernen. +- Index, Dateiinhalte und Synchronisationsstatus validieren, dann Commit und Push. +- **Kein** komplett neuer Git-Checkout bei jedem Lauf: permanenter Checkout unter `/var/lib/act_runner/cache/vuln-bsi/repo`. + +### Einschränkungen + +- Geänderte ältere Dokumente werden erst beim rotierenden Prüflauf erkannt. `changes.csv` wird aktuell nicht interpretiert; für schnellere Updates kann diese Quelle später ergänzt werden. +- Nach einem abgebrochenen Lauf bleibt das Arbeitsverzeichnis bewusst unverändert und muss geprüft werden, bevor ein weiterer Lauf startet. Das verhindert versehentliche Datenverluste. +- Ein Git-Index mit vielen Rohdaten kann erhebliche Größe erreichen. +- Der Runner arbeitet direkt auf der Gitea-VM. Nur vertrauenswürdige Workflows verwenden. +- Ein vollständiger Live-Abruf ist noch nicht verifiziert. + +### Manuell lokal synchronisieren + +```bash +python3 scripts/sync.py --since-year 2017 --full +python3 scripts/validate.py +``` diff --git a/scripts/run-cached.sh b/scripts/run-cached.sh new file mode 100644 index 0000000..88b0ff8 --- /dev/null +++ b/scripts/run-cached.sh @@ -0,0 +1,46 @@ +#!/usr/bin/env bash +set -Eeuo pipefail +umask 077 +: "${GITEA_JOB_TOKEN:?GITEA job token missing}" +CACHE_ROOT="${CACHE_ROOT:-/var/lib/act_runner/cache/vuln-bsi}" +REPO_URL="${REPO_URL:-https://git.apex-project.de/VULN/vuln-bsi.git}" +mkdir -p "$CACHE_ROOT" +exec 9>"$CACHE_ROOT/.sync.lock" +flock -w 300 9 || { echo 'Another mirror run holds the lock' >&2; exit 1; } +ASKPASS_FILE="$(mktemp "$CACHE_ROOT/.askpass.XXXXXX")" +cleanup() { rm -f "$ASKPASS_FILE"; } +trap cleanup EXIT +cat > "$ASKPASS_FILE" <<'EOF' +#!/bin/sh +case "$1" in + *Username*) printf '%s\n' 'oauth2' ;; + *Password*) printf '%s\n' "$GITEA_JOB_TOKEN" ;; +esac +EOF +chmod 700 "$ASKPASS_FILE" +export GIT_ASKPASS="$ASKPASS_FILE" GIT_TERMINAL_PROMPT=0 +export GIT_CONFIG_NOSYSTEM=0 +if [[ ! -d "$CACHE_ROOT/repo/.git" ]]; then + git clone --depth 1 --branch main "$REPO_URL" "$CACHE_ROOT/repo" +fi +cd "$CACHE_ROOT/repo" +# Never overwrite a previous interrupted sync silently. +if [[ -n "$(git status --porcelain)" ]]; then + echo 'Cached working tree is dirty. Inspect/recover it before retrying.' >&2 + exit 1 +fi +git remote set-url origin "$REPO_URL" +git fetch --depth 1 origin main +git checkout main +git merge --ff-only FETCH_HEAD +python3 scripts/sync.py --since-year 2017 --refresh-old 300 +python3 scripts/validate.py +git config user.name 'Vulnerability Bot' +git config user.email 'vulnerability-bot@users.noreply.local' +git add -A data state +if git diff --cached --quiet; then + echo 'No changes to commit' + exit 0 +fi +git commit -m "Update BSI WID CSAF $(date -u +%Y-%m-%dT%H:%M:%SZ)" +git push origin HEAD:main diff --git a/scripts/sync.py b/scripts/sync.py new file mode 100644 index 0000000..c284840 --- /dev/null +++ b/scripts/sync.py @@ -0,0 +1,139 @@ +#!/usr/bin/env python3 +"""Mirror public BSI WID CSAF WHITE advisories using authoritative index.txt.""" +import argparse +import hashlib +import json +import logging +import os +from pathlib import Path +import random +import re +import sys +import time +from datetime import datetime, timezone +from urllib.error import HTTPError, URLError +from urllib.request import Request, urlopen + +BASE = 'https://wid.cert-bund.de/.well-known/csaf/white/' +PATH_RE = re.compile(r'^([12][0-9]{3})/([a-zA-Z0-9_.-]+\.json)$') +LOG = logging.getLogger('bsi') + + +def fetch(url, timeout=90, retries=5): + for attempt in range(retries): + try: + req = Request(url, headers={'User-Agent': 'vuln-bsi-mirror/1.0 (+https://git.apex-project.de/VULN/vuln-bsi)', 'Accept': 'application/json,text/plain,*/*'}) + with urlopen(req, timeout=timeout) as response: + return response.read() + except (HTTPError, URLError, TimeoutError, OSError) as exc: + if attempt + 1 == retries: + raise RuntimeError(f'Failed to download {url}: {exc}') from exc + delay = min(90, 2 ** attempt * 3 + random.random() * 2) + LOG.warning('Download failed (%s), retry in %.1fs: %s', exc, delay, url) + time.sleep(delay) + + +def parse_index(raw, min_year): + try: + lines = raw.decode('utf-8-sig').splitlines() + except UnicodeError as exc: + raise ValueError('index.txt is not valid UTF-8') from exc + all_entries = [x.strip() for x in lines if x.strip()] + if not all_entries: + raise ValueError('Empty index; refusing to remove documents') + if any(not PATH_RE.fullmatch(x) for x in all_entries): + bad = next(x for x in all_entries if not PATH_RE.fullmatch(x)) + raise ValueError(f'Unexpected index entry: {bad[:140]!r}') + if len(all_entries) != len(set(all_entries)): + raise ValueError('Duplicate entries in index') + selected = sorted(x for x in all_entries if int(x.split('/', 1)[0]) >= min_year) + if not selected: + raise ValueError('No entries in requested years') + return selected + + +def validate_document(raw, path): + try: + obj = json.loads(raw) + except (ValueError, UnicodeError) as exc: + raise ValueError(f'Invalid JSON in {path}') from exc + if not isinstance(obj, dict) or not isinstance(obj.get('document'), dict): + raise ValueError(f'Not a CSAF document: {path}') + doc = obj['document'] + tracking = doc.get('tracking') + if not isinstance(tracking, dict) or not tracking.get('id') or not tracking.get('current_release_date'): + raise ValueError(f'Missing CSAF tracking fields: {path}') + return obj + + +def atomic_bytes(path, raw): + path.parent.mkdir(parents=True, exist_ok=True) + temp = path.with_name(path.name + '.tmp') + try: + temp.write_bytes(raw) + os.replace(temp, path) + finally: + temp.unlink(missing_ok=True) + + +def main(): + p = argparse.ArgumentParser() + p.add_argument('--since-year', type=int, default=2017) + p.add_argument('--refresh-old', type=int, default=300, help='Rotate old documents to check changed advisories each run') + p.add_argument('--full', action='store_true', help='Re-fetch every indexed document') + p.add_argument('--delay', type=float, default=0.15) + p.add_argument('--root', default='.') + args = p.parse_args() + logging.basicConfig(level=logging.INFO, format='%(asctime)s %(levelname)s %(message)s') + root = Path(args.root).resolve() + statefile = root / 'state' / 'sync.json' + state = json.loads(statefile.read_text()) if statefile.exists() else {} + index_raw = fetch(BASE + 'index.txt') + entries = parse_index(index_raw, args.since_year) + if state.get('count', 0) and len(entries) < max(1, int(state['count'] * .70)): + raise RuntimeError('Index unexpectedly shrunk by over 30%; refusing unsafe sync') + current = set(entries) + stored = set(state.get('entries', [])) + pending = sorted(current - stored) + refresh_n = max(0, args.refresh_old) + cursor = int(state.get('cursor', 0)) + old = sorted(current & stored) + revisit = [] + if old and refresh_n: + revisit = [old[(cursor + n) % len(old)] for n in range(min(refresh_n, len(old)))] + missing = [item for item in entries if not (root / 'data' / 'csaf' / item).is_file()] + targets = entries if args.full else sorted(set(pending) | set(missing) | set(revisit)) + LOG.info('Index: %s entries, new=%s missing=%s refreshing=%s total downloads=%s', len(entries), len(pending), len(missing), len(revisit), len(targets)) + updates = 0 + for i, item in enumerate(targets, 1): + dest = root / 'data' / 'csaf' / item + raw = fetch(BASE + item) + validate_document(raw, item) + if not dest.is_file() or hashlib.sha256(raw).digest() != hashlib.sha256(dest.read_bytes()).digest(): + atomic_bytes(dest, raw) + updates += 1 + if i % 100 == 0: + LOG.info('Processed %s/%s (new/updated %s)', i, len(targets), updates) + if args.delay and i < len(targets): + time.sleep(args.delay) + # Apply removals only after ALL downloads succeeded. No state changes on partial failures. + removed = stored - current + for item in removed: + (root / 'data' / 'csaf' / item).unlink(missing_ok=True) + atomic_bytes(root / 'data' / 'index.txt', index_raw) + new_state = { + 'source': BASE, 'since_year': args.since_year, + 'last_success_utc': datetime.now(timezone.utc).isoformat(), + 'index_sha256': hashlib.sha256(index_raw).hexdigest(), + 'count': len(entries), 'cursor': (cursor + len(revisit)) % max(len(old), 1), + 'entries': entries, + } + atomic_bytes(statefile, (json.dumps(new_state, ensure_ascii=False, sort_keys=True, indent=2) + '\n').encode()) + LOG.info('Done: %s downloaded, %s changed, %s removed', len(targets), updates, len(removed)) + +if __name__ == '__main__': + try: + main() + except Exception as exc: + LOG.exception('Sync failed: %s', exc) + sys.exit(1) diff --git a/scripts/validate.py b/scripts/validate.py new file mode 100644 index 0000000..c38a3ab --- /dev/null +++ b/scripts/validate.py @@ -0,0 +1,25 @@ +#!/usr/bin/env python3 +import argparse +import json +from pathlib import Path +from sync import parse_index, validate_document + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument('--root', default='.') + args = parser.parse_args() + root = Path(args.root) + state = json.loads((root / 'state/sync.json').read_text()) + selected = parse_index((root / 'data/index.txt').read_bytes(), state['since_year']) + if selected != state['entries']: + raise ValueError('Index and state disagree') + for item in selected: + p = root / 'data/csaf' / item + if not p.is_file(): + raise ValueError(f'Missing CSAF document: {item}') + validate_document(p.read_bytes(), item) + print(f'Validated {len(selected)} CSAF documents') + +if __name__ == '__main__': + main() diff --git a/tests/test_sync.py b/tests/test_sync.py new file mode 100644 index 0000000..23e3bcf --- /dev/null +++ b/tests/test_sync.py @@ -0,0 +1,19 @@ +import sys +from pathlib import Path +import unittest +sys.path.insert(0, str(Path(__file__).parents[1] / 'scripts')) +from sync import parse_index, validate_document + +class TestBSI(unittest.TestCase): + def test_index_filter(self): + self.assertEqual(parse_index(b'2016/a.json\n2017/b.json\n2026/c.json\n', 2017), ['2017/b.json','2026/c.json']) + def test_bad_index(self): + with self.assertRaises(ValueError): + parse_index(b'../../evil.json\n', 2017) + def test_doc(self): + validate_document(b'{"document":{"tracking":{"id":"WID-1","current_release_date":"2026-01-01"}}}', '2026/a.json') + with self.assertRaises(ValueError): + validate_document(b'{}', '2026/a.json') + +if __name__ == '__main__': + unittest.main()