From 41e6676c2d70f546cf24333233b2aa669cf2f245 Mon Sep 17 00:00:00 2001 From: root Date: Tue, 10 Nov 2026 22:17:16 +0100 Subject: [PATCH] Initial SUSE CSAF mirror --- .gitea/workflows/update.yml | 31 +++++++++ .gitignore | 4 ++ README.md | 33 +++++++++ scripts/run-cached.sh | 34 ++++++++++ scripts/sync.py | 132 ++++++++++++++++++++++++++++++++++++ scripts/validate.py | 17 +++++ tests/test_sync.py | 16 +++++ 7 files changed, 267 insertions(+) create mode 100644 .gitea/workflows/update.yml create mode 100644 .gitignore create mode 100644 README.md create mode 100644 scripts/run-cached.sh create mode 100644 scripts/sync.py create mode 100644 scripts/validate.py create mode 100644 tests/test_sync.py diff --git a/.gitea/workflows/update.yml b/.gitea/workflows/update.yml new file mode 100644 index 0000000..6a55809 --- /dev/null +++ b/.gitea/workflows/update.yml @@ -0,0 +1,31 @@ +name: SUSE CSAF Mirror +on: + schedule: + - cron: "31 */6 * * *" + workflow_dispatch: +permissions: + contents: write +concurrency: + group: vuln-suse + cancel-in-progress: false +jobs: + mirror: + runs-on: vuln-runner + timeout-minutes: 240 + steps: + - name: SUSE CSAF aktualisieren (persistenter Git-Checkout) + env: + GITEA_JOB_TOKEN: ${{ secrets.GITEA_TOKEN }} + CACHE_ROOT: /var/lib/act_runner/cache/vuln-suse + REPO_URL: https://git.apex-project.de/VULN/vuln-suse.git + run: | + set -Eeuo pipefail + # Only the first run needs to bootstrap the local checkout. + if [ ! -f "$CACHE_ROOT/repo/scripts/run-cached.sh" ]; then + mkdir -p "$CACHE_ROOT" + export GIT_CONFIG_COUNT=1 + export GIT_CONFIG_KEY_0=http.extraheader + export GIT_CONFIG_VALUE_0="Authorization: Basic $(printf 'oauth2:%s' "$GITEA_JOB_TOKEN" | base64 -w0)" + git clone --depth 1 --branch main "$REPO_URL" "$CACHE_ROOT/repo" + fi + bash "$CACHE_ROOT/repo/scripts/run-cached.sh" diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..fc024c0 --- /dev/null +++ b/.gitignore @@ -0,0 +1,4 @@ +__pycache__/ +*.pyc +*.tmp +.env diff --git a/README.md b/README.md new file mode 100644 index 0000000..fc6b7e6 --- /dev/null +++ b/README.md @@ -0,0 +1,33 @@ +# vuln-suse – SUSE/openSUSE CSAF mirror + +Spiegelt die offiziellen **CSAF 2.0 Advisory-Rohdaten** (SUSE und openSUSE) ab Advisory-Jahr 2017 direkt nach Git. + +Quelle (GWDG-Mirror): https://ftp2.gwdg.de/pub/linux/suse/projects/security/csaf/ + +- `data/index.txt`: unveränderter offizieller Index. +- `data/csaf//.json`: unveränderte CSAF-Dokumente. +- `state/sync.json`: Indexstand und Rotation für die Rückprüfung alter Dateien. + +**Nicht enthalten:** Die separate SUSE CSAF-VEX-Sammlung nach CVE. Diese kann als eigenes Repository `vuln-suse-vex` folgen. + +## Gitea-Installation + +1. Leeres Repository `VULN/vuln-suse` erstellen. +2. Dateien initial nach `main` pushen. +3. Auf dem Gitea-Runner-Host: + `sudo install -d -o act_runner -g act_runner -m 700 /var/lib/act_runner/cache/vuln-suse` +4. `git`, `python3`, `flock` (`util-linux`) installieren. +5. Workflow `SUSE CSAF Mirror` manuell starten. Automatisch alle 6 Stunden um Minute 31 (UTC). +6. Gitea-Jobtoken `secrets.GITEA_TOKEN` benötigt Schreibrecht auf das Repository. + +## Ablauf + +- Erstimport: alle Advisory-JSONs ab 2017 aus `index.txt`. +- Weitere Läufe: neue/fehlende Dokumente und 300 ältere Dokumente zur Änderungserkennung. +- Vollabgleich: `python3 scripts/sync.py --full` im persistenten Checkout. +- Fehlgeschlagene Downloads verhindern Statusfortschreibung und Löschungen. +- Alte Dateien werden nur entfernt, wenn sie nach erfolgreichem Indexabruf nicht mehr gelistet sind. + +**Grenze:** Änderungen an alten Dokumenten werden bei der rotierenden Kontrolle verzögert erkannt. `changes.csv` wird aktuell nicht ausgewertet. Der öffentliche GWDG-Mirror kann dem SUSE-Ursprung zeitlich hinterherhinken. Große Erstimports benötigen entsprechend Speicher und Git-Commit-Zeit. Für produktive Nutzung empfiehlt sich zusätzlich ein regelmäßiger `--full`-Abgleich. + +Sicherheit: Ein `:host`-Runner führt Workflows direkt auf der Gitea-VM aus. Repository-Schreibrechte beschränken und nur vertrauenswürdige Branches ausführen. diff --git a/scripts/run-cached.sh b/scripts/run-cached.sh new file mode 100644 index 0000000..9546a02 --- /dev/null +++ b/scripts/run-cached.sh @@ -0,0 +1,34 @@ +#!/usr/bin/env bash +set -Eeuo pipefail +umask 077 +: "${CACHE_ROOT:=/var/lib/act_runner/cache/vuln-suse}" +: "${GITEA_JOB_TOKEN:?GITEA_JOB_TOKEN fehlt}" +: "${REPO_URL:=https://git.apex-project.de/VULN/vuln-suse.git}" +mkdir -p "$CACHE_ROOT" +exec 9>"$CACHE_ROOT/.sync.lock" +flock -w 300 9 || { echo 'Ein anderer SUSE-Lauf ist aktiv' >&2; exit 1; } +# Temporary, environment-only Git authentication. No token embedded in remote URL or git config file. +GIT_CONFIG_COUNT=1 +GIT_CONFIG_KEY_0=http.extraheader +GIT_CONFIG_VALUE_0="Authorization: Basic $(printf 'oauth2:%s' "$GITEA_JOB_TOKEN" | base64 -w0)" +export GIT_CONFIG_COUNT GIT_CONFIG_KEY_0 GIT_CONFIG_VALUE_0 +if [[ ! -d "$CACHE_ROOT/repo/.git" ]]; then + git clone --branch main --single-branch --depth 1 "$REPO_URL" "$CACHE_ROOT/repo" +fi +cd "$CACHE_ROOT/repo" +git remote set-url origin "$REPO_URL" +git fetch --depth 1 origin main +git reset --hard FETCH_HEAD +# Recover tracked files that might have been created during a failed previous run. +git clean -fd -- scripts/ state/ || true +python3 scripts/sync.py --since-year 2017 +python3 scripts/validate.py +git config user.name 'Vulnerability Bot' +git config user.email 'vulnerability-bot@users.noreply.local' +git add -A data/ state/ +if git diff --cached --quiet; then + echo 'Keine Git-Änderungen' + exit 0 +fi +git commit -m "Update SUSE CSAF $(date -u +%Y-%m-%dT%H:%M:%SZ)" +git push origin HEAD:main diff --git a/scripts/sync.py b/scripts/sync.py new file mode 100644 index 0000000..0f3f0c7 --- /dev/null +++ b/scripts/sync.py @@ -0,0 +1,132 @@ +#!/usr/bin/env python3 +"""Mirror SUSE/openSUSE CSAF advisory documents using the official index.""" +import argparse +import concurrent.futures +import hashlib +import json +import os +from pathlib import Path +import re +import sys +import time +from datetime import datetime, timezone +from urllib.error import HTTPError, URLError +from urllib.parse import quote +from urllib.request import Request, urlopen + +BASE = os.getenv('SUSE_CSAF_URL', 'https://ftp2.gwdg.de/pub/linux/suse/projects/security/csaf/').rstrip('/') + '/' +NAME = re.compile(r'^(?:opensuse|suse)-[a-z0-9_-]*?(20\d\d)[_-][a-z0-9_.-]+\.json$', re.I) +# Handles 'suse-su-2026_...' etc; do not interpret CVE year as advisory year. +USER_AGENT='apex-project-vuln-suse-mirror/1.0' + +def fetch(url, tries=5): + for i in range(tries): + try: + with urlopen(Request(url, headers={'User-Agent': USER_AGENT, 'Accept':'application/json,text/plain,*/*'}), timeout=90) as r: + return r.read() + except (HTTPError, URLError, TimeoutError) as e: + if i == tries-1 or (isinstance(e, HTTPError) and e.code not in (429,500,502,503,504)): + raise + time.sleep(min(60, 2**(i+1))) + +def eligible(name, year): + if '/' in name or '\\' in name or '..' in name or not name.endswith('.json'): + return False + m = NAME.match(name) + return bool(m and int(m.group(1)) >= year) + +def load_index(data, year): + text=data.decode('utf-8-sig') + names=set() + for line in text.splitlines(): + name=line.strip() + if eligible(name,year): names.add(name) + if not names: + raise ValueError('Index liefert keine passenden CSAF-Dateien; kein Abgleich durchgeführt') + return names + +def valid_doc(blob): + doc=json.loads(blob) + if not isinstance(doc,dict) or not isinstance(doc.get('document'),dict): + raise ValueError('Ungültiges CSAF-Dokument (document fehlt)') + if not isinstance(doc.get('product_tree'),dict) and not isinstance(doc.get('vulnerabilities'),list): + raise ValueError('CSAF enthält weder product_tree noch vulnerabilities') + return doc + +def atomic_write(path,blob): + path.parent.mkdir(parents=True,exist_ok=True) + tmp=path.with_name(path.name+'.tmp') + try: + tmp.write_bytes(blob) + tmp.replace(path) + finally: + tmp.unlink(missing_ok=True) + +def main(argv=None): + ap=argparse.ArgumentParser() + ap.add_argument('--since-year',type=int,default=2017) + ap.add_argument('--workers',type=int,default=3) + ap.add_argument('--review',type=int,default=300,help='Anzahl älterer Advisories pro Lauf erneut herunterladen') + ap.add_argument('--full',action='store_true',help='Alle Dokumente erneut abgleichen') + ap.add_argument('--max-files',type=int,default=0,help='Nur für Tests, keine Löschungen') + args=ap.parse_args(argv) + root=Path(__file__).resolve().parent.parent + out=root/'data'/'csaf' + state_file=root/'state'/'sync.json' + state=json.loads(state_file.read_text()) if state_file.exists() else {} + index=fetch(BASE+'index.txt') + names=load_index(index,args.since_year) + previous=set(state.get('index',[])) + known={p.name for p in out.glob('*/*.json')} + missing=names-known + remaining=sorted(names-missing) + pos=int(state.get('review_cursor',0)) + review=[] + if args.full: + targets=sorted(names) + else: + if remaining: + count=min(args.review,len(remaining)) + review=[remaining[(pos+i)%len(remaining)] for i in range(count)] + targets=sorted(missing|set(review)) + if args.max_files: + targets=targets[:args.max_files] + print(f'Index={len(names)} vorhanden={len(known)} neu={len(missing)} Abrufe={len(targets)}',flush=True) + errors=[]; changed=0 + def worker(name): + m=NAME.match(name) + path=out/m.group(1)/name + blob=fetch(BASE+quote(name)) + valid_doc(blob) + if path.exists() and hashlib.sha256(path.read_bytes()).digest()==hashlib.sha256(blob).digest(): + return False + atomic_write(path,blob) + return True + with concurrent.futures.ThreadPoolExecutor(max_workers=max(1,min(8,args.workers))) as pool: + futures={pool.submit(worker,n):n for n in targets} + for fut in concurrent.futures.as_completed(futures): + try: changed+=int(fut.result()) + except Exception as e: errors.append(f'{futures[fut]}: {e}') + if errors: + for msg in errors[:30]:print('ERROR',msg,file=sys.stderr) + raise RuntimeError(f'{len(errors)} Downloads fehlgeschlagen. Keine Löschungen und kein Status-Commit.') + if not args.max_files: + deleted=0 + # The public index is authoritative only after a successful complete fetch of the index and all requested documents. + for old in sorted(known-names): + for path in out.glob('*/'+old): + path.unlink(); deleted+=1 + atomic_write(root/'data'/'index.txt',index) + new_state={'source':BASE,'since_year':args.since_year,'updated_at':datetime.now(timezone.utc).isoformat(), + 'index':sorted(names),'review_cursor':(pos+len(review)) % max(1,len(remaining)), + 'documents':len(names)} + atomic_write(state_file,(json.dumps(new_state,ensure_ascii=False,indent=2)+'\n').encode()) + print(f'Änderungen={changed}, Löschungen={deleted}') + else: + print('TESTMODUS: Status und Löschungen unverändert') + return 0 + +if __name__=='__main__': + try:sys.exit(main()) + except Exception as exc: + print(f'FEHLER: {exc}',file=sys.stderr); sys.exit(1) diff --git a/scripts/validate.py b/scripts/validate.py new file mode 100644 index 0000000..aa28546 --- /dev/null +++ b/scripts/validate.py @@ -0,0 +1,17 @@ +#!/usr/bin/env python3 +import json +import re +from pathlib import Path +from sys import exit +root=Path(__file__).resolve().parent.parent +state=root/'state'/'sync.json' +if not state.exists():raise SystemExit('Kein Sync-Status vorhanden') +s=json.loads(state.read_text()) +assert s.get('documents')==len(s.get('index',[])) +files=list((root/'data'/'csaf').glob('*/*.json')) +if len(files)!=s['documents']: + raise SystemExit(f'Unvollständig: {len(files)} Dateien, {s["documents"]} erwartet') +for p in files: + obj=json.loads(p.read_bytes()) + if not isinstance(obj.get('document'),dict):raise SystemExit(f'CSAF ungültig: {p}') +print(f'VALID: {len(files)} CSAF-Dokumente') diff --git a/tests/test_sync.py b/tests/test_sync.py new file mode 100644 index 0000000..b815abd --- /dev/null +++ b/tests/test_sync.py @@ -0,0 +1,16 @@ +import importlib.util +from pathlib import Path +import unittest +spec=importlib.util.spec_from_file_location('sync',Path(__file__).resolve().parents[1]/'scripts/sync.py') +mod=importlib.util.module_from_spec(spec);spec.loader.exec_module(mod) +class TestIndex(unittest.TestCase): + def test_filter(self): + self.assertTrue(mod.eligible('opensuse-su-2017_0034-1.json',2017)) + self.assertTrue(mod.eligible('suse-su-2026_9999-1.json',2017)) + self.assertFalse(mod.eligible('opensuse-su-2016_2223-1.json',2017)) + self.assertFalse(mod.eligible('../invalid-2026.json',2017)) + self.assertEqual(len(mod.load_index(b'opensuse-su-2016_2223-1.json\nopensuse-su-2017_0034-1.json\n',2017)),1) + def test_csaf(self): + mod.valid_doc(b'{"document":{},"vulnerabilities":[]}') + with self.assertRaises(ValueError):mod.valid_doc(b'{"hello":"world"}') +if __name__=='__main__':unittest.main()