Initial Red Hat CSAF mirror

2026-11-10 21:28:10 +01:00
commit e6a108b4b0
9 changed files with 340 additions and 0 deletions
@@ -0,0 +1,33 @@
+#!/usr/bin/env bash
+set -Eeuo pipefail
+umask 077
+BASE="${VULN_CACHE_BASE:-/var/lib/act_runner/cache/vuln-redhat}"
+REPO="$BASE/repo"
+URL="${GITEA_REPO_URL:-https://git.apex-project.de/VULN/vuln-redhat.git}"
+mkdir -p "$BASE"
+exec 9>"$BASE/.sync.lock"
+flock -w 120 9 || { echo 'Another run holds the Red Hat lock' >&2; exit 1; }
+: "${GITEA_JOB_TOKEN:?Missing GITEA_JOB_TOKEN secret}"
+AUTH="Authorization: Basic $(printf 'x-access-token:%s' "$GITEA_JOB_TOKEN" | base64 | tr -d '\n')"
+git_auth() { git -c "http.extraHeader=$AUTH" "$@"; }
+if [[ ! -d "$REPO/.git" ]]; then
+  git_auth clone --branch main --depth 1 "$URL" "$REPO"
+fi
+cd "$REPO"
+git remote set-url origin "$URL"
+if [[ -n "$(git status --porcelain)" ]]; then
+  echo 'Dirty cached checkout. Refusing to overwrite local data.' >&2
+  exit 1
+fi
+git_auth fetch --depth 1 origin main
+git checkout main
+git merge --ff-only FETCH_HEAD
+python3 scripts/sync.py --since-year 2017 --rotate 250 --prune
+python3 scripts/validate.py
+git add -A data state
+if git diff --cached --quiet; then
+  echo 'No changes'
+  exit 0
+fi
+git -c user.name='Vulnerability Bot' -c user.email='vulnerability-bot@users.noreply.local' commit -m "Red Hat CSAF update $(date -u +%Y-%m-%dT%H:%M:%SZ)"
+git_auth push origin HEAD:main
@@ -0,0 +1,165 @@
+#!/usr/bin/env python3
+"""Mirror official Red Hat CSAF advisories, retaining the original JSON bytes."""
+import argparse
+import csv
+import hashlib
+import io
+import json
+import os
+import re
+import sys
+import time
+from datetime import datetime, timezone
+from pathlib import Path
+from urllib.error import HTTPError, URLError
+from urllib.parse import quote
+from urllib.request import Request, urlopen
+
+BASE = 'https://security.access.redhat.com/data/csaf/v2/advisories/'
+PATH_RE = re.compile(r'(?<![0-9a-z])(?:20\d{2})/(?:rhsa|rhba|rhea)-20\d{2}_[0-9a-z_-]+\.json', re.I)
+NAME_RE = re.compile(r'\b(?:RHSA|RHBA|RHEA)[-:]\s*(20\d{2})[-:]([0-9]{1,5})\b', re.I)
+
+
+def valid_paths(raw, year):
+    out = set()
+    for line in raw.decode('utf-8-sig').splitlines():
+        v = line.strip().lstrip('/')
+        if PATH_RE.fullmatch(v) and int(v[:4]) >= year:
+            out.add(v.lower())
+    if not out:
+        raise ValueError('Index empty or incompatible format: refusing to synchronize')
+    return sorted(out)
+
+
+def extract_changed(raw, known):
+    """Parse path or advisory ID from change/deletion feed without assuming CSV columns."""
+    data = raw.decode('utf-8-sig', errors='replace')
+    exact = set()
+    name_map = {}
+    for p in known:
+        name_map.setdefault(p.rsplit('/', 1)[-1], set()).add(p)
+    for row in csv.reader(io.StringIO(data)):
+        for value in row:
+            for match in PATH_RE.finditer(value):
+                p = match.group(0).lower()
+                if p in known:
+                    exact.add(p)
+            for match in NAME_RE.finditer(value):
+                kind = match.group(0).split('-')[0].split(':')[0].lower()
+                filename = f'{kind}-{match.group(1)}_{int(match.group(2)):03d}.json'
+                exact.update(name_map.get(filename, ()))
+    return exact
+
+
+def get(url, attempts=5, timeout=90):
+    for i in range(attempts):
+        req = Request(url, headers={'User-Agent': 'vuln-redhat-mirror/1.0', 'Accept': 'application/json, text/plain, text/csv, */*'})
+        try:
+            with urlopen(req, timeout=timeout) as response:
+                return response.read()
+        except (HTTPError, URLError, TimeoutError) as exc:
+            if isinstance(exc, HTTPError) and exc.code not in (408, 429, 500, 502, 503, 504):
+                raise
+            if i == attempts - 1:
+                raise
+            delay = min(45, 2 ** i * 3)
+            print(f'Network error {url}: {exc}; retry in {delay}s', file=sys.stderr)
+            time.sleep(delay)
+    raise RuntimeError('Unreachable')
+
+
+def write_if_changed(dst, content):
+    if dst.exists() and dst.read_bytes() == content:
+        return False
+    dst.parent.mkdir(parents=True, exist_ok=True)
+    tmp = dst.with_suffix('.tmp')
+    tmp.write_bytes(content)
+    os.replace(tmp, dst)
+    return True
+
+
+def sync(root, year=2017, rotate=250, prune=False, limit=0):
+    root = Path(root)
+    data_dir = root / 'data' / 'csaf'
+    state_path = root / 'state' / 'sync.json'
+    old = json.loads(state_path.read_text()) if state_path.exists() else {}
+    index_raw = get(BASE + 'index.txt')
+    paths = valid_paths(index_raw, year)
+    known = set(paths)
+    changed = set()
+    feed_status = {}
+    for feed in ('changes.csv', 'deletions.csv'):
+        try:
+            raw = get(BASE + feed)
+            feed_status[feed] = hashlib.sha256(raw).hexdigest()
+            if feed == 'changes.csv':
+                changed |= extract_changed(raw, known)
+            else:
+                # Deletions are handled through the authoritative index below.
+                # Preserve hash to observe feed changes in state.
+                pass
+        except Exception as exc:
+            print(f'Warning: {feed} not available: {exc}', file=sys.stderr)
+            feed_status[feed] = None
+    missing = {p for p in paths if not (data_dir / p).exists()}
+    cursor = int(old.get('rotation_cursor', 0)) % len(paths)
+    rotating = {paths[(cursor + n) % len(paths)] for n in range(min(rotate, len(paths)))}
+    todo = sorted(missing | changed | rotating)
+    if limit:
+        todo = todo[:limit]
+    count_new = count_changed = count_same = 0
+    for index, rel in enumerate(todo, 1):
+        raw = get(BASE + quote(rel, safe='/'))
+        record = json.loads(raw)
+        if not isinstance(record, dict) or not isinstance(record.get('document'), dict):
+            raise ValueError(f'Invalid CSAF structure: {rel}')
+        out = data_dir / rel
+        was_present = out.exists()
+        if write_if_changed(out, raw):
+            if was_present:
+                count_changed += 1
+            else:
+                count_new += 1
+        else:
+            count_same += 1
+        if index % 250 == 0:
+            print(f'Processed {index}/{len(todo)}')
+    removed = 0
+    if prune and not limit:
+        existing = [p for p in data_dir.glob('*/*.json') if int(p.parent.name) >= year]
+        doomed = [p for p in existing if f'{p.parent.name}/{p.name}'.lower() not in known]
+        if len(doomed) > max(100, int(len(existing) * .05)):
+            raise RuntimeError(f'Refusing unexpected mass deletion: {len(doomed)} paths')
+        for p in doomed:
+            p.unlink()
+        removed = len(doomed)
+    state = {
+        'schema': 1, 'source': BASE, 'since_year': year,
+        'last_success_utc': datetime.now(timezone.utc).isoformat(),
+        'index_sha256': hashlib.sha256(index_raw).hexdigest(),
+        'index_count': len(paths), 'rotation_cursor': (cursor + (0 if limit else rotate)) % len(paths),
+        'feed_sha256': feed_status,
+        'last_run': {'downloaded': len(todo), 'new': count_new, 'updated': count_changed,
+                     'unchanged': count_same, 'removed': removed, 'limited': bool(limit)}
+    }
+    # A limited developer run must not advance the durable synchronization status.
+    if not limit:
+        state_path.parent.mkdir(parents=True, exist_ok=True)
+        write_if_changed(state_path, (json.dumps(state, indent=2, ensure_ascii=False) + '\n').encode())
+    print(json.dumps(state['last_run'], sort_keys=True))
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument('--root', default='.')
+    parser.add_argument('--since-year', type=int, default=2017)
+    parser.add_argument('--rotate', type=int, default=250, help='Number of older files to recheck per run')
+    parser.add_argument('--prune', action='store_true', help='Remove files missing from index after safety checks')
+    parser.add_argument('--limit', type=int, default=0, help='Developer test: do not advance state')
+    args = parser.parse_args()
+    if args.rotate < 0 or args.limit < 0:
+        parser.error('rotate and limit must not be negative')
+    sync(args.root, args.since_year, args.rotate, args.prune, args.limit)
+
+if __name__ == '__main__':
+    main()
@@ -0,0 +1,26 @@
+#!/usr/bin/env python3
+"""Validate the local Red Hat CSAF mirror."""
+import argparse
+import json
+from pathlib import Path
+
+
+def validate(root):
+    root = Path(root)
+    state = root / 'state/sync.json'
+    if not state.exists():
+        raise RuntimeError('Sync state missing')
+    obj = json.loads(state.read_text())
+    files = list((root / 'data/csaf').glob('*/*.json'))
+    if not files:
+        raise RuntimeError('No CSAF documents present')
+    for path in files:
+        data = json.loads(path.read_bytes())
+        if not isinstance(data, dict) or not isinstance(data.get('document'), dict):
+            raise RuntimeError(f'Invalid CSAF document: {path}')
+    print(f'Validated {len(files)} Red Hat CSAF documents; source index entries: {obj["index_count"]}')
+
+if __name__ == '__main__':
+    ap = argparse.ArgumentParser()
+    ap.add_argument('--root', default='.')
+    validate(ap.parse_args().root)