Initial Red Hat CSAF mirror

.gitea/workflows/update.yml

@@ -0,0 +1,34 @@
+name: Red Hat CSAF Mirror
+
+on:
+  schedule:
+    - cron: '53 */4 * * *'
+  workflow_dispatch:
+
+permissions:
+  contents: write
+
+concurrency:
+  group: redhat-mirror
+  cancel-in-progress: false
+
+jobs:
+  mirror:
+    runs-on: vuln-runner
+    timeout-minutes: 240
+    env:
+      GITEA_JOB_TOKEN: ${{ secrets.GITEA_TOKEN }}
+      GITEA_REPO_URL: https://git.apex-project.de/VULN/vuln-redhat.git
+      VULN_CACHE_BASE: /var/lib/act_runner/cache/vuln-redhat
+    steps:
+      - name: Red Hat CSAF mirror
+        shell: bash
+        run: |
+          set -Eeuo pipefail
+          BASE="$VULN_CACHE_BASE"
+          mkdir -p "$BASE"
+          if [ ! -f "$BASE/repo/scripts/run-cached.sh" ]; then
+            AUTH="Authorization: Basic $(printf 'x-access-token:%s' "$GITEA_JOB_TOKEN" | base64 | tr -d '\n')"
+            git -c "http.extraHeader=$AUTH" clone --depth 1 --branch main "$GITEA_REPO_URL" "$BASE/repo"
+          fi
+          bash "$BASE/repo/scripts/run-cached.sh"

.gitignore

@@ -0,0 +1,5 @@
+__pycache__/
+*.pyc
+*.tmp
+.env
+.cache/

README.md

@@ -0,0 +1,45 @@
+# vuln-redhat – Red Hat CSAF mirror
+
+Mirrors the **original Red Hat CSAF JSON** from [the official advisory feed](https://security.access.redhat.com/data/csaf/v2/advisories/). The repository retains advisories by directory year starting in 2017. No transformation is applied to the JSON bytes. The authoritative `index.txt` is fetched every run; `changes.csv` is used as an update hint, and a rotating set of 250 entries is always rechecked as additional reconciliation.
+
+## Server installation (Debian/Ubuntu, host-based Gitea runner)
+
+```bash
+sudo apt install -y python3 git util-linux ca-certificates
+sudo install -d -o act_runner -g act_runner -m 700 /var/lib/act_runner/cache/vuln-redhat
+```
+
+Create `VULN/vuln-redhat` in Gitea and push the contents to `main`. The workflow runs on the already-registered **`vuln-runner`**. It uses the built-in `${{ secrets.GITEA_TOKEN }}`; no additional PAT is required if this already works for other repos. Allow job token repository write access.
+
+## Run
+
+Open `https://git.apex-project.de/VULN/vuln-redhat/actions`, select **Red Hat CSAF Mirror**, and run manually. Subsequent runs are scheduled every four hours at minute 53 (Gitea scheduler time zone).
+
+```bash
+# Direct sync for development, without Git:
+python3 scripts/sync.py --since-year 2017
+python3 scripts/validate.py
+
+# Only first 5 records, without advancing stored state:
+python3 scripts/sync.py --limit 5 --rotate 0
+```
+
+## Output
+
+```text
+data/csaf/2017/rhsa-2017_001.json
+...
+state/sync.json
+```
+
+### Behavior and limitations
+
+- The first sync can download tens of thousands of JSON documents and create a very large Git commit; leave enough disk space and allow several hours.
+- New advisory files are discovered using `index.txt`. Changed entries are identified opportunistically via `changes.csv` plus a rotating audit. **This is not a guaranteed near-real-time change feed**: a historical advisory can take multiple runs to be revisited if it is not recognizable in the changes feed.
+- Files missing from a successfully retrieved authoritative index are pruned only when the count is below an explicit mass-deletion safety threshold. A suspicious deletion count fails the job instead.
+- Any HTTP/JSON error stops the run, and neither the state nor Git commit advances. The persistent checkout is retained for investigation on failures; after fixing, revert only unintended local edits before re-running.
+- Run jobs on trusted branches only, because a host runner executes scripts as its local Linux account.
+- License: Red Hat security data is CC BY 4.0. Preserve attribution and link to the original source if you redistribute it.
+- Full live download is not part of the provided local tests.
+
+Source: https://access.redhat.com/security/data

scripts/run-cached.sh

@@ -0,0 +1,33 @@
+#!/usr/bin/env bash
+set -Eeuo pipefail
+umask 077
+BASE="${VULN_CACHE_BASE:-/var/lib/act_runner/cache/vuln-redhat}"
+REPO="$BASE/repo"
+URL="${GITEA_REPO_URL:-https://git.apex-project.de/VULN/vuln-redhat.git}"
+mkdir -p "$BASE"
+exec 9>"$BASE/.sync.lock"
+flock -w 120 9 || { echo 'Another run holds the Red Hat lock' >&2; exit 1; }
+: "${GITEA_JOB_TOKEN:?Missing GITEA_JOB_TOKEN secret}"
+AUTH="Authorization: Basic $(printf 'x-access-token:%s' "$GITEA_JOB_TOKEN" | base64 | tr -d '\n')"
+git_auth() { git -c "http.extraHeader=$AUTH" "$@"; }
+if [[ ! -d "$REPO/.git" ]]; then
+  git_auth clone --branch main --depth 1 "$URL" "$REPO"
+fi
+cd "$REPO"
+git remote set-url origin "$URL"
+if [[ -n "$(git status --porcelain)" ]]; then
+  echo 'Dirty cached checkout. Refusing to overwrite local data.' >&2
+  exit 1
+fi
+git_auth fetch --depth 1 origin main
+git checkout main
+git merge --ff-only FETCH_HEAD
+python3 scripts/sync.py --since-year 2017 --rotate 250 --prune
+python3 scripts/validate.py
+git add -A data state
+if git diff --cached --quiet; then
+  echo 'No changes'
+  exit 0
+fi
+git -c user.name='Vulnerability Bot' -c user.email='vulnerability-bot@users.noreply.local' commit -m "Red Hat CSAF update $(date -u +%Y-%m-%dT%H:%M:%SZ)"
+git_auth push origin HEAD:main

scripts/sync.py

@@ -0,0 +1,165 @@
+#!/usr/bin/env python3
+"""Mirror official Red Hat CSAF advisories, retaining the original JSON bytes."""
+import argparse
+import csv
+import hashlib
+import io
+import json
+import os
+import re
+import sys
+import time
+from datetime import datetime, timezone
+from pathlib import Path
+from urllib.error import HTTPError, URLError
+from urllib.parse import quote
+from urllib.request import Request, urlopen
+
+BASE = 'https://security.access.redhat.com/data/csaf/v2/advisories/'
+PATH_RE = re.compile(r'(?<![0-9a-z])(?:20\d{2})/(?:rhsa|rhba|rhea)-20\d{2}_[0-9a-z_-]+\.json', re.I)
+NAME_RE = re.compile(r'\b(?:RHSA|RHBA|RHEA)[-:]\s*(20\d{2})[-:]([0-9]{1,5})\b', re.I)
+
+
+def valid_paths(raw, year):
+    out = set()
+    for line in raw.decode('utf-8-sig').splitlines():
+        v = line.strip().lstrip('/')
+        if PATH_RE.fullmatch(v) and int(v[:4]) >= year:
+            out.add(v.lower())
+    if not out:
+        raise ValueError('Index empty or incompatible format: refusing to synchronize')
+    return sorted(out)
+
+
+def extract_changed(raw, known):
+    """Parse path or advisory ID from change/deletion feed without assuming CSV columns."""
+    data = raw.decode('utf-8-sig', errors='replace')
+    exact = set()
+    name_map = {}
+    for p in known:
+        name_map.setdefault(p.rsplit('/', 1)[-1], set()).add(p)
+    for row in csv.reader(io.StringIO(data)):
+        for value in row:
+            for match in PATH_RE.finditer(value):
+                p = match.group(0).lower()
+                if p in known:
+                    exact.add(p)
+            for match in NAME_RE.finditer(value):
+                kind = match.group(0).split('-')[0].split(':')[0].lower()
+                filename = f'{kind}-{match.group(1)}_{int(match.group(2)):03d}.json'
+                exact.update(name_map.get(filename, ()))
+    return exact
+
+
+def get(url, attempts=5, timeout=90):
+    for i in range(attempts):
+        req = Request(url, headers={'User-Agent': 'vuln-redhat-mirror/1.0', 'Accept': 'application/json, text/plain, text/csv, */*'})
+        try:
+            with urlopen(req, timeout=timeout) as response:
+                return response.read()
+        except (HTTPError, URLError, TimeoutError) as exc:
+            if isinstance(exc, HTTPError) and exc.code not in (408, 429, 500, 502, 503, 504):
+                raise
+            if i == attempts - 1:
+                raise
+            delay = min(45, 2 ** i * 3)
+            print(f'Network error {url}: {exc}; retry in {delay}s', file=sys.stderr)
+            time.sleep(delay)
+    raise RuntimeError('Unreachable')
+
+
+def write_if_changed(dst, content):
+    if dst.exists() and dst.read_bytes() == content:
+        return False
+    dst.parent.mkdir(parents=True, exist_ok=True)
+    tmp = dst.with_suffix('.tmp')
+    tmp.write_bytes(content)
+    os.replace(tmp, dst)
+    return True
+
+
+def sync(root, year=2017, rotate=250, prune=False, limit=0):
+    root = Path(root)
+    data_dir = root / 'data' / 'csaf'
+    state_path = root / 'state' / 'sync.json'
+    old = json.loads(state_path.read_text()) if state_path.exists() else {}
+    index_raw = get(BASE + 'index.txt')
+    paths = valid_paths(index_raw, year)
+    known = set(paths)
+    changed = set()
+    feed_status = {}
+    for feed in ('changes.csv', 'deletions.csv'):
+        try:
+            raw = get(BASE + feed)
+            feed_status[feed] = hashlib.sha256(raw).hexdigest()
+            if feed == 'changes.csv':
+                changed |= extract_changed(raw, known)
+            else:
+                # Deletions are handled through the authoritative index below.
+                # Preserve hash to observe feed changes in state.
+                pass
+        except Exception as exc:
+            print(f'Warning: {feed} not available: {exc}', file=sys.stderr)
+            feed_status[feed] = None
+    missing = {p for p in paths if not (data_dir / p).exists()}
+    cursor = int(old.get('rotation_cursor', 0)) % len(paths)
+    rotating = {paths[(cursor + n) % len(paths)] for n in range(min(rotate, len(paths)))}
+    todo = sorted(missing | changed | rotating)
+    if limit:
+        todo = todo[:limit]
+    count_new = count_changed = count_same = 0
+    for index, rel in enumerate(todo, 1):
+        raw = get(BASE + quote(rel, safe='/'))
+        record = json.loads(raw)
+        if not isinstance(record, dict) or not isinstance(record.get('document'), dict):
+            raise ValueError(f'Invalid CSAF structure: {rel}')
+        out = data_dir / rel
+        was_present = out.exists()
+        if write_if_changed(out, raw):
+            if was_present:
+                count_changed += 1
+            else:
+                count_new += 1
+        else:
+            count_same += 1
+        if index % 250 == 0:
+            print(f'Processed {index}/{len(todo)}')
+    removed = 0
+    if prune and not limit:
+        existing = [p for p in data_dir.glob('*/*.json') if int(p.parent.name) >= year]
+        doomed = [p for p in existing if f'{p.parent.name}/{p.name}'.lower() not in known]
+        if len(doomed) > max(100, int(len(existing) * .05)):
+            raise RuntimeError(f'Refusing unexpected mass deletion: {len(doomed)} paths')
+        for p in doomed:
+            p.unlink()
+        removed = len(doomed)
+    state = {
+        'schema': 1, 'source': BASE, 'since_year': year,
+        'last_success_utc': datetime.now(timezone.utc).isoformat(),
+        'index_sha256': hashlib.sha256(index_raw).hexdigest(),
+        'index_count': len(paths), 'rotation_cursor': (cursor + (0 if limit else rotate)) % len(paths),
+        'feed_sha256': feed_status,
+        'last_run': {'downloaded': len(todo), 'new': count_new, 'updated': count_changed,
+                     'unchanged': count_same, 'removed': removed, 'limited': bool(limit)}
+    }
+    # A limited developer run must not advance the durable synchronization status.
+    if not limit:
+        state_path.parent.mkdir(parents=True, exist_ok=True)
+        write_if_changed(state_path, (json.dumps(state, indent=2, ensure_ascii=False) + '\n').encode())
+    print(json.dumps(state['last_run'], sort_keys=True))
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument('--root', default='.')
+    parser.add_argument('--since-year', type=int, default=2017)
+    parser.add_argument('--rotate', type=int, default=250, help='Number of older files to recheck per run')
+    parser.add_argument('--prune', action='store_true', help='Remove files missing from index after safety checks')
+    parser.add_argument('--limit', type=int, default=0, help='Developer test: do not advance state')
+    args = parser.parse_args()
+    if args.rotate < 0 or args.limit < 0:
+        parser.error('rotate and limit must not be negative')
+    sync(args.root, args.since_year, args.rotate, args.prune, args.limit)
+
+if __name__ == '__main__':
+    main()

scripts/validate.py

@@ -0,0 +1,26 @@
+#!/usr/bin/env python3
+"""Validate the local Red Hat CSAF mirror."""
+import argparse
+import json
+from pathlib import Path
+
+
+def validate(root):
+    root = Path(root)
+    state = root / 'state/sync.json'
+    if not state.exists():
+        raise RuntimeError('Sync state missing')
+    obj = json.loads(state.read_text())
+    files = list((root / 'data/csaf').glob('*/*.json'))
+    if not files:
+        raise RuntimeError('No CSAF documents present')
+    for path in files:
+        data = json.loads(path.read_bytes())
+        if not isinstance(data, dict) or not isinstance(data.get('document'), dict):
+            raise RuntimeError(f'Invalid CSAF document: {path}')
+    print(f'Validated {len(files)} Red Hat CSAF documents; source index entries: {obj["index_count"]}')
+
+if __name__ == '__main__':
+    ap = argparse.ArgumentParser()
+    ap.add_argument('--root', default='.')
+    validate(ap.parse_args().root)

tests/test_sync.py

@@ -0,0 +1,32 @@
+import sys
+import tempfile
+import unittest
+from pathlib import Path
+from unittest.mock import patch
+sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'scripts'))
+import sync
+
+class RedHatSyncTests(unittest.TestCase):
+    def test_index_year_filter(self):
+        data = b'2016/rhsa-2016_001.json\n2017/rhsa-2017_001.json\n2026/rhsa-2026_012.json\n'
+        self.assertEqual(sync.valid_paths(data, 2017), ['2017/rhsa-2017_001.json', '2026/rhsa-2026_012.json'])
+
+    def test_changes_csv(self):
+        items = {'2025/rhsa-2025_012.json'}
+        self.assertEqual(sync.extract_changed(b'url,info\nhttps://example.test/2025/rhsa-2025_012.json,updated\n', items), items)
+
+    def test_first_sync_and_second_no_change(self):
+        with tempfile.TemporaryDirectory() as d:
+            idx = b'2026/rhsa-2026_001.json\n'
+            doc = b'{"document":{"title":"RHSA"}}'
+            def fetch(url):
+                if url.endswith('index.txt'): return idx
+                if url.endswith('.csv'): return b''
+                return doc
+            with patch.object(sync, 'get', side_effect=fetch):
+                sync.sync(d, 2017, rotate=1)
+                sync.sync(d, 2017, rotate=1)
+            self.assertEqual((Path(d)/'data/csaf/2026/rhsa-2026_001.json').read_bytes(), doc)
+
+if __name__ == '__main__':
+    unittest.main()