commit e6a108b4b0e12d1b44d565407ab4f0909cd240ef Author: root Date: Tue Nov 10 21:28:10 2026 +0100 Initial Red Hat CSAF mirror diff --git a/.gitea/workflows/update.yml b/.gitea/workflows/update.yml new file mode 100644 index 0000000..c501486 --- /dev/null +++ b/.gitea/workflows/update.yml @@ -0,0 +1,34 @@ +name: Red Hat CSAF Mirror + +on: + schedule: + - cron: '53 */4 * * *' + workflow_dispatch: + +permissions: + contents: write + +concurrency: + group: redhat-mirror + cancel-in-progress: false + +jobs: + mirror: + runs-on: vuln-runner + timeout-minutes: 240 + env: + GITEA_JOB_TOKEN: ${{ secrets.GITEA_TOKEN }} + GITEA_REPO_URL: https://git.apex-project.de/VULN/vuln-redhat.git + VULN_CACHE_BASE: /var/lib/act_runner/cache/vuln-redhat + steps: + - name: Red Hat CSAF mirror + shell: bash + run: | + set -Eeuo pipefail + BASE="$VULN_CACHE_BASE" + mkdir -p "$BASE" + if [ ! -f "$BASE/repo/scripts/run-cached.sh" ]; then + AUTH="Authorization: Basic $(printf 'x-access-token:%s' "$GITEA_JOB_TOKEN" | base64 | tr -d '\n')" + git -c "http.extraHeader=$AUTH" clone --depth 1 --branch main "$GITEA_REPO_URL" "$BASE/repo" + fi + bash "$BASE/repo/scripts/run-cached.sh" diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..b51d1fd --- /dev/null +++ b/.gitignore @@ -0,0 +1,5 @@ +__pycache__/ +*.pyc +*.tmp +.env +.cache/ diff --git a/README.md b/README.md new file mode 100644 index 0000000..e3f63ee --- /dev/null +++ b/README.md @@ -0,0 +1,45 @@ +# vuln-redhat – Red Hat CSAF mirror + +Mirrors the **original Red Hat CSAF JSON** from [the official advisory feed](https://security.access.redhat.com/data/csaf/v2/advisories/). The repository retains advisories by directory year starting in 2017. No transformation is applied to the JSON bytes. The authoritative `index.txt` is fetched every run; `changes.csv` is used as an update hint, and a rotating set of 250 entries is always rechecked as additional reconciliation. + +## Server installation (Debian/Ubuntu, host-based Gitea runner) + +```bash +sudo apt install -y python3 git util-linux ca-certificates +sudo install -d -o act_runner -g act_runner -m 700 /var/lib/act_runner/cache/vuln-redhat +``` + +Create `VULN/vuln-redhat` in Gitea and push the contents to `main`. The workflow runs on the already-registered **`vuln-runner`**. It uses the built-in `${{ secrets.GITEA_TOKEN }}`; no additional PAT is required if this already works for other repos. Allow job token repository write access. + +## Run + +Open `https://git.apex-project.de/VULN/vuln-redhat/actions`, select **Red Hat CSAF Mirror**, and run manually. Subsequent runs are scheduled every four hours at minute 53 (Gitea scheduler time zone). + +```bash +# Direct sync for development, without Git: +python3 scripts/sync.py --since-year 2017 +python3 scripts/validate.py + +# Only first 5 records, without advancing stored state: +python3 scripts/sync.py --limit 5 --rotate 0 +``` + +## Output + +```text +data/csaf/2017/rhsa-2017_001.json +... +state/sync.json +``` + +### Behavior and limitations + +- The first sync can download tens of thousands of JSON documents and create a very large Git commit; leave enough disk space and allow several hours. +- New advisory files are discovered using `index.txt`. Changed entries are identified opportunistically via `changes.csv` plus a rotating audit. **This is not a guaranteed near-real-time change feed**: a historical advisory can take multiple runs to be revisited if it is not recognizable in the changes feed. +- Files missing from a successfully retrieved authoritative index are pruned only when the count is below an explicit mass-deletion safety threshold. A suspicious deletion count fails the job instead. +- Any HTTP/JSON error stops the run, and neither the state nor Git commit advances. The persistent checkout is retained for investigation on failures; after fixing, revert only unintended local edits before re-running. +- Run jobs on trusted branches only, because a host runner executes scripts as its local Linux account. +- License: Red Hat security data is CC BY 4.0. Preserve attribution and link to the original source if you redistribute it. +- Full live download is not part of the provided local tests. + +Source: https://access.redhat.com/security/data diff --git a/data/csaf/.gitkeep b/data/csaf/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/scripts/run-cached.sh b/scripts/run-cached.sh new file mode 100644 index 0000000..fa19f3c --- /dev/null +++ b/scripts/run-cached.sh @@ -0,0 +1,33 @@ +#!/usr/bin/env bash +set -Eeuo pipefail +umask 077 +BASE="${VULN_CACHE_BASE:-/var/lib/act_runner/cache/vuln-redhat}" +REPO="$BASE/repo" +URL="${GITEA_REPO_URL:-https://git.apex-project.de/VULN/vuln-redhat.git}" +mkdir -p "$BASE" +exec 9>"$BASE/.sync.lock" +flock -w 120 9 || { echo 'Another run holds the Red Hat lock' >&2; exit 1; } +: "${GITEA_JOB_TOKEN:?Missing GITEA_JOB_TOKEN secret}" +AUTH="Authorization: Basic $(printf 'x-access-token:%s' "$GITEA_JOB_TOKEN" | base64 | tr -d '\n')" +git_auth() { git -c "http.extraHeader=$AUTH" "$@"; } +if [[ ! -d "$REPO/.git" ]]; then + git_auth clone --branch main --depth 1 "$URL" "$REPO" +fi +cd "$REPO" +git remote set-url origin "$URL" +if [[ -n "$(git status --porcelain)" ]]; then + echo 'Dirty cached checkout. Refusing to overwrite local data.' >&2 + exit 1 +fi +git_auth fetch --depth 1 origin main +git checkout main +git merge --ff-only FETCH_HEAD +python3 scripts/sync.py --since-year 2017 --rotate 250 --prune +python3 scripts/validate.py +git add -A data state +if git diff --cached --quiet; then + echo 'No changes' + exit 0 +fi +git -c user.name='Vulnerability Bot' -c user.email='vulnerability-bot@users.noreply.local' commit -m "Red Hat CSAF update $(date -u +%Y-%m-%dT%H:%M:%SZ)" +git_auth push origin HEAD:main diff --git a/scripts/sync.py b/scripts/sync.py new file mode 100644 index 0000000..4444a7a --- /dev/null +++ b/scripts/sync.py @@ -0,0 +1,165 @@ +#!/usr/bin/env python3 +"""Mirror official Red Hat CSAF advisories, retaining the original JSON bytes.""" +import argparse +import csv +import hashlib +import io +import json +import os +import re +import sys +import time +from datetime import datetime, timezone +from pathlib import Path +from urllib.error import HTTPError, URLError +from urllib.parse import quote +from urllib.request import Request, urlopen + +BASE = 'https://security.access.redhat.com/data/csaf/v2/advisories/' +PATH_RE = re.compile(r'(?= year: + out.add(v.lower()) + if not out: + raise ValueError('Index empty or incompatible format: refusing to synchronize') + return sorted(out) + + +def extract_changed(raw, known): + """Parse path or advisory ID from change/deletion feed without assuming CSV columns.""" + data = raw.decode('utf-8-sig', errors='replace') + exact = set() + name_map = {} + for p in known: + name_map.setdefault(p.rsplit('/', 1)[-1], set()).add(p) + for row in csv.reader(io.StringIO(data)): + for value in row: + for match in PATH_RE.finditer(value): + p = match.group(0).lower() + if p in known: + exact.add(p) + for match in NAME_RE.finditer(value): + kind = match.group(0).split('-')[0].split(':')[0].lower() + filename = f'{kind}-{match.group(1)}_{int(match.group(2)):03d}.json' + exact.update(name_map.get(filename, ())) + return exact + + +def get(url, attempts=5, timeout=90): + for i in range(attempts): + req = Request(url, headers={'User-Agent': 'vuln-redhat-mirror/1.0', 'Accept': 'application/json, text/plain, text/csv, */*'}) + try: + with urlopen(req, timeout=timeout) as response: + return response.read() + except (HTTPError, URLError, TimeoutError) as exc: + if isinstance(exc, HTTPError) and exc.code not in (408, 429, 500, 502, 503, 504): + raise + if i == attempts - 1: + raise + delay = min(45, 2 ** i * 3) + print(f'Network error {url}: {exc}; retry in {delay}s', file=sys.stderr) + time.sleep(delay) + raise RuntimeError('Unreachable') + + +def write_if_changed(dst, content): + if dst.exists() and dst.read_bytes() == content: + return False + dst.parent.mkdir(parents=True, exist_ok=True) + tmp = dst.with_suffix('.tmp') + tmp.write_bytes(content) + os.replace(tmp, dst) + return True + + +def sync(root, year=2017, rotate=250, prune=False, limit=0): + root = Path(root) + data_dir = root / 'data' / 'csaf' + state_path = root / 'state' / 'sync.json' + old = json.loads(state_path.read_text()) if state_path.exists() else {} + index_raw = get(BASE + 'index.txt') + paths = valid_paths(index_raw, year) + known = set(paths) + changed = set() + feed_status = {} + for feed in ('changes.csv', 'deletions.csv'): + try: + raw = get(BASE + feed) + feed_status[feed] = hashlib.sha256(raw).hexdigest() + if feed == 'changes.csv': + changed |= extract_changed(raw, known) + else: + # Deletions are handled through the authoritative index below. + # Preserve hash to observe feed changes in state. + pass + except Exception as exc: + print(f'Warning: {feed} not available: {exc}', file=sys.stderr) + feed_status[feed] = None + missing = {p for p in paths if not (data_dir / p).exists()} + cursor = int(old.get('rotation_cursor', 0)) % len(paths) + rotating = {paths[(cursor + n) % len(paths)] for n in range(min(rotate, len(paths)))} + todo = sorted(missing | changed | rotating) + if limit: + todo = todo[:limit] + count_new = count_changed = count_same = 0 + for index, rel in enumerate(todo, 1): + raw = get(BASE + quote(rel, safe='/')) + record = json.loads(raw) + if not isinstance(record, dict) or not isinstance(record.get('document'), dict): + raise ValueError(f'Invalid CSAF structure: {rel}') + out = data_dir / rel + was_present = out.exists() + if write_if_changed(out, raw): + if was_present: + count_changed += 1 + else: + count_new += 1 + else: + count_same += 1 + if index % 250 == 0: + print(f'Processed {index}/{len(todo)}') + removed = 0 + if prune and not limit: + existing = [p for p in data_dir.glob('*/*.json') if int(p.parent.name) >= year] + doomed = [p for p in existing if f'{p.parent.name}/{p.name}'.lower() not in known] + if len(doomed) > max(100, int(len(existing) * .05)): + raise RuntimeError(f'Refusing unexpected mass deletion: {len(doomed)} paths') + for p in doomed: + p.unlink() + removed = len(doomed) + state = { + 'schema': 1, 'source': BASE, 'since_year': year, + 'last_success_utc': datetime.now(timezone.utc).isoformat(), + 'index_sha256': hashlib.sha256(index_raw).hexdigest(), + 'index_count': len(paths), 'rotation_cursor': (cursor + (0 if limit else rotate)) % len(paths), + 'feed_sha256': feed_status, + 'last_run': {'downloaded': len(todo), 'new': count_new, 'updated': count_changed, + 'unchanged': count_same, 'removed': removed, 'limited': bool(limit)} + } + # A limited developer run must not advance the durable synchronization status. + if not limit: + state_path.parent.mkdir(parents=True, exist_ok=True) + write_if_changed(state_path, (json.dumps(state, indent=2, ensure_ascii=False) + '\n').encode()) + print(json.dumps(state['last_run'], sort_keys=True)) + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument('--root', default='.') + parser.add_argument('--since-year', type=int, default=2017) + parser.add_argument('--rotate', type=int, default=250, help='Number of older files to recheck per run') + parser.add_argument('--prune', action='store_true', help='Remove files missing from index after safety checks') + parser.add_argument('--limit', type=int, default=0, help='Developer test: do not advance state') + args = parser.parse_args() + if args.rotate < 0 or args.limit < 0: + parser.error('rotate and limit must not be negative') + sync(args.root, args.since_year, args.rotate, args.prune, args.limit) + +if __name__ == '__main__': + main() diff --git a/scripts/validate.py b/scripts/validate.py new file mode 100644 index 0000000..fceb4ca --- /dev/null +++ b/scripts/validate.py @@ -0,0 +1,26 @@ +#!/usr/bin/env python3 +"""Validate the local Red Hat CSAF mirror.""" +import argparse +import json +from pathlib import Path + + +def validate(root): + root = Path(root) + state = root / 'state/sync.json' + if not state.exists(): + raise RuntimeError('Sync state missing') + obj = json.loads(state.read_text()) + files = list((root / 'data/csaf').glob('*/*.json')) + if not files: + raise RuntimeError('No CSAF documents present') + for path in files: + data = json.loads(path.read_bytes()) + if not isinstance(data, dict) or not isinstance(data.get('document'), dict): + raise RuntimeError(f'Invalid CSAF document: {path}') + print(f'Validated {len(files)} Red Hat CSAF documents; source index entries: {obj["index_count"]}') + +if __name__ == '__main__': + ap = argparse.ArgumentParser() + ap.add_argument('--root', default='.') + validate(ap.parse_args().root) diff --git a/state/.gitkeep b/state/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/tests/test_sync.py b/tests/test_sync.py new file mode 100644 index 0000000..761defa --- /dev/null +++ b/tests/test_sync.py @@ -0,0 +1,32 @@ +import sys +import tempfile +import unittest +from pathlib import Path +from unittest.mock import patch +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'scripts')) +import sync + +class RedHatSyncTests(unittest.TestCase): + def test_index_year_filter(self): + data = b'2016/rhsa-2016_001.json\n2017/rhsa-2017_001.json\n2026/rhsa-2026_012.json\n' + self.assertEqual(sync.valid_paths(data, 2017), ['2017/rhsa-2017_001.json', '2026/rhsa-2026_012.json']) + + def test_changes_csv(self): + items = {'2025/rhsa-2025_012.json'} + self.assertEqual(sync.extract_changed(b'url,info\nhttps://example.test/2025/rhsa-2025_012.json,updated\n', items), items) + + def test_first_sync_and_second_no_change(self): + with tempfile.TemporaryDirectory() as d: + idx = b'2026/rhsa-2026_001.json\n' + doc = b'{"document":{"title":"RHSA"}}' + def fetch(url): + if url.endswith('index.txt'): return idx + if url.endswith('.csv'): return b'' + return doc + with patch.object(sync, 'get', side_effect=fetch): + sync.sync(d, 2017, rotate=1) + sync.sync(d, 2017, rotate=1) + self.assertEqual((Path(d)/'data/csaf/2026/rhsa-2026_001.json').read_bytes(), doc) + +if __name__ == '__main__': + unittest.main()