Initial Red Hat CSAF mirror
This commit is contained in:
root
+165
@@ -0,0 +1,165 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Mirror official Red Hat CSAF advisories, retaining the original JSON bytes."""
|
||||
import argparse
|
||||
import csv
|
||||
import hashlib
|
||||
import io
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
import time
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
from urllib.error import HTTPError, URLError
|
||||
from urllib.parse import quote
|
||||
from urllib.request import Request, urlopen
|
||||
|
||||
BASE = 'https://security.access.redhat.com/data/csaf/v2/advisories/'
|
||||
PATH_RE = re.compile(r'(?<![0-9a-z])(?:20\d{2})/(?:rhsa|rhba|rhea)-20\d{2}_[0-9a-z_-]+\.json', re.I)
|
||||
NAME_RE = re.compile(r'\b(?:RHSA|RHBA|RHEA)[-:]\s*(20\d{2})[-:]([0-9]{1,5})\b', re.I)
|
||||
|
||||
|
||||
def valid_paths(raw, year):
|
||||
out = set()
|
||||
for line in raw.decode('utf-8-sig').splitlines():
|
||||
v = line.strip().lstrip('/')
|
||||
if PATH_RE.fullmatch(v) and int(v[:4]) >= year:
|
||||
out.add(v.lower())
|
||||
if not out:
|
||||
raise ValueError('Index empty or incompatible format: refusing to synchronize')
|
||||
return sorted(out)
|
||||
|
||||
|
||||
def extract_changed(raw, known):
|
||||
"""Parse path or advisory ID from change/deletion feed without assuming CSV columns."""
|
||||
data = raw.decode('utf-8-sig', errors='replace')
|
||||
exact = set()
|
||||
name_map = {}
|
||||
for p in known:
|
||||
name_map.setdefault(p.rsplit('/', 1)[-1], set()).add(p)
|
||||
for row in csv.reader(io.StringIO(data)):
|
||||
for value in row:
|
||||
for match in PATH_RE.finditer(value):
|
||||
p = match.group(0).lower()
|
||||
if p in known:
|
||||
exact.add(p)
|
||||
for match in NAME_RE.finditer(value):
|
||||
kind = match.group(0).split('-')[0].split(':')[0].lower()
|
||||
filename = f'{kind}-{match.group(1)}_{int(match.group(2)):03d}.json'
|
||||
exact.update(name_map.get(filename, ()))
|
||||
return exact
|
||||
|
||||
|
||||
def get(url, attempts=5, timeout=90):
|
||||
for i in range(attempts):
|
||||
req = Request(url, headers={'User-Agent': 'vuln-redhat-mirror/1.0', 'Accept': 'application/json, text/plain, text/csv, */*'})
|
||||
try:
|
||||
with urlopen(req, timeout=timeout) as response:
|
||||
return response.read()
|
||||
except (HTTPError, URLError, TimeoutError) as exc:
|
||||
if isinstance(exc, HTTPError) and exc.code not in (408, 429, 500, 502, 503, 504):
|
||||
raise
|
||||
if i == attempts - 1:
|
||||
raise
|
||||
delay = min(45, 2 ** i * 3)
|
||||
print(f'Network error {url}: {exc}; retry in {delay}s', file=sys.stderr)
|
||||
time.sleep(delay)
|
||||
raise RuntimeError('Unreachable')
|
||||
|
||||
|
||||
def write_if_changed(dst, content):
|
||||
if dst.exists() and dst.read_bytes() == content:
|
||||
return False
|
||||
dst.parent.mkdir(parents=True, exist_ok=True)
|
||||
tmp = dst.with_suffix('.tmp')
|
||||
tmp.write_bytes(content)
|
||||
os.replace(tmp, dst)
|
||||
return True
|
||||
|
||||
|
||||
def sync(root, year=2017, rotate=250, prune=False, limit=0):
|
||||
root = Path(root)
|
||||
data_dir = root / 'data' / 'csaf'
|
||||
state_path = root / 'state' / 'sync.json'
|
||||
old = json.loads(state_path.read_text()) if state_path.exists() else {}
|
||||
index_raw = get(BASE + 'index.txt')
|
||||
paths = valid_paths(index_raw, year)
|
||||
known = set(paths)
|
||||
changed = set()
|
||||
feed_status = {}
|
||||
for feed in ('changes.csv', 'deletions.csv'):
|
||||
try:
|
||||
raw = get(BASE + feed)
|
||||
feed_status[feed] = hashlib.sha256(raw).hexdigest()
|
||||
if feed == 'changes.csv':
|
||||
changed |= extract_changed(raw, known)
|
||||
else:
|
||||
# Deletions are handled through the authoritative index below.
|
||||
# Preserve hash to observe feed changes in state.
|
||||
pass
|
||||
except Exception as exc:
|
||||
print(f'Warning: {feed} not available: {exc}', file=sys.stderr)
|
||||
feed_status[feed] = None
|
||||
missing = {p for p in paths if not (data_dir / p).exists()}
|
||||
cursor = int(old.get('rotation_cursor', 0)) % len(paths)
|
||||
rotating = {paths[(cursor + n) % len(paths)] for n in range(min(rotate, len(paths)))}
|
||||
todo = sorted(missing | changed | rotating)
|
||||
if limit:
|
||||
todo = todo[:limit]
|
||||
count_new = count_changed = count_same = 0
|
||||
for index, rel in enumerate(todo, 1):
|
||||
raw = get(BASE + quote(rel, safe='/'))
|
||||
record = json.loads(raw)
|
||||
if not isinstance(record, dict) or not isinstance(record.get('document'), dict):
|
||||
raise ValueError(f'Invalid CSAF structure: {rel}')
|
||||
out = data_dir / rel
|
||||
was_present = out.exists()
|
||||
if write_if_changed(out, raw):
|
||||
if was_present:
|
||||
count_changed += 1
|
||||
else:
|
||||
count_new += 1
|
||||
else:
|
||||
count_same += 1
|
||||
if index % 250 == 0:
|
||||
print(f'Processed {index}/{len(todo)}')
|
||||
removed = 0
|
||||
if prune and not limit:
|
||||
existing = [p for p in data_dir.glob('*/*.json') if int(p.parent.name) >= year]
|
||||
doomed = [p for p in existing if f'{p.parent.name}/{p.name}'.lower() not in known]
|
||||
if len(doomed) > max(100, int(len(existing) * .05)):
|
||||
raise RuntimeError(f'Refusing unexpected mass deletion: {len(doomed)} paths')
|
||||
for p in doomed:
|
||||
p.unlink()
|
||||
removed = len(doomed)
|
||||
state = {
|
||||
'schema': 1, 'source': BASE, 'since_year': year,
|
||||
'last_success_utc': datetime.now(timezone.utc).isoformat(),
|
||||
'index_sha256': hashlib.sha256(index_raw).hexdigest(),
|
||||
'index_count': len(paths), 'rotation_cursor': (cursor + (0 if limit else rotate)) % len(paths),
|
||||
'feed_sha256': feed_status,
|
||||
'last_run': {'downloaded': len(todo), 'new': count_new, 'updated': count_changed,
|
||||
'unchanged': count_same, 'removed': removed, 'limited': bool(limit)}
|
||||
}
|
||||
# A limited developer run must not advance the durable synchronization status.
|
||||
if not limit:
|
||||
state_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
write_if_changed(state_path, (json.dumps(state, indent=2, ensure_ascii=False) + '\n').encode())
|
||||
print(json.dumps(state['last_run'], sort_keys=True))
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument('--root', default='.')
|
||||
parser.add_argument('--since-year', type=int, default=2017)
|
||||
parser.add_argument('--rotate', type=int, default=250, help='Number of older files to recheck per run')
|
||||
parser.add_argument('--prune', action='store_true', help='Remove files missing from index after safety checks')
|
||||
parser.add_argument('--limit', type=int, default=0, help='Developer test: do not advance state')
|
||||
args = parser.parse_args()
|
||||
if args.rotate < 0 or args.limit < 0:
|
||||
parser.error('rotate and limit must not be negative')
|
||||
sync(args.root, args.since_year, args.rotate, args.prune, args.limit)
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
Reference in New Issue
Block a user