diff --git a/scripts/sync.py b/scripts/sync.py index 66541e0068..dbb7e9a489 100644 --- a/scripts/sync.py +++ b/scripts/sync.py @@ -1,70 +1,130 @@ #!/usr/bin/env python3 -"""Mirror official CVE List V5 current snapshot (2017+) into this repository.""" +"""Incrementally mirror CVEProject/cvelistV5 CVE records into data/. + +Requires Linux, git >=2.25, rsync, Python 3.9+. +Persistent upstream clone is protected by flock; no new clone after first run. +""" + import argparse import datetime as dt +import fcntl import json import os from pathlib import Path import re -import shutil import subprocess -import tempfile -UPSTREAM = 'https://github.com/CVEProject/cvelistV5.git' ROOT = Path(__file__).resolve().parents[1] +UPSTREAM_URL = 'https://github.com/CVEProject/cvelistV5.git' +DEFAULT_CACHE = Path('/var/cache/vuln-cve') + + +def command(*args, cwd=None, capture=False): + return subprocess.run( + list(map(str, args)), cwd=cwd, check=True, + text=True, capture_output=capture, + ) + + +def git(cwd, *args): + return command('git', '-C', cwd, *args, capture=True).stdout.strip() + + +def prepare_cache(path, source_url, years): + if not (path / '.git').is_dir(): + if path.exists() and any(path.iterdir()): + raise RuntimeError(f'Incomplete cache directory: {path}; move it aside and retry') + path.parent.mkdir(parents=True, exist_ok=True) + command('git', 'clone', '--depth=1', '--filter=blob:none', '--sparse', + '--branch=main', source_url, path) + command('git', '-C', path, 'sparse-checkout', 'set', *years) + else: + # Refuse accidentally reusing a different upstream repository. + remote_url = git(path, 'remote', 'get-url', 'origin') + if remote_url != source_url: + raise RuntimeError(f'Cache remote mismatch: {remote_url}') + existing = git(path, 'sparse-checkout', 'list').splitlines() + if existing != years: + command('git', '-C', path, 'sparse-checkout', 'set', *years) + + # Shallow fetch downloads objects missing from persistent cache. + command('git', '-C', path, 'fetch', '--depth=1', '--no-tags', 'origin', 'main') + command('git', '-C', path, 'reset', '--hard', 'FETCH_HEAD') + return git(path, 'rev-parse', 'HEAD') + + +def validate_records(source_root, minimum): + count = 0 + for file in source_root.rglob('CVE-*.json'): + if not re.fullmatch(r'CVE-\d{4}-\d{4,}', file.stem): + raise RuntimeError(f'Invalid filename: {file}') + try: + with file.open('r', encoding='utf-8') as handle: + doc = json.load(handle) + except (ValueError, OSError) as exc: + raise RuntimeError(f'Invalid JSON file {file}: {exc}') from exc + if doc.get('cveMetadata', {}).get('cveId') != file.stem: + raise RuntimeError(f'CVE ID mismatch: {file}') + count += 1 + if count < minimum: + raise RuntimeError(f'Only {count} records (< {minimum}): refusing destructive sync') + return count -def run(*args, cwd=None): - return subprocess.check_output(args, cwd=cwd, text=True).strip() def main(): - ap = argparse.ArgumentParser() - ap.add_argument('--since-year', type=int, default=2017) - ap.add_argument('--source-url', default=UPSTREAM) - ap.add_argument('--minimum-records', type=int, default=1000) - args = ap.parse_args() - now_year = dt.datetime.now(dt.timezone.utc).year - if not 1999 <= args.since_year <= now_year: - ap.error('invalid since-year') - if not shutil.which('git') or not shutil.which('rsync'): - raise SystemExit('git and rsync are required') - source_dir = ROOT / 'data' - state_dir = ROOT / 'state' - source_dir.mkdir(exist_ok=True) - state_dir.mkdir(exist_ok=True) - with tempfile.TemporaryDirectory(prefix='vuln-cve-') as tmp: - clone = Path(tmp) / 'source' - subprocess.run(['git', 'clone', '--depth=1', '--filter=blob:none', '--sparse', args.source_url, str(clone)], check=True) - years = [f'cves/{y}' for y in range(args.since_year, now_year + 1)] - subprocess.run(['git', '-C', str(clone), 'sparse-checkout', 'set', *years], check=True) - revision = run('git', 'rev-parse', 'HEAD', cwd=clone) - record_root = clone / 'cves' - candidates = list(record_root.rglob('CVE-*.json')) - if len(candidates) < args.minimum_records: - raise RuntimeError(f'Upstream has only {len(candidates)} CVE records: refusing destructive synchronization') - # Validate JSON structure and identifier across the entire source before deleting local entries. - validated = 0 - for p in candidates: - if not re.fullmatch(r'CVE-\d{4}-\d{4,}', p.stem): - raise RuntimeError(f'Unexpected file name: {p}') - try: - with p.open(encoding='utf-8') as handle: - record = json.load(handle) - if record.get('cveMetadata', {}).get('cveId') != p.stem: - raise ValueError('metadata CVE ID mismatch') - except Exception as exc: - raise RuntimeError(f'Invalid upstream CVE record {p}: {exc}') from exc - validated += 1 - # Delete files not present upstream, including records withdrawn or removed. - subprocess.run(['rsync', '-a', '--delete', '--exclude=.gitkeep', str(record_root) + '/', str(source_dir) + '/'], check=True) + parser = argparse.ArgumentParser() + parser.add_argument('--since-year', type=int, default=2017) + parser.add_argument('--cache-dir', type=Path, default=Path(os.environ.get('CVE_CACHE_DIR', DEFAULT_CACHE))) + parser.add_argument('--source-url', default=UPSTREAM_URL) + parser.add_argument('--minimum-records', type=int, default=1000) + parser.add_argument('--force-verify', action='store_true', help='Validate and rsync even if upstream commit has not changed') + args = parser.parse_args() + + current_year = dt.datetime.now(dt.timezone.utc).year + if not 1999 <= args.since_year <= current_year: + parser.error('Invalid --since-year') + years = [f'cves/{year}' for year in range(args.since_year, current_year + 1)] + + cache_root = args.cache_dir.resolve() + cache_root.mkdir(parents=True, exist_ok=True) + with (cache_root / 'sync.lock').open('a+') as lock: + fcntl.flock(lock.fileno(), fcntl.LOCK_EX) + upstream = cache_root / 'upstream' + revision = prepare_cache(upstream, args.source_url, years) + + data_dir = ROOT / 'data' + state_file = ROOT / 'state' / 'sync.json' + previous = {} + if state_file.is_file(): + previous = json.loads(state_file.read_text(encoding='utf-8')) + if ( + not args.force_verify + and previous.get('upstream_commit') == revision + and previous.get('since_year') == args.since_year + and data_dir.is_dir() + and any(data_dir.rglob('CVE-*.json')) + ): + print(f'No upstream changes; cached revision {revision}') + return + + src = upstream / 'cves' + records = validate_records(src, args.minimum_records) + data_dir.mkdir(parents=True, exist_ok=True) + command('rsync', '-a', '--delete', '--exclude=.gitkeep', f'{src}/', f'{data_dir}/') + + state_file.parent.mkdir(parents=True, exist_ok=True) state = { 'source': args.source_url, 'upstream_commit': revision, 'since_year': args.since_year, - 'records': validated, + 'records': records, 'synced_at_utc': dt.datetime.now(dt.timezone.utc).isoformat(), } - (state_dir / 'sync.json').write_text(json.dumps(state, ensure_ascii=False, indent=2) + '\n', encoding='utf-8') - print(f'Mirrored {validated} CVE records at {revision}') + tmp = state_file.with_suffix('.json.tmp') + tmp.write_text(json.dumps(state, indent=2, ensure_ascii=False) + '\n', encoding='utf-8') + os.replace(tmp, state_file) + print(f'Mirrored {records} records from {revision}') + if __name__ == '__main__': main()