scripts/sync.py aktualisiert

scripts/sync.py

@@ -1,70 +1,130 @@
 #!/usr/bin/env python3
-"""Mirror official CVE List V5 current snapshot (2017+) into this repository."""
+"""Incrementally mirror CVEProject/cvelistV5 CVE records into data/.
+
+Requires Linux, git >=2.25, rsync, Python 3.9+.
+Persistent upstream clone is protected by flock; no new clone after first run.
+"""
+
 import argparse
 import datetime as dt
+import fcntl
 import json
 import os
 from pathlib import Path
 import re
-import shutil
 import subprocess
-import tempfile
 
-UPSTREAM = 'https://github.com/CVEProject/cvelistV5.git'
 ROOT = Path(__file__).resolve().parents[1]
+UPSTREAM_URL = 'https://github.com/CVEProject/cvelistV5.git'
+DEFAULT_CACHE = Path('/var/cache/vuln-cve')
+
+
+def command(*args, cwd=None, capture=False):
+    return subprocess.run(
+        list(map(str, args)), cwd=cwd, check=True,
+        text=True, capture_output=capture,
+    )
+
+
+def git(cwd, *args):
+    return command('git', '-C', cwd, *args, capture=True).stdout.strip()
+
+
+def prepare_cache(path, source_url, years):
+    if not (path / '.git').is_dir():
+        if path.exists() and any(path.iterdir()):
+            raise RuntimeError(f'Incomplete cache directory: {path}; move it aside and retry')
+        path.parent.mkdir(parents=True, exist_ok=True)
+        command('git', 'clone', '--depth=1', '--filter=blob:none', '--sparse',
+                '--branch=main', source_url, path)
+        command('git', '-C', path, 'sparse-checkout', 'set', *years)
+    else:
+        # Refuse accidentally reusing a different upstream repository.
+        remote_url = git(path, 'remote', 'get-url', 'origin')
+        if remote_url != source_url:
+            raise RuntimeError(f'Cache remote mismatch: {remote_url}')
+        existing = git(path, 'sparse-checkout', 'list').splitlines()
+        if existing != years:
+            command('git', '-C', path, 'sparse-checkout', 'set', *years)
+
+    # Shallow fetch downloads objects missing from persistent cache.
+    command('git', '-C', path, 'fetch', '--depth=1', '--no-tags', 'origin', 'main')
+    command('git', '-C', path, 'reset', '--hard', 'FETCH_HEAD')
+    return git(path, 'rev-parse', 'HEAD')
+
+
+def validate_records(source_root, minimum):
+    count = 0
+    for file in source_root.rglob('CVE-*.json'):
+        if not re.fullmatch(r'CVE-\d{4}-\d{4,}', file.stem):
+            raise RuntimeError(f'Invalid filename: {file}')
+        try:
+            with file.open('r', encoding='utf-8') as handle:
+                doc = json.load(handle)
+        except (ValueError, OSError) as exc:
+            raise RuntimeError(f'Invalid JSON file {file}: {exc}') from exc
+        if doc.get('cveMetadata', {}).get('cveId') != file.stem:
+            raise RuntimeError(f'CVE ID mismatch: {file}')
+        count += 1
+    if count < minimum:
+        raise RuntimeError(f'Only {count} records (< {minimum}): refusing destructive sync')
+    return count
 
-def run(*args, cwd=None):
-    return subprocess.check_output(args, cwd=cwd, text=True).strip()
 
 def main():
-    ap = argparse.ArgumentParser()
+    parser = argparse.ArgumentParser()
-    ap.add_argument('--since-year', type=int, default=2017)
+    parser.add_argument('--since-year', type=int, default=2017)
-    ap.add_argument('--source-url', default=UPSTREAM)
+    parser.add_argument('--cache-dir', type=Path, default=Path(os.environ.get('CVE_CACHE_DIR', DEFAULT_CACHE)))
-    ap.add_argument('--minimum-records', type=int, default=1000)
+    parser.add_argument('--source-url', default=UPSTREAM_URL)
-    args = ap.parse_args()
+    parser.add_argument('--minimum-records', type=int, default=1000)
-    now_year = dt.datetime.now(dt.timezone.utc).year
+    parser.add_argument('--force-verify', action='store_true', help='Validate and rsync even if upstream commit has not changed')
-    if not 1999 <= args.since_year <= now_year:
+    args = parser.parse_args()
-        ap.error('invalid since-year')
+
-    if not shutil.which('git') or not shutil.which('rsync'):
+    current_year = dt.datetime.now(dt.timezone.utc).year
-        raise SystemExit('git and rsync are required')
+    if not 1999 <= args.since_year <= current_year:
-    source_dir = ROOT / 'data'
+        parser.error('Invalid --since-year')
-    state_dir = ROOT / 'state'
+    years = [f'cves/{year}' for year in range(args.since_year, current_year + 1)]
-    source_dir.mkdir(exist_ok=True)
+
-    state_dir.mkdir(exist_ok=True)
+    cache_root = args.cache_dir.resolve()
-    with tempfile.TemporaryDirectory(prefix='vuln-cve-') as tmp:
+    cache_root.mkdir(parents=True, exist_ok=True)
-        clone = Path(tmp) / 'source'
+    with (cache_root / 'sync.lock').open('a+') as lock:
-        subprocess.run(['git', 'clone', '--depth=1', '--filter=blob:none', '--sparse', args.source_url, str(clone)], check=True)
+        fcntl.flock(lock.fileno(), fcntl.LOCK_EX)
-        years = [f'cves/{y}' for y in range(args.since_year, now_year + 1)]
+        upstream = cache_root / 'upstream'
-        subprocess.run(['git', '-C', str(clone), 'sparse-checkout', 'set', *years], check=True)
+        revision = prepare_cache(upstream, args.source_url, years)
-        revision = run('git', 'rev-parse', 'HEAD', cwd=clone)
+
-        record_root = clone / 'cves'
+        data_dir = ROOT / 'data'
-        candidates = list(record_root.rglob('CVE-*.json'))
+        state_file = ROOT / 'state' / 'sync.json'
-        if len(candidates) < args.minimum_records:
+        previous = {}
-            raise RuntimeError(f'Upstream has only {len(candidates)} CVE records: refusing destructive synchronization')
+        if state_file.is_file():
-        # Validate JSON structure and identifier across the entire source before deleting local entries.
+            previous = json.loads(state_file.read_text(encoding='utf-8'))
-        validated = 0
+        if (
-        for p in candidates:
+            not args.force_verify
-            if not re.fullmatch(r'CVE-\d{4}-\d{4,}', p.stem):
+            and previous.get('upstream_commit') == revision
-                raise RuntimeError(f'Unexpected file name: {p}')
+            and previous.get('since_year') == args.since_year
-            try:
+            and data_dir.is_dir()
-                with p.open(encoding='utf-8') as handle:
+            and any(data_dir.rglob('CVE-*.json'))
-                    record = json.load(handle)
+        ):
-                if record.get('cveMetadata', {}).get('cveId') != p.stem:
+            print(f'No upstream changes; cached revision {revision}')
-                    raise ValueError('metadata CVE ID mismatch')
+            return
-            except Exception as exc:
+
-                raise RuntimeError(f'Invalid upstream CVE record {p}: {exc}') from exc
+        src = upstream / 'cves'
-            validated += 1
+        records = validate_records(src, args.minimum_records)
-        # Delete files not present upstream, including records withdrawn or removed.
+        data_dir.mkdir(parents=True, exist_ok=True)
-        subprocess.run(['rsync', '-a', '--delete', '--exclude=.gitkeep', str(record_root) + '/', str(source_dir) + '/'], check=True)
+        command('rsync', '-a', '--delete', '--exclude=.gitkeep', f'{src}/', f'{data_dir}/')
+
+        state_file.parent.mkdir(parents=True, exist_ok=True)
         state = {
             'source': args.source_url,
             'upstream_commit': revision,
             'since_year': args.since_year,
-            'records': validated,
+            'records': records,
             'synced_at_utc': dt.datetime.now(dt.timezone.utc).isoformat(),
         }
-        (state_dir / 'sync.json').write_text(json.dumps(state, ensure_ascii=False, indent=2) + '\n', encoding='utf-8')
+        tmp = state_file.with_suffix('.json.tmp')
-        print(f'Mirrored {validated} CVE records at {revision}')
+        tmp.write_text(json.dumps(state, indent=2, ensure_ascii=False) + '\n', encoding='utf-8')
+        os.replace(tmp, state_file)
+        print(f'Mirrored {records} records from {revision}')
+
 
 if __name__ == '__main__':
     main()