scripts/sync.py aktualisiert
This commit is contained in:
+108
-48
@@ -1,70 +1,130 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Mirror official CVE List V5 current snapshot (2017+) into this repository."""
|
||||
"""Incrementally mirror CVEProject/cvelistV5 CVE records into data/.
|
||||
|
||||
Requires Linux, git >=2.25, rsync, Python 3.9+.
|
||||
Persistent upstream clone is protected by flock; no new clone after first run.
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import datetime as dt
|
||||
import fcntl
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import re
|
||||
import shutil
|
||||
import subprocess
|
||||
import tempfile
|
||||
|
||||
UPSTREAM = 'https://github.com/CVEProject/cvelistV5.git'
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
UPSTREAM_URL = 'https://github.com/CVEProject/cvelistV5.git'
|
||||
DEFAULT_CACHE = Path('/var/cache/vuln-cve')
|
||||
|
||||
|
||||
def command(*args, cwd=None, capture=False):
|
||||
return subprocess.run(
|
||||
list(map(str, args)), cwd=cwd, check=True,
|
||||
text=True, capture_output=capture,
|
||||
)
|
||||
|
||||
|
||||
def git(cwd, *args):
|
||||
return command('git', '-C', cwd, *args, capture=True).stdout.strip()
|
||||
|
||||
|
||||
def prepare_cache(path, source_url, years):
|
||||
if not (path / '.git').is_dir():
|
||||
if path.exists() and any(path.iterdir()):
|
||||
raise RuntimeError(f'Incomplete cache directory: {path}; move it aside and retry')
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
command('git', 'clone', '--depth=1', '--filter=blob:none', '--sparse',
|
||||
'--branch=main', source_url, path)
|
||||
command('git', '-C', path, 'sparse-checkout', 'set', *years)
|
||||
else:
|
||||
# Refuse accidentally reusing a different upstream repository.
|
||||
remote_url = git(path, 'remote', 'get-url', 'origin')
|
||||
if remote_url != source_url:
|
||||
raise RuntimeError(f'Cache remote mismatch: {remote_url}')
|
||||
existing = git(path, 'sparse-checkout', 'list').splitlines()
|
||||
if existing != years:
|
||||
command('git', '-C', path, 'sparse-checkout', 'set', *years)
|
||||
|
||||
# Shallow fetch downloads objects missing from persistent cache.
|
||||
command('git', '-C', path, 'fetch', '--depth=1', '--no-tags', 'origin', 'main')
|
||||
command('git', '-C', path, 'reset', '--hard', 'FETCH_HEAD')
|
||||
return git(path, 'rev-parse', 'HEAD')
|
||||
|
||||
|
||||
def validate_records(source_root, minimum):
|
||||
count = 0
|
||||
for file in source_root.rglob('CVE-*.json'):
|
||||
if not re.fullmatch(r'CVE-\d{4}-\d{4,}', file.stem):
|
||||
raise RuntimeError(f'Invalid filename: {file}')
|
||||
try:
|
||||
with file.open('r', encoding='utf-8') as handle:
|
||||
doc = json.load(handle)
|
||||
except (ValueError, OSError) as exc:
|
||||
raise RuntimeError(f'Invalid JSON file {file}: {exc}') from exc
|
||||
if doc.get('cveMetadata', {}).get('cveId') != file.stem:
|
||||
raise RuntimeError(f'CVE ID mismatch: {file}')
|
||||
count += 1
|
||||
if count < minimum:
|
||||
raise RuntimeError(f'Only {count} records (< {minimum}): refusing destructive sync')
|
||||
return count
|
||||
|
||||
def run(*args, cwd=None):
|
||||
return subprocess.check_output(args, cwd=cwd, text=True).strip()
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument('--since-year', type=int, default=2017)
|
||||
ap.add_argument('--source-url', default=UPSTREAM)
|
||||
ap.add_argument('--minimum-records', type=int, default=1000)
|
||||
args = ap.parse_args()
|
||||
now_year = dt.datetime.now(dt.timezone.utc).year
|
||||
if not 1999 <= args.since_year <= now_year:
|
||||
ap.error('invalid since-year')
|
||||
if not shutil.which('git') or not shutil.which('rsync'):
|
||||
raise SystemExit('git and rsync are required')
|
||||
source_dir = ROOT / 'data'
|
||||
state_dir = ROOT / 'state'
|
||||
source_dir.mkdir(exist_ok=True)
|
||||
state_dir.mkdir(exist_ok=True)
|
||||
with tempfile.TemporaryDirectory(prefix='vuln-cve-') as tmp:
|
||||
clone = Path(tmp) / 'source'
|
||||
subprocess.run(['git', 'clone', '--depth=1', '--filter=blob:none', '--sparse', args.source_url, str(clone)], check=True)
|
||||
years = [f'cves/{y}' for y in range(args.since_year, now_year + 1)]
|
||||
subprocess.run(['git', '-C', str(clone), 'sparse-checkout', 'set', *years], check=True)
|
||||
revision = run('git', 'rev-parse', 'HEAD', cwd=clone)
|
||||
record_root = clone / 'cves'
|
||||
candidates = list(record_root.rglob('CVE-*.json'))
|
||||
if len(candidates) < args.minimum_records:
|
||||
raise RuntimeError(f'Upstream has only {len(candidates)} CVE records: refusing destructive synchronization')
|
||||
# Validate JSON structure and identifier across the entire source before deleting local entries.
|
||||
validated = 0
|
||||
for p in candidates:
|
||||
if not re.fullmatch(r'CVE-\d{4}-\d{4,}', p.stem):
|
||||
raise RuntimeError(f'Unexpected file name: {p}')
|
||||
try:
|
||||
with p.open(encoding='utf-8') as handle:
|
||||
record = json.load(handle)
|
||||
if record.get('cveMetadata', {}).get('cveId') != p.stem:
|
||||
raise ValueError('metadata CVE ID mismatch')
|
||||
except Exception as exc:
|
||||
raise RuntimeError(f'Invalid upstream CVE record {p}: {exc}') from exc
|
||||
validated += 1
|
||||
# Delete files not present upstream, including records withdrawn or removed.
|
||||
subprocess.run(['rsync', '-a', '--delete', '--exclude=.gitkeep', str(record_root) + '/', str(source_dir) + '/'], check=True)
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument('--since-year', type=int, default=2017)
|
||||
parser.add_argument('--cache-dir', type=Path, default=Path(os.environ.get('CVE_CACHE_DIR', DEFAULT_CACHE)))
|
||||
parser.add_argument('--source-url', default=UPSTREAM_URL)
|
||||
parser.add_argument('--minimum-records', type=int, default=1000)
|
||||
parser.add_argument('--force-verify', action='store_true', help='Validate and rsync even if upstream commit has not changed')
|
||||
args = parser.parse_args()
|
||||
|
||||
current_year = dt.datetime.now(dt.timezone.utc).year
|
||||
if not 1999 <= args.since_year <= current_year:
|
||||
parser.error('Invalid --since-year')
|
||||
years = [f'cves/{year}' for year in range(args.since_year, current_year + 1)]
|
||||
|
||||
cache_root = args.cache_dir.resolve()
|
||||
cache_root.mkdir(parents=True, exist_ok=True)
|
||||
with (cache_root / 'sync.lock').open('a+') as lock:
|
||||
fcntl.flock(lock.fileno(), fcntl.LOCK_EX)
|
||||
upstream = cache_root / 'upstream'
|
||||
revision = prepare_cache(upstream, args.source_url, years)
|
||||
|
||||
data_dir = ROOT / 'data'
|
||||
state_file = ROOT / 'state' / 'sync.json'
|
||||
previous = {}
|
||||
if state_file.is_file():
|
||||
previous = json.loads(state_file.read_text(encoding='utf-8'))
|
||||
if (
|
||||
not args.force_verify
|
||||
and previous.get('upstream_commit') == revision
|
||||
and previous.get('since_year') == args.since_year
|
||||
and data_dir.is_dir()
|
||||
and any(data_dir.rglob('CVE-*.json'))
|
||||
):
|
||||
print(f'No upstream changes; cached revision {revision}')
|
||||
return
|
||||
|
||||
src = upstream / 'cves'
|
||||
records = validate_records(src, args.minimum_records)
|
||||
data_dir.mkdir(parents=True, exist_ok=True)
|
||||
command('rsync', '-a', '--delete', '--exclude=.gitkeep', f'{src}/', f'{data_dir}/')
|
||||
|
||||
state_file.parent.mkdir(parents=True, exist_ok=True)
|
||||
state = {
|
||||
'source': args.source_url,
|
||||
'upstream_commit': revision,
|
||||
'since_year': args.since_year,
|
||||
'records': validated,
|
||||
'records': records,
|
||||
'synced_at_utc': dt.datetime.now(dt.timezone.utc).isoformat(),
|
||||
}
|
||||
(state_dir / 'sync.json').write_text(json.dumps(state, ensure_ascii=False, indent=2) + '\n', encoding='utf-8')
|
||||
print(f'Mirrored {validated} CVE records at {revision}')
|
||||
tmp = state_file.with_suffix('.json.tmp')
|
||||
tmp.write_text(json.dumps(state, indent=2, ensure_ascii=False) + '\n', encoding='utf-8')
|
||||
os.replace(tmp, state_file)
|
||||
print(f'Mirrored {records} records from {revision}')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
|
||||
Reference in New Issue
Block a user