130 lines
4.9 KiB
Python
130 lines
4.9 KiB
Python
#!/usr/bin/env python3
|
|
"""Mirror Canonical's OSV JSON dataset to the local Gitea repository.
|
|
|
|
The upstream Git checkout lives outside the repository in a persistent cache.
|
|
The initial clone is large; subsequent runs fetch only new Git objects.
|
|
"""
|
|
import argparse
|
|
import datetime as dt
|
|
import hashlib
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import shutil
|
|
import subprocess
|
|
import sys
|
|
import tempfile
|
|
|
|
UPSTREAM = 'https://github.com/canonical/ubuntu-security-notices.git'
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
|
|
|
|
def run(*args, cwd=None):
|
|
print('+', ' '.join(map(str,args)), flush=True)
|
|
subprocess.run(args, cwd=cwd, check=True)
|
|
|
|
|
|
def upstream_update(path: Path):
|
|
path.parent.mkdir(parents=True, exist_ok=True)
|
|
if not (path / '.git').is_dir():
|
|
if path.exists():
|
|
if any(path.iterdir()):
|
|
raise RuntimeError(f'Upstream-Cache ist kein Git-Repository: {path}')
|
|
path.rmdir()
|
|
run('git', 'clone', '--depth', '1', '--filter=blob:none', '--sparse', UPSTREAM, str(path))
|
|
run('git', 'sparse-checkout', 'set', 'osv', cwd=path)
|
|
else:
|
|
run('git', 'fetch', '--depth', '1', 'origin', 'main', cwd=path)
|
|
run('git', 'reset', '--hard', 'FETCH_HEAD', cwd=path)
|
|
run('git', 'sparse-checkout', 'set', 'osv', cwd=path)
|
|
sha = subprocess.check_output(['git','rev-parse','HEAD'], cwd=path, text=True).strip()
|
|
return sha
|
|
|
|
|
|
def collect(src: Path, since_year: int):
|
|
"""Select OSV CVEs from year >= since_year, plus all USNs/LSNs."""
|
|
if not (src/'cve').is_dir():
|
|
raise RuntimeError('Upstream osv/cve/ nicht vorhanden: Abbruch ohne Löschung')
|
|
count = 0
|
|
for p in sorted(src.rglob('*.json')):
|
|
rel = p.relative_to(src)
|
|
if rel.parts[0] == 'cve':
|
|
if len(rel.parts) < 3 or not rel.parts[1].isdigit():
|
|
continue
|
|
if int(rel.parts[1]) < since_year:
|
|
continue
|
|
elif rel.parts[0] not in ('usn', 'lsn'):
|
|
continue
|
|
count += 1
|
|
yield p, rel
|
|
if count == 0:
|
|
raise RuntimeError('Keine OSV-JSON-Dateien gefunden')
|
|
|
|
|
|
def is_valid(payload, name):
|
|
if not isinstance(payload, dict) or not isinstance(payload.get('id'), str) or not payload['id']:
|
|
raise ValueError(f'Ungültiges OSV-Dokument: {name}')
|
|
|
|
|
|
def sync(src: Path, dst: Path, since_year: int):
|
|
dst.mkdir(parents=True, exist_ok=True)
|
|
present = set()
|
|
updated = 0
|
|
unchanged = 0
|
|
for p, rel in collect(src, since_year):
|
|
data = p.read_bytes()
|
|
is_valid(json.loads(data), p)
|
|
target = dst / rel
|
|
present.add(rel.as_posix())
|
|
if target.exists() and target.read_bytes() == data:
|
|
unchanged += 1
|
|
continue
|
|
target.parent.mkdir(parents=True, exist_ok=True)
|
|
with tempfile.NamedTemporaryFile(dir=target.parent, delete=False) as f:
|
|
tmp = Path(f.name)
|
|
f.write(data)
|
|
os.replace(tmp, target)
|
|
updated += 1
|
|
# Only delete after fully successful fetch and validation of all inputs.
|
|
removed = 0
|
|
for p in dst.rglob('*.json'):
|
|
if p.relative_to(dst).as_posix() not in present:
|
|
p.unlink()
|
|
removed += 1
|
|
for folder in sorted((p for p in dst.rglob('*') if p.is_dir()), key=lambda x:len(x.parts), reverse=True):
|
|
if not any(folder.iterdir()):
|
|
folder.rmdir()
|
|
return {'files':len(present), 'updated':updated,'unchanged':unchanged,'removed':removed}
|
|
|
|
|
|
def main():
|
|
ap=argparse.ArgumentParser()
|
|
ap.add_argument('--upstream-cache', default=os.environ.get('UPSTREAM_CACHE','/var/lib/act_runner/cache/vuln-ubuntu/upstream'))
|
|
ap.add_argument('--since-year', type=int, default=2017)
|
|
args=ap.parse_args()
|
|
if args.since_year < 1999 or args.since_year > dt.datetime.now(dt.timezone.utc).year:
|
|
ap.error('Ungültiges Jahr')
|
|
sha = upstream_update(Path(args.upstream_cache))
|
|
stats = sync(Path(args.upstream_cache)/'osv', ROOT/'data'/'osv', args.since_year)
|
|
state={'source':UPSTREAM,'upstream_commit':sha,'since_year':args.since_year,
|
|
'last_success_utc':dt.datetime.now(dt.timezone.utc).isoformat(), **stats}
|
|
(ROOT/'state').mkdir(exist_ok=True)
|
|
state_path = ROOT/'state'/'sync.json'
|
|
# Do not produce a Git commit if the upstream commit and dataset are unchanged.
|
|
previous = {}
|
|
if state_path.exists():
|
|
try:
|
|
previous = json.loads(state_path.read_text())
|
|
except (OSError, ValueError):
|
|
pass
|
|
if previous.get('upstream_commit') != sha or stats['updated'] or stats['removed']:
|
|
state_path.write_text(json.dumps(state,indent=2,ensure_ascii=False)+'\n')
|
|
print(json.dumps(state, indent=2))
|
|
|
|
if __name__=='__main__':
|
|
try:
|
|
main()
|
|
except (RuntimeError, subprocess.CalledProcessError, OSError, ValueError, json.JSONDecodeError) as e:
|
|
print(f'Synchronisierung fehlgeschlagen: {e}',file=sys.stderr)
|
|
sys.exit(1)
|