Initial Ubuntu security mirror

2026-11-10 21:58:22 +01:00
commit e0be5fda89
7 changed files with 275 additions and 0 deletions
@@ -0,0 +1,32 @@
+#!/usr/bin/env bash
+set -Eeuo pipefail
+umask 077
+: "${GITEA_JOB_TOKEN:?GITEA_JOB_TOKEN fehlt}"
+CACHE_ROOT="${CACHE_ROOT:-/var/lib/act_runner/cache/vuln-ubuntu}"
+REPO_URL="${REPO_URL:-https://git.apex-project.de/VULN/vuln-ubuntu.git}"
+mkdir -p "$CACHE_ROOT"
+exec 9>"$CACHE_ROOT/.sync.lock"
+flock -w 300 9 || { echo 'Schon ein anderer Lauf aktiv' >&2; exit 1; }
+cd "$CACHE_ROOT"
+if [[ ! -d repo/.git ]]; then
+  [[ ! -e repo ]] || { echo 'repo existiert ohne gültiges Git-Verzeichnis' >&2; exit 1; }
+  git -c "http.extraHeader=$AUTH_HEADER" clone --depth 1 "$REPO_URL" repo
+fi
+cd repo
+# No credentials written to disk. Authentication is supplied on individual
+# Git requests through an HTTP header.
+AUTH_HEADER="Authorization: Basic $(printf 'oauth2:%s' "$GITEA_JOB_TOKEN" | base64 -w0)"
+git -c "http.extraHeader=$AUTH_HEADER" fetch --depth 1 origin main
+git checkout -B main FETCH_HEAD
+git reset --hard FETCH_HEAD
+# Keep the large upstream checkout outside the Gitea repository.
+python3 scripts/sync.py --upstream-cache "$CACHE_ROOT/upstream" --since-year 2017
+python3 scripts/validate.py
+git add -A data state
+if git diff --cached --quiet; then
+  echo 'Keine Rohdaten-Änderungen'
+  exit 0
+fi
+git -c user.name='Vulnerability Bot' -c user.email='vulnerability-bot@localhost.invalid' \
+  commit -m "Update Ubuntu OSV $(date -u +%Y-%m-%dT%H:%M:%SZ)"
+git -c "http.extraHeader=$AUTH_HEADER" push origin HEAD:main
@@ -0,0 +1,129 @@
+#!/usr/bin/env python3
+"""Mirror Canonical's OSV JSON dataset to the local Gitea repository.
+
+The upstream Git checkout lives outside the repository in a persistent cache.
+The initial clone is large; subsequent runs fetch only new Git objects.
+"""
+import argparse
+import datetime as dt
+import hashlib
+import json
+import os
+from pathlib import Path
+import shutil
+import subprocess
+import sys
+import tempfile
+
+UPSTREAM = 'https://github.com/canonical/ubuntu-security-notices.git'
+ROOT = Path(__file__).resolve().parents[1]
+
+
+def run(*args, cwd=None):
+    print('+', ' '.join(map(str,args)), flush=True)
+    subprocess.run(args, cwd=cwd, check=True)
+
+
+def upstream_update(path: Path):
+    path.parent.mkdir(parents=True, exist_ok=True)
+    if not (path / '.git').is_dir():
+        if path.exists():
+            if any(path.iterdir()):
+                raise RuntimeError(f'Upstream-Cache ist kein Git-Repository: {path}')
+            path.rmdir()
+        run('git', 'clone', '--depth', '1', '--filter=blob:none', '--sparse', UPSTREAM, str(path))
+        run('git', 'sparse-checkout', 'set', 'osv', cwd=path)
+    else:
+        run('git', 'fetch', '--depth', '1', 'origin', 'main', cwd=path)
+        run('git', 'reset', '--hard', 'FETCH_HEAD', cwd=path)
+        run('git', 'sparse-checkout', 'set', 'osv', cwd=path)
+    sha = subprocess.check_output(['git','rev-parse','HEAD'], cwd=path, text=True).strip()
+    return sha
+
+
+def collect(src: Path, since_year: int):
+    """Select OSV CVEs from year >= since_year, plus all USNs/LSNs."""
+    if not (src/'cve').is_dir():
+        raise RuntimeError('Upstream osv/cve/ nicht vorhanden: Abbruch ohne Löschung')
+    count = 0
+    for p in sorted(src.rglob('*.json')):
+        rel = p.relative_to(src)
+        if rel.parts[0] == 'cve':
+            if len(rel.parts) < 3 or not rel.parts[1].isdigit():
+                continue
+            if int(rel.parts[1]) < since_year:
+                continue
+        elif rel.parts[0] not in ('usn', 'lsn'):
+            continue
+        count += 1
+        yield p, rel
+    if count == 0:
+        raise RuntimeError('Keine OSV-JSON-Dateien gefunden')
+
+
+def is_valid(payload, name):
+    if not isinstance(payload, dict) or not isinstance(payload.get('id'), str) or not payload['id']:
+        raise ValueError(f'Ungültiges OSV-Dokument: {name}')
+
+
+def sync(src: Path, dst: Path, since_year: int):
+    dst.mkdir(parents=True, exist_ok=True)
+    present = set()
+    updated = 0
+    unchanged = 0
+    for p, rel in collect(src, since_year):
+        data = p.read_bytes()
+        is_valid(json.loads(data), p)
+        target = dst / rel
+        present.add(rel.as_posix())
+        if target.exists() and target.read_bytes() == data:
+            unchanged += 1
+            continue
+        target.parent.mkdir(parents=True, exist_ok=True)
+        with tempfile.NamedTemporaryFile(dir=target.parent, delete=False) as f:
+            tmp = Path(f.name)
+            f.write(data)
+        os.replace(tmp, target)
+        updated += 1
+    # Only delete after fully successful fetch and validation of all inputs.
+    removed = 0
+    for p in dst.rglob('*.json'):
+        if p.relative_to(dst).as_posix() not in present:
+            p.unlink()
+            removed += 1
+    for folder in sorted((p for p in dst.rglob('*') if p.is_dir()), key=lambda x:len(x.parts), reverse=True):
+        if not any(folder.iterdir()):
+            folder.rmdir()
+    return {'files':len(present), 'updated':updated,'unchanged':unchanged,'removed':removed}
+
+
+def main():
+    ap=argparse.ArgumentParser()
+    ap.add_argument('--upstream-cache', default=os.environ.get('UPSTREAM_CACHE','/var/lib/act_runner/cache/vuln-ubuntu/upstream'))
+    ap.add_argument('--since-year', type=int, default=2017)
+    args=ap.parse_args()
+    if args.since_year < 1999 or args.since_year > dt.datetime.now(dt.timezone.utc).year:
+        ap.error('Ungültiges Jahr')
+    sha = upstream_update(Path(args.upstream_cache))
+    stats = sync(Path(args.upstream_cache)/'osv', ROOT/'data'/'osv', args.since_year)
+    state={'source':UPSTREAM,'upstream_commit':sha,'since_year':args.since_year,
+           'last_success_utc':dt.datetime.now(dt.timezone.utc).isoformat(), **stats}
+    (ROOT/'state').mkdir(exist_ok=True)
+    state_path = ROOT/'state'/'sync.json'
+    # Do not produce a Git commit if the upstream commit and dataset are unchanged.
+    previous = {}
+    if state_path.exists():
+        try:
+            previous = json.loads(state_path.read_text())
+        except (OSError, ValueError):
+            pass
+    if previous.get('upstream_commit') != sha or stats['updated'] or stats['removed']:
+        state_path.write_text(json.dumps(state,indent=2,ensure_ascii=False)+'\n')
+    print(json.dumps(state, indent=2))
+
+if __name__=='__main__':
+    try:
+        main()
+    except (RuntimeError, subprocess.CalledProcessError, OSError, ValueError, json.JSONDecodeError) as e:
+        print(f'Synchronisierung fehlgeschlagen: {e}',file=sys.stderr)
+        sys.exit(1)
@@ -0,0 +1,15 @@
+#!/usr/bin/env python3
+import json
+from pathlib import Path
+import sys
+root=Path(__file__).resolve().parents[1]
+files=list((root/'data'/'osv').rglob('*.json'))
+if not files:
+    sys.exit('Fehler: Keine OSV-Dateien gefunden')
+for p in files:
+    try:
+        v=json.loads(p.read_bytes())
+        assert isinstance(v,dict) and isinstance(v.get('id'),str) and v['id']
+    except (ValueError, AssertionError) as e:
+        sys.exit(f'Ungültig: {p}: {e}')
+print(f'{len(files)} Ubuntu OSV-Datensätze gültig')