From fd80ba6152421f767ff36da508c40f7c55a730de Mon Sep 17 00:00:00 2001 From: root Date: Tue, 10 Nov 2026 20:41:03 +0100 Subject: [PATCH] Initial OSV mirror --- .gitea/workflows/update.yml | 71 ++++++++++++++++++++ .gitignore | 5 ++ README.md | 21 ++++++ data/.gitkeep | 0 scripts/sync.py | 126 ++++++++++++++++++++++++++++++++++++ scripts/validate.py | 20 ++++++ state/.gitkeep | 0 7 files changed, 243 insertions(+) create mode 100644 .gitea/workflows/update.yml create mode 100644 .gitignore create mode 100644 README.md create mode 100644 data/.gitkeep create mode 100644 scripts/sync.py create mode 100644 scripts/validate.py create mode 100644 state/.gitkeep diff --git a/.gitea/workflows/update.yml b/.gitea/workflows/update.yml new file mode 100644 index 0000000000..baf95e0491 --- /dev/null +++ b/.gitea/workflows/update.yml @@ -0,0 +1,71 @@ +name: vuln-osv Synchronisation (lokaler Git-Cache) + +on: + schedule: + - cron: "41 */6 * * *" + workflow_dispatch: + +permissions: + contents: write + +concurrency: + group: vuln-osv-sync + cancel-in-progress: false + +jobs: + mirror: + runs-on: vuln-runner + timeout-minutes: 360 + steps: + - name: Rohdaten aktualisieren + shell: bash + env: + GITEA_JOB_TOKEN: ${{ secrets.GITEA_TOKEN }} + CACHE_ROOT: /var/lib/act_runner/cache/vuln-osv + run: | + set -Eeuo pipefail + umask 077 + : "${GITEA_JOB_TOKEN:?GITEA_TOKEN fehlt}" + mkdir -p "$CACHE_ROOT" + exec 9>"$CACHE_ROOT/.sync.lock" + flock -w 300 9 || { echo "Bereits ein Lauf aktiv" >&2; exit 1; } + + ASKPASS="$(mktemp "$CACHE_ROOT/.askpass.XXXXXX")" + trap 'rm -f "$ASKPASS"' EXIT + cat > "$ASKPASS" <<'ASKPASS_SCRIPT' + #!/bin/sh + case "$1" in + *Username*|*username*) printf '%s\n' 'oauth2' ;; + *Password*|*password*) printf '%s\n' "$GITEA_JOB_TOKEN" ;; + *) exit 1 ;; + esac + ASKPASS_SCRIPT + chmod 700 "$ASKPASS" + export GIT_ASKPASS="$ASKPASS" GIT_TERMINAL_PROMPT=0 + REMOTE="https://git.apex-project.de/VULN/vuln-osv.git" + REPO="$CACHE_ROOT/repo" + if [ ! -d "$REPO/.git" ]; then + [ ! -e "$REPO" ] || { echo "Unvollstaendiger Cache: $REPO" >&2; exit 1; } + git -c credential.helper= clone --branch main --single-branch --depth 1 "$REMOTE" "$REPO" + else + # Never reset an unfinished local commit or changes after a failed push. + if [ -n "$(git -C "$REPO" status --porcelain)" ] || [ "$(git -C "$REPO" rev-list --count origin/main..HEAD)" != 0 ]; then + echo "Lokale Aenderungen/Commits vorhanden: manuell pruefen, nicht ueberschreiben" >&2 + exit 1 + fi + git -C "$REPO" remote set-url origin "$REMOTE" + git -C "$REPO" -c credential.helper= fetch --no-tags --depth 1 origin main + git -C "$REPO" checkout -B main FETCH_HEAD + fi + cd "$REPO" + python3 scripts/sync.py + python3 scripts/validate.py + git config user.name "Vulnerability Bot" + git config user.email "vuln-bot@users.noreply.local" + git add -A -- data/ state/ + if git diff --cached --quiet; then + echo "Keine neuen Daten" + exit 0 + fi + git commit -m "vuln-osv update $(date -u +%Y-%m-%dT%H:%M:%SZ)" + git -c credential.helper= push origin HEAD:main diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000000..1a6b2a7940 --- /dev/null +++ b/.gitignore @@ -0,0 +1,5 @@ +__pycache__/ +*.pyc +*.tmp +.sync.lock +.DS_Store diff --git a/README.md b/README.md new file mode 100644 index 0000000000..852cd1b2fb --- /dev/null +++ b/README.md @@ -0,0 +1,21 @@ +# vuln-osv – Gitea OSV raw-data mirror + +Speichert Original-JSON aus OSV-Ökosystem-Snapshots direkt in Git unter `data//.json`. + +## Installation +1. Leeres Gitea-Repository `VULN/vuln-osv` erstellen, dieses Verzeichnis pushen. +2. Auf der Gitea-VM: `sudo install -d -o act_runner -g act_runner -m 700 /var/lib/act_runner/cache/vuln-osv`. +3. `vuln-runner:host` muss registriert sein. `python3`, `git` und `flock` müssen verfügbar sein. +4. Actions-Job-Token benötigt Schreibrechte. Workflow `.gitea/workflows/update.yml` starten. + +## Updates +- Zeitplan: alle sechs Stunden (Minute 41). +- HTTP `If-None-Match` / `If-Modified-Since`: Bei unverändertem Export Antwort `304`, kein ZIP-Download. +- Wenn sich ein ZIP ändert, wird **das gesamte Ökosystem-ZIP erneut geladen**, nicht nur die Änderungen. Ein späterer Umbau auf `modified_id.csv` spart weitere Bandbreite. +- Nur geänderte JSON-Dateien werden geschrieben und committed. +- Standard-Ökosysteme: PyPI, npm, Go, Maven, NuGet, crates.io, Packagist, RubyGems, GitHub Actions, OSS-Fuzz. +- Für andere Ökosysteme `python3 scripts/sync.py --ecosystems 'PyPI,npm,Go'` ausführen, beziehungsweise Workflow-Aufruf anpassen. +- **Löschungen**: Standardmäßig werden keine Dateien entfernt. Erst nach belastbar bestätigtem Full-Snapshot mit `--allow-deletions` aktivieren. Ein fehlender Snapshot darf niemals als Leeren interpretiert werden. +- Sehr große Git-Repos: Speicherplatz, Inodes und Gitea-Push-Limits kontrollieren. Für npm/Go sind mehrere hunderttausend Dateien möglich. + +Offizielle Quelle: https://google.github.io/osv.dev/data/ diff --git a/data/.gitkeep b/data/.gitkeep new file mode 100644 index 0000000000..e69de29bb2 diff --git a/scripts/sync.py b/scripts/sync.py new file mode 100644 index 0000000000..c125da1cba --- /dev/null +++ b/scripts/sync.py @@ -0,0 +1,126 @@ +#!/usr/bin/env python3 +"""Mirror full raw OSV JSON files per ecosystem with conditional HTTP requests. +No third-party Python dependencies. All-or-nothing validation per ecosystem. +""" +import argparse +import io +import json +import os +import re +import shutil +import sys +import tempfile +import time +import urllib.error +import urllib.parse +import urllib.request +import zipfile +from datetime import datetime, timezone +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +BASE = 'https://storage.googleapis.com/osv-vulnerabilities' +DEFAULT = 'PyPI,npm,Go,Maven,NuGet,crates.io,Packagist,RubyGems,GitHub Actions,OSS-Fuzz' +VALID_ECOSYSTEM = re.compile(r'^[A-Za-z0-9._ -]{1,60}$') +VALID_ID = re.compile(r'^[A-Za-z0-9._-]{2,150}$') + +def atomic_write(path, data): + path.parent.mkdir(parents=True, exist_ok=True) + if path.exists() and path.read_bytes() == data: + return False + tmp = path.with_name(path.name + '.tmp') + try: + tmp.write_bytes(data) + os.replace(tmp, path) + finally: + tmp.unlink(missing_ok=True) + return True + +def fetch(url, etag=None, last_modified=None): + headers = {'User-Agent': 'vuln-osv-gitea-mirror/1.0'} + if etag: + headers['If-None-Match'] = etag + elif last_modified: + headers['If-Modified-Since'] = last_modified + for attempt in range(5): + try: + with urllib.request.urlopen(urllib.request.Request(url, headers=headers), timeout=180) as response: + # Download into a temp file; don't hold huge archives in RAM. + with tempfile.TemporaryFile() as temp: + shutil.copyfileobj(response, temp) + temp.seek(0) + body = temp.read() + return body, response.headers.get('ETag'), response.headers.get('Last-Modified') + except urllib.error.HTTPError as exc: + if exc.code == 304: + return None, etag, last_modified + if exc.code not in (408, 429, 500, 502, 503, 504) or attempt == 4: + raise + except (urllib.error.URLError, TimeoutError): + if attempt == 4: + raise + time.sleep(min(120, 3 * 2**attempt)) + raise RuntimeError('Unreachable') + +def process(ecosystem, state, allow_deletions): + if not VALID_ECOSYSTEM.fullmatch(ecosystem): + raise ValueError(f'Invalid ecosystem name: {ecosystem!r}') + url = f'{BASE}/{urllib.parse.quote(ecosystem)}/all.zip' + previous = state.get('ecosystems', {}).get(ecosystem, {}) + body, etag, modified = fetch(url, previous.get('etag'), previous.get('last_modified')) + if body is None: + print(f'{ecosystem}: unchanged (304)', flush=True) + return False + # Validate all records before changing the working tree. + records = {} + with zipfile.ZipFile(io.BytesIO(body)) as archive: + for info in archive.infolist(): + if info.is_dir() or not info.filename.endswith('.json'): + continue + if info.file_size > 32*1024*1024: + raise ValueError(f'OSV JSON too large: {info.filename}') + identifier = Path(info.filename).stem + if not VALID_ID.fullmatch(identifier): + raise ValueError(f'Unsafe ID: {identifier!r}') + content = archive.read(info) + entry = json.loads(content) + if not isinstance(entry, dict) or entry.get('id') != identifier: + raise ValueError(f'Invalid OSV record: {info.filename}') + if identifier in records and records[identifier] != content: + raise ValueError(f'Duplicate OSV ID: {identifier}') + records[identifier] = content + if not records: + raise ValueError('Empty OSV archive: refusing to delete data') + directory = ROOT / 'data' / ecosystem + existing = {p.stem: p for p in directory.glob('*.json')} + written = sum(atomic_write(directory / f'{ident}.json', raw) for ident, raw in records.items()) + deleted = 0 + if allow_deletions: + for ident, path in existing.items(): + if ident not in records: + path.unlink() + deleted += 1 + state.setdefault('ecosystems', {})[ecosystem] = { + 'etag': etag, 'last_modified': modified, + 'records': len(records), 'last_success': datetime.now(timezone.utc).isoformat() + } + print(f'{ecosystem}: {len(records)} records, {written} written, {deleted} deleted', flush=True) + return True + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument('--ecosystems', default=os.getenv('OSV_ECOSYSTEMS', DEFAULT)) + parser.add_argument('--allow-deletions', action='store_true', help='Remove records no longer in verified full snapshots') + args = parser.parse_args() + ecosystems = [s.strip() for s in args.ecosystems.split(',') if s.strip()] + if len(set(ecosystems)) != len(ecosystems): + raise ValueError('Duplicate ecosystem') + state_path = ROOT / 'state' / 'sync.json' + state = json.loads(state_path.read_text()) if state_path.exists() else {'version': 1, 'ecosystems': {}} + for ecosystem in ecosystems: + # Persist each successfully downloaded ecosystem. A retry resumes the rest. + if process(ecosystem, state, args.allow_deletions): + atomic_write(state_path, (json.dumps(state, ensure_ascii=False, sort_keys=True, indent=2) + '\n').encode()) + +if __name__ == '__main__': + main() diff --git a/scripts/validate.py b/scripts/validate.py new file mode 100644 index 0000000000..9905e0617c --- /dev/null +++ b/scripts/validate.py @@ -0,0 +1,20 @@ +#!/usr/bin/env python3 +import json +import sys +from pathlib import Path +root = Path(__file__).resolve().parents[1] +state_file = root / 'state/sync.json' +if not state_file.exists(): + sys.exit('No OSV sync state') +state = json.loads(state_file.read_text()) +for ecosystem, metadata in state.get('ecosystems', {}).items(): + files = list((root / 'data' / ecosystem).glob('*.json')) + if len(files) < 1: + sys.exit(f'Empty ecosystem: {ecosystem}') + # Files are checked on import; spot-check retained data here to avoid reparsing millions. + for path in files[:10]: + item = json.loads(path.read_text()) + if item.get('id') != path.stem: + sys.exit(f'Invalid {path}') + print(ecosystem, len(files), 'records') +print('Validation OK') diff --git a/state/.gitkeep b/state/.gitkeep new file mode 100644 index 0000000000..e69de29bb2